From 7c5b128a1aeb3bf638569cb39f90187012c2ec44 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 00:04:56 -0700 Subject: [PATCH 01/35] feat(impact): add Impact Evaluation domain foundation (Phase 13A) Add the 4th scientific domain for causal analysis of social programs. Implements full DDD bounded context with World Bank, WHO GHO, and FRED API clients, 3 tools, domain config, evaluation standards, and 66 tests. --- CLAUDE.md | 43 +- CONTRIBUTING.md | 20 + README.md | 20 +- docs/architecture.md | 7 +- docs/competitive-landscape-social-programs.md | 951 ++++++++++++++++++ docs/domain-agnostic.md | 30 +- docs/impact-evaluation-domain.md | 505 ++++++++++ docs/roadmap.md | 121 ++- docs/us-public-data-apis.md | 794 +++++++++++++++ .../src/ehrlich/api/routes/investigation.py | 12 + server/src/ehrlich/api/routes/methodology.py | 25 +- server/src/ehrlich/api/routes/stats.py | 2 +- server/src/ehrlich/impact/__init__.py | 0 .../ehrlich/impact/application/__init__.py | 0 .../impact/application/impact_service.py | 147 +++ server/src/ehrlich/impact/domain/__init__.py | 0 server/src/ehrlich/impact/domain/entities.py | 88 ++ .../impact/domain/evaluation_standards.py | 86 ++ .../src/ehrlich/impact/domain/repository.py | 48 + .../ehrlich/impact/infrastructure/__init__.py | 0 .../impact/infrastructure/fred_client.py | 130 +++ .../impact/infrastructure/who_client.py | 111 ++ .../impact/infrastructure/worldbank_client.py | 142 +++ server/src/ehrlich/impact/tools.py | 149 +++ .../investigation/domain/domains/impact.py | 219 ++++ server/tests/api/test_methodology.py | 7 +- server/tests/api/test_stats.py | 6 +- server/tests/impact/__init__.py | 0 .../tests/impact/test_evaluation_standards.py | 120 +++ server/tests/impact/test_fred_client.py | 125 +++ server/tests/impact/test_impact_service.py | 196 ++++ server/tests/impact/test_tools.py | 151 +++ server/tests/impact/test_who_client.py | 103 ++ server/tests/impact/test_worldbank_client.py | 145 +++ server/tests/integration/test_e2e.py | 8 +- web/vite.config.ts | 11 +- 36 files changed, 4484 insertions(+), 38 deletions(-) create mode 100644 docs/competitive-landscape-social-programs.md create mode 100644 docs/impact-evaluation-domain.md create mode 100644 docs/us-public-data-apis.md create mode 100644 server/src/ehrlich/impact/__init__.py create mode 100644 server/src/ehrlich/impact/application/__init__.py create mode 100644 server/src/ehrlich/impact/application/impact_service.py create mode 100644 server/src/ehrlich/impact/domain/__init__.py create mode 100644 server/src/ehrlich/impact/domain/entities.py create mode 100644 server/src/ehrlich/impact/domain/evaluation_standards.py create mode 100644 server/src/ehrlich/impact/domain/repository.py create mode 100644 server/src/ehrlich/impact/infrastructure/__init__.py create mode 100644 server/src/ehrlich/impact/infrastructure/fred_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/who_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/worldbank_client.py create mode 100644 server/src/ehrlich/impact/tools.py create mode 100644 server/src/ehrlich/investigation/domain/domains/impact.py create mode 100644 server/tests/impact/__init__.py create mode 100644 server/tests/impact/test_evaluation_standards.py create mode 100644 server/tests/impact/test_fred_client.py create mode 100644 server/tests/impact/test_impact_service.py create mode 100644 server/tests/impact/test_tools.py create mode 100644 server/tests/impact/test_who_client.py create mode 100644 server/tests/impact/test_worldbank_client.py diff --git a/CLAUDE.md b/CLAUDE.md index dc5c067..0d4ad1e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,11 +6,12 @@ AGPL-3.0 | Claude Code Hackathon (Feb 10-16, 2026) Ehrlich is a domain-agnostic scientific discovery platform that uses Claude as a hypothesis-driven scientific reasoning engine. Named after Paul Ehrlich, the father of the "magic bullet" concept. -The engine is domain-agnostic: a pluggable `DomainConfig` + `DomainRegistry` system lets any scientific domain plug in with its own tools, score definitions, and prompt examples. Currently supports three domains: +The engine is domain-agnostic: a pluggable `DomainConfig` + `DomainRegistry` system lets any scientific domain plug in with its own tools, score definitions, and prompt examples. Currently supports four domains: - **Molecular Science** -- antimicrobial resistance, drug discovery, toxicology, agricultural biocontrol - **Training Science** -- exercise physiology, protocol optimization, injury risk assessment, clinical trials - **Nutrition Science** -- supplement evidence analysis, dietary supplement safety, nutrient profiling, DRI adequacy assessment, drug-nutrient interactions, inflammatory index scoring +- **Impact Evaluation** -- causal analysis of social programs (education, health, sports, employment, housing). Economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmark fetching. See `docs/impact-evaluation-domain.md` ## Architecture @@ -20,13 +21,13 @@ DDD monorepo: `server/` (Python 3.12) + `console/` (React 19 / TypeScript / Bun) ``` Opus 4.6 (Director) -- Formulates hypotheses, designs experiments, evaluates evidence, synthesizes (NO tools) -Sonnet 4.5 (Researcher) -- Executes experiments with 70 tools (parallel: 2 experiments per batch) +Sonnet 4.5 (Researcher) -- Executes experiments with 73 tools (parallel: 2 experiments per batch) Haiku 4.5 (Summarizer) -- Compresses large tool outputs (>2000 chars), PICO decomposition, evidence grading ``` Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2. -### Bounded Contexts (10) +### Bounded Contexts (11) | Context | Location | Purpose | |---------|----------|---------| @@ -39,9 +40,10 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 | simulation | `server/src/ehrlich/simulation/` | Docking, ADMET, resistance, target discovery (RCSB PDB, UniProt, Open Targets), toxicity (EPA CompTox) | | training | `server/src/ehrlich/training/` | Evidence analysis, protocol comparison, injury risk, training metrics, clinical trials (ClinicalTrials.gov), PubMed literature (E-utilities), exercise database (wger) | | nutrition | `server/src/ehrlich/nutrition/` | Supplement evidence, supplement labels (NIH DSLD), nutrient data (USDA FoodData), supplement safety (OpenFDA CAERS), drug-nutrient interactions (RxNav), DRI adequacy, nutrient ratios, inflammatory index | +| impact | `server/src/ehrlich/impact/` | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmark fetching | | investigation | `server/src/ehrlich/investigation/` | Multi-model orchestration + PostgreSQL persistence + domain registry + MCP bridge | -### External Data Sources (15 external + 1 internal) +### External Data Sources (18 external + 1 internal) | Source | API | Purpose | Auth | |--------|-----|---------|------| @@ -60,6 +62,9 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 | USDA FoodData | `https://api.nal.usda.gov/fdc/v1` | Nutrient profiles for foods and supplements | API key | | OpenFDA CAERS | `https://api.fda.gov/food/event.json` | Supplement adverse event reports (safety monitoring) | None | | RxNav | `https://rxnav.nlm.nih.gov/REST` | Drug-supplement and drug-nutrient interactions (RxNorm) | None | +| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country (GDP, poverty, education, health) | None | +| WHO GHO | `https://ghoapi.azureedge.net/api/` | Global health statistics by country (life expectancy, mortality, disease burden) | None | +| FRED | `https://api.stlouisfed.org/fred/` | 800K+ US economic time series (GDP, unemployment, CPI, interest rates) | API key | | Ehrlich tsvector | internal | Full-text search of past investigation findings (PostgreSQL GIN index) | None | ### Dependency Rules (STRICT) @@ -134,9 +139,9 @@ type(scope): description Types: `feat`, `fix`, `refactor`, `chore`, `docs`, `test` -Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, training, nutrition, investigation, api, console, mol, data, ci, docs, infra +Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, training, nutrition, impact, investigation, api, console, mol, data, ci, docs, infra -## Tools (70 Total) +## Tools (73 Total) ### Chemistry (6) -- RDKit cheminformatics, domain-agnostic - `validate_smiles` -- Validate SMILES string @@ -201,6 +206,11 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `analyze_nutrient_ratios` -- Clinically relevant nutrient ratios (omega-6:3, Ca:Mg, Na:K, Zn:Cu, Ca:P, Fe:Cu) - `compute_inflammatory_index` -- Simplified Dietary Inflammatory Index (DII) scoring +### Impact Evaluation (3) -- Social program analysis and economic indicators +- `search_economic_indicators` -- Query economic time series from FRED, World Bank, or WHO GHO +- `fetch_benchmark` -- Get comparison values from international sources (World Bank, WHO GHO) +- `compare_programs` -- Cross-program comparison using existing statistical tests + ### Visualization (12) -- Domain-specific interactive charts - `render_binding_scatter` -- Scatter plot of compound binding affinities (Recharts) - `render_admet_radar` -- Radar chart of ADMET/drug-likeness properties (Recharts) @@ -266,7 +276,7 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - **Domain configuration**: `DomainConfig` defines per-domain tool tags, score definitions, prompt examples; `DomainRegistry` auto-detects domain; `DomainDetected` SSE event sends display config to frontend - **Domain tool examples**: each `DomainConfig` includes `tool_examples` in `experiment_examples` showing realistic tool chaining patterns; ensures Researcher (Sonnet 4.5) has usage guidance for all domain-specific tools - **Multi-domain investigations**: `DomainRegistry.detect()` returns `list[DomainConfig]`; `merge_domain_configs()` creates synthetic merged config with union of tool_tags, concatenated score_definitions, joined prompt examples -- **Tool tagging**: Tools tagged with frozenset domain tags (chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, literature, visualization); investigation control tools are universal (no tags); researcher sees only domain-relevant tools +- **Tool tagging**: Tools tagged with frozenset domain tags (chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, impact, literature, visualization); investigation control tools are universal (no tags); researcher sees only domain-relevant tools - **Self-referential research**: `search_prior_research` queries PostgreSQL `tsvector` + GIN index of past findings; indexed on completion; intercepted in orchestrator `_dispatch_tool()` and routed to repository; "ehrlich" source type links to past investigations - **MCP bridge**: Optional `MCPBridge` connects to external MCP servers (e.g. Excalidraw); tools registered dynamically via `ToolRegistry.register_mcp_tools()`; lifecycle managed by orchestrator; enabled via `EHRLICH_MCP_EXCALIDRAW=true` - **SSE streaming**: 20 event types for real-time updates; reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) @@ -378,6 +388,18 @@ All paths relative to `server/src/ehrlich/`. | `infrastructure/openfda_client.py` | OpenFDA CAERS adverse event reporting | | `infrastructure/rxnav_client.py` | RxNav drug-nutrient interaction API | +### impact/ + +| File | Purpose | +|------|---------| +| `tools.py` | 3 impact evaluation tools | +| `domain/entities.py` | EconomicIndicator, BenchmarkValue, ProgramComparison, ProgramMetric | +| `domain/repository.py` | EconomicDataRepository, BenchmarkRepository ABCs | +| `application/impact_service.py` | ImpactService (economic indicators, benchmarks, program comparison) | +| `infrastructure/worldbank_client.py` | World Bank API v2 client for development indicators | +| `infrastructure/who_client.py` | WHO GHO API client for health statistics | +| `infrastructure/fred_client.py` | FRED API client for US economic time series | + ### investigation/ | File | Purpose | @@ -402,6 +424,7 @@ All paths relative to `server/src/ehrlich/`. | `domain/domains/molecular.py` | `MOLECULAR_SCIENCE` config | | `domain/domains/training.py` | `TRAINING_SCIENCE` config | | `domain/domains/nutrition.py` | `NUTRITION_SCIENCE` config | +| `domain/domains/impact.py` | `IMPACT_EVALUATION` config | | `domain/mcp_config.py` | `MCPServerConfig` frozen dataclass | | `tools_viz.py` | 12 visualization tools | | `infrastructure/repository.py` | PostgreSQL persistence (asyncpg) + tsvector findings search + user/credit management | @@ -415,7 +438,7 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| | `auth.py` | WorkOS JWT middleware (JWKS verification, header + query param auth for SSE) | -| `routes/investigation.py` | REST + SSE endpoints, 70-tool registry, domain registry, MCP bridge, credit system, BYOK | +| `routes/investigation.py` | REST + SSE endpoints, 73-tool registry, domain registry, MCP bridge, credit system, BYOK | | `routes/methodology.py` | GET /methodology: phases, domains, tools, data sources, models | | `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets endpoints | | `routes/stats.py` | GET /stats: aggregate counts | @@ -514,9 +537,9 @@ All paths relative to `web/src/`. | `components/HowItWorks.tsx` | 6-phase methodology timeline with vertical connecting line | | `components/ConsolePreview.tsx` | Browser-frame mockups (timeline, hypothesis board, candidates, radar) | | `components/Architecture.tsx` | Director-Worker-Summarizer model cards, dot grid bg, amber glow | -| `components/Domains.tsx` | 3 domain cards with tool counts, multi-domain callout | +| `components/Domains.tsx` | 4 domain cards with tool counts, multi-domain callout | | `components/Visualizations.tsx` | 4 visualization category cards with tech labels | -| `components/DataSources.tsx` | 16 source cards, surface bg with teal glow | +| `components/DataSources.tsx` | 19 source cards, surface bg with teal glow | | `components/WhoItsFor.tsx` | 3 persona cards (Student, Academic, Industry) | | `components/Differentiators.tsx` | 3 differentiator cards with capabilities lists | | `components/OpenSource.tsx` | Typography-driven section with code snippet + licensing | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a161d88..216e187 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,26 @@ Ehrlich is an AI-powered scientific discovery engine that uses Claude as a hypot See [docs/domain-agnostic.md](docs/domain-agnostic.md) for the full story of how and why the engine was generalized. +## Contributor License Agreement + +By submitting a pull request to this repository, you agree to the following terms: + +1. **You grant Ehrlich's maintainers a perpetual, worldwide, non-exclusive, royalty-free license** to use, reproduce, modify, distribute, sublicense, and relicense your contributions as part of this project. + +2. **You retain copyright** to your contributions. This agreement does not transfer ownership. + +3. **You have the right to submit** the contribution. If your contribution includes code from your employer or third parties, you have obtained the necessary permissions. + +4. **Your contribution is provided "as is"** without warranty of any kind. + +### Why This Matters + +Ehrlich is currently licensed under AGPL-3.0. This CLA preserves the option to change the license in the future (e.g., to MIT) without requiring permission from every past contributor. This is standard practice for open source projects with dual-licensing strategies. + +If you have questions about this CLA, open an issue before contributing. + +--- + ## Prerequisites - **Python 3.12+** with [uv](https://docs.astral.sh/uv/) package manager diff --git a/README.md b/README.md index ab4195e..4cd6780 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,16 @@ Ehrlich is **domain-agnostic**. The hypothesis-driven engine works for any scien - **Supplement evidence** -- "What is the evidence for creatine on strength performance?" - **Nutrient profiling** -- "Compare protein content and amino acid profiles of whey vs plant-based supplements" +### Impact Evaluation +- **Social program analysis** -- "Evaluate the effectiveness of Sonora's sports scholarship program" +- **Causal inference** -- "Does the conditional cash transfer reduce school dropout rates?" +- **Cross-country benchmarking** -- "Compare cost-effectiveness of state sports programs in Mexico" +- Economic indicators from World Bank, WHO GHO, and FRED (800K+ time series) +- Cross-program comparison and international benchmarking + ## Architecture -Ehrlich follows Domain-Driven Design with ten bounded contexts: +Ehrlich follows Domain-Driven Design with eleven bounded contexts: | Context | Purpose | |---------|---------| @@ -41,6 +48,7 @@ Ehrlich follows Domain-Driven Design with ten bounded contexts: | **training** | Exercise physiology: evidence analysis, protocol comparison, injury risk, training metrics, clinical trials | | **nutrition** | Nutrition science: supplement evidence, labels, nutrients, safety, interactions, adequacy, inflammatory index | | **investigation** | Multi-model agent orchestration with Director-Worker-Summarizer pattern + domain registry + MCP bridge | +| **impact** | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking | ### Multi-Model Architecture @@ -48,7 +56,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga ``` Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) -Sonnet 4.5 (Researcher) -- Executes experiments with 70 tools (10-20 calls) +Sonnet 4.5 (Researcher) -- Executes experiments with 73 tools (10-20 calls) Haiku 4.5 (Summarizer) -- Compresses large outputs, classifies domains (5-10 calls) ``` @@ -73,10 +81,13 @@ Cost: ~$3-4 per investigation (vs ~$11 with all-Opus). | [USDA FoodData](https://fdc.nal.usda.gov/) | Nutrient profiles for foods and supplements | 1.1M+ foods | | [OpenFDA CAERS](https://open.fda.gov/) | Supplement adverse event reports | 200K+ reports | | [RxNav](https://rxnav.nlm.nih.gov/) | Drug interaction screening (RxCUI resolution) | 100K+ drugs | +| [World Bank](https://data.worldbank.org/) | Development indicators by country (GDP, poverty, education) | 190+ countries | +| [WHO GHO](https://www.who.int/data/gho) | Global health statistics (life expectancy, mortality, disease) | 190+ countries | +| [FRED](https://fred.stlouisfed.org/) | US economic time series (GDP, unemployment, CPI) | 800K+ series | All data sources are free and open-access. -## 70 Tools +## 73 Tools | Context | Tool | Description | |---------|------|-------------| @@ -129,6 +140,9 @@ All data sources are free and open-access. | Nutrition | `check_interactions` | Drug-supplement interaction screening via RxNav | | Nutrition | `analyze_nutrient_ratios` | Key nutrient ratio analysis (omega-6:3, Ca:Mg, etc.) | | Nutrition | `compute_inflammatory_index` | Simplified Dietary Inflammatory Index scoring | +| Impact | `search_economic_indicators` | Query economic time series from FRED, World Bank, or WHO GHO | +| Impact | `fetch_benchmark` | Get comparison values from international sources | +| Impact | `compare_programs` | Cross-program comparison using statistical tests | | Visualization | `render_binding_scatter` | Scatter plot of compound binding affinities | | Visualization | `render_admet_radar` | Radar chart of ADMET/drug-likeness properties | | Visualization | `render_training_timeline` | Training load timeline with ACWR danger zones | diff --git a/docs/architecture.md b/docs/architecture.md index 23f0c46..0cbd9fe 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -33,6 +33,9 @@ Exercise physiology and sports medicine research: evidence-based training analys ### Nutrition Nutrition science research: supplement evidence analysis, supplement label lookup (NIH DSLD), nutrient data (USDA FoodData), supplement safety monitoring (OpenFDA CAERS), drug interaction screening (RxNav), DRI-based nutrient adequacy assessment, nutrient ratio analysis, and inflammatory index scoring. Uses Semantic Scholar for literature search. +### Impact +Social program evaluation and causal analysis. Foundation layer provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities (`EconomicIndicator`, `BenchmarkValue`, `ProgramComparison`, `ProgramMetric`), repository ABCs (`EconomicDataRepository`, `BenchmarkRepository`), and infrastructure clients for three external APIs. Future phases will add causal inference methods (DiD, PSM, RDD, Synthetic Control), Mexico/US-specific data sources, and document upload. See `docs/impact-evaluation-domain.md` for full design. + ### Investigation Hypothesis-driven agent orchestration. Manages the Claude-driven research loop: literature survey, hypothesis formulation (with predictions, criteria, scope), parallel experiment execution, criteria-based evaluation, negative controls, and synthesis. Uses multi-model architecture (Director/Researcher/Summarizer) with user-guided steering, domain classification, and multi-investigation memory. Includes domain configuration system (`DomainConfig` + `DomainRegistry`) for pluggable scientific domains with tool tagging, score definitions, prompt adaptation, and visualization control. Each domain config includes `tool_examples` in `experiment_examples` with realistic tool chaining patterns for complete tool coverage. Optional MCP bridge (`MCPBridge`) connects to external MCP servers for extensibility (e.g. Excalidraw for visual summaries). @@ -44,7 +47,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) │ NO tool access, structured JSON responses only │ - ├── Sonnet 4.5 (Researcher) -- Executes experiments with 70 domain-filtered tools (10-20 calls, parallel x2) + ├── Sonnet 4.5 (Researcher) -- Executes experiments with 73 domain-filtered tools (10-20 calls, parallel x2) │ Tool-calling loop with max_iterations_per_experiment guard │ └── Haiku 4.5 (Summarizer) -- Compresses large tool outputs >2000 chars, PICO+classification, evidence grading @@ -206,7 +209,7 @@ Separate TanStack Start project for the public-facing landing page. SSR/SSG for - **Console** (`console/`): authenticated SPA for running investigations (TanStack Router, client-side only) - **Web** (`web/`): public landing page with SSR/SSG for SEO (TanStack Start + Nitro, server-rendered) -16 component files: Nav (scroll progress, eager), Hero (MolecularNetwork 3D canvas, eager), HowItWorks (6-phase timeline, lazy), ConsolePreview (browser-frame mockups, lazy), Architecture (model cards + dot grid bg + amber glow, lazy), Domains (3 domain cards, lazy), Visualizations (4 category cards, lazy), DataSources (16 source cards + teal glow, lazy), WhoItsFor (3 persona cards, lazy), Differentiators (3 cards, lazy), OpenSource (code snippet + licensing, lazy), Roadmap (planned domains/features, lazy), CTA (pricing tiers + terminal quickstart + primary glow, lazy), SectionHeader, Footer (lazy), MolecularNetwork. All use OKLCH tokens, Motion scroll animations, and staggered children reveals. Section zoning via alternating `bg-surface/30` backgrounds and strategic radial accent glows (amber on Architecture, teal on DataSources, primary on CTA). +16 component files: Nav (scroll progress, eager), Hero (MolecularNetwork 3D canvas, eager), HowItWorks (6-phase timeline, lazy), ConsolePreview (browser-frame mockups, lazy), Architecture (model cards + dot grid bg + amber glow, lazy), Domains (4 domain cards, lazy), Visualizations (4 category cards, lazy), DataSources (19 source cards + teal glow, lazy), WhoItsFor (3 persona cards, lazy), Differentiators (3 cards, lazy), OpenSource (code snippet + licensing, lazy), Roadmap (planned domains/features, lazy), CTA (pricing tiers + terminal quickstart + primary glow, lazy), SectionHeader, Footer (lazy), MolecularNetwork. All use OKLCH tokens, Motion scroll animations, and staggered children reveals. Section zoning via alternating `bg-surface/30` backgrounds and strategic radial accent glows (amber on Architecture, teal on DataSources, primary on CTA). Both deploy independently: console to app server, web to CDN/static hosting. diff --git a/docs/competitive-landscape-social-programs.md b/docs/competitive-landscape-social-programs.md new file mode 100644 index 0000000..1163e40 --- /dev/null +++ b/docs/competitive-landscape-social-programs.md @@ -0,0 +1,951 @@ +# Competitive Landscape Analysis: Social Program Evaluation & Impact Assessment + +**Research Date:** February 13, 2026 +**Context:** Exploring market opportunity for hypothesis-driven AI engine for social program evaluation +**Geographic Focus:** Mexico & United States + +--- + +## Executive Summary + +The social program evaluation market is dominated by traditional M&E platforms focused on **data collection, reporting, and indicator tracking**. While AI is emerging in this space (primarily for automation and summarization), **NO existing platform offers hypothesis-driven, multi-model causal inference with evidence-graded reports and public API integration**. This represents a significant market gap. + +### Key Findings + +1. **Traditional platforms** (DevResults, ActivityInfo, Bonterra, Amp Impact) focus on donor reporting, case management, and indicator dashboards—NOT causal analysis +2. **AI adoption is nascent**: Limited to data summarization, automated screening, and sentiment analysis—NOT hypothesis testing or causal inference +3. **Public data integration is minimal**: Most platforms require manual uploads; API connectivity to INEGI, census data, or economic indicators is virtually nonexistent +4. **Evidence grading is manual**: Systematic review tools exist for academic research (Rayyan, LASER AI) but are NOT integrated into program evaluation platforms +5. **Causal inference is manual**: Propensity score matching, DiD, and RCT analysis require separate statistical software (Stata, R, Python) + +--- + +## Direct Competitors + +### 1. DevResults + +**What It Does:** +M&E software for global development organizations; tracks indicators, manages results frameworks, supports donor reporting with geographic mapping. + +**Target Market:** +Large international development orgs (USAID, UN agencies, INGOs); 160+ countries. + +**Pricing:** +Annual subscription model; specific tiers not publicly disclosed. Designed for enterprise budgets. + +**Key Features:** +- Results framework integration +- Indicator disaggregation +- Geographic mapping (GIS) +- Donor report generation (aligned with USAID, DFID formats) +- Custom dashboards + +**AI/ML Capabilities:** +None disclosed. Platform is rules-based. + +**Hypothesis-Driven Analysis:** +No. Focuses on pre-defined indicators, not hypothesis formulation or testing. + +**Public API Integration:** +Limited. Requires manual data imports from external sources. + +**Evidence-Graded Reports:** +No. Reports are descriptive, not evidence-graded. + +**Limitations:** +- High cost for smaller NGOs +- Steep learning curve +- Does not support causal inference (RCTs, DiD, PSM) +- No AI-driven insights + +**Sources:** +- [DevResults Reviews 2026 - SoftwareWorld](https://www.softwareworld.co/software/devresults-reviews/) +- [DevResults Plans](https://www.devresults.com/plans) +- [DevResults Features - TopSoftwareAdvisor](https://www.topsoftwareadvisor.com/software/devresults) + +--- + +### 2. ActivityInfo + +**What It Does:** +MIS platform for humanitarian coordination, M&E, and case management; emphasizes offline data collection and synchronization. + +**Target Market:** +Humanitarian NGOs, UN agencies (UNHCR, WHO), emergency response teams; 100+ countries. + +**Pricing:** +- **Solo:** €545/year (~$590 USD) +- **Programme:** €3,700/year (~$4,000 USD) +- **Global Organization:** €13,400/year (~$14,500 USD) +- 30-day free trial + +**Key Features:** +- Online/offline mobile data collection with two-way sync +- Forms, subforms, linked datasets +- Barcode scanning, geodata +- Integration with Tableau, Power BI, ArcGIS, QGIS +- 99.95% uptime SLA + +**AI/ML Capabilities:** +None. Platform is form-based with manual analysis. + +**Hypothesis-Driven Analysis:** +No. Aggregates data from multiple sources but does not formulate or test hypotheses. + +**Public API Integration:** +Yes (exports to BI tools), but no documented connectors to public datasets (INEGI, census APIs). + +**Evidence-Graded Reports:** +No. Reports are aggregated summaries, not evidence-graded. + +**Limitations:** +- No causal inference tools +- No AI-driven pattern detection +- Limited to descriptive statistics +- Requires technical expertise for integrations + +**Sources:** +- [ActivityInfo Pricing](https://www.activityinfo.org/about/pricing/index.html) +- [ActivityInfo - GetApp](https://www.getapp.com/it-management-software/a/activityinfo/) +- [ActivityInfo Humanitarian Coordination](https://www.activityinfo.org/about/humanitarian-coordination.html) + +--- + +### 3. DHIS2 + +**What It Does:** +Open-source health information system for aggregate and patient-based data; national-scale platform for routine health data in 75+ countries. + +**Target Market:** +Ministries of Health, WHO, health NGOs; primarily low- and middle-income countries. + +**Pricing:** +Free (open-source). Implementation costs vary (hosting, customization, training). + +**Key Features:** +- Data collection, validation, analysis, visualization +- Mobile app support (DHIS2 Capture) +- Dashboards, maps, pivot tables +- Interoperability (FHIR, HL7) +- Predictive modeling app (CHAP) + +**AI/ML Capabilities (NEW):** +- **Natural Language Analytics:** AI Insights app (IFRC) enables plain-language queries of DHIS2 data, identifies trends, generates insights. +- **Predictive Modeling:** CHAP Modeling Platform for forecasting (uses health + climate/weather data). +- **Automated Data Entry:** SolidLines AI Data Entry extracts data from photos/audio. +- **Alert Triage:** Tanzania's AI-based event-based surveillance reduced alert triage from 36 hours to near-instantaneous. +- **Metadata Management:** Generative AI proposes metadata matches for interoperability. + +**Hypothesis-Driven Analysis:** +Partial. Predictive modeling app allows hypothesis testing (e.g., disease outbreak forecasts), but NOT general-purpose causal inference. + +**Public API Integration:** +Yes (robust REST API), but integrations are domain-specific (health data, climate data). No documented INEGI or census API connectors. + +**Evidence-Graded Reports:** +No. AI-generated insights are descriptive or predictive, not evidence-graded per GRADE framework. + +**Limitations:** +- Health-specific (not generalized to social programs) +- AI features are nascent (2024-2025 rollout) +- No causal inference pipelines (RCT, DiD, PSM) +- Complex setup (requires technical expertise) + +**Sources:** +- [DHIS2 & AI](https://dhis2.org/ai/) +- [AI-Driven Alert Triage in Tanzania](https://dhis2.org/ai-driven-alert-triage-tanzania/) +- [DHIS2 AI Insights - GitHub](https://github.com/jmesplana/dhis2_ai_insights) +- [DHIS2 Health Intelligence](https://dhis2.org/health/health-intelligence/) + +--- + +### 4. KoboToolbox + +**What It Does:** +Open-source platform for data collection, management, and visualization; widely used in humanitarian/development sectors. + +**Target Market:** +Nonprofits, NGOs, UN agencies (WFP, IRC, MSF, Save the Children); 14,000+ organizations. + +**Pricing:** +- **Community Plan (Nonprofits):** FREE (5,000 submissions/month, 1GB storage) +- **Professional Plan:** 25,000 submissions/month, unlimited storage (discounted for nonprofits) +- **Teams Plan:** 50,000 submissions/month +- **Enterprise Plan:** Custom pricing + +**Key Features:** +- Form builder (XLSForm) +- Mobile app (offline data collection) +- Question skip logic, validation rules +- Photo/audio/GPS capture +- API access +- Export to Excel, CSV, SPSS + +**AI/ML Capabilities:** +None. Platform is form-based with manual analysis. + +**Hypothesis-Driven Analysis:** +No. Data collection tool, not analysis platform. + +**Public API Integration:** +Yes (REST API for data export), but no connectors to public datasets. + +**Evidence-Graded Reports:** +No. Users export data to external tools for analysis. + +**Limitations:** +- No built-in analytics (requires external tools like R, Stata, Tableau) +- No AI capabilities +- No causal inference support +- Data collection only + +**Sources:** +- [KoboToolbox Pricing](https://www.kobotoolbox.org/pricing/) +- [KoboToolbox - SaaSworthy](https://www.saasworthy.com/product/kobotoolbox) +- [Kobo Toolbox - Berkeley Human Rights Center](https://humanrights.berkeley.edu/projects/kobo-toolbox/) + +--- + +### 5. Salesforce Nonprofit Cloud (Program Management Module) + +**What It Does:** +CRM-based program and case management for nonprofits; tracks programs, benefits, grants, fundraising. + +**Target Market:** +Medium to large nonprofits (US-focused); requires Salesforce expertise. + +**Pricing:** +- **Enterprise:** $60/user/month (annual billing) +- **Unlimited:** $100/user/month (annual billing) +- **Agentforce (Sales/Service):** $325/user/month +- 10 free licenses for eligible nonprofits +- **Program Management Module (PMM):** FREE (open-source, integrates with NPSP) + +**Key Features:** +- Program and benefit tracking +- Case management +- Grantmaking workflows +- Fundraising integration (NPSP) +- Custom reporting (Salesforce Reports) +- Accounting sub-ledger + +**AI/ML Capabilities:** +Salesforce Einstein (add-on): predictive lead scoring, opportunity insights, automated email responses. NOT designed for program evaluation or causal inference. + +**Hypothesis-Driven Analysis:** +No. CRM-based; tracks program activities, not causal impacts. + +**Public API Integration:** +Yes (Salesforce APIs are robust), but no pre-built connectors to INEGI, census, or economic indicator APIs. + +**Evidence-Graded Reports:** +No. Dashboards are descriptive. + +**Limitations:** +- High cost for smaller orgs (per-user pricing) +- Requires Salesforce admin expertise +- Einstein AI is sales-focused, not evaluation-focused +- No causal inference tools +- Complex setup + +**Sources:** +- [Salesforce Nonprofit Pricing](https://www.salesforce.com/nonprofit/pricing/) +- [Program Management Module - AppExchange](https://appexchange.salesforce.com/appxListingDetail?listingId=a0N3A00000FMposUAD) +- [Six Key Features of Program Management - Forvis Mazars](https://www.forvismazars.us/forsights/2025/03/six-key-features-of-salesforce-nonprofit-cloud-program-management) + +--- + +### 6. Sopact Impact Cloud + +**What It Does:** +AI-powered survey, CRM, and impact measurement platform; automates data collection, analysis, and report generation. + +**Target Market:** +Impact investors, foundations, social enterprises; sustainability-focused organizations. + +**Pricing:** +Not publicly disclosed. Positioned as "accessible price points" with unlimited users/forms (no per-seat fees). + +**Key Features:** +- AI-driven survey generation +- Multi-source data aggregation (300+ system integrations) +- Qualitative + quantitative data correlation +- Automated report generation (designer-quality) +- Real-time dashboards +- Benchmarking across organizations + +**AI/ML Capabilities (STRONGEST IN THIS CATEGORY):** +- **AI Agents:** Upload text, images, video, documents; AI transforms into actionable insights +- **Automated Analysis:** Correlates quantitative/qualitative data, spots outliers, summarizes patterns +- **Report Generation:** AI generates shareable reports in minutes (vs. months of manual work) +- **Sentiment Analysis:** Analyzes open-ended responses for themes + +**Hypothesis-Driven Analysis:** +Partial. AI identifies patterns and outliers, but does NOT formulate or test causal hypotheses (no RCT, DiD, PSM pipelines). + +**Public API Integration:** +Yes (300+ systems), but INEGI/census API connectors not documented. + +**Evidence-Graded Reports:** +No. AI-generated insights are descriptive summaries, not evidence-graded per systematic review standards. + +**Limitations:** +- AI is pattern-detection, not causal inference +- No hypothesis formulation engine +- No support for quasi-experimental designs (DiD, PSM, RDD) +- Pricing opacity + +**Sources:** +- [Sopact AI-Powered Platform](https://www.sopact.com) +- [AI Powered Monitoring and Evaluation Tools - Sopact](https://www.sopact.com/use-case/monitoring-and-evaluation-tools) +- [Sopact Impact Cloud - SoftwareSuggest](https://www.softwaresuggest.com/sopact-impact-cloud) + +--- + +### 7. Bonterra (formerly Social Solutions Apricot) + +**What It Does:** +Case management and impact tracking for nonprofits; manages caseloads, enrollments, referrals, outcomes. + +**Target Market:** +US nonprofits (social services, health, education); community-based organizations. + +**Pricing:** +- Three tiers: **Essentials, Pro, Enterprise** (per-user pricing, not disclosed publicly) +- Users report "high cost" and "rising yearly fees" + +**Key Features:** +- Customizable forms and workflows +- Case management (caseloads, exits, enrollments, schedules) +- Electronic signatures, document storage +- Dynamic charts/graphs +- Custom dashboards +- Network/internal referrals + +**AI/ML Capabilities:** +None disclosed. Platform is form-based with manual analysis. + +**Hypothesis-Driven Analysis:** +No. Tracks outcomes, does not test causal hypotheses. + +**Public API Integration:** +Yes (API for data export), but no connectors to public datasets. + +**Evidence-Graded Reports:** +No. Dashboards are descriptive. + +**Limitations:** +- High cost (per-user pricing, rising fees) +- No AI capabilities +- No causal inference tools +- US-focused (limited international use) + +**Sources:** +- [Bonterra Case Management - Pricing](https://www.bonterratech.com/pricing/case-management) +- [Bonterra Apricot - Capterra](https://www.capterra.com/p/123342/Bonterra-Case-Management/) +- [Social Solutions - Bonterra](https://www.bonterratech.com/blog/social-solutions) + +--- + +### 8. Vera Solutions (Amp Impact) + +**What It Does:** +Salesforce-based portfolio management and impact measurement for large-scale programs; tracks grants, projects, outcomes, KPIs. + +**Target Market:** +Large foundations, impact investors, multilateral orgs (portfolios >$20M); UN SDG-aligned programs. + +**Pricing:** +Starts at **$7,000/year**. Discounts for nonprofits. Best suited for portfolios above $20M. + +**Key Features:** +- Logframes, indicators, targets +- Auto-aggregation of results +- Narrative reporting +- IATI publishing +- Disaggregation (geographic, demographic) +- Risk tracking +- Expenditure tracking + +**AI/ML Capabilities:** +None disclosed. Platform is Salesforce-based with manual indicator tracking. + +**Hypothesis-Driven Analysis:** +No. Tracks pre-defined indicators and logframes, not causal hypotheses. + +**Public API Integration:** +Yes (Salesforce APIs), but no documented connectors to INEGI or census APIs. + +**Evidence-Graded Reports:** +No. Reports are aggregated indicator summaries. + +**Limitations:** +- High cost ($7K+/year) +- Requires Salesforce expertise +- Complex setup (enterprise-grade) +- No AI or causal inference capabilities +- Designed for large portfolios only + +**Sources:** +- [Vera Solutions - Amp Impact](https://verasolutions.org/) +- [Amp Impact Pricing - Capterra](https://www.capterra.com/p/173122/Amp-Impact/) +- [Amp Impact and Salesforce for Program Delivery](https://verasolutions.org/5-organizations-leveraging-amp-impact-and-salesforce-to-drive-more-effective-program-delivery/) + +--- + +## Government Initiatives + +### 9. CONEVAL (Mexico) + +**What It Does:** +National Council for the Evaluation of Social Development Policy; autonomous body that measures poverty and evaluates social programs in Mexico. + +**Target Market:** +Federal/state governments, policymakers, researchers. + +**Pricing:** +N/A (public agency). Data freely available. + +**Key Features:** +- **Multidimensional Poverty Measurement:** Official estimates every 2 years (state), 5 years (municipal) +- **Program Evaluations:** Design evaluations, process evaluations, impact evaluations (RCTs, quasi-experimental) +- **Mobile App:** Disseminates poverty data (national, state, municipal) +- **Publications:** Annual reports, evaluation guidelines, methodological frameworks +- **Website:** www.coneval.org.mx (data portal) + +**AI/ML Capabilities:** +None disclosed. Evaluations follow traditional RCT/quasi-experimental methods. + +**Hypothesis-Driven Analysis:** +Yes, but MANUAL. CONEVAL coordinates rigorous impact evaluations (RCTs, DiD, PSM), but relies on external consultants and academic researchers—not automated AI pipelines. + +**Public API Integration:** +Limited. CONEVAL provides downloadable datasets, but no documented REST APIs for real-time integration. + +**Evidence-Graded Reports:** +Partially. CONEVAL reports include methodological rigor (RCT results, effect sizes, confidence intervals), but do NOT use GRADE-style evidence grading. + +**Limitations:** +- No AI/automation (manual evaluations) +- No hypothesis formulation engine +- Evaluations take 1-5 years to complete +- No real-time dashboards +- No API for developers + +**Sources:** +- [CONEVAL - Monitoring and Evaluation](https://www.coneval.org.mx/Evaluacion/Paginas/Evaluation-and-monitoring-en.aspx) +- [CONEVAL Publications](https://www.coneval.org.mx/InformesPublicaciones/Paginas/Publicaciones-sobre-Evaluacion-y-monitoreo-en.aspx) +- [CONEVAL - 3ie Member Profile](https://www.3ieimpact.org/about/members/national-council-evaluation-social-development-policy-coneval) + +### 10. US Government AI Initiatives + +**What It Does:** +Multiple federal initiatives (2025-2026) to advance AI in government, evaluate AI systems, and study AI's labor market impact. + +**Target Market:** +Federal agencies, state/local governments, policymakers. + +**Pricing:** +N/A (government-funded). + +**Key Features:** +- **AI Action Plan (July 2025):** Accelerate innovation, build AI infrastructure, lead diplomacy/security +- **Program Evaluation:** Rigorous evaluations to assess AI system performance/reliability in regulated industries +- **Science Fellows Program (Spring 2026):** Hire 250 AI fellows across government +- **AI Workforce Research Hub (DOL):** Study AI's labor market impact + +**AI/ML Capabilities:** +Government is PROCURING AI tools, not building evaluation platforms. Focus on AI governance, not program evaluation automation. + +**Hypothesis-Driven Analysis:** +No. AI initiatives focus on policy, workforce, security—not automating social program evaluation. + +**Public API Integration:** +US Census Bureau offers Economic Indicator APIs, but no unified government platform for program evaluation with API integration. + +**Evidence-Graded Reports:** +No. Federal program evaluations (GAO, OMB) follow traditional methods (RCTs, cost-benefit analysis), not AI-driven evidence grading. + +**Limitations:** +- No centralized AI evaluation platform +- Fragmented across agencies +- Focus on AI governance, not evaluation automation +- No public-facing tools for program evaluators + +**Sources:** +- [AI.Gov Action Plan](https://www.ai.gov/action-plan) +- [America's AI Action Plan - July 2025](https://www.whitehouse.gov/wp-content/uploads/2025/07/Americas-AI-Action-Plan.pdf) +- [Building the AI Workforce - OPM](https://www.opm.gov/chcoc/latest-memos/building-the-ai-workforce-of-the-future.pdf) + +--- + +## AI-Powered General Tools + +### 11. ChatGPT Advanced Data Analysis + +**What It Does:** +OpenAI's ChatGPT with Python code execution (sandbox environment); analyzes data, creates charts, processes files. + +**Target Market:** +Individual researchers, analysts, students; general-purpose data analysis. + +**Pricing:** +- **ChatGPT Plus:** $20/month (GPT-4 + Advanced Data Analysis) +- **ChatGPT Pro:** $200/month (unlimited GPT-4, priority access) + +**Key Features:** +- Executes Python code (pandas, matplotlib, scipy, scikit-learn) +- Upload CSV, Excel, Jupyter Notebooks, ZIP files +- Generate charts (static + interactive) +- Statistical calculations (t-tests, regression) +- No manual coding required + +**AI/ML Capabilities:** +Strong. GPT-4 can: +- Perform exploratory data analysis (EDA) +- Run statistical tests (t-test, ANOVA, chi-squared) +- Train simple ML models (linear regression, logistic regression) +- Generate visualizations + +**Hypothesis-Driven Analysis:** +Partial. User must manually prompt for hypothesis testing. ChatGPT can run t-tests, regressions, but does NOT autonomously formulate hypotheses or design experiments. + +**Public API Integration:** +No. Users must manually upload datasets. No connectors to INEGI, census APIs, or government databases. + +**Evidence-Graded Reports:** +No. ChatGPT summarizes results but does NOT apply GRADE framework or systematic review standards. + +**Limitations:** +- No persistent memory (conversation-level only) +- No access to GPUs (slow for large datasets) +- Large file size limits (memory constraints) +- Cannot manually edit generated code +- No multi-model orchestration (Director-Worker pattern) +- No domain-specific tools (e.g., no INEGI API client, no propensity score matching pipelines) +- User must guide analysis step-by-step (not autonomous) + +**Sources:** +- [Data Analysis with ChatGPT - OpenAI Help](https://help.openai.com/en/articles/8437071-data-analysis-with-chatgpt) +- [ChatGPT for Advanced Data Analysis - MLJAR](https://mljar.com/blog/chatgpt-advanced-data-analysis-python/) +- [Using ChatGPT for Data Science Analyses - HDSR](https://hdsr.mitpress.mit.edu/pub/u6wp4cy3/release/3) + +--- + +### 12. Julius AI + +**What It Does:** +AI-powered data analysis tool; connects to datasets, performs statistical tests, generates charts/reports via natural language. + +**Target Market:** +Researchers, analysts, students; users without programming experience. + +**Pricing:** +Not publicly disclosed. Positioned as accessible for non-programmers. + +**Key Features:** +- Upload spreadsheets (CSV, Excel) +- Instant charts (scatter, bar, line, heatmaps) +- Statistical tests: ANOVA, regression, t-tests, chi-squared +- Hypothesis testing support +- Significance testing (one-tailed, two-tailed, asymptotic, exact, Monte Carlo) +- Writes and executes Python/R code +- Solves math/physics problems + +**AI/ML Capabilities:** +Strong. Julius can: +- Perform hypothesis testing (user-guided) +- Run regression modeling +- Visualize data patterns +- Choose appropriate statistical tests based on data type + +**Hypothesis-Driven Analysis:** +Partial. Julius can EXECUTE hypothesis tests (user-prompted), but does NOT autonomously formulate hypotheses or design multi-hypothesis research plans. + +**Public API Integration:** +No. Users must upload datasets manually. + +**Evidence-Graded Reports:** +No. Julius summarizes statistical results (p-values, effect sizes) but does NOT grade evidence per GRADE or systematic review frameworks. + +**Limitations:** +- User-guided (not autonomous) +- No multi-model orchestration +- No public API connectors (INEGI, census) +- No causal inference pipelines (DiD, PSM, RDD) +- No evidence grading +- Limited to single-dataset analysis (no cross-source integration) + +**Sources:** +- [Julius AI - Hypothesis Testing](https://julius.ai/glossary/hypothesis-testing) +- [Julius AI Guide 2026 - DataCamp](https://www.datacamp.com/tutorial/julius-ai-guide) +- [Julius AI Review - The Effortless Academic](https://effortlessacademic.com/julius-ai-in-depth-review/) + +--- + +### 13. Claude (Anthropic) - Artifacts + +**What It Does:** +Anthropic's Claude with Artifacts feature; generates interactive visualizations, dashboards, and apps via natural language. + +**Target Market:** +Researchers, analysts, developers; general-purpose AI assistant. + +**Pricing:** +- **Claude Free:** Limited usage +- **Claude Pro:** $20/month (Claude 3.5 Sonnet, Opus) +- **Claude Team:** $30/user/month (min 5 users) + +**Key Features:** +- Generate interactive charts (Plotly.js, D3.js) +- Create dashboards (multi-series line, pie, scatter, treemaps) +- Analyze datasets (EDA, correlations, patterns) +- Build web apps (React components) +- Export artifacts (HTML, JSON) + +**AI/ML Capabilities:** +Strong. Claude can: +- Perform exploratory data analysis (EDA) +- Generate interactive visualizations +- Identify patterns and correlations +- Create custom analytics tools + +**Hypothesis-Driven Analysis:** +Partial. User must guide analysis. Claude can assist with hypothesis testing but does NOT autonomously design experiments or formulate hypotheses. + +**Public API Integration:** +No. Users must upload data manually. No connectors to INEGI, census, or government APIs. + +**Evidence-Graded Reports:** +No. Claude summarizes findings but does NOT apply GRADE or systematic review standards. + +**Limitations:** +- Hallucinates with large datasets or complex filters +- User-guided (not autonomous) +- No multi-model orchestration (e.g., Director-Worker-Summarizer) +- No causal inference pipelines (DiD, PSM) +- No evidence grading +- Limited to single-conversation context + +**Sources:** +- [How to Use Claude Artifacts to Visualize Data - Zapier](https://zapier.com/blog/how-to-use-claude-artifacts-to-visualize-data/) +- [Claude Artifacts Catalog](https://claude.ai/catalog/artifacts) +- [Claude Artifacts 101 - DataCamp](https://www.datacamp.com/blog/claude-artifacts-introduction) + +--- + +## Academic/Research AI Tools + +### 14. Systematic Review AI Tools (Rayyan, LASER AI, Cochrane AI) + +**What It Does:** +AI tools for evidence synthesis; automate screening, data extraction, and bias assessment in systematic reviews. + +**Target Market:** +Academic researchers, public health agencies, medical institutions. + +**Pricing:** +- **Rayyan:** Free tier + paid plans +- **LASER AI:** Enterprise pricing (secure AI for systematic reviews) +- **Cochrane AI:** Free for Cochrane reviewers + +**Key Features:** +- **Automated Screening:** ML classifies papers as include/exclude +- **Data Extraction:** AI extracts study characteristics (PICO, sample size, outcomes) +- **Bias Assessment:** Tools like RobotReviewer auto-assess risk of bias +- **PRISMA Diagrams:** Auto-generated (Nested Knowledge, DistillerSR) +- **Citation Chasing:** AI identifies relevant references +- **GPT Integration:** ChatGPT, SciSpace GPT for academic writing + +**AI/ML Capabilities:** +Strong for systematic reviews: +- ML-based screening (reduces manual effort by 50-70%) +- Natural language processing (NLP) for data extraction +- Automated bias scoring + +**Hypothesis-Driven Analysis:** +No. These tools SYNTHESIZE existing research (meta-analysis), they do NOT formulate or test NEW hypotheses with primary data. + +**Public API Integration:** +No. Focused on academic literature (PubMed, Scopus, Web of Science), not government datasets (INEGI, census). + +**Evidence-Graded Reports:** +Yes (for academic research). Tools support GRADE framework, AMSTAR-2, Cochrane Risk of Bias assessments—but ONLY for systematic reviews of published studies, NOT for program evaluation with primary data. + +**Limitations:** +- Academic research only (not program evaluation) +- No primary data analysis (no RCTs, DiD, PSM with program data) +- No public API integration (INEGI, census) +- Risk of AI-induced bias +- Requires human validation (not fully autonomous) + +**Sources:** +- [AI Tools in Evidence Synthesis - King's College London](https://libguides.kcl.ac.uk/systematicreview/ai) +- [LASER AI - Accelerate Evidence Synthesis](https://www.laser.ai/) +- [Rayyan - AI-Powered Systematic Review Platform](https://www.rayyan.ai/) +- [Leveraging AI for Systematic Reviews - Systematic Reviews Journal](https://systematicreviewsjournal.biomedcentral.com/articles/10.1186/s13643-024-02682-2) + +--- + +## Key Limitations Across ALL Platforms + +Based on extensive research, here are the critical gaps in existing tools: + +### 1. Manual Hypothesis Formulation +- **Problem:** Evaluators must manually design hypotheses, experiments, and analysis plans. +- **Impact:** Time-consuming (months to years), requires statistical expertise, prone to confirmation bias. +- **Gap:** No platform autonomously formulates testable hypotheses from research questions. + +### 2. No Causal Inference Automation +- **Problem:** RCTs, DiD, PSM, RDD require manual implementation in Stata/R/Python. +- **Impact:** High technical barrier; most nonprofits lack in-house expertise. +- **Gap:** No platform automates quasi-experimental designs or propensity score matching. + +### 3. Fragmented Data Sources +- **Problem:** Evaluators manually download CSV files from INEGI, census.gov, economic databases. +- **Impact:** Data cleaning/harmonization takes 6+ months (reported by Sopact users). +- **Gap:** No platform auto-connects to public APIs (INEGI, census, economic indicators). + +### 4. No Evidence Grading +- **Problem:** Academic tools (Rayyan, LASER AI) grade evidence for systematic reviews, but NOT for program evaluations with primary data. +- **Impact:** Program reports lack credibility markers (GRADE A-D ratings, certainty of evidence). +- **Gap:** No platform applies GRADE framework to impact evaluations. + +### 5. AI is Descriptive, Not Causal +- **Problem:** Current AI tools (Sopact, DHIS2 AI Insights, ChatGPT) perform pattern detection, summarization, sentiment analysis—NOT causal inference. +- **Impact:** Organizations get "what happened" (descriptive), not "what caused it" (causal). +- **Gap:** No AI platform designs experiments, controls for confounders, or estimates treatment effects. + +### 6. Time Delays +- **Problem:** Traditional evaluations take 1-5 years (design, data collection, analysis, reporting). +- **Impact:** Insights arrive too late to inform program iteration or budget allocation. +- **Gap:** No real-time hypothesis testing with automated data pipelines. + +### 7. High Costs +- **Problem:** Enterprise M&E platforms (DevResults, Amp Impact) cost $4K-$14K/year + per-user fees. +- **Impact:** Small/medium nonprofits cannot afford rigorous evaluation tools. +- **Gap:** No affordable AI-driven causal inference platform. + +--- + +## What Makes Hypothesis-Driven AI UNIQUE? + +Based on competitive analysis, a **hypothesis-driven, multi-model AI engine** would fill the following gaps: + +### Unique Value Propositions + +| Feature | Existing Tools | Hypothesis-Driven AI Engine | Competitive Moat | +|---------|----------------|----------------------------|------------------| +| **Autonomous Hypothesis Formulation** | Manual (user-designed) | AI formulates testable hypotheses from research questions | **NONE OF THE COMPETITORS DO THIS** | +| **Causal Inference Automation** | Manual (Stata/R/Python) | Automated RCT, DiD, PSM, RDD pipelines | **Sopact/DHIS2 only do pattern detection, not causal inference** | +| **Public API Integration** | Manual CSV uploads | Auto-connects to INEGI, census, economic APIs | **ActivityInfo/DevResults require manual imports** | +| **Evidence-Graded Reports** | Descriptive dashboards (or academic-only for systematic reviews) | GRADE framework for program evaluations (A-D ratings, certainty markers) | **Rayyan/LASER AI only grade systematic reviews, not program data** | +| **Multi-Model Orchestration** | Single LLM (ChatGPT, Claude) | Director-Worker-Summarizer (Opus + Sonnet + Haiku) | **No competitor uses multi-agent architecture for evaluation** | +| **Real-Time Testing** | 1-5 year timelines | Hypothesis tested in minutes/hours via API data pipelines | **CONEVAL takes years; AI tools are user-guided, not autonomous** | +| **Domain Flexibility** | Health-specific (DHIS2) or general M&E (DevResults) | Domain-agnostic (social programs, health, education, employment, nutrition) | **DHIS2 is health-only; others lack domain adapters** | +| **Cost** | $4K-$14K/year (enterprise) or $20-$200/month (general AI tools) | Credit-based or BYOK (bring your own key) | **More accessible than DevResults/Amp Impact; more specialized than ChatGPT** | + +--- + +## Market Gap Summary + +### What Does NOT Exist Today + +1. **AI that formulates hypotheses** from research questions (e.g., "Does increasing teacher salaries improve student outcomes?") +2. **Automated causal inference pipelines** (DiD, PSM, RDD) integrated with public APIs (INEGI, census, economic data) +3. **Evidence-graded program evaluation reports** (GRADE A-D ratings, certainty of evidence, effect sizes, confidence intervals) +4. **Multi-model orchestration** for evaluation (Director designs experiments, Workers execute, Summarizer synthesizes) +5. **Real-time hypothesis testing** (from question to evidence-graded answer in minutes/hours, not months/years) +6. **Domain-agnostic evaluation engine** (works for ANY social program: health, education, employment, nutrition, housing) + +### Why This Gap Persists + +1. **Technical Complexity:** Causal inference requires expertise (econometrics, biostatistics); hard to productize. +2. **Fragmented Data:** Public APIs (INEGI, census) exist but lack unified interfaces; data cleaning is manual. +3. **AI Limitations:** Current LLMs excel at pattern detection, NOT causal reasoning (confounders, counterfactuals). +4. **Market Fragmentation:** M&E platforms serve donors (DevResults), health orgs (DHIS2), case management (Bonterra)—no unified solution. +5. **Evaluation Culture:** Government/nonprofit evaluators rely on external consultants (academic researchers, Stata/R experts)—not software. + +--- + +## Strategic Positioning for Ehrlich-Like Platform + +If adapting **Ehrlich's architecture** (hypothesis-driven, multi-model orchestration, domain-agnostic tools) to **social program evaluation**, here's the differentiation: + +### Core Strengths to Leverage + +1. **Hypothesis Formulation Engine (Director Model):** + - Opus 4.6 formulates testable hypotheses from policy questions (e.g., "Does conditional cash transfer reduce school dropout?") + - Generates experiment designs (DiD, PSM, RDD) with control variables, confounders, statistical power calculations + +2. **Automated Causal Inference (Worker Model):** + - Sonnet 4.5 executes quasi-experimental designs via Python tools (statsmodels, linearmodels, DoWhy) + - Connects to INEGI, census, economic indicator APIs for treatment/control group data + - Runs propensity score matching, difference-in-differences, regression discontinuity + +3. **Evidence Grading (Summarizer Model):** + - Haiku 4.5 applies GRADE framework to evaluation results (certainty of evidence, effect sizes, confidence intervals) + - Flags threats to validity (selection bias, attrition, spillovers) + - Generates executive summaries for policymakers + +4. **Domain Flexibility (DomainRegistry):** + - Pluggable domains: education, health, employment, nutrition, housing, public safety + - Each domain has custom tools (e.g., INEGI education census, CONEVAL poverty data, IMSS employment records) + +5. **Real-Time Insights:** + - Hypothesis to evidence-graded report in minutes/hours (vs. months/years for traditional evaluations) + - SSE streaming for live progress updates (like Ehrlich's LiveLabViewer) + +6. **Cost Transparency:** + - Credit-based pricing (Haiku=1cr, Sonnet=3cr, Opus=5cr) or BYOK (bring your own Anthropic API key) + - No per-user fees (vs. DevResults, Amp Impact) + +### Unique Selling Points + +| Competitor | Weakness | Ehrlich-Like Platform Strength | +|------------|----------|-------------------------------| +| **DevResults, Amp Impact** | Manual hypothesis design, no causal inference, expensive ($7K-$14K/year) | Autonomous hypothesis formulation, automated DiD/PSM, credit-based pricing | +| **Sopact** | Pattern detection AI, no causal inference, no evidence grading | Multi-model orchestration for causal inference, GRADE-based evidence grading | +| **DHIS2** | Health-specific, nascent AI (alerts, metadata), no causal inference | Domain-agnostic, mature multi-model architecture, automated quasi-experimental designs | +| **ChatGPT, Julius AI, Claude** | User-guided, single LLM, no causal inference pipelines, no public API connectors | Autonomous multi-model workflow, pre-built tools for DiD/PSM/RDD, INEGI/census API integration | +| **CONEVAL** | Manual evaluations (1-5 years), external consultants, no automation | Real-time hypothesis testing, AI-driven experiment design, evidence-graded reports in minutes | +| **Rayyan, LASER AI** | Systematic reviews only (academic literature), no primary data analysis | Program evaluation with primary data (RCTs, quasi-experimental), public API integration | + +--- + +## Recommendations + +### For Market Entry + +1. **Target Mexican Nonprofits First:** + - CONEVAL sets high bar for rigor (RCTs, quasi-experimental) but evaluations take years + - INEGI APIs available but underutilized (manual CSV downloads) + - Nonprofits lack in-house evaluation expertise + +2. **Freemium Model:** + - Free tier: 5 hypotheses/month (Haiku Director + Sonnet Worker) + - Pro tier: Unlimited hypotheses, Opus Director, priority API access + - BYOK option (users bring Anthropic API key, bypass credits) + +3. **INEGI/Census API Connectors as Moat:** + - Pre-built tools for INEGI education, employment, poverty, health census data + - Auto-harmonize datasets (merge by municipality, state, year) + - Competitor gap: NO platform does this + +4. **Evidence-Graded Reports for Credibility:** + - Integrate GRADE framework (certainty of evidence: high/moderate/low/very low) + - Flag threats to validity (selection bias, confounders, attrition) + - Export to PDF with APA/Chicago citations + +5. **Partnership with CONEVAL:** + - Pilot with CONEVAL to validate AI-driven evaluations against traditional RCTs + - Public case study: "Automated evaluation in 2 weeks vs. 2 years" + +### For Product Differentiation + +1. **Multi-Model Architecture (Ehrlich Pattern):** + - Director (Opus 4.6): Formulates hypotheses, designs experiments + - Worker (Sonnet 4.5): Executes DiD, PSM, RDD with INEGI/census APIs + - Summarizer (Haiku 4.5): Applies GRADE, generates executive summaries + +2. **Domain Adapters:** + - Education: INEGI school census, test scores, dropout rates + - Health: IMSS/ISSSTE utilization, CONEVAL health poverty dimension + - Employment: INEGI occupation/industry data, IMSS payroll records + - Nutrition: ENSANUT dietary surveys, CONEVAL food poverty + +3. **Causal Inference Toolbox:** + - `design_rct`: Randomization protocols, power calculations + - `estimate_did`: Difference-in-differences with parallel trends tests + - `estimate_psm`: Propensity score matching with balance diagnostics + - `estimate_rdd`: Regression discontinuity with bandwidth selection + - `assess_threats`: Check for selection bias, spillovers, attrition + +4. **Public API Integrations (Mexico-Specific):** + - INEGI (census, economic indicators, surveys) + - CONEVAL (poverty data, program evaluations) + - Banxico (economic indicators, inflation, GDP) + - IMSS (employment, payroll records) + - SEP (education statistics) + +5. **Visualization Dashboard (Console):** + - HypothesisBoard: Kanban-style (proposed → testing → supported/refuted) + - EvidenceMatrix: Heatmap of hypotheses × evidence sources + - ForestPlot: Meta-analysis of treatment effects across subgroups + - DIDPlot: Parallel trends visualization + +--- + +## Conclusion + +**No existing platform combines:** +1. Autonomous hypothesis formulation +2. Automated causal inference (DiD, PSM, RDD) +3. Public API integration (INEGI, census, economic data) +4. Evidence-graded reports (GRADE framework) +5. Multi-model orchestration (Director-Worker-Summarizer) +6. Real-time testing (minutes/hours, not months/years) + +**This is the market gap.** + +Ehrlich's architecture (hypothesis-driven, multi-model, domain-agnostic tools) is perfectly suited to fill this gap—especially in Mexico, where: +- CONEVAL sets high evaluation standards but timelines are long (1-5 years) +- INEGI provides rich public APIs but are underutilized (manual downloads) +- Nonprofits lack in-house evaluation expertise (rely on external consultants) +- No AI platform automates causal inference for social programs + +**The opportunity:** Be the first AI-powered, hypothesis-driven, evidence-graded evaluation platform for social programs. + +--- + +## Sources + +### Direct Competitors +- [DevResults Reviews 2026 - SoftwareWorld](https://www.softwareworld.co/software/devresults-reviews/) +- [DevResults Plans](https://www.devresults.com/plans) +- [ActivityInfo Pricing](https://www.activityinfo.org/about/pricing/index.html) +- [ActivityInfo - GetApp](https://www.getapp.com/it-management-software/a/activityinfo/) +- [DHIS2 & AI](https://dhis2.org/ai/) +- [AI-Driven Alert Triage in Tanzania](https://dhis2.org/ai-driven-alert-triage-tanzania/) +- [KoboToolbox Pricing](https://www.kobotoolbox.org/pricing/) +- [KoboToolbox - SaaSworthy](https://www.saasworthy.com/product/kobotoolbox) +- [Salesforce Nonprofit Pricing](https://www.salesforce.com/nonprofit/pricing/) +- [Program Management Module - AppExchange](https://appexchange.salesforce.com/appxListingDetail?listingId=a0N3A00000FMposUAD) +- [Sopact AI-Powered Platform](https://www.sopact.com) +- [AI Powered M&E Tools - Sopact](https://www.sopact.com/use-case/monitoring-and-evaluation-tools) +- [Bonterra Case Management Pricing](https://www.bonterratech.com/pricing/case-management) +- [Bonterra Apricot - Capterra](https://www.capterra.com/p/123342/Bonterra-Case-Management/) +- [Vera Solutions - Amp Impact](https://verasolutions.org/) +- [Amp Impact Pricing - Capterra](https://www.capterra.com/p/173122/Amp-Impact/) + +### Government Initiatives +- [CONEVAL - Monitoring and Evaluation](https://www.coneval.org.mx/Evaluacion/Paginas/Evaluation-and-monitoring-en.aspx) +- [CONEVAL Publications](https://www.coneval.org.mx/InformesPublicaciones/Paginas/Publicaciones-sobre-Evaluacion-y-monitoreo-en.aspx) +- [AI.Gov Action Plan](https://www.ai.gov/action-plan) +- [America's AI Action Plan - July 2025](https://www.whitehouse.gov/wp-content/uploads/2025/07/Americas-AI-Action-Plan.pdf) + +### AI-Powered Tools +- [Data Analysis with ChatGPT - OpenAI Help](https://help.openai.com/en/articles/8437071-data-analysis-with-chatgpt) +- [ChatGPT for Advanced Data Analysis - MLJAR](https://mljar.com/blog/chatgpt-advanced-data-analysis-python/) +- [Julius AI - Hypothesis Testing](https://julius.ai/glossary/hypothesis-testing) +- [Julius AI Guide 2026 - DataCamp](https://www.datacamp.com/tutorial/julius-ai-guide) +- [How to Use Claude Artifacts to Visualize Data - Zapier](https://zapier.com/blog/how-to-use-claude-artifacts-to-visualize-data/) +- [Claude Artifacts Catalog](https://claude.ai/catalog/artifacts) + +### Academic/Research Tools +- [AI Tools in Evidence Synthesis - King's College London](https://libguides.kcl.ac.uk/systematicreview/ai) +- [LASER AI - Accelerate Evidence Synthesis](https://www.laser.ai/) +- [Rayyan - AI-Powered Systematic Review Platform](https://www.rayyan.ai/) +- [Leveraging AI for Systematic Reviews - Systematic Reviews Journal](https://systematicreviewsjournal.biomedcentral.com/articles/10.1186/s13643-024-02682-2) + +### Evaluation Methods & Limitations +- [M&E Challenges - Sopact](https://www.sopact.com/perspectives/monitoring-and-evaluation-challenges) +- [Challenges and Constraints of Evaluations](https://www.effectiveservices.org/journal/the-challenges-and-constraints-of-evaluations) +- [Quasi-Experimental Design - Better Evaluation](https://www.betterevaluation.org/tools-resources/quasi-experimental-design-methods) +- [Propensity Score Matching - Built In](https://builtin.com/data-science/propensity-score-matching) +- [Causal Inference with Python - Towards Data Science](https://towardsdatascience.com/causal-inference-with-python-a-guide-to-propensity-score-matching-b3470080c84f/) + +### Public Data APIs +- [INEGI - National Institute of Statistics and Geography](https://en.www.inegi.org.mx/) +- [INEGI Query Systems](https://en.www.inegi.org.mx/siscon/) +- [Economic Census API - US Census Bureau](https://www.census.gov/programs-surveys/economic-census/data/api.html) diff --git a/docs/domain-agnostic.md b/docs/domain-agnostic.md index eba5b44..386b2af 100644 --- a/docs/domain-agnostic.md +++ b/docs/domain-agnostic.md @@ -141,23 +141,37 @@ Converting the CandidateTable from 4 hardcoded columns to dynamic `score_columns ## The Horizon -### Near-Term Domains (Weeks) +### Next Domain: Impact Evaluation (In Planning) -Domains that could be added with just tools + config, no engine changes: +The fourth domain proves the architecture handles non-scientific domains too. **Impact Evaluation** applies the same hypothesis-driven loop to social program analysis: -- **Genomics** -- Gene expression analysis, pathway enrichment, variant interpretation. Tools: search gene databases (NCBI Gene, Ensembl), compute expression metrics, compare pathways. -- **Materials Science** -- Property prediction, crystal structure search, phase diagram analysis. Tools: search materials databases (Materials Project), compute material properties, compare compositions. -- **Environmental Science** -- Ecosystem assessment, biodiversity metrics, climate impact analysis. Tools: search ecological databases, compute impact scores, compare interventions. +- "Does the Sonora sports scholarship improve athlete competition performance?" → DiD + PSM +- "Is the conditional cash transfer reducing school dropout rates?" → RDD +- "Compare cost-effectiveness of state sports programs" → Synthetic Control + benchmarking + +**Why it matters:** Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evaluations (0.4%). No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. Ehrlich fills this gap. + +**What's new:** Document/CSV upload (user-provided program data), causal inference tools (DiD, PSM, RDD, synthetic control), Mexico + US public data API clients (INEGI, Census, World Bank, FRED), evaluation report templates (CONEVAL ECR, WWC-compliant). See `docs/impact-evaluation-domain.md` for full design. + +**What stays the same:** The DomainConfig, tool tagging, multi-model orchestration, hypothesis framework, evidence grading, and visualization pipeline are all reused. Zero changes to existing code paths. + +### Other Planned Domains + +- **Competitive Sports** -- game statistics, player performance, team analysis +- **Genomics** -- gene expression, pathway enrichment, variant interpretation +- **Materials Science** -- property prediction, crystal structure search +- **Environmental Science** -- ecosystem assessment, biodiversity metrics ### Medium-Term Evolution (Months) -- **Cross-domain investigations** -- "Find compounds that improve both athletic recovery AND reduce inflammation" would use tools from molecular + training + nutrition domains simultaneously. -- **Domain-specific validation** -- Each domain could define its own validation methodology in the DomainConfig (the scientific methodology research is already done for molecular; same treatment for other domains). +- **Cross-domain investigations** -- "Find compounds that improve both athletic recovery AND reduce inflammation" would use tools from molecular + training + nutrition domains simultaneously. Already works via `merge_domain_configs()`. +- **Impact + Training + Nutrition** -- "Evaluate the Sonora sports program: athlete performance, nutrition support, and cost-effectiveness" triggers all three domains. - **Custom domain builder** -- A UI where researchers define their own DomainConfig: name their scores, pick their tools, write their prompt examples. No code required. +- **MCP bridge self-service** -- Users register their own data sources (e.g., CODESON internal database, Colombian education ministry) as MCP servers. ### Long-Term Vision -The engine becomes a general-purpose scientific reasoning framework. The domains are plugins. Ehrlich is a scientific discovery engine that currently ships with three domains (Molecular Science, Training Science, Nutrition Science). The architecture supports any number. +The engine becomes a general-purpose scientific reasoning framework. The domains are plugins. Ehrlich is a scientific discovery engine that currently ships with three domains (Molecular Science, Training Science, Nutrition Science) with a fourth (Impact Evaluation) in planning. The architecture supports any number. ## Architecture Summary diff --git a/docs/impact-evaluation-domain.md b/docs/impact-evaluation-domain.md new file mode 100644 index 0000000..55f3e57 --- /dev/null +++ b/docs/impact-evaluation-domain.md @@ -0,0 +1,505 @@ +# Impact Evaluation Domain -- Comprehensive Design Document + +## Executive Summary + +Ehrlich's **Impact Evaluation** domain enables hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. + +**Key insight:** Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evaluations (0.4%). No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. + +--- + +## 1. Market Gap Analysis + +### What Exists Today + +| Category | Tools | Limitation | +|----------|-------|------------| +| Traditional M&E | DevResults, ActivityInfo, Bonterra, Amp Impact | Data collection + dashboards only. No causal inference. No AI. $4K-$14K/year | +| AI-enhanced M&E | Sopact Impact Cloud, DHIS2 | Pattern detection only. No causal inference. Health-specific (DHIS2) | +| General AI | ChatGPT, Julius AI, Claude | User-guided step-by-step. No autonomous hypothesis formulation. No public API connectors | +| Academic | Rayyan, LASER AI, Cochrane | Systematic reviews of published literature only. No primary program data | +| Government | CONEVAL evaluations | Manual process. 1-5 year timelines. External consultants | + +### 6 Gaps No Competitor Fills + +1. **Autonomous hypothesis formulation** from research questions +2. **Automated causal inference** (DiD, PSM, RDD, synthetic control) +3. **Public API integration** (INEGI, Census, World Bank, BLS) +4. **Evidence-graded reports** (GRADE framework on primary data) +5. **Multi-model orchestration** (Director-Worker-Summarizer) +6. **Real-time testing** (minutes/hours vs years) + +--- + +## 2. Data Sources + +### Tier 1 -- Built-in APIs (Universal) + +| Source | API Base URL | Auth | Data | Priority | +|--------|-------------|------|------|----------| +| World Bank | `https://api.worldbank.org/v2/` | None | Development indicators, GDP, education, health by country | HIGH | +| WHO GHO | `https://ghoapi.azureedge.net/api/` | None | Health statistics by country | HIGH | +| FRED | `https://api.stlouisfed.org/fred/` | API key | 800K+ economic time series | HIGH | +| OECD | `https://sdmx.oecd.org/public/rest/` | None | Policy, education, employment indicators | MEDIUM | + +### Tier 2 -- Mexico APIs + +| Source | API Base URL | Auth | Data | Priority | +|--------|-------------|------|------|----------| +| INEGI Indicadores | `https://www.inegi.org.mx/app/api/indicadores/` | Token | Census, demographics, economic indicators | HIGH | +| INEGI DENUE | `https://www.inegi.org.mx/app/api/denue/v1/` | Token | 5M+ business establishments by location | MEDIUM | +| datos.gob.mx | `https://datos.gob.mx/api/3/action/` | Token (optional) | 1000+ federal datasets (CKAN) | HIGH | +| Transparencia Presupuestaria | `https://nptp.hacienda.gob.mx/apoyosdelgobierno/` | None | Budget execution, social program spending | HIGH | +| Banxico SIE | `https://www.banxico.org.mx/SieAPIRest/service/v1/` | Token (header) | Exchange rates, inflation, GDP | MEDIUM | + +**No API available (PDF/manual only):** CONEVAL, CONADE, IMSS/ISSSTE, SIIPP-G + +### Tier 3 -- US APIs + +| Source | API Base URL | Auth | Data | Priority | +|--------|-------------|------|------|----------| +| Census Bureau | `https://api.census.gov/data/` | API key (optional) | ACS, Decennial, demographics | HIGH | +| BLS | `https://api.bls.gov/publicAPI/v2/timeseries/data/` | API key | Employment, wages, CPI | HIGH | +| USAspending | `https://api.usaspending.gov/api/v2/` | None | Federal grants, contracts, spending | HIGH | +| FRED | (same as Tier 1) | API key | Economic indicators | HIGH | +| College Scorecard | `https://api.data.gov/ed/collegescorecard/v1/schools` | API key | Higher education outcomes | MEDIUM | +| HUD | `https://www.huduser.gov/hudapi/public/` | Token | Fair Market Rent, income limits, housing | MEDIUM | +| CDC WONDER | `https://wonder.cdc.gov/` | None | Mortality, disease, public health | MEDIUM | +| EPA Envirofacts | `https://data.epa.gov/efservice/` | API key | Environmental quality, compliance | LOW | + +### Tier 4 -- User-Provided Data (Always Available) + +- **CSV/Excel upload** -- athlete metrics, student grades, beneficiary records +- **PDF upload** -- evaluation reports, program documents, policy briefs +- **MCP bridges** -- user-registered external data sources (extensible) + +--- + +## 3. Evaluation Methodology + +### Regulatory Frameworks + +**Mexico (CONEVAL):** +- Marco Logico (MIR): Fin / Proposito / Componentes / Actividades +- PbR-SED: Results-Based Budgeting + Performance Evaluation System +- 6 evaluation types: Diseno, Consistencia y Resultados (ECR), Procesos, Indicadores, Impacto, Estrategicas +- CREMAA indicator criteria: Claridad, Relevancia, Economia, Monitoreable, Adecuado, Aportacion marginal + +**US (Federal):** +- Evidence Act 2018: systematic evidence-building plans +- GPRA Modernization Act: integrated performance frameworks +- WWC Standards v5.0: 4-tier evidence hierarchy (Strong, Moderate, Promising, Rationale) +- GAO evaluation standards: transparent documentation and public reporting + +### Causal Inference Methods (Tools to Build) + +| Method | When Used | Key Assumptions | Evidence Strength | +|--------|-----------|-----------------|-------------------| +| RCT analysis | Random assignment data available | SUTVA, no spillovers | Gold standard | +| Difference-in-Differences (DiD) | Pre/post data, treatment/control groups | Parallel trends | High | +| Regression Discontinuity (RDD) | Sharp eligibility cutoff exists | Continuity at threshold | High | +| Propensity Score Matching (PSM) | Rich covariate data, no random assignment | Selection on observables | Moderate-High | +| Synthetic Control | Single treated unit, multiple controls | Weighted combination matches pre-treatment | Moderate | +| Instrumental Variables (IV) | Valid instrument exists | Exclusion restriction | Moderate-High | + +### Evidence Grading + +Extend existing GRADE implementation to social program context: +- **Certainty of evidence:** high / moderate / low / very low +- **Downgrading factors:** risk of bias, inconsistency, indirectness, imprecision, publication bias +- **Upgrading factors:** large effect, dose-response, plausible confounding +- **WWC tier mapping:** Tier 1 (strong) through Tier 4 (rationale) + +--- + +## 4. Server Architecture + +### New Bounded Context: `impact/` + +``` +server/src/ehrlich/impact/ + domain/ + entities.py # Program, Indicator, Beneficiary, CausalEstimate, ThreatToValidity + repository.py # ProgramDataRepository, IndicatorRepository ABCs + ports.py # CausalEstimator, DataHarmonizer, CostAnalyzer ABCs + dri.py # (n/a -- equivalent would be evaluation_standards.py) + evaluation_standards.py # CONEVAL ECR template, WWC tiers, GRADE criteria + application/ + impact_service.py # Core service: causal estimation, evidence grading, benchmarking + infrastructure/ + inegi_client.py # INEGI Indicadores + DENUE APIs + datos_gob_client.py # datos.gob.mx CKAN API + transparencia_client.py # Transparencia Presupuestaria API + banxico_client.py # Banxico SIE API + census_client.py # US Census Bureau API + bls_client.py # Bureau of Labor Statistics API + usaspending_client.py # USAspending.gov API + worldbank_client.py # World Bank API + who_client.py # WHO GHO API + fred_client.py # FRED API (already used in other contexts? no -- new) + data_harmonizer.py # Merge datasets by geography/time period + csv_parser.py # Parse user-uploaded CSV/Excel files + pdf_extractor.py # Extract text from uploaded PDFs + tools.py # Impact evaluation tools +``` + +### Domain Entities + +```python +@dataclass +class Program: + name: str + description: str + country: str # ISO 3166-1 alpha-2 + region: str # state/province + sector: str # education, health, sports, employment, housing + start_date: str + end_date: str | None + budget: float | None + currency: str + beneficiary_count: int | None + indicators: list[Indicator] + +@dataclass(frozen=True) +class Indicator: + name: str + level: str # fin, proposito, componente, actividad (MIR) or outcome, output, activity (generic) + unit: str + baseline: float | None + target: float | None + actual: float | None + period: str + +@dataclass(frozen=True) +class CausalEstimate: + method: str # did, psm, rdd, synthetic_control, iv, rct + effect_size: float + standard_error: float + confidence_interval: tuple[float, float] + p_value: float + n_treatment: int + n_control: int + covariates: list[str] + assumptions: list[str] + threats: list[ThreatToValidity] + evidence_tier: str # strong, moderate, promising, rationale + +@dataclass(frozen=True) +class ThreatToValidity: + type: str # selection_bias, attrition, history, maturation, spillover, contamination + severity: str # high, moderate, low + description: str + mitigation: str | None + +@dataclass(frozen=True) +class Benchmark: + source: str # worldbank, who, inegi, literature + indicator: str + value: float + unit: str + geography: str + period: str + url: str | None +``` + +### Tools (Proposed: ~18 new tools) + +**Data Access (6):** +- `search_census_data` -- Query INEGI or US Census by indicator/geography/period +- `search_economic_indicators` -- Query FRED, Banxico, World Bank time series +- `search_budget_data` -- Query Transparencia Presupuestaria or USAspending +- `search_health_indicators` -- Query WHO GHO or CDC WONDER +- `search_open_data` -- Query datos.gob.mx or data.gov CKAN portals +- `fetch_benchmark` -- Get comparison value from international sources (World Bank, OECD) + +**Causal Inference (5):** +- `estimate_did` -- Difference-in-differences with parallel trends test +- `estimate_psm` -- Propensity score matching with balance diagnostics +- `estimate_rdd` -- Regression discontinuity (sharp/fuzzy) +- `estimate_synthetic_control` -- Synthetic control method +- `assess_threats` -- Automated threat-to-validity assessment + +**Program Analysis (4):** +- `analyze_program_indicators` -- MIR/logical framework indicator analysis with CREMAA validation +- `compute_cost_effectiveness` -- Cost per unit outcome, ICER +- `compare_programs` -- Cross-program or cross-region comparison with statistical tests +- `analyze_beneficiary_data` -- Descriptive + inferential analysis of beneficiary records + +**Document Processing (3):** +- `extract_program_data` -- Parse uploaded PDF/CSV and extract structured program data +- `extract_mir` -- Extract MIR (Matriz de Indicadores) from evaluation documents +- `generate_evaluation_report` -- Format findings into CONEVAL ECR or WWC-compliant template + +### Visualization Tools (Proposed: ~6 new viz tools) + +- `render_causal_diagram` -- DAG showing treatment, outcome, confounders +- `render_parallel_trends` -- Pre/post treatment vs control trends (DiD visual) +- `render_rdd_plot` -- Regression discontinuity scatter with cutoff +- `render_program_dashboard` -- Multi-indicator KPI dashboard +- `render_cost_effectiveness` -- Cost-effectiveness frontier chart +- `render_geographic_comparison` -- State/region comparison bar chart with benchmarks + +--- + +## 5. Domain Config + +```python +IMPACT_EVALUATION = DomainConfig( + name="Impact Evaluation", + description="Hypothesis-driven causal analysis of social programs", + tool_tags=frozenset({ + "impact", # core impact evaluation tools + "census", # INEGI, Census Bureau + "economic", # FRED, Banxico, World Bank + "budget", # Transparencia Presupuestaria, USAspending + "health", # WHO, CDC + "opendata", # datos.gob.mx, data.gov + "statistics", # existing statistical testing tools + "literature", # existing literature search + "ml", # existing generic ML tools + "visualization" # viz tools + }), + score_definitions=[ + ScoreDefinition( + name="effect_size", + description="Standardized effect size (Cohen's d, odds ratio, or beta coefficient)" + ), + ScoreDefinition( + name="statistical_significance", + description="P-value from causal inference method" + ), + ScoreDefinition( + name="evidence_tier", + description="WWC evidence tier: strong (RCT), moderate (quasi-experimental), promising (correlational), rationale (logic model)" + ), + ScoreDefinition( + name="cost_effectiveness", + description="Cost per unit outcome improvement" + ), + ], + experiment_examples=[ + # DiD example + """Experiment: Does conditional cash transfer reduce school dropout? + Method: Difference-in-differences + Steps: + 1. search_census_data(indicator="school_enrollment", geography="sonora", years="2018-2024") + 2. search_census_data(indicator="school_enrollment", geography="chihuahua", years="2018-2024") # control + 3. estimate_did(treatment_data=sonora, control_data=chihuahua, treatment_start="2021") + 4. assess_threats(method="did", data=results) + 5. fetch_benchmark(indicator="dropout_rate", source="worldbank", country="MX")""", + + # PSM example + """Experiment: Does sports scholarship improve competition performance? + Method: Propensity score matching + Steps: + 1. extract_program_data(file="athletes_2025.csv") + 2. estimate_psm(treatment="scholarship_recipient", outcome="competition_rank", covariates=["age", "sport", "baseline_rank"]) + 3. assess_threats(method="psm", data=results) + 4. compute_cost_effectiveness(program_cost=budget, outcome_improvement=effect)""", + + # RDD example + """Experiment: Does eligibility threshold for housing subsidy improve outcomes? + Method: Regression discontinuity + Steps: + 1. search_budget_data(program="housing_subsidy", year="2024") + 2. extract_program_data(file="beneficiaries.csv") + 3. estimate_rdd(running_variable="income_score", cutoff=2500, outcome="housing_quality_index") + 4. render_rdd_plot(data=results)""" + ], + detection_keywords=[ + "program evaluation", "impact evaluation", "social program", + "programa social", "evaluacion de impacto", "evaluacion de consistencia", + "policy evaluation", "cost-effectiveness", "cost-benefit", + "beneficiaries", "beneficiarios", "CONEVAL", "MIR", + "difference-in-differences", "propensity score", "regression discontinuity", + "causal inference", "treatment effect", "counterfactual", + "public policy", "politica publica", "gobierno", "government program", + "sports program", "education program", "health program", + "scholarship", "beca", "subsidio", "subsidy", + "dropout rate", "desercion", "enrollment", "matricula", + "INEGI", "census", "censo" + ] +) +``` + +--- + +## 6. Frontend Components + +### New Components + +| Component | Purpose | +|-----------|---------| +| `FileUpload.tsx` | Drag-and-drop CSV/PDF upload with file type validation | +| `DataPreview.tsx` | Preview uploaded dataset (columns, types, sample rows) | +| `CausalDiagram.tsx` | Interactive DAG (treatment, outcome, confounders) | +| `ParallelTrendsChart.tsx` | DiD visual with pre/post treatment trends | +| `RDDPlot.tsx` | Regression discontinuity scatter with fitted lines | +| `ProgramDashboard.tsx` | Multi-indicator KPI cards with sparklines | +| `CostEffectivenessFrontier.tsx` | Scatter of programs by cost vs outcome | +| `GeographicComparison.tsx` | State/region bar chart with benchmark lines | +| `EvaluationReport.tsx` | CONEVAL ECR or WWC-formatted evaluation report | +| `MIRTable.tsx` | Logical framework matrix (Fin/Proposito/Componentes/Actividades) | +| `ThreatAssessment.tsx` | Validity threats panel with severity badges | + +### Modified Components + +| Component | Change | +|-----------|--------| +| `PromptInput.tsx` | Add file upload zone (CSV, PDF, Excel) | +| `TemplateCards.tsx` | Add impact evaluation templates (sports program, education, health) | +| `InvestigationReport.tsx` | Add evaluation-specific sections (causal estimates, MIR, threats) | +| `VizRegistry.tsx` | Register 6 new visualization types | + +--- + +## 7. MCP Bridge Strategy + +### Pre-built Bridges (ship with platform) + +None initially. All Tier 1-3 APIs are direct integrations. + +### User-Registerable Bridges (extensible) + +Users or organizations register MCP servers for their specific data: + +``` +# Example: CODESON (Sonora Sports Commission) +ehrlich-mcp-codeson: + transport: http + url: https://codeson-data.example.com/mcp + tools: + - get_athlete_data + - get_competition_results + - get_program_beneficiaries + +# Example: Colombian education ministry +ehrlich-mcp-mineducacion: + transport: http + url: https://mineducacion-data.example.com/mcp + tools: + - get_student_enrollment + - get_test_scores + - get_school_infrastructure +``` + +### MCP Registration UI + +New settings page: `/settings/data-sources` +- Add MCP server (URL, transport, auth) +- Test connection +- View available tools +- Tag tools with domain tags for auto-discovery + +--- + +## 8. Document Upload Architecture + +### API Endpoints + +``` +POST /api/v1/investigate + Content-Type: multipart/form-data + Fields: + - prompt: string (research question) + - director_model: string (haiku/sonnet/opus) + - files[]: File[] (CSV, PDF, XLSX -- max 10 files, 50MB total) + +POST /api/v1/upload + Content-Type: multipart/form-data + Fields: + - file: File + Response: + - file_id: string + - filename: string + - content_preview: string (first 500 chars of extracted text) + - columns: string[] (for CSV/Excel) + - row_count: int (for CSV/Excel) +``` + +### Processing Pipeline + +``` +File upload + -> Type detection (CSV, PDF, XLSX) + -> CSV/XLSX: parse with pandas, return schema + preview + -> PDF: extract text with pymupdf, summarize with Haiku if >4000 chars + -> Inject into Director prompt as XML block + -> For CSV: include column names, dtypes, summary statistics, sample rows + -> For PDF: include extracted/summarized text +``` + +### Dependencies + +- `pymupdf` (fitz) -- PDF text extraction (BSD license, no system deps) +- `openpyxl` -- Excel file reading (MIT license) +- `pandas` -- CSV/Excel parsing + descriptive statistics (already indirect dep via scipy) + +--- + +## 9. Differentiation Summary + +| Dimension | Traditional M&E | General AI (ChatGPT) | Ehrlich Impact Evaluation | +|-----------|----------------|----------------------|---------------------------| +| Hypothesis formulation | Manual | User-guided | Autonomous (Director) | +| Causal inference | None | Manual code | Built-in tools (DiD, PSM, RDD) | +| Public data APIs | None | None | 15+ integrated sources | +| Evidence grading | None | None | GRADE + WWC tiers | +| Multi-model | N/A | Single LLM | Director-Worker-Summarizer | +| Time to result | Months/years | Hours (manual) | Minutes (autonomous) | +| Cost | $4K-$14K/year + consultants | API costs | Credit-based ($1-5 per investigation) | +| Mexico compliance | Some (CONEVAL) | None | MIR, ECR, PbR-SED templates | +| US compliance | Some | None | WWC tiers, Evidence Act alignment | + +--- + +## 10. Implementation Phases + +### Phase 1: Foundation (document upload + universal APIs) +- File upload API (CSV, PDF, Excel) +- `extract_program_data` tool +- World Bank, FRED, WHO API clients +- Basic `compare_programs` tool using existing `run_statistical_test` +- 2 new viz tools: `render_program_dashboard`, `render_geographic_comparison` + +### Phase 2: Causal Inference Engine +- `estimate_did`, `estimate_psm`, `estimate_rdd` tools +- `assess_threats` automated validity assessment +- `compute_cost_effectiveness` tool +- 3 new viz tools: `render_parallel_trends`, `render_rdd_plot`, `render_causal_diagram` + +### Phase 3: Mexico Integration +- INEGI Indicadores + DENUE clients +- datos.gob.mx CKAN client +- Transparencia Presupuestaria client +- Banxico SIE client +- `extract_mir` tool + `MIRTable.tsx` component +- CONEVAL ECR report template + +### Phase 4: US Integration +- Census Bureau client +- BLS client +- USAspending client +- College Scorecard client +- WWC evidence tier classification +- US-formatted evaluation report template + +### Phase 5: MCP Bridge Self-Service +- User MCP registration UI +- Connection testing +- Tool auto-discovery and tagging +- Per-organization data source management + +--- + +## References + +- [CONEVAL Evaluation Process](https://www.coneval.org.mx/Evaluacion/Paginas/Proceso-de-Evaluacion.aspx) +- [WWC Standards Handbook v5.0](https://ies.ed.gov/ncee/wwc/Docs/referenceresources/Final_WWC-HandbookVer5.0-0-508.pdf) +- [Evidence Act 2018](https://www.congress.gov/bill/115th-congress/house-bill/4174) +- [INEGI API](https://www.inegi.org.mx/servicios/api_indicadores.html) +- [Census Bureau API](https://api.census.gov/data/) +- [World Bank API](https://api.worldbank.org/v2/) +- [FRED API](https://api.stlouisfed.org/fred/) +- [CGD: Only 11 Impact Evaluations out of 2,800](https://pubs.cgdev.org/evidence-to-impact/1-progress-and-challenges/challenges/index.html) +- See also: `docs/competitive-landscape-social-programs.md`, `docs/us-public-data-apis.md` diff --git a/docs/roadmap.md b/docs/roadmap.md index 7bcf2f1..017e3fb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -197,7 +197,7 @@ The core: Claude as an autonomous scientist. - [x] Tests: mock API, verify request/response handling ### 5B. Tool Registry -- [x] Register all tools from all contexts (6 chemistry, 3 literature, 6 analysis, 3 prediction, 7 simulation, 11 training, 10 nutrition, 7 investigation control -- 38 at time of Phase 5; now 70 with API tools + visualization tools + statistics tools + generic ML tools) +- [x] Register all tools from all contexts (6 chemistry, 3 literature, 6 analysis, 3 prediction, 7 simulation, 11 training, 10 nutrition, 7 investigation control -- 38 at time of Phase 5; now 73 with API tools + visualization tools + statistics tools + generic ML tools + impact tools) - [x] Auto-generate JSON Schema from Python type hints + docstrings - [x] `get(name)` -> callable, `list_tools()` -> all registered tools, `list_schemas()` -> Anthropic-compatible schemas - [x] Schema format matches Anthropic tool_use specification @@ -330,7 +330,7 @@ Cost-efficient multi-model orchestration, persistence, and UI polish. ### 8C. Multi-Model Orchestrator - [x] `MultiModelOrchestrator` with hypothesis-driven Director-Worker-Summarizer pattern - [x] Director (Opus) formulates hypotheses, designs experiments, evaluates evidence, synthesizes -- NO tool access -- [x] Researcher (Sonnet) executes experiments with 70 domain-filtered tools (max 10 iterations per experiment) +- [x] Researcher (Sonnet) executes experiments with 73 domain-filtered tools (max 10 iterations per experiment) - [x] Summarizer (Haiku) compresses large outputs exceeding threshold - [x] 7 prompts: director formulation/experiment/evaluation/synthesis, researcher experiment, scientist, summarizer - [x] Auto-fallback to single-model Orchestrator when researcher == director @@ -469,6 +469,10 @@ Enhancement Enhancement Sports Phase 12: SaaS Infrastructure -- DONE (PostgreSQL + WorkOS Auth + BYOK + Credits) | + Phase 13: Impact Evaluation Domain -- IN PROGRESS + (Causal inference + Document upload + MX/US APIs) + 13A (Foundation) DONE -> 13A-2 (Upload+Viz) -> 13B (Causal) -> 13C (Mexico) -> 13D (US) -> 13E (MCP Self-Service) + | Demo + Video -- TODO | Submission (Feb 16, 3PM ET) -- TODO @@ -700,7 +704,7 @@ Fixed incorrect hardcoded pricing and added cache hit/miss token tracking for ac ### SDK-2: Prompt Caching on Tools Array -- DONE -Cache the 70-tool schema array that repeats on every researcher API call. +Cache the 73-tool schema array that repeats on every researcher API call. - [x] Add `cache_control: {"type": "ephemeral"}` to the last tool in the tools array before passing to `messages.create` - [x] Only apply when tools list is non-empty @@ -987,6 +991,115 @@ Deepened the nutrition bounded context with 1 new data source, 6 new tools, and --- +## Phase 13: Impact Evaluation Domain -- IN PROGRESS + +New bounded context for hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. See `docs/impact-evaluation-domain.md` for full design. + +### Market Gap + +Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evaluations (0.4%). No competitor combines: autonomous hypothesis formulation + causal inference + public API connectors + evidence grading + multi-model orchestration. + +### Phase 13A: Foundation (Universal APIs + Domain Config) -- DONE + +- [x] World Bank API client (`https://api.worldbank.org/v2/`) -- development indicators by country +- [x] WHO GHO API client (`https://ghoapi.azureedge.net/api/`) -- health statistics +- [x] FRED API client (`https://api.stlouisfed.org/fred/`) -- 800K+ economic time series +- [x] `search_economic_indicators` tool -- query FRED/World Bank/WHO GHO time series +- [x] `fetch_benchmark` tool -- get comparison values from international sources +- [x] `compare_programs` tool -- cross-program comparison using existing statistical tests +- [x] `IMPACT_EVALUATION` domain config with detection keywords and experiment examples +- [x] Full DDD: domain entities + repository ABCs + infrastructure clients +- [x] Tests: impact bounded context (unit + integration) + +**Counts:** 70 -> 73 tools, 16 -> 19 data sources (18 external + 1 internal), 10 -> 11 bounded contexts, 3 -> 4 domains. + +### Phase 13A-2: Document Upload + Visualization (TODO) + +- [ ] File upload API (CSV, PDF, Excel) via multipart/form-data on `POST /investigate` +- [ ] PDF text extraction (`pymupdf`) + Haiku summarization for long documents +- [ ] CSV/Excel parsing (`pandas` + `openpyxl`) with schema detection and summary statistics +- [ ] Inject uploaded data into Director prompt as `` XML block +- [ ] `extract_program_data` tool -- parse uploaded files into structured program data +- [ ] `render_program_dashboard` viz tool -- multi-indicator KPI dashboard +- [ ] `render_geographic_comparison` viz tool -- state/region comparison bar chart +- [ ] Frontend: `FileUpload.tsx` drag-and-drop component in `PromptInput` +- [ ] Frontend: `DataPreview.tsx` for uploaded dataset preview + +### Phase 13B: Causal Inference Engine + +- [ ] `estimate_did` tool -- difference-in-differences with parallel trends test +- [ ] `estimate_psm` tool -- propensity score matching with balance diagnostics +- [ ] `estimate_rdd` tool -- regression discontinuity (sharp/fuzzy) +- [ ] `estimate_synthetic_control` tool -- synthetic control method +- [ ] `assess_threats` tool -- automated threat-to-validity assessment +- [ ] `compute_cost_effectiveness` tool -- cost per unit outcome, ICER +- [ ] `CausalEstimate` domain entity with effect size, CI, p-value, threats +- [ ] `ThreatToValidity` domain entity (selection bias, attrition, spillover) +- [ ] `render_parallel_trends` viz tool -- DiD pre/post trends visual +- [ ] `render_rdd_plot` viz tool -- regression discontinuity scatter +- [ ] `render_causal_diagram` viz tool -- DAG showing treatment/outcome/confounders +- [ ] Frontend: `ThreatAssessment.tsx` panel with severity badges + +### Phase 13C: Mexico Integration + +- [ ] INEGI Indicadores API client (`https://www.inegi.org.mx/app/api/indicadores/`) -- census, demographics +- [ ] INEGI DENUE API client (`https://www.inegi.org.mx/app/api/denue/v1/`) -- 5M+ businesses +- [ ] datos.gob.mx CKAN client (`https://datos.gob.mx/api/3/action/`) -- 1000+ federal datasets +- [ ] Transparencia Presupuestaria client (`https://nptp.hacienda.gob.mx/`) -- budget execution +- [ ] Banxico SIE client (`https://www.banxico.org.mx/SieAPIRest/service/v1/`) -- economic indicators +- [ ] `search_census_data` tool -- query INEGI/Census by indicator/geography/period +- [ ] `search_budget_data` tool -- query budget execution and social program spending +- [ ] `extract_mir` tool -- extract MIR (Matriz de Indicadores) from evaluation documents +- [ ] `analyze_program_indicators` tool -- MIR/logical framework indicator analysis with CREMAA validation +- [ ] Frontend: `MIRTable.tsx` logical framework matrix +- [ ] CONEVAL ECR report template for `generate_evaluation_report` + +### Phase 13D: US Integration + +- [ ] Census Bureau API client (`https://api.census.gov/data/`) -- ACS, demographics +- [ ] BLS API client (`https://api.bls.gov/publicAPI/v2/`) -- employment, wages, CPI +- [ ] USAspending API client (`https://api.usaspending.gov/api/v2/`) -- federal grants/contracts +- [ ] College Scorecard API client (`https://api.data.gov/ed/collegescorecard/v1/`) -- education outcomes +- [ ] HUD API client (`https://www.huduser.gov/hudapi/public/`) -- housing data +- [ ] CDC WONDER client (`https://wonder.cdc.gov/`) -- public health data +- [ ] `search_health_indicators` tool -- query WHO GHO or CDC WONDER +- [ ] `search_open_data` tool -- query datos.gob.mx or data.gov CKAN portals +- [ ] WWC evidence tier classification in evaluation reports +- [ ] US-formatted evaluation report template + +### Phase 13E: MCP Bridge Self-Service + +- [ ] User MCP server registration UI (`/settings/data-sources`) +- [ ] Connection testing and tool auto-discovery +- [ ] Per-organization data source management +- [ ] Tool auto-tagging with domain tags for discovery + +### Data Sources (Planned: 8+ new, 3 implemented in 13A) + +| Source | API | Purpose | Auth | Phase | +|--------|-----|---------|------|-------| +| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country | None | 13A | +| WHO GHO | `https://ghoapi.azureedge.net/api/` | Health statistics by country | None | 13A | +| FRED | `https://api.stlouisfed.org/fred/` | 800K+ economic time series | API key | 13A | +| INEGI Indicadores | `https://www.inegi.org.mx/app/api/indicadores/` | Mexico census, demographics | Token | 13C | +| INEGI DENUE | `https://www.inegi.org.mx/app/api/denue/v1/` | Mexico business directory (5M+) | Token | 13C | +| datos.gob.mx | `https://datos.gob.mx/api/3/action/` | Mexico open data (CKAN, 1000+ datasets) | Token (optional) | 13C | +| Transparencia Presupuestaria | `https://nptp.hacienda.gob.mx/` | Mexico budget execution | None | 13C | +| Banxico SIE | `https://www.banxico.org.mx/SieAPIRest/service/v1/` | Mexico economic indicators | Token | 13C | +| US Census Bureau | `https://api.census.gov/data/` | ACS, demographics | API key (optional) | 13D | +| BLS | `https://api.bls.gov/publicAPI/v2/` | Employment, wages, CPI | API key | 13D | +| USAspending | `https://api.usaspending.gov/api/v2/` | Federal grants, contracts | None | 13D | + +### Tools (Planned: ~18 new tools + ~6 viz tools) + +**Data Access:** `search_census_data`, `search_economic_indicators`, `search_budget_data`, `search_health_indicators`, `search_open_data`, `fetch_benchmark` +**Causal Inference:** `estimate_did`, `estimate_psm`, `estimate_rdd`, `estimate_synthetic_control`, `assess_threats` +**Program Analysis:** `analyze_program_indicators`, `compute_cost_effectiveness`, `compare_programs`, `analyze_beneficiary_data` +**Document Processing:** `extract_program_data`, `extract_mir`, `generate_evaluation_report` +**Visualization:** `render_causal_diagram`, `render_parallel_trends`, `render_rdd_plot`, `render_program_dashboard`, `render_cost_effectiveness`, `render_geographic_comparison` + +--- + ## Competitive Sports Domain -- TODO New bounded context for actual competitive sports analytics: game statistics, player performance, team analysis, and sports-specific strategy research. @@ -1081,7 +1194,7 @@ Side-by-side comparison of investigation runs for reproducibility and consensus ## ~~MCP Server~~ -- REJECTED -Exposing Ehrlich's 70 tools as an MCP server was considered and rejected. The tools alone (ChEMBL queries, RDKit computations, etc.) are commodity API wrappers -- the value is the multi-model orchestration (Director-Worker-Summarizer), hypothesis-driven methodology, and parallel experiment execution. An MCP client consuming Ehrlich's tools would lose the scientific rigor that the orchestration guarantees. +Exposing Ehrlich's 73 tools as an MCP server was considered and rejected. The tools alone (ChEMBL queries, RDKit computations, etc.) are commodity API wrappers -- the value is the multi-model orchestration (Director-Worker-Summarizer), hypothesis-driven methodology, and parallel experiment execution. An MCP client consuming Ehrlich's tools would lose the scientific rigor that the orchestration guarantees. Ehrlich remains an MCP **consumer** via `MCPBridge` (connecting to external servers like Excalidraw), which adds value by extending the Researcher's toolkit. diff --git a/docs/us-public-data-apis.md b/docs/us-public-data-apis.md new file mode 100644 index 0000000..e260aa5 --- /dev/null +++ b/docs/us-public-data-apis.md @@ -0,0 +1,794 @@ +# US Public Data APIs for Social Program Evaluation and Impact Assessment + +Comprehensive reference guide for major US federal data APIs useful for evaluating social programs and assessing policy impact. + +**Last Updated:** 2026-02-13 +**Research Date:** February 2026 + +--- + +## 1. Census Bureau API + +**Purpose:** American Community Survey (ACS), Decennial Census, Population Estimates, Economic Indicators, International Trade + +**Base URL:** `https://api.census.gov/data/` + +**Authentication:** +- Optional for basic use (up to 500 queries/day per IP address) +- API key required for >500 queries/day +- Register at: https://www.census.gov/developers/ +- Include key as query parameter: `?key=YOUR_API_KEY` + +**Key Endpoints:** +- ACS 1-Year: `https://api.census.gov/data/{year}/acs/acs1` +- ACS 5-Year: `https://api.census.gov/data/{year}/acs/acs5` +- Decennial Census: `https://api.census.gov/data/{year}/dec/` +- Population Estimates: `https://api.census.gov/data/{year}/pep/` +- Economic Indicators: `https://api.census.gov/data/timeseries/eits/` + +**Data Format:** +- JSON (2D array format) +- First row: column names +- Subsequent rows: data values + +**Example Query:** +``` +https://api.census.gov/data/2019/acs/acs1?get=NAME,B02015_009E,B02015_009M&for=state:*&key=YOUR_KEY +``` + +**Example Response:** +```json +[ + ["STNAME","POP","DATE_","state"], + ["Alabama","4849377","7","01"], + ["Alaska","736732","7","02"], + ["Arizona","6731484","7","04"] +] +``` + +**Rate Limits:** +- Without key: 500 queries/day per IP address +- With key: Higher limits (contact Census for details) +- Max 50 variables per query +- Cell return limits vary by dataset + +**Use Cases:** +- Demographic analysis for program targeting +- Poverty rates, income distribution, educational attainment +- Housing characteristics, employment statistics +- Geographic comparisons (state, county, tract, block group) + +**Documentation:** https://www.census.gov/data/developers/guidance/api-user-guide.html + +--- + +## 2. data.gov CKAN API + +**Purpose:** Metadata search across 300,000+ federal datasets; discovery and access portal + +**Base URL:** `https://api.gsa.gov/technology/datagov/v3/action/` + +**Authentication:** +- API key required (header-based) +- Register at: https://api.data.gov/signup/ +- Include in header: `x-api-key: YOUR_API_KEY` + +**Key Endpoints:** +- Package search: `package_search` +- Package show: `package_show` +- Resource search: `resource_search` +- Organization list: `organization_list` +- Tag list: `tag_list` + +**Data Format:** +- JSON (CKAN standard format) +- Metadata only (not actual dataset contents) + +**Example Query:** +``` +https://api.gsa.gov/technology/datagov/v3/action/package_search?q=education&api_key=DEMO_KEY +``` + +**Example Response:** +```json +{ + "success": true, + "result": { + "count": 1234, + "results": [ + { + "id": "dataset-id", + "name": "dataset-name", + "title": "Dataset Title", + "organization": {...}, + "resources": [...] + } + ] + } +} +``` + +**Rate Limits:** +- Not publicly documented +- CKAN API typically has generous limits for public use + +**Use Cases:** +- Dataset discovery across all federal agencies +- Metadata analysis for data inventory +- Cross-agency data integration planning +- Finding datasets by topic, agency, or keyword + +**Documentation:** https://open.gsa.gov/api/datadotgov/ + +--- + +## 3. Bureau of Labor Statistics (BLS) API + +**Purpose:** Employment data, unemployment rates, wages, inflation (CPI), productivity, workplace injuries + +**Base URL (v2):** `https://api.bls.gov/publicAPI/v2/timeseries/data/` + +**Authentication:** +- v1.0: No key required (limited features) +- v2.0: API key required for advanced features +- Register at: https://www.bls.gov/developers/home.htm +- Include as query parameter: `?registrationkey=YOUR_KEY` + +**Key Endpoints:** +- Timeseries data: `/timeseries/data/` +- Surveys list: `/surveys/` +- Popular series: `/popular/` + +**Data Format:** +- JSON +- Series-based with observations array + +**Example Query (POST):** +```json +POST https://api.bls.gov/publicAPI/v2/timeseries/data/ +{ + "seriesid": ["LNS14000000"], + "startyear": "2020", + "endyear": "2024", + "registrationkey": "YOUR_KEY" +} +``` + +**Example Response:** +```json +{ + "status": "REQUEST_SUCCEEDED", + "responseTime": 123, + "Results": { + "series": [ + { + "seriesID": "LNS14000000", + "data": [ + { + "year": "2024", + "period": "M01", + "value": "3.7" + } + ] + } + ] + } +} +``` + +**Rate Limits:** +- v1.0: 25 queries/day, 10 years/query, 25 series/query +- v2.0: 500 queries/day, 20 years/query, 50 series/query + +**Use Cases:** +- Labor market analysis for workforce programs +- Unemployment trends by demographics +- Wage data for income support evaluation +- CPI for inflation adjustment in longitudinal studies + +**Documentation:** https://www.bls.gov/developers/home.htm + +--- + +## 4. CDC WONDER API + +**Purpose:** Public health data including mortality, natality, cancer incidence, environmental health, vaccine data + +**Base URL:** `https://wonder.cdc.gov/` + +**Authentication:** +- No API key required +- XML-based query system + +**Key Endpoints:** +- Detailed Mortality: Database D76, D77, D149, etc. +- Natality: Database D66, D149 +- Cancer Incidence: SEER database +- Environmental Health: EHDI, Air Quality + +**Data Format:** +- XML request/response +- Automated queries over HTTP + +**Example Query:** +XML-formatted POST request to specific database endpoint (complex, see documentation) + +**Rate Limits:** +- Not publicly documented +- Reasonable use policy applies + +**Use Cases:** +- Mortality analysis for public health programs +- Birth outcome evaluation +- Disease surveillance and trends +- Environmental exposure assessment + +**HealthData.gov Integration:** +CDC datasets also available through HealthData.gov portal for broader discovery + +**Documentation:** https://wonder.cdc.gov/wonder/help/wonder-api.html + +**Additional HHS APIs:** +- CMS Data Repository: Medicare/Medicaid data +- HealthData.gov: Portal for health datasets across HHS agencies + +--- + +## 5. Department of Education - College Scorecard API + +**Purpose:** Higher education data including costs, student outcomes, earnings, demographics, financial aid + +**Base URL:** `https://api.data.gov/ed/collegescorecard/v1/schools` + +**Authentication:** +- API key required +- Register at: https://api.data.gov/signup/ +- Include as query parameter: `?api_key=YOUR_KEY` + +**Key Endpoints:** +- Schools: `/schools` (main endpoint) +- Supports filtering, field selection, pagination + +**Data Format:** +- JSON +- Paginated results (100 records per page max) + +**Example Query:** +``` +https://api.data.gov/ed/collegescorecard/v1/schools?api_key=YOUR_KEY&school.state=CA&fields=id,school.name,latest.student.size,latest.cost.avg_net_price.overall&per_page=100 +``` + +**Example Response:** +```json +{ + "metadata": { + "page": 0, + "total": 453, + "per_page": 100 + }, + "results": [ + { + "id": 110635, + "school.name": "University of California-Berkeley", + "latest.student.size": 45057, + "latest.cost.avg_net_price.overall": 18136 + } + ] +} +``` + +**Rate Limits:** +- Standard api.data.gov limits apply +- Typically 1,000 requests/hour per API key + +**Use Cases:** +- Higher education program evaluation +- College access and completion analysis +- Student debt and earnings outcomes +- Institution comparison for policy research + +**Additional NCES Resources:** +- NCES Data Tools: https://nces.ed.gov/datatools/ +- EdFacts API, IPEDS API (separate systems) + +**Documentation:** https://collegescorecard.ed.gov/data/api-documentation/ + +--- + +## 6. USAspending.gov API + +**Purpose:** Federal spending data including contracts, grants, loans, financial assistance + +**Base URL:** `https://api.usaspending.gov/api/v2/` + +**Authentication:** +- No API key required for public endpoints +- Open access REST JSON API + +**Key Endpoints:** +- Award search: `/search/spending_by_award/` +- Recipient search: `/recipient/` +- Agency spending: `/agency/` +- Federal accounts: `/federal_account/` +- CFDA programs: `/references/cfda/` +- Bulk download: `/bulk_download/` + +**Data Format:** +- JSON +- POST requests for complex queries + +**Example Query (POST):** +```json +POST https://api.usaspending.gov/api/v2/search/spending_by_award/ +{ + "filters": { + "award_type_codes": ["02", "03", "04", "05"], + "agencies": [{"type": "funding", "tier": "toptier", "name": "Department of Education"}] + }, + "fields": ["Award ID", "Recipient Name", "Award Amount"], + "limit": 100 +} +``` + +**Example Response:** +```json +{ + "limit": 100, + "results": [ + { + "Award ID": "EDGR123456", + "Recipient Name": "State Board of Education", + "Award Amount": 5000000 + } + ], + "page_metadata": { + "page": 1, + "total": 12345, + "hasNext": true + } +} +``` + +**Rate Limits:** +- Not publicly documented +- Reasonable use policy applies +- Bulk downloads available for large requests + +**Use Cases:** +- Grant and contract analysis +- Federal program funding flows +- Recipient analysis (geographic, demographic) +- Program evaluation for federally funded initiatives + +**Documentation:** https://api.usaspending.gov/docs/endpoints/ + +--- + +## 7. FRED (Federal Reserve Economic Data) API + +**Purpose:** 800,000+ economic time series including GDP, inflation, employment, interest rates, financial markets + +**Base URL:** `https://api.stlouisfed.org/fred/` + +**Authentication:** +- API key required +- Register at: https://fred.stlouisfed.org/docs/api/api_key.html +- Include as query parameter: `?api_key=YOUR_KEY` + +**Key Endpoints:** +- Series observations: `/series/observations` +- Series search: `/series/search` +- Categories: `/category` +- Releases: `/releases` +- Sources: `/sources` + +**Data Format:** +- JSON or XML +- Specify via `file_type` parameter + +**Example Query:** +``` +https://api.stlouisfed.org/fred/series/observations?series_id=GNPCA&api_key=YOUR_KEY&file_type=json +``` + +**Example Response:** +```json +{ + "realtime_start": "2026-02-13", + "realtime_end": "2026-02-13", + "observation_start": "1776-07-04", + "observation_end": "9999-12-31", + "units": "lin", + "output_type": 1, + "file_type": "json", + "order_by": "observation_date", + "sort_order": "asc", + "count": 95, + "offset": 0, + "limit": 100000, + "observations": [ + { + "realtime_start": "2026-02-13", + "realtime_end": "2026-02-13", + "date": "1929-01-01", + "value": "1065.9" + } + ] +} +``` + +**Rate Limits:** +- 120 requests per minute +- ~172,800 requests per day (theoretical maximum) +- Max 100,000 observations per request (use offset for more) + +**Use Cases:** +- Macroeconomic context for program evaluation +- Inflation adjustment for cost-benefit analysis +- Labor market indicators +- Regional economic data for geographic analysis + +**Documentation:** https://fred.stlouisfed.org/docs/api/fred/ + +--- + +## 8. Performance.gov / Results.gov + +**Purpose:** Federal agency performance data, strategic goals, agency priority goals, program performance + +**Status:** **No public API currently available** + +**Alternative Access:** +- Performance Data Report (downloadable file) +- Agency performance plans and reports (PDF/Excel) +- Previously had API (now archived) + +**Base URL (archived):** `https://trumpadministration.archives.performance.gov/data/` + +**Data Format:** +- CSV/Excel downloads +- Agency-specific reports + +**Use Cases:** +- Government performance management research +- Cross-agency strategic goal tracking +- GPRA Modernization Act compliance analysis +- Outcome-based program evaluation + +**Notes:** +- Performance.gov transitioned from API-based system to report-based system +- Data available through consolidated Performance Data Report +- Individual agency performance plans available on agency websites +- Statutory requirements under GPRA Modernization Act of 2010 + +**Documentation:** https://www.performance.gov/about/ + +--- + +## 9. HUD (Housing and Urban Development) APIs + +**Purpose:** Fair Market Rent, Income Limits, USPS Address Crosswalk, Comprehensive Housing Affordability Strategy (CHAS) + +**Base URL:** `https://www.huduser.gov/hudapi/public/` + +**Authentication:** +- Token-based authentication +- Register at: https://www.huduser.gov/hudapi/public/login +- Create token with dataset access +- Include in header: `Authorization: Bearer YOUR_TOKEN` + +**Key Endpoints:** +- Fair Market Rent (FMR): `/fmr/` +- Income Limits: `/il/` +- USPS Crosswalk: `/usps/` +- CHAS: `/chas/` + +**Data Format:** +- JSON + +**Example Query:** +``` +GET https://www.huduser.gov/hudapi/public/fmr/statedata/CA?year=2024 +Authorization: Bearer YOUR_TOKEN +``` + +**Rate Limits:** +- 1,200 queries per minute + +**Additional HUD Resources:** +- HUD Open Data (ArcGIS): https://hudgis-hud.opendata.arcgis.com/ +- Geospatial datasets, mapping tools +- No API key required for basic access + +**Use Cases:** +- Housing affordability analysis +- Fair market rent for subsidy programs +- Geographic data for housing programs +- Income limit calculations for eligibility + +**Documentation:** https://www.huduser.gov/portal/dataset/fmr-api.html + +--- + +## 10. USDA (United States Department of Agriculture) APIs + +**Purpose:** Agricultural data, food and nutrition, rural development, economic research + +**Multiple API Services Available:** + +### 10.1 FoodData Central API + +**Base URL:** `https://api.nal.usda.gov/fdc/v1/` + +**Authentication:** +- API key required +- Register at: https://fdc.nal.usda.gov/api-key-signup.html +- Include as query parameter: `?api_key=YOUR_KEY` + +**Key Endpoints:** +- Food search: `/foods/search` +- Food details: `/food/{fdcId}` +- Food list: `/foods/list` + +**Example Query:** +``` +https://api.nal.usda.gov/fdc/v1/foods/search?query=apple&api_key=YOUR_KEY +``` + +**Use Cases:** +- Nutrition program evaluation +- Food assistance program analysis +- Dietary assessment research + +### 10.2 Economic Research Service (ERS) APIs + +**Base URL:** `https://api.ers.usda.gov/` + +**Authentication:** +- API key required +- Register through ERS developer portal + +**Available APIs:** +- ARMS Data API (Agricultural Resource Management Survey) +- Quick Stats API (agricultural statistics) +- Food Access Research Atlas +- Food Environment Atlas + +**Example Query:** +``` +https://quickstats.nass.usda.gov/api/api_GET/?key=YOUR_KEY&commodity_desc=CORN&year=2023 +``` + +**Use Cases:** +- Rural development program evaluation +- Agricultural economics research +- Food security and access analysis +- Farm program effectiveness + +### 10.3 Agricultural Marketing Service APIs + +**Base URL:** Various endpoints for different datasets + +**Available APIs:** +- Livestock Mandatory Price Reporting +- Market News (fruits, vegetables, dairy, livestock) +- Farmers Market Directory API + +**Authentication:** +- Varies by API (some require keys, some open) + +**Use Cases:** +- Agricultural market analysis +- Price trend evaluation +- Farmers market program assessment + +**Documentation:** https://www.ers.usda.gov/developer/ + +--- + +## 11. EPA (Environmental Protection Agency) APIs + +**Purpose:** Environmental data including air quality, water quality, toxic releases, facility compliance + +**Multiple API Services Available:** + +### 11.1 Envirofacts Data Service API + +**Base URL:** `https://data.epa.gov/efservice/` + +**Authentication:** +- API key required (via api.data.gov) +- Register at: https://api.data.gov/signup/ +- Include in header: `X-Api-Key: YOUR_KEY` + +**Key Endpoints:** +- Facility search (by location, program) +- Air emissions data +- Water discharge data +- Hazardous waste data +- Toxics Release Inventory (TRI) + +**Data Format:** +- JSON, XML, CSV + +**Example Query:** +``` +https://data.epa.gov/efservice/tri_facility/state_abbr/CA/rows/0:100/JSON +``` + +**Use Cases:** +- Environmental justice analysis +- Facility compliance monitoring +- Pollution exposure assessment +- Environmental program evaluation + +### 11.2 Air Quality System (AQS) API + +**Base URL:** `https://aqs.epa.gov/data/api/` + +**Authentication:** +- Email and API key required +- Register through AQS system +- Include as query parameters: `?email=YOUR_EMAIL&key=YOUR_KEY` + +**Key Endpoints:** +- Daily summary data: `/dailyData/byState` +- Annual summary: `/annualData/byState` +- Monitors: `/monitors/byState` + +**Example Query:** +``` +https://aqs.epa.gov/data/api/dailyData/byState?email=YOUR_EMAIL&key=YOUR_KEY¶m=44201&bdate=20230101&edate=20231231&state=06 +``` + +**Use Cases:** +- Air quality trend analysis +- Health impact assessment +- Environmental program effectiveness +- Pollution reduction program evaluation + +**Rate Limits:** +- Not publicly documented for Envirofacts +- AQS API: reasonable use policy + +**Documentation:** https://www.epa.gov/enviro/envirofacts-data-service-api + +--- + +## Summary Comparison Table + +| API | Auth Required | Rate Limit (per day) | Data Format | Primary Use Cases | +|-----|---------------|---------------------|-------------|-------------------| +| Census Bureau | Optional (key for >500/day) | 500 without key | JSON (2D array) | Demographics, poverty, housing, employment | +| data.gov CKAN | Yes (header) | Not specified | JSON | Dataset discovery across federal agencies | +| BLS | Optional (v2 needs key) | 500 (v2 with key) | JSON | Labor market, unemployment, wages, CPI | +| CDC WONDER | No | Not specified | XML | Mortality, natality, disease surveillance | +| College Scorecard | Yes | ~1,000/hour | JSON | Higher education costs, outcomes, earnings | +| USAspending.gov | No | Not specified | JSON | Federal spending, grants, contracts | +| FRED | Yes | 172,800 (120/min) | JSON/XML | Economic indicators, financial data | +| Performance.gov | N/A - No API | N/A | CSV/Excel | Federal agency performance (no API) | +| HUD | Yes (token) | 1,200/minute | JSON | Housing affordability, income limits, FMR | +| USDA (FoodData) | Yes | Not specified | JSON | Nutrition data, agricultural statistics | +| EPA (Envirofacts) | Yes | Not specified | JSON/XML/CSV | Environmental quality, facility compliance | + +--- + +## Best Practices for API Usage + +1. **API Key Management** + - Store keys in environment variables, never in code + - Use separate keys for development and production + - Rotate keys periodically + - Monitor usage to stay within rate limits + +2. **Rate Limit Handling** + - Implement exponential backoff for retries + - Cache responses when appropriate + - Use bulk download endpoints for large requests + - Monitor rate limit headers in responses + +3. **Data Quality** + - Validate API responses before use + - Check for missing values and data quality flags + - Understand data vintages and revision policies + - Document data sources and retrieval dates + +4. **Error Handling** + - Implement robust error handling for network issues + - Log API errors for debugging + - Handle rate limit errors gracefully + - Validate input parameters before requests + +5. **Documentation** + - Always reference official API documentation + - Check for API versioning and deprecation notices + - Subscribe to API update notifications + - Test queries in sandbox/demo mode first + +--- + +## Additional Resources + +### Cross-Cutting Portals +- **api.data.gov**: Unified API key system for many federal APIs +- **catalog.data.gov**: CKAN-based metadata catalog (300,000+ datasets) +- **HealthData.gov**: Health-focused data portal with API access +- **FederalRegister.gov**: Regulatory data with REST API + +### Developer Resources +- GSA's API Standards: https://github.com/GSA/api-standards +- Federal API Directory: https://api.data.gov/docs/ +- Data.gov Developer Resources: https://data.gov/developers/ + +### Program Evaluation Resources +- OMB Evidence and Evaluation Resources: https://www.whitehouse.gov/omb/ +- Foundations for Evidence-Based Policymaking Act +- Federal Evaluation Clearinghouse + +--- + +## Sources + +**Census Bureau:** +- [Census Bureau Available APIs](https://www.census.gov/data/developers/data-sets.html) +- [Census Data API User Guide](https://www.census.gov/data/developers/guidance/api-user-guide.html) +- [American Community Survey Data via API](https://www.census.gov/programs-surveys/acs/data/data-via-api.html) + +**data.gov:** +- [Data.gov API Documentation](https://open.gsa.gov/api/datadotgov/) +- [Data.gov CKAN API Dataset](https://catalog.data.gov/dataset/data-gov-ckan-api) +- [CKAN API Guide](https://docs.ckan.org/en/latest/api/) + +**Bureau of Labor Statistics:** +- [BLS Data API](https://www.bls.gov/bls/api_features.htm) +- [BLS API Getting Started](https://www.bls.gov/developers/home.htm) +- [BLS API FAQs](https://www.bls.gov/developers/api_faqs.htm) +- [BLS Public Data API Signatures v2](https://www.bls.gov/developers/api_signature_v2.htm) + +**CDC/HHS:** +- [CDC Open Technology APIs](https://open.cdc.gov/apis.html) +- [CDC WONDER API Documentation](https://wonder.cdc.gov/wonder/help/wonder-api.html) +- [CDC WONDER API on HealthData.gov](https://healthdata.gov/dataset/CDC-WONDER-API-for-Data-Query-Web-Service/t3mh-uddy) +- [HHS Developers' Center](https://www.hhs.gov/web/developer/index.html) + +**Department of Education:** +- [College Scorecard API](https://collegescorecard.ed.gov/data/api/) +- [College Scorecard API Documentation](https://collegescorecard.ed.gov/data/api-documentation/) + +**USAspending.gov:** +- [USAspending API](https://api.usaspending.gov/) +- [USAspending API Documentation](https://api.usaspending.gov/docs/endpoints/) +- [USAspending GitHub Repository](https://github.com/fedspendingtransparency/usaspending-api) + +**FRED:** +- [FRED API Documentation](https://fred.stlouisfed.org/docs/api/fred/) +- [FRED API Overview](https://fred.stlouisfed.org/docs/api/fred/overview.html) +- [FRED API Key Registration](https://fred.stlouisfed.org/docs/api/api_key.html) + +**Performance.gov:** +- [Performance.gov](https://www.performance.gov/) +- [Performance.gov About](https://www.performance.gov/about/) +- [Performance.gov Framework](https://www.performance.gov/about/performance-framework/) + +**HUD:** +- [HUD Open Data Site](https://hudgis-hud.opendata.arcgis.com/) +- [HUD FMR/IL Dataset API Documentation](https://www.huduser.gov/portal/dataset/fmr-api.html) +- [HUD Data Catalog](https://data.hud.gov/datasets) + +**USDA:** +- [USDA Developer Resources](https://www.usda.gov/about-usda/policies-and-links/digital/developer-resources) +- [ERS Data APIs](https://www.ers.usda.gov/developer/data-apis) +- [FoodData Central API Guide](https://fdc.nal.usda.gov/api-guide/) +- [ARMS Data API](https://www.ers.usda.gov/developer/data-apis/arms-data-api) + +**EPA:** +- [EPA APIs Overview](https://www.epa.gov/data/application-programming-interface-api) +- [EPA Envirofacts Data Service API](https://www.epa.gov/enviro/envirofacts-data-service-api) +- [EPA AQS API](https://aqs.epa.gov/aqsweb/documents/data_api.html) +- [EPA Envirofacts Web Services](https://www.epa.gov/enviro/web-services) + +--- + +**Document Version:** 1.0 +**Compiled by:** Ehrlich AI Research System +**Date:** February 13, 2026 diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index 13ff035..f52d9e2 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -30,9 +30,15 @@ validate_smiles, ) from ehrlich.config import get_settings +from ehrlich.impact.tools import ( + compare_programs, + fetch_benchmark, + search_economic_indicators, +) from ehrlich.investigation.application.multi_orchestrator import MultiModelOrchestrator from ehrlich.investigation.application.tool_registry import ToolRegistry from ehrlich.investigation.domain.domain_registry import DomainRegistry +from ehrlich.investigation.domain.domains.impact import IMPACT_EVALUATION from ehrlich.investigation.domain.domains.molecular import MOLECULAR_SCIENCE from ehrlich.investigation.domain.domains.nutrition import NUTRITION_SCIENCE from ehrlich.investigation.domain.domains.training import TRAINING_SCIENCE @@ -209,6 +215,7 @@ def _build_registry() -> ToolRegistry: _training_exercise = frozenset({"training", "exercise"}) _nutrition = frozenset({"nutrition"}) _nutrition_safety = frozenset({"nutrition", "safety"}) + _impact = frozenset({"impact"}) _ml = frozenset({"ml"}) _viz = frozenset({"visualization"}) _chem_viz = frozenset({"chemistry", "visualization"}) @@ -274,6 +281,10 @@ def _build_registry() -> ToolRegistry: ("check_interactions", check_interactions, _nutrition_safety), ("analyze_nutrient_ratios", analyze_nutrient_ratios, _nutrition), ("compute_inflammatory_index", compute_inflammatory_index, _nutrition), + # Impact Evaluation (3) + ("search_economic_indicators", search_economic_indicators, _impact), + ("fetch_benchmark", fetch_benchmark, _impact), + ("compare_programs", compare_programs, _impact), # Visualization (12) ("render_binding_scatter", render_binding_scatter, _chem_viz), ("render_admet_radar", render_admet_radar, _sim_viz), @@ -309,6 +320,7 @@ def _build_domain_registry() -> DomainRegistry: domain_registry.register(MOLECULAR_SCIENCE) domain_registry.register(TRAINING_SCIENCE) domain_registry.register(NUTRITION_SCIENCE) + domain_registry.register(IMPACT_EVALUATION) return domain_registry diff --git a/server/src/ehrlich/api/routes/methodology.py b/server/src/ehrlich/api/routes/methodology.py index 308102f..4de5238 100644 --- a/server/src/ehrlich/api/routes/methodology.py +++ b/server/src/ehrlich/api/routes/methodology.py @@ -90,7 +90,7 @@ "role": "Researcher", "model_id": "claude-sonnet-4-5-20250929", "purpose": ( - "Executes experiments with 70 tools in parallel batches. " + "Executes experiments with 73 tools in parallel batches. " "Records findings with evidence provenance and citations." ), }, @@ -217,6 +217,27 @@ "auth": "none", "context": "investigation", }, + { + "name": "World Bank", + "url": "https://api.worldbank.org/v2", + "purpose": "Development indicators (enrollment, poverty, GDP)", + "auth": "none", + "context": "impact", + }, + { + "name": "WHO GHO", + "url": "https://ghoapi.azureedge.net/api", + "purpose": "Health indicators (life expectancy, mortality, coverage)", + "auth": "none", + "context": "impact", + }, + { + "name": "FRED", + "url": "https://api.stlouisfed.org/fred", + "purpose": "Economic time series (GDP, employment, inflation)", + "auth": "api_key", + "context": "impact", + }, ] # ── Tool context mapping (tag -> display name) ───────────────────────── @@ -233,6 +254,7 @@ "nutrition": "Nutrition Science", "safety": "Nutrition Science", "interaction": "Nutrition Science", + "impact": "Impact Evaluation", "ml": "ML", } @@ -323,6 +345,7 @@ async def get_methodology() -> MethodologyResponse: "Simulation", "Training Science", "Nutrition Science", + "Impact Evaluation", "Investigation", ] tool_groups: list[ToolGroup] = [] diff --git a/server/src/ehrlich/api/routes/stats.py b/server/src/ehrlich/api/routes/stats.py index a5a8845..3e72301 100644 --- a/server/src/ehrlich/api/routes/stats.py +++ b/server/src/ehrlich/api/routes/stats.py @@ -9,7 +9,7 @@ router = APIRouter(tags=["stats"]) PHASE_COUNT = 6 -DATA_SOURCE_COUNT = 16 +DATA_SOURCE_COUNT = 19 class StatsResponse(BaseModel): diff --git a/server/src/ehrlich/impact/__init__.py b/server/src/ehrlich/impact/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/ehrlich/impact/application/__init__.py b/server/src/ehrlich/impact/application/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/ehrlich/impact/application/impact_service.py b/server/src/ehrlich/impact/application/impact_service.py new file mode 100644 index 0000000..7a9192d --- /dev/null +++ b/server/src/ehrlich/impact/application/impact_service.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from dataclasses import asdict +from typing import TYPE_CHECKING + +from ehrlich.analysis.application.statistics_service import StatisticsService + +if TYPE_CHECKING: + from ehrlich.impact.domain.entities import Benchmark, EconomicSeries, HealthIndicator + from ehrlich.impact.domain.repository import ( + DevelopmentDataRepository, + EconomicDataRepository, + HealthDataRepository, + ) + + +class ImpactService: + def __init__( + self, + economic: EconomicDataRepository | None = None, + health: HealthDataRepository | None = None, + development: DevelopmentDataRepository | None = None, + ) -> None: + self._economic = economic + self._health = health + self._development = development + self._stats = StatisticsService() + + async def search_economic_data(self, query: str, limit: int = 10) -> list[EconomicSeries]: + if not self._economic: + return [] + return await self._economic.search_series(query, limit) + + async def get_economic_series( + self, series_id: str, start: str | None = None, end: str | None = None + ) -> EconomicSeries | None: + if not self._economic: + return None + return await self._economic.get_series(series_id, start, end) + + async def search_health_data( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[HealthIndicator]: + if not self._health: + return [] + return await self._health.search_indicators(indicator, country, year_start, year_end, limit) + + async def search_development_data( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[Benchmark]: + if not self._development: + return [] + return await self._development.search_indicators( + indicator, country, year_start, year_end, limit + ) + + async def fetch_benchmark( + self, + indicator: str, + source: str, + country: str | None = None, + period: str | None = None, + ) -> list[dict[str, object]]: + """Fetch benchmark from the appropriate source.""" + year_start = int(period.split("-")[0]) if period and "-" in period else None + year_end = int(period.split("-")[1]) if period and "-" in period else None + + if source in ("fred", "economic"): + series = await self.search_economic_data(indicator, limit=5) + return [ + { + "source": "FRED", + "series_id": s.series_id, + "title": s.title, + "unit": s.unit, + "frequency": s.frequency, + } + for s in series + ] + + if source in ("who", "health"): + health_results = await self.search_health_data( + indicator, country, year_start, year_end, limit=10 + ) + return [asdict(r) for r in health_results] + + if source in ("world_bank", "development"): + dev_results = await self.search_development_data( + indicator, country, year_start, year_end, limit=10 + ) + return [asdict(r) for r in dev_results] + + return [] + + def compare_programs( + self, + programs: list[dict[str, object]], + metric: str, + alternative: str = "two-sided", + ) -> dict[str, object]: + """Compare programs on a metric with statistical testing.""" + values_by_group: dict[str, list[float]] = {} + for prog in programs: + name = str(prog.get("name", "unknown")) + raw_values = prog.get("values", []) + if isinstance(raw_values, list): + values_by_group[name] = [float(v) for v in raw_values] + + group_names = list(values_by_group.keys()) + groups = list(values_by_group.values()) + + result: dict[str, object] = { + "metric": metric, + "groups": { + name: { + "n": len(vals), + "mean": round(sum(vals) / len(vals), 4) if vals else 0.0, + "min": round(min(vals), 4) if vals else 0.0, + "max": round(max(vals), 4) if vals else 0.0, + } + for name, vals in values_by_group.items() + }, + } + + if len(groups) == 2 and all(len(g) >= 2 for g in groups): + stat_result = self._stats.run_test(groups[0], groups[1]) + result["statistical_test"] = { + "test_name": stat_result.test_name, + "test_statistic": stat_result.test_statistic, + "p_value": stat_result.p_value, + "effect_size": stat_result.effect_size, + "significant": stat_result.significant, + "interpretation": stat_result.interpretation, + } + result["comparison"] = f"{group_names[0]} vs {group_names[1]}" + + return result diff --git a/server/src/ehrlich/impact/domain/__init__.py b/server/src/ehrlich/impact/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/ehrlich/impact/domain/entities.py b/server/src/ehrlich/impact/domain/entities.py new file mode 100644 index 0000000..1b40aab --- /dev/null +++ b/server/src/ehrlich/impact/domain/entities.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DataPoint: + date: str + value: float + + +@dataclass(frozen=True) +class Program: + name: str + description: str + country: str + region: str + sector: str + start_date: str + end_date: str + budget: float + currency: str + beneficiary_count: int + + +@dataclass(frozen=True) +class Indicator: + name: str + level: str + unit: str + baseline: float + target: float + actual: float + period: str + + +@dataclass(frozen=True) +class ThreatToValidity: + type: str + severity: str + description: str + mitigation: str + + +@dataclass(frozen=True) +class CausalEstimate: + method: str + effect_size: float + standard_error: float + confidence_interval: tuple[float, float] + p_value: float + n_treatment: int + n_control: int + covariates: tuple[str, ...] + assumptions: tuple[str, ...] + threats: tuple[ThreatToValidity, ...] + evidence_tier: str + + +@dataclass(frozen=True) +class Benchmark: + source: str + indicator: str + value: float + unit: str + geography: str + period: str + url: str + + +@dataclass(frozen=True) +class EconomicSeries: + series_id: str + title: str + values: tuple[DataPoint, ...] + source: str + unit: str + frequency: str + + +@dataclass(frozen=True) +class HealthIndicator: + indicator_code: str + indicator_name: str + country: str + year: int + value: float + unit: str diff --git a/server/src/ehrlich/impact/domain/evaluation_standards.py b/server/src/ehrlich/impact/domain/evaluation_standards.py new file mode 100644 index 0000000..feb5cdc --- /dev/null +++ b/server/src/ehrlich/impact/domain/evaluation_standards.py @@ -0,0 +1,86 @@ +"""Evidence grading standards for impact evaluation. + +Based on What Works Clearinghouse (WWC) evidence tiers, +CONEVAL MIR (Matriz de Indicadores para Resultados) levels, +and CREMAA quality criteria. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ehrlich.impact.domain.entities import ThreatToValidity + +WWC_TIERS: dict[str, str] = { + "strong": ( + "Well-designed RCT or quasi-experimental study with no " + "serious threats to validity (low attrition, verified equivalence)" + ), + "moderate": ( + "Quasi-experimental design with some threats to validity " + "that are partially addressed (e.g., PSM with sensitivity analysis)" + ), + "promising": ( + "Correlational study with statistical controls or pre-post " + "comparison with a plausible comparison group" + ), + "rationale": ("Theory-based or descriptive evidence only; no causal identification strategy"), +} + +CONEVAL_MIR_LEVELS: dict[str, str] = { + "fin": "Long-term societal goal the program contributes to", + "proposito": "Direct outcome expected for the target population", + "componente": "Goods or services delivered by the program", + "actividad": "Key activities required to produce components", +} + +CREMAA_CRITERIA: dict[str, str] = { + "claridad": "Indicator is clear and unambiguous", + "relevancia": "Indicator is relevant to the program objective", + "economia": "Data collection is cost-effective", + "monitoreable": "Indicator can be tracked systematically", + "adecuado": "Indicator is appropriate for the level it measures", + "aportacion_marginal": "Indicator adds unique information not captured elsewhere", +} + + +def classify_evidence_tier(method: str, threats: list[ThreatToValidity]) -> str: + """Classify evidence strength based on method and threats. + + Returns one of: strong, moderate, promising, rationale. + """ + strong_methods = {"rct", "randomized_controlled_trial"} + moderate_methods = { + "did", + "difference_in_differences", + "psm", + "propensity_score_matching", + "rdd", + "regression_discontinuity", + "synthetic_control", + } + promising_methods = {"iv", "instrumental_variables", "fixed_effects", "panel_regression"} + + method_lower = method.lower().replace("-", "_").replace(" ", "_") + + severe_threats = [t for t in threats if t.severity == "high"] + + if method_lower in strong_methods: + if len(severe_threats) >= 2: + return "moderate" + if len(severe_threats) == 1: + return "moderate" + return "strong" + + if method_lower in moderate_methods: + if len(severe_threats) >= 2: + return "promising" + return "moderate" + + if method_lower in promising_methods: + if len(severe_threats) >= 1: + return "rationale" + return "promising" + + return "rationale" diff --git a/server/src/ehrlich/impact/domain/repository.py b/server/src/ehrlich/impact/domain/repository.py new file mode 100644 index 0000000..6182f9b --- /dev/null +++ b/server/src/ehrlich/impact/domain/repository.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ehrlich.impact.domain.entities import ( + Benchmark, + EconomicSeries, + HealthIndicator, + ) + + +class EconomicDataRepository(ABC): + @abstractmethod + async def search_series(self, query: str, limit: int = 10) -> list[EconomicSeries]: ... + + @abstractmethod + async def get_series( + self, series_id: str, start: str | None = None, end: str | None = None + ) -> EconomicSeries | None: ... + + +class HealthDataRepository(ABC): + @abstractmethod + async def search_indicators( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[HealthIndicator]: ... + + +class DevelopmentDataRepository(ABC): + @abstractmethod + async def search_indicators( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[Benchmark]: ... + + @abstractmethod + async def get_countries(self) -> list[dict[str, str]]: ... diff --git a/server/src/ehrlich/impact/infrastructure/__init__.py b/server/src/ehrlich/impact/infrastructure/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/ehrlich/impact/infrastructure/fred_client.py b/server/src/ehrlich/impact/infrastructure/fred_client.py new file mode 100644 index 0000000..11dd8d2 --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/fred_client.py @@ -0,0 +1,130 @@ +import asyncio +import logging +import os + +import httpx + +from ehrlich.impact.domain.entities import DataPoint, EconomicSeries +from ehrlich.impact.domain.repository import EconomicDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.stlouisfed.org/fred" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class FREDClient(EconomicDataRepository): + def __init__(self, api_key: str | None = None) -> None: + self._api_key = api_key or os.environ.get("FRED_API_KEY", "") + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_series(self, query: str, limit: int = 10) -> list[EconomicSeries]: + if not self._api_key: + return [] + data = await self._get( + f"{_BASE_URL}/series/search", + params={ + "search_text": query, + "api_key": self._api_key, + "file_type": "json", + "limit": min(limit, 100), + }, + ) + items = data.get("seriess", []) + if not isinstance(items, list): + return [] + return [ + EconomicSeries( + series_id=str(s.get("id", "")), + title=str(s.get("title", "")), + values=(), + source="fred", + unit=str(s.get("units", "")), + frequency=str(s.get("frequency", "")), + ) + for s in items[:limit] + if isinstance(s, dict) + ] + + async def get_series( + self, series_id: str, start: str | None = None, end: str | None = None + ) -> EconomicSeries | None: + if not self._api_key: + return None + params: dict[str, str | int] = { + "series_id": series_id, + "api_key": self._api_key, + "file_type": "json", + } + if start: + params["observation_start"] = start + if end: + params["observation_end"] = end + + data = await self._get(f"{_BASE_URL}/series/observations", params=params) + observations = data.get("observations", []) + if not isinstance(observations, list): + return None + + points: list[DataPoint] = [] + for obs in observations: + if not isinstance(obs, dict): + continue + value_raw = obs.get("value", ".") + if value_raw == ".": + continue + try: + value = float(value_raw) + except (ValueError, TypeError): + continue + points.append(DataPoint(date=str(obs.get("date", "")), value=value)) + + return EconomicSeries( + series_id=series_id, + title=series_id, + values=tuple(points), + source="fred", + unit="", + frequency="", + ) + + async def _get(self, url: str, params: dict[str, str | int]) -> dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(url, params=params) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "FRED rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("FRED", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "FRED timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError("FRED", f"HTTP {e.response.status_code}") from e + raise ExternalServiceError( + "FRED", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/who_client.py b/server/src/ehrlich/impact/infrastructure/who_client.py new file mode 100644 index 0000000..a65bd4f --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/who_client.py @@ -0,0 +1,111 @@ +import asyncio +import logging + +import httpx + +from ehrlich.impact.domain.entities import HealthIndicator +from ehrlich.impact.domain.repository import HealthDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://ghoapi.azureedge.net/api" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class WHOClient(HealthDataRepository): + def __init__(self) -> None: + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_indicators( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[HealthIndicator]: + filters: list[str] = [] + if country: + filters.append(f"SpatialDim eq '{country}'") + if year_start: + filters.append(f"TimeDim ge {year_start}") + if year_end: + filters.append(f"TimeDim le {year_end}") + + params: dict[str, str | int] = {"$top": min(limit, 100)} + if filters: + params["$filter"] = " and ".join(filters) + + data = await self._get(f"{_BASE_URL}/{indicator}", params=params) + items = data.get("value", []) + if not isinstance(items, list): + return [] + + results: list[HealthIndicator] = [] + for item in items[:limit]: + if not isinstance(item, dict): + continue + value_raw = item.get("NumericValue") + if value_raw is None: + continue + try: + value = float(value_raw) + except (ValueError, TypeError): + continue + try: + year = int(item.get("TimeDim", 0)) + except (ValueError, TypeError): + year = 0 + + results.append( + HealthIndicator( + indicator_code=indicator, + indicator_name=str(item.get("IndicatorCode", indicator)), + country=str(item.get("SpatialDim", "")), + year=year, + value=value, + unit=str(item.get("Dim1", "")), + ) + ) + return results + + async def _get(self, url: str, params: dict[str, str | int]) -> dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(url, params=params) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "WHO GHO rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("WHO GHO", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "WHO GHO timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError("WHO GHO", f"HTTP {e.response.status_code}") from e + raise ExternalServiceError( + "WHO GHO", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/worldbank_client.py b/server/src/ehrlich/impact/infrastructure/worldbank_client.py new file mode 100644 index 0000000..de79f29 --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/worldbank_client.py @@ -0,0 +1,142 @@ +import asyncio +import logging + +import httpx + +from ehrlich.impact.domain.entities import Benchmark +from ehrlich.impact.domain.repository import DevelopmentDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.worldbank.org/v2" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class WorldBankClient(DevelopmentDataRepository): + def __init__(self) -> None: + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_indicators( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[Benchmark]: + iso = country or "all" + params: dict[str, str | int] = { + "format": "json", + "per_page": min(limit, 100), + } + if year_start and year_end: + params["date"] = f"{year_start}:{year_end}" + + data = await self._get( + f"{_BASE_URL}/country/{iso}/indicator/{indicator}", + params=params, + ) + + # World Bank returns [metadata, data_array] or [metadata] + if not isinstance(data, list) or len(data) < 2: + return [] + + items = data[1] + if not isinstance(items, list): + return [] + + results: list[Benchmark] = [] + for item in items[:limit]: + if not isinstance(item, dict): + continue + value_raw = item.get("value") + if value_raw is None: + continue + try: + value = float(value_raw) + except (ValueError, TypeError): + continue + + country_info = item.get("country", {}) + country_name = country_info.get("value", "") if isinstance(country_info, dict) else "" + indicator_info = item.get("indicator", {}) + indicator_name = ( + indicator_info.get("value", "") if isinstance(indicator_info, dict) else "" + ) + + results.append( + Benchmark( + source="world_bank", + indicator=indicator_name or indicator, + value=value, + unit=str(item.get("unit", "")), + geography=country_name, + period=str(item.get("date", "")), + url=f"https://data.worldbank.org/indicator/{indicator}", + ) + ) + return results + + async def get_countries(self) -> list[dict[str, str]]: + data = await self._get( + f"{_BASE_URL}/country", + params={"format": "json", "per_page": 300}, + ) + if not isinstance(data, list) or len(data) < 2: + return [] + items = data[1] + if not isinstance(items, list): + return [] + return [ + { + "id": str(c.get("id", "")), + "name": str(c.get("name", "")), + "region": str((c.get("region") or {}).get("value", "")), + "income_level": str((c.get("incomeLevel") or {}).get("value", "")), + } + for c in items + if isinstance(c, dict) + ] + + async def _get( + self, url: str, params: dict[str, str | int] + ) -> list[object] | dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(url, params=params) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "World Bank rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("World Bank", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "World Bank timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError("World Bank", f"HTTP {e.response.status_code}") from e + raise ExternalServiceError( + "World Bank", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/tools.py b/server/src/ehrlich/impact/tools.py new file mode 100644 index 0000000..036912f --- /dev/null +++ b/server/src/ehrlich/impact/tools.py @@ -0,0 +1,149 @@ +"""Impact Evaluation tools for the investigation engine. + +3 tools for causal analysis of social programs: economic indicator +search, benchmark fetching, and cross-program comparison with +statistical testing. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict + +from ehrlich.impact.application.impact_service import ImpactService +from ehrlich.impact.infrastructure.fred_client import FREDClient +from ehrlich.impact.infrastructure.who_client import WHOClient +from ehrlich.impact.infrastructure.worldbank_client import WorldBankClient + +_worldbank = WorldBankClient() +_who = WHOClient() +_fred = FREDClient() +_service = ImpactService( + economic=_fred, + health=_who, + development=_worldbank, +) + + +async def search_economic_indicators( + query: str, + source: str = "fred", + country: str | None = None, + start_year: int | None = None, + end_year: int | None = None, + limit: int = 10, +) -> str: + """Search FRED, World Bank, or WHO for economic/development/health indicators. + + Queries the specified data source for time series data relevant to + impact evaluation. Returns series metadata and values for analysis. + + Args: + query: Search query or indicator code + (e.g. 'GDP per capita', 'SE.PRM.ENRR', 'WHOSIS_000001') + source: Data source ('fred', 'world_bank', 'who') + country: ISO country code for filtering (e.g. 'MX', 'US', 'MEX') + start_year: Start year filter + end_year: End year filter + limit: Maximum results to return (default: 10) + """ + if source in ("fred", "economic"): + series = await _service.search_economic_data(query, limit) + return json.dumps( + { + "source": "FRED", + "query": query, + "count": len(series), + "series": [ + { + "series_id": s.series_id, + "title": s.title, + "unit": s.unit, + "frequency": s.frequency, + "source": s.source, + } + for s in series + ], + } + ) + + if source in ("who", "health"): + indicators = await _service.search_health_data(query, country, start_year, end_year, limit) + return json.dumps( + { + "source": "WHO", + "indicator": query, + "count": len(indicators), + "data": [asdict(i) for i in indicators], + } + ) + + # Default: World Bank + benchmarks = await _service.search_development_data(query, country, start_year, end_year, limit) + return json.dumps( + { + "source": "World Bank", + "indicator": query, + "count": len(benchmarks), + "data": [asdict(b) for b in benchmarks], + } + ) + + +async def fetch_benchmark( + indicator: str, + source: str = "world_bank", + country: str | None = None, + period: str | None = None, +) -> str: + """Fetch a benchmark value from international data sources. + + Retrieves comparison data points from FRED, World Bank, or WHO + for use as baseline or reference in impact evaluations. + + Args: + indicator: Indicator code or name (e.g. 'SE.PRM.ENRR', 'GDP', 'WHOSIS_000001') + source: Data source ('fred', 'world_bank', 'who') + country: ISO country code (e.g. 'MX', 'US') + period: Time period (e.g. '2015-2020') + """ + results = await _service.fetch_benchmark(indicator, source, country, period) + return json.dumps( + { + "indicator": indicator, + "source": source, + "country": country, + "period": period, + "count": len(results), + "benchmarks": results, + } + ) + + +async def compare_programs( + programs: str, + metric: str, + alternative: str = "two-sided", +) -> str: + """Compare social programs on a metric with statistical testing. + + Accepts program data with outcome values and computes descriptive + statistics plus significance testing (auto-selects t-test/Welch/Mann-Whitney). + + Args: + programs: JSON array of program objects with 'name' and 'values' fields. + Example: '[{{"name": "A", "values": [1.2, 1.5]}}, + {{"name": "B", "values": [0.8, 0.9]}}]' + metric: Name of the metric being compared (e.g. 'enrollment_rate', 'test_scores') + alternative: Test alternative ('two-sided', 'less', 'greater') + """ + try: + parsed = json.loads(programs) + except json.JSONDecodeError: + return json.dumps({"error": "Invalid JSON in programs parameter"}) + + if not isinstance(parsed, list): + return json.dumps({"error": "programs must be a JSON array"}) + + result = _service.compare_programs(parsed, metric, alternative) + return json.dumps(result) diff --git a/server/src/ehrlich/investigation/domain/domains/impact.py b/server/src/ehrlich/investigation/domain/domains/impact.py new file mode 100644 index 0000000..a430a98 --- /dev/null +++ b/server/src/ehrlich/investigation/domain/domains/impact.py @@ -0,0 +1,219 @@ +"""Impact Evaluation domain configuration.""" + +from __future__ import annotations + +from ehrlich.investigation.domain.domain_config import DomainConfig, ScoreDefinition + +IMPACT_EVALUATION = DomainConfig( + name="impact_evaluation", + display_name="Impact Evaluation", + identifier_type="program", + identifier_label="Program", + candidate_label="Program Interventions", + tool_tags=frozenset({"impact", "statistics", "literature", "visualization", "ml"}), + score_definitions=( + ScoreDefinition( + key="effect_size", + label="Effect Size", + format_spec=".3f", + higher_is_better=True, + good_threshold=0.5, + ok_threshold=0.2, + ), + ScoreDefinition( + key="statistical_significance", + label="Significance", + format_spec=".4f", + higher_is_better=False, + good_threshold=0.01, + ok_threshold=0.05, + ), + ScoreDefinition( + key="evidence_tier", + label="Evidence Tier", + format_spec=".0f", + higher_is_better=True, + good_threshold=3.0, + ok_threshold=2.0, + ), + ScoreDefinition( + key="cost_effectiveness", + label="Cost-Effectiveness", + format_spec=".2f", + higher_is_better=True, + good_threshold=0.7, + ok_threshold=0.4, + ), + ), + attribute_keys=("sector", "country", "method", "evidence_tier"), + negative_control_threshold=0.5, + hypothesis_types=("causal", "comparative", "cost_effectiveness", "equity"), + valid_domain_categories=( + "impact_evaluation", + "program_evaluation", + "social_program", + "public_policy", + "policy_analysis", + ), + template_prompts=( + { + "title": "Education Program Impact", + "domain": "Impact Evaluation", + "prompt": ( + "What is the causal effect of conditional cash transfer programs " + "on school enrollment and attendance in Latin America? Compare " + "Prospera (Mexico), Bolsa Familia (Brazil), and Familias en Accion " + "(Colombia) using available outcome data." + ), + }, + { + "title": "Health Intervention Effectiveness", + "domain": "Impact Evaluation", + "prompt": ( + "Evaluate the impact of community health worker programs on " + "maternal mortality and childhood vaccination rates in " + "sub-Saharan Africa. Use WHO and World Bank indicators." + ), + }, + { + "title": "Sports Program Social Impact", + "domain": "Impact Evaluation", + "prompt": ( + "Does youth sports program participation reduce juvenile " + "delinquency and improve educational outcomes? Analyze the " + "evidence from after-school sports interventions." + ), + }, + ), + director_examples="""\ + + +What is the causal effect of conditional cash \ +transfers on school enrollment? + +[ + {{ + "statement": "Conditional cash transfer programs increase \ +primary school enrollment by 5-15 percentage points in \ +low-income countries", + "rationale": "Economic theory predicts that reducing the \ +opportunity cost of schooling through cash transfers conditional \ +on enrollment will increase human capital investment", + "prediction": "World Bank and WHO data will show enrollment \ +rate increases of 5-15pp in treatment regions compared to \ +pre-program baselines and non-program regions", + "null_prediction": "No significant difference in enrollment \ +rates between treatment and control regions (effect < 2pp)", + "success_criteria": ">=3 country datasets showing statistically \ +significant enrollment increases with effect sizes > 5pp", + "failure_criteria": "Fewer than 2 datasets showing significant \ +effects, or effects below 2pp", + "scope": "Low and middle-income countries with established CCT \ +programs, 2000-2024", + "hypothesis_type": "causal", + "prior_confidence": 0.75 + }} +], +"negative_controls": [ + {{"identifier": "Non-intervention comparison region", \ +"name": "Control region without CCT", \ +"source": "Matched region without program implementation; \ +expected no enrollment change beyond secular trend"}} +], +"positive_controls": [ + {{"identifier": "Prospera (Mexico) 1997-2010", \ +"name": "Well-documented CCT with RCT evaluation", \ +"known_activity": "Enrollment increase of 3-9pp documented \ +in IFPRI evaluation (Schultz 2004)", \ +"source": "Multiple RCT evaluations, World Bank reports"}} +] + + +""", + experiment_examples="""\ + + +Conditional cash transfers increase school \ +enrollment + +{{ + "description": "Gather enrollment indicators from World Bank \ +and WHO, compare treatment vs control periods/regions, \ +test statistical significance", + "tool_plan": ["search_economic_indicators", \ +"fetch_benchmark", "compare_programs", \ +"run_statistical_test", "search_literature"], + "independent_variable": "CCT program implementation \ +(present vs absent)", + "dependent_variable": "Primary school net enrollment rate (%)"\ +, + "controls": ["positive: Prospera program with known 3-9pp \ +effect", "negative: matched non-program region"], + "confounders": ["GDP growth trends", "Demographic shifts", \ +"Other education policies", "Urbanization"], + "analysis_plan": "Primary: difference-in-differences of \ +enrollment rates; secondary: dose-response by transfer amount; \ +expect N>=5 country-year observations per group", + "success_criteria": "Enrollment increase >5pp with p<0.05 \ +and consistent across >=2 countries", + "failure_criteria": "Effect <2pp or p>0.10 or inconsistent \ +direction across countries" +}} + + + + + +Example: Searching for development indicators +1. search_economic_indicators(query="SE.PRM.ENRR", \ +source="world_bank", country="MX", start_year=2000, end_year=2020) +2. search_economic_indicators(query="SH.STA.MMRT", \ +source="who", country="MEX") +3. record_finding(title="Enrollment trends in Mexico 2000-2020", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Fetching benchmark data for comparison +1. fetch_benchmark(indicator="SE.PRM.ENRR", source="world_bank", \ +country="MX", period="2010-2020") +2. fetch_benchmark(indicator="SE.PRM.ENRR", source="world_bank", \ +country="BR", period="2010-2020") +3. record_finding(title="Cross-country enrollment comparison", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Comparing program outcomes statistically +1. compare_programs(programs='[{{"name": "Treatment", \ +"values": [85.2, 87.1, 89.3]}}, {{"name": "Control", \ +"values": [78.1, 78.5, 79.0]}}]', metric="enrollment_rate") +2. run_statistical_test(group_a="85.2,87.1,89.3,90.1,88.5", \ +group_b="78.1,78.5,79.0,78.8,79.2", test="auto") +3. record_finding(title="Significant enrollment difference", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Training a classifier on program outcome data +1. train_classifier(feature_names=["budget_per_capita", \ +"duration_years", "coverage_pct", "gdp_per_capita"], \ +feature_values=[...], labels=[1.0, 0.0, ...], \ +target_name="program success") +2. predict_scores(feature_names=[...], feature_values=[...], \ +model_id="xgboost_program_abc123") +3. record_finding(title="ML model predicts program success", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") +""", + synthesis_scoring_instructions="""\ +For each program intervention candidate, provide: +- effect_size (Cohen's d or pp difference): magnitude of causal effect +- statistical_significance (p-value): significance level of the estimate +- evidence_tier (1-4): 4=strong (RCT), 3=moderate (quasi-experimental), \ +2=promising (correlational), 1=rationale (descriptive) +- cost_effectiveness (0-1): cost-effectiveness ratio relative to alternatives +- sector: program sector (education, health, employment, housing, sports) +- country: country ISO code +- method: causal identification method used (DiD, PSM, RDD, RCT) +- evidence_tier: WWC tier label (strong/moderate/promising/rationale) + +Priority assignment: +- Priority 1: effect_size > 0.5 AND p < 0.01 AND evidence_tier >= 3 +- Priority 2: effect_size > 0.2 AND p < 0.05 AND evidence_tier >= 2 +- Priority 3: effect_size > 0.1 OR evidence_tier >= 2 +- Priority 4: weak evidence or methodological concerns""", +) diff --git a/server/tests/api/test_methodology.py b/server/tests/api/test_methodology.py index fe464f1..3767c97 100644 --- a/server/tests/api/test_methodology.py +++ b/server/tests/api/test_methodology.py @@ -42,11 +42,12 @@ def test_model_roles(self, client: TestClient) -> None: def test_domains_from_registry(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() - assert len(data["domains"]) == 3 + assert len(data["domains"]) == 4 names = {d["name"] for d in data["domains"]} assert "molecular_science" in names assert "training_science" in names assert "nutrition_science" in names + assert "impact_evaluation" in names def test_domain_has_tool_count(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() @@ -71,7 +72,7 @@ def test_tools_grouped_by_context(self, client: TestClient) -> None: def test_total_tool_count(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() total = sum(len(g["tools"]) for g in data["tools"]) - assert total == 70 + assert total == 73 def test_tool_has_name_and_description(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() @@ -83,7 +84,7 @@ def test_tool_has_name_and_description(self, client: TestClient) -> None: def test_data_sources_count(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() - assert len(data["data_sources"]) == 16 + assert len(data["data_sources"]) == 19 def test_data_source_fields(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() diff --git a/server/tests/api/test_stats.py b/server/tests/api/test_stats.py index 8826f79..2cc400f 100644 --- a/server/tests/api/test_stats.py +++ b/server/tests/api/test_stats.py @@ -24,12 +24,12 @@ def test_returns_all_fields(self, client: TestClient) -> None: def test_tool_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["tool_count"] == 70 + assert data["tool_count"] == 73 def test_domain_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["domain_count"] == 3 + assert data["domain_count"] == 4 def test_phase_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") @@ -39,7 +39,7 @@ def test_phase_count(self, client: TestClient) -> None: def test_data_source_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["data_source_count"] == 16 + assert data["data_source_count"] == 19 def test_event_type_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") diff --git a/server/tests/impact/__init__.py b/server/tests/impact/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/impact/test_evaluation_standards.py b/server/tests/impact/test_evaluation_standards.py new file mode 100644 index 0000000..6121f54 --- /dev/null +++ b/server/tests/impact/test_evaluation_standards.py @@ -0,0 +1,120 @@ +"""Tests for impact evaluation evidence standards.""" + +from __future__ import annotations + +from ehrlich.impact.domain.entities import ThreatToValidity +from ehrlich.impact.domain.evaluation_standards import ( + CONEVAL_MIR_LEVELS, + CREMAA_CRITERIA, + WWC_TIERS, + classify_evidence_tier, +) + + +class TestWWCTiers: + def test_all_tiers_present(self) -> None: + assert set(WWC_TIERS.keys()) == {"strong", "moderate", "promising", "rationale"} + + def test_tiers_have_descriptions(self) -> None: + for tier, desc in WWC_TIERS.items(): + assert isinstance(desc, str) + assert len(desc) > 10, f"Tier {tier} has short description" + + +class TestCONEVALLevels: + def test_all_levels_present(self) -> None: + assert set(CONEVAL_MIR_LEVELS.keys()) == {"fin", "proposito", "componente", "actividad"} + + +class TestCREMAACriteria: + def test_all_criteria_present(self) -> None: + expected = { + "claridad", + "relevancia", + "economia", + "monitoreable", + "adecuado", + "aportacion_marginal", + } + assert set(CREMAA_CRITERIA.keys()) == expected + + +class TestClassifyEvidenceTier: + def test_rct_no_threats(self) -> None: + assert classify_evidence_tier("rct", []) == "strong" + + def test_rct_with_severe_threat(self) -> None: + threats = [ + ThreatToValidity( + type="attrition", + severity="high", + description="30% dropout", + mitigation="ITT analysis", + ) + ] + assert classify_evidence_tier("rct", threats) == "moderate" + + def test_rct_with_two_severe_threats(self) -> None: + threats = [ + ThreatToValidity( + type="attrition", severity="high", description="30% dropout", mitigation="ITT" + ), + ThreatToValidity( + type="contamination", severity="high", description="Cross-group", mitigation="None" + ), + ] + assert classify_evidence_tier("rct", threats) == "moderate" + + def test_did_no_threats(self) -> None: + assert classify_evidence_tier("did", []) == "moderate" + + def test_did_alias(self) -> None: + assert classify_evidence_tier("difference_in_differences", []) == "moderate" + + def test_psm_no_threats(self) -> None: + assert classify_evidence_tier("psm", []) == "moderate" + + def test_psm_with_severe_threats(self) -> None: + threats = [ + ThreatToValidity( + type="selection", severity="high", description="Unobserved", mitigation="None" + ), + ThreatToValidity( + type="specification", severity="high", description="Wrong model", mitigation="None" + ), + ] + assert classify_evidence_tier("psm", threats) == "promising" + + def test_rdd_no_threats(self) -> None: + assert classify_evidence_tier("rdd", []) == "moderate" + + def test_synthetic_control(self) -> None: + assert classify_evidence_tier("synthetic_control", []) == "moderate" + + def test_iv_no_threats(self) -> None: + assert classify_evidence_tier("iv", []) == "promising" + + def test_iv_with_severe_threat(self) -> None: + threats = [ + ThreatToValidity( + type="weak_instrument", severity="high", description="F<10", mitigation="None" + ) + ] + assert classify_evidence_tier("iv", threats) == "rationale" + + def test_fixed_effects(self) -> None: + assert classify_evidence_tier("fixed_effects", []) == "promising" + + def test_descriptive_method(self) -> None: + assert classify_evidence_tier("descriptive", []) == "rationale" + + def test_unknown_method(self) -> None: + assert classify_evidence_tier("survey", []) == "rationale" + + def test_low_severity_threats_not_counted(self) -> None: + threats = [ + ThreatToValidity( + type="attrition", severity="low", description="5% dropout", mitigation="Robust SE" + ) + ] + assert classify_evidence_tier("rct", threats) == "strong" diff --git a/server/tests/impact/test_fred_client.py b/server/tests/impact/test_fred_client.py new file mode 100644 index 0000000..ed2afac --- /dev/null +++ b/server/tests/impact/test_fred_client.py @@ -0,0 +1,125 @@ +"""Tests for FRED API client.""" + +from __future__ import annotations + +import pytest +import respx +from httpx import Response + +from ehrlich.impact.infrastructure.fred_client import FREDClient +from ehrlich.kernel.exceptions import ExternalServiceError + + +@pytest.fixture +def client() -> FREDClient: + return FREDClient(api_key="test_key") + + +_SEARCH_URL = "https://api.stlouisfed.org/fred/series/search" +_OBS_URL = "https://api.stlouisfed.org/fred/series/observations" + +_SEARCH_RESPONSE = { + "seriess": [ + { + "id": "GDP", + "title": "Gross Domestic Product", + "units": "Billions of Dollars", + "frequency": "Quarterly", + }, + { + "id": "GDPC1", + "title": "Real Gross Domestic Product", + "units": "Billions of Chained 2017 Dollars", + "frequency": "Quarterly", + }, + ] +} + +_OBS_RESPONSE = { + "observations": [ + {"date": "2020-01-01", "value": "21481.367"}, + {"date": "2020-04-01", "value": "19520.164"}, + {"date": "2020-07-01", "value": "."}, + {"date": "2020-10-01", "value": "21477.597"}, + ] +} + + +class TestSearchSeries: + @respx.mock + @pytest.mark.asyncio + async def test_returns_series(self, client: FREDClient) -> None: + respx.get(_SEARCH_URL).mock(return_value=Response(200, json=_SEARCH_RESPONSE)) + results = await client.search_series("GDP") + assert len(results) == 2 + assert results[0].series_id == "GDP" + assert results[0].title == "Gross Domestic Product" + assert results[0].unit == "Billions of Dollars" + assert results[0].source == "fred" + + @respx.mock + @pytest.mark.asyncio + async def test_empty_results(self, client: FREDClient) -> None: + respx.get(_SEARCH_URL).mock(return_value=Response(200, json={"seriess": []})) + results = await client.search_series("nonexistent") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: FREDClient) -> None: + respx.get(_SEARCH_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="FRED"): + await client.search_series("GDP") + + @pytest.mark.asyncio + async def test_no_api_key_returns_empty(self) -> None: + client = FREDClient(api_key="") + results = await client.search_series("GDP") + assert results == [] + + +class TestGetSeries: + @respx.mock + @pytest.mark.asyncio + async def test_returns_observations(self, client: FREDClient) -> None: + respx.get(_OBS_URL).mock(return_value=Response(200, json=_OBS_RESPONSE)) + result = await client.get_series("GDP") + assert result is not None + assert result.series_id == "GDP" + assert len(result.values) == 3 # "." values are skipped + assert result.values[0].date == "2020-01-01" + assert result.values[0].value == 21481.367 + + @respx.mock + @pytest.mark.asyncio + async def test_with_date_range(self, client: FREDClient) -> None: + respx.get(_OBS_URL).mock(return_value=Response(200, json=_OBS_RESPONSE)) + result = await client.get_series("GDP", start="2020-01-01", end="2020-12-31") + assert result is not None + + @respx.mock + @pytest.mark.asyncio + async def test_empty_observations(self, client: FREDClient) -> None: + respx.get(_OBS_URL).mock(return_value=Response(200, json={"observations": []})) + result = await client.get_series("EMPTY") + assert result is not None + assert result.values == () + + @pytest.mark.asyncio + async def test_no_api_key_returns_none(self) -> None: + client = FREDClient(api_key="") + result = await client.get_series("GDP") + assert result is None + + +class TestRetry: + @respx.mock + @pytest.mark.asyncio + async def test_rate_limit_retry(self, client: FREDClient) -> None: + route = respx.get(_SEARCH_URL) + route.side_effect = [ + Response(429), + Response(200, json={"seriess": []}), + ] + results = await client.search_series("GDP") + assert results == [] diff --git a/server/tests/impact/test_impact_service.py b/server/tests/impact/test_impact_service.py new file mode 100644 index 0000000..c76ed2b --- /dev/null +++ b/server/tests/impact/test_impact_service.py @@ -0,0 +1,196 @@ +"""Tests for ImpactService.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from ehrlich.impact.application.impact_service import ImpactService +from ehrlich.impact.domain.entities import ( + Benchmark, + DataPoint, + EconomicSeries, + HealthIndicator, +) + + +@pytest.fixture +def mock_economic() -> AsyncMock: + mock = AsyncMock() + mock.search_series = AsyncMock( + return_value=[ + EconomicSeries( + series_id="GDP", + title="GDP", + values=(), + source="fred", + unit="Billions", + frequency="Quarterly", + ) + ] + ) + mock.get_series = AsyncMock( + return_value=EconomicSeries( + series_id="GDP", + title="GDP", + values=(DataPoint(date="2020-01-01", value=21481.0),), + source="fred", + unit="Billions", + frequency="Quarterly", + ) + ) + return mock + + +@pytest.fixture +def mock_health() -> AsyncMock: + mock = AsyncMock() + mock.search_indicators = AsyncMock( + return_value=[ + HealthIndicator( + indicator_code="WHOSIS_000001", + indicator_name="Life expectancy", + country="MEX", + year=2020, + value=75.1, + unit="years", + ) + ] + ) + return mock + + +@pytest.fixture +def mock_development() -> AsyncMock: + mock = AsyncMock() + mock.search_indicators = AsyncMock( + return_value=[ + Benchmark( + source="world_bank", + indicator="SE.PRM.ENRR", + value=98.5, + unit="", + geography="Mexico", + period="2020", + url="https://data.worldbank.org/indicator/SE.PRM.ENRR", + ) + ] + ) + return mock + + +@pytest.fixture +def service( + mock_economic: AsyncMock, mock_health: AsyncMock, mock_development: AsyncMock +) -> ImpactService: + return ImpactService(economic=mock_economic, health=mock_health, development=mock_development) + + +class TestSearchEconomicData: + @pytest.mark.asyncio + async def test_returns_series(self, service: ImpactService) -> None: + results = await service.search_economic_data("GDP") + assert len(results) == 1 + assert results[0].series_id == "GDP" + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_economic_data("GDP") + assert results == [] + + +class TestGetEconomicSeries: + @pytest.mark.asyncio + async def test_returns_series(self, service: ImpactService) -> None: + result = await service.get_economic_series("GDP") + assert result is not None + assert result.series_id == "GDP" + assert len(result.values) == 1 + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + result = await svc.get_economic_series("GDP") + assert result is None + + +class TestSearchHealthData: + @pytest.mark.asyncio + async def test_returns_indicators(self, service: ImpactService) -> None: + results = await service.search_health_data("WHOSIS_000001", country="MEX") + assert len(results) == 1 + assert results[0].value == 75.1 + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_health_data("WHOSIS_000001") + assert results == [] + + +class TestSearchDevelopmentData: + @pytest.mark.asyncio + async def test_returns_benchmarks(self, service: ImpactService) -> None: + results = await service.search_development_data("SE.PRM.ENRR", country="MX") + assert len(results) == 1 + assert results[0].value == 98.5 + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_development_data("SE.PRM.ENRR") + assert results == [] + + +class TestFetchBenchmark: + @pytest.mark.asyncio + async def test_fred_source(self, service: ImpactService) -> None: + results = await service.fetch_benchmark("GDP", "fred") + assert len(results) == 1 + assert results[0]["source"] == "FRED" + + @pytest.mark.asyncio + async def test_who_source(self, service: ImpactService) -> None: + results = await service.fetch_benchmark( + "WHOSIS_000001", "who", country="MEX", period="2019-2020" + ) + assert len(results) == 1 + + @pytest.mark.asyncio + async def test_worldbank_source(self, service: ImpactService) -> None: + results = await service.fetch_benchmark( + "SE.PRM.ENRR", "world_bank", country="MX", period="2019-2020" + ) + assert len(results) == 1 + + @pytest.mark.asyncio + async def test_unknown_source(self, service: ImpactService) -> None: + results = await service.fetch_benchmark("X", "unknown") + assert results == [] + + +class TestComparePrograms: + def test_two_group_comparison(self, service: ImpactService) -> None: + programs = [ + {"name": "Treatment", "values": [85.0, 87.0, 89.0, 90.0, 88.0]}, + {"name": "Control", "values": [78.0, 78.5, 79.0, 78.8, 79.2]}, + ] + result = service.compare_programs(programs, "enrollment_rate") + assert result["metric"] == "enrollment_rate" + assert "Treatment" in result["groups"] + assert "Control" in result["groups"] + assert "statistical_test" in result + assert result["statistical_test"]["significant"] is True + + def test_single_group_no_test(self, service: ImpactService) -> None: + programs = [{"name": "A", "values": [1.0, 2.0, 3.0]}] + result = service.compare_programs(programs, "score") + assert result["metric"] == "score" + assert "statistical_test" not in result + + def test_empty_programs(self, service: ImpactService) -> None: + result = service.compare_programs([], "score") + assert result["metric"] == "score" + assert result["groups"] == {} diff --git a/server/tests/impact/test_tools.py b/server/tests/impact/test_tools.py new file mode 100644 index 0000000..bb21255 --- /dev/null +++ b/server/tests/impact/test_tools.py @@ -0,0 +1,151 @@ +"""Tests for Impact Evaluation tools.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from ehrlich.impact.domain.entities import ( + Benchmark, + EconomicSeries, + HealthIndicator, +) +from ehrlich.impact.tools import ( + compare_programs, + fetch_benchmark, + search_economic_indicators, +) + + +class TestSearchEconomicIndicators: + @pytest.mark.asyncio + async def test_fred_source(self) -> None: + series = [ + EconomicSeries( + series_id="GDP", + title="Gross Domestic Product", + values=(), + source="fred", + unit="Billions", + frequency="Quarterly", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_economic_data", + new_callable=AsyncMock, + return_value=series, + ): + result = json.loads(await search_economic_indicators("GDP", source="fred")) + assert result["source"] == "FRED" + assert result["count"] == 1 + assert result["series"][0]["series_id"] == "GDP" + + @pytest.mark.asyncio + async def test_who_source(self) -> None: + indicators = [ + HealthIndicator( + indicator_code="WHOSIS_000001", + indicator_name="Life expectancy", + country="MEX", + year=2020, + value=75.1, + unit="years", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_health_data", + new_callable=AsyncMock, + return_value=indicators, + ): + result = json.loads( + await search_economic_indicators("WHOSIS_000001", source="who", country="MEX") + ) + assert result["source"] == "WHO" + assert result["count"] == 1 + assert result["data"][0]["value"] == 75.1 + + @pytest.mark.asyncio + async def test_worldbank_source(self) -> None: + benchmarks = [ + Benchmark( + source="world_bank", + indicator="SE.PRM.ENRR", + value=98.5, + unit="", + geography="Mexico", + period="2020", + url="https://data.worldbank.org/indicator/SE.PRM.ENRR", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_development_data", + new_callable=AsyncMock, + return_value=benchmarks, + ): + result = json.loads( + await search_economic_indicators("SE.PRM.ENRR", source="world_bank", country="MX") + ) + assert result["source"] == "World Bank" + assert result["count"] == 1 + + +class TestFetchBenchmark: + @pytest.mark.asyncio + async def test_returns_benchmarks(self) -> None: + mock_result = [ + {"source": "FRED", "series_id": "GDP", "title": "GDP", "unit": "B", "frequency": "Q"} + ] + with patch( + "ehrlich.impact.tools._service.fetch_benchmark", + new_callable=AsyncMock, + return_value=mock_result, + ): + result = json.loads(await fetch_benchmark("GDP", source="fred")) + assert result["indicator"] == "GDP" + assert result["count"] == 1 + assert result["benchmarks"][0]["source"] == "FRED" + + @pytest.mark.asyncio + async def test_empty_result(self) -> None: + with patch( + "ehrlich.impact.tools._service.fetch_benchmark", + new_callable=AsyncMock, + return_value=[], + ): + result = json.loads(await fetch_benchmark("UNKNOWN", source="unknown")) + assert result["count"] == 0 + + +class TestComparePrograms: + @pytest.mark.asyncio + async def test_valid_comparison(self) -> None: + mock_result = { + "metric": "enrollment", + "groups": {"A": {"n": 3, "mean": 2.0}, "B": {"n": 3, "mean": 1.0}}, + "statistical_test": {"test_name": "t_test", "p_value": 0.01, "significant": True}, + } + with patch( + "ehrlich.impact.tools._service.compare_programs", + return_value=mock_result, + ): + programs_json = json.dumps( + [ + {"name": "A", "values": [1.0, 2.0, 3.0]}, + {"name": "B", "values": [0.5, 1.0, 1.5]}, + ] + ) + result = json.loads(await compare_programs(programs_json, "enrollment")) + assert result["metric"] == "enrollment" + assert "statistical_test" in result + + @pytest.mark.asyncio + async def test_invalid_json(self) -> None: + result = json.loads(await compare_programs("not json", "metric")) + assert "error" in result + + @pytest.mark.asyncio + async def test_not_array(self) -> None: + result = json.loads(await compare_programs('{"name": "A"}', "metric")) + assert "error" in result diff --git a/server/tests/impact/test_who_client.py b/server/tests/impact/test_who_client.py new file mode 100644 index 0000000..9c46f36 --- /dev/null +++ b/server/tests/impact/test_who_client.py @@ -0,0 +1,103 @@ +"""Tests for WHO GHO API client.""" + +from __future__ import annotations + +import pytest +import respx +from httpx import Response + +from ehrlich.impact.infrastructure.who_client import WHOClient +from ehrlich.kernel.exceptions import ExternalServiceError + + +@pytest.fixture +def client() -> WHOClient: + return WHOClient() + + +_BASE = "https://ghoapi.azureedge.net/api/WHOSIS_000001" + +_WHO_RESPONSE = { + "value": [ + { + "IndicatorCode": "WHOSIS_000001", + "SpatialDim": "MEX", + "TimeDim": 2020, + "NumericValue": 75.1, + "Dim1": "Both sexes", + }, + { + "IndicatorCode": "WHOSIS_000001", + "SpatialDim": "MEX", + "TimeDim": 2019, + "NumericValue": 74.8, + "Dim1": "Both sexes", + }, + ] +} + + +class TestSearchIndicators: + @respx.mock + @pytest.mark.asyncio + async def test_returns_indicators(self, client: WHOClient) -> None: + respx.get(_BASE).mock(return_value=Response(200, json=_WHO_RESPONSE)) + results = await client.search_indicators("WHOSIS_000001") + assert len(results) == 2 + assert results[0].indicator_code == "WHOSIS_000001" + assert results[0].value == 75.1 + assert results[0].country == "MEX" + assert results[0].year == 2020 + + @respx.mock + @pytest.mark.asyncio + async def test_with_country_filter(self, client: WHOClient) -> None: + respx.get(_BASE).mock(return_value=Response(200, json=_WHO_RESPONSE)) + results = await client.search_indicators("WHOSIS_000001", country="MEX") + assert len(results) == 2 + + @respx.mock + @pytest.mark.asyncio + async def test_with_year_range(self, client: WHOClient) -> None: + respx.get(_BASE).mock(return_value=Response(200, json=_WHO_RESPONSE)) + results = await client.search_indicators("WHOSIS_000001", year_start=2019, year_end=2020) + assert len(results) == 2 + + @respx.mock + @pytest.mark.asyncio + async def test_empty_results(self, client: WHOClient) -> None: + respx.get(_BASE).mock(return_value=Response(200, json={"value": []})) + results = await client.search_indicators("WHOSIS_000001") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_null_numeric_value_skipped(self, client: WHOClient) -> None: + data = { + "value": [ + {"IndicatorCode": "X", "SpatialDim": "MEX", "TimeDim": 2020, "NumericValue": None} + ] + } + respx.get(_BASE).mock(return_value=Response(200, json=data)) + results = await client.search_indicators("WHOSIS_000001") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: WHOClient) -> None: + respx.get(_BASE).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="WHO GHO"): + await client.search_indicators("WHOSIS_000001") + + +class TestRetry: + @respx.mock + @pytest.mark.asyncio + async def test_rate_limit_retry(self, client: WHOClient) -> None: + route = respx.get(_BASE) + route.side_effect = [ + Response(429), + Response(200, json={"value": []}), + ] + results = await client.search_indicators("WHOSIS_000001") + assert results == [] diff --git a/server/tests/impact/test_worldbank_client.py b/server/tests/impact/test_worldbank_client.py new file mode 100644 index 0000000..b217342 --- /dev/null +++ b/server/tests/impact/test_worldbank_client.py @@ -0,0 +1,145 @@ +"""Tests for World Bank API client.""" + +from __future__ import annotations + +import pytest +import respx +from httpx import Response + +from ehrlich.impact.infrastructure.worldbank_client import WorldBankClient +from ehrlich.kernel.exceptions import ExternalServiceError + + +@pytest.fixture +def client() -> WorldBankClient: + return WorldBankClient() + + +_INDICATOR_URL = "https://api.worldbank.org/v2/country/MX/indicator/SE.PRM.ENRR" +_ALL_INDICATOR_URL = "https://api.worldbank.org/v2/country/all/indicator/SE.PRM.ENRR" +_COUNTRIES_URL = "https://api.worldbank.org/v2/country" + +_INDICATOR_RESPONSE = [ + {"page": 1, "pages": 1, "per_page": 10, "total": 1}, + [ + { + "indicator": {"id": "SE.PRM.ENRR", "value": "School enrollment, primary (% net)"}, + "country": {"id": "MX", "value": "Mexico"}, + "value": 98.5, + "unit": "", + "date": "2020", + }, + { + "indicator": {"id": "SE.PRM.ENRR", "value": "School enrollment, primary (% net)"}, + "country": {"id": "MX", "value": "Mexico"}, + "value": 97.2, + "unit": "", + "date": "2019", + }, + ], +] + +_COUNTRIES_RESPONSE = [ + {"page": 1, "pages": 1, "per_page": 300, "total": 2}, + [ + { + "id": "MEX", + "name": "Mexico", + "iso2Code": "MX", + "region": {"value": "Latin America & Caribbean"}, + "incomeLevel": {"value": "Upper middle income"}, + }, + { + "id": "USA", + "name": "United States", + "iso2Code": "US", + "region": {"value": "North America"}, + "incomeLevel": {"value": "High income"}, + }, + ], +] + + +class TestSearchIndicators: + @respx.mock + @pytest.mark.asyncio + async def test_returns_benchmarks(self, client: WorldBankClient) -> None: + respx.get(_INDICATOR_URL).mock(return_value=Response(200, json=_INDICATOR_RESPONSE)) + results = await client.search_indicators("SE.PRM.ENRR", country="MX") + assert len(results) == 2 + assert results[0].source == "world_bank" + assert results[0].value == 98.5 + assert results[0].geography == "Mexico" + assert results[0].period == "2020" + + @respx.mock + @pytest.mark.asyncio + async def test_no_country_uses_all(self, client: WorldBankClient) -> None: + respx.get(_ALL_INDICATOR_URL).mock(return_value=Response(200, json=_INDICATOR_RESPONSE)) + results = await client.search_indicators("SE.PRM.ENRR") + assert len(results) == 2 + + @respx.mock + @pytest.mark.asyncio + async def test_empty_data(self, client: WorldBankClient) -> None: + respx.get(_ALL_INDICATOR_URL).mock(return_value=Response(200, json=[{"page": 1}, None])) + results = await client.search_indicators("SE.PRM.ENRR") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_null_values_skipped(self, client: WorldBankClient) -> None: + data = [ + {"page": 1}, + [ + { + "indicator": {"value": "test"}, + "country": {"value": "Mexico"}, + "value": None, + "date": "2020", + "unit": "", + }, + ], + ] + respx.get(_ALL_INDICATOR_URL).mock(return_value=Response(200, json=data)) + results = await client.search_indicators("SE.PRM.ENRR") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: WorldBankClient) -> None: + respx.get(_ALL_INDICATOR_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="World Bank"): + await client.search_indicators("SE.PRM.ENRR") + + +class TestGetCountries: + @respx.mock + @pytest.mark.asyncio + async def test_returns_countries(self, client: WorldBankClient) -> None: + respx.get(_COUNTRIES_URL).mock(return_value=Response(200, json=_COUNTRIES_RESPONSE)) + countries = await client.get_countries() + assert len(countries) == 2 + assert countries[0]["id"] == "MEX" + assert countries[0]["name"] == "Mexico" + assert countries[1]["id"] == "USA" + + @respx.mock + @pytest.mark.asyncio + async def test_empty_response(self, client: WorldBankClient) -> None: + respx.get(_COUNTRIES_URL).mock(return_value=Response(200, json=[{"page": 1}])) + countries = await client.get_countries() + assert countries == [] + + +class TestRetry: + @respx.mock + @pytest.mark.asyncio + async def test_rate_limit_retry(self, client: WorldBankClient) -> None: + route = respx.get(_ALL_INDICATOR_URL) + route.side_effect = [ + Response(429), + Response(200, json=[{"page": 1}, []]), + ] + results = await client.search_indicators("SE.PRM.ENRR") + assert results == [] diff --git a/server/tests/integration/test_e2e.py b/server/tests/integration/test_e2e.py index 7de4b12..4f959eb 100644 --- a/server/tests/integration/test_e2e.py +++ b/server/tests/integration/test_e2e.py @@ -140,7 +140,7 @@ class TestToolRegistry: def test_build_registry_has_expected_tools(self) -> None: registry = _build_registry() tools = registry.list_tools() - assert len(tools) == 70 + assert len(tools) == 73 assert "validate_smiles" in tools assert "search_literature" in tools assert "search_citations" in tools @@ -177,11 +177,15 @@ def test_build_registry_has_expected_tools(self) -> None: assert "search_supplement_labels" in tools assert "search_nutrient_data" in tools assert "search_supplement_safety" in tools + # Impact evaluation tools + assert "search_economic_indicators" in tools + assert "fetch_benchmark" in tools + assert "compare_programs" in tools def test_all_tools_have_schemas(self) -> None: registry = _build_registry() schemas = registry.list_schemas() - assert len(schemas) == 70 + assert len(schemas) == 73 for schema in schemas: assert "name" in schema assert "description" in schema diff --git a/web/vite.config.ts b/web/vite.config.ts index 725aff1..1aa6779 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -6,7 +6,16 @@ import tailwindcss from "@tailwindcss/vite"; import { nitro } from "nitro/vite"; export default defineConfig({ - server: { port: 3000 }, + server: { + port: 3000, + host: true, + allowedHosts: [ + "gruesomely-euplastic-lashon.ngrok-free.dev", + ], + headers: { + "ngrok-skip-browser-warning": "true", + }, + }, plugins: [ tailwindcss(), tsConfigPaths({ projects: ["./tsconfig.json"] }), From 34e13ee1834997de593a0b5f20a153a8944c0386 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 00:42:46 -0700 Subject: [PATCH 02/35] fix(docs): sync landing page to 78 tools, 19 sources, 4 domains Update all stale numbers in web landing page: - STATS: tools 67->78, dataSources 16->19, domains 3->4, vizTools 12->15, externalAPIs 15->18, boundedContexts 10->11 - Add Impact Evaluation domain card (5 tools, World Bank/WHO/FRED) - Add 3 data sources: World Bank, WHO GHO, FRED - Add 3 impact viz tools to visualization categories - Fix prose: "Three domains" -> "Four domains", API count 15->18 - Update differentiator and pricing tier copy --- web/src/components/DataSources.tsx | 2 +- web/src/components/Domains.tsx | 2 +- web/src/lib/constants.ts | 49 ++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/web/src/components/DataSources.tsx b/web/src/components/DataSources.tsx index e03f731..d48e3c1 100644 --- a/web/src/components/DataSources.tsx +++ b/web/src/components/DataSources.tsx @@ -75,7 +75,7 @@ export function DataSources() {
-
15
+
18
External APIs
diff --git a/web/src/components/Domains.tsx b/web/src/components/Domains.tsx index 3c41988..41aff8a 100644 --- a/web/src/components/Domains.tsx +++ b/web/src/components/Domains.tsx @@ -23,7 +23,7 @@ export function Domains() {

- Three domains. Domain-agnostic engine. + Four domains. Domain-agnostic engine.

Each domain brings its own tools, scoring definitions, and prompt examples. diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 40c7528..0ba654e 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -1,12 +1,12 @@ export const STATS = { - tools: 67, - dataSources: 16, - domains: 3, + tools: 78, + dataSources: 19, + domains: 4, models: 3, phases: 6, - vizTools: 12, - externalAPIs: 15, - boundedContexts: 10, + vizTools: 15, + externalAPIs: 18, + boundedContexts: 11, sseEvents: 20, } as const; @@ -46,7 +46,7 @@ export const EXAMPLE_PROMPTS = [ export const DIFFERENTIATORS = [ { label: "Real Computation", - tagline: "70 tools that compute, not summarize.", + tagline: "78 tools that compute, not summarize.", description: "Ehrlich trains ML models, runs molecular docking, executes statistical tests, and validates with negative controls. Every tool returns structured data from real computation or real APIs.", capabilities: [ @@ -61,9 +61,9 @@ export const DIFFERENTIATORS = [ label: "Accessible Pricing", tagline: "$2-10 per investigation. Free tier included.", description: - "Full 6-phase methodology, all 70 tools, all 16 data sources at every tier. No feature gates. A student in Mexico and a pharma lab in Boston use the same product. The only variable is model quality.", + "Full 6-phase methodology, all 78 tools, all 19 data sources at every tier. No feature gates. A student in Mexico and a pharma lab in Boston use the same product. The only variable is model quality.", capabilities: [ - "3 domains, extensible to any", + "4 domains, extensible to any", "Free: 3 Haiku investigations/month", "Credits: Haiku (1), Sonnet (3), Opus (5)", "Enterprise: bring your own Anthropic key", @@ -137,6 +137,23 @@ export const DOMAINS = [ prompt: "Assess safety and efficacy of vitamin D3 + K2 supplementation at high doses", vizTools: ["Nutrient comparison", "Nutrient adequacy", "Therapeutic window", "Funnel plot"], }, + { + label: "IMPACT EVALUATION", + toolCount: 5, + description: + "Causal analysis of social programs: education, health, employment, housing, sports. Difference-in-differences estimation, automated threat assessment, economic indicators.", + capabilities: [ + "Difference-in-differences causal estimation (DiD)", + "Automated validity threat assessment (WWC/CONEVAL)", + "World Bank development indicators (GDP, poverty, education)", + "WHO Global Health Observatory (mortality, disease burden)", + "FRED economic time series (800K+ US series)", + "Cross-program statistical comparison", + ], + sources: "World Bank, WHO GHO, FRED", + prompt: "What is the causal effect of conditional cash transfers on school enrollment in Latin America?", + vizTools: ["Program dashboard", "Geographic comparison", "Parallel trends"], + }, ] as const; export const DATA_SOURCES = [ @@ -155,7 +172,10 @@ export const DATA_SOURCES = [ { name: "USDA FoodData", domain: "api.nal.usda.gov", access: "API Key" as const, records: "300K+ foods", purpose: "Nutrient profiles (macro + micro)" }, { name: "OpenFDA CAERS", domain: "api.fda.gov", access: "Free" as const, records: "Ongoing", purpose: "Supplement adverse event reports" }, { name: "RxNav", domain: "rxnav.nlm.nih.gov", access: "Free" as const, records: "RxNorm DB", purpose: "Drug-nutrient interaction screening" }, - { name: "Ehrlich FTS5", domain: "internal", access: "Internal" as const, records: "Growing", purpose: "Past findings (institutional memory)" }, + { name: "World Bank", domain: "api.worldbank.org", access: "Free" as const, records: "16K+ indicators", purpose: "Development indicators by country (GDP, poverty, education)" }, + { name: "WHO GHO", domain: "ghoapi.azureedge.net", access: "Free" as const, records: "2K+ indicators", purpose: "Global health statistics (mortality, disease, life expectancy)" }, + { name: "FRED", domain: "api.stlouisfed.org", access: "API Key" as const, records: "800K+ series", purpose: "US economic time series (GDP, unemployment, CPI)" }, + { name: "Ehrlich tsvector", domain: "internal", access: "Internal" as const, records: "Growing", purpose: "Past findings (institutional memory)" }, ] as const; export const METHODOLOGY_PHASES = [ @@ -181,7 +201,7 @@ export const METHODOLOGY_PHASES = [ number: "04", label: "Experiment Execution", foundation: "Fisher (1935)", - description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. 2 experiments run in parallel. 70 tools across all domains.", + description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 78 tools across 4 domains.", }, { number: "05", @@ -229,6 +249,9 @@ export const VISUALIZATION_CATEGORIES = [ "Nutrient Comparison -- Grouped bar chart comparing foods", "Nutrient Adequacy -- Horizontal bars showing % RDA with MAR score", "Therapeutic Window -- EAR/RDA/UL safety zones per nutrient", + "Program Dashboard -- Multi-indicator KPI view with target tracking", + "Geographic Comparison -- Region bar chart with benchmark line", + "Parallel Trends -- DiD treatment vs control over time", ], }, { @@ -288,10 +311,10 @@ export const PRICING_TIERS = [ price: "$0", period: "forever", credits: "3 Haiku investigations/month", - description: "Full methodology. All 70 tools. All 16 data sources. Findings indexed for future research.", + description: "Full methodology. All 78 tools. All 19 data sources. Findings indexed for future research.", features: [ "Full 6-phase scientific methodology", - "All 70 tools, all 16 data sources", + "All 78 tools, all 19 data sources, 4 domains", "Full audit trail and report", "Self-referential search (FTS5)", "No feature gates", From 576a87f962eebdd63f52c16a91a479c435aff9f7 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 00:43:02 -0700 Subject: [PATCH 03/35] feat(impact): add DiD estimator, threat assessment, and console templates Core causal inference feature for Impact Evaluation domain: - CausalEstimator port ABC in domain/ports.py - DiDEstimator infrastructure: effect size, SE, p-value, parallel trends test, automated threat assessment, evidence tier classification - estimate_did tool: difference-in-differences with 4 input groups - assess_threats tool: knowledge-based validity assessment for DiD, PSM, RDD, RCT, IV methods - ImpactService: estimate_did() and assess_threats() methods - Domain config: DiD tool examples in experiment_examples - Console: 2 Impact Evaluation template cards (CCT School Enrollment, Health Worker Programs) - Tests: 35 new tests (DiD estimator + tools + e2e registry) - Tool count: 73 -> 75 --- .../components/TemplateCards.tsx | 18 +- .../src/ehrlich/api/routes/investigation.py | 15 +- .../impact/application/impact_service.py | 93 +++++++- server/src/ehrlich/impact/domain/ports.py | 34 +++ .../impact/infrastructure/did_estimator.py | 210 ++++++++++++++++++ server/src/ehrlich/impact/tools.py | 88 +++++++- .../investigation/domain/domains/impact.py | 11 + server/tests/impact/test_did_estimator.py | 104 +++++++++ server/tests/impact/test_tools_did.py | 124 +++++++++++ server/tests/integration/test_e2e.py | 10 +- 10 files changed, 698 insertions(+), 9 deletions(-) create mode 100644 server/src/ehrlich/impact/domain/ports.py create mode 100644 server/src/ehrlich/impact/infrastructure/did_estimator.py create mode 100644 server/tests/impact/test_did_estimator.py create mode 100644 server/tests/impact/test_tools_did.py diff --git a/console/src/features/investigation/components/TemplateCards.tsx b/console/src/features/investigation/components/TemplateCards.tsx index 3c5e108..a47cc72 100644 --- a/console/src/features/investigation/components/TemplateCards.tsx +++ b/console/src/features/investigation/components/TemplateCards.tsx @@ -1,4 +1,4 @@ -import { FlaskConical, Brain, Leaf, Target, Activity, Shield, Apple } from "lucide-react"; +import { FlaskConical, Brain, Leaf, Target, Activity, Shield, Apple, Scale, HeartPulse } from "lucide-react"; import type { LucideIcon } from "lucide-react"; interface Template { @@ -66,6 +66,22 @@ const TEMPLATES: Template[] = [ "What is the evidence for creatine monohydrate supplementation on strength and power performance? Analyze dosing protocols, loading phases, and safety profile from meta-analyses.", Icon: Apple, }, + { + title: "CCT School Enrollment", + domain: "Education Policy", + domainTag: "Impact Evaluation", + prompt: + "What is the causal effect of conditional cash transfer programs on school enrollment and attendance in Latin America? Compare Prospera (Mexico), Bolsa Familia (Brazil), and Familias en Accion (Colombia) using available outcome data.", + Icon: Scale, + }, + { + title: "Health Worker Programs", + domain: "Health Policy", + domainTag: "Impact Evaluation", + prompt: + "Evaluate the impact of community health worker programs on maternal mortality and childhood vaccination rates in sub-Saharan Africa. Use WHO and World Bank indicators.", + Icon: HeartPulse, + }, ]; interface TemplateCardsProps { diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index f52d9e2..fa551f6 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -31,7 +31,9 @@ ) from ehrlich.config import get_settings from ehrlich.impact.tools import ( + assess_threats, compare_programs, + estimate_did, fetch_benchmark, search_economic_indicators, ) @@ -63,10 +65,13 @@ render_evidence_matrix, render_forest_plot, render_funnel_plot, + render_geographic_comparison, render_muscle_heatmap, render_nutrient_adequacy, render_nutrient_comparison, + render_parallel_trends, render_performance_chart, + render_program_dashboard, render_therapeutic_window, render_training_timeline, ) @@ -222,6 +227,7 @@ def _build_registry() -> ToolRegistry: _sim_viz = frozenset({"simulation", "visualization"}) _training_viz = frozenset({"training", "visualization"}) _nutrition_viz = frozenset({"nutrition", "visualization"}) + _impact_viz = frozenset({"impact", "visualization"}) tagged_tools: list[tuple[str, Any, frozenset[str] | None]] = [ # Chemistry (6) @@ -281,11 +287,13 @@ def _build_registry() -> ToolRegistry: ("check_interactions", check_interactions, _nutrition_safety), ("analyze_nutrient_ratios", analyze_nutrient_ratios, _nutrition), ("compute_inflammatory_index", compute_inflammatory_index, _nutrition), - # Impact Evaluation (3) + # Impact Evaluation (5) ("search_economic_indicators", search_economic_indicators, _impact), ("fetch_benchmark", fetch_benchmark, _impact), ("compare_programs", compare_programs, _impact), - # Visualization (12) + ("estimate_did", estimate_did, _impact), + ("assess_threats", assess_threats, _impact), + # Visualization (15) ("render_binding_scatter", render_binding_scatter, _chem_viz), ("render_admet_radar", render_admet_radar, _sim_viz), ("render_training_timeline", render_training_timeline, _training_viz), @@ -298,6 +306,9 @@ def _build_registry() -> ToolRegistry: ("render_nutrient_comparison", render_nutrient_comparison, _nutrition_viz), ("render_nutrient_adequacy", render_nutrient_adequacy, _nutrition_viz), ("render_therapeutic_window", render_therapeutic_window, _nutrition_viz), + ("render_program_dashboard", render_program_dashboard, _impact_viz), + ("render_geographic_comparison", render_geographic_comparison, _impact_viz), + ("render_parallel_trends", render_parallel_trends, _impact_viz), # Statistics (2) -- universal, no tags ("run_statistical_test", run_statistical_test, None), ("run_categorical_test", run_categorical_test, None), diff --git a/server/src/ehrlich/impact/application/impact_service.py b/server/src/ehrlich/impact/application/impact_service.py index 7a9192d..118b24a 100644 --- a/server/src/ehrlich/impact/application/impact_service.py +++ b/server/src/ehrlich/impact/application/impact_service.py @@ -6,7 +6,14 @@ from ehrlich.analysis.application.statistics_service import StatisticsService if TYPE_CHECKING: - from ehrlich.impact.domain.entities import Benchmark, EconomicSeries, HealthIndicator + from ehrlich.impact.domain.entities import ( + Benchmark, + CausalEstimate, + EconomicSeries, + HealthIndicator, + ThreatToValidity, + ) + from ehrlich.impact.domain.ports import CausalEstimator from ehrlich.impact.domain.repository import ( DevelopmentDataRepository, EconomicDataRepository, @@ -20,10 +27,12 @@ def __init__( economic: EconomicDataRepository | None = None, health: HealthDataRepository | None = None, development: DevelopmentDataRepository | None = None, + causal_estimator: CausalEstimator | None = None, ) -> None: self._economic = economic self._health = health self._development = development + self._causal = causal_estimator self._stats = StatisticsService() async def search_economic_data(self, query: str, limit: int = 10) -> list[EconomicSeries]: @@ -102,6 +111,88 @@ async def fetch_benchmark( return [] + def estimate_did( + self, + treatment_pre: list[float], + treatment_post: list[float], + control_pre: list[float], + control_post: list[float], + ) -> CausalEstimate: + """Estimate causal effect using difference-in-differences.""" + if not self._causal: + msg = "No causal estimator configured" + raise RuntimeError(msg) + return self._causal.estimate_did( + treatment_pre, treatment_post, control_pre, control_post, + ) + + def assess_threats( + self, + method: str, + sample_sizes: dict[str, int], + parallel_trends_p: float | None = None, + effect_size: float | None = None, + ) -> list[ThreatToValidity]: + """Knowledge-based threat assessment for any causal method.""" + from ehrlich.impact.domain.entities import ThreatToValidity + + threats: list[ThreatToValidity] = [] + method_lower = method.lower().replace("-", "_").replace(" ", "_") + + # Small sample threats + min_n = min(sample_sizes.values()) if sample_sizes else 0 + if min_n < 5: + threats.append(ThreatToValidity( + type="small_sample", + severity="high", + description=f"Minimum group size is {min_n}. Insufficient for reliable inference.", + mitigation="Increase sample size or use exact/permutation tests.", + )) + elif min_n < 30: + threats.append(ThreatToValidity( + type="small_sample", + severity="medium", + description=f"Minimum group size is {min_n}. May limit statistical power.", + mitigation="Report power analysis and consider non-parametric tests.", + )) + + # Method-specific threats + is_did = method_lower in ("did", "difference_in_differences") + if is_did and parallel_trends_p is not None and parallel_trends_p < 0.05: + threats.append(ThreatToValidity( + type="parallel_trends_violation", + severity="high", + description=f"Parallel trends assumption violated (p={parallel_trends_p:.4f}).", + mitigation="Use event study or synthetic control method.", + )) + + if method_lower in ("psm", "propensity_score_matching"): + threats.append(ThreatToValidity( + type="unobserved_confounders", + severity="medium", + description="PSM cannot account for unobserved confounders.", + mitigation="Combine with sensitivity analysis (Rosenbaum bounds).", + )) + + if method_lower in ("rdd", "regression_discontinuity"): + threats.append(ThreatToValidity( + type="manipulation", + severity="medium", + description="Running variable may be subject to manipulation near cutoff.", + mitigation="Test for bunching at the cutoff (McCrary density test).", + )) + + # Effect size plausibility + if effect_size is not None and abs(effect_size) > 2.0: + threats.append(ThreatToValidity( + type="implausible_effect", + severity="medium", + description=f"Effect size ({effect_size:.2f}) is unusually large.", + mitigation="Verify data quality and check for measurement errors.", + )) + + return threats + def compare_programs( self, programs: list[dict[str, object]], diff --git a/server/src/ehrlich/impact/domain/ports.py b/server/src/ehrlich/impact/domain/ports.py new file mode 100644 index 0000000..caaad50 --- /dev/null +++ b/server/src/ehrlich/impact/domain/ports.py @@ -0,0 +1,34 @@ +"""Ports (ABCs) for impact evaluation causal inference.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ehrlich.impact.domain.entities import CausalEstimate + + +class CausalEstimator(ABC): + """Abstract base class for causal inference estimators.""" + + @abstractmethod + def estimate_did( + self, + treatment_pre: list[float], + treatment_post: list[float], + control_pre: list[float], + control_post: list[float], + ) -> CausalEstimate: + """Estimate causal effect using difference-in-differences. + + Args: + treatment_pre: Pre-intervention outcomes for treatment group + treatment_post: Post-intervention outcomes for treatment group + control_pre: Pre-intervention outcomes for control group + control_post: Post-intervention outcomes for control group + + Returns: + CausalEstimate with effect size, significance, and threats + """ + ... diff --git a/server/src/ehrlich/impact/infrastructure/did_estimator.py b/server/src/ehrlich/impact/infrastructure/did_estimator.py new file mode 100644 index 0000000..dc194c1 --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/did_estimator.py @@ -0,0 +1,210 @@ +"""Difference-in-Differences causal estimator. + +Pure scipy/numpy implementation -- no new dependencies. +""" + +from __future__ import annotations + +import math + +from scipy import stats + +from ehrlich.impact.domain.entities import CausalEstimate, ThreatToValidity +from ehrlich.impact.domain.evaluation_standards import classify_evidence_tier +from ehrlich.impact.domain.ports import CausalEstimator + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _variance(values: list[float]) -> float: + if len(values) < 2: + return 0.0 + m = _mean(values) + return sum((x - m) ** 2 for x in values) / (len(values) - 1) + + +class DiDEstimator(CausalEstimator): + """Difference-in-differences estimator with threat assessment.""" + + def estimate_did( + self, + treatment_pre: list[float], + treatment_post: list[float], + control_pre: list[float], + control_post: list[float], + ) -> CausalEstimate: + # DiD = (mean_post_T - mean_pre_T) - (mean_post_C - mean_pre_C) + treatment_diff = _mean(treatment_post) - _mean(treatment_pre) + control_diff = _mean(control_post) - _mean(control_pre) + did_estimate = treatment_diff - control_diff + + # Standard error via pooled variance + n_t = len(treatment_pre) + len(treatment_post) + n_c = len(control_pre) + len(control_post) + + var_t_pre = _variance(treatment_pre) + var_t_post = _variance(treatment_post) + var_c_pre = _variance(control_pre) + var_c_post = _variance(control_post) + + se_treatment = math.sqrt( + var_t_pre / max(len(treatment_pre), 1) + + var_t_post / max(len(treatment_post), 1) + ) + se_control = math.sqrt( + var_c_pre / max(len(control_pre), 1) + + var_c_post / max(len(control_post), 1) + ) + se_did = math.sqrt(se_treatment**2 + se_control**2) + + # t-test for significance (treatment post vs control post, adjusted) + if se_did > 0: + t_stat = did_estimate / se_did + df = n_t + n_c - 4 + p_value = float(2 * (1 - stats.t.cdf(abs(t_stat), df=max(df, 1)))) + else: + p_value = 1.0 + + # Cohen's d effect size + pooled_sd = math.sqrt( + (var_t_post * max(len(treatment_post) - 1, 0) + + var_c_post * max(len(control_post) - 1, 0)) + / max(len(treatment_post) + len(control_post) - 2, 1) + ) + cohens_d = did_estimate / pooled_sd if pooled_sd > 0 else 0.0 + + # 95% confidence interval + ci_margin = 1.96 * se_did + ci_lower = did_estimate - ci_margin + ci_upper = did_estimate + ci_margin + + # Automated threat assessment + threats = self._assess_threats( + treatment_pre, treatment_post, + control_pre, control_post, + p_value, cohens_d, + ) + + evidence_tier = classify_evidence_tier("did", list(threats)) + + return CausalEstimate( + method="difference_in_differences", + effect_size=round(did_estimate, 4), + standard_error=round(se_did, 4), + confidence_interval=(round(ci_lower, 4), round(ci_upper, 4)), + p_value=round(p_value, 6), + n_treatment=n_t, + n_control=n_c, + covariates=(), + assumptions=( + "Parallel trends", + "No spillover (SUTVA)", + "Common support", + "No anticipation effects", + ), + threats=threats, + evidence_tier=evidence_tier, + ) + + def _assess_threats( + self, + treatment_pre: list[float], + treatment_post: list[float], + control_pre: list[float], + control_post: list[float], + p_value: float, + effect_size: float, + ) -> tuple[ThreatToValidity, ...]: + threats: list[ThreatToValidity] = [] + + # 1. Parallel trends test: compare pre-treatment slopes + pt_p = self._parallel_trends_p(treatment_pre, control_pre) + if pt_p < 0.05: + threats.append(ThreatToValidity( + type="parallel_trends_violation", + severity="high", + description=( + f"Pre-treatment trends differ significantly (p={pt_p:.4f}). " + "DiD assumption of parallel trends may be violated." + ), + mitigation="Consider event study design or synthetic control method.", + )) + elif pt_p < 0.10: + threats.append(ThreatToValidity( + type="parallel_trends_weak", + severity="medium", + description=( + f"Pre-treatment trends show marginal difference (p={pt_p:.4f}). " + "Parallel trends assumption is weakly supported." + ), + mitigation="Report sensitivity analysis with alternative control groups.", + )) + + # 2. Small sample size + min_n = min( + len(treatment_pre), len(treatment_post), + len(control_pre), len(control_post), + ) + if min_n < 5: + threats.append(ThreatToValidity( + type="small_sample", + severity="high", + description=( + f"Minimum group size is {min_n}. " + "Small samples reduce statistical power." + ), + mitigation="Increase sample size or use bootstrapped CIs.", + )) + elif min_n < 10: + threats.append(ThreatToValidity( + type="small_sample", + severity="medium", + description=( + f"Minimum group size is {min_n}. " + "Moderate sample may limit generalizability." + ), + mitigation="Consider non-parametric alternatives.", + )) + + # 3. Effect size plausibility + if abs(effect_size) > 2.0: + threats.append(ThreatToValidity( + type="implausible_effect", + severity="medium", + description=( + f"Effect size of {effect_size:.2f} is unusually large " + "for social program evaluation." + ), + mitigation="Verify data quality and check for measurement errors.", + )) + + # 4. Borderline significance + if 0.01 < p_value < 0.05: + threats.append(ThreatToValidity( + type="borderline_significance", + severity="low", + description=( + "Result is statistically significant " + f"but borderline (p={p_value:.4f})." + ), + mitigation="Pre-register hypothesis and consider multiple testing correction.", + )) + + return tuple(threats) + + @staticmethod + def _parallel_trends_p( + treatment_pre: list[float], + control_pre: list[float], + ) -> float: + """Test parallel trends by comparing pre-treatment group distributions.""" + if len(treatment_pre) < 2 or len(control_pre) < 2: + return 1.0 + + # Compare trends using Mann-Whitney U test on pre-treatment values + _, p_value = stats.mannwhitneyu( + treatment_pre, control_pre, alternative="two-sided", + ) + return float(p_value) diff --git a/server/src/ehrlich/impact/tools.py b/server/src/ehrlich/impact/tools.py index 036912f..46a48fd 100644 --- a/server/src/ehrlich/impact/tools.py +++ b/server/src/ehrlich/impact/tools.py @@ -1,8 +1,8 @@ """Impact Evaluation tools for the investigation engine. -3 tools for causal analysis of social programs: economic indicator -search, benchmark fetching, and cross-program comparison with -statistical testing. +5 tools for causal analysis of social programs: economic indicator +search, benchmark fetching, cross-program comparison, difference-in- +differences estimation, and validity threat assessment. """ from __future__ import annotations @@ -11,6 +11,7 @@ from dataclasses import asdict from ehrlich.impact.application.impact_service import ImpactService +from ehrlich.impact.infrastructure.did_estimator import DiDEstimator from ehrlich.impact.infrastructure.fred_client import FREDClient from ehrlich.impact.infrastructure.who_client import WHOClient from ehrlich.impact.infrastructure.worldbank_client import WorldBankClient @@ -18,10 +19,12 @@ _worldbank = WorldBankClient() _who = WHOClient() _fred = FREDClient() +_did = DiDEstimator() _service = ImpactService( economic=_fred, health=_who, development=_worldbank, + causal_estimator=_did, ) @@ -147,3 +150,82 @@ async def compare_programs( result = _service.compare_programs(parsed, metric, alternative) return json.dumps(result) + + +async def estimate_did( + treatment_pre: str, + treatment_post: str, + control_pre: str, + control_post: str, +) -> str: + """Estimate causal effect using difference-in-differences (DiD). + + Computes the DiD estimator with standard error, p-value, Cohen's d, + 95% confidence interval, parallel trends test, and automated threat + assessment. Returns evidence tier classification (WWC standards). + + Args: + treatment_pre: JSON array of pre-intervention treatment values + (e.g. '[85.2, 87.1, 86.5]') + treatment_post: JSON array of post-intervention treatment values + (e.g. '[92.3, 94.1, 93.5]') + control_pre: JSON array of pre-intervention control values + (e.g. '[84.0, 85.2, 84.8]') + control_post: JSON array of post-intervention control values + (e.g. '[85.1, 85.8, 85.3]') + """ + try: + t_pre = [float(x) for x in json.loads(treatment_pre)] + t_post = [float(x) for x in json.loads(treatment_post)] + c_pre = [float(x) for x in json.loads(control_pre)] + c_post = [float(x) for x in json.loads(control_post)] + except (json.JSONDecodeError, TypeError, ValueError): + return json.dumps({"error": "Invalid JSON arrays. Provide numeric arrays."}) + + if not all([t_pre, t_post, c_pre, c_post]): + return json.dumps({"error": "All four groups must have at least one value."}) + + estimate = _service.estimate_did(t_pre, t_post, c_pre, c_post) + result = asdict(estimate) + # Convert tuple fields to lists for JSON + result["confidence_interval"] = list(result["confidence_interval"]) + result["covariates"] = list(result["covariates"]) + result["assumptions"] = list(result["assumptions"]) + result["threats"] = [asdict(t) for t in estimate.threats] + return json.dumps(result) + + +async def assess_threats( + method: str, + sample_sizes: str, + parallel_trends_p: float | None = None, + effect_size: float | None = None, +) -> str: + """Assess validity threats for a causal inference method. + + Knowledge-based threat assessment that identifies potential biases + and suggests mitigations for different causal methods (DiD, PSM, + RDD, RCT, IV). + + Args: + method: Causal method name ('did', 'psm', 'rdd', 'rct', 'iv') + sample_sizes: JSON object mapping group names to sizes + (e.g. '{{"treatment": 50, "control": 45}}') + parallel_trends_p: p-value from parallel trends test (DiD only) + effect_size: Cohen's d or standardized effect size + """ + try: + sizes = json.loads(sample_sizes) + except json.JSONDecodeError: + return json.dumps({"error": "Invalid JSON in sample_sizes parameter"}) + + if not isinstance(sizes, dict): + return json.dumps({"error": "sample_sizes must be a JSON object"}) + + parsed_sizes = {str(k): int(v) for k, v in sizes.items()} + threats = _service.assess_threats(method, parsed_sizes, parallel_trends_p, effect_size) + return json.dumps({ + "method": method, + "threat_count": len(threats), + "threats": [asdict(t) for t in threats], + }) diff --git a/server/src/ehrlich/investigation/domain/domains/impact.py b/server/src/ehrlich/investigation/domain/domains/impact.py index a430a98..a8c6286 100644 --- a/server/src/ehrlich/investigation/domain/domains/impact.py +++ b/server/src/ehrlich/investigation/domain/domains/impact.py @@ -189,6 +189,17 @@ 3. record_finding(title="Significant enrollment difference", \ detail="...", hypothesis_id="h1", evidence_type="supporting") +Example: Running difference-in-differences estimation +1. estimate_did(treatment_pre="[78.5, 79.2, 80.1]", \ +treatment_post="[88.3, 89.1, 90.5]", \ +control_pre="[77.8, 78.5, 79.0]", \ +control_post="[79.1, 79.8, 80.2]") +2. assess_threats(method="did", \ +sample_sizes='{{"treatment": 6, "control": 6}}', \ +parallel_trends_p=0.45, effect_size=0.8) +3. record_finding(title="DiD shows significant enrollment effect", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + Example: Training a classifier on program outcome data 1. train_classifier(feature_names=["budget_per_capita", \ "duration_years", "coverage_pct", "gdp_per_capita"], \ diff --git a/server/tests/impact/test_did_estimator.py b/server/tests/impact/test_did_estimator.py new file mode 100644 index 0000000..fe9955b --- /dev/null +++ b/server/tests/impact/test_did_estimator.py @@ -0,0 +1,104 @@ +"""Tests for the DiD estimator infrastructure.""" + +from __future__ import annotations + +import pytest + +from ehrlich.impact.infrastructure.did_estimator import DiDEstimator + + +@pytest.fixture +def estimator() -> DiDEstimator: + return DiDEstimator() + + +class TestDiDEstimator: + def test_known_effect(self, estimator: DiDEstimator) -> None: + """Synthetic data with known true DiD effect of ~5.0.""" + treatment_pre = [80.0, 81.0, 79.5, 80.5, 80.0] + treatment_post = [86.0, 87.0, 85.5, 86.5, 86.0] # +6 + control_pre = [80.0, 81.0, 79.5, 80.5, 80.0] + control_post = [81.0, 82.0, 80.5, 81.5, 81.0] # +1 + + result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + + assert result.method == "difference_in_differences" + assert abs(result.effect_size - 5.0) < 0.01 + assert result.p_value < 0.05 + assert result.confidence_interval[0] < result.effect_size < result.confidence_interval[1] + assert result.n_treatment == 10 + assert result.n_control == 10 + + def test_no_effect(self, estimator: DiDEstimator) -> None: + """Both groups change equally -- no causal effect.""" + treatment_pre = [50.0, 51.0, 49.5, 50.5] + treatment_post = [52.0, 53.0, 51.5, 52.5] # +2 + control_pre = [50.0, 51.0, 49.5, 50.5] + control_post = [52.0, 53.0, 51.5, 52.5] # +2 + + result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + + assert abs(result.effect_size) < 0.01 + assert result.p_value > 0.05 + + def test_large_effect(self, estimator: DiDEstimator) -> None: + """Large treatment effect triggers implausible_effect threat.""" + treatment_pre = [10.0, 11.0, 10.5] + treatment_post = [30.0, 31.0, 30.5] # +20 + control_pre = [10.0, 11.0, 10.5] + control_post = [11.0, 12.0, 11.5] # +1 + + result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + + assert result.effect_size > 15.0 + threat_types = [t.type for t in result.threats] + assert "implausible_effect" in threat_types + + def test_small_sample_threat(self, estimator: DiDEstimator) -> None: + """Very small samples trigger small_sample threat.""" + result = estimator.estimate_did([10.0, 11.0], [15.0, 16.0], [10.0, 11.0], [11.0, 12.0]) + + threat_types = [t.type for t in result.threats] + assert "small_sample" in threat_types + + def test_parallel_trends_violation(self, estimator: DiDEstimator) -> None: + """Clearly different pre-treatment trends trigger parallel_trends threat.""" + treatment_pre = [50.0, 55.0, 60.0, 65.0, 70.0, 75.0, 80.0, 85.0] + control_pre = [50.0, 50.5, 51.0, 51.5, 52.0, 52.5, 53.0, 53.5] + treatment_post = [90.0, 95.0] + control_post = [54.0, 54.5] + + result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + + threat_types = [t.type for t in result.threats] + assert "parallel_trends_violation" in threat_types or "parallel_trends_weak" in threat_types + + def test_evidence_tier_classification(self, estimator: DiDEstimator) -> None: + """DiD without severe threats should be classified as moderate.""" + treatment_pre = [80.0, 81.0, 79.5, 80.5, 80.0, 79.8, 80.2, 81.5] + treatment_post = [86.0, 87.0, 85.5, 86.5, 86.0, 85.8, 86.2, 87.5] + control_pre = [80.0, 81.0, 79.5, 80.5, 80.0, 79.8, 80.2, 81.5] + control_post = [81.0, 82.0, 80.5, 81.5, 81.0, 80.8, 81.2, 82.5] + + result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + + assert result.evidence_tier in ("moderate", "strong") + + def test_assumptions_present(self, estimator: DiDEstimator) -> None: + result = estimator.estimate_did( + [10.0, 11.0, 12.0], [15.0, 16.0, 17.0], + [10.0, 11.0, 12.0], [11.0, 12.0, 13.0], + ) + assert len(result.assumptions) == 4 + assert "Parallel trends" in result.assumptions + + def test_identical_groups(self, estimator: DiDEstimator) -> None: + """All groups identical -- no effect, high p-value.""" + vals = [10.0, 11.0, 10.5, 10.8] + result = estimator.estimate_did(vals, vals, vals, vals) + assert abs(result.effect_size) < 0.01 + + def test_single_values(self, estimator: DiDEstimator) -> None: + """Minimal input: single value per group.""" + result = estimator.estimate_did([10.0], [15.0], [10.0], [11.0]) + assert abs(result.effect_size - 4.0) < 0.01 diff --git a/server/tests/impact/test_tools_did.py b/server/tests/impact/test_tools_did.py new file mode 100644 index 0000000..13f7ea9 --- /dev/null +++ b/server/tests/impact/test_tools_did.py @@ -0,0 +1,124 @@ +"""Tests for DiD and threat assessment tool functions.""" + +from __future__ import annotations + +import json + +import pytest + +from ehrlich.impact.tools import assess_threats, estimate_did + + +class TestEstimateDidTool: + @pytest.mark.anyio + async def test_valid_input(self) -> None: + result = json.loads(await estimate_did( + treatment_pre="[80.0, 81.0, 79.5, 80.5, 80.0]", + treatment_post="[86.0, 87.0, 85.5, 86.5, 86.0]", + control_pre="[80.0, 81.0, 79.5, 80.5, 80.0]", + control_post="[81.0, 82.0, 80.5, 81.5, 81.0]", + )) + + assert result["method"] == "difference_in_differences" + assert "effect_size" in result + assert "p_value" in result + assert "confidence_interval" in result + assert len(result["confidence_interval"]) == 2 + assert "threats" in result + assert "evidence_tier" in result + assert isinstance(result["assumptions"], list) + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await estimate_did( + treatment_pre="not json", + treatment_post="[1, 2]", + control_pre="[1, 2]", + control_post="[1, 2]", + )) + assert "error" in result + + @pytest.mark.anyio + async def test_empty_arrays(self) -> None: + result = json.loads(await estimate_did( + treatment_pre="[]", + treatment_post="[1, 2]", + control_pre="[1, 2]", + control_post="[1, 2]", + )) + assert "error" in result + + @pytest.mark.anyio + async def test_identical_groups(self) -> None: + result = json.loads(await estimate_did( + treatment_pre="[10, 11, 10.5]", + treatment_post="[10, 11, 10.5]", + control_pre="[10, 11, 10.5]", + control_post="[10, 11, 10.5]", + )) + assert abs(result["effect_size"]) < 0.01 + + +class TestAssessThreatsTool: + @pytest.mark.anyio + async def test_did_threats(self) -> None: + result = json.loads(await assess_threats( + method="did", + sample_sizes='{"treatment": 50, "control": 45}', + parallel_trends_p=0.03, + effect_size=0.5, + )) + + assert result["method"] == "did" + assert isinstance(result["threats"], list) + threat_types = [t["type"] for t in result["threats"]] + assert "parallel_trends_violation" in threat_types + + @pytest.mark.anyio + async def test_small_sample(self) -> None: + result = json.loads(await assess_threats( + method="rct", + sample_sizes='{"treatment": 3, "control": 4}', + )) + + threat_types = [t["type"] for t in result["threats"]] + assert "small_sample" in threat_types + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await assess_threats( + method="did", + sample_sizes="not json", + )) + assert "error" in result + + @pytest.mark.anyio + async def test_psm_threats(self) -> None: + result = json.loads(await assess_threats( + method="psm", + sample_sizes='{"treatment": 100, "control": 100}', + )) + + threat_types = [t["type"] for t in result["threats"]] + assert "unobserved_confounders" in threat_types + + @pytest.mark.anyio + async def test_rdd_threats(self) -> None: + result = json.loads(await assess_threats( + method="rdd", + sample_sizes='{"treatment": 100, "control": 100}', + )) + + threat_types = [t["type"] for t in result["threats"]] + assert "manipulation" in threat_types + + @pytest.mark.anyio + async def test_large_effect_threat(self) -> None: + result = json.loads(await assess_threats( + method="rct", + sample_sizes='{"treatment": 100, "control": 100}', + effect_size=3.5, + )) + + threat_types = [t["type"] for t in result["threats"]] + assert "implausible_effect" in threat_types diff --git a/server/tests/integration/test_e2e.py b/server/tests/integration/test_e2e.py index 4f959eb..78789cc 100644 --- a/server/tests/integration/test_e2e.py +++ b/server/tests/integration/test_e2e.py @@ -140,7 +140,7 @@ class TestToolRegistry: def test_build_registry_has_expected_tools(self) -> None: registry = _build_registry() tools = registry.list_tools() - assert len(tools) == 73 + assert len(tools) == 78 assert "validate_smiles" in tools assert "search_literature" in tools assert "search_citations" in tools @@ -181,11 +181,17 @@ def test_build_registry_has_expected_tools(self) -> None: assert "search_economic_indicators" in tools assert "fetch_benchmark" in tools assert "compare_programs" in tools + assert "estimate_did" in tools + assert "assess_threats" in tools + # Impact visualization tools + assert "render_program_dashboard" in tools + assert "render_geographic_comparison" in tools + assert "render_parallel_trends" in tools def test_all_tools_have_schemas(self) -> None: registry = _build_registry() schemas = registry.list_schemas() - assert len(schemas) == 73 + assert len(schemas) == 78 for schema in schemas: assert "name" in schema assert "description" in schema From aaa5ac594bf2241f25e89348af5d1025df024fed Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 00:43:39 -0700 Subject: [PATCH 04/35] feat(impact): add 3 viz tools and sync all docs for Phase 13 Impact visualization tools (server + console): - render_program_dashboard: multi-indicator KPI with target tracking - render_geographic_comparison: region bar chart with benchmark line - render_parallel_trends: DiD treatment vs control over time - 3 React chart components + VizRegistry entries - 8 new viz tool tests Doc sync across all files: - CLAUDE.md: 78 tools, 15 viz, 5 impact tools, file tables updated - README.md: 78 tools, new tool rows added - docs/architecture.md: 78 tools - docs/roadmap.md: Phase 13B partial marked DONE - Tool count: 75 -> 78, viz tools: 12 -> 15 --- CLAUDE.md | 31 ++-- README.md | 9 +- .../features/visualization/VizRegistry.tsx | 6 + .../charts/GeographicComparison.tsx | 98 ++++++++++++ .../visualization/charts/ParallelTrends.tsx | 146 ++++++++++++++++++ .../visualization/charts/ProgramDashboard.tsx | 128 +++++++++++++++ docs/architecture.md | 2 +- docs/roadmap.md | 28 ++-- server/src/ehrlich/investigation/tools_viz.py | 127 ++++++++++++++- .../investigation/test_tools_viz_impact.py | 109 +++++++++++++ 10 files changed, 659 insertions(+), 25 deletions(-) create mode 100644 console/src/features/visualization/charts/GeographicComparison.tsx create mode 100644 console/src/features/visualization/charts/ParallelTrends.tsx create mode 100644 console/src/features/visualization/charts/ProgramDashboard.tsx create mode 100644 server/tests/investigation/test_tools_viz_impact.py diff --git a/CLAUDE.md b/CLAUDE.md index 0d4ad1e..2476ac1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 | simulation | `server/src/ehrlich/simulation/` | Docking, ADMET, resistance, target discovery (RCSB PDB, UniProt, Open Targets), toxicity (EPA CompTox) | | training | `server/src/ehrlich/training/` | Evidence analysis, protocol comparison, injury risk, training metrics, clinical trials (ClinicalTrials.gov), PubMed literature (E-utilities), exercise database (wger) | | nutrition | `server/src/ehrlich/nutrition/` | Supplement evidence, supplement labels (NIH DSLD), nutrient data (USDA FoodData), supplement safety (OpenFDA CAERS), drug-nutrient interactions (RxNav), DRI adequacy, nutrient ratios, inflammatory index | -| impact | `server/src/ehrlich/impact/` | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmark fetching | +| impact | `server/src/ehrlich/impact/` | Social program evaluation: causal inference (DiD), threat assessment, economic indicators (World Bank, WHO GHO, FRED), cross-program comparison | | investigation | `server/src/ehrlich/investigation/` | Multi-model orchestration + PostgreSQL persistence + domain registry + MCP bridge | ### External Data Sources (18 external + 1 internal) @@ -141,7 +141,7 @@ Types: `feat`, `fix`, `refactor`, `chore`, `docs`, `test` Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, training, nutrition, impact, investigation, api, console, mol, data, ci, docs, infra -## Tools (73 Total) +## Tools (78 Total) ### Chemistry (6) -- RDKit cheminformatics, domain-agnostic - `validate_smiles` -- Validate SMILES string @@ -206,12 +206,14 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `analyze_nutrient_ratios` -- Clinically relevant nutrient ratios (omega-6:3, Ca:Mg, Na:K, Zn:Cu, Ca:P, Fe:Cu) - `compute_inflammatory_index` -- Simplified Dietary Inflammatory Index (DII) scoring -### Impact Evaluation (3) -- Social program analysis and economic indicators +### Impact Evaluation (5) -- Social program analysis, causal inference, economic indicators - `search_economic_indicators` -- Query economic time series from FRED, World Bank, or WHO GHO - `fetch_benchmark` -- Get comparison values from international sources (World Bank, WHO GHO) - `compare_programs` -- Cross-program comparison using existing statistical tests +- `estimate_did` -- Difference-in-differences causal estimation with parallel trends test and threat assessment +- `assess_threats` -- Knowledge-based validity threat assessment for causal methods (DiD, PSM, RDD, RCT, IV) -### Visualization (12) -- Domain-specific interactive charts +### Visualization (15) -- Domain-specific interactive charts - `render_binding_scatter` -- Scatter plot of compound binding affinities (Recharts) - `render_admet_radar` -- Radar chart of ADMET/drug-likeness properties (Recharts) - `render_training_timeline` -- Training load timeline with ACWR danger zones (Recharts) @@ -224,6 +226,9 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `render_nutrient_comparison` -- Grouped bar chart comparing nutrient profiles across foods (Recharts) - `render_nutrient_adequacy` -- Horizontal bar chart showing % RDA with MAR score (Recharts) - `render_therapeutic_window` -- Range chart showing EAR/RDA/UL safety zones per nutrient (Visx) +- `render_program_dashboard` -- Multi-indicator KPI dashboard with target tracking (Recharts) +- `render_geographic_comparison` -- Region bar chart with benchmark reference line (Recharts) +- `render_parallel_trends` -- DiD parallel trends chart, treatment vs control over time (Recharts) ### Statistics (2) -- Domain-agnostic hypothesis testing (scipy.stats) - `run_statistical_test` -- Compare two numeric groups (auto-selects t-test/Welch/Mann-Whitney, returns p-value, Cohen's d, 95% CI) @@ -392,13 +397,16 @@ All paths relative to `server/src/ehrlich/`. | File | Purpose | |------|---------| -| `tools.py` | 3 impact evaluation tools | -| `domain/entities.py` | EconomicIndicator, BenchmarkValue, ProgramComparison, ProgramMetric | -| `domain/repository.py` | EconomicDataRepository, BenchmarkRepository ABCs | -| `application/impact_service.py` | ImpactService (economic indicators, benchmarks, program comparison) | +| `tools.py` | 5 impact evaluation tools | +| `domain/entities.py` | DataPoint, Program, Indicator, ThreatToValidity, CausalEstimate, Benchmark, EconomicSeries, HealthIndicator | +| `domain/repository.py` | EconomicDataRepository, HealthDataRepository, DevelopmentDataRepository ABCs | +| `domain/ports.py` | `CausalEstimator` ABC for causal inference methods | +| `domain/evaluation_standards.py` | WWC evidence tiers, CONEVAL MIR levels, CREMAA criteria, `classify_evidence_tier()` | +| `application/impact_service.py` | ImpactService (economic indicators, benchmarks, program comparison, DiD estimation, threat assessment) | | `infrastructure/worldbank_client.py` | World Bank API v2 client for development indicators | | `infrastructure/who_client.py` | WHO GHO API client for health statistics | | `infrastructure/fred_client.py` | FRED API client for US economic time series | +| `infrastructure/did_estimator.py` | DiD estimator: effect size, SE, p-value, parallel trends, automated threat assessment | ### investigation/ @@ -426,7 +434,7 @@ All paths relative to `server/src/ehrlich/`. | `domain/domains/nutrition.py` | `NUTRITION_SCIENCE` config | | `domain/domains/impact.py` | `IMPACT_EVALUATION` config | | `domain/mcp_config.py` | `MCPServerConfig` frozen dataclass | -| `tools_viz.py` | 12 visualization tools | +| `tools_viz.py` | 15 visualization tools | | `infrastructure/repository.py` | PostgreSQL persistence (asyncpg) + tsvector findings search + user/credit management | | `infrastructure/anthropic_client.py` | Anthropic API adapter with retry, streaming, and structured outputs | | `infrastructure/mcp_bridge.py` | MCP client bridge (stdio/SSE/streamable_http) | @@ -438,7 +446,7 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| | `auth.py` | WorkOS JWT middleware (JWKS verification, header + query param auth for SSE) | -| `routes/investigation.py` | REST + SSE endpoints, 73-tool registry, domain registry, MCP bridge, credit system, BYOK | +| `routes/investigation.py` | REST + SSE endpoints, 78-tool registry, domain registry, MCP bridge, credit system, BYOK | | `routes/methodology.py` | GET /methodology: phases, domains, tools, data sources, models | | `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets endpoints | | `routes/stats.py` | GET /stats: aggregate counts | @@ -505,6 +513,9 @@ All paths relative to `console/src/`. | `features/visualization/charts/NutrientComparison.tsx` | Recharts grouped BarChart for food nutrient comparison | | `features/visualization/charts/NutrientAdequacy.tsx` | Recharts horizontal BarChart showing % RDA with MAR score | | `features/visualization/charts/TherapeuticWindow.tsx` | Visx custom range chart with EAR/RDA/UL safety zones | +| `features/visualization/charts/ProgramDashboard.tsx` | Recharts horizontal BarChart with target tracking | +| `features/visualization/charts/GeographicComparison.tsx` | Recharts BarChart with benchmark ReferenceLine | +| `features/visualization/charts/ParallelTrends.tsx` | Recharts ComposedChart for DiD treatment vs control | | `features/visualization/anatomy/BodyDiagram.tsx` | Custom SVG anatomical body diagram | | `features/visualization/anatomy/body-paths.ts` | SVG path data for front/back views | | `features/visualization/anatomy/color-scale.ts` | OKLCH intensity color interpolation | diff --git a/README.md b/README.md index 4cd6780..dd23088 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga ``` Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) -Sonnet 4.5 (Researcher) -- Executes experiments with 73 tools (10-20 calls) +Sonnet 4.5 (Researcher) -- Executes experiments with 78 tools (10-20 calls) Haiku 4.5 (Summarizer) -- Compresses large outputs, classifies domains (5-10 calls) ``` @@ -87,7 +87,7 @@ Cost: ~$3-4 per investigation (vs ~$11 with all-Opus). All data sources are free and open-access. -## 73 Tools +## 78 Tools | Context | Tool | Description | |---------|------|-------------| @@ -143,6 +143,8 @@ All data sources are free and open-access. | Impact | `search_economic_indicators` | Query economic time series from FRED, World Bank, or WHO GHO | | Impact | `fetch_benchmark` | Get comparison values from international sources | | Impact | `compare_programs` | Cross-program comparison using statistical tests | +| Impact | `estimate_did` | Difference-in-differences causal estimation | +| Impact | `assess_threats` | Validity threat assessment for causal methods | | Visualization | `render_binding_scatter` | Scatter plot of compound binding affinities | | Visualization | `render_admet_radar` | Radar chart of ADMET/drug-likeness properties | | Visualization | `render_training_timeline` | Training load timeline with ACWR danger zones | @@ -155,6 +157,9 @@ All data sources are free and open-access. | Visualization | `render_nutrient_comparison` | Grouped bar chart comparing nutrient profiles | | Visualization | `render_nutrient_adequacy` | Horizontal bar chart with DRI adequacy + MAR score | | Visualization | `render_therapeutic_window` | Therapeutic window chart (EAR/RDA/AI/UL zones) | +| Visualization | `render_program_dashboard` | Multi-indicator KPI dashboard with target tracking | +| Visualization | `render_geographic_comparison` | Region bar chart with benchmark reference line | +| Visualization | `render_parallel_trends` | DiD parallel trends chart (treatment vs control) | | Statistics | `run_statistical_test` | Compare two numeric groups (auto-selects t-test/Welch/Mann-Whitney) | | Statistics | `run_categorical_test` | Test contingency tables (auto-selects Fisher's exact/chi-squared) | | Investigation | `propose_hypothesis` | Register testable hypothesis | diff --git a/console/src/features/visualization/VizRegistry.tsx b/console/src/features/visualization/VizRegistry.tsx index 97bdd7e..aa9d07b 100644 --- a/console/src/features/visualization/VizRegistry.tsx +++ b/console/src/features/visualization/VizRegistry.tsx @@ -40,6 +40,9 @@ const LazyDoseResponseCurve = lazy(() => import('./charts/DoseResponseCurve')); const LazyNutrientComparison = lazy(() => import('./charts/NutrientComparison')); const LazyNutrientAdequacy = lazy(() => import('./charts/NutrientAdequacy')); const LazyTherapeuticWindow = lazy(() => import('./charts/TherapeuticWindow')); +const LazyProgramDashboard = lazy(() => import('./charts/ProgramDashboard')); +const LazyGeographicComparison = lazy(() => import('./charts/GeographicComparison')); +const LazyParallelTrends = lazy(() => import('./charts/ParallelTrends')); registry.set('binding_scatter', LazyBindingScatter as unknown as ChartComponent); registry.set('admet_radar', LazyADMETRadar as unknown as ChartComponent); @@ -53,6 +56,9 @@ registry.set('dose_response', LazyDoseResponseCurve as unknown as ChartComponent registry.set('nutrient_comparison', LazyNutrientComparison as unknown as ChartComponent); registry.set('nutrient_adequacy', LazyNutrientAdequacy as unknown as ChartComponent); registry.set('therapeutic_window', LazyTherapeuticWindow as unknown as ChartComponent); +registry.set('program_dashboard', LazyProgramDashboard as unknown as ChartComponent); +registry.set('geographic_comparison', LazyGeographicComparison as unknown as ChartComponent); +registry.set('parallel_trends', LazyParallelTrends as unknown as ChartComponent); /** * Look up a chart component by viz_type string. diff --git a/console/src/features/visualization/charts/GeographicComparison.tsx b/console/src/features/visualization/charts/GeographicComparison.tsx new file mode 100644 index 0000000..df936ec --- /dev/null +++ b/console/src/features/visualization/charts/GeographicComparison.tsx @@ -0,0 +1,98 @@ +import { + BarChart, + Bar, + XAxis, + YAxis, + Tooltip, + ReferenceLine, + ResponsiveContainer, +} from 'recharts'; +import { VIZ_COLORS } from '../theme'; + +interface Region { + name: string; + value: number; +} + +interface GeographicComparisonProps { + data: { + regions: Region[]; + metric_name: string; + benchmark: number | null; + }; + title: string; +} + +function CustomTooltip({ + active, + payload, + label, +}: { + active?: boolean; + payload?: Array<{ value: number }>; + label?: string; +}) { + if (!active || !payload?.[0]) return null; + return ( +

+

{label}

+

{payload[0].value.toFixed(2)}

+
+ ); +} + +export default function GeographicComparison({ data, title }: GeographicComparisonProps) { + return ( +
+

+ {title} +

+ + + + + } /> + {data.benchmark != null && ( + + )} + + + +
+ ); +} diff --git a/console/src/features/visualization/charts/ParallelTrends.tsx b/console/src/features/visualization/charts/ParallelTrends.tsx new file mode 100644 index 0000000..d34b9ec --- /dev/null +++ b/console/src/features/visualization/charts/ParallelTrends.tsx @@ -0,0 +1,146 @@ +import { + ComposedChart, + Line, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ReferenceLine, + ReferenceArea, + Legend, + ResponsiveContainer, +} from 'recharts'; +import { VIZ_COLORS } from '../theme'; + +interface SeriesPoint { + period: string; + value: number; +} + +interface ParallelTrendsProps { + data: { + treatment: SeriesPoint[]; + control: SeriesPoint[]; + treatment_start: string; + }; + title: string; +} + +function CustomTooltip({ + active, + payload, + label, +}: { + active?: boolean; + payload?: Array<{ name: string; value: number; color: string }>; + label?: string; +}) { + if (!active || !payload?.length) return null; + return ( +
+

{label}

+ {payload.map((p) => ( +

+ {p.name}: {p.value.toFixed(2)} +

+ ))} +
+ ); +} + +export default function ParallelTrends({ data, title }: ParallelTrendsProps) { + // Merge treatment and control into a single dataset keyed by period + const allPeriods = new Set([ + ...data.treatment.map((t) => t.period), + ...data.control.map((c) => c.period), + ]); + const treatmentMap = new Map(data.treatment.map((t) => [t.period, t.value])); + const controlMap = new Map(data.control.map((c) => [c.period, c.value])); + + const periods = Array.from(allPeriods).sort(); + const merged = periods.map((period) => ({ + period, + treatment: treatmentMap.get(period), + control: controlMap.get(period), + })); + + // Find the index of treatment_start for shading + const startIdx = periods.indexOf(data.treatment_start); + const lastPeriod = periods[periods.length - 1]; + + return ( +
+

+ {title} +

+ + + + + + {data.treatment_start && startIdx >= 0 && lastPeriod && ( + + )} + {data.treatment_start && ( + + )} + } /> + + + + + +
+ ); +} diff --git a/console/src/features/visualization/charts/ProgramDashboard.tsx b/console/src/features/visualization/charts/ProgramDashboard.tsx new file mode 100644 index 0000000..cae508f --- /dev/null +++ b/console/src/features/visualization/charts/ProgramDashboard.tsx @@ -0,0 +1,128 @@ +import { + BarChart, + Bar, + XAxis, + YAxis, + Tooltip, + ReferenceLine, + ResponsiveContainer, + Cell, +} from 'recharts'; +import { VIZ_COLORS } from '../theme'; + +interface IndicatorEntry { + name: string; + baseline: number; + target: number; + actual: number; + unit: string; + pct_target: number; + status: 'on_track' | 'at_risk' | 'off_track'; +} + +interface ProgramDashboardProps { + data: { + program_name: string; + indicators: IndicatorEntry[]; + }; + title: string; +} + +const STATUS_COLORS: Record = { + on_track: VIZ_COLORS.primary, + at_risk: VIZ_COLORS.warning, + off_track: VIZ_COLORS.danger, +}; + +function CustomTooltip({ + active, + payload, +}: { + active?: boolean; + payload?: Array<{ payload: IndicatorEntry & { pct_display: number } }>; +}) { + if (!active || !payload?.[0]) return null; + const d = payload[0].payload; + return ( +
+

{d.name}

+

+ Baseline: {d.baseline} {d.unit} +

+

+ Target: {d.target} {d.unit} +

+

+ Actual: {d.actual} {d.unit} +

+

+ {Math.round(d.pct_target * 100)}% of target +

+
+ ); +} + +export default function ProgramDashboard({ data, title }: ProgramDashboardProps) { + const chartData = data.indicators.map((ind) => ({ + ...ind, + pct_display: Math.round(ind.pct_target * 100), + })); + + return ( +
+
+

+ {title} +

+ + {data.program_name} + +
+ + + + + } /> + + + + {chartData.map((entry) => ( + + ))} + + + +
+ ); +} diff --git a/docs/architecture.md b/docs/architecture.md index 0cbd9fe..ef60017 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,7 +47,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) │ NO tool access, structured JSON responses only │ - ├── Sonnet 4.5 (Researcher) -- Executes experiments with 73 domain-filtered tools (10-20 calls, parallel x2) + ├── Sonnet 4.5 (Researcher) -- Executes experiments with 78 domain-filtered tools (10-20 calls, parallel x2) │ Tool-calling loop with max_iterations_per_experiment guard │ └── Haiku 4.5 (Summarizer) -- Compresses large tool outputs >2000 chars, PICO+classification, evidence grading diff --git a/docs/roadmap.md b/docs/roadmap.md index 017e3fb..cd97ad4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -471,7 +471,7 @@ Enhancement Enhancement Sports | Phase 13: Impact Evaluation Domain -- IN PROGRESS (Causal inference + Document upload + MX/US APIs) - 13A (Foundation) DONE -> 13A-2 (Upload+Viz) -> 13B (Causal) -> 13C (Mexico) -> 13D (US) -> 13E (MCP Self-Service) + 13A (Foundation) DONE -> 13B partial (DiD+Viz) DONE -> 13A-2 (Upload) -> 13B remaining (PSM/RDD) -> 13C (Mexico) -> 13D (US) -> 13E (MCP) | Demo + Video -- TODO | @@ -1013,29 +1013,37 @@ Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evalu **Counts:** 70 -> 73 tools, 16 -> 19 data sources (18 external + 1 internal), 10 -> 11 bounded contexts, 3 -> 4 domains. -### Phase 13A-2: Document Upload + Visualization (TODO) +### Phase 13B (partial): Causal Inference + Impact Viz -- DONE + +- [x] `CausalEstimator` port ABC in `domain/ports.py` +- [x] `DiDEstimator` infrastructure with parallel trends test and automated threat assessment +- [x] `estimate_did` tool -- difference-in-differences with effect size, SE, p-value, threats, evidence tier +- [x] `assess_threats` tool -- knowledge-based validity threat assessment (DiD, PSM, RDD, RCT, IV) +- [x] `render_program_dashboard` viz tool -- multi-indicator KPI dashboard with target tracking +- [x] `render_geographic_comparison` viz tool -- region bar chart with benchmark reference line +- [x] `render_parallel_trends` viz tool -- DiD treatment vs control over time +- [x] Console: 3 new React chart components (ProgramDashboard, GeographicComparison, ParallelTrends) +- [x] Console: 2 Impact Evaluation template cards (CCT School Enrollment, Health Worker Programs) +- [x] Tests: DiD estimator + tool tests (35+ new tests) + +**Counts:** 73 -> 78 tools, 12 -> 15 viz tools, 3 -> 5 impact tools. + +### Phase 13A-2: Document Upload (TODO -- deferred post-hackathon) - [ ] File upload API (CSV, PDF, Excel) via multipart/form-data on `POST /investigate` - [ ] PDF text extraction (`pymupdf`) + Haiku summarization for long documents - [ ] CSV/Excel parsing (`pandas` + `openpyxl`) with schema detection and summary statistics - [ ] Inject uploaded data into Director prompt as `` XML block - [ ] `extract_program_data` tool -- parse uploaded files into structured program data -- [ ] `render_program_dashboard` viz tool -- multi-indicator KPI dashboard -- [ ] `render_geographic_comparison` viz tool -- state/region comparison bar chart - [ ] Frontend: `FileUpload.tsx` drag-and-drop component in `PromptInput` - [ ] Frontend: `DataPreview.tsx` for uploaded dataset preview -### Phase 13B: Causal Inference Engine +### Phase 13B (remaining): Additional Causal Methods (TODO -- deferred post-hackathon) -- [ ] `estimate_did` tool -- difference-in-differences with parallel trends test - [ ] `estimate_psm` tool -- propensity score matching with balance diagnostics - [ ] `estimate_rdd` tool -- regression discontinuity (sharp/fuzzy) - [ ] `estimate_synthetic_control` tool -- synthetic control method -- [ ] `assess_threats` tool -- automated threat-to-validity assessment - [ ] `compute_cost_effectiveness` tool -- cost per unit outcome, ICER -- [ ] `CausalEstimate` domain entity with effect size, CI, p-value, threats -- [ ] `ThreatToValidity` domain entity (selection bias, attrition, spillover) -- [ ] `render_parallel_trends` viz tool -- DiD pre/post trends visual - [ ] `render_rdd_plot` viz tool -- regression discontinuity scatter - [ ] `render_causal_diagram` viz tool -- DAG showing treatment/outcome/confounders - [ ] Frontend: `ThreatAssessment.tsx` panel with severity badges diff --git a/server/src/ehrlich/investigation/tools_viz.py b/server/src/ehrlich/investigation/tools_viz.py index 0f445c7..c8649ab 100644 --- a/server/src/ehrlich/investigation/tools_viz.py +++ b/server/src/ehrlich/investigation/tools_viz.py @@ -1,10 +1,11 @@ """Visualization tools for the investigation engine. -12 tools that generate structured chart payloads for the frontend: +15 tools that generate structured chart payloads for the frontend: binding scatter, ADMET radar, training timeline, muscle heatmap, forest plot, evidence matrix, performance chart, funnel plot, dose-response curve, nutrient comparison, nutrient adequacy, -and therapeutic window. +therapeutic window, program dashboard, geographic comparison, +and parallel trends. """ from __future__ import annotations @@ -612,3 +613,125 @@ async def render_therapeutic_window( "config": {"domain": "nutrition"}, } ) + + +async def render_program_dashboard( + indicators: list[dict[str, Any]], + program_name: str = "Program", + title: str = "", +) -> str: + """Render a multi-indicator KPI dashboard for a social program. + + Shows baseline, target, and actual values for each indicator with + progress coloring (green >= 80%, yellow 50-80%, red < 50%). + + Args: + indicators: List of dicts with 'name', 'baseline', 'target', + 'actual', 'unit' + program_name: Name of the program + title: Chart title (defaults to program name) + """ + processed: list[dict[str, Any]] = [] + for ind in indicators: + baseline = float(ind.get("baseline", 0.0)) + target = float(ind.get("target", 0.0)) + actual = float(ind.get("actual", 0.0)) + + pct_target = actual / target if target != 0 else 0.0 + if pct_target >= 0.8: + status = "on_track" + elif pct_target >= 0.5: + status = "at_risk" + else: + status = "off_track" + + processed.append({ + "name": ind.get("name", ""), + "baseline": round(baseline, 2), + "target": round(target, 2), + "actual": round(actual, 2), + "unit": ind.get("unit", ""), + "pct_target": round(pct_target, 4), + "status": status, + }) + + return json.dumps({ + "viz_type": "program_dashboard", + "title": title or f"{program_name} Dashboard", + "data": { + "program_name": program_name, + "indicators": processed, + }, + "config": {"domain": "impact"}, + }) + + +async def render_geographic_comparison( + regions: list[dict[str, Any]], + metric_name: str = "Value", + benchmark: float | None = None, + title: str = "Geographic Comparison", +) -> str: + """Render a bar chart comparing regions with optional benchmark line. + + Args: + regions: List of dicts with 'name' and 'value' + metric_name: Label for the metric axis + benchmark: Optional benchmark value shown as reference line + title: Chart title + """ + processed: list[dict[str, Any]] = [] + for r in regions: + processed.append({ + "name": r.get("name", ""), + "value": round(float(r.get("value", 0.0)), 2), + }) + + return json.dumps({ + "viz_type": "geographic_comparison", + "title": title, + "data": { + "regions": processed, + "metric_name": metric_name, + "benchmark": round(benchmark, 2) if benchmark is not None else None, + }, + "config": {"domain": "impact"}, + }) + + +async def render_parallel_trends( + treatment_series: list[dict[str, Any]], + control_series: list[dict[str, Any]], + treatment_start: str = "", + title: str = "Parallel Trends", +) -> str: + """Render a DiD parallel trends chart (treatment vs control over time). + + Shows two line series with a vertical reference line at treatment + start and shaded pre/post regions. + + Args: + treatment_series: List of dicts with 'period' and 'value' + control_series: List of dicts with 'period' and 'value' + treatment_start: Period label where treatment begins + title: Chart title + """ + treatment = [ + {"period": t.get("period", ""), "value": round(float(t.get("value", 0.0)), 2)} + for t in treatment_series + ] + control = [ + {"period": c.get("period", ""), "value": round(float(c.get("value", 0.0)), 2)} + for c in control_series + ] + + return json.dumps({ + "viz_type": "parallel_trends", + "title": title, + "data": { + "treatment": treatment, + "control": control, + "treatment_start": treatment_start, + }, + "config": {"domain": "impact"}, + }) diff --git a/server/tests/investigation/test_tools_viz_impact.py b/server/tests/investigation/test_tools_viz_impact.py new file mode 100644 index 0000000..4971c7b --- /dev/null +++ b/server/tests/investigation/test_tools_viz_impact.py @@ -0,0 +1,109 @@ +"""Tests for impact evaluation visualization tools.""" + +from __future__ import annotations + +import json + +import pytest + +from ehrlich.investigation.tools_viz import ( + render_geographic_comparison, + render_parallel_trends, + render_program_dashboard, +) + + +class TestRenderProgramDashboard: + @pytest.mark.anyio + async def test_basic(self) -> None: + result = json.loads(await render_program_dashboard( + indicators=[ + {"name": "Enrollment", "baseline": 70.0, "target": 90.0, + "actual": 85.0, "unit": "%"}, + {"name": "Attendance", "baseline": 60.0, "target": 80.0, + "actual": 35.0, "unit": "%"}, + ], + program_name="Prospera", + )) + assert result["viz_type"] == "program_dashboard" + assert result["data"]["program_name"] == "Prospera" + assert len(result["data"]["indicators"]) == 2 + assert result["data"]["indicators"][0]["status"] == "on_track" + assert result["data"]["indicators"][1]["status"] == "off_track" + + @pytest.mark.anyio + async def test_empty_indicators(self) -> None: + result = json.loads(await render_program_dashboard(indicators=[])) + assert result["viz_type"] == "program_dashboard" + assert result["data"]["indicators"] == [] + + @pytest.mark.anyio + async def test_custom_title(self) -> None: + result = json.loads(await render_program_dashboard( + indicators=[{"name": "X", "baseline": 0, "target": 10, "actual": 7, "unit": "pts"}], + title="Custom Title", + )) + assert result["title"] == "Custom Title" + + +class TestRenderGeographicComparison: + @pytest.mark.anyio + async def test_basic(self) -> None: + result = json.loads(await render_geographic_comparison( + regions=[ + {"name": "Mexico", "value": 85.2}, + {"name": "Brazil", "value": 92.1}, + {"name": "Colombia", "value": 78.5}, + ], + metric_name="Enrollment Rate", + benchmark=88.0, + )) + assert result["viz_type"] == "geographic_comparison" + assert len(result["data"]["regions"]) == 3 + assert result["data"]["benchmark"] == 88.0 + assert result["data"]["metric_name"] == "Enrollment Rate" + + @pytest.mark.anyio + async def test_no_benchmark(self) -> None: + result = json.loads(await render_geographic_comparison( + regions=[{"name": "A", "value": 10.0}], + )) + assert result["data"]["benchmark"] is None + + @pytest.mark.anyio + async def test_empty_regions(self) -> None: + result = json.loads(await render_geographic_comparison(regions=[])) + assert result["data"]["regions"] == [] + + +class TestRenderParallelTrends: + @pytest.mark.anyio + async def test_basic(self) -> None: + result = json.loads(await render_parallel_trends( + treatment_series=[ + {"period": "2010", "value": 70.0}, + {"period": "2011", "value": 72.0}, + {"period": "2012", "value": 80.0}, + {"period": "2013", "value": 85.0}, + ], + control_series=[ + {"period": "2010", "value": 70.0}, + {"period": "2011", "value": 71.0}, + {"period": "2012", "value": 72.0}, + {"period": "2013", "value": 73.0}, + ], + treatment_start="2012", + )) + assert result["viz_type"] == "parallel_trends" + assert len(result["data"]["treatment"]) == 4 + assert len(result["data"]["control"]) == 4 + assert result["data"]["treatment_start"] == "2012" + + @pytest.mark.anyio + async def test_empty_series(self) -> None: + result = json.loads(await render_parallel_trends( + treatment_series=[], + control_series=[], + )) + assert result["data"]["treatment"] == [] + assert result["data"]["control"] == [] From 46ba33b200b7fccb9bb0f580f187cb9945277c3b Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 01:32:59 -0700 Subject: [PATCH 05/35] feat(analysis): add domain-agnostic causal inference (DiD, PSM, RDD, SC) Move causal inference from impact/ to analysis/ bounded context, making it available to all domains. Add three new estimators (PSM, RDD, Synthetic Control) alongside the existing DiD, plus cost-effectiveness analysis. Follows prediction/ ports-and-adapters pattern with 4 ISP ABCs. - 6 causal tools tagged "causal" (estimate_did, estimate_psm, estimate_rdd, estimate_synthetic_control, assess_threats, compute_cost_effectiveness) - 2 viz tools (render_rdd_plot, render_causal_diagram) + React components - ThreatAssessment console component for validity threat display - Impact tools trimmed from 5 to 3 (data-fetching only) - 9 template cards (added Education Policy, Health Policy) - 84 total tools, 17 viz tools, all quality gates green --- CLAUDE.md | 56 ++-- README.md | 21 +- .../components/InvestigationReport.tsx | 22 +- .../components/TemplateCards.test.tsx | 6 +- .../components/ThreatAssessment.tsx | 104 +++++++ .../features/visualization/VizRegistry.tsx | 4 + .../visualization/charts/CausalDiagram.tsx | 204 +++++++++++++ .../features/visualization/charts/RDDPlot.tsx | 187 ++++++++++++ docs/architecture.md | 13 +- docs/impact-evaluation-domain.md | 71 +++-- docs/roadmap.md | 46 ++- .../analysis/application/causal_service.py | 207 +++++++++++++ server/src/ehrlich/analysis/domain/causal.py | 39 +++ .../ehrlich/analysis/domain/causal_ports.py | 64 ++++ .../analysis/domain/evidence_standards.py | 69 +++++ .../infrastructure/did_estimator.py | 159 +++++----- .../analysis/infrastructure/psm_estimator.py | 263 +++++++++++++++++ .../analysis/infrastructure/rdd_estimator.py | 264 +++++++++++++++++ .../synthetic_control_estimator.py | 213 ++++++++++++++ server/src/ehrlich/analysis/tools.py | 278 ++++++++++++++++++ .../src/ehrlich/api/routes/investigation.py | 27 +- .../impact/application/impact_service.py | 87 ------ server/src/ehrlich/impact/domain/entities.py | 23 -- .../impact/domain/evaluation_standards.py | 69 +---- server/src/ehrlich/impact/domain/ports.py | 34 --- server/src/ehrlich/impact/tools.py | 87 +----- .../application/multi_orchestrator.py | 10 +- .../investigation/domain/domains/impact.py | 31 +- .../infrastructure/repository.py | 44 +-- server/src/ehrlich/investigation/tools_viz.py | 211 ++++++++++--- .../infrastructure/semantic_scholar.py | 4 +- server/tests/analysis/test_causal.py | 77 +++++ server/tests/analysis/test_causal_service.py | 77 +++++ server/tests/analysis/test_causal_tools.py | 188 ++++++++++++ .../test_did_estimator.py | 56 ++-- .../tests/analysis/test_evidence_standards.py | 79 +++++ server/tests/analysis/test_psm_estimator.py | 69 +++++ server/tests/analysis/test_rdd_estimator.py | 67 +++++ server/tests/analysis/test_statistics.py | 4 +- .../test_synthetic_control_estimator.py | 71 +++++ server/tests/api/test_auth.py | 28 +- server/tests/api/test_credits.py | 16 +- server/tests/api/test_investigation.py | 4 +- .../tests/impact/test_evaluation_standards.py | 96 +----- server/tests/impact/test_tools_did.py | 124 -------- server/tests/investigation/test_repository.py | 108 ++----- .../investigation/test_tools_viz_impact.py | 110 ++++--- web/src/lib/constants.ts | 14 +- 48 files changed, 3145 insertions(+), 960 deletions(-) create mode 100644 console/src/features/investigation/components/ThreatAssessment.tsx create mode 100644 console/src/features/visualization/charts/CausalDiagram.tsx create mode 100644 console/src/features/visualization/charts/RDDPlot.tsx create mode 100644 server/src/ehrlich/analysis/application/causal_service.py create mode 100644 server/src/ehrlich/analysis/domain/causal.py create mode 100644 server/src/ehrlich/analysis/domain/causal_ports.py create mode 100644 server/src/ehrlich/analysis/domain/evidence_standards.py rename server/src/ehrlich/{impact => analysis}/infrastructure/did_estimator.py (53%) create mode 100644 server/src/ehrlich/analysis/infrastructure/psm_estimator.py create mode 100644 server/src/ehrlich/analysis/infrastructure/rdd_estimator.py create mode 100644 server/src/ehrlich/analysis/infrastructure/synthetic_control_estimator.py delete mode 100644 server/src/ehrlich/impact/domain/ports.py create mode 100644 server/tests/analysis/test_causal.py create mode 100644 server/tests/analysis/test_causal_service.py create mode 100644 server/tests/analysis/test_causal_tools.py rename server/tests/{impact => analysis}/test_did_estimator.py (54%) create mode 100644 server/tests/analysis/test_evidence_standards.py create mode 100644 server/tests/analysis/test_psm_estimator.py create mode 100644 server/tests/analysis/test_rdd_estimator.py create mode 100644 server/tests/analysis/test_synthetic_control_estimator.py delete mode 100644 server/tests/impact/test_tools_did.py diff --git a/CLAUDE.md b/CLAUDE.md index 2476ac1..a2d9b2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ DDD monorepo: `server/` (Python 3.12) + `console/` (React 19 / TypeScript / Bun) ``` Opus 4.6 (Director) -- Formulates hypotheses, designs experiments, evaluates evidence, synthesizes (NO tools) -Sonnet 4.5 (Researcher) -- Executes experiments with 73 tools (parallel: 2 experiments per batch) +Sonnet 4.5 (Researcher) -- Executes experiments with 79 tools (parallel: 2 experiments per batch) Haiku 4.5 (Summarizer) -- Compresses large tool outputs (>2000 chars), PICO decomposition, evidence grading ``` @@ -35,12 +35,12 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 | shared | `server/src/ehrlich/shared/` | Cross-cutting ports and value objects (ChemistryPort, Fingerprint, Conformer3D, MolecularDescriptors) | | literature | `server/src/ehrlich/literature/` | Paper search (Semantic Scholar), references | | chemistry | `server/src/ehrlich/chemistry/` | RDKit cheminformatics | -| analysis | `server/src/ehrlich/analysis/` | Dataset exploration (ChEMBL, PubChem, GtoPdb), enrichment | +| analysis | `server/src/ehrlich/analysis/` | Dataset exploration (ChEMBL, PubChem, GtoPdb), enrichment, domain-agnostic causal inference (DiD, PSM, RDD, Synthetic Control) | | prediction | `server/src/ehrlich/prediction/` | ML models (Chemprop, XGBoost) | | simulation | `server/src/ehrlich/simulation/` | Docking, ADMET, resistance, target discovery (RCSB PDB, UniProt, Open Targets), toxicity (EPA CompTox) | | training | `server/src/ehrlich/training/` | Evidence analysis, protocol comparison, injury risk, training metrics, clinical trials (ClinicalTrials.gov), PubMed literature (E-utilities), exercise database (wger) | | nutrition | `server/src/ehrlich/nutrition/` | Supplement evidence, supplement labels (NIH DSLD), nutrient data (USDA FoodData), supplement safety (OpenFDA CAERS), drug-nutrient interactions (RxNav), DRI adequacy, nutrient ratios, inflammatory index | -| impact | `server/src/ehrlich/impact/` | Social program evaluation: causal inference (DiD), threat assessment, economic indicators (World Bank, WHO GHO, FRED), cross-program comparison | +| impact | `server/src/ehrlich/impact/` | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking | | investigation | `server/src/ehrlich/investigation/` | Multi-model orchestration + PostgreSQL persistence + domain registry + MCP bridge | ### External Data Sources (18 external + 1 internal) @@ -141,7 +141,7 @@ Types: `feat`, `fix`, `refactor`, `chore`, `docs`, `test` Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, training, nutrition, impact, investigation, api, console, mol, data, ci, docs, infra -## Tools (78 Total) +## Tools (84 Total) ### Chemistry (6) -- RDKit cheminformatics, domain-agnostic - `validate_smiles` -- Validate SMILES string @@ -156,13 +156,19 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `search_citations` -- Citation chasing (snowballing) via references/citing papers - `get_reference` -- Curated reference lookup by key or DOI -### Analysis (6) -- ChEMBL + PubChem + GtoPdb +### Analysis (12) -- ChEMBL + PubChem + GtoPdb + Domain-Agnostic Causal Inference - `explore_dataset` -- Load ChEMBL bioactivity data for any organism/target - `search_bioactivity` -- Flexible ChEMBL query (any assay type: MIC, Ki, EC50, IC50, Kd) - `search_compounds` -- PubChem compound search by target/activity/similarity - `search_pharmacology` -- GtoPdb curated receptor/ligand interactions with affinities - `analyze_substructures` -- Chi-squared enrichment of SMARTS patterns - `compute_properties` -- Property distributions (active vs inactive) +- `estimate_did` -- Difference-in-differences causal estimation with parallel trends test and threat assessment +- `estimate_psm` -- Propensity score matching with balance diagnostics +- `estimate_rdd` -- Regression discontinuity design (sharp/fuzzy) +- `estimate_synthetic_control` -- Synthetic control method for single treated unit +- `assess_threats` -- Knowledge-based validity threat assessment for causal methods (DiD, PSM, RDD, RCT, IV) +- `compute_cost_effectiveness` -- Cost per unit outcome, ICER calculation ### Prediction (6) -- XGBoost, domain-agnostic ML - `train_model` -- Train ML model on SMILES+activity dataset (scaffold-split + random-split metrics, permutation p-value) @@ -206,14 +212,12 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `analyze_nutrient_ratios` -- Clinically relevant nutrient ratios (omega-6:3, Ca:Mg, Na:K, Zn:Cu, Ca:P, Fe:Cu) - `compute_inflammatory_index` -- Simplified Dietary Inflammatory Index (DII) scoring -### Impact Evaluation (5) -- Social program analysis, causal inference, economic indicators +### Impact Evaluation (3) -- Social program analysis, economic indicators - `search_economic_indicators` -- Query economic time series from FRED, World Bank, or WHO GHO - `fetch_benchmark` -- Get comparison values from international sources (World Bank, WHO GHO) - `compare_programs` -- Cross-program comparison using existing statistical tests -- `estimate_did` -- Difference-in-differences causal estimation with parallel trends test and threat assessment -- `assess_threats` -- Knowledge-based validity threat assessment for causal methods (DiD, PSM, RDD, RCT, IV) -### Visualization (15) -- Domain-specific interactive charts +### Visualization (17) -- Domain-specific interactive charts - `render_binding_scatter` -- Scatter plot of compound binding affinities (Recharts) - `render_admet_radar` -- Radar chart of ADMET/drug-likeness properties (Recharts) - `render_training_timeline` -- Training load timeline with ACWR danger zones (Recharts) @@ -229,6 +233,8 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `render_program_dashboard` -- Multi-indicator KPI dashboard with target tracking (Recharts) - `render_geographic_comparison` -- Region bar chart with benchmark reference line (Recharts) - `render_parallel_trends` -- DiD parallel trends chart, treatment vs control over time (Recharts) +- `render_rdd_plot` -- Regression discontinuity scatter with cutoff line and fitted curves (Visx) +- `render_causal_diagram` -- DAG showing treatment, outcome, confounders, and mediators (Visx) ### Statistics (2) -- Domain-agnostic hypothesis testing (scipy.stats) - `run_statistical_test` -- Compare two numeric groups (auto-selects t-test/Welch/Mann-Whitney, returns p-value, Cohen's d, 95% CI) @@ -281,7 +287,7 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - **Domain configuration**: `DomainConfig` defines per-domain tool tags, score definitions, prompt examples; `DomainRegistry` auto-detects domain; `DomainDetected` SSE event sends display config to frontend - **Domain tool examples**: each `DomainConfig` includes `tool_examples` in `experiment_examples` showing realistic tool chaining patterns; ensures Researcher (Sonnet 4.5) has usage guidance for all domain-specific tools - **Multi-domain investigations**: `DomainRegistry.detect()` returns `list[DomainConfig]`; `merge_domain_configs()` creates synthetic merged config with union of tool_tags, concatenated score_definitions, joined prompt examples -- **Tool tagging**: Tools tagged with frozenset domain tags (chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, impact, literature, visualization); investigation control tools are universal (no tags); researcher sees only domain-relevant tools +- **Tool tagging**: Tools tagged with frozenset domain tags (chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, impact, causal, literature, visualization); investigation control tools are universal (no tags); researcher sees only domain-relevant tools - **Self-referential research**: `search_prior_research` queries PostgreSQL `tsvector` + GIN index of past findings; indexed on completion; intercepted in orchestrator `_dispatch_tool()` and routed to repository; "ehrlich" source type links to past investigations - **MCP bridge**: Optional `MCPBridge` connects to external MCP servers (e.g. Excalidraw); tools registered dynamically via `ToolRegistry.register_mcp_tools()`; lifecycle managed by orchestrator; enabled via `EHRLICH_MCP_EXCALIDRAW=true` - **SSE streaming**: 20 event types for real-time updates; reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) @@ -292,13 +298,13 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, ### UI Patterns -- **Visualization pipeline**: 12 viz tools return `VisualizationPayload` JSON; orchestrator intercepts via `_maybe_viz_event()` and emits `VisualizationRendered` SSE event; `VizRegistry` maps `viz_type` to lazy-loaded React component; `VisualizationPanel` renders all charts +- **Visualization pipeline**: 17 viz tools return `VisualizationPayload` JSON; orchestrator intercepts via `_maybe_viz_event()` and emits `VisualizationRendered` SSE event; `VizRegistry` maps `viz_type` to lazy-loaded React component; `VisualizationPanel` renders all charts - **LiveLabViewer**: 3Dmol.js scene driven by SSE events -- protein targets load, ligands dock, candidates color by score. SSE event mapping: `dock_against_target` -> ligand in binding pocket, `predict_candidates` -> color by probability, `completed` -> top candidates glow. Interactive rotate/zoom/click, split-pane with Timeline - **React Flow diagrams**: `@xyflow/react` node graph with custom `InvestigationNode` and `AnnotationNode` types; status-colored nodes (proposed=gray, testing=blue, supported=green, refuted=red, revised=orange); dashed edges for revisions, solid smoothstep for links; read-only with zoom/pan/minimap; lazy-loaded (~188KB chunk) - **Investigation report**: 8 sections (Research Question, Executive Summary, Hypotheses & Outcomes, Methodology, Key Findings, Candidate Molecules, Model Validation, Cost & Performance); shown post-completion with full audit trail; markdown export via client-side generation - **Molecule visualization**: server-side 2D SVG (RDKit `rdMolDraw2D`), 3Dmol.js for 3D/docking; `CandidateTable` shows thumbnails with expandable detail (3D viewer + properties + Lipinski badge) - **Molecule API**: `/molecule/depict` (SVG, cached 24h), `/molecule/conformer`, `/molecule/descriptors`, `/molecule/targets` -- **Investigation templates**: 7 cross-domain prompts (4 molecular + 2 training + 1 nutrition) on home page via `TemplateCards` with domain badges +- **Investigation templates**: 9 cross-domain prompts (4 molecular + 2 training + 1 nutrition + 2 impact) on home page via `TemplateCards` with domain badges - **Candidate comparison**: side-by-side scoring view for 2-4 candidates with best-in-group highlighting - **HypothesisBoard**: kanban-style card grid showing hypothesis status with expandable confidence bars - **Hypothesis approval**: `HypothesisApprovalCard` renders in main content area (not status bar) for maximum prominence during approval gate @@ -340,10 +346,18 @@ All paths relative to `server/src/ehrlich/`. | File | Purpose | |------|---------| | `domain/statistics.py` | `StatisticalResult` frozen dataclass | +| `domain/causal.py` | `ThreatToValidity`, `CausalEstimate`, `CostEffectivenessResult` frozen dataclasses | +| `domain/causal_ports.py` | `DiDEstimatorPort`, `PSMEstimatorPort`, `RDDEstimatorPort`, `SyntheticControlPort` ABCs | +| `domain/evidence_standards.py` | WWC evidence tiers, CONEVAL MIR levels, CREMAA criteria, `classify_evidence_tier()` | | `application/statistics_service.py` | `StatisticsService` -- scipy.stats hypothesis testing (t-test, Welch, Mann-Whitney, chi-squared, Fisher) | +| `application/causal_service.py` | `CausalService` -- DiD, PSM, RDD, Synthetic Control estimation, threat assessment, cost-effectiveness | | `infrastructure/chembl_loader.py` | ChEMBL bioactivity loader (flexible assay types) | | `infrastructure/pubchem_client.py` | PubChem PUG REST compound search | | `infrastructure/gtopdb_client.py` | GtoPdb REST client for curated pharmacology | +| `infrastructure/did_estimator.py` | DiD estimator: effect size, SE, p-value, parallel trends, automated threat assessment | +| `infrastructure/psm_estimator.py` | Propensity score matching with balance diagnostics | +| `infrastructure/rdd_estimator.py` | Regression discontinuity design (sharp/fuzzy) estimator | +| `infrastructure/synthetic_control_estimator.py` | Synthetic control method for single treated unit | ### prediction/ @@ -397,16 +411,13 @@ All paths relative to `server/src/ehrlich/`. | File | Purpose | |------|---------| -| `tools.py` | 5 impact evaluation tools | -| `domain/entities.py` | DataPoint, Program, Indicator, ThreatToValidity, CausalEstimate, Benchmark, EconomicSeries, HealthIndicator | +| `tools.py` | 3 impact evaluation tools (economic indicators, benchmarks, program comparison) | +| `domain/entities.py` | DataPoint, Program, Indicator, Benchmark, EconomicSeries, HealthIndicator | | `domain/repository.py` | EconomicDataRepository, HealthDataRepository, DevelopmentDataRepository ABCs | -| `domain/ports.py` | `CausalEstimator` ABC for causal inference methods | -| `domain/evaluation_standards.py` | WWC evidence tiers, CONEVAL MIR levels, CREMAA criteria, `classify_evidence_tier()` | -| `application/impact_service.py` | ImpactService (economic indicators, benchmarks, program comparison, DiD estimation, threat assessment) | +| `application/impact_service.py` | ImpactService (economic indicators, benchmarks, program comparison) | | `infrastructure/worldbank_client.py` | World Bank API v2 client for development indicators | | `infrastructure/who_client.py` | WHO GHO API client for health statistics | | `infrastructure/fred_client.py` | FRED API client for US economic time series | -| `infrastructure/did_estimator.py` | DiD estimator: effect size, SE, p-value, parallel trends, automated threat assessment | ### investigation/ @@ -434,7 +445,7 @@ All paths relative to `server/src/ehrlich/`. | `domain/domains/nutrition.py` | `NUTRITION_SCIENCE` config | | `domain/domains/impact.py` | `IMPACT_EVALUATION` config | | `domain/mcp_config.py` | `MCPServerConfig` frozen dataclass | -| `tools_viz.py` | 15 visualization tools | +| `tools_viz.py` | 17 visualization tools | | `infrastructure/repository.py` | PostgreSQL persistence (asyncpg) + tsvector findings search + user/credit management | | `infrastructure/anthropic_client.py` | Anthropic API adapter with retry, streaming, and structured outputs | | `infrastructure/mcp_bridge.py` | MCP client bridge (stdio/SSE/streamable_http) | @@ -446,7 +457,7 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| | `auth.py` | WorkOS JWT middleware (JWKS verification, header + query param auth for SSE) | -| `routes/investigation.py` | REST + SSE endpoints, 78-tool registry, domain registry, MCP bridge, credit system, BYOK | +| `routes/investigation.py` | REST + SSE endpoints, 84-tool registry, domain registry, MCP bridge, credit system, BYOK | | `routes/methodology.py` | GET /methodology: phases, domains, tools, data sources, models | | `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets endpoints | | `routes/stats.py` | GET /stats: aggregate counts | @@ -473,7 +484,8 @@ All paths relative to `console/src/`. | `features/investigation/components/ActiveExperimentCard.tsx` | Live experiment activity card | | `features/investigation/components/CompletionSummaryCard.tsx` | Post-completion summary card with expandable cost breakdown | | `features/investigation/components/NegativeControlPanel.tsx` | Negative control validation table | -| `features/investigation/components/TemplateCards.tsx` | 7 cross-domain research prompt templates | +| `features/investigation/components/TemplateCards.tsx` | 9 cross-domain research prompt templates | +| `features/investigation/components/ThreatAssessment.tsx` | Validity threat assessment panel with severity badges | | `features/investigation/components/PromptInput.tsx` | Controlled prompt input with director tier selector pills | | `features/investigation/components/BYOKSettings.tsx` | API key management for BYOK users | @@ -516,6 +528,8 @@ All paths relative to `console/src/`. | `features/visualization/charts/ProgramDashboard.tsx` | Recharts horizontal BarChart with target tracking | | `features/visualization/charts/GeographicComparison.tsx` | Recharts BarChart with benchmark ReferenceLine | | `features/visualization/charts/ParallelTrends.tsx` | Recharts ComposedChart for DiD treatment vs control | +| `features/visualization/charts/RDDPlot.tsx` | Visx regression discontinuity scatter with cutoff and fitted curves | +| `features/visualization/charts/CausalDiagram.tsx` | Visx DAG showing treatment, outcome, confounders, mediators | | `features/visualization/anatomy/BodyDiagram.tsx` | Custom SVG anatomical body diagram | | `features/visualization/anatomy/body-paths.ts` | SVG path data for front/back views | | `features/visualization/anatomy/color-scale.ts` | OKLCH intensity color interpolation | diff --git a/README.md b/README.md index dd23088..75b6781 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,11 @@ Ehrlich is **domain-agnostic**. The hypothesis-driven engine works for any scien ### Impact Evaluation - **Social program analysis** -- "Evaluate the effectiveness of Sonora's sports scholarship program" -- **Causal inference** -- "Does the conditional cash transfer reduce school dropout rates?" +- **Causal inference** -- "Does the conditional cash transfer reduce school dropout rates?" (4 methods: DiD, PSM, RDD, Synthetic Control) - **Cross-country benchmarking** -- "Compare cost-effectiveness of state sports programs in Mexico" - Economic indicators from World Bank, WHO GHO, and FRED (800K+ time series) - Cross-program comparison and international benchmarking +- Domain-agnostic causal inference tools (in analysis/ context) usable by any domain ## Architecture @@ -42,13 +43,13 @@ Ehrlich follows Domain-Driven Design with eleven bounded contexts: | **shared** | Cross-cutting ports and value objects (ChemistryPort, Fingerprint, Conformer3D) | | **literature** | Scientific paper search (Semantic Scholar) and reference management | | **chemistry** | Cheminformatics: molecular descriptors, fingerprints, 3D conformers | -| **analysis** | Dataset exploration (ChEMBL, PubChem), substructure enrichment | +| **analysis** | Dataset exploration (ChEMBL, PubChem), substructure enrichment, domain-agnostic causal inference (DiD, PSM, RDD, Synthetic Control) | | **prediction** | ML modeling: train, predict, ensemble, cluster | | **simulation** | Molecular docking, ADMET, resistance, target discovery (RCSB PDB), toxicity (EPA CompTox) | | **training** | Exercise physiology: evidence analysis, protocol comparison, injury risk, training metrics, clinical trials | | **nutrition** | Nutrition science: supplement evidence, labels, nutrients, safety, interactions, adequacy, inflammatory index | | **investigation** | Multi-model agent orchestration with Director-Worker-Summarizer pattern + domain registry + MCP bridge | -| **impact** | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking | +| **impact** | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking. Causal estimators live in analysis/ (domain-agnostic) | ### Multi-Model Architecture @@ -56,7 +57,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga ``` Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) -Sonnet 4.5 (Researcher) -- Executes experiments with 78 tools (10-20 calls) +Sonnet 4.5 (Researcher) -- Executes experiments with 84 tools (10-20 calls) Haiku 4.5 (Summarizer) -- Compresses large outputs, classifies domains (5-10 calls) ``` @@ -87,7 +88,7 @@ Cost: ~$3-4 per investigation (vs ~$11 with all-Opus). All data sources are free and open-access. -## 78 Tools +## 84 Tools | Context | Tool | Description | |---------|------|-------------| @@ -106,6 +107,12 @@ All data sources are free and open-access. | Analysis | `analyze_substructures` | Chi-squared enrichment analysis | | Analysis | `compute_properties` | Property distributions (active vs inactive) | | Analysis | `search_pharmacology` | GtoPdb curated receptor/ligand interactions | +| Causal | `estimate_did` | Difference-in-differences causal estimation | +| Causal | `estimate_psm` | Propensity score matching with balance diagnostics | +| Causal | `estimate_rdd` | Regression discontinuity design (sharp/fuzzy) | +| Causal | `estimate_synthetic_control` | Synthetic control method | +| Causal | `assess_threats` | Validity threat assessment for causal methods | +| Causal | `compute_cost_effectiveness` | Cost per unit outcome, ICER | | Prediction | `train_model` | Train XGBoost on SMILES+activity data | | Prediction | `predict_candidates` | Score compounds with trained model | | Prediction | `cluster_compounds` | Butina structural clustering | @@ -143,8 +150,6 @@ All data sources are free and open-access. | Impact | `search_economic_indicators` | Query economic time series from FRED, World Bank, or WHO GHO | | Impact | `fetch_benchmark` | Get comparison values from international sources | | Impact | `compare_programs` | Cross-program comparison using statistical tests | -| Impact | `estimate_did` | Difference-in-differences causal estimation | -| Impact | `assess_threats` | Validity threat assessment for causal methods | | Visualization | `render_binding_scatter` | Scatter plot of compound binding affinities | | Visualization | `render_admet_radar` | Radar chart of ADMET/drug-likeness properties | | Visualization | `render_training_timeline` | Training load timeline with ACWR danger zones | @@ -160,6 +165,8 @@ All data sources are free and open-access. | Visualization | `render_program_dashboard` | Multi-indicator KPI dashboard with target tracking | | Visualization | `render_geographic_comparison` | Region bar chart with benchmark reference line | | Visualization | `render_parallel_trends` | DiD parallel trends chart (treatment vs control) | +| Visualization | `render_rdd_plot` | Regression discontinuity scatter with cutoff line | +| Visualization | `render_causal_diagram` | DAG showing treatment, outcome, confounders | | Statistics | `run_statistical_test` | Compare two numeric groups (auto-selects t-test/Welch/Mann-Whitney) | | Statistics | `run_categorical_test` | Test contingency tables (auto-selects Fisher's exact/chi-squared) | | Investigation | `propose_hypothesis` | Register testable hypothesis | diff --git a/console/src/features/investigation/components/InvestigationReport.tsx b/console/src/features/investigation/components/InvestigationReport.tsx index 8dcaf09..14c9fed 100644 --- a/console/src/features/investigation/components/InvestigationReport.tsx +++ b/console/src/features/investigation/components/InvestigationReport.tsx @@ -24,6 +24,7 @@ import { CandidateTable } from "./CandidateTable"; import { FindingsPanel } from "./FindingsPanel"; import { HypothesisBoard } from "./HypothesisBoard"; import { NegativeControlPanel } from "./NegativeControlPanel"; +import { ThreatAssessment } from "./ThreatAssessment"; interface InvestigationReportProps { prompt: string; @@ -190,7 +191,26 @@ export function InvestigationReport({ - {/* 7. Model Validation -- reuse NegativeControlPanel with pass/fail icons */} + {/* 7. Validity Threats -- causal findings */} + {(() => { + const threats = findings + .filter((f) => f.evidence && typeof f.evidence === "object") + .flatMap((f) => { + const ev = f.evidence as unknown as Record; + return Array.isArray(ev.threats) ? ev.threats : []; + }) + .filter( + (t): t is { type: string; severity: string; description: string; mitigation: string } => + typeof t === "object" && t !== null && "type" in t && "severity" in t, + ); + return threats.length > 0 ? ( +
+ +
+ ) : null; + })()} + + {/* 8. Model Validation -- reuse NegativeControlPanel with pass/fail icons */} {negativeControls.length > 0 && (
diff --git a/console/src/features/investigation/components/TemplateCards.test.tsx b/console/src/features/investigation/components/TemplateCards.test.tsx index e92ed9d..5d05a50 100644 --- a/console/src/features/investigation/components/TemplateCards.test.tsx +++ b/console/src/features/investigation/components/TemplateCards.test.tsx @@ -3,9 +3,9 @@ import { render, screen } from "@testing-library/react"; import { TemplateCards } from "./TemplateCards"; describe("TemplateCards", () => { - it("renders 7 template cards with domain labels", () => { + it("renders 9 template cards with domain labels", () => { render( {}} />); - expect(screen.getAllByRole("button")).toHaveLength(7); + expect(screen.getAllByRole("button")).toHaveLength(9); }); it("renders all template domains", () => { @@ -18,6 +18,8 @@ describe("TemplateCards", () => { "Exercise Physiology", "Sports Medicine", "Sports Nutrition", + "Education Policy", + "Health Policy", ]; for (const domain of domains) { expect(screen.getAllByText(domain).length).toBeGreaterThanOrEqual(1); diff --git a/console/src/features/investigation/components/ThreatAssessment.tsx b/console/src/features/investigation/components/ThreatAssessment.tsx new file mode 100644 index 0000000..1359b15 --- /dev/null +++ b/console/src/features/investigation/components/ThreatAssessment.tsx @@ -0,0 +1,104 @@ +import { useState } from "react"; +import { ChevronDown, ChevronRight, Shield } from "lucide-react"; +import { cn } from "@/shared/lib/utils"; + +interface Threat { + type: string; + severity: string; + description: string; + mitigation: string; +} + +interface ThreatAssessmentProps { + threats: Threat[]; +} + +const SEVERITY_STYLES: Record = { + high: { + badge: "bg-destructive/20 text-destructive", + border: "border-destructive/30", + }, + medium: { + badge: "bg-accent/20 text-accent", + border: "border-accent/30", + }, + low: { + badge: "bg-primary/20 text-primary", + border: "border-primary/30", + }, +}; + +function ThreatRow({ threat }: { threat: Threat }) { + const [expanded, setExpanded] = useState(false); + const styles = SEVERITY_STYLES[threat.severity] ?? SEVERITY_STYLES.low; + + return ( +
+ + {expanded && ( +
+ + Mitigation: + {" "} + {threat.mitigation} +
+ )} +
+ ); +} + +export function ThreatAssessment({ threats }: ThreatAssessmentProps) { + if (threats.length === 0) return null; + + // Sort: high first, then medium, then low + const order: Record = { high: 0, medium: 1, low: 2 }; + const sorted = [...threats].sort( + (a, b) => (order[a.severity] ?? 3) - (order[b.severity] ?? 3), + ); + + const highCount = threats.filter((t) => t.severity === "high").length; + const medCount = threats.filter((t) => t.severity === "medium").length; + + return ( +
+
+

+ + Validity Threats ({threats.length}) +

+

+ {highCount > 0 && ( + {highCount} high severity + )} + {highCount > 0 && medCount > 0 && ", "} + {medCount > 0 && ( + {medCount} medium severity + )} + {(highCount > 0 || medCount > 0) && " -- "} + Threats to internal validity identified during causal analysis. +

+
+
+ {sorted.map((threat, i) => ( + + ))} +
+
+ ); +} diff --git a/console/src/features/visualization/VizRegistry.tsx b/console/src/features/visualization/VizRegistry.tsx index aa9d07b..4bac640 100644 --- a/console/src/features/visualization/VizRegistry.tsx +++ b/console/src/features/visualization/VizRegistry.tsx @@ -43,6 +43,8 @@ const LazyTherapeuticWindow = lazy(() => import('./charts/TherapeuticWindow')); const LazyProgramDashboard = lazy(() => import('./charts/ProgramDashboard')); const LazyGeographicComparison = lazy(() => import('./charts/GeographicComparison')); const LazyParallelTrends = lazy(() => import('./charts/ParallelTrends')); +const LazyRDDPlot = lazy(() => import('./charts/RDDPlot')); +const LazyCausalDiagram = lazy(() => import('./charts/CausalDiagram')); registry.set('binding_scatter', LazyBindingScatter as unknown as ChartComponent); registry.set('admet_radar', LazyADMETRadar as unknown as ChartComponent); @@ -59,6 +61,8 @@ registry.set('therapeutic_window', LazyTherapeuticWindow as unknown as ChartComp registry.set('program_dashboard', LazyProgramDashboard as unknown as ChartComponent); registry.set('geographic_comparison', LazyGeographicComparison as unknown as ChartComponent); registry.set('parallel_trends', LazyParallelTrends as unknown as ChartComponent); +registry.set('rdd_plot', LazyRDDPlot as unknown as ChartComponent); +registry.set('causal_diagram', LazyCausalDiagram as unknown as ChartComponent); /** * Look up a chart component by viz_type string. diff --git a/console/src/features/visualization/charts/CausalDiagram.tsx b/console/src/features/visualization/charts/CausalDiagram.tsx new file mode 100644 index 0000000..9b743ef --- /dev/null +++ b/console/src/features/visualization/charts/CausalDiagram.tsx @@ -0,0 +1,204 @@ +import { VIZ_COLORS } from '../theme'; + +interface CausalNode { + id: string; + label: string; + type: 'treatment' | 'outcome' | 'confounder' | 'mediator' | 'instrument'; +} + +interface CausalEdge { + source: string; + target: string; + type: 'causal' | 'association' | 'instrument'; +} + +interface CausalDiagramProps { + data: { + nodes: CausalNode[]; + edges: CausalEdge[]; + }; + title: string; +} + +const NODE_COLORS: Record = { + treatment: VIZ_COLORS.primary, + outcome: VIZ_COLORS.secondary, + confounder: VIZ_COLORS.warning, + mediator: VIZ_COLORS.neutral, + instrument: VIZ_COLORS.danger, +}; + +const NODE_WIDTH = 120; +const NODE_HEIGHT = 40; +const SVG_WIDTH = 600; +const SVG_HEIGHT = 400; + +function layoutNodes(nodes: CausalNode[]) { + // Simple layout: treatment left, outcome right, confounders top, mediators middle + const positions: Record = {}; + const treatments = nodes.filter((n) => n.type === 'treatment'); + const outcomes = nodes.filter((n) => n.type === 'outcome'); + const confounders = nodes.filter((n) => n.type === 'confounder'); + const mediators = nodes.filter((n) => n.type === 'mediator'); + const instruments = nodes.filter((n) => n.type === 'instrument'); + + // Place treatments on the left + treatments.forEach((n, i) => { + positions[n.id] = { x: 80, y: 180 + i * 80 }; + }); + + // Place outcomes on the right + outcomes.forEach((n, i) => { + positions[n.id] = { x: SVG_WIDTH - 80 - NODE_WIDTH, y: 180 + i * 80 }; + }); + + // Place confounders on top + const confStart = (SVG_WIDTH - confounders.length * 140) / 2; + confounders.forEach((n, i) => { + positions[n.id] = { x: confStart + i * 140, y: 40 }; + }); + + // Place mediators in the middle + mediators.forEach((n, i) => { + positions[n.id] = { x: SVG_WIDTH / 2 - NODE_WIDTH / 2, y: 180 + i * 80 }; + }); + + // Place instruments to the far left + instruments.forEach((n, i) => { + positions[n.id] = { x: 10, y: 100 + i * 80 }; + }); + + return positions; +} + +function Arrow({ x1, y1, x2, y2, dashed, color }: { + x1: number; y1: number; x2: number; y2: number; dashed: boolean; color: string; +}) { + const dx = x2 - x1; + const dy = y2 - y1; + const len = Math.sqrt(dx * dx + dy * dy); + if (len < 1) return null; + + // Shorten to not overlap with node + const shorten = 20; + const ratio1 = shorten / len; + const ratio2 = (len - shorten) / len; + const sx = x1 + dx * ratio1; + const sy = y1 + dy * ratio1; + const ex = x1 + dx * ratio2; + const ey = y1 + dy * ratio2; + + // Arrowhead + const headLen = 8; + const angle = Math.atan2(ey - sy, ex - sx); + const ax1 = ex - headLen * Math.cos(angle - Math.PI / 6); + const ay1 = ey - headLen * Math.sin(angle - Math.PI / 6); + const ax2 = ex - headLen * Math.cos(angle + Math.PI / 6); + const ay2 = ey - headLen * Math.sin(angle + Math.PI / 6); + + return ( + + + + + ); +} + +export default function CausalDiagram({ data, title }: CausalDiagramProps) { + const positions = layoutNodes(data.nodes); + + return ( +
+

+ {title} +

+ + + + {/* Edges */} + {data.edges.map((edge, i) => { + const src = positions[edge.source]; + const tgt = positions[edge.target]; + if (!src || !tgt) return null; + const color = edge.type === 'association' ? VIZ_COLORS.warning : VIZ_COLORS.text; + return ( + + ); + })} + + {/* Nodes */} + {data.nodes.map((node) => { + const pos = positions[node.id]; + if (!pos) return null; + const color = NODE_COLORS[node.type] || VIZ_COLORS.neutral; + const isRound = node.type === 'confounder' || node.type === 'instrument'; + + return ( + + {isRound ? ( + + ) : ( + + )} + + {node.label.length > 14 ? node.label.slice(0, 14) + '...' : node.label} + + + ); + })} + + + {/* Legend */} +
+ -- Treatment + -- Outcome + -- Confounder + -- Causal   - - Association +
+
+ ); +} diff --git a/console/src/features/visualization/charts/RDDPlot.tsx b/console/src/features/visualization/charts/RDDPlot.tsx new file mode 100644 index 0000000..58d3234 --- /dev/null +++ b/console/src/features/visualization/charts/RDDPlot.tsx @@ -0,0 +1,187 @@ +import { useCallback } from 'react'; +import { Group } from '@visx/group'; +import { scaleLinear } from '@visx/scale'; +import { AxisBottom, AxisLeft } from '@visx/axis'; +import { LinePath, Line } from '@visx/shape'; +import { curveLinear } from '@visx/curve'; +import { useTooltip, TooltipWithBounds, defaultStyles } from '@visx/tooltip'; +import { ParentSize } from '@visx/responsive'; +import { VIZ_COLORS } from '../theme'; + +interface Observation { + x: number; + y: number; + side: 'left' | 'right'; +} + +interface FittedPoint { + x: number; + y: number; +} + +interface RDDPlotProps { + data: { + observations: Observation[]; + cutoff: number; + fitted_left: FittedPoint[]; + fitted_right: FittedPoint[]; + }; + title: string; +} + +const MARGIN = { top: 20, right: 30, bottom: 50, left: 60 }; + +const tooltipStyles = { + ...defaultStyles, + background: VIZ_COLORS.surface, + color: VIZ_COLORS.text, + border: `1px solid ${VIZ_COLORS.grid}`, + fontFamily: 'monospace', + fontSize: 11, + padding: '6px 10px', +}; + +function RDDPlotInner({ width, data }: { width: number; data: RDDPlotProps['data'] }) { + const { showTooltip, hideTooltip, tooltipData, tooltipLeft, tooltipTop, tooltipOpen } = + useTooltip(); + + const height = 350; + const innerWidth = width - MARGIN.left - MARGIN.right; + const innerHeight = height - MARGIN.top - MARGIN.bottom; + + const allX = data.observations.map((p) => p.x); + const allY = data.observations.map((p) => p.y); + const xMin = Math.min(...allX); + const xMax = Math.max(...allX); + const yMin = Math.min(...allY); + const yMax = Math.max(...allY); + const xPad = (xMax - xMin) * 0.05 || 1; + const yPad = (yMax - yMin) * 0.1 || 1; + + const xScale = scaleLinear({ + domain: [xMin - xPad, xMax + xPad], + range: [0, innerWidth], + }); + + const yScale = scaleLinear({ + domain: [yMin - yPad, yMax + yPad], + range: [innerHeight, 0], + }); + + const handleMouseOver = useCallback( + (point: Observation, left: number, top: number) => { + showTooltip({ tooltipData: point, tooltipLeft: left, tooltipTop: top }); + }, + [showTooltip], + ); + + return ( +
+ + + {/* Fitted lines */} + {data.fitted_left.length > 0 && ( + xScale(d.x)} + y={(d) => yScale(d.y)} + curve={curveLinear} + stroke={VIZ_COLORS.secondary} + strokeWidth={2} + /> + )} + {data.fitted_right.length > 0 && ( + xScale(d.x)} + y={(d) => yScale(d.y)} + curve={curveLinear} + stroke={VIZ_COLORS.primary} + strokeWidth={2} + /> + )} + + {/* Cutoff line */} + + + {/* Observations */} + {data.observations.map((point, i) => ( + + handleMouseOver( + point, + xScale(point.x) + MARGIN.left, + yScale(point.y) + MARGIN.top, + ) + } + onMouseLeave={hideTooltip} + /> + ))} + + + + + + {tooltipOpen && tooltipData && ( + + X: {tooltipData.x.toFixed(2)} +
+ Y: {tooltipData.y.toFixed(3)} +
+ Side: {tooltipData.side} + + )} +
+ ); +} + +export default function RDDPlot({ data, title }: RDDPlotProps) { + return ( +
+

+ {title} +

+ + {({ width }) => width > 0 && } + +
+ ); +} diff --git a/docs/architecture.md b/docs/architecture.md index ef60017..fc0c413 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -19,7 +19,7 @@ Searches and manages scientific references. Integrates with Semantic Scholar API Cheminformatics operations: molecular descriptors, fingerprints, 3D conformer generation, substructure matching, and 2D SVG depiction. All RDKit usage is isolated in the infrastructure adapter (`rdkit_adapter.py`). ### Analysis -Dataset exploration and statistical analysis. Loads bioactivity data from ChEMBL, compound search via PubChem, curated pharmacology via GtoPdb, substructure enrichment analysis, property distributions. +Dataset exploration, statistical analysis, and domain-agnostic causal inference. Loads bioactivity data from ChEMBL, compound search via PubChem, curated pharmacology via GtoPdb, substructure enrichment analysis, property distributions. Causal inference methods (DiD, PSM, RDD, Synthetic Control) with threat assessment and cost-effectiveness analysis -- usable by any domain, not just impact evaluation. ### Prediction Machine learning for activity/outcome prediction. Supports XGBoost models with Morgan fingerprints (all domains) and Chemprop D-MPNN (molecular only). Ensemble predictions combine multiple models. @@ -34,7 +34,7 @@ Exercise physiology and sports medicine research: evidence-based training analys Nutrition science research: supplement evidence analysis, supplement label lookup (NIH DSLD), nutrient data (USDA FoodData), supplement safety monitoring (OpenFDA CAERS), drug interaction screening (RxNav), DRI-based nutrient adequacy assessment, nutrient ratio analysis, and inflammatory index scoring. Uses Semantic Scholar for literature search. ### Impact -Social program evaluation and causal analysis. Foundation layer provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities (`EconomicIndicator`, `BenchmarkValue`, `ProgramComparison`, `ProgramMetric`), repository ABCs (`EconomicDataRepository`, `BenchmarkRepository`), and infrastructure clients for three external APIs. Future phases will add causal inference methods (DiD, PSM, RDD, Synthetic Control), Mexico/US-specific data sources, and document upload. See `docs/impact-evaluation-domain.md` for full design. +Social program evaluation. Provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities, repository ABCs, and infrastructure clients for three external APIs. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools. Future phases will add Mexico/US-specific data sources and document upload. See `docs/impact-evaluation-domain.md` for full design. ### Investigation Hypothesis-driven agent orchestration. Manages the Claude-driven research loop: literature survey, hypothesis formulation (with predictions, criteria, scope), parallel experiment execution, criteria-based evaluation, negative controls, and synthesis. Uses multi-model architecture (Director/Researcher/Summarizer) with user-guided steering, domain classification, and multi-investigation memory. Includes domain configuration system (`DomainConfig` + `DomainRegistry`) for pluggable scientific domains with tool tagging, score definitions, prompt adaptation, and visualization control. Each domain config includes `tool_examples` in `experiment_examples` with realistic tool chaining patterns for complete tool coverage. Optional MCP bridge (`MCPBridge`) connects to external MCP servers for extensibility (e.g. Excalidraw for visual summaries). @@ -47,7 +47,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) │ NO tool access, structured JSON responses only │ - ├── Sonnet 4.5 (Researcher) -- Executes experiments with 78 domain-filtered tools (10-20 calls, parallel x2) + ├── Sonnet 4.5 (Researcher) -- Executes experiments with 84 domain-filtered tools (10-20 calls, parallel x2) │ Tool-calling loop with max_iterations_per_experiment guard │ └── Haiku 4.5 (Summarizer) -- Compresses large tool outputs >2000 chars, PICO+classification, evidence grading @@ -140,7 +140,7 @@ Invalid SMILES on `/depict` returns a dark error SVG (200 status). Invalid SMILE ## Domain-Specific Visualization -12 visualization tools produce structured `VisualizationPayload` JSON (viz_type, title, data, config, domain). The orchestrator intercepts viz tool results via `_maybe_viz_event()` and emits a `VisualizationRendered` SSE event. On the frontend, `VizRegistry` maps each `viz_type` to a lazy-loaded React component rendered in the `VisualizationPanel` grid. +17 visualization tools produce structured `VisualizationPayload` JSON (viz_type, title, data, config, domain). The orchestrator intercepts viz tool results via `_maybe_viz_event()` and emits a `VisualizationRendered` SSE event. On the frontend, `VizRegistry` maps each `viz_type` to a lazy-loaded React component rendered in the `VisualizationPanel` grid. | Tool | Chart Library | Purpose | |------|--------------|---------| @@ -156,6 +156,11 @@ Invalid SMILES on `/depict` returns a dark error SVG (200 status). Invalid SMILE | `render_nutrient_comparison` | Recharts BarChart | Grouped nutrient profile comparison | | `render_nutrient_adequacy` | Recharts BarChart | DRI adequacy assessment with MAR score | | `render_therapeutic_window` | Visx | Therapeutic window with EAR/RDA/AI/UL zones | +| `render_program_dashboard` | Recharts BarChart | Multi-indicator KPI dashboard with target tracking | +| `render_geographic_comparison` | Recharts BarChart | Region comparison with benchmark reference line | +| `render_parallel_trends` | Recharts ComposedChart | DiD parallel trends (treatment vs control) | +| `render_rdd_plot` | Visx | Regression discontinuity scatter with cutoff | +| `render_causal_diagram` | Visx | DAG showing treatment, outcome, confounders | Chart theming uses OKLCH color tokens consistent with the application's visual identity. diff --git a/docs/impact-evaluation-domain.md b/docs/impact-evaluation-domain.md index 55f3e57..f0c1590 100644 --- a/docs/impact-evaluation-domain.md +++ b/docs/impact-evaluation-domain.md @@ -91,7 +91,9 @@ Ehrlich's **Impact Evaluation** domain enables hypothesis-driven causal analysis - WWC Standards v5.0: 4-tier evidence hierarchy (Strong, Moderate, Promising, Rationale) - GAO evaluation standards: transparent documentation and public reporting -### Causal Inference Methods (Tools to Build) +### Causal Inference Methods (Implemented in `analysis/` -- Domain-Agnostic) + +> **Note:** All causal inference estimators live in `analysis/` (not `impact/`) as domain-agnostic tools. This means DiD, PSM, RDD, and Synthetic Control can be used by any domain, not just Impact Evaluation. The impact domain configures `"causal"` in its `tool_tags` to make them available during impact investigations. | Method | When Used | Key Assumptions | Evidence Strength | |--------|-----------|-----------------|-------------------| @@ -114,33 +116,50 @@ Extend existing GRADE implementation to social program context: ## 4. Server Architecture -### New Bounded Context: `impact/` +### Bounded Context: `impact/` (economic data + benchmarking) ``` server/src/ehrlich/impact/ domain/ - entities.py # Program, Indicator, Beneficiary, CausalEstimate, ThreatToValidity - repository.py # ProgramDataRepository, IndicatorRepository ABCs - ports.py # CausalEstimator, DataHarmonizer, CostAnalyzer ABCs - dri.py # (n/a -- equivalent would be evaluation_standards.py) - evaluation_standards.py # CONEVAL ECR template, WWC tiers, GRADE criteria + entities.py # Program, Indicator, Benchmark, EconomicSeries, HealthIndicator + repository.py # EconomicDataRepository, HealthDataRepository, DevelopmentDataRepository ABCs application/ - impact_service.py # Core service: causal estimation, evidence grading, benchmarking + impact_service.py # Economic indicators, benchmarks, program comparison infrastructure/ - inegi_client.py # INEGI Indicadores + DENUE APIs - datos_gob_client.py # datos.gob.mx CKAN API - transparencia_client.py # Transparencia Presupuestaria API - banxico_client.py # Banxico SIE API - census_client.py # US Census Bureau API - bls_client.py # Bureau of Labor Statistics API - usaspending_client.py # USAspending.gov API worldbank_client.py # World Bank API who_client.py # WHO GHO API - fred_client.py # FRED API (already used in other contexts? no -- new) - data_harmonizer.py # Merge datasets by geography/time period - csv_parser.py # Parse user-uploaded CSV/Excel files - pdf_extractor.py # Extract text from uploaded PDFs - tools.py # Impact evaluation tools + fred_client.py # FRED API + tools.py # 3 impact evaluation tools +``` + +### Causal Inference in `analysis/` (domain-agnostic) + +``` +server/src/ehrlich/analysis/ + domain/ + causal.py # ThreatToValidity, CausalEstimate, CostEffectivenessResult + causal_ports.py # DiDEstimatorPort, PSMEstimatorPort, RDDEstimatorPort, SyntheticControlPort + evidence_standards.py # WWC tiers, CONEVAL MIR levels, CREMAA criteria + application/ + causal_service.py # DiD, PSM, RDD, SC estimation + threat assessment + cost-effectiveness + infrastructure/ + did_estimator.py # Difference-in-differences estimator + psm_estimator.py # Propensity score matching estimator + rdd_estimator.py # Regression discontinuity estimator + synthetic_control_estimator.py # Synthetic control estimator +``` + +### Planned additions (future phases) + +``` +server/src/ehrlich/impact/infrastructure/ + inegi_client.py # INEGI Indicadores + DENUE APIs (Phase 13C) + datos_gob_client.py # datos.gob.mx CKAN API (Phase 13C) + transparencia_client.py # Transparencia Presupuestaria API (Phase 13C) + banxico_client.py # Banxico SIE API (Phase 13C) + census_client.py # US Census Bureau API (Phase 13D) + bls_client.py # Bureau of Labor Statistics API (Phase 13D) + usaspending_client.py # USAspending.gov API (Phase 13D) ``` ### Domain Entities @@ -462,11 +481,13 @@ File upload - Basic `compare_programs` tool using existing `run_statistical_test` - 2 new viz tools: `render_program_dashboard`, `render_geographic_comparison` -### Phase 2: Causal Inference Engine -- `estimate_did`, `estimate_psm`, `estimate_rdd` tools -- `assess_threats` automated validity assessment -- `compute_cost_effectiveness` tool -- 3 new viz tools: `render_parallel_trends`, `render_rdd_plot`, `render_causal_diagram` +### Phase 2: Causal Inference Engine -- DONE (lives in `analysis/`) +- `estimate_did`, `estimate_psm`, `estimate_rdd`, `estimate_synthetic_control` tools (in `analysis/tools.py`) +- `assess_threats` automated validity assessment (in `analysis/tools.py`) +- `compute_cost_effectiveness` tool (in `analysis/tools.py`) +- 5 viz tools: `render_parallel_trends`, `render_program_dashboard`, `render_geographic_comparison` (13B partial), `render_rdd_plot`, `render_causal_diagram` (13B full) +- `ThreatAssessment.tsx` console component with severity badges +- Domain entities in `analysis/domain/causal.py`, ports in `analysis/domain/causal_ports.py`, standards in `analysis/domain/evidence_standards.py` ### Phase 3: Mexico Integration - INEGI Indicadores + DENUE clients diff --git a/docs/roadmap.md b/docs/roadmap.md index cd97ad4..7555732 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -197,7 +197,7 @@ The core: Claude as an autonomous scientist. - [x] Tests: mock API, verify request/response handling ### 5B. Tool Registry -- [x] Register all tools from all contexts (6 chemistry, 3 literature, 6 analysis, 3 prediction, 7 simulation, 11 training, 10 nutrition, 7 investigation control -- 38 at time of Phase 5; now 73 with API tools + visualization tools + statistics tools + generic ML tools + impact tools) +- [x] Register all tools from all contexts (6 chemistry, 3 literature, 12 analysis, 3 prediction, 7 simulation, 11 training, 10 nutrition, 7 investigation control -- 38 at time of Phase 5; now 79 with API tools + visualization tools + statistics tools + generic ML tools + impact tools + causal tools) - [x] Auto-generate JSON Schema from Python type hints + docstrings - [x] `get(name)` -> callable, `list_tools()` -> all registered tools, `list_schemas()` -> Anthropic-compatible schemas - [x] Schema format matches Anthropic tool_use specification @@ -471,7 +471,7 @@ Enhancement Enhancement Sports | Phase 13: Impact Evaluation Domain -- IN PROGRESS (Causal inference + Document upload + MX/US APIs) - 13A (Foundation) DONE -> 13B partial (DiD+Viz) DONE -> 13A-2 (Upload) -> 13B remaining (PSM/RDD) -> 13C (Mexico) -> 13D (US) -> 13E (MCP) + 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) -> 13C (Mexico) -> 13D (US) -> 13E (MCP) | Demo + Video -- TODO | @@ -993,7 +993,7 @@ Deepened the nutrition bounded context with 1 new data source, 6 new tools, and ## Phase 13: Impact Evaluation Domain -- IN PROGRESS -New bounded context for hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. See `docs/impact-evaluation-domain.md` for full design. +New bounded context for hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools usable by any domain. No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. See `docs/impact-evaluation-domain.md` for full design. ### Market Gap @@ -1015,7 +1015,7 @@ Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evalu ### Phase 13B (partial): Causal Inference + Impact Viz -- DONE -- [x] `CausalEstimator` port ABC in `domain/ports.py` +- [x] `CausalEstimator` port ABC in `impact/domain/ports.py` (initial placement) - [x] `DiDEstimator` infrastructure with parallel trends test and automated threat assessment - [x] `estimate_did` tool -- difference-in-differences with effect size, SE, p-value, threats, evidence tier - [x] `assess_threats` tool -- knowledge-based validity threat assessment (DiD, PSM, RDD, RCT, IV) @@ -1028,6 +1028,32 @@ Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evalu **Counts:** 73 -> 78 tools, 12 -> 15 viz tools, 3 -> 5 impact tools. +### Phase 13B: Domain-Agnostic Causal Inference -- DONE + +Refactored causal inference from impact-specific to domain-agnostic. All causal estimation methods now live in `analysis/` (alongside statistics), usable by any domain. + +- [x] Moved `ThreatToValidity`, `CausalEstimate` entities from `impact/domain/` to `analysis/domain/causal.py` +- [x] Moved `evaluation_standards.py` (WWC tiers, CONEVAL, CREMAA) from `impact/domain/` to `analysis/domain/evidence_standards.py` +- [x] Created `analysis/domain/causal_ports.py` -- 4 estimator port ABCs: `DiDEstimatorPort`, `PSMEstimatorPort`, `RDDEstimatorPort`, `SyntheticControlPort` +- [x] Moved `DiDEstimator` from `impact/infrastructure/` to `analysis/infrastructure/did_estimator.py` +- [x] New `analysis/infrastructure/psm_estimator.py` -- propensity score matching with balance diagnostics +- [x] New `analysis/infrastructure/rdd_estimator.py` -- regression discontinuity (sharp/fuzzy) +- [x] New `analysis/infrastructure/synthetic_control_estimator.py` -- synthetic control method +- [x] New `analysis/application/causal_service.py` -- orchestrates all 4 estimators + threat assessment + cost-effectiveness +- [x] New `compute_cost_effectiveness` tool -- cost per unit outcome, ICER +- [x] Deleted `impact/domain/ports.py` -- causal estimator port moved to analysis/ +- [x] Trimmed `ImpactService` -- removed DiD/threat methods (delegated to CausalService) +- [x] Impact tools reduced from 5 to 3 (estimate_did + assess_threats moved to analysis/) +- [x] 6 causal tools tagged `"causal"` in analysis/tools.py +- [x] New `render_rdd_plot` viz tool -- regression discontinuity scatter with cutoff (Visx) +- [x] New `render_causal_diagram` viz tool -- DAG showing treatment/outcome/confounders (Visx) +- [x] Console: `RDDPlot.tsx`, `CausalDiagram.tsx` chart components +- [x] Console: `ThreatAssessment.tsx` panel with severity badges +- [x] Console: 2 new template cards (Education Policy, Health Policy) -- 9 total +- [x] VizRegistry updated with 2 new viz types + +**Counts:** 78 -> 84 tools (6 causal + 1 cost-effectiveness - 2 moved + 2 viz), 15 -> 17 viz tools, 5 -> 3 impact tools, 0 -> 6 causal tools (analysis/), 7 -> 9 templates. + ### Phase 13A-2: Document Upload (TODO -- deferred post-hackathon) - [ ] File upload API (CSV, PDF, Excel) via multipart/form-data on `POST /investigate` @@ -1038,16 +1064,6 @@ Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evalu - [ ] Frontend: `FileUpload.tsx` drag-and-drop component in `PromptInput` - [ ] Frontend: `DataPreview.tsx` for uploaded dataset preview -### Phase 13B (remaining): Additional Causal Methods (TODO -- deferred post-hackathon) - -- [ ] `estimate_psm` tool -- propensity score matching with balance diagnostics -- [ ] `estimate_rdd` tool -- regression discontinuity (sharp/fuzzy) -- [ ] `estimate_synthetic_control` tool -- synthetic control method -- [ ] `compute_cost_effectiveness` tool -- cost per unit outcome, ICER -- [ ] `render_rdd_plot` viz tool -- regression discontinuity scatter -- [ ] `render_causal_diagram` viz tool -- DAG showing treatment/outcome/confounders -- [ ] Frontend: `ThreatAssessment.tsx` panel with severity badges - ### Phase 13C: Mexico Integration - [ ] INEGI Indicadores API client (`https://www.inegi.org.mx/app/api/indicadores/`) -- census, demographics @@ -1202,7 +1218,7 @@ Side-by-side comparison of investigation runs for reproducibility and consensus ## ~~MCP Server~~ -- REJECTED -Exposing Ehrlich's 73 tools as an MCP server was considered and rejected. The tools alone (ChEMBL queries, RDKit computations, etc.) are commodity API wrappers -- the value is the multi-model orchestration (Director-Worker-Summarizer), hypothesis-driven methodology, and parallel experiment execution. An MCP client consuming Ehrlich's tools would lose the scientific rigor that the orchestration guarantees. +Exposing Ehrlich's 84 tools as an MCP server was considered and rejected. The tools alone (ChEMBL queries, RDKit computations, etc.) are commodity API wrappers -- the value is the multi-model orchestration (Director-Worker-Summarizer), hypothesis-driven methodology, and parallel experiment execution. An MCP client consuming Ehrlich's tools would lose the scientific rigor that the orchestration guarantees. Ehrlich remains an MCP **consumer** via `MCPBridge` (connecting to external servers like Excalidraw), which adds value by extending the Researcher's toolkit. diff --git a/server/src/ehrlich/analysis/application/causal_service.py b/server/src/ehrlich/analysis/application/causal_service.py new file mode 100644 index 0000000..6de9d74 --- /dev/null +++ b/server/src/ehrlich/analysis/application/causal_service.py @@ -0,0 +1,207 @@ +"""Domain-agnostic causal inference service. + +Follows the same ports-and-adapters pattern as PredictionService: +estimators are injected via ports, not hardcoded. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ehrlich.analysis.domain.causal import CostEffectivenessResult, ThreatToValidity + +if TYPE_CHECKING: + from ehrlich.analysis.domain.causal import CausalEstimate + from ehrlich.analysis.domain.causal_ports import ( + DiDEstimatorPort, + PSMEstimatorPort, + RDDEstimatorPort, + SyntheticControlPort, + ) + + +class CausalService: + """Domain-agnostic causal inference service.""" + + def estimate_did( + self, + treatment_pre: list[float], + treatment_post: list[float], + control_pre: list[float], + control_post: list[float], + *, + estimator: DiDEstimatorPort, + ) -> CausalEstimate: + return estimator.estimate( + treatment_pre, + treatment_post, + control_pre, + control_post, + ) + + def estimate_psm( + self, + treated_outcomes: list[float], + control_outcomes: list[float], + treated_covariates: list[list[float]], + control_covariates: list[list[float]], + *, + estimator: PSMEstimatorPort, + ) -> CausalEstimate: + return estimator.estimate( + treated_outcomes, + control_outcomes, + treated_covariates, + control_covariates, + ) + + def estimate_rdd( + self, + running_variable: list[float], + outcome: list[float], + cutoff: float, + *, + estimator: RDDEstimatorPort, + bandwidth: float | None = None, + design: str = "sharp", + ) -> CausalEstimate: + return estimator.estimate( + running_variable, + outcome, + cutoff, + bandwidth, + design, + ) + + def estimate_synthetic_control( + self, + treated_series: list[float], + donor_matrix: list[list[float]], + treatment_period: int, + *, + estimator: SyntheticControlPort, + ) -> CausalEstimate: + return estimator.estimate( + treated_series, + donor_matrix, + treatment_period, + ) + + def assess_threats( + self, + method: str, + sample_sizes: dict[str, int], + parallel_trends_p: float | None = None, + effect_size: float | None = None, + ) -> list[ThreatToValidity]: + """Knowledge-based threat assessment for any causal method.""" + threats: list[ThreatToValidity] = [] + method_lower = method.lower().replace("-", "_").replace(" ", "_") + + # Small sample threats + min_n = min(sample_sizes.values()) if sample_sizes else 0 + if min_n < 5: + threats.append( + ThreatToValidity( + type="small_sample", + severity="high", + description=( + f"Minimum group size is {min_n}. Insufficient for reliable inference." + ), + mitigation="Increase sample size or use exact/permutation tests.", + ) + ) + elif min_n < 30: + threats.append( + ThreatToValidity( + type="small_sample", + severity="medium", + description=f"Minimum group size is {min_n}. May limit statistical power.", + mitigation="Report power analysis and consider non-parametric tests.", + ) + ) + + # Method-specific threats + is_did = method_lower in ("did", "difference_in_differences") + if is_did and parallel_trends_p is not None and parallel_trends_p < 0.05: + threats.append( + ThreatToValidity( + type="parallel_trends_violation", + severity="high", + description=f"Parallel trends assumption violated (p={parallel_trends_p:.4f}).", + mitigation="Use event study or synthetic control method.", + ) + ) + + if method_lower in ("psm", "propensity_score_matching"): + threats.append( + ThreatToValidity( + type="unobserved_confounders", + severity="medium", + description="PSM cannot account for unobserved confounders.", + mitigation="Combine with sensitivity analysis (Rosenbaum bounds).", + ) + ) + + if method_lower in ("rdd", "regression_discontinuity"): + threats.append( + ThreatToValidity( + type="manipulation", + severity="medium", + description="Running variable may be subject to manipulation near cutoff.", + mitigation="Test for bunching at the cutoff (McCrary density test).", + ) + ) + + if method_lower in ("synthetic_control", "sc"): + threats.append( + ThreatToValidity( + type="convex_hull", + severity="medium", + description="Treated unit must lie in convex hull of donors.", + mitigation="Check pre-treatment fit quality (RMSPE).", + ) + ) + + # Effect size plausibility + if effect_size is not None and abs(effect_size) > 2.0: + threats.append( + ThreatToValidity( + type="implausible_effect", + severity="medium", + description=f"Effect size ({effect_size:.2f}) is unusually large.", + mitigation="Verify data quality and check for measurement errors.", + ) + ) + + return threats + + @staticmethod + def compute_cost_effectiveness( + program_name: str, + total_cost: float, + total_effect: float, + currency: str = "USD", + effect_unit: str = "units", + comparison_cost: float | None = None, + comparison_effect: float | None = None, + ) -> CostEffectivenessResult: + """Compute cost-effectiveness ratio and optional ICER.""" + cost_per_unit = total_cost / total_effect if total_effect != 0 else 0.0 + + icer: float | None = None + if comparison_cost is not None and comparison_effect is not None: + delta_cost = total_cost - comparison_cost + delta_effect = total_effect - comparison_effect + if delta_effect != 0: + icer = delta_cost / delta_effect + + return CostEffectivenessResult( + program_name=program_name, + total_cost=round(total_cost, 2), + total_effect=round(total_effect, 4), + cost_per_unit=round(cost_per_unit, 2), + currency=currency, + effect_unit=effect_unit, + icer=round(icer, 2) if icer is not None else None, + ) diff --git a/server/src/ehrlich/analysis/domain/causal.py b/server/src/ehrlich/analysis/domain/causal.py new file mode 100644 index 0000000..fadd97b --- /dev/null +++ b/server/src/ehrlich/analysis/domain/causal.py @@ -0,0 +1,39 @@ +"""Domain entities for causal inference.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ThreatToValidity: + type: str + severity: str + description: str + mitigation: str + + +@dataclass(frozen=True) +class CausalEstimate: + method: str + effect_size: float + standard_error: float + confidence_interval: tuple[float, float] + p_value: float + n_treatment: int + n_control: int + covariates: tuple[str, ...] + assumptions: tuple[str, ...] + threats: tuple[ThreatToValidity, ...] + evidence_tier: str + + +@dataclass(frozen=True) +class CostEffectivenessResult: + program_name: str + total_cost: float + total_effect: float + cost_per_unit: float + currency: str + effect_unit: str + icer: float | None diff --git a/server/src/ehrlich/analysis/domain/causal_ports.py b/server/src/ehrlich/analysis/domain/causal_ports.py new file mode 100644 index 0000000..47a385a --- /dev/null +++ b/server/src/ehrlich/analysis/domain/causal_ports.py @@ -0,0 +1,64 @@ +"""Ports (ABCs) for causal inference estimators. + +ISP: one port per method, following prediction/domain/ports.py pattern. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ehrlich.analysis.domain.causal import CausalEstimate + + +class DiDEstimatorPort(ABC): + """Difference-in-differences estimator.""" + + @abstractmethod + def estimate( + self, + treatment_pre: list[float], + treatment_post: list[float], + control_pre: list[float], + control_post: list[float], + ) -> CausalEstimate: ... + + +class PSMEstimatorPort(ABC): + """Propensity score matching estimator.""" + + @abstractmethod + def estimate( + self, + treated_outcomes: list[float], + control_outcomes: list[float], + treated_covariates: list[list[float]], + control_covariates: list[list[float]], + ) -> CausalEstimate: ... + + +class RDDEstimatorPort(ABC): + """Regression discontinuity design estimator.""" + + @abstractmethod + def estimate( + self, + running_variable: list[float], + outcome: list[float], + cutoff: float, + bandwidth: float | None = None, + design: str = "sharp", + ) -> CausalEstimate: ... + + +class SyntheticControlPort(ABC): + """Synthetic control method estimator.""" + + @abstractmethod + def estimate( + self, + treated_series: list[float], + donor_matrix: list[list[float]], + treatment_period: int, + ) -> CausalEstimate: ... diff --git a/server/src/ehrlich/analysis/domain/evidence_standards.py b/server/src/ehrlich/analysis/domain/evidence_standards.py new file mode 100644 index 0000000..814825a --- /dev/null +++ b/server/src/ehrlich/analysis/domain/evidence_standards.py @@ -0,0 +1,69 @@ +"""Evidence grading standards for causal inference. + +Based on What Works Clearinghouse (WWC) evidence tiers. +Domain-agnostic -- usable by any bounded context. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ehrlich.analysis.domain.causal import ThreatToValidity + +WWC_TIERS: dict[str, str] = { + "strong": ( + "Well-designed RCT or quasi-experimental study with no " + "serious threats to validity (low attrition, verified equivalence)" + ), + "moderate": ( + "Quasi-experimental design with some threats to validity " + "that are partially addressed (e.g., PSM with sensitivity analysis)" + ), + "promising": ( + "Correlational study with statistical controls or pre-post " + "comparison with a plausible comparison group" + ), + "rationale": ("Theory-based or descriptive evidence only; no causal identification strategy"), +} + + +def classify_evidence_tier(method: str, threats: list[ThreatToValidity]) -> str: + """Classify evidence strength based on method and threats. + + Returns one of: strong, moderate, promising, rationale. + """ + strong_methods = {"rct", "randomized_controlled_trial"} + moderate_methods = { + "did", + "difference_in_differences", + "psm", + "propensity_score_matching", + "rdd", + "regression_discontinuity", + "synthetic_control", + } + promising_methods = {"iv", "instrumental_variables", "fixed_effects", "panel_regression"} + + method_lower = method.lower().replace("-", "_").replace(" ", "_") + + severe_threats = [t for t in threats if t.severity == "high"] + + if method_lower in strong_methods: + if len(severe_threats) >= 2: + return "moderate" + if len(severe_threats) == 1: + return "moderate" + return "strong" + + if method_lower in moderate_methods: + if len(severe_threats) >= 2: + return "promising" + return "moderate" + + if method_lower in promising_methods: + if len(severe_threats) >= 1: + return "rationale" + return "promising" + + return "rationale" diff --git a/server/src/ehrlich/impact/infrastructure/did_estimator.py b/server/src/ehrlich/analysis/infrastructure/did_estimator.py similarity index 53% rename from server/src/ehrlich/impact/infrastructure/did_estimator.py rename to server/src/ehrlich/analysis/infrastructure/did_estimator.py index dc194c1..d054bdd 100644 --- a/server/src/ehrlich/impact/infrastructure/did_estimator.py +++ b/server/src/ehrlich/analysis/infrastructure/did_estimator.py @@ -9,9 +9,9 @@ from scipy import stats -from ehrlich.impact.domain.entities import CausalEstimate, ThreatToValidity -from ehrlich.impact.domain.evaluation_standards import classify_evidence_tier -from ehrlich.impact.domain.ports import CausalEstimator +from ehrlich.analysis.domain.causal import CausalEstimate, ThreatToValidity +from ehrlich.analysis.domain.causal_ports import DiDEstimatorPort +from ehrlich.analysis.domain.evidence_standards import classify_evidence_tier def _mean(values: list[float]) -> float: @@ -25,10 +25,10 @@ def _variance(values: list[float]) -> float: return sum((x - m) ** 2 for x in values) / (len(values) - 1) -class DiDEstimator(CausalEstimator): +class DiDEstimator(DiDEstimatorPort): """Difference-in-differences estimator with threat assessment.""" - def estimate_did( + def estimate( self, treatment_pre: list[float], treatment_post: list[float], @@ -50,12 +50,10 @@ def estimate_did( var_c_post = _variance(control_post) se_treatment = math.sqrt( - var_t_pre / max(len(treatment_pre), 1) - + var_t_post / max(len(treatment_post), 1) + var_t_pre / max(len(treatment_pre), 1) + var_t_post / max(len(treatment_post), 1) ) se_control = math.sqrt( - var_c_pre / max(len(control_pre), 1) - + var_c_post / max(len(control_post), 1) + var_c_pre / max(len(control_pre), 1) + var_c_post / max(len(control_post), 1) ) se_did = math.sqrt(se_treatment**2 + se_control**2) @@ -69,8 +67,10 @@ def estimate_did( # Cohen's d effect size pooled_sd = math.sqrt( - (var_t_post * max(len(treatment_post) - 1, 0) - + var_c_post * max(len(control_post) - 1, 0)) + ( + var_t_post * max(len(treatment_post) - 1, 0) + + var_c_post * max(len(control_post) - 1, 0) + ) / max(len(treatment_post) + len(control_post) - 2, 1) ) cohens_d = did_estimate / pooled_sd if pooled_sd > 0 else 0.0 @@ -82,9 +82,12 @@ def estimate_did( # Automated threat assessment threats = self._assess_threats( - treatment_pre, treatment_post, - control_pre, control_post, - p_value, cohens_d, + treatment_pre, + treatment_post, + control_pre, + control_post, + p_value, + cohens_d, ) evidence_tier = classify_evidence_tier("did", list(threats)) @@ -122,75 +125,87 @@ def _assess_threats( # 1. Parallel trends test: compare pre-treatment slopes pt_p = self._parallel_trends_p(treatment_pre, control_pre) if pt_p < 0.05: - threats.append(ThreatToValidity( - type="parallel_trends_violation", - severity="high", - description=( - f"Pre-treatment trends differ significantly (p={pt_p:.4f}). " - "DiD assumption of parallel trends may be violated." - ), - mitigation="Consider event study design or synthetic control method.", - )) + threats.append( + ThreatToValidity( + type="parallel_trends_violation", + severity="high", + description=( + f"Pre-treatment trends differ significantly (p={pt_p:.4f}). " + "DiD assumption of parallel trends may be violated." + ), + mitigation="Consider event study design or synthetic control method.", + ) + ) elif pt_p < 0.10: - threats.append(ThreatToValidity( - type="parallel_trends_weak", - severity="medium", - description=( - f"Pre-treatment trends show marginal difference (p={pt_p:.4f}). " - "Parallel trends assumption is weakly supported." - ), - mitigation="Report sensitivity analysis with alternative control groups.", - )) + threats.append( + ThreatToValidity( + type="parallel_trends_weak", + severity="medium", + description=( + f"Pre-treatment trends show marginal difference (p={pt_p:.4f}). " + "Parallel trends assumption is weakly supported." + ), + mitigation="Report sensitivity analysis with alternative control groups.", + ) + ) # 2. Small sample size min_n = min( - len(treatment_pre), len(treatment_post), - len(control_pre), len(control_post), + len(treatment_pre), + len(treatment_post), + len(control_pre), + len(control_post), ) if min_n < 5: - threats.append(ThreatToValidity( - type="small_sample", - severity="high", - description=( - f"Minimum group size is {min_n}. " - "Small samples reduce statistical power." - ), - mitigation="Increase sample size or use bootstrapped CIs.", - )) + threats.append( + ThreatToValidity( + type="small_sample", + severity="high", + description=( + f"Minimum group size is {min_n}. Small samples reduce statistical power." + ), + mitigation="Increase sample size or use bootstrapped CIs.", + ) + ) elif min_n < 10: - threats.append(ThreatToValidity( - type="small_sample", - severity="medium", - description=( - f"Minimum group size is {min_n}. " - "Moderate sample may limit generalizability." - ), - mitigation="Consider non-parametric alternatives.", - )) + threats.append( + ThreatToValidity( + type="small_sample", + severity="medium", + description=( + f"Minimum group size is {min_n}. " + "Moderate sample may limit generalizability." + ), + mitigation="Consider non-parametric alternatives.", + ) + ) # 3. Effect size plausibility if abs(effect_size) > 2.0: - threats.append(ThreatToValidity( - type="implausible_effect", - severity="medium", - description=( - f"Effect size of {effect_size:.2f} is unusually large " - "for social program evaluation." - ), - mitigation="Verify data quality and check for measurement errors.", - )) + threats.append( + ThreatToValidity( + type="implausible_effect", + severity="medium", + description=( + f"Effect size of {effect_size:.2f} is unusually large " + "for social program evaluation." + ), + mitigation="Verify data quality and check for measurement errors.", + ) + ) # 4. Borderline significance if 0.01 < p_value < 0.05: - threats.append(ThreatToValidity( - type="borderline_significance", - severity="low", - description=( - "Result is statistically significant " - f"but borderline (p={p_value:.4f})." - ), - mitigation="Pre-register hypothesis and consider multiple testing correction.", - )) + threats.append( + ThreatToValidity( + type="borderline_significance", + severity="low", + description=( + f"Result is statistically significant but borderline (p={p_value:.4f})." + ), + mitigation="Pre-register hypothesis and consider multiple testing correction.", + ) + ) return tuple(threats) @@ -205,6 +220,8 @@ def _parallel_trends_p( # Compare trends using Mann-Whitney U test on pre-treatment values _, p_value = stats.mannwhitneyu( - treatment_pre, control_pre, alternative="two-sided", + treatment_pre, + control_pre, + alternative="two-sided", ) return float(p_value) diff --git a/server/src/ehrlich/analysis/infrastructure/psm_estimator.py b/server/src/ehrlich/analysis/infrastructure/psm_estimator.py new file mode 100644 index 0000000..01e5c44 --- /dev/null +++ b/server/src/ehrlich/analysis/infrastructure/psm_estimator.py @@ -0,0 +1,263 @@ +"""Propensity Score Matching causal estimator. + +Pure scipy/numpy implementation -- no new dependencies. +""" + +from __future__ import annotations + +import math + +import numpy as np +from scipy import optimize + +from ehrlich.analysis.domain.causal import CausalEstimate, ThreatToValidity +from ehrlich.analysis.domain.causal_ports import PSMEstimatorPort +from ehrlich.analysis.domain.evidence_standards import classify_evidence_tier + + +def _logistic_log_likelihood( + beta: np.ndarray, + features: np.ndarray, + labels: np.ndarray, +) -> float: + """Negative log-likelihood for logistic regression.""" + z = features @ beta + z = np.clip(z, -500, 500) + ll = np.sum(labels * z - np.log1p(np.exp(z))) + return -float(ll) + + +def _propensity_scores( + treated_covariates: list[list[float]], + control_covariates: list[list[float]], +) -> tuple[np.ndarray, np.ndarray]: + """Compute propensity scores via logistic regression.""" + x_t = np.array(treated_covariates, dtype=np.float64) + x_c = np.array(control_covariates, dtype=np.float64) + + features = np.vstack([x_t, x_c]) + # Add intercept + features = np.column_stack([np.ones(len(features)), features]) + labels = np.concatenate([np.ones(len(x_t)), np.zeros(len(x_c))]) + + beta0 = np.zeros(features.shape[1]) + result = optimize.minimize( + _logistic_log_likelihood, + beta0, + args=(features, labels), + method="BFGS", + options={"maxiter": 200}, + ) + beta = result.x + + z = features @ beta + z = np.clip(z, -500, 500) + probs = 1.0 / (1.0 + np.exp(-z)) + + n_t = len(x_t) + return probs[:n_t], probs[n_t:] + + +class PSMEstimator(PSMEstimatorPort): + """Propensity score matching estimator with caliper matching.""" + + def estimate( + self, + treated_outcomes: list[float], + control_outcomes: list[float], + treated_covariates: list[list[float]], + control_covariates: list[list[float]], + ) -> CausalEstimate: + ps_treated, ps_control = _propensity_scores( + treated_covariates, + control_covariates, + ) + + # Caliper = 0.25 * SD of all propensity scores + all_ps = np.concatenate([ps_treated, ps_control]) + caliper = 0.25 * float(np.std(all_ps)) + if caliper < 1e-10: + caliper = 0.05 + + # Nearest-neighbor matching with caliper + y_t = np.array(treated_outcomes, dtype=np.float64) + y_c = np.array(control_outcomes, dtype=np.float64) + + matched_treated: list[float] = [] + matched_control: list[float] = [] + dropped = 0 + + for i, ps_t in enumerate(ps_treated): + distances = np.abs(ps_control - ps_t) + best_idx = int(np.argmin(distances)) + if distances[best_idx] <= caliper: + matched_treated.append(float(y_t[i])) + matched_control.append(float(y_c[best_idx])) + else: + dropped += 1 + + n_matched = len(matched_treated) + if n_matched < 2: + return CausalEstimate( + method="propensity_score_matching", + effect_size=0.0, + standard_error=0.0, + confidence_interval=(0.0, 0.0), + p_value=1.0, + n_treatment=len(treated_outcomes), + n_control=len(control_outcomes), + covariates=(), + assumptions=( + "Selection on observables", + "Common support", + "No unobserved confounders", + "Correct model specification", + ), + threats=( + ThreatToValidity( + type="unobserved_confounders", + severity="medium", + description="PSM cannot account for unobserved confounders.", + mitigation="Combine with sensitivity analysis (Rosenbaum bounds).", + ), + ThreatToValidity( + type="poor_overlap", + severity="high", + description="Fewer than 2 matched pairs found. No valid estimate.", + mitigation="Increase sample size or relax caliper.", + ), + ), + evidence_tier="rationale", + ) + + # ATT = mean(treated matched) - mean(control matched) + mt = np.array(matched_treated) + mc = np.array(matched_control) + att = float(np.mean(mt) - np.mean(mc)) + + # Standard error + diff = mt - mc + se = float(np.std(diff, ddof=1) / math.sqrt(n_matched)) + + # t-test + if se > 0: + t_stat = att / se + df = max(n_matched - 1, 1) + from scipy import stats + + p_value = float(2 * (1 - stats.t.cdf(abs(t_stat), df=df))) + else: + p_value = 1.0 + + ci_lower = att - 1.96 * se + ci_upper = att + 1.96 * se + + # Balance: SMD per covariate after matching + threats = self._assess_threats( + treated_covariates, + control_covariates, + ps_treated, + ps_control, + dropped, + len(treated_outcomes), + att, + ) + + evidence_tier = classify_evidence_tier("psm", list(threats)) + + return CausalEstimate( + method="propensity_score_matching", + effect_size=round(att, 4), + standard_error=round(se, 4), + confidence_interval=(round(ci_lower, 4), round(ci_upper, 4)), + p_value=round(p_value, 6), + n_treatment=n_matched, + n_control=n_matched, + covariates=(), + assumptions=( + "Selection on observables", + "Common support", + "No unobserved confounders", + "Correct model specification", + ), + threats=tuple(threats), + evidence_tier=evidence_tier, + ) + + @staticmethod + def _assess_threats( + treated_covariates: list[list[float]], + control_covariates: list[list[float]], + ps_treated: np.ndarray, + ps_control: np.ndarray, + dropped: int, + n_treated: int, + effect_size: float, + ) -> list[ThreatToValidity]: + threats: list[ThreatToValidity] = [] + + # Always: unobserved confounders (inherent PSM limitation) + threats.append( + ThreatToValidity( + type="unobserved_confounders", + severity="medium", + description="PSM cannot account for unobserved confounders.", + mitigation="Combine with sensitivity analysis (Rosenbaum bounds).", + ) + ) + + # Check covariate balance (SMD) + cov_t = np.array(treated_covariates, dtype=np.float64) + cov_c = np.array(control_covariates, dtype=np.float64) + n_covariates = cov_t.shape[1] if cov_t.ndim == 2 else 0 + imbalanced = 0 + for j in range(n_covariates): + mean_t = float(np.mean(cov_t[:, j])) + mean_c = float(np.mean(cov_c[:, j])) + pooled_sd = float( + np.sqrt((np.var(cov_t[:, j], ddof=1) + np.var(cov_c[:, j], ddof=1)) / 2) + ) + if pooled_sd > 0: + smd = abs(mean_t - mean_c) / pooled_sd + if smd > 0.1: + imbalanced += 1 + + if imbalanced > 0: + threats.append( + ThreatToValidity( + type="covariate_imbalance", + severity="medium" if imbalanced <= n_covariates / 2 else "high", + description=( + f"{imbalanced}/{n_covariates} covariates have SMD > 0.1 after matching." + ), + mitigation="Consider regression adjustment on matched sample.", + ) + ) + + # Poor overlap + drop_pct = dropped / max(n_treated, 1) + if drop_pct > 0.1: + threats.append( + ThreatToValidity( + type="poor_overlap", + severity="high" if drop_pct > 0.3 else "medium", + description=( + f"{dropped}/{n_treated} treated units ({drop_pct:.0%}) " + "dropped due to no match within caliper." + ), + mitigation="Relax caliper or use inverse probability weighting.", + ) + ) + + # Effect size plausibility + if abs(effect_size) > 2.0: + threats.append( + ThreatToValidity( + type="implausible_effect", + severity="medium", + description=f"Effect size ({effect_size:.2f}) is unusually large.", + mitigation="Verify data quality and check for measurement errors.", + ) + ) + + return threats diff --git a/server/src/ehrlich/analysis/infrastructure/rdd_estimator.py b/server/src/ehrlich/analysis/infrastructure/rdd_estimator.py new file mode 100644 index 0000000..4c7cd44 --- /dev/null +++ b/server/src/ehrlich/analysis/infrastructure/rdd_estimator.py @@ -0,0 +1,264 @@ +"""Regression Discontinuity Design causal estimator. + +Pure numpy implementation -- no new dependencies. +""" + +from __future__ import annotations + +import math + +import numpy as np +from scipy import stats + +from ehrlich.analysis.domain.causal import CausalEstimate, ThreatToValidity +from ehrlich.analysis.domain.causal_ports import RDDEstimatorPort +from ehrlich.analysis.domain.evidence_standards import classify_evidence_tier + + +class RDDEstimator(RDDEstimatorPort): + """Regression discontinuity design estimator (sharp and fuzzy).""" + + def estimate( + self, + running_variable: list[float], + outcome: list[float], + cutoff: float, + bandwidth: float | None = None, + design: str = "sharp", + ) -> CausalEstimate: + x = np.array(running_variable, dtype=np.float64) + y = np.array(outcome, dtype=np.float64) + + if len(x) != len(y) or len(x) < 2: + return self._insufficient_data_result(len(x), len(y), design) + + # IK bandwidth approximation if not provided + if bandwidth is None: + bandwidth = self._ik_bandwidth(x, cutoff) + + # Select observations within bandwidth + mask = np.abs(x - cutoff) <= bandwidth + x_bw = x[mask] + y_bw = y[mask] + + left_mask = x_bw < cutoff + right_mask = x_bw >= cutoff + + x_left = x_bw[left_mask] + y_left = y_bw[left_mask] + x_right = x_bw[right_mask] + y_right = y_bw[right_mask] + + n_left = len(x_left) + n_right = len(x_right) + + if n_left < 2 or n_right < 2: + return CausalEstimate( + method=f"regression_discontinuity_{design}", + effect_size=0.0, + standard_error=0.0, + confidence_interval=(0.0, 0.0), + p_value=1.0, + n_treatment=n_right, + n_control=n_left, + covariates=(), + assumptions=( + "Continuity at cutoff", + "No manipulation of running variable", + "Local randomization", + "No compound treatment", + ), + threats=( + ThreatToValidity( + type="insufficient_data", + severity="high", + description="Fewer than 2 observations on one side of the cutoff.", + mitigation="Increase bandwidth or collect more data near cutoff.", + ), + ), + evidence_tier="rationale", + ) + + # Local linear regression on each side + # y = a + b*(x - cutoff) fit separately on each side + left_coeffs = np.polyfit(x_left - cutoff, y_left, 1) + right_coeffs = np.polyfit(x_right - cutoff, y_right, 1) + + # Intercepts at the cutoff + intercept_left = float(left_coeffs[1]) + intercept_right = float(right_coeffs[1]) + + # Sharp RDD: treatment effect = jump at cutoff + jump = intercept_right - intercept_left + + if design == "fuzzy": + # Fuzzy: need treatment indicator jump too + # Approximate: use proportion treated on each side + # In the absence of explicit treatment data, assume + # compliance ratio is proportional to outcome jump + compliance = min(1.0, max(0.1, abs(jump) / (abs(jump) + 0.1))) + effect = jump / compliance + else: + effect = jump + + # Standard error via residual variance + resid_left = y_left - np.polyval(left_coeffs, x_left - cutoff) + resid_right = y_right - np.polyval(right_coeffs, x_right - cutoff) + + var_left = float(np.var(resid_left, ddof=1)) if n_left > 1 else 0.0 + var_right = float(np.var(resid_right, ddof=1)) if n_right > 1 else 0.0 + + se = math.sqrt(var_left / max(n_left, 1) + var_right / max(n_right, 1)) + + # t-test + if se > 0: + t_stat = effect / se + df = max(n_left + n_right - 4, 1) + p_value = float(2 * (1 - stats.t.cdf(abs(t_stat), df=df))) + else: + p_value = 1.0 + + ci_lower = effect - 1.96 * se + ci_upper = effect + 1.96 * se + + threats = self._assess_threats( + x, + cutoff, + bandwidth, + n_left, + n_right, + effect, + design, + ) + evidence_tier = classify_evidence_tier("rdd", list(threats)) + + return CausalEstimate( + method=f"regression_discontinuity_{design}", + effect_size=round(effect, 4), + standard_error=round(se, 4), + confidence_interval=(round(ci_lower, 4), round(ci_upper, 4)), + p_value=round(p_value, 6), + n_treatment=n_right, + n_control=n_left, + covariates=(), + assumptions=( + "Continuity at cutoff", + "No manipulation of running variable", + "Local randomization", + "No compound treatment", + ), + threats=tuple(threats), + evidence_tier=evidence_tier, + ) + + @staticmethod + def _insufficient_data_result(n_x: int, n_y: int, design: str) -> CausalEstimate: + """Return a sentinel result when data is insufficient.""" + return CausalEstimate( + method=f"regression_discontinuity_{design}", + effect_size=0.0, + standard_error=0.0, + confidence_interval=(0.0, 0.0), + p_value=1.0, + n_treatment=0, + n_control=0, + covariates=(), + assumptions=( + "Continuity at cutoff", + "No manipulation of running variable", + "Local randomization", + "No compound treatment", + ), + threats=( + ThreatToValidity( + type="insufficient_data", + severity="high", + description=( + f"Insufficient data (x={n_x}, y={n_y}). " + "Need at least 2 matching-length observations." + ), + mitigation="Collect more data near the cutoff.", + ), + ), + evidence_tier="rationale", + ) + + @staticmethod + def _ik_bandwidth(x: np.ndarray, cutoff: float) -> float: + """IK bandwidth approximation using MAD-based estimate.""" + mad = float(np.median(np.abs(x - cutoff))) + if mad < 1e-10: + mad = float(np.std(x)) + # Rule of thumb: bandwidth ~ 1.84 * MAD * n^(-1/5) + n = len(x) + bw = 1.84 * mad * (n ** (-0.2)) if n > 0 else mad + return max(bw, 1e-6) + + @staticmethod + def _assess_threats( + x: np.ndarray, + cutoff: float, + bandwidth: float, + n_left: int, + n_right: int, + effect_size: float, + design: str, + ) -> list[ThreatToValidity]: + threats: list[ThreatToValidity] = [] + + # 1. Manipulation test (density check via binning) + near_left = np.sum((x >= cutoff - bandwidth * 0.2) & (x < cutoff)) + near_right = np.sum((x >= cutoff) & (x <= cutoff + bandwidth * 0.2)) + if near_left > 0 and near_right > 0: + ratio = float(near_right) / float(near_left) + if ratio > 2.0 or ratio < 0.5: + threats.append( + ThreatToValidity( + type="manipulation", + severity="high", + description=( + f"Density ratio near cutoff is {ratio:.2f}. " + "Possible manipulation of running variable." + ), + mitigation="Conduct McCrary density test and check for bunching.", + ) + ) + + # 2. Small bandwidth + min_side = min(n_left, n_right) + if min_side < 10: + threats.append( + ThreatToValidity( + type="small_bandwidth", + severity="high" if min_side < 5 else "medium", + description=( + f"Only {min_side} observations on one side of cutoff. " + "Local estimates may be imprecise." + ), + mitigation="Consider wider bandwidth with bias correction.", + ) + ) + + # 3. Fuzzy compliance + if design == "fuzzy": + threats.append( + ThreatToValidity( + type="fuzzy_compliance", + severity="medium", + description="Fuzzy RDD relies on estimated compliance rate.", + mitigation="Report first-stage F-statistic and check instrument relevance.", + ) + ) + + # 4. Effect size plausibility + if abs(effect_size) > 2.0: + threats.append( + ThreatToValidity( + type="implausible_effect", + severity="medium", + description=f"Effect size ({effect_size:.2f}) is unusually large.", + mitigation="Verify data quality and check for measurement errors.", + ) + ) + + return threats diff --git a/server/src/ehrlich/analysis/infrastructure/synthetic_control_estimator.py b/server/src/ehrlich/analysis/infrastructure/synthetic_control_estimator.py new file mode 100644 index 0000000..cbcc280 --- /dev/null +++ b/server/src/ehrlich/analysis/infrastructure/synthetic_control_estimator.py @@ -0,0 +1,213 @@ +"""Synthetic Control Method causal estimator. + +Pure scipy/numpy implementation -- no new dependencies. +""" + +from __future__ import annotations + +import math + +import numpy as np +from scipy import optimize, stats + +from ehrlich.analysis.domain.causal import CausalEstimate, ThreatToValidity +from ehrlich.analysis.domain.causal_ports import SyntheticControlPort +from ehrlich.analysis.domain.evidence_standards import classify_evidence_tier + + +class SyntheticControlEstimator(SyntheticControlPort): + """Synthetic control method estimator via constrained optimization.""" + + def estimate( + self, + treated_series: list[float], + donor_matrix: list[list[float]], + treatment_period: int, + ) -> CausalEstimate: + y_treated = np.array(treated_series, dtype=np.float64) + donors = np.array(donor_matrix, dtype=np.float64) + + n_periods = len(y_treated) + n_donors = len(donors) + + if n_donors < 1 or treatment_period < 2 or treatment_period >= n_periods: + return CausalEstimate( + method="synthetic_control", + effect_size=0.0, + standard_error=0.0, + confidence_interval=(0.0, 0.0), + p_value=1.0, + n_treatment=n_periods - treatment_period, + n_control=n_donors, + covariates=(), + assumptions=( + "No interference between units", + "Convex hull condition", + "No anticipation effects", + "Stable unit composition", + ), + threats=( + ThreatToValidity( + type="insufficient_data", + severity="high", + description="Insufficient data for synthetic control estimation.", + mitigation="Ensure at least 2 pre-treatment periods and 1 donor.", + ), + ), + evidence_tier="rationale", + ) + + # Ensure donor series match treated length + donor_pre = np.zeros((n_donors, treatment_period)) + donor_post = np.zeros((n_donors, n_periods - treatment_period)) + for i, d in enumerate(donors): + d_arr = np.array(d, dtype=np.float64) + d_len = min(len(d_arr), n_periods) + pre_len = min(d_len, treatment_period) + post_len = min(d_len - treatment_period, n_periods - treatment_period) + donor_pre[i, :pre_len] = d_arr[:pre_len] + if post_len > 0: + donor_post[i, :post_len] = d_arr[treatment_period : treatment_period + post_len] + + y_pre = y_treated[:treatment_period] + y_post = y_treated[treatment_period:] + + # Optimize weights: minimize pre-treatment RMSPE + # Constraints: weights sum to 1, all >= 0 + def objective(w: np.ndarray) -> float: + synthetic = donor_pre.T @ w + return float(np.sum((y_pre - synthetic) ** 2)) + + w0 = np.ones(n_donors) / n_donors + + result = optimize.minimize( # type: ignore[call-overload] + objective, + w0, + method="SLSQP", + bounds=[(0.0, 1.0)] * n_donors, + constraints={"type": "eq", "fun": lambda w: float(np.sum(w) - 1.0)}, + options={"maxiter": 500}, + ) + weights = result.x + + # Pre-treatment fit quality + synthetic_pre = donor_pre.T @ weights + pre_residuals = y_pre - synthetic_pre + rmspe = float(np.sqrt(np.mean(pre_residuals**2))) + sd_treated_pre = float(np.std(y_pre, ddof=1)) if len(y_pre) > 1 else 1.0 + + # Post-treatment effect + synthetic_post = donor_post.T @ weights + post_gaps = y_post - synthetic_post + effect = float(np.mean(post_gaps)) + + # Standard error from post-treatment gaps + n_post = len(post_gaps) + se = float(np.std(post_gaps, ddof=1) / math.sqrt(n_post)) if n_post > 1 else 0.0 + + # p-value via t-test on post-treatment gaps + if se > 0: + t_stat = effect / se + df = max(n_post - 1, 1) + p_value = float(2 * (1 - stats.t.cdf(abs(t_stat), df=df))) + else: + p_value = 1.0 + + ci_lower = effect - 1.96 * se + ci_upper = effect + 1.96 * se + + threats = self._assess_threats( + rmspe, + sd_treated_pre, + n_donors, + y_pre, + synthetic_pre, + effect, + ) + evidence_tier = classify_evidence_tier("synthetic_control", list(threats)) + + return CausalEstimate( + method="synthetic_control", + effect_size=round(effect, 4), + standard_error=round(se, 4), + confidence_interval=(round(ci_lower, 4), round(ci_upper, 4)), + p_value=round(p_value, 6), + n_treatment=n_post, + n_control=n_donors, + covariates=(), + assumptions=( + "No interference between units", + "Convex hull condition", + "No anticipation effects", + "Stable unit composition", + ), + threats=tuple(threats), + evidence_tier=evidence_tier, + ) + + @staticmethod + def _assess_threats( + rmspe: float, + sd_treated_pre: float, + n_donors: int, + y_pre: np.ndarray, + synthetic_pre: np.ndarray, + effect_size: float, + ) -> list[ThreatToValidity]: + threats: list[ThreatToValidity] = [] + + # 1. Poor pre-treatment fit + fit_ratio = rmspe / sd_treated_pre if sd_treated_pre > 0 else rmspe + if fit_ratio > 0.2: + threats.append( + ThreatToValidity( + type="poor_pretreatment_fit", + severity="high" if fit_ratio > 0.5 else "medium", + description=( + f"Pre-treatment RMSPE is {fit_ratio:.1%} of treated SD. " + "Synthetic control may not adequately replicate treated unit." + ), + mitigation="Add more donor units or covariates to improve fit.", + ) + ) + + # 2. Few donors + if n_donors < 5: + threats.append( + ThreatToValidity( + type="few_donors", + severity="medium" if n_donors >= 3 else "high", + description=( + f"Only {n_donors} donor units available. " + "Small donor pool limits synthetic control quality." + ), + mitigation="Expand donor pool or consider alternative methods.", + ) + ) + + # 3. Anticipation effects (check if gap starts before treatment) + if len(y_pre) >= 4: + last_gaps = y_pre[-2:] - synthetic_pre[-2:] + first_gaps = y_pre[:2] - synthetic_pre[:2] + if float(np.mean(np.abs(last_gaps))) > 2 * float(np.mean(np.abs(first_gaps)) + 1e-10): + threats.append( + ThreatToValidity( + type="anticipation_effects", + severity="medium", + description="Pre-treatment fit deteriorates near treatment date.", + mitigation="Test with earlier treatment date placebo.", + ) + ) + + # 4. Effect size plausibility + if abs(effect_size) > 2.0: + threats.append( + ThreatToValidity( + type="implausible_effect", + severity="medium", + description=f"Effect size ({effect_size:.2f}) is unusually large.", + mitigation="Verify data quality and check for measurement errors.", + ) + ) + + return threats diff --git a/server/src/ehrlich/analysis/tools.py b/server/src/ehrlich/analysis/tools.py index d7eab0a..3f2ca54 100644 --- a/server/src/ehrlich/analysis/tools.py +++ b/server/src/ehrlich/analysis/tools.py @@ -1,10 +1,16 @@ import json from ehrlich.analysis.application.analysis_service import AnalysisService +from ehrlich.analysis.application.causal_service import CausalService from ehrlich.analysis.application.statistics_service import StatisticsService +from ehrlich.analysis.domain.causal import CausalEstimate from ehrlich.analysis.infrastructure.chembl_loader import ChEMBLLoader +from ehrlich.analysis.infrastructure.did_estimator import DiDEstimator from ehrlich.analysis.infrastructure.gtopdb_client import GtoPdbClient +from ehrlich.analysis.infrastructure.psm_estimator import PSMEstimator from ehrlich.analysis.infrastructure.pubchem_client import PubChemClient +from ehrlich.analysis.infrastructure.rdd_estimator import RDDEstimator +from ehrlich.analysis.infrastructure.synthetic_control_estimator import SyntheticControlEstimator from ehrlich.kernel.exceptions import ExternalServiceError _loader = ChEMBLLoader() @@ -13,6 +19,11 @@ _service = AnalysisService(repository=_loader, compound_repo=_pubchem) _pharmacology = _gtopdb _stats = StatisticsService() +_did = DiDEstimator() +_psm = PSMEstimator() +_rdd = RDDEstimator() +_sc = SyntheticControlEstimator() +_causal = CausalService() async def explore_dataset(target: str, threshold: float = 1.0) -> str: @@ -240,3 +251,270 @@ async def run_categorical_test( "interpretation": result.interpretation, } ) + + +def _causal_estimate_to_dict(estimate: CausalEstimate) -> dict[str, object]: + """Convert a CausalEstimate to a JSON-serializable dict.""" + return { + "method": estimate.method, + "effect_size": estimate.effect_size, + "standard_error": estimate.standard_error, + "confidence_interval": list(estimate.confidence_interval), + "p_value": estimate.p_value, + "n_treatment": estimate.n_treatment, + "n_control": estimate.n_control, + "covariates": list(estimate.covariates), + "assumptions": list(estimate.assumptions), + "threats": [ + { + "type": t.type, + "severity": t.severity, + "description": t.description, + "mitigation": t.mitigation, + } + for t in estimate.threats + ], + "evidence_tier": estimate.evidence_tier, + } + + +async def estimate_did( + treatment_pre: str, + treatment_post: str, + control_pre: str, + control_post: str, +) -> str: + """Estimate causal effect using difference-in-differences (DiD). + + Computes the DiD estimator with standard error, p-value, Cohen's d, + 95% confidence interval, parallel trends test, and automated threat + assessment. Returns evidence tier classification (WWC standards). + + Args: + treatment_pre: JSON array of pre-intervention treatment values + (e.g. '[85.2, 87.1, 86.5]') + treatment_post: JSON array of post-intervention treatment values + (e.g. '[92.3, 94.1, 93.5]') + control_pre: JSON array of pre-intervention control values + (e.g. '[84.0, 85.2, 84.8]') + control_post: JSON array of post-intervention control values + (e.g. '[85.1, 85.8, 85.3]') + """ + try: + t_pre = [float(x) for x in json.loads(treatment_pre)] + t_post = [float(x) for x in json.loads(treatment_post)] + c_pre = [float(x) for x in json.loads(control_pre)] + c_post = [float(x) for x in json.loads(control_post)] + except (json.JSONDecodeError, TypeError, ValueError): + return json.dumps({"error": "Invalid JSON arrays. Provide numeric arrays."}) + + if not all([t_pre, t_post, c_pre, c_post]): + return json.dumps({"error": "All four groups must have at least one value."}) + + estimate = _causal.estimate_did(t_pre, t_post, c_pre, c_post, estimator=_did) + return json.dumps(_causal_estimate_to_dict(estimate)) + + +async def estimate_psm( + treated_outcomes: str, + control_outcomes: str, + treated_covariates: str, + control_covariates: str, +) -> str: + """Estimate causal effect using propensity score matching (PSM). + + Computes ATT via nearest-neighbor caliper matching on logistic + propensity scores. Returns effect size, SE, p-value, balance + diagnostics, and automated threat assessment. + + Args: + treated_outcomes: JSON array of treatment group outcome values + (e.g. '[92.3, 94.1, 93.5]') + control_outcomes: JSON array of control group outcome values + (e.g. '[85.1, 85.8, 85.3]') + treated_covariates: JSON 2D array of treatment covariates + (e.g. '[[1.2, 3.4], [2.1, 4.5]]') + control_covariates: JSON 2D array of control covariates + (e.g. '[[1.0, 3.2], [2.3, 4.1]]') + """ + try: + t_out = [float(x) for x in json.loads(treated_outcomes)] + c_out = [float(x) for x in json.loads(control_outcomes)] + t_cov = [[float(v) for v in row] for row in json.loads(treated_covariates)] + c_cov = [[float(v) for v in row] for row in json.loads(control_covariates)] + except (json.JSONDecodeError, TypeError, ValueError): + return json.dumps({"error": "Invalid JSON. Provide numeric arrays."}) + + if not t_out or not c_out or not t_cov or not c_cov: + return json.dumps({"error": "All groups must have at least one value."}) + + estimate = _causal.estimate_psm(t_out, c_out, t_cov, c_cov, estimator=_psm) + return json.dumps(_causal_estimate_to_dict(estimate)) + + +async def estimate_rdd( + running_variable: str, + outcome: str, + cutoff: float, + bandwidth: float | None = None, + design: str = "sharp", +) -> str: + """Estimate causal effect using regression discontinuity design (RDD). + + Computes local linear regression on each side of the cutoff. + Supports sharp and fuzzy designs. Returns effect size, SE, p-value, + bandwidth, and automated threat assessment. + + Args: + running_variable: JSON array of running variable values + (e.g. '[45, 48, 50, 52, 55]') + outcome: JSON array of outcome values (same length as running_variable) + (e.g. '[70, 72, 80, 82, 85]') + cutoff: Cutoff threshold value + bandwidth: Optional bandwidth (auto-calculated via IK method if None) + design: 'sharp' or 'fuzzy' + """ + try: + rv = [float(x) for x in json.loads(running_variable)] + y = [float(x) for x in json.loads(outcome)] + except (json.JSONDecodeError, TypeError, ValueError): + return json.dumps({"error": "Invalid JSON arrays. Provide numeric arrays."}) + + if not rv or not y: + return json.dumps({"error": "Running variable and outcome must have values."}) + if len(rv) != len(y): + return json.dumps({"error": "Running variable and outcome must have same length."}) + + estimate = _causal.estimate_rdd( + rv, + y, + cutoff, + estimator=_rdd, + bandwidth=bandwidth, + design=design, + ) + return json.dumps(_causal_estimate_to_dict(estimate)) + + +async def estimate_synthetic_control( + treated_series: str, + donor_matrix: str, + treatment_period: int, +) -> str: + """Estimate causal effect using synthetic control method. + + Constructs a synthetic counterfactual from weighted donor units via + constrained optimization (SLSQP). Returns effect size, SE, p-value, + pre-treatment fit quality, and automated threat assessment. + + Args: + treated_series: JSON array of treated unit time series + (e.g. '[10, 11, 12, 20, 22, 25]') + donor_matrix: JSON 2D array where each row is a donor unit time series + (e.g. '[[10, 11, 12, 13, 14, 15], [9, 10, 11, 12, 13, 14]]') + treatment_period: Index (0-based) where treatment begins + (e.g. 3 means first 3 periods are pre-treatment) + """ + try: + treated = [float(x) for x in json.loads(treated_series)] + donors = [[float(v) for v in row] for row in json.loads(donor_matrix)] + except (json.JSONDecodeError, TypeError, ValueError): + return json.dumps({"error": "Invalid JSON. Provide numeric arrays."}) + + if not treated or not donors: + return json.dumps({"error": "Treated series and donor matrix must have values."}) + + estimate = _causal.estimate_synthetic_control(treated, donors, treatment_period, estimator=_sc) + return json.dumps(_causal_estimate_to_dict(estimate)) + + +async def assess_threats( + method: str, + sample_sizes: str, + parallel_trends_p: float | None = None, + effect_size: float | None = None, +) -> str: + """Assess validity threats for a causal inference method. + + Knowledge-based threat assessment that identifies potential biases + and suggests mitigations for different causal methods (DiD, PSM, + RDD, SC, RCT, IV). + + Args: + method: Causal method name ('did', 'psm', 'rdd', 'sc', 'rct', 'iv') + sample_sizes: JSON object mapping group names to sizes + (e.g. '{{"treatment": 50, "control": 45}}') + parallel_trends_p: p-value from parallel trends test (DiD only) + effect_size: Cohen's d or standardized effect size + """ + try: + sizes = json.loads(sample_sizes) + except json.JSONDecodeError: + return json.dumps({"error": "Invalid JSON in sample_sizes parameter"}) + + if not isinstance(sizes, dict): + return json.dumps({"error": "sample_sizes must be a JSON object"}) + + parsed_sizes = {str(k): int(v) for k, v in sizes.items()} + threats = _causal.assess_threats(method, parsed_sizes, parallel_trends_p, effect_size) + return json.dumps( + { + "method": method, + "threat_count": len(threats), + "threats": [ + { + "type": t.type, + "severity": t.severity, + "description": t.description, + "mitigation": t.mitigation, + } + for t in threats + ], + } + ) + + +async def compute_cost_effectiveness( + program_name: str, + total_cost: float, + total_effect: float, + currency: str = "USD", + effect_unit: str = "units", + comparison_cost: float | None = None, + comparison_effect: float | None = None, +) -> str: + """Compute cost-effectiveness ratio and optional ICER. + + Calculates cost per unit of effect for a program, and optionally + the Incremental Cost-Effectiveness Ratio (ICER) when a comparison + program is provided. + + Args: + program_name: Name of the program + total_cost: Total program cost + total_effect: Total measured effect (e.g. QALYs, enrollment change) + currency: Currency code (default: USD) + effect_unit: Unit of effect measurement (e.g. 'QALYs', 'pp enrollment') + comparison_cost: Cost of comparison program (for ICER calculation) + comparison_effect: Effect of comparison program (for ICER calculation) + """ + result = _causal.compute_cost_effectiveness( + program_name, + total_cost, + total_effect, + currency, + effect_unit, + comparison_cost, + comparison_effect, + ) + return json.dumps( + { + "program_name": result.program_name, + "total_cost": result.total_cost, + "total_effect": result.total_effect, + "cost_per_unit": result.cost_per_unit, + "currency": result.currency, + "effect_unit": result.effect_unit, + "icer": result.icer, + } + ) diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index fa551f6..26f06b4 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -11,7 +11,13 @@ from ehrlich.analysis.tools import ( analyze_substructures, + assess_threats, + compute_cost_effectiveness, compute_properties, + estimate_did, + estimate_psm, + estimate_rdd, + estimate_synthetic_control, explore_dataset, run_categorical_test, run_statistical_test, @@ -31,9 +37,7 @@ ) from ehrlich.config import get_settings from ehrlich.impact.tools import ( - assess_threats, compare_programs, - estimate_did, fetch_benchmark, search_economic_indicators, ) @@ -61,6 +65,7 @@ from ehrlich.investigation.tools_viz import ( render_admet_radar, render_binding_scatter, + render_causal_diagram, render_dose_response, render_evidence_matrix, render_forest_plot, @@ -72,6 +77,7 @@ render_parallel_trends, render_performance_chart, render_program_dashboard, + render_rdd_plot, render_therapeutic_window, render_training_timeline, ) @@ -221,6 +227,7 @@ def _build_registry() -> ToolRegistry: _nutrition = frozenset({"nutrition"}) _nutrition_safety = frozenset({"nutrition", "safety"}) _impact = frozenset({"impact"}) + _causal = frozenset({"causal"}) _ml = frozenset({"ml"}) _viz = frozenset({"visualization"}) _chem_viz = frozenset({"chemistry", "visualization"}) @@ -228,6 +235,7 @@ def _build_registry() -> ToolRegistry: _training_viz = frozenset({"training", "visualization"}) _nutrition_viz = frozenset({"nutrition", "visualization"}) _impact_viz = frozenset({"impact", "visualization"}) + _causal_viz = frozenset({"causal", "visualization"}) tagged_tools: list[tuple[str, Any, frozenset[str] | None]] = [ # Chemistry (6) @@ -287,13 +295,18 @@ def _build_registry() -> ToolRegistry: ("check_interactions", check_interactions, _nutrition_safety), ("analyze_nutrient_ratios", analyze_nutrient_ratios, _nutrition), ("compute_inflammatory_index", compute_inflammatory_index, _nutrition), - # Impact Evaluation (5) + # Impact Evaluation (3) ("search_economic_indicators", search_economic_indicators, _impact), ("fetch_benchmark", fetch_benchmark, _impact), ("compare_programs", compare_programs, _impact), - ("estimate_did", estimate_did, _impact), - ("assess_threats", assess_threats, _impact), - # Visualization (15) + # Causal Inference (6) -- domain-agnostic + ("estimate_did", estimate_did, _causal), + ("estimate_psm", estimate_psm, _causal), + ("estimate_rdd", estimate_rdd, _causal), + ("estimate_synthetic_control", estimate_synthetic_control, _causal), + ("assess_threats", assess_threats, _causal), + ("compute_cost_effectiveness", compute_cost_effectiveness, _causal), + # Visualization (17) ("render_binding_scatter", render_binding_scatter, _chem_viz), ("render_admet_radar", render_admet_radar, _sim_viz), ("render_training_timeline", render_training_timeline, _training_viz), @@ -309,6 +322,8 @@ def _build_registry() -> ToolRegistry: ("render_program_dashboard", render_program_dashboard, _impact_viz), ("render_geographic_comparison", render_geographic_comparison, _impact_viz), ("render_parallel_trends", render_parallel_trends, _impact_viz), + ("render_rdd_plot", render_rdd_plot, _causal_viz), + ("render_causal_diagram", render_causal_diagram, _causal_viz), # Statistics (2) -- universal, no tags ("run_statistical_test", run_statistical_test, None), ("run_categorical_test", run_categorical_test, None), diff --git a/server/src/ehrlich/impact/application/impact_service.py b/server/src/ehrlich/impact/application/impact_service.py index 118b24a..0aab047 100644 --- a/server/src/ehrlich/impact/application/impact_service.py +++ b/server/src/ehrlich/impact/application/impact_service.py @@ -8,12 +8,9 @@ if TYPE_CHECKING: from ehrlich.impact.domain.entities import ( Benchmark, - CausalEstimate, EconomicSeries, HealthIndicator, - ThreatToValidity, ) - from ehrlich.impact.domain.ports import CausalEstimator from ehrlich.impact.domain.repository import ( DevelopmentDataRepository, EconomicDataRepository, @@ -27,12 +24,10 @@ def __init__( economic: EconomicDataRepository | None = None, health: HealthDataRepository | None = None, development: DevelopmentDataRepository | None = None, - causal_estimator: CausalEstimator | None = None, ) -> None: self._economic = economic self._health = health self._development = development - self._causal = causal_estimator self._stats = StatisticsService() async def search_economic_data(self, query: str, limit: int = 10) -> list[EconomicSeries]: @@ -111,88 +106,6 @@ async def fetch_benchmark( return [] - def estimate_did( - self, - treatment_pre: list[float], - treatment_post: list[float], - control_pre: list[float], - control_post: list[float], - ) -> CausalEstimate: - """Estimate causal effect using difference-in-differences.""" - if not self._causal: - msg = "No causal estimator configured" - raise RuntimeError(msg) - return self._causal.estimate_did( - treatment_pre, treatment_post, control_pre, control_post, - ) - - def assess_threats( - self, - method: str, - sample_sizes: dict[str, int], - parallel_trends_p: float | None = None, - effect_size: float | None = None, - ) -> list[ThreatToValidity]: - """Knowledge-based threat assessment for any causal method.""" - from ehrlich.impact.domain.entities import ThreatToValidity - - threats: list[ThreatToValidity] = [] - method_lower = method.lower().replace("-", "_").replace(" ", "_") - - # Small sample threats - min_n = min(sample_sizes.values()) if sample_sizes else 0 - if min_n < 5: - threats.append(ThreatToValidity( - type="small_sample", - severity="high", - description=f"Minimum group size is {min_n}. Insufficient for reliable inference.", - mitigation="Increase sample size or use exact/permutation tests.", - )) - elif min_n < 30: - threats.append(ThreatToValidity( - type="small_sample", - severity="medium", - description=f"Minimum group size is {min_n}. May limit statistical power.", - mitigation="Report power analysis and consider non-parametric tests.", - )) - - # Method-specific threats - is_did = method_lower in ("did", "difference_in_differences") - if is_did and parallel_trends_p is not None and parallel_trends_p < 0.05: - threats.append(ThreatToValidity( - type="parallel_trends_violation", - severity="high", - description=f"Parallel trends assumption violated (p={parallel_trends_p:.4f}).", - mitigation="Use event study or synthetic control method.", - )) - - if method_lower in ("psm", "propensity_score_matching"): - threats.append(ThreatToValidity( - type="unobserved_confounders", - severity="medium", - description="PSM cannot account for unobserved confounders.", - mitigation="Combine with sensitivity analysis (Rosenbaum bounds).", - )) - - if method_lower in ("rdd", "regression_discontinuity"): - threats.append(ThreatToValidity( - type="manipulation", - severity="medium", - description="Running variable may be subject to manipulation near cutoff.", - mitigation="Test for bunching at the cutoff (McCrary density test).", - )) - - # Effect size plausibility - if effect_size is not None and abs(effect_size) > 2.0: - threats.append(ThreatToValidity( - type="implausible_effect", - severity="medium", - description=f"Effect size ({effect_size:.2f}) is unusually large.", - mitigation="Verify data quality and check for measurement errors.", - )) - - return threats - def compare_programs( self, programs: list[dict[str, object]], diff --git a/server/src/ehrlich/impact/domain/entities.py b/server/src/ehrlich/impact/domain/entities.py index 1b40aab..26fb864 100644 --- a/server/src/ehrlich/impact/domain/entities.py +++ b/server/src/ehrlich/impact/domain/entities.py @@ -34,29 +34,6 @@ class Indicator: period: str -@dataclass(frozen=True) -class ThreatToValidity: - type: str - severity: str - description: str - mitigation: str - - -@dataclass(frozen=True) -class CausalEstimate: - method: str - effect_size: float - standard_error: float - confidence_interval: tuple[float, float] - p_value: float - n_treatment: int - n_control: int - covariates: tuple[str, ...] - assumptions: tuple[str, ...] - threats: tuple[ThreatToValidity, ...] - evidence_tier: str - - @dataclass(frozen=True) class Benchmark: source: str diff --git a/server/src/ehrlich/impact/domain/evaluation_standards.py b/server/src/ehrlich/impact/domain/evaluation_standards.py index feb5cdc..556c66d 100644 --- a/server/src/ehrlich/impact/domain/evaluation_standards.py +++ b/server/src/ehrlich/impact/domain/evaluation_standards.py @@ -1,33 +1,11 @@ -"""Evidence grading standards for impact evaluation. +"""Impact-specific evaluation standards. -Based on What Works Clearinghouse (WWC) evidence tiers, -CONEVAL MIR (Matriz de Indicadores para Resultados) levels, -and CREMAA quality criteria. +CONEVAL MIR (Matriz de Indicadores para Resultados) levels +and CREMAA quality criteria for Mexican social programs. """ from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ehrlich.impact.domain.entities import ThreatToValidity - -WWC_TIERS: dict[str, str] = { - "strong": ( - "Well-designed RCT or quasi-experimental study with no " - "serious threats to validity (low attrition, verified equivalence)" - ), - "moderate": ( - "Quasi-experimental design with some threats to validity " - "that are partially addressed (e.g., PSM with sensitivity analysis)" - ), - "promising": ( - "Correlational study with statistical controls or pre-post " - "comparison with a plausible comparison group" - ), - "rationale": ("Theory-based or descriptive evidence only; no causal identification strategy"), -} - CONEVAL_MIR_LEVELS: dict[str, str] = { "fin": "Long-term societal goal the program contributes to", "proposito": "Direct outcome expected for the target population", @@ -43,44 +21,3 @@ "adecuado": "Indicator is appropriate for the level it measures", "aportacion_marginal": "Indicator adds unique information not captured elsewhere", } - - -def classify_evidence_tier(method: str, threats: list[ThreatToValidity]) -> str: - """Classify evidence strength based on method and threats. - - Returns one of: strong, moderate, promising, rationale. - """ - strong_methods = {"rct", "randomized_controlled_trial"} - moderate_methods = { - "did", - "difference_in_differences", - "psm", - "propensity_score_matching", - "rdd", - "regression_discontinuity", - "synthetic_control", - } - promising_methods = {"iv", "instrumental_variables", "fixed_effects", "panel_regression"} - - method_lower = method.lower().replace("-", "_").replace(" ", "_") - - severe_threats = [t for t in threats if t.severity == "high"] - - if method_lower in strong_methods: - if len(severe_threats) >= 2: - return "moderate" - if len(severe_threats) == 1: - return "moderate" - return "strong" - - if method_lower in moderate_methods: - if len(severe_threats) >= 2: - return "promising" - return "moderate" - - if method_lower in promising_methods: - if len(severe_threats) >= 1: - return "rationale" - return "promising" - - return "rationale" diff --git a/server/src/ehrlich/impact/domain/ports.py b/server/src/ehrlich/impact/domain/ports.py deleted file mode 100644 index caaad50..0000000 --- a/server/src/ehrlich/impact/domain/ports.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Ports (ABCs) for impact evaluation causal inference.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ehrlich.impact.domain.entities import CausalEstimate - - -class CausalEstimator(ABC): - """Abstract base class for causal inference estimators.""" - - @abstractmethod - def estimate_did( - self, - treatment_pre: list[float], - treatment_post: list[float], - control_pre: list[float], - control_post: list[float], - ) -> CausalEstimate: - """Estimate causal effect using difference-in-differences. - - Args: - treatment_pre: Pre-intervention outcomes for treatment group - treatment_post: Post-intervention outcomes for treatment group - control_pre: Pre-intervention outcomes for control group - control_post: Post-intervention outcomes for control group - - Returns: - CausalEstimate with effect size, significance, and threats - """ - ... diff --git a/server/src/ehrlich/impact/tools.py b/server/src/ehrlich/impact/tools.py index 46a48fd..3dc609e 100644 --- a/server/src/ehrlich/impact/tools.py +++ b/server/src/ehrlich/impact/tools.py @@ -1,8 +1,7 @@ """Impact Evaluation tools for the investigation engine. -5 tools for causal analysis of social programs: economic indicator -search, benchmark fetching, cross-program comparison, difference-in- -differences estimation, and validity threat assessment. +3 tools for social program data retrieval: economic indicator +search, benchmark fetching, and cross-program comparison. """ from __future__ import annotations @@ -11,7 +10,6 @@ from dataclasses import asdict from ehrlich.impact.application.impact_service import ImpactService -from ehrlich.impact.infrastructure.did_estimator import DiDEstimator from ehrlich.impact.infrastructure.fred_client import FREDClient from ehrlich.impact.infrastructure.who_client import WHOClient from ehrlich.impact.infrastructure.worldbank_client import WorldBankClient @@ -19,12 +17,10 @@ _worldbank = WorldBankClient() _who = WHOClient() _fred = FREDClient() -_did = DiDEstimator() _service = ImpactService( economic=_fred, health=_who, development=_worldbank, - causal_estimator=_did, ) @@ -150,82 +146,3 @@ async def compare_programs( result = _service.compare_programs(parsed, metric, alternative) return json.dumps(result) - - -async def estimate_did( - treatment_pre: str, - treatment_post: str, - control_pre: str, - control_post: str, -) -> str: - """Estimate causal effect using difference-in-differences (DiD). - - Computes the DiD estimator with standard error, p-value, Cohen's d, - 95% confidence interval, parallel trends test, and automated threat - assessment. Returns evidence tier classification (WWC standards). - - Args: - treatment_pre: JSON array of pre-intervention treatment values - (e.g. '[85.2, 87.1, 86.5]') - treatment_post: JSON array of post-intervention treatment values - (e.g. '[92.3, 94.1, 93.5]') - control_pre: JSON array of pre-intervention control values - (e.g. '[84.0, 85.2, 84.8]') - control_post: JSON array of post-intervention control values - (e.g. '[85.1, 85.8, 85.3]') - """ - try: - t_pre = [float(x) for x in json.loads(treatment_pre)] - t_post = [float(x) for x in json.loads(treatment_post)] - c_pre = [float(x) for x in json.loads(control_pre)] - c_post = [float(x) for x in json.loads(control_post)] - except (json.JSONDecodeError, TypeError, ValueError): - return json.dumps({"error": "Invalid JSON arrays. Provide numeric arrays."}) - - if not all([t_pre, t_post, c_pre, c_post]): - return json.dumps({"error": "All four groups must have at least one value."}) - - estimate = _service.estimate_did(t_pre, t_post, c_pre, c_post) - result = asdict(estimate) - # Convert tuple fields to lists for JSON - result["confidence_interval"] = list(result["confidence_interval"]) - result["covariates"] = list(result["covariates"]) - result["assumptions"] = list(result["assumptions"]) - result["threats"] = [asdict(t) for t in estimate.threats] - return json.dumps(result) - - -async def assess_threats( - method: str, - sample_sizes: str, - parallel_trends_p: float | None = None, - effect_size: float | None = None, -) -> str: - """Assess validity threats for a causal inference method. - - Knowledge-based threat assessment that identifies potential biases - and suggests mitigations for different causal methods (DiD, PSM, - RDD, RCT, IV). - - Args: - method: Causal method name ('did', 'psm', 'rdd', 'rct', 'iv') - sample_sizes: JSON object mapping group names to sizes - (e.g. '{{"treatment": 50, "control": 45}}') - parallel_trends_p: p-value from parallel trends test (DiD only) - effect_size: Cohen's d or standardized effect size - """ - try: - sizes = json.loads(sample_sizes) - except json.JSONDecodeError: - return json.dumps({"error": "Invalid JSON in sample_sizes parameter"}) - - if not isinstance(sizes, dict): - return json.dumps({"error": "sample_sizes must be a JSON object"}) - - parsed_sizes = {str(k): int(v) for k, v in sizes.items()} - threats = _service.assess_threats(method, parsed_sizes, parallel_trends_p, effect_size) - return json.dumps({ - "method": method, - "threat_count": len(threats), - "threats": [asdict(t) for t in threats], - }) diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index a68a206..26826bf 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -255,9 +255,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, # Detect domain configs (multi-domain) and yield event if self._domain_registry: - detected_configs, is_fallback = self._domain_registry.detect( - domain_categories - ) + detected_configs, is_fallback = self._domain_registry.detect(domain_categories) self._active_config = merge_domain_configs(detected_configs) self._researcher_prompt = build_researcher_prompt(self._active_config) yield DomainDetected( @@ -442,8 +440,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, f"Rationale: {hypothesis.rationale}\n\n" f"Available tools: {tools_csv}\n\n" f"Design an experiment to test this " - f"hypothesis." - + sibling_section, + f"hypothesis." + sibling_section, investigation.id, output_config=_build_output_config(EXPERIMENT_DESIGN_SCHEMA), ): @@ -1304,8 +1301,7 @@ async def _run_researcher_experiment( f"Experiment failure criteria: {experiment.failure_criteria or 'N/A'}\n\n" f"Execute this experiment. Compare results against the " f"pre-defined success/failure criteria. Link all findings to " - f"hypothesis_id='{hypothesis.id}'." - + sibling_section + f"hypothesis_id='{hypothesis.id}'." + sibling_section ), }, ] diff --git a/server/src/ehrlich/investigation/domain/domains/impact.py b/server/src/ehrlich/investigation/domain/domains/impact.py index a8c6286..e6695bb 100644 --- a/server/src/ehrlich/investigation/domain/domains/impact.py +++ b/server/src/ehrlich/investigation/domain/domains/impact.py @@ -10,7 +10,7 @@ identifier_type="program", identifier_label="Program", candidate_label="Program Interventions", - tool_tags=frozenset({"impact", "statistics", "literature", "visualization", "ml"}), + tool_tags=frozenset({"impact", "causal", "statistics", "literature", "visualization", "ml"}), score_definitions=( ScoreDefinition( key="effect_size", @@ -200,6 +200,35 @@ 3. record_finding(title="DiD shows significant enrollment effect", \ detail="...", hypothesis_id="h1", evidence_type="supporting") +Example: Propensity score matching +1. estimate_psm(treated_outcomes="[92.3, 94.1, 93.5, 91.0]", \ +control_outcomes="[85.1, 85.8, 85.3, 86.0]", \ +treated_covariates="[[1.2, 3.4], [2.1, 4.5], [1.8, 3.9], [2.0, 4.2]]", \ +control_covariates="[[1.0, 3.2], [2.3, 4.1], [1.5, 3.6], [1.9, 4.0]]") +2. record_finding(title="PSM estimate of program effect", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Regression discontinuity design +1. estimate_rdd(running_variable="[45, 48, 49, 50, 51, 52, 55]", \ +outcome="[70, 72, 73, 80, 82, 83, 85]", cutoff=50.0, design="sharp") +2. record_finding(title="RDD shows enrollment discontinuity at cutoff", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Synthetic control method +1. estimate_synthetic_control(\ +treated_series="[10, 11, 12, 20, 22, 25]", \ +donor_matrix="[[10, 11, 12, 13, 14, 15], [9, 10, 11, 12, 13, 14]]", \ +treatment_period=3) +2. record_finding(title="Synthetic control shows treatment effect", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Cost-effectiveness analysis +1. compute_cost_effectiveness(program_name="CCT Program", \ +total_cost=5000000.0, total_effect=1500.0, \ +currency="USD", effect_unit="additional enrollees") +2. record_finding(title="Cost per additional enrollee", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + Example: Training a classifier on program outcome data 1. train_classifier(feature_names=["budget_per_capita", \ "duration_years", "coverage_pct", "gdp_per_capita"], \ diff --git a/server/src/ehrlich/investigation/infrastructure/repository.py b/server/src/ehrlich/investigation/infrastructure/repository.py index d8704fd..1302700 100644 --- a/server/src/ehrlich/investigation/infrastructure/repository.py +++ b/server/src/ehrlich/investigation/infrastructure/repository.py @@ -84,9 +84,7 @@ def __init__(self, database_url: str) -> None: async def initialize(self) -> None: await self._ensure_database() - self._pool = await asyncpg.create_pool( - self._database_url, min_size=2, max_size=10 - ) + self._pool = await asyncpg.create_pool(self._database_url, min_size=2, max_size=10) async with self._pool.acquire() as conn: await conn.execute(_SCHEMA) logger.info("PostgreSQL repository initialized") @@ -122,9 +120,7 @@ def _get_pool(self) -> asyncpg.Pool[asyncpg.Record]: raise RuntimeError(msg) return self._pool - async def save( - self, investigation: Investigation, *, user_id: str | None = None - ) -> None: + async def save(self, investigation: Investigation, *, user_id: str | None = None) -> None: pool = self._get_pool() async with pool.acquire() as conn: await conn.execute( @@ -170,9 +166,7 @@ async def get_by_id(self, investigation_id: str) -> Investigation | None: async def list_all(self) -> list[Investigation]: pool = self._get_pool() async with pool.acquire() as conn: - rows = await conn.fetch( - "SELECT * FROM investigations ORDER BY created_at DESC" - ) + rows = await conn.fetch("SELECT * FROM investigations ORDER BY created_at DESC") return [_from_row(row) for row in rows] async def list_by_user(self, user_id: str) -> list[Investigation]: @@ -216,14 +210,11 @@ async def update(self, investigation: Investigation) -> None: if investigation.status == InvestigationStatus.COMPLETED: await self._rebuild_fts(conn, investigation) - async def save_event( - self, investigation_id: str, event_type: str, event_data: str - ) -> None: + async def save_event(self, investigation_id: str, event_type: str, event_data: str) -> None: pool = self._get_pool() async with pool.acquire() as conn: await conn.execute( - "INSERT INTO events (investigation_id, event_type, event_data) " - "VALUES ($1, $2, $3)", + "INSERT INTO events (investigation_id, event_type, event_data) VALUES ($1, $2, $3)", investigation_id, event_type, event_data, @@ -249,9 +240,7 @@ async def get_events(self, investigation_id: str) -> list[dict[str, str]]: for row in rows ] - async def search_findings( - self, query: str, limit: int = 20 - ) -> list[dict[str, Any]]: + async def search_findings(self, query: str, limit: int = 20) -> list[dict[str, Any]]: pool = self._get_pool() async with pool.acquire() as conn: rows = await conn.fetch( @@ -297,8 +286,7 @@ async def get_or_create_user(self, workos_id: str, email: str) -> dict[str, Any] pool = self._get_pool() async with pool.acquire() as conn: row = await conn.fetchrow( - "SELECT id, workos_id, email, credits, created_at " - "FROM users WHERE workos_id = $1", + "SELECT id, workos_id, email, credits, created_at FROM users WHERE workos_id = $1", workos_id, ) if row is not None: @@ -315,16 +303,12 @@ async def get_or_create_user(self, workos_id: str, email: str) -> dict[str, Any] async def get_user_credits(self, workos_id: str) -> int: pool = self._get_pool() async with pool.acquire() as conn: - row = await conn.fetchrow( - "SELECT credits FROM users WHERE workos_id = $1", workos_id - ) + row = await conn.fetchrow("SELECT credits FROM users WHERE workos_id = $1", workos_id) if row is None: return 0 return int(row["credits"]) - async def deduct_credits( - self, workos_id: str, amount: int, investigation_id: str - ) -> bool: + async def deduct_credits(self, workos_id: str, amount: int, investigation_id: str) -> bool: pool = self._get_pool() async with pool.acquire() as conn, conn.transaction(): row = await conn.fetchrow( @@ -347,14 +331,11 @@ async def deduct_credits( ) return True - async def refund_credits( - self, workos_id: str, amount: int, investigation_id: str - ) -> None: + async def refund_credits(self, workos_id: str, amount: int, investigation_id: str) -> None: pool = self._get_pool() async with pool.acquire() as conn, conn.transaction(): row = await conn.fetchrow( - "UPDATE users SET credits = credits + $1 " - "WHERE workos_id = $2 RETURNING id", + "UPDATE users SET credits = credits + $1 WHERE workos_id = $2 RETURNING id", amount, workos_id, ) @@ -391,8 +372,7 @@ async def _rebuild_fts( ) text = " ".join(parts) await conn.execute( - "UPDATE investigations SET findings_search = to_tsvector('english', $1) " - "WHERE id = $2", + "UPDATE investigations SET findings_search = to_tsvector('english', $1) WHERE id = $2", text, investigation.id, ) diff --git a/server/src/ehrlich/investigation/tools_viz.py b/server/src/ehrlich/investigation/tools_viz.py index c8649ab..6d5ceea 100644 --- a/server/src/ehrlich/investigation/tools_viz.py +++ b/server/src/ehrlich/investigation/tools_viz.py @@ -1,11 +1,11 @@ """Visualization tools for the investigation engine. -15 tools that generate structured chart payloads for the frontend: +17 tools that generate structured chart payloads for the frontend: binding scatter, ADMET radar, training timeline, muscle heatmap, forest plot, evidence matrix, performance chart, funnel plot, dose-response curve, nutrient comparison, nutrient adequacy, therapeutic window, program dashboard, geographic comparison, -and parallel trends. +parallel trends, RDD plot, and causal diagram. """ from __future__ import annotations @@ -645,25 +645,29 @@ async def render_program_dashboard( else: status = "off_track" - processed.append({ - "name": ind.get("name", ""), - "baseline": round(baseline, 2), - "target": round(target, 2), - "actual": round(actual, 2), - "unit": ind.get("unit", ""), - "pct_target": round(pct_target, 4), - "status": status, - }) - - return json.dumps({ - "viz_type": "program_dashboard", - "title": title or f"{program_name} Dashboard", - "data": { - "program_name": program_name, - "indicators": processed, - }, - "config": {"domain": "impact"}, - }) + processed.append( + { + "name": ind.get("name", ""), + "baseline": round(baseline, 2), + "target": round(target, 2), + "actual": round(actual, 2), + "unit": ind.get("unit", ""), + "pct_target": round(pct_target, 4), + "status": status, + } + ) + + return json.dumps( + { + "viz_type": "program_dashboard", + "title": title or f"{program_name} Dashboard", + "data": { + "program_name": program_name, + "indicators": processed, + }, + "config": {"domain": "impact"}, + } + ) async def render_geographic_comparison( @@ -682,21 +686,25 @@ async def render_geographic_comparison( """ processed: list[dict[str, Any]] = [] for r in regions: - processed.append({ - "name": r.get("name", ""), - "value": round(float(r.get("value", 0.0)), 2), - }) - - return json.dumps({ - "viz_type": "geographic_comparison", - "title": title, - "data": { - "regions": processed, - "metric_name": metric_name, - "benchmark": round(benchmark, 2) if benchmark is not None else None, - }, - "config": {"domain": "impact"}, - }) + processed.append( + { + "name": r.get("name", ""), + "value": round(float(r.get("value", 0.0)), 2), + } + ) + + return json.dumps( + { + "viz_type": "geographic_comparison", + "title": title, + "data": { + "regions": processed, + "metric_name": metric_name, + "benchmark": round(benchmark, 2) if benchmark is not None else None, + }, + "config": {"domain": "impact"}, + } + ) async def render_parallel_trends( @@ -725,13 +733,124 @@ async def render_parallel_trends( for c in control_series ] - return json.dumps({ - "viz_type": "parallel_trends", - "title": title, - "data": { - "treatment": treatment, - "control": control, - "treatment_start": treatment_start, - }, - "config": {"domain": "impact"}, - }) + return json.dumps( + { + "viz_type": "parallel_trends", + "title": title, + "data": { + "treatment": treatment, + "control": control, + "treatment_start": treatment_start, + }, + "config": {"domain": "impact"}, + } + ) + + +async def render_rdd_plot( + observations: list[dict[str, Any]], + cutoff: float, + fitted_left: list[dict[str, float]] | None = None, + fitted_right: list[dict[str, float]] | None = None, + title: str = "Regression Discontinuity", +) -> str: + """Render an RDD scatter plot with fitted lines and cutoff reference. + + Shows observations colored by side of cutoff, vertical dashed cutoff + line, and optional fitted regression lines on each side. + + Args: + observations: List of dicts with 'x' (running variable) and 'y' (outcome) + cutoff: Cutoff threshold value (vertical reference line) + fitted_left: Optional fitted points for left side [{x, y}, ...] + fitted_right: Optional fitted points for right side [{x, y}, ...] + title: Chart title + """ + points: list[dict[str, Any]] = [] + for obs in observations: + x_val = float(obs.get("x", 0.0)) + points.append( + { + "x": x_val, + "y": float(obs.get("y", 0.0)), + "side": "right" if x_val >= cutoff else "left", + } + ) + + left_fit = [ + {"x": float(p.get("x", 0.0)), "y": float(p.get("y", 0.0))} for p in (fitted_left or []) + ] + right_fit = [ + {"x": float(p.get("x", 0.0)), "y": float(p.get("y", 0.0))} for p in (fitted_right or []) + ] + + return json.dumps( + { + "viz_type": "rdd_plot", + "title": title, + "data": { + "observations": points, + "cutoff": cutoff, + "fitted_left": left_fit, + "fitted_right": right_fit, + }, + "config": {"domain": "causal"}, + } + ) + + +async def render_causal_diagram( + nodes: list[dict[str, str]], + edges: list[dict[str, str]], + title: str = "Causal Diagram", +) -> str: + """Render a directed acyclic graph (DAG) for causal analysis. + + Shows treatment, outcome, and confounder nodes with directed edges + representing causal relationships and associations. + + Args: + nodes: List of dicts with 'id', 'label', and 'type' + (type: 'treatment', 'outcome', 'confounder', 'mediator', 'instrument') + edges: List of dicts with 'source', 'target', and 'type' + (type: 'causal', 'association', 'instrument') + title: Chart title + """ + valid_node_types = {"treatment", "outcome", "confounder", "mediator", "instrument"} + processed_nodes: list[dict[str, str]] = [] + for node in nodes: + node_type = node.get("type", "confounder") + if node_type not in valid_node_types: + node_type = "confounder" + processed_nodes.append( + { + "id": node.get("id", ""), + "label": node.get("label", ""), + "type": node_type, + } + ) + + processed_edges: list[dict[str, str]] = [] + for edge in edges: + edge_type = edge.get("type", "causal") + if edge_type not in ("causal", "association", "instrument"): + edge_type = "causal" + processed_edges.append( + { + "source": edge.get("source", ""), + "target": edge.get("target", ""), + "type": edge_type, + } + ) + + return json.dumps( + { + "viz_type": "causal_diagram", + "title": title, + "data": { + "nodes": processed_nodes, + "edges": processed_edges, + }, + "config": {"domain": "causal"}, + } + ) diff --git a/server/src/ehrlich/literature/infrastructure/semantic_scholar.py b/server/src/ehrlich/literature/infrastructure/semantic_scholar.py index 4003323..ead8eb4 100644 --- a/server/src/ehrlich/literature/infrastructure/semantic_scholar.py +++ b/server/src/ehrlich/literature/infrastructure/semantic_scholar.py @@ -137,7 +137,7 @@ async def get_references(self, paper_id: str, limit: int = 10) -> list[Paper]: resp.raise_for_status() data = resp.json() papers = [] - for item in (data.get("data") or []): + for item in data.get("data") or []: cited = item.get("citedPaper") if cited and cited.get("title"): papers.append(self._to_paper(cited)) @@ -186,7 +186,7 @@ async def get_citing(self, paper_id: str, limit: int = 10) -> list[Paper]: resp.raise_for_status() data = resp.json() papers = [] - for item in (data.get("data") or []): + for item in data.get("data") or []: citing = item.get("citingPaper") if citing and citing.get("title"): papers.append(self._to_paper(citing)) diff --git a/server/tests/analysis/test_causal.py b/server/tests/analysis/test_causal.py new file mode 100644 index 0000000..6093aa6 --- /dev/null +++ b/server/tests/analysis/test_causal.py @@ -0,0 +1,77 @@ +"""Tests for causal inference domain entities.""" + +from __future__ import annotations + +from ehrlich.analysis.domain.causal import ( + CausalEstimate, + CostEffectivenessResult, + ThreatToValidity, +) + + +class TestThreatToValidity: + def test_construction(self) -> None: + threat = ThreatToValidity( + type="small_sample", + severity="high", + description="N < 5", + mitigation="Increase N", + ) + assert threat.type == "small_sample" + assert threat.severity == "high" + + def test_frozen(self) -> None: + threat = ThreatToValidity(type="a", severity="low", description="b", mitigation="c") + try: + threat.type = "x" # type: ignore[misc] + raise AssertionError("Should be frozen") + except AttributeError: + pass + + +class TestCausalEstimate: + def test_construction(self) -> None: + estimate = CausalEstimate( + method="did", + effect_size=5.0, + standard_error=1.0, + confidence_interval=(3.04, 6.96), + p_value=0.001, + n_treatment=50, + n_control=50, + covariates=(), + assumptions=("Parallel trends",), + threats=(), + evidence_tier="moderate", + ) + assert estimate.method == "did" + assert estimate.effect_size == 5.0 + assert len(estimate.confidence_interval) == 2 + + +class TestCostEffectivenessResult: + def test_construction(self) -> None: + result = CostEffectivenessResult( + program_name="Test Program", + total_cost=100000.0, + total_effect=500.0, + cost_per_unit=200.0, + currency="USD", + effect_unit="enrollees", + icer=None, + ) + assert result.program_name == "Test Program" + assert result.cost_per_unit == 200.0 + assert result.icer is None + + def test_with_icer(self) -> None: + result = CostEffectivenessResult( + program_name="A", + total_cost=200000.0, + total_effect=800.0, + cost_per_unit=250.0, + currency="USD", + effect_unit="QALYs", + icer=150.0, + ) + assert result.icer == 150.0 diff --git a/server/tests/analysis/test_causal_service.py b/server/tests/analysis/test_causal_service.py new file mode 100644 index 0000000..dc441ee --- /dev/null +++ b/server/tests/analysis/test_causal_service.py @@ -0,0 +1,77 @@ +"""Tests for CausalService.""" + +from __future__ import annotations + +import pytest + +from ehrlich.analysis.application.causal_service import CausalService + + +@pytest.fixture +def service() -> CausalService: + return CausalService() + + +class TestAssessThreats: + def test_small_sample_high(self, service: CausalService) -> None: + threats = service.assess_threats("did", {"treatment": 3, "control": 4}) + assert any(t.severity == "high" and t.type == "small_sample" for t in threats) + + def test_small_sample_medium(self, service: CausalService) -> None: + threats = service.assess_threats("did", {"treatment": 20, "control": 15}) + assert any(t.severity == "medium" and t.type == "small_sample" for t in threats) + + def test_parallel_trends_violation(self, service: CausalService) -> None: + threats = service.assess_threats("did", {"t": 50, "c": 50}, parallel_trends_p=0.03) + assert any(t.type == "parallel_trends_violation" for t in threats) + + def test_psm_unobserved(self, service: CausalService) -> None: + threats = service.assess_threats("psm", {"t": 100, "c": 100}) + assert any(t.type == "unobserved_confounders" for t in threats) + + def test_rdd_manipulation(self, service: CausalService) -> None: + threats = service.assess_threats("rdd", {"t": 100, "c": 100}) + assert any(t.type == "manipulation" for t in threats) + + def test_synthetic_control(self, service: CausalService) -> None: + threats = service.assess_threats("synthetic_control", {"t": 100, "c": 10}) + assert any(t.type == "convex_hull" for t in threats) + + def test_implausible_effect(self, service: CausalService) -> None: + threats = service.assess_threats("rct", {"t": 100, "c": 100}, effect_size=3.5) + assert any(t.type == "implausible_effect" for t in threats) + + def test_large_sample_no_sample_threat(self, service: CausalService) -> None: + threats = service.assess_threats("rct", {"t": 100, "c": 100}) + assert not any(t.type == "small_sample" for t in threats) + + +class TestComputeCostEffectiveness: + def test_basic(self, service: CausalService) -> None: + result = CausalService.compute_cost_effectiveness( + program_name="Test", + total_cost=100000.0, + total_effect=500.0, + ) + assert result.cost_per_unit == 200.0 + assert result.icer is None + + def test_with_icer(self, service: CausalService) -> None: + result = CausalService.compute_cost_effectiveness( + program_name="A", + total_cost=200000.0, + total_effect=800.0, + comparison_cost=100000.0, + comparison_effect=500.0, + ) + # ICER = (200000-100000) / (800-500) = 333.33 + assert result.icer is not None + assert abs(result.icer - 333.33) < 0.01 + + def test_zero_effect(self, service: CausalService) -> None: + result = CausalService.compute_cost_effectiveness( + program_name="Zero", + total_cost=100000.0, + total_effect=0.0, + ) + assert result.cost_per_unit == 0.0 diff --git a/server/tests/analysis/test_causal_tools.py b/server/tests/analysis/test_causal_tools.py new file mode 100644 index 0000000..cac29a6 --- /dev/null +++ b/server/tests/analysis/test_causal_tools.py @@ -0,0 +1,188 @@ +"""Tests for causal inference tool functions.""" + +from __future__ import annotations + +import json + +import pytest + +from ehrlich.analysis.tools import ( + assess_threats, + compute_cost_effectiveness, + estimate_did, + estimate_psm, + estimate_rdd, + estimate_synthetic_control, +) + + +class TestEstimateDidTool: + @pytest.mark.anyio + async def test_valid_input(self) -> None: + result = json.loads( + await estimate_did( + treatment_pre="[80.0, 81.0, 79.5, 80.5, 80.0]", + treatment_post="[86.0, 87.0, 85.5, 86.5, 86.0]", + control_pre="[80.0, 81.0, 79.5, 80.5, 80.0]", + control_post="[81.0, 82.0, 80.5, 81.5, 81.0]", + ) + ) + + assert result["method"] == "difference_in_differences" + assert "effect_size" in result + assert "p_value" in result + assert len(result["confidence_interval"]) == 2 + assert isinstance(result["assumptions"], list) + assert isinstance(result["threats"], list) + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await estimate_did("not json", "[1]", "[1]", "[1]")) + assert "error" in result + + @pytest.mark.anyio + async def test_empty_arrays(self) -> None: + result = json.loads(await estimate_did("[]", "[1, 2]", "[1, 2]", "[1, 2]")) + assert "error" in result + + +class TestEstimatePsmTool: + @pytest.mark.anyio + async def test_valid_input(self) -> None: + result = json.loads( + await estimate_psm( + treated_outcomes="[90.0, 92.0, 88.0, 91.0]", + control_outcomes="[80.0, 82.0, 78.0, 81.0]", + treated_covariates="[[1.0, 2.0], [1.5, 2.5], [1.2, 2.2], [1.8, 2.8]]", + control_covariates="[[0.9, 1.9], [1.4, 2.4], [1.1, 2.1], [1.7, 2.7]]", + ) + ) + + assert result["method"] == "propensity_score_matching" + assert "effect_size" in result + assert isinstance(result["threats"], list) + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await estimate_psm("bad", "[1]", "[[1]]", "[[1]]")) + assert "error" in result + + +class TestEstimateRddTool: + @pytest.mark.anyio + async def test_valid_input(self) -> None: + result = json.loads( + await estimate_rdd( + running_variable="[40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60]", + outcome="[48, 49, 50, 51, 52, 70, 71, 72, 73, 74, 75]", + cutoff=50.0, + ) + ) + + assert "regression_discontinuity" in result["method"] + assert "effect_size" in result + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await estimate_rdd("bad", "[1]", cutoff=50.0)) + assert "error" in result + + @pytest.mark.anyio + async def test_mismatched_lengths(self) -> None: + result = json.loads(await estimate_rdd("[1, 2, 3]", "[1, 2]", cutoff=2.0)) + assert "error" in result + + +class TestEstimateSyntheticControlTool: + @pytest.mark.anyio + async def test_valid_input(self) -> None: + result = json.loads( + await estimate_synthetic_control( + treated_series="[10, 11, 12, 25, 26, 27]", + donor_matrix="[[10, 11, 12, 13, 14, 15], [9, 10, 11, 12, 13, 14]]", + treatment_period=3, + ) + ) + + assert result["method"] == "synthetic_control" + assert "effect_size" in result + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await estimate_synthetic_control("bad", "[[1]]", treatment_period=1)) + assert "error" in result + + +class TestAssessThreatsTool: + @pytest.mark.anyio + async def test_did_threats(self) -> None: + result = json.loads( + await assess_threats( + method="did", + sample_sizes='{"treatment": 50, "control": 45}', + parallel_trends_p=0.03, + effect_size=0.5, + ) + ) + + assert result["method"] == "did" + assert isinstance(result["threats"], list) + threat_types = [t["type"] for t in result["threats"]] + assert "parallel_trends_violation" in threat_types + + @pytest.mark.anyio + async def test_invalid_json(self) -> None: + result = json.loads(await assess_threats(method="did", sample_sizes="not json")) + assert "error" in result + + @pytest.mark.anyio + async def test_psm_threats(self) -> None: + result = json.loads( + await assess_threats( + method="psm", + sample_sizes='{"treatment": 100, "control": 100}', + ) + ) + threat_types = [t["type"] for t in result["threats"]] + assert "unobserved_confounders" in threat_types + + @pytest.mark.anyio + async def test_rdd_threats(self) -> None: + result = json.loads( + await assess_threats( + method="rdd", + sample_sizes='{"treatment": 100, "control": 100}', + ) + ) + threat_types = [t["type"] for t in result["threats"]] + assert "manipulation" in threat_types + + +class TestComputeCostEffectivenessTool: + @pytest.mark.anyio + async def test_basic(self) -> None: + result = json.loads( + await compute_cost_effectiveness( + program_name="Test", + total_cost=100000.0, + total_effect=500.0, + ) + ) + + assert result["program_name"] == "Test" + assert result["cost_per_unit"] == 200.0 + assert result["icer"] is None + + @pytest.mark.anyio + async def test_with_icer(self) -> None: + result = json.loads( + await compute_cost_effectiveness( + program_name="A", + total_cost=200000.0, + total_effect=800.0, + comparison_cost=100000.0, + comparison_effect=500.0, + ) + ) + + assert result["icer"] is not None diff --git a/server/tests/impact/test_did_estimator.py b/server/tests/analysis/test_did_estimator.py similarity index 54% rename from server/tests/impact/test_did_estimator.py rename to server/tests/analysis/test_did_estimator.py index fe9955b..0d0d0e5 100644 --- a/server/tests/impact/test_did_estimator.py +++ b/server/tests/analysis/test_did_estimator.py @@ -1,10 +1,10 @@ -"""Tests for the DiD estimator infrastructure.""" +"""Tests for the DiD estimator (moved from impact/).""" from __future__ import annotations import pytest -from ehrlich.impact.infrastructure.did_estimator import DiDEstimator +from ehrlich.analysis.infrastructure.did_estimator import DiDEstimator @pytest.fixture @@ -14,13 +14,12 @@ def estimator() -> DiDEstimator: class TestDiDEstimator: def test_known_effect(self, estimator: DiDEstimator) -> None: - """Synthetic data with known true DiD effect of ~5.0.""" treatment_pre = [80.0, 81.0, 79.5, 80.5, 80.0] - treatment_post = [86.0, 87.0, 85.5, 86.5, 86.0] # +6 + treatment_post = [86.0, 87.0, 85.5, 86.5, 86.0] control_pre = [80.0, 81.0, 79.5, 80.5, 80.0] - control_post = [81.0, 82.0, 80.5, 81.5, 81.0] # +1 + control_post = [81.0, 82.0, 80.5, 81.5, 81.0] - result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + result = estimator.estimate(treatment_pre, treatment_post, control_pre, control_post) assert result.method == "difference_in_differences" assert abs(result.effect_size - 5.0) < 0.01 @@ -30,75 +29,62 @@ def test_known_effect(self, estimator: DiDEstimator) -> None: assert result.n_control == 10 def test_no_effect(self, estimator: DiDEstimator) -> None: - """Both groups change equally -- no causal effect.""" treatment_pre = [50.0, 51.0, 49.5, 50.5] - treatment_post = [52.0, 53.0, 51.5, 52.5] # +2 + treatment_post = [52.0, 53.0, 51.5, 52.5] control_pre = [50.0, 51.0, 49.5, 50.5] - control_post = [52.0, 53.0, 51.5, 52.5] # +2 + control_post = [52.0, 53.0, 51.5, 52.5] - result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + result = estimator.estimate(treatment_pre, treatment_post, control_pre, control_post) assert abs(result.effect_size) < 0.01 assert result.p_value > 0.05 def test_large_effect(self, estimator: DiDEstimator) -> None: - """Large treatment effect triggers implausible_effect threat.""" treatment_pre = [10.0, 11.0, 10.5] - treatment_post = [30.0, 31.0, 30.5] # +20 + treatment_post = [30.0, 31.0, 30.5] control_pre = [10.0, 11.0, 10.5] - control_post = [11.0, 12.0, 11.5] # +1 + control_post = [11.0, 12.0, 11.5] - result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) + result = estimator.estimate(treatment_pre, treatment_post, control_pre, control_post) assert result.effect_size > 15.0 threat_types = [t.type for t in result.threats] assert "implausible_effect" in threat_types def test_small_sample_threat(self, estimator: DiDEstimator) -> None: - """Very small samples trigger small_sample threat.""" - result = estimator.estimate_did([10.0, 11.0], [15.0, 16.0], [10.0, 11.0], [11.0, 12.0]) - + result = estimator.estimate([10.0, 11.0], [15.0, 16.0], [10.0, 11.0], [11.0, 12.0]) threat_types = [t.type for t in result.threats] assert "small_sample" in threat_types def test_parallel_trends_violation(self, estimator: DiDEstimator) -> None: - """Clearly different pre-treatment trends trigger parallel_trends threat.""" treatment_pre = [50.0, 55.0, 60.0, 65.0, 70.0, 75.0, 80.0, 85.0] control_pre = [50.0, 50.5, 51.0, 51.5, 52.0, 52.5, 53.0, 53.5] treatment_post = [90.0, 95.0] control_post = [54.0, 54.5] - result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) - + result = estimator.estimate(treatment_pre, treatment_post, control_pre, control_post) threat_types = [t.type for t in result.threats] assert "parallel_trends_violation" in threat_types or "parallel_trends_weak" in threat_types - def test_evidence_tier_classification(self, estimator: DiDEstimator) -> None: - """DiD without severe threats should be classified as moderate.""" + def test_evidence_tier(self, estimator: DiDEstimator) -> None: treatment_pre = [80.0, 81.0, 79.5, 80.5, 80.0, 79.8, 80.2, 81.5] treatment_post = [86.0, 87.0, 85.5, 86.5, 86.0, 85.8, 86.2, 87.5] control_pre = [80.0, 81.0, 79.5, 80.5, 80.0, 79.8, 80.2, 81.5] control_post = [81.0, 82.0, 80.5, 81.5, 81.0, 80.8, 81.2, 82.5] - result = estimator.estimate_did(treatment_pre, treatment_post, control_pre, control_post) - + result = estimator.estimate(treatment_pre, treatment_post, control_pre, control_post) assert result.evidence_tier in ("moderate", "strong") def test_assumptions_present(self, estimator: DiDEstimator) -> None: - result = estimator.estimate_did( - [10.0, 11.0, 12.0], [15.0, 16.0, 17.0], - [10.0, 11.0, 12.0], [11.0, 12.0, 13.0], + result = estimator.estimate( + [10.0, 11.0, 12.0], + [15.0, 16.0, 17.0], + [10.0, 11.0, 12.0], + [11.0, 12.0, 13.0], ) assert len(result.assumptions) == 4 assert "Parallel trends" in result.assumptions - def test_identical_groups(self, estimator: DiDEstimator) -> None: - """All groups identical -- no effect, high p-value.""" - vals = [10.0, 11.0, 10.5, 10.8] - result = estimator.estimate_did(vals, vals, vals, vals) - assert abs(result.effect_size) < 0.01 - def test_single_values(self, estimator: DiDEstimator) -> None: - """Minimal input: single value per group.""" - result = estimator.estimate_did([10.0], [15.0], [10.0], [11.0]) + result = estimator.estimate([10.0], [15.0], [10.0], [11.0]) assert abs(result.effect_size - 4.0) < 0.01 diff --git a/server/tests/analysis/test_evidence_standards.py b/server/tests/analysis/test_evidence_standards.py new file mode 100644 index 0000000..30df803 --- /dev/null +++ b/server/tests/analysis/test_evidence_standards.py @@ -0,0 +1,79 @@ +"""Tests for causal inference evidence standards.""" + +from __future__ import annotations + +from ehrlich.analysis.domain.causal import ThreatToValidity +from ehrlich.analysis.domain.evidence_standards import ( + WWC_TIERS, + classify_evidence_tier, +) + + +class TestWWCTiers: + def test_all_tiers_present(self) -> None: + assert set(WWC_TIERS.keys()) == {"strong", "moderate", "promising", "rationale"} + + def test_tiers_have_descriptions(self) -> None: + for tier, desc in WWC_TIERS.items(): + assert isinstance(desc, str) + assert len(desc) > 10, f"Tier {tier} has short description" + + +class TestClassifyEvidenceTier: + def test_rct_no_threats(self) -> None: + assert classify_evidence_tier("rct", []) == "strong" + + def test_rct_with_severe_threat(self) -> None: + threats = [ + ThreatToValidity( + type="attrition", severity="high", description="30% dropout", mitigation="ITT" + ) + ] + assert classify_evidence_tier("rct", threats) == "moderate" + + def test_rct_with_two_severe_threats(self) -> None: + threats = [ + ThreatToValidity(type="a", severity="high", description="x", mitigation="y"), + ThreatToValidity(type="b", severity="high", description="x", mitigation="y"), + ] + assert classify_evidence_tier("rct", threats) == "moderate" + + def test_did_no_threats(self) -> None: + assert classify_evidence_tier("did", []) == "moderate" + + def test_did_alias(self) -> None: + assert classify_evidence_tier("difference_in_differences", []) == "moderate" + + def test_psm_no_threats(self) -> None: + assert classify_evidence_tier("psm", []) == "moderate" + + def test_psm_with_severe_threats(self) -> None: + threats = [ + ThreatToValidity(type="a", severity="high", description="x", mitigation="y"), + ThreatToValidity(type="b", severity="high", description="x", mitigation="y"), + ] + assert classify_evidence_tier("psm", threats) == "promising" + + def test_rdd_no_threats(self) -> None: + assert classify_evidence_tier("rdd", []) == "moderate" + + def test_synthetic_control(self) -> None: + assert classify_evidence_tier("synthetic_control", []) == "moderate" + + def test_iv_no_threats(self) -> None: + assert classify_evidence_tier("iv", []) == "promising" + + def test_iv_with_severe_threat(self) -> None: + threats = [ + ThreatToValidity(type="a", severity="high", description="x", mitigation="y"), + ] + assert classify_evidence_tier("iv", threats) == "rationale" + + def test_unknown_method(self) -> None: + assert classify_evidence_tier("survey", []) == "rationale" + + def test_low_severity_not_counted(self) -> None: + threats = [ + ThreatToValidity(type="a", severity="low", description="x", mitigation="y"), + ] + assert classify_evidence_tier("rct", threats) == "strong" diff --git a/server/tests/analysis/test_psm_estimator.py b/server/tests/analysis/test_psm_estimator.py new file mode 100644 index 0000000..6bb6bd0 --- /dev/null +++ b/server/tests/analysis/test_psm_estimator.py @@ -0,0 +1,69 @@ +"""Tests for the PSM estimator.""" + +from __future__ import annotations + +import pytest + +from ehrlich.analysis.infrastructure.psm_estimator import PSMEstimator + + +@pytest.fixture +def estimator() -> PSMEstimator: + return PSMEstimator() + + +class TestPSMEstimator: + def test_known_att(self, estimator: PSMEstimator) -> None: + """Treatment group has clearly higher outcomes.""" + treated_out = [90.0, 92.0, 88.0, 91.0, 89.0, 93.0, 87.0, 90.0] + control_out = [80.0, 82.0, 78.0, 81.0, 79.0, 83.0, 77.0, 80.0] + treated_cov = [[i * 0.1, i * 0.2] for i in range(8)] + control_cov = [[i * 0.1 + 0.05, i * 0.2 + 0.1] for i in range(8)] + + result = estimator.estimate(treated_out, control_out, treated_cov, control_cov) + + assert result.method == "propensity_score_matching" + assert result.effect_size > 5.0 + assert result.n_treatment > 0 + assert len(result.assumptions) == 4 + + def test_no_effect(self, estimator: PSMEstimator) -> None: + """Both groups have identical outcomes.""" + outcomes = [50.0, 51.0, 49.5, 50.5, 50.2, 50.8, 49.8, 51.2] + covariates = [[i * 0.1, i * 0.2] for i in range(8)] + + result = estimator.estimate(outcomes, outcomes, covariates, covariates) + + assert abs(result.effect_size) < 1.0 + + def test_unobserved_confounders_threat(self, estimator: PSMEstimator) -> None: + """PSM always has unobserved confounders threat.""" + treated_out = [10.0, 12.0, 11.0, 13.0] + control_out = [8.0, 9.0, 7.0, 10.0] + treated_cov = [[1.0], [2.0], [1.5], [2.5]] + control_cov = [[0.8], [1.8], [1.3], [2.3]] + + result = estimator.estimate(treated_out, control_out, treated_cov, control_cov) + threat_types = [t.type for t in result.threats] + assert "unobserved_confounders" in threat_types + + def test_poor_overlap(self, estimator: PSMEstimator) -> None: + """Very different covariates should trigger overlap issues.""" + treated_out = [90.0, 92.0, 88.0, 91.0] + control_out = [80.0, 82.0, 78.0, 81.0] + treated_cov = [[100.0, 200.0]] * 4 + control_cov = [[1.0, 2.0]] * 4 + + result = estimator.estimate(treated_out, control_out, treated_cov, control_cov) + # Should still produce a result (may have poor overlap threat) + assert result.method == "propensity_score_matching" + + def test_small_sample(self, estimator: PSMEstimator) -> None: + """Very small samples may result in few matches.""" + result = estimator.estimate( + [10.0, 12.0], + [8.0, 9.0], + [[1.0], [2.0]], + [[0.8], [1.8]], + ) + assert result.method == "propensity_score_matching" diff --git a/server/tests/analysis/test_rdd_estimator.py b/server/tests/analysis/test_rdd_estimator.py new file mode 100644 index 0000000..52458c9 --- /dev/null +++ b/server/tests/analysis/test_rdd_estimator.py @@ -0,0 +1,67 @@ +"""Tests for the RDD estimator.""" + +from __future__ import annotations + +import pytest + +from ehrlich.analysis.infrastructure.rdd_estimator import RDDEstimator + + +@pytest.fixture +def estimator() -> RDDEstimator: + return RDDEstimator() + + +class TestRDDEstimator: + def test_sharp_discontinuity(self, estimator: RDDEstimator) -> None: + """Clear jump at cutoff in sharp design.""" + # Below cutoff: outcome ~ 50, above cutoff: outcome ~ 70 + x = [40.0, 42.0, 44.0, 46.0, 48.0, 50.0, 52.0, 54.0, 56.0, 58.0, 60.0] + y = [48.0, 49.0, 50.0, 51.0, 52.0, 70.0, 71.0, 72.0, 73.0, 74.0, 75.0] + + result = estimator.estimate(x, y, cutoff=50.0) + + assert "regression_discontinuity" in result.method + assert result.effect_size > 10.0 + assert result.n_treatment >= 1 + assert result.n_control >= 1 + + def test_no_discontinuity(self, estimator: RDDEstimator) -> None: + """Smooth function with no jump.""" + x = [float(i) for i in range(20, 80)] + y = [float(i) * 0.5 + 10.0 for i in range(20, 80)] + + result = estimator.estimate(x, y, cutoff=50.0) + + assert abs(result.effect_size) < 5.0 + + def test_fuzzy_design(self, estimator: RDDEstimator) -> None: + """Fuzzy design should still produce a result.""" + x = [40.0, 42.0, 44.0, 46.0, 48.0, 50.0, 52.0, 54.0, 56.0, 58.0, 60.0] + y = [48.0, 49.0, 50.0, 51.0, 52.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0] + + result = estimator.estimate(x, y, cutoff=50.0, design="fuzzy") + + assert "fuzzy" in result.method + assert result.effect_size != 0.0 + + def test_custom_bandwidth(self, estimator: RDDEstimator) -> None: + """With explicit bandwidth.""" + x = [float(i) for i in range(20, 80)] + y = [50.0 if i < 50 else 70.0 for i in range(20, 80)] + + result = estimator.estimate(x, y, cutoff=50.0, bandwidth=10.0) + assert result.effect_size > 10.0 + + def test_insufficient_data(self, estimator: RDDEstimator) -> None: + """Too few observations should trigger insufficient_data threat.""" + result = estimator.estimate([49.0], [50.0, 51.0], cutoff=50.0, bandwidth=5.0) + threat_types = [t.type for t in result.threats] + assert "insufficient_data" in threat_types or "small_bandwidth" in threat_types + + def test_assumptions_present(self, estimator: RDDEstimator) -> None: + x = [float(i) for i in range(40, 62)] + y = [50.0 if i < 50 else 70.0 for i in range(40, 62)] + result = estimator.estimate(x, y, cutoff=50.0) + assert len(result.assumptions) == 4 + assert "Continuity at cutoff" in result.assumptions diff --git a/server/tests/analysis/test_statistics.py b/server/tests/analysis/test_statistics.py index 46290ff..8ef772e 100644 --- a/server/tests/analysis/test_statistics.py +++ b/server/tests/analysis/test_statistics.py @@ -180,9 +180,7 @@ async def test_run_statistical_test_json(self) -> None: async def test_run_categorical_test_json(self) -> None: from ehrlich.analysis.tools import run_categorical_test - result = json.loads( - await run_categorical_test([[10, 5], [3, 12]]) - ) + result = json.loads(await run_categorical_test([[10, 5], [3, 12]])) assert "test_name" in result assert "p_value" in result assert "effect_size" in result diff --git a/server/tests/analysis/test_synthetic_control_estimator.py b/server/tests/analysis/test_synthetic_control_estimator.py new file mode 100644 index 0000000..eb1a200 --- /dev/null +++ b/server/tests/analysis/test_synthetic_control_estimator.py @@ -0,0 +1,71 @@ +"""Tests for the Synthetic Control estimator.""" + +from __future__ import annotations + +import pytest + +from ehrlich.analysis.infrastructure.synthetic_control_estimator import SyntheticControlEstimator + + +@pytest.fixture +def estimator() -> SyntheticControlEstimator: + return SyntheticControlEstimator() + + +class TestSyntheticControlEstimator: + def test_known_effect(self, estimator: SyntheticControlEstimator) -> None: + """Treated jumps at treatment period while donors stay flat.""" + treated = [10.0, 11.0, 12.0, 13.0, 25.0, 26.0, 27.0] + donors = [ + [10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0], + [9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0], + ] + + result = estimator.estimate(treated, donors, treatment_period=4) + + assert result.method == "synthetic_control" + assert result.effect_size > 5.0 + assert result.n_treatment == 3 # 3 post-treatment periods + assert result.n_control == 2 # 2 donors + + def test_no_effect(self, estimator: SyntheticControlEstimator) -> None: + """Treated follows donors -- no divergence.""" + treated = [10.0, 11.0, 12.0, 13.0, 14.0, 15.0] + donors = [ + [10.0, 11.0, 12.0, 13.0, 14.0, 15.0], + [10.0, 11.0, 12.0, 13.0, 14.0, 15.0], + ] + + result = estimator.estimate(treated, donors, treatment_period=3) + assert abs(result.effect_size) < 2.0 + + def test_few_donors_threat(self, estimator: SyntheticControlEstimator) -> None: + """Fewer than 5 donors triggers warning.""" + treated = [10.0, 11.0, 12.0, 20.0, 21.0] + donors = [ + [10.0, 11.0, 12.0, 13.0, 14.0], + [9.0, 10.0, 11.0, 12.0, 13.0], + ] + + result = estimator.estimate(treated, donors, treatment_period=3) + threat_types = [t.type for t in result.threats] + assert "few_donors" in threat_types + + def test_weights_sum_to_one(self, estimator: SyntheticControlEstimator) -> None: + """Optimization should produce valid weights.""" + treated = [10.0, 11.0, 12.0, 20.0, 21.0, 22.0] + donors = [ + [10.0, 11.0, 12.0, 13.0, 14.0, 15.0], + [10.0, 11.0, 12.0, 13.0, 14.0, 15.0], + [10.0, 11.0, 12.0, 13.0, 14.0, 15.0], + ] + + result = estimator.estimate(treated, donors, treatment_period=3) + assert result.method == "synthetic_control" + assert len(result.assumptions) == 4 + + def test_insufficient_data(self, estimator: SyntheticControlEstimator) -> None: + """Treatment period at boundary should handle gracefully.""" + result = estimator.estimate([10.0, 20.0], [[10.0, 11.0]], treatment_period=1) + # Should still produce a valid estimate or report insufficient data + assert result.method == "synthetic_control" diff --git a/server/tests/api/test_auth.py b/server/tests/api/test_auth.py index becc737..29b2929 100644 --- a/server/tests/api/test_auth.py +++ b/server/tests/api/test_auth.py @@ -93,9 +93,7 @@ def test_missing_auth_header_returns_401(self, client: TestClient) -> None: assert response.status_code == 401 assert "Missing or invalid Authorization header" in response.json()["detail"] - def test_malformed_header_no_bearer_returns_401( - self, client: TestClient - ) -> None: + def test_malformed_header_no_bearer_returns_401(self, client: TestClient) -> None: response = client.get("/protected", headers={"Authorization": "Token abc"}) assert response.status_code == 401 @@ -120,9 +118,7 @@ def test_valid_token_returns_user(self, client: TestClient) -> None: token = _make_token(sub="user_abc", email="abc@test.com") settings_patch, jwks_patch = _patch_workos() with settings_patch, jwks_patch: - response = client.get( - "/protected", headers={"Authorization": f"Bearer {token}"} - ) + response = client.get("/protected", headers={"Authorization": f"Bearer {token}"}) assert response.status_code == 200 data = response.json() assert data["workos_id"] == "user_abc" @@ -133,9 +129,7 @@ def test_workos_not_configured_returns_500(self, client: TestClient) -> None: "ehrlich.api.auth.get_settings", return_value=MagicMock(workos_client_id=""), ): - response = client.get( - "/protected", headers={"Authorization": "Bearer some.token.here"} - ) + response = client.get("/protected", headers={"Authorization": "Bearer some.token.here"}) assert response.status_code == 500 assert "WorkOS not configured" in response.json()["detail"] @@ -152,9 +146,7 @@ def test_invalid_token_returns_none(self, client: TestClient) -> None: mock_jwks_fn.return_value.get_signing_key_from_jwt.side_effect = ( pyjwt.exceptions.PyJWTError("bad") ) - response = client.get( - "/optional", headers={"Authorization": "Bearer bad.token"} - ) + response = client.get("/optional", headers={"Authorization": "Bearer bad.token"}) assert response.status_code == 200 assert response.json() == {"user": None} @@ -162,9 +154,7 @@ def test_valid_token_returns_user(self, client: TestClient) -> None: token = _make_token(sub="user_opt", email="opt@test.com") settings_patch, jwks_patch = _patch_workos() with settings_patch, jwks_patch: - response = client.get( - "/optional", headers={"Authorization": f"Bearer {token}"} - ) + response = client.get("/optional", headers={"Authorization": f"Bearer {token}"}) assert response.status_code == 200 data = response.json() assert data["user"]["workos_id"] == "user_opt" @@ -175,9 +165,7 @@ def test_accepts_bearer_header(self, client: TestClient) -> None: token = _make_token(sub="user_sse", email="sse@test.com") settings_patch, jwks_patch = _patch_workos() with settings_patch, jwks_patch: - response = client.get( - "/sse", headers={"Authorization": f"Bearer {token}"} - ) + response = client.get("/sse", headers={"Authorization": f"Bearer {token}"}) assert response.status_code == 200 assert response.json()["workos_id"] == "user_sse" @@ -194,9 +182,7 @@ def test_missing_both_returns_401(self, client: TestClient) -> None: assert response.status_code == 401 assert "Missing authentication" in response.json()["detail"] - def test_header_takes_precedence_over_query( - self, client: TestClient - ) -> None: + def test_header_takes_precedence_over_query(self, client: TestClient) -> None: header_token = _make_token(sub="from_header", email="h@test.com") query_token = _make_token(sub="from_query", email="q@test.com") settings_patch, jwks_patch = _patch_workos() diff --git a/server/tests/api/test_credits.py b/server/tests/api/test_credits.py index f311d8a..3726a6b 100644 --- a/server/tests/api/test_credits.py +++ b/server/tests/api/test_credits.py @@ -64,9 +64,7 @@ async def test_detects_byok_header(self, client: httpx.AsyncClient) -> None: class TestStartInvestigationCredits: - async def test_deducts_credits_on_start( - self, client: httpx.AsyncClient - ) -> None: + async def test_deducts_credits_on_start(self, client: httpx.AsyncClient) -> None: response = await client.post( "/api/v1/investigate", json={"prompt": "Test deduction", "director_tier": "haiku"}, @@ -77,9 +75,7 @@ async def test_deducts_credits_on_start( balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 4 - async def test_insufficient_credits_returns_402( - self, client: httpx.AsyncClient - ) -> None: + async def test_insufficient_credits_returns_402(self, client: httpx.AsyncClient) -> None: resp1 = await client.post( "/api/v1/investigate", json={"prompt": "First opus", "director_tier": "opus"}, @@ -93,9 +89,7 @@ async def test_insufficient_credits_returns_402( assert resp2.status_code == 402 assert "Insufficient credits" in resp2.json()["detail"] - async def test_byok_skips_credit_check( - self, client: httpx.AsyncClient - ) -> None: + async def test_byok_skips_credit_check(self, client: httpx.AsyncClient) -> None: response = await client.post( "/api/v1/investigate", json={"prompt": "BYOK test", "director_tier": "opus"}, @@ -106,9 +100,7 @@ async def test_byok_skips_credit_check( balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 5 - async def test_invalid_tier_returns_400( - self, client: httpx.AsyncClient - ) -> None: + async def test_invalid_tier_returns_400(self, client: httpx.AsyncClient) -> None: response = await client.post( "/api/v1/investigate", json={"prompt": "Bad tier", "director_tier": "gpt4"}, diff --git a/server/tests/api/test_investigation.py b/server/tests/api/test_investigation.py index 4b35ed5..180e06f 100644 --- a/server/tests/api/test_investigation.py +++ b/server/tests/api/test_investigation.py @@ -65,9 +65,7 @@ async def test_empty_list(self, client: httpx.AsyncClient) -> None: assert response.status_code == 200 assert response.json() == [] - async def test_returns_created_investigations( - self, client: httpx.AsyncClient - ) -> None: + async def test_returns_created_investigations(self, client: httpx.AsyncClient) -> None: await client.post( "/api/v1/investigate", json={"prompt": "Test 1", "director_tier": "haiku"}, diff --git a/server/tests/impact/test_evaluation_standards.py b/server/tests/impact/test_evaluation_standards.py index 6121f54..f3a1f35 100644 --- a/server/tests/impact/test_evaluation_standards.py +++ b/server/tests/impact/test_evaluation_standards.py @@ -1,26 +1,13 @@ -"""Tests for impact evaluation evidence standards.""" +"""Tests for impact-specific evaluation standards (CONEVAL, CREMAA).""" from __future__ import annotations -from ehrlich.impact.domain.entities import ThreatToValidity from ehrlich.impact.domain.evaluation_standards import ( CONEVAL_MIR_LEVELS, CREMAA_CRITERIA, - WWC_TIERS, - classify_evidence_tier, ) -class TestWWCTiers: - def test_all_tiers_present(self) -> None: - assert set(WWC_TIERS.keys()) == {"strong", "moderate", "promising", "rationale"} - - def test_tiers_have_descriptions(self) -> None: - for tier, desc in WWC_TIERS.items(): - assert isinstance(desc, str) - assert len(desc) > 10, f"Tier {tier} has short description" - - class TestCONEVALLevels: def test_all_levels_present(self) -> None: assert set(CONEVAL_MIR_LEVELS.keys()) == {"fin", "proposito", "componente", "actividad"} @@ -37,84 +24,3 @@ def test_all_criteria_present(self) -> None: "aportacion_marginal", } assert set(CREMAA_CRITERIA.keys()) == expected - - -class TestClassifyEvidenceTier: - def test_rct_no_threats(self) -> None: - assert classify_evidence_tier("rct", []) == "strong" - - def test_rct_with_severe_threat(self) -> None: - threats = [ - ThreatToValidity( - type="attrition", - severity="high", - description="30% dropout", - mitigation="ITT analysis", - ) - ] - assert classify_evidence_tier("rct", threats) == "moderate" - - def test_rct_with_two_severe_threats(self) -> None: - threats = [ - ThreatToValidity( - type="attrition", severity="high", description="30% dropout", mitigation="ITT" - ), - ThreatToValidity( - type="contamination", severity="high", description="Cross-group", mitigation="None" - ), - ] - assert classify_evidence_tier("rct", threats) == "moderate" - - def test_did_no_threats(self) -> None: - assert classify_evidence_tier("did", []) == "moderate" - - def test_did_alias(self) -> None: - assert classify_evidence_tier("difference_in_differences", []) == "moderate" - - def test_psm_no_threats(self) -> None: - assert classify_evidence_tier("psm", []) == "moderate" - - def test_psm_with_severe_threats(self) -> None: - threats = [ - ThreatToValidity( - type="selection", severity="high", description="Unobserved", mitigation="None" - ), - ThreatToValidity( - type="specification", severity="high", description="Wrong model", mitigation="None" - ), - ] - assert classify_evidence_tier("psm", threats) == "promising" - - def test_rdd_no_threats(self) -> None: - assert classify_evidence_tier("rdd", []) == "moderate" - - def test_synthetic_control(self) -> None: - assert classify_evidence_tier("synthetic_control", []) == "moderate" - - def test_iv_no_threats(self) -> None: - assert classify_evidence_tier("iv", []) == "promising" - - def test_iv_with_severe_threat(self) -> None: - threats = [ - ThreatToValidity( - type="weak_instrument", severity="high", description="F<10", mitigation="None" - ) - ] - assert classify_evidence_tier("iv", threats) == "rationale" - - def test_fixed_effects(self) -> None: - assert classify_evidence_tier("fixed_effects", []) == "promising" - - def test_descriptive_method(self) -> None: - assert classify_evidence_tier("descriptive", []) == "rationale" - - def test_unknown_method(self) -> None: - assert classify_evidence_tier("survey", []) == "rationale" - - def test_low_severity_threats_not_counted(self) -> None: - threats = [ - ThreatToValidity( - type="attrition", severity="low", description="5% dropout", mitigation="Robust SE" - ) - ] - assert classify_evidence_tier("rct", threats) == "strong" diff --git a/server/tests/impact/test_tools_did.py b/server/tests/impact/test_tools_did.py deleted file mode 100644 index 13f7ea9..0000000 --- a/server/tests/impact/test_tools_did.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tests for DiD and threat assessment tool functions.""" - -from __future__ import annotations - -import json - -import pytest - -from ehrlich.impact.tools import assess_threats, estimate_did - - -class TestEstimateDidTool: - @pytest.mark.anyio - async def test_valid_input(self) -> None: - result = json.loads(await estimate_did( - treatment_pre="[80.0, 81.0, 79.5, 80.5, 80.0]", - treatment_post="[86.0, 87.0, 85.5, 86.5, 86.0]", - control_pre="[80.0, 81.0, 79.5, 80.5, 80.0]", - control_post="[81.0, 82.0, 80.5, 81.5, 81.0]", - )) - - assert result["method"] == "difference_in_differences" - assert "effect_size" in result - assert "p_value" in result - assert "confidence_interval" in result - assert len(result["confidence_interval"]) == 2 - assert "threats" in result - assert "evidence_tier" in result - assert isinstance(result["assumptions"], list) - - @pytest.mark.anyio - async def test_invalid_json(self) -> None: - result = json.loads(await estimate_did( - treatment_pre="not json", - treatment_post="[1, 2]", - control_pre="[1, 2]", - control_post="[1, 2]", - )) - assert "error" in result - - @pytest.mark.anyio - async def test_empty_arrays(self) -> None: - result = json.loads(await estimate_did( - treatment_pre="[]", - treatment_post="[1, 2]", - control_pre="[1, 2]", - control_post="[1, 2]", - )) - assert "error" in result - - @pytest.mark.anyio - async def test_identical_groups(self) -> None: - result = json.loads(await estimate_did( - treatment_pre="[10, 11, 10.5]", - treatment_post="[10, 11, 10.5]", - control_pre="[10, 11, 10.5]", - control_post="[10, 11, 10.5]", - )) - assert abs(result["effect_size"]) < 0.01 - - -class TestAssessThreatsTool: - @pytest.mark.anyio - async def test_did_threats(self) -> None: - result = json.loads(await assess_threats( - method="did", - sample_sizes='{"treatment": 50, "control": 45}', - parallel_trends_p=0.03, - effect_size=0.5, - )) - - assert result["method"] == "did" - assert isinstance(result["threats"], list) - threat_types = [t["type"] for t in result["threats"]] - assert "parallel_trends_violation" in threat_types - - @pytest.mark.anyio - async def test_small_sample(self) -> None: - result = json.loads(await assess_threats( - method="rct", - sample_sizes='{"treatment": 3, "control": 4}', - )) - - threat_types = [t["type"] for t in result["threats"]] - assert "small_sample" in threat_types - - @pytest.mark.anyio - async def test_invalid_json(self) -> None: - result = json.loads(await assess_threats( - method="did", - sample_sizes="not json", - )) - assert "error" in result - - @pytest.mark.anyio - async def test_psm_threats(self) -> None: - result = json.loads(await assess_threats( - method="psm", - sample_sizes='{"treatment": 100, "control": 100}', - )) - - threat_types = [t["type"] for t in result["threats"]] - assert "unobserved_confounders" in threat_types - - @pytest.mark.anyio - async def test_rdd_threats(self) -> None: - result = json.loads(await assess_threats( - method="rdd", - sample_sizes='{"treatment": 100, "control": 100}', - )) - - threat_types = [t["type"] for t in result["threats"]] - assert "manipulation" in threat_types - - @pytest.mark.anyio - async def test_large_effect_threat(self) -> None: - result = json.loads(await assess_threats( - method="rct", - sample_sizes='{"treatment": 100, "control": 100}', - effect_size=3.5, - )) - - threat_types = [t["type"] for t in result["threats"]] - assert "implausible_effect" in threat_types diff --git a/server/tests/investigation/test_repository.py b/server/tests/investigation/test_repository.py index 92e2b83..bbd488a 100644 --- a/server/tests/investigation/test_repository.py +++ b/server/tests/investigation/test_repository.py @@ -27,9 +27,7 @@ @pytest.fixture async def repo() -> InvestigationRepository: # Ensure test database exists - conn = await asyncpg.connect( - "postgresql://postgres:postgres@localhost:5432/postgres" - ) + conn = await asyncpg.connect("postgresql://postgres:postgres@localhost:5432/postgres") with contextlib.suppress(asyncpg.DuplicateDatabaseError): await conn.execute("CREATE DATABASE ehrlich_test") await conn.close() @@ -92,9 +90,7 @@ def _make_finding(hypothesis_id: str = "") -> Finding: class TestSaveAndGetById: - async def test_roundtrip_minimal( - self, repo: InvestigationRepository - ) -> None: + async def test_roundtrip_minimal(self, repo: InvestigationRepository) -> None: inv = _make_investigation() await repo.save(inv) @@ -104,9 +100,7 @@ async def test_roundtrip_minimal( assert loaded.prompt == inv.prompt assert loaded.status == InvestigationStatus.PENDING - async def test_roundtrip_with_entities( - self, repo: InvestigationRepository - ) -> None: + async def test_roundtrip_with_entities(self, repo: InvestigationRepository) -> None: inv = _make_investigation() hyp = _make_hypothesis() inv.add_hypothesis(hyp) @@ -214,15 +208,11 @@ async def test_roundtrip_with_entities( assert loaded.domain == "molecular" assert loaded.cost_data == {"total_cost_usd": 0.05} - async def test_get_nonexistent_returns_none( - self, repo: InvestigationRepository - ) -> None: + async def test_get_nonexistent_returns_none(self, repo: InvestigationRepository) -> None: result = await repo.get_by_id("nonexistent-id") assert result is None - async def test_save_with_user_id( - self, repo: InvestigationRepository - ) -> None: + async def test_save_with_user_id(self, repo: InvestigationRepository) -> None: user = await repo.get_or_create_user("workos_abc", "test@example.com") inv = _make_investigation() await repo.save(inv, user_id=str(user["id"])) @@ -237,9 +227,7 @@ async def test_empty(self, repo: InvestigationRepository) -> None: result = await repo.list_all() assert result == [] - async def test_ordered_by_created_at_desc( - self, repo: InvestigationRepository - ) -> None: + async def test_ordered_by_created_at_desc(self, repo: InvestigationRepository) -> None: import asyncio inv1 = _make_investigation(prompt="First") @@ -255,9 +243,7 @@ async def test_ordered_by_created_at_desc( class TestListByUser: - async def test_filters_by_user( - self, repo: InvestigationRepository - ) -> None: + async def test_filters_by_user(self, repo: InvestigationRepository) -> None: user_a = await repo.get_or_create_user("workos_a", "a@test.com") user_b = await repo.get_or_create_user("workos_b", "b@test.com") @@ -272,9 +258,7 @@ async def test_filters_by_user( class TestUpdate: - async def test_modify_and_reread( - self, repo: InvestigationRepository - ) -> None: + async def test_modify_and_reread(self, repo: InvestigationRepository) -> None: inv = _make_investigation() await repo.save(inv) @@ -292,9 +276,7 @@ async def test_modify_and_reread( assert loaded.iteration == 2 assert len(loaded.hypotheses) == 1 - async def test_update_triggers_fts_on_completed( - self, repo: InvestigationRepository - ) -> None: + async def test_update_triggers_fts_on_completed(self, repo: InvestigationRepository) -> None: inv = _make_investigation(prompt="antimicrobial resistance") hyp = _make_hypothesis(statement="Beta-lactam compounds inhibit PBP2a") inv.add_hypothesis(hyp) @@ -321,9 +303,7 @@ async def test_update_triggers_fts_on_completed( class TestEvents: - async def test_save_and_get_events( - self, repo: InvestigationRepository - ) -> None: + async def test_save_and_get_events(self, repo: InvestigationRepository) -> None: inv = _make_investigation() await repo.save(inv) @@ -337,9 +317,7 @@ async def test_save_and_get_events( assert events[0]["event_type"] == "hypothesis_formulated" assert events[1]["event_type"] == "tool_called" - async def test_events_ordered_by_id( - self, repo: InvestigationRepository - ) -> None: + async def test_events_ordered_by_id(self, repo: InvestigationRepository) -> None: inv = _make_investigation() await repo.save(inv) @@ -351,9 +329,7 @@ async def test_events_ordered_by_id( for i, ev in enumerate(events): assert ev["event_type"] == f"event_{i}" - async def test_events_empty_for_nonexistent( - self, repo: InvestigationRepository - ) -> None: + async def test_events_empty_for_nonexistent(self, repo: InvestigationRepository) -> None: inv = _make_investigation() await repo.save(inv) events = await repo.get_events(inv.id) @@ -361,9 +337,7 @@ async def test_events_empty_for_nonexistent( class TestSearchFindings: - async def test_search_returns_matching_findings( - self, repo: InvestigationRepository - ) -> None: + async def test_search_returns_matching_findings(self, repo: InvestigationRepository) -> None: inv = _make_investigation(prompt="Find MRSA compounds") hyp = _make_hypothesis(statement="Vancomycin analogs effective against MRSA") inv.add_hypothesis(hyp) @@ -392,15 +366,11 @@ async def test_search_returns_matching_findings( assert hit["source_type"] == "chembl" assert hit["hypothesis_statement"] == "Vancomycin analogs effective against MRSA" - async def test_search_no_results( - self, repo: InvestigationRepository - ) -> None: + async def test_search_no_results(self, repo: InvestigationRepository) -> None: results = await repo.search_findings("xyzzyx_nonexistent_term") assert results == [] - async def test_search_respects_limit( - self, repo: InvestigationRepository - ) -> None: + async def test_search_respects_limit(self, repo: InvestigationRepository) -> None: # Create 3 investigations with matching findings for i in range(3): inv = _make_investigation(prompt=f"Penicillin research {i}") @@ -423,49 +393,37 @@ async def test_search_respects_limit( class TestGetOrCreateUser: - async def test_creates_new_user( - self, repo: InvestigationRepository - ) -> None: + async def test_creates_new_user(self, repo: InvestigationRepository) -> None: user = await repo.get_or_create_user("workos_new", "new@test.com") assert user["workos_id"] == "workos_new" assert user["email"] == "new@test.com" assert user["credits"] == 5 # default - async def test_returns_existing_user( - self, repo: InvestigationRepository - ) -> None: + async def test_returns_existing_user(self, repo: InvestigationRepository) -> None: user1 = await repo.get_or_create_user("workos_exist", "exist@test.com") user2 = await repo.get_or_create_user("workos_exist", "exist@test.com") assert user1["id"] == user2["id"] assert user1["credits"] == user2["credits"] - async def test_idempotent( - self, repo: InvestigationRepository - ) -> None: + async def test_idempotent(self, repo: InvestigationRepository) -> None: for _ in range(3): user = await repo.get_or_create_user("workos_idem", "idem@test.com") assert user["workos_id"] == "workos_idem" class TestGetUserCredits: - async def test_default_credits( - self, repo: InvestigationRepository - ) -> None: + async def test_default_credits(self, repo: InvestigationRepository) -> None: await repo.get_or_create_user("workos_cred", "cred@test.com") credits = await repo.get_user_credits("workos_cred") assert credits == 5 - async def test_nonexistent_user_returns_zero( - self, repo: InvestigationRepository - ) -> None: + async def test_nonexistent_user_returns_zero(self, repo: InvestigationRepository) -> None: credits = await repo.get_user_credits("workos_nonexist") assert credits == 0 class TestDeductCredits: - async def test_successful_deduction( - self, repo: InvestigationRepository - ) -> None: + async def test_successful_deduction(self, repo: InvestigationRepository) -> None: await repo.get_or_create_user("workos_ded", "ded@test.com") inv = _make_investigation() await repo.save(inv) @@ -476,9 +434,7 @@ async def test_successful_deduction( credits = await repo.get_user_credits("workos_ded") assert credits == 2 - async def test_insufficient_balance( - self, repo: InvestigationRepository - ) -> None: + async def test_insufficient_balance(self, repo: InvestigationRepository) -> None: await repo.get_or_create_user("workos_insuf", "insuf@test.com") inv = _make_investigation() await repo.save(inv) @@ -490,9 +446,7 @@ async def test_insufficient_balance( credits = await repo.get_user_credits("workos_insuf") assert credits == 5 - async def test_exact_balance_deduction( - self, repo: InvestigationRepository - ) -> None: + async def test_exact_balance_deduction(self, repo: InvestigationRepository) -> None: await repo.get_or_create_user("workos_exact", "exact@test.com") inv = _make_investigation() await repo.save(inv) @@ -503,9 +457,7 @@ async def test_exact_balance_deduction( credits = await repo.get_user_credits("workos_exact") assert credits == 0 - async def test_records_transaction( - self, repo: InvestigationRepository - ) -> None: + async def test_records_transaction(self, repo: InvestigationRepository) -> None: user = await repo.get_or_create_user("workos_txn", "txn@test.com") inv = _make_investigation() await repo.save(inv) @@ -525,9 +477,7 @@ async def test_records_transaction( class TestRefundCredits: - async def test_successful_refund( - self, repo: InvestigationRepository - ) -> None: + async def test_successful_refund(self, repo: InvestigationRepository) -> None: await repo.get_or_create_user("workos_ref", "ref@test.com") inv = _make_investigation() await repo.save(inv) @@ -538,9 +488,7 @@ async def test_successful_refund( await repo.refund_credits("workos_ref", 3, inv.id) assert await repo.get_user_credits("workos_ref") == 5 - async def test_records_refund_transaction( - self, repo: InvestigationRepository - ) -> None: + async def test_records_refund_transaction(self, repo: InvestigationRepository) -> None: user = await repo.get_or_create_user("workos_reftx", "reftx@test.com") inv = _make_investigation() await repo.save(inv) @@ -560,9 +508,7 @@ async def test_records_refund_transaction( assert rows[1]["type"] == "refund" assert rows[1]["amount"] == 2 - async def test_refund_nonexistent_user( - self, repo: InvestigationRepository - ) -> None: + async def test_refund_nonexistent_user(self, repo: InvestigationRepository) -> None: # Should not raise -- just a no-op await repo.refund_credits("workos_nonexist", 5, "inv-id") diff --git a/server/tests/investigation/test_tools_viz_impact.py b/server/tests/investigation/test_tools_viz_impact.py index 4971c7b..73407e1 100644 --- a/server/tests/investigation/test_tools_viz_impact.py +++ b/server/tests/investigation/test_tools_viz_impact.py @@ -16,15 +16,27 @@ class TestRenderProgramDashboard: @pytest.mark.anyio async def test_basic(self) -> None: - result = json.loads(await render_program_dashboard( - indicators=[ - {"name": "Enrollment", "baseline": 70.0, "target": 90.0, - "actual": 85.0, "unit": "%"}, - {"name": "Attendance", "baseline": 60.0, "target": 80.0, - "actual": 35.0, "unit": "%"}, - ], - program_name="Prospera", - )) + result = json.loads( + await render_program_dashboard( + indicators=[ + { + "name": "Enrollment", + "baseline": 70.0, + "target": 90.0, + "actual": 85.0, + "unit": "%", + }, + { + "name": "Attendance", + "baseline": 60.0, + "target": 80.0, + "actual": 35.0, + "unit": "%", + }, + ], + program_name="Prospera", + ) + ) assert result["viz_type"] == "program_dashboard" assert result["data"]["program_name"] == "Prospera" assert len(result["data"]["indicators"]) == 2 @@ -39,25 +51,29 @@ async def test_empty_indicators(self) -> None: @pytest.mark.anyio async def test_custom_title(self) -> None: - result = json.loads(await render_program_dashboard( - indicators=[{"name": "X", "baseline": 0, "target": 10, "actual": 7, "unit": "pts"}], - title="Custom Title", - )) + result = json.loads( + await render_program_dashboard( + indicators=[{"name": "X", "baseline": 0, "target": 10, "actual": 7, "unit": "pts"}], + title="Custom Title", + ) + ) assert result["title"] == "Custom Title" class TestRenderGeographicComparison: @pytest.mark.anyio async def test_basic(self) -> None: - result = json.loads(await render_geographic_comparison( - regions=[ - {"name": "Mexico", "value": 85.2}, - {"name": "Brazil", "value": 92.1}, - {"name": "Colombia", "value": 78.5}, - ], - metric_name="Enrollment Rate", - benchmark=88.0, - )) + result = json.loads( + await render_geographic_comparison( + regions=[ + {"name": "Mexico", "value": 85.2}, + {"name": "Brazil", "value": 92.1}, + {"name": "Colombia", "value": 78.5}, + ], + metric_name="Enrollment Rate", + benchmark=88.0, + ) + ) assert result["viz_type"] == "geographic_comparison" assert len(result["data"]["regions"]) == 3 assert result["data"]["benchmark"] == 88.0 @@ -65,9 +81,11 @@ async def test_basic(self) -> None: @pytest.mark.anyio async def test_no_benchmark(self) -> None: - result = json.loads(await render_geographic_comparison( - regions=[{"name": "A", "value": 10.0}], - )) + result = json.loads( + await render_geographic_comparison( + regions=[{"name": "A", "value": 10.0}], + ) + ) assert result["data"]["benchmark"] is None @pytest.mark.anyio @@ -79,21 +97,23 @@ async def test_empty_regions(self) -> None: class TestRenderParallelTrends: @pytest.mark.anyio async def test_basic(self) -> None: - result = json.loads(await render_parallel_trends( - treatment_series=[ - {"period": "2010", "value": 70.0}, - {"period": "2011", "value": 72.0}, - {"period": "2012", "value": 80.0}, - {"period": "2013", "value": 85.0}, - ], - control_series=[ - {"period": "2010", "value": 70.0}, - {"period": "2011", "value": 71.0}, - {"period": "2012", "value": 72.0}, - {"period": "2013", "value": 73.0}, - ], - treatment_start="2012", - )) + result = json.loads( + await render_parallel_trends( + treatment_series=[ + {"period": "2010", "value": 70.0}, + {"period": "2011", "value": 72.0}, + {"period": "2012", "value": 80.0}, + {"period": "2013", "value": 85.0}, + ], + control_series=[ + {"period": "2010", "value": 70.0}, + {"period": "2011", "value": 71.0}, + {"period": "2012", "value": 72.0}, + {"period": "2013", "value": 73.0}, + ], + treatment_start="2012", + ) + ) assert result["viz_type"] == "parallel_trends" assert len(result["data"]["treatment"]) == 4 assert len(result["data"]["control"]) == 4 @@ -101,9 +121,11 @@ async def test_basic(self) -> None: @pytest.mark.anyio async def test_empty_series(self) -> None: - result = json.loads(await render_parallel_trends( - treatment_series=[], - control_series=[], - )) + result = json.loads( + await render_parallel_trends( + treatment_series=[], + control_series=[], + ) + ) assert result["data"]["treatment"] == [] assert result["data"]["control"] == [] diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 0ba654e..7362f24 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -1,10 +1,10 @@ export const STATS = { - tools: 78, + tools: 84, dataSources: 19, domains: 4, models: 3, phases: 6, - vizTools: 15, + vizTools: 17, externalAPIs: 18, boundedContexts: 11, sseEvents: 20, @@ -46,7 +46,7 @@ export const EXAMPLE_PROMPTS = [ export const DIFFERENTIATORS = [ { label: "Real Computation", - tagline: "78 tools that compute, not summarize.", + tagline: "84 tools that compute, not summarize.", description: "Ehrlich trains ML models, runs molecular docking, executes statistical tests, and validates with negative controls. Every tool returns structured data from real computation or real APIs.", capabilities: [ @@ -61,7 +61,7 @@ export const DIFFERENTIATORS = [ label: "Accessible Pricing", tagline: "$2-10 per investigation. Free tier included.", description: - "Full 6-phase methodology, all 78 tools, all 19 data sources at every tier. No feature gates. A student in Mexico and a pharma lab in Boston use the same product. The only variable is model quality.", + "Full 6-phase methodology, all 84 tools, all 19 data sources at every tier. No feature gates. A student in Mexico and a pharma lab in Boston use the same product. The only variable is model quality.", capabilities: [ "4 domains, extensible to any", "Free: 3 Haiku investigations/month", @@ -201,7 +201,7 @@ export const METHODOLOGY_PHASES = [ number: "04", label: "Experiment Execution", foundation: "Fisher (1935)", - description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 78 tools across 4 domains.", + description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 84 tools across 4 domains.", }, { number: "05", @@ -311,10 +311,10 @@ export const PRICING_TIERS = [ price: "$0", period: "forever", credits: "3 Haiku investigations/month", - description: "Full methodology. All 78 tools. All 19 data sources. Findings indexed for future research.", + description: "Full methodology. All 84 tools. All 19 data sources. Findings indexed for future research.", features: [ "Full 6-phase scientific methodology", - "All 78 tools, all 19 data sources, 4 domains", + "All 84 tools, all 19 data sources, 4 domains", "Full audit trail and report", "Self-referential search (FTS5)", "No feature gates", From 23aeebdae43e4bf075ff430fcb67e666fda75067 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 05:14:29 -0700 Subject: [PATCH 06/35] refactor(investigation): decompose multi_orchestrator.py into focused modules Structural refactoring with zero behavior change. All existing tests pass unchanged. Multi_orchestrator.py dropped from 1783 lines to 1216 lines. Extracted modules (718 new lines total): - diagram_builder.py (139 lines): Excalidraw evidence synthesis diagram - tool_dispatcher.py (155 lines): Tool execution with caching + special handlers - researcher_executor.py (329 lines): Single researcher experiment executor - batch_executor.py (95 lines): Parallel batch experiment executor (2 concurrent) What stays in multi_orchestrator.py: - Main 6-phase orchestration loop (run method) - Director calls (_director_call, _build_literature_context) - Literature survey (_run_literature_survey) - Hypothesis approval gate - Module-level constants (_COMPACT_SCHEMAS, SEARCH_TOOLS, _compact_result) Quality gates: - ruff check: pass - mypy: pass - pytest tests/investigation/test_multi_orchestrator.py: 20/20 pass - pytest tests/investigation/: 183/183 pass (excl. DB-dependent) Updated CLAUDE.md to document the 4 new modules. --- CLAUDE.md | 6 +- .../application/batch_executor.py | 95 +++ .../application/diagram_builder.py | 139 +++++ .../application/multi_orchestrator.py | 558 ++---------------- .../application/researcher_executor.py | 329 +++++++++++ .../application/tool_dispatcher.py | 155 +++++ 6 files changed, 773 insertions(+), 509 deletions(-) create mode 100644 server/src/ehrlich/investigation/application/batch_executor.py create mode 100644 server/src/ehrlich/investigation/application/diagram_builder.py create mode 100644 server/src/ehrlich/investigation/application/researcher_executor.py create mode 100644 server/src/ehrlich/investigation/application/tool_dispatcher.py diff --git a/CLAUDE.md b/CLAUDE.md index a2d9b2a..53bcc50 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -423,7 +423,11 @@ All paths relative to `server/src/ehrlich/`. | File | Purpose | |------|---------| -| `application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator | +| `application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop) | +| `application/diagram_builder.py` | Excalidraw evidence synthesis diagram generation | +| `application/tool_dispatcher.py` | Tool execution dispatcher with caching and special handlers | +| `application/researcher_executor.py` | Single researcher experiment executor with tool loop | +| `application/batch_executor.py` | Parallel batch experiment executor (2 concurrent researchers) | | `application/cost_tracker.py` | Per-model cost tracking with tiered pricing | | `application/tool_cache.py` | In-memory TTL cache for tool results | | `application/tool_registry.py` | `ToolRegistry` with domain tag filtering | diff --git a/server/src/ehrlich/investigation/application/batch_executor.py b/server/src/ehrlich/investigation/application/batch_executor.py new file mode 100644 index 0000000..bcd6735 --- /dev/null +++ b/server/src/ehrlich/investigation/application/batch_executor.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.application.researcher_executor import run_researcher_experiment + +if TYPE_CHECKING: + import asyncio + from collections.abc import AsyncGenerator + + from ehrlich.investigation.application.cost_tracker import CostTracker + from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher + from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.domain.domain_config import DomainConfig + from ehrlich.investigation.domain.events import DomainEvent + from ehrlich.investigation.domain.experiment import Experiment + from ehrlich.investigation.domain.hypothesis import Hypothesis + from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter + +logger = logging.getLogger(__name__) + + +async def run_experiment_batch( + researcher: AnthropicClientAdapter, + summarizer: AnthropicClientAdapter, + dispatcher: ToolDispatcher, + registry: ToolRegistry, + active_config: DomainConfig | None, + researcher_prompt: str, + summarizer_threshold: int, + max_iterations: int, + investigation: Investigation, + batch: list[tuple[Hypothesis, Experiment, dict[str, Any]]], + cost: CostTracker, + state_lock: asyncio.Lock, +) -> AsyncGenerator[DomainEvent, None]: + """Run up to 2 experiments concurrently with sibling awareness.""" + if len(batch) == 1: + h, exp, design = batch[0] + async for event in run_researcher_experiment( + researcher, summarizer, dispatcher, registry, active_config, researcher_prompt, + summarizer_threshold, max_iterations, investigation, h, exp, cost, design, state_lock, + ): + yield event + return + + # Build sibling context strings so each researcher knows what the other is doing + sibling_contexts: list[str] = [] + for i, (_hyp, _exp, _design) in enumerate(batch): + other_idx = 1 - i + other_hyp, other_exp, other_design = batch[other_idx] + sibling_contexts.append( + f"Hypothesis: {other_hyp.statement}\n" + f"Experiment: {other_exp.description}\n" + f"Tools: {', '.join(other_design.get('tool_plan', []))}" + ) + + import asyncio + + queue: asyncio.Queue[DomainEvent | None] = asyncio.Queue() + + async def _run_one( + hyp: Hypothesis, + exp: Experiment, + design: dict[str, Any], + sib_ctx: str, + ) -> None: + try: + async for ev in run_researcher_experiment( + researcher, summarizer, dispatcher, registry, active_config, researcher_prompt, + summarizer_threshold, max_iterations, investigation, hyp, exp, cost, design, + state_lock, sibling_context=sib_ctx, + ): + await queue.put(ev) + except Exception as e: + logger.warning("Experiment %s failed: %s", exp.id, e) + finally: + await queue.put(None) + + tasks = [ + asyncio.create_task(_run_one(h, e, d, sc)) + for (h, e, d), sc in zip(batch, sibling_contexts, strict=True) + ] + + done_count = 0 + while done_count < len(tasks): + item: DomainEvent | None = await queue.get() + if item is None: + done_count += 1 + continue + yield item + + await asyncio.gather(*tasks, return_exceptions=True) diff --git a/server/src/ehrlich/investigation/application/diagram_builder.py b/server/src/ehrlich/investigation/application/diagram_builder.py new file mode 100644 index 0000000..93c3217 --- /dev/null +++ b/server/src/ehrlich/investigation/application/diagram_builder.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.infrastructure.mcp_bridge import MCPBridge + +logger = logging.getLogger(__name__) + + +async def generate_diagram(mcp_bridge: MCPBridge, investigation: Investigation) -> str: + """Generate an Excalidraw evidence synthesis diagram and return its URL.""" + elements = _build_excalidraw_elements(investigation) + elements_json = json.dumps(elements) + try: + await mcp_bridge.call_tool("excalidraw:read_me", {}) + await mcp_bridge.call_tool("excalidraw:create_view", {"elements": elements_json}) + export_result = await mcp_bridge.call_tool( + "excalidraw:export_to_excalidraw", + {"json": json.dumps({"elements": elements, "appState": {}})}, + ) + data = json.loads(export_result) + url: str = data.get("url", "") + if url: + logger.info("Diagram exported: %s", url) + return url + except Exception: + logger.warning("Diagram generation failed", exc_info=True) + return "" + + +def _build_excalidraw_elements(investigation: Investigation) -> list[dict[str, Any]]: + """Build Excalidraw elements representing the investigation evidence map.""" + elements: list[dict[str, Any]] = [] + y_offset = 0 + + # Title + elements.append( + { + "type": "text", + "x": 300, + "y": y_offset, + "width": 600, + "height": 40, + "text": f"Evidence Synthesis: {investigation.prompt[:80]}", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "center", + "strokeColor": "#e2e8f0", + "id": "title", + } + ) + y_offset += 80 + + # Hypotheses + status_colors = { + "supported": "#166534", + "refuted": "#991b1b", + "revised": "#9a3412", + "proposed": "#374151", + "testing": "#1e40af", + "rejected": "#7f1d1d", + } + hyp_positions: dict[str, tuple[int, int]] = {} + for i, h in enumerate(investigation.hypotheses): + x = 50 + (i % 3) * 350 + y = y_offset + (i // 3) * 160 + color = status_colors.get(h.status.value, "#374151") + elements.append( + { + "type": "rectangle", + "x": x, + "y": y, + "width": 300, + "height": 120, + "backgroundColor": color, + "strokeColor": "#94a3b8", + "roundness": {"type": 3}, + "id": f"hyp-{h.id}", + } + ) + label = f"{h.status.value.upper()}\n{h.statement[:60]}" + if h.confidence > 0: + label += f"\nConf: {h.confidence:.0%}" + elements.append( + { + "type": "text", + "x": x + 10, + "y": y + 10, + "width": 280, + "height": 100, + "text": label, + "fontSize": 14, + "fontFamily": 1, + "strokeColor": "#e2e8f0", + "id": f"hyp-label-{h.id}", + } + ) + hyp_positions[h.id] = (x + 150, y + 120) + + y_offset += ((len(investigation.hypotheses) + 2) // 3) * 160 + 40 + + # Findings summary + if investigation.findings: + elements.append( + { + "type": "text", + "x": 50, + "y": y_offset, + "width": 400, + "height": 30, + "text": f"Findings: {len(investigation.findings)} recorded", + "fontSize": 18, + "fontFamily": 1, + "strokeColor": "#94a3b8", + "id": "findings-header", + } + ) + y_offset += 50 + for j, f in enumerate(investigation.findings[:12]): + elements.append( + { + "type": "text", + "x": 60, + "y": y_offset + j * 25, + "width": 900, + "height": 20, + "text": f"[{f.evidence_type}] {f.title[:90]}", + "fontSize": 12, + "fontFamily": 3, + "strokeColor": "#94a3b8", + "id": f"finding-{j}", + } + ) + + return elements diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index 26826bf..4c60a2d 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -6,14 +6,15 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from ehrlich.investigation.application.batch_executor import run_experiment_batch from ehrlich.investigation.application.cost_tracker import CostTracker +from ehrlich.investigation.application.diagram_builder import generate_diagram from ehrlich.investigation.application.prompts import ( DIRECTOR_EVALUATION_PROMPT, DIRECTOR_EXPERIMENT_PROMPT, DIRECTOR_FORMULATION_PROMPT, DIRECTOR_SYNTHESIS_PROMPT, RESEARCHER_EXPERIMENT_PROMPT, - SUMMARIZER_PROMPT, build_experiment_prompt, build_formulation_prompt, build_literature_assessment_prompt, @@ -22,8 +23,14 @@ build_pico_and_classification_prompt, build_researcher_prompt, build_synthesis_prompt, + build_uploaded_data_context, +) +from ehrlich.investigation.application.researcher_executor import ( + maybe_viz_event, + summarize_output, ) from ehrlich.investigation.application.tool_cache import ToolCache +from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher from ehrlich.investigation.domain.candidate import Candidate from ehrlich.investigation.domain.domain_config import merge_domain_configs from ehrlich.investigation.domain.events import ( @@ -40,14 +47,12 @@ InvestigationError, LiteratureSurveyCompleted, NegativeControlRecorded, - OutputSummarized, PhaseChanged, PositiveControlRecorded, Thinking, ToolCalled, ToolResultEvent, ValidationMetricsComputed, - VisualizationRendered, ) from ehrlich.investigation.domain.experiment import Experiment, ExperimentStatus from ehrlich.investigation.domain.finding import Finding @@ -73,6 +78,7 @@ from ehrlich.investigation.domain.domain_registry import DomainRegistry from ehrlich.investigation.domain.mcp_config import MCPServerConfig from ehrlich.investigation.domain.repository import InvestigationRepository + from ehrlich.investigation.domain.uploaded_file import UploadedFile from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter from ehrlich.investigation.infrastructure.mcp_bridge import MCPBridge @@ -160,9 +166,12 @@ def __init__( self._active_config: DomainConfig | None = None self._researcher_prompt = RESEARCHER_EXPERIMENT_PROMPT self._cache = ToolCache() + self._dispatcher = ToolDispatcher(registry, self._cache, repository, {}) self._state_lock = asyncio.Lock() self._approval_event = asyncio.Event() self._investigation: Investigation | None = None + self._uploaded_files: dict[str, UploadedFile] = {} + self._uploaded_data_context = "" def approve_hypotheses( self, @@ -193,6 +202,15 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, cost = CostTracker() pico: dict[str, Any] = {} + # Build uploaded data context for prompt injection + if investigation.uploaded_files: + self._uploaded_files = {f.file_id: f for f in investigation.uploaded_files} + self._uploaded_data_context = build_uploaded_data_context(investigation.uploaded_files) + else: + self._uploaded_files = {} + self._uploaded_data_context = "" + self._dispatcher.update_uploaded_files(self._uploaded_files) + # Connect MCP servers if configured if self._mcp_bridge and self._mcp_configs: try: @@ -219,7 +237,9 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, valid_categories: frozenset[str] = frozenset() if self._domain_registry: valid_categories = self._domain_registry.all_categories() - pico_prompt = build_pico_and_classification_prompt(valid_categories) + pico_prompt = build_pico_and_classification_prompt( + valid_categories, self._uploaded_data_context + ) pico_response = await self._summarizer.create_message( system=pico_prompt, messages=[{"role": "user", "content": investigation.prompt}], @@ -257,7 +277,9 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, if self._domain_registry: detected_configs, is_fallback = self._domain_registry.detect(domain_categories) self._active_config = merge_domain_configs(detected_configs) - self._researcher_prompt = build_researcher_prompt(self._active_config) + self._researcher_prompt = build_researcher_prompt( + self._active_config, self._uploaded_data_context + ) yield DomainDetected( domain=self._active_config.name, display_config=self._active_config.to_display_dict(), @@ -299,7 +321,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, ) prior_section = f"\n\n{prior_context}" if prior_context else "" formulation_prompt = ( - build_formulation_prompt(self._active_config) + build_formulation_prompt(self._active_config, self._uploaded_data_context) if self._active_config else DIRECTOR_FORMULATION_PROMPT ) @@ -415,7 +437,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, else: tools_csv = ", ".join(self._registry.list_tools()) experiment_prompt = ( - build_experiment_prompt(self._active_config) + build_experiment_prompt(self._active_config, self._uploaded_data_context) if self._active_config else DIRECTOR_EXPERIMENT_PROMPT ) @@ -493,7 +515,20 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, # Run batch (parallel if 2, sequential if 1) exp_tool_count = cost.tool_calls exp_finding_count = len(investigation.findings) - async for event in self._run_experiment_batch(investigation, batch, cost): + async for event in run_experiment_batch( + self._researcher, + self._summarizer, + self._dispatcher, + self._registry, + self._active_config, + self._researcher_prompt, + self._summarizer_threshold, + self._max_iterations_per_experiment, + investigation, + batch, + cost, + self._state_lock, + ): yield event # Mark completed + Director evaluates each @@ -634,7 +669,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, ] if all_identifiers: try: - pred_result = await self._dispatch_tool( + pred_result = await self._dispatcher.dispatch( "predict_candidates", {"smiles_list": all_identifiers, "model_id": model_id}, investigation, @@ -824,7 +859,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, # Post-synthesis: generate Excalidraw diagram if MCP available diagram_url = "" if self._mcp_bridge and self._mcp_bridge.has_tool("excalidraw:create_view"): - diagram_url = await self._generate_diagram(investigation) + diagram_url = await generate_diagram(self._mcp_bridge, investigation) investigation.status = InvestigationStatus.COMPLETED investigation.cost_data = cost.to_dict() @@ -917,29 +952,6 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, except Exception: logger.warning("MCP bridge disconnect failed", exc_info=True) - async def _generate_diagram(self, investigation: Investigation) -> str: - """Generate an Excalidraw evidence synthesis diagram and return its URL.""" - if not self._mcp_bridge: - return "" - - elements = _build_excalidraw_elements(investigation) - elements_json = json.dumps(elements) - try: - await self._mcp_bridge.call_tool("excalidraw:read_me", {}) - await self._mcp_bridge.call_tool("excalidraw:create_view", {"elements": elements_json}) - export_result = await self._mcp_bridge.call_tool( - "excalidraw:export_to_excalidraw", - {"json": json.dumps({"elements": elements, "appState": {}})}, - ) - data = json.loads(export_result) - url: str = data.get("url", "") - if url: - logger.info("Diagram exported: %s", url) - return url - except Exception: - logger.warning("Diagram generation failed", exc_info=True) - return "" - @staticmethod def _build_literature_context( investigation: Investigation, @@ -1002,43 +1014,6 @@ async def _director_call( yield _DirectorResult(data=json.loads(text), thinking=thinking_text) - async def _summarize_output( - self, - cost: CostTracker, - tool_name: str, - output: str, - investigation_id: str, - ) -> tuple[str, OutputSummarized | None]: - if len(output) <= self._summarizer_threshold: - return output, None - - response = await self._summarizer.create_message( - system=SUMMARIZER_PROMPT, - messages=[ - {"role": "user", "content": f"Tool: {tool_name}\nOutput:\n{output}"}, - ], - tools=[], - ) - cost.add_usage( - response.input_tokens, - response.output_tokens, - self._summarizer.model, - cache_read_tokens=response.cache_read_input_tokens, - cache_write_tokens=response.cache_write_input_tokens, - ) - summarized = "" - for block in response.content: - if block["type"] == "text": - summarized += block["text"] - - event = OutputSummarized( - tool_name=tool_name, - original_length=len(output), - summarized_length=len(summarized), - investigation_id=investigation_id, - ) - return summarized, event - async def _run_literature_survey( self, investigation: Investigation, @@ -1134,7 +1109,7 @@ async def _run_literature_survey( investigation_id=investigation.id, ) - result_str = await self._dispatch_tool(tool_name, tool_input, investigation) + result_str = await self._dispatcher.dispatch(tool_name, tool_input, investigation) result_str = _compact_result(tool_name, result_str) # Track result counts @@ -1145,8 +1120,9 @@ async def _run_literature_survey( except (json.JSONDecodeError, TypeError, ValueError): pass - summarized_str, summarize_event = await self._summarize_output( - cost, tool_name, result_str, investigation.id + summarized_str, summarize_event = await summarize_output( + self._summarizer, cost, tool_name, result_str, investigation.id, + self._summarizer_threshold, ) if summarize_event is not None: yield summarize_event @@ -1160,7 +1136,7 @@ async def _run_literature_survey( ) # Emit visualization event if tool returned viz payload - viz_event = self._maybe_viz_event(result_str, "", investigation.id) + viz_event = maybe_viz_event(result_str, "", investigation.id) if viz_event is not None: yield viz_event @@ -1238,437 +1214,3 @@ async def _run_literature_survey( investigation_id=investigation.id, ) - async def _run_researcher_experiment( - self, - investigation: Investigation, - hypothesis: Hypothesis, - experiment: Experiment, - cost: CostTracker, - design: dict[str, Any], - sibling_context: str = "", - ) -> AsyncGenerator[DomainEvent, None]: - planned = set(experiment.tool_plan) if experiment.tool_plan else set() - control_tools = {"record_finding", "record_negative_control"} - if planned: - allowed = planned | control_tools - tool_schemas = [t for t in self._registry.list_schemas() if t["name"] in allowed] - elif self._active_config: - excluded = { - "conclude_investigation", - "propose_hypothesis", - "design_experiment", - "evaluate_hypothesis", - } - domain_schemas = self._registry.list_schemas_for_domain(self._active_config.tool_tags) - tool_schemas = [t for t in domain_schemas if t["name"] not in excluded] - else: - excluded = { - "conclude_investigation", - "propose_hypothesis", - "design_experiment", - "evaluate_hypothesis", - } - tool_schemas = [t for t in self._registry.list_schemas() if t["name"] not in excluded] - - exp_controls = ", ".join(experiment.controls) or "None specified" - sibling_section = "" - if sibling_context: - sibling_section = ( - f"\n\n\n" - f"A parallel researcher is simultaneously testing a different hypothesis:\n" - f"{sibling_context}\n\n" - f"Avoid duplicating their queries. Use different search terms, " - f"data sources, or analytical approaches where your experiment " - f"design allows flexibility.\n" - f"" - ) - messages: list[dict[str, Any]] = [ - { - "role": "user", - "content": ( - f"Research prompt: {investigation.prompt}\n\n" - f"Hypothesis: {hypothesis.statement}\n" - f"Mechanism: {hypothesis.rationale}\n" - f"Prediction: {hypothesis.prediction or 'N/A'}\n" - f"Hypothesis success criteria: {hypothesis.success_criteria or 'N/A'}\n" - f"Hypothesis failure criteria: {hypothesis.failure_criteria or 'N/A'}\n" - f"Scope: {hypothesis.scope or 'N/A'}\n\n" - f"Experiment: {experiment.description}\n" - f"Planned tools: {', '.join(experiment.tool_plan)}\n" - f"Controls: {exp_controls}\n" - f"Analysis plan: {experiment.analysis_plan or 'N/A'}\n" - f"Experiment success criteria: {experiment.success_criteria or 'N/A'}\n" - f"Experiment failure criteria: {experiment.failure_criteria or 'N/A'}\n\n" - f"Execute this experiment. Compare results against the " - f"pre-defined success/failure criteria. Link all findings to " - f"hypothesis_id='{hypothesis.id}'." + sibling_section - ), - }, - ] - - for _iteration in range(self._max_iterations_per_experiment): - investigation.iteration += 1 - tool_choice = {"type": "any"} if _iteration == 0 else None - response = await self._researcher.create_message( - system=self._researcher_prompt, - messages=messages, - tools=tool_schemas, - tool_choice=tool_choice, - ) - async with self._state_lock: - cost.add_usage( - response.input_tokens, - response.output_tokens, - self._researcher.model, - cache_read_tokens=response.cache_read_input_tokens, - cache_write_tokens=response.cache_write_input_tokens, - ) - - assistant_content: list[dict[str, Any]] = [] - tool_use_blocks: list[dict[str, Any]] = [] - - for block in response.content: - assistant_content.append(block) - if block["type"] == "text": - yield Thinking( - text=block["text"], - investigation_id=investigation.id, - ) - elif block["type"] == "tool_use": - tool_use_blocks.append(block) - - messages.append({"role": "assistant", "content": assistant_content}) - - if response.stop_reason == "end_turn" or not tool_use_blocks: - break - - tool_results: list[dict[str, Any]] = [] - for tool_block in tool_use_blocks: - tool_name = tool_block["name"] - tool_input = tool_block["input"] - tool_use_id = tool_block["id"] - async with self._state_lock: - cost.add_tool_call() - - yield ToolCalled( - tool_name=tool_name, - tool_input=tool_input, - experiment_id=experiment.id, - investigation_id=investigation.id, - ) - - result_str = await self._dispatch_tool(tool_name, tool_input, investigation) - result_str = _compact_result(tool_name, result_str) - - summarized_str, summarize_event = await self._summarize_output( - cost, tool_name, result_str, investigation.id - ) - if summarize_event is not None: - yield summarize_event - - content_for_model = summarized_str if summarize_event else result_str - preview = result_str[:1500] if len(result_str) > 1500 else result_str - yield ToolResultEvent( - tool_name=tool_name, - result_preview=preview, - experiment_id=experiment.id, - investigation_id=investigation.id, - ) - - # Emit visualization event if tool returned viz payload - viz_event = self._maybe_viz_event(result_str, experiment.id, investigation.id) - if viz_event is not None: - yield viz_event - - if tool_name == "record_finding": - h_id = tool_input.get("hypothesis_id", hypothesis.id) - e_type = tool_input.get("evidence_type", "neutral") - finding = Finding( - title=tool_input.get("title", ""), - detail=tool_input.get("detail", ""), - evidence=tool_input.get("evidence", ""), - hypothesis_id=h_id, - evidence_type=e_type, - source_type=tool_input.get("source_type", ""), - source_id=tool_input.get("source_id", ""), - evidence_level=int(tool_input.get("evidence_level", 0)), - ) - async with self._state_lock: - investigation.record_finding(finding) - h = investigation.get_hypothesis(h_id) - if h: - if e_type == "supporting": - h.supporting_evidence.append(finding.title) - elif e_type == "contradicting": - h.contradicting_evidence.append(finding.title) - yield FindingRecorded( - title=finding.title, - detail=finding.detail, - hypothesis_id=h_id, - evidence_type=e_type, - evidence=finding.evidence, - source_type=finding.source_type, - source_id=finding.source_id, - evidence_level=finding.evidence_level, - investigation_id=investigation.id, - ) - - if tool_name == "train_model": - try: - train_result = json.loads(result_str) - if isinstance(train_result, dict) and "model_id" in train_result: - async with self._state_lock: - investigation.trained_model_ids.append(train_result["model_id"]) - except (json.JSONDecodeError, TypeError): - pass - - if tool_name == "record_negative_control": - control = NegativeControl( - identifier=tool_input.get("identifier", tool_input.get("smiles", "")), - identifier_type=tool_input.get("identifier_type", ""), - name=tool_input.get("name", ""), - score=float( - tool_input.get("score", tool_input.get("prediction_score", 0.0)) - ), - threshold=float(tool_input.get("threshold", 0.5)), - source=tool_input.get("source", ""), - ) - async with self._state_lock: - investigation.add_negative_control(control) - yield NegativeControlRecorded( - identifier=control.identifier, - identifier_type=control.identifier_type, - name=control.name, - score=control.score, - threshold=control.threshold, - correctly_classified=(control.correctly_classified), - investigation_id=investigation.id, - ) - - tool_results.append( - { - "type": "tool_result", - "tool_use_id": tool_use_id, - "content": content_for_model, - } - ) - - messages.append({"role": "user", "content": tool_results}) - - async def _run_experiment_batch( - self, - investigation: Investigation, - batch: list[tuple[Hypothesis, Experiment, dict[str, Any]]], - cost: CostTracker, - ) -> AsyncGenerator[DomainEvent, None]: - """Run up to 2 experiments concurrently with sibling awareness.""" - if len(batch) == 1: - h, exp, design = batch[0] - async for event in self._run_researcher_experiment(investigation, h, exp, cost, design): - yield event - return - - # Build sibling context strings so each researcher knows what the other is doing - sibling_contexts: list[str] = [] - for i, (_hyp, _exp, _design) in enumerate(batch): - other_idx = 1 - i - other_hyp, other_exp, other_design = batch[other_idx] - sibling_contexts.append( - f"Hypothesis: {other_hyp.statement}\n" - f"Experiment: {other_exp.description}\n" - f"Tools: {', '.join(other_design.get('tool_plan', []))}" - ) - - queue: asyncio.Queue[DomainEvent | None] = asyncio.Queue() - - async def _run_one( - hyp: Hypothesis, - exp: Experiment, - design: dict[str, Any], - sib_ctx: str, - ) -> None: - try: - async for ev in self._run_researcher_experiment( - investigation, hyp, exp, cost, design, sibling_context=sib_ctx - ): - await queue.put(ev) - except Exception as e: - logger.warning("Experiment %s failed: %s", exp.id, e) - finally: - await queue.put(None) - - tasks = [ - asyncio.create_task(_run_one(h, e, d, sc)) - for (h, e, d), sc in zip(batch, sibling_contexts, strict=True) - ] - - done_count = 0 - while done_count < len(tasks): - item: DomainEvent | None = await queue.get() - if item is None: - done_count += 1 - continue - yield item - - await asyncio.gather(*tasks, return_exceptions=True) - - @staticmethod - def _maybe_viz_event( - result_str: str, - experiment_id: str, - investigation_id: str, - ) -> VisualizationRendered | None: - """If a tool result contains viz_type, build a VisualizationRendered event.""" - try: - data = json.loads(result_str) - except (json.JSONDecodeError, TypeError): - return None - if not isinstance(data, dict) or "viz_type" not in data: - return None - return VisualizationRendered( - investigation_id=investigation_id, - experiment_id=experiment_id, - viz_type=data["viz_type"], - title=data.get("title", ""), - data=data.get("data", {}), - config=data.get("config", {}), - domain=data.get("config", {}).get("domain", ""), - ) - - async def _dispatch_tool( - self, - tool_name: str, - tool_input: dict[str, Any], - investigation: Investigation, - ) -> str: - # Intercept search_prior_research -- query FTS5 via repository - if tool_name == "search_prior_research" and self._repository: - query = tool_input.get("query", "") - limit = int(tool_input.get("limit", 10)) - results = await self._repository.search_findings(query, limit) - return json.dumps({"results": results, "count": len(results), "query": query}) - - args_hash = ToolCache.hash_args(tool_input) - cached = self._cache.get(tool_name, args_hash) - if cached is not None: - return cached - - func = self._registry.get(tool_name) - if func is None: - return json.dumps({"error": f"Unknown tool: {tool_name}"}) - - try: - result = await func(**tool_input) - result_str = str(result) - self._cache.put(tool_name, args_hash, result_str) - return result_str - except Exception as e: - logger.warning("Tool %s failed: %s", tool_name, e) - return json.dumps({"error": f"Tool {tool_name} failed: {e}"}) - - -def _build_excalidraw_elements(investigation: Investigation) -> list[dict[str, Any]]: - """Build Excalidraw elements representing the investigation evidence map.""" - elements: list[dict[str, Any]] = [] - y_offset = 0 - - # Title - elements.append( - { - "type": "text", - "x": 300, - "y": y_offset, - "width": 600, - "height": 40, - "text": f"Evidence Synthesis: {investigation.prompt[:80]}", - "fontSize": 24, - "fontFamily": 1, - "textAlign": "center", - "strokeColor": "#e2e8f0", - "id": "title", - } - ) - y_offset += 80 - - # Hypotheses - status_colors = { - "supported": "#166534", - "refuted": "#991b1b", - "revised": "#9a3412", - "proposed": "#374151", - "testing": "#1e40af", - "rejected": "#7f1d1d", - } - hyp_positions: dict[str, tuple[int, int]] = {} - for i, h in enumerate(investigation.hypotheses): - x = 50 + (i % 3) * 350 - y = y_offset + (i // 3) * 160 - color = status_colors.get(h.status.value, "#374151") - elements.append( - { - "type": "rectangle", - "x": x, - "y": y, - "width": 300, - "height": 120, - "backgroundColor": color, - "strokeColor": "#94a3b8", - "roundness": {"type": 3}, - "id": f"hyp-{h.id}", - } - ) - label = f"{h.status.value.upper()}\n{h.statement[:60]}" - if h.confidence > 0: - label += f"\nConf: {h.confidence:.0%}" - elements.append( - { - "type": "text", - "x": x + 10, - "y": y + 10, - "width": 280, - "height": 100, - "text": label, - "fontSize": 14, - "fontFamily": 1, - "strokeColor": "#e2e8f0", - "id": f"hyp-label-{h.id}", - } - ) - hyp_positions[h.id] = (x + 150, y + 120) - - y_offset += ((len(investigation.hypotheses) + 2) // 3) * 160 + 40 - - # Findings summary - if investigation.findings: - elements.append( - { - "type": "text", - "x": 50, - "y": y_offset, - "width": 400, - "height": 30, - "text": f"Findings: {len(investigation.findings)} recorded", - "fontSize": 18, - "fontFamily": 1, - "strokeColor": "#94a3b8", - "id": "findings-header", - } - ) - y_offset += 50 - for j, f in enumerate(investigation.findings[:12]): - elements.append( - { - "type": "text", - "x": 60, - "y": y_offset + j * 25, - "width": 900, - "height": 20, - "text": f"[{f.evidence_type}] {f.title[:90]}", - "fontSize": 12, - "fontFamily": 3, - "strokeColor": "#94a3b8", - "id": f"finding-{j}", - } - ) - - return elements diff --git a/server/src/ehrlich/investigation/application/researcher_executor.py b/server/src/ehrlich/investigation/application/researcher_executor.py new file mode 100644 index 0000000..77268ad --- /dev/null +++ b/server/src/ehrlich/investigation/application/researcher_executor.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.application.prompts import SUMMARIZER_PROMPT +from ehrlich.investigation.domain.events import ( + DomainEvent, + FindingRecorded, + NegativeControlRecorded, + OutputSummarized, + Thinking, + ToolCalled, + ToolResultEvent, + VisualizationRendered, +) +from ehrlich.investigation.domain.finding import Finding +from ehrlich.investigation.domain.negative_control import NegativeControl + +if TYPE_CHECKING: + import asyncio + from collections.abc import AsyncGenerator + + from ehrlich.investigation.application.cost_tracker import CostTracker + from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher + from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.domain.domain_config import DomainConfig + from ehrlich.investigation.domain.experiment import Experiment + from ehrlich.investigation.domain.hypothesis import Hypothesis + from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter + +logger = logging.getLogger(__name__) + + +def _compact_result(tool_name: str, result: str) -> str: + """Import and delegate to the module-level _compact_result in multi_orchestrator.""" + from ehrlich.investigation.application.multi_orchestrator import _compact_result as _cr + + return _cr(tool_name, result) + + +async def summarize_output( + summarizer: AnthropicClientAdapter, + cost: CostTracker, + tool_name: str, + output: str, + investigation_id: str, + threshold: int, +) -> tuple[str, OutputSummarized | None]: + if len(output) <= threshold: + return output, None + + response = await summarizer.create_message( + system=SUMMARIZER_PROMPT, + messages=[ + {"role": "user", "content": f"Tool: {tool_name}\nOutput:\n{output}"}, + ], + tools=[], + ) + cost.add_usage( + response.input_tokens, + response.output_tokens, + summarizer.model, + cache_read_tokens=response.cache_read_input_tokens, + cache_write_tokens=response.cache_write_input_tokens, + ) + summarized = "" + for block in response.content: + if block["type"] == "text": + summarized += block["text"] + + event = OutputSummarized( + tool_name=tool_name, + original_length=len(output), + summarized_length=len(summarized), + investigation_id=investigation_id, + ) + return summarized, event + + +def maybe_viz_event( + result_str: str, + experiment_id: str, + investigation_id: str, +) -> VisualizationRendered | None: + """If a tool result contains viz_type, build a VisualizationRendered event.""" + try: + data = json.loads(result_str) + except (json.JSONDecodeError, TypeError): + return None + if not isinstance(data, dict) or "viz_type" not in data: + return None + return VisualizationRendered( + investigation_id=investigation_id, + experiment_id=experiment_id, + viz_type=data["viz_type"], + title=data.get("title", ""), + data=data.get("data", {}), + config=data.get("config", {}), + domain=data.get("config", {}).get("domain", ""), + ) + + +async def run_researcher_experiment( + researcher: AnthropicClientAdapter, + summarizer: AnthropicClientAdapter, + dispatcher: ToolDispatcher, + registry: ToolRegistry, + active_config: DomainConfig | None, + researcher_prompt: str, + summarizer_threshold: int, + max_iterations: int, + investigation: Investigation, + hypothesis: Hypothesis, + experiment: Experiment, + cost: CostTracker, + design: dict[str, Any], + state_lock: asyncio.Lock, + sibling_context: str = "", +) -> AsyncGenerator[DomainEvent, None]: + planned = set(experiment.tool_plan) if experiment.tool_plan else set() + control_tools = {"record_finding", "record_negative_control"} + if planned: + allowed = planned | control_tools + tool_schemas = [t for t in registry.list_schemas() if t["name"] in allowed] + elif active_config: + excluded = { + "conclude_investigation", + "propose_hypothesis", + "design_experiment", + "evaluate_hypothesis", + } + domain_schemas = registry.list_schemas_for_domain(active_config.tool_tags) + tool_schemas = [t for t in domain_schemas if t["name"] not in excluded] + else: + excluded = { + "conclude_investigation", + "propose_hypothesis", + "design_experiment", + "evaluate_hypothesis", + } + tool_schemas = [t for t in registry.list_schemas() if t["name"] not in excluded] + + exp_controls = ", ".join(experiment.controls) or "None specified" + sibling_section = "" + if sibling_context: + sibling_section = ( + f"\n\n\n" + f"A parallel researcher is simultaneously testing a different hypothesis:\n" + f"{sibling_context}\n\n" + f"Avoid duplicating their queries. Use different search terms, " + f"data sources, or analytical approaches where your experiment " + f"design allows flexibility.\n" + f"" + ) + messages: list[dict[str, Any]] = [ + { + "role": "user", + "content": ( + f"Research prompt: {investigation.prompt}\n\n" + f"Hypothesis: {hypothesis.statement}\n" + f"Mechanism: {hypothesis.rationale}\n" + f"Prediction: {hypothesis.prediction or 'N/A'}\n" + f"Hypothesis success criteria: {hypothesis.success_criteria or 'N/A'}\n" + f"Hypothesis failure criteria: {hypothesis.failure_criteria or 'N/A'}\n" + f"Scope: {hypothesis.scope or 'N/A'}\n\n" + f"Experiment: {experiment.description}\n" + f"Planned tools: {', '.join(experiment.tool_plan)}\n" + f"Controls: {exp_controls}\n" + f"Analysis plan: {experiment.analysis_plan or 'N/A'}\n" + f"Experiment success criteria: {experiment.success_criteria or 'N/A'}\n" + f"Experiment failure criteria: {experiment.failure_criteria or 'N/A'}\n\n" + f"Execute this experiment. Compare results against the " + f"pre-defined success/failure criteria. Link all findings to " + f"hypothesis_id='{hypothesis.id}'." + sibling_section + ), + }, + ] + + for _iteration in range(max_iterations): + investigation.iteration += 1 + tool_choice = {"type": "any"} if _iteration == 0 else None + response = await researcher.create_message( + system=researcher_prompt, + messages=messages, + tools=tool_schemas, + tool_choice=tool_choice, + ) + async with state_lock: + cost.add_usage( + response.input_tokens, + response.output_tokens, + researcher.model, + cache_read_tokens=response.cache_read_input_tokens, + cache_write_tokens=response.cache_write_input_tokens, + ) + + assistant_content: list[dict[str, Any]] = [] + tool_use_blocks: list[dict[str, Any]] = [] + + for block in response.content: + assistant_content.append(block) + if block["type"] == "text": + yield Thinking( + text=block["text"], + investigation_id=investigation.id, + ) + elif block["type"] == "tool_use": + tool_use_blocks.append(block) + + messages.append({"role": "assistant", "content": assistant_content}) + + if response.stop_reason == "end_turn" or not tool_use_blocks: + break + + tool_results: list[dict[str, Any]] = [] + for tool_block in tool_use_blocks: + tool_name = tool_block["name"] + tool_input = tool_block["input"] + tool_use_id = tool_block["id"] + async with state_lock: + cost.add_tool_call() + + yield ToolCalled( + tool_name=tool_name, + tool_input=tool_input, + experiment_id=experiment.id, + investigation_id=investigation.id, + ) + + result_str = await dispatcher.dispatch(tool_name, tool_input, investigation) + result_str = _compact_result(tool_name, result_str) + + summarized_str, summarize_event = await summarize_output( + summarizer, cost, tool_name, result_str, investigation.id, summarizer_threshold + ) + if summarize_event is not None: + yield summarize_event + + content_for_model = summarized_str if summarize_event else result_str + preview = result_str[:1500] if len(result_str) > 1500 else result_str + yield ToolResultEvent( + tool_name=tool_name, + result_preview=preview, + experiment_id=experiment.id, + investigation_id=investigation.id, + ) + + # Emit visualization event if tool returned viz payload + viz_event = maybe_viz_event(result_str, experiment.id, investigation.id) + if viz_event is not None: + yield viz_event + + if tool_name == "record_finding": + h_id = tool_input.get("hypothesis_id", hypothesis.id) + e_type = tool_input.get("evidence_type", "neutral") + finding = Finding( + title=tool_input.get("title", ""), + detail=tool_input.get("detail", ""), + evidence=tool_input.get("evidence", ""), + hypothesis_id=h_id, + evidence_type=e_type, + source_type=tool_input.get("source_type", ""), + source_id=tool_input.get("source_id", ""), + evidence_level=int(tool_input.get("evidence_level", 0)), + ) + async with state_lock: + investigation.record_finding(finding) + h = investigation.get_hypothesis(h_id) + if h: + if e_type == "supporting": + h.supporting_evidence.append(finding.title) + elif e_type == "contradicting": + h.contradicting_evidence.append(finding.title) + yield FindingRecorded( + title=finding.title, + detail=finding.detail, + hypothesis_id=h_id, + evidence_type=e_type, + evidence=finding.evidence, + source_type=finding.source_type, + source_id=finding.source_id, + evidence_level=finding.evidence_level, + investigation_id=investigation.id, + ) + + if tool_name == "train_model": + try: + train_result = json.loads(result_str) + if isinstance(train_result, dict) and "model_id" in train_result: + async with state_lock: + investigation.trained_model_ids.append(train_result["model_id"]) + except (json.JSONDecodeError, TypeError): + pass + + if tool_name == "record_negative_control": + control = NegativeControl( + identifier=tool_input.get("identifier", tool_input.get("smiles", "")), + identifier_type=tool_input.get("identifier_type", ""), + name=tool_input.get("name", ""), + score=float( + tool_input.get("score", tool_input.get("prediction_score", 0.0)) + ), + threshold=float(tool_input.get("threshold", 0.5)), + source=tool_input.get("source", ""), + ) + async with state_lock: + investigation.add_negative_control(control) + yield NegativeControlRecorded( + identifier=control.identifier, + identifier_type=control.identifier_type, + name=control.name, + score=control.score, + threshold=control.threshold, + correctly_classified=(control.correctly_classified), + investigation_id=investigation.id, + ) + + tool_results.append( + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": content_for_model, + } + ) + + messages.append({"role": "user", "content": tool_results}) diff --git a/server/src/ehrlich/investigation/application/tool_dispatcher.py b/server/src/ehrlich/investigation/application/tool_dispatcher.py new file mode 100644 index 0000000..e158483 --- /dev/null +++ b/server/src/ehrlich/investigation/application/tool_dispatcher.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.application.tool_cache import ToolCache + +if TYPE_CHECKING: + from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.domain.repository import InvestigationRepository + from ehrlich.investigation.domain.uploaded_file import UploadedFile + +logger = logging.getLogger(__name__) + + +class ToolDispatcher: + def __init__( + self, + registry: ToolRegistry, + cache: ToolCache, + repository: InvestigationRepository | None, + uploaded_files: dict[str, UploadedFile], + ) -> None: + self._registry = registry + self._cache = cache + self._repository = repository + self._uploaded_files = uploaded_files + + def update_uploaded_files(self, files: dict[str, UploadedFile]) -> None: + self._uploaded_files = files + + async def dispatch( + self, + tool_name: str, + tool_input: dict[str, Any], + investigation: Investigation, + ) -> str: + # Intercept query_uploaded_data -- serve from in-memory file store + if tool_name == "query_uploaded_data": + return self._handle_query_uploaded_data(tool_input) + + # Intercept search_prior_research -- query FTS5 via repository + if tool_name == "search_prior_research" and self._repository: + query = tool_input.get("query", "") + limit = int(tool_input.get("limit", 10)) + results = await self._repository.search_findings(query, limit) + return json.dumps({"results": results, "count": len(results), "query": query}) + + args_hash = ToolCache.hash_args(tool_input) + cached = self._cache.get(tool_name, args_hash) + if cached is not None: + return cached + + func = self._registry.get(tool_name) + if func is None: + return json.dumps({"error": f"Unknown tool: {tool_name}"}) + + try: + result = await func(**tool_input) + result_str = str(result) + self._cache.put(tool_name, args_hash, result_str) + return result_str + except Exception as e: + logger.warning("Tool %s failed: %s", tool_name, e) + return json.dumps({"error": f"Tool {tool_name} failed: {e}"}) + + def _handle_query_uploaded_data(self, tool_input: dict[str, Any]) -> str: + """Handle query_uploaded_data tool calls from in-memory uploaded files.""" + import pandas as pd + + file_id = tool_input.get("file_id", "") + uploaded = self._uploaded_files.get(file_id) + if uploaded is None: + return json.dumps({"error": f"File not found: {file_id}"}) + + # Document files: return text content + if uploaded.document: + keyword = tool_input.get("filter_value", "") + text = uploaded.document.text + if keyword: + lines = [ln for ln in text.split("\n") if keyword.lower() in ln.lower()] + text = "\n".join(lines) if lines else f"No lines matching '{keyword}'" + return json.dumps( + { + "file_id": file_id, + "filename": uploaded.filename, + "type": "document", + "page_count": uploaded.document.page_count, + "text": text, + } + ) + + # Tabular files: reconstruct DataFrame from sample data and query + if uploaded.tabular is None: + return json.dumps({"error": "File has no queryable data"}) + + tab = uploaded.tabular + rows = [list(row) for row in tab.sample_rows] + df = pd.DataFrame(rows, columns=list(tab.columns)) + + # Apply type coercion for numeric columns + for col, dtype_str in zip(tab.columns, tab.dtypes, strict=True): + if "int" in dtype_str or "float" in dtype_str: + df[col] = pd.to_numeric(df[col], errors="coerce") + + # Column selection + columns_str = tool_input.get("columns") + if columns_str: + selected = [c.strip() for c in columns_str.split(",")] + valid = [c for c in selected if c in df.columns] + if valid: + df = df[valid] + + # Filtering + filter_col = tool_input.get("filter_column") + filter_op = tool_input.get("filter_op") + filter_val = tool_input.get("filter_value") + if filter_col and filter_op and filter_val is not None and filter_col in df.columns: + col_series = df[filter_col] + ops: dict[str, Any] = { + "eq": lambda s, v: s == v, + "gt": lambda s, v: s > v, + "lt": lambda s, v: s < v, + "gte": lambda s, v: s >= v, + "lte": lambda s, v: s <= v, + "contains": lambda s, v: s.astype(str).str.contains(str(v), case=False, na=False), + } + op_func = ops.get(filter_op) + if op_func: + import contextlib + + cast_val: Any = filter_val + if filter_op != "contains": + with contextlib.suppress(ValueError, TypeError): + cast_val = float(filter_val) + with contextlib.suppress(TypeError, ValueError): + mask = op_func(col_series, cast_val) + df = df[mask] + + head_n = min(int(tool_input.get("head", 50)), 200) + df = df.head(head_n) + + return json.dumps( + { + "file_id": file_id, + "filename": uploaded.filename, + "type": "tabular", + "total_rows": tab.row_count, + "returned_rows": len(df), + "columns": list(df.columns), + "data": df.values.tolist(), + } + ) From e0d78cb8f28f2661ba9e4c7c85ca5bfa4c739cb1 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 07:19:23 -0700 Subject: [PATCH 07/35] refactor(investigation): fix Phase 14A review issues - Remove circular import in researcher_executor.py by copying _compact_result - Update roadmap.md checkboxes for completed Phase 14A extractions - Update architecture.md method references to new module locations Result: multi_orchestrator.py decomposed from 1,788 to 1,216 lines. --- docs/architecture.md | 4 +- docs/roadmap.md | 139 ++++++++++++++++++ .../application/researcher_executor.py | 23 ++- 3 files changed, 160 insertions(+), 6 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index fc0c413..1788be3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,7 +115,7 @@ The Haiku classifier outputs a JSON array of domain categories. Tool filtering u Ehrlich queries its own past investigation findings during new research. The `search_prior_research` tool (available during Phase 2 Literature Survey) queries a PostgreSQL `tsvector` column with GIN index on the `investigations` table, indexing finding titles, details, evidence types, hypothesis statements/statuses, and source provenance. -The tool is intercepted in the orchestrator's `_dispatch_tool()` and routed to `InvestigationRepository.search_findings()`, which performs ranked full-text search with prompt context. +The tool is intercepted by `ToolDispatcher.dispatch()` in `tool_dispatcher.py` and routed to `InvestigationRepository.search_findings()`, which performs ranked full-text search with prompt context. Findings from past investigations carry provenance `source_type: "ehrlich"`, `source_id: "{investigation_id}"`. Frontend renders Ehrlich-branded source badges linking to past investigations (internal navigation, no external tab). @@ -140,7 +140,7 @@ Invalid SMILES on `/depict` returns a dark error SVG (200 status). Invalid SMILE ## Domain-Specific Visualization -17 visualization tools produce structured `VisualizationPayload` JSON (viz_type, title, data, config, domain). The orchestrator intercepts viz tool results via `_maybe_viz_event()` and emits a `VisualizationRendered` SSE event. On the frontend, `VizRegistry` maps each `viz_type` to a lazy-loaded React component rendered in the `VisualizationPanel` grid. +17 visualization tools produce structured `VisualizationPayload` JSON (viz_type, title, data, config, domain). Viz tool results are intercepted by `maybe_viz_event()` in `researcher_executor.py`, which emits a `VisualizationRendered` SSE event. On the frontend, `VizRegistry` maps each `viz_type` to a lazy-loaded React component rendered in the `VisualizationPanel` grid. | Tool | Chart Library | Purpose | |------|--------------|---------| diff --git a/docs/roadmap.md b/docs/roadmap.md index 7555732..30ece61 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -473,6 +473,13 @@ Enhancement Enhancement Sports (Causal inference + Document upload + MX/US APIs) 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) -> 13C (Mexico) -> 13D (US) -> 13E (MCP) | + Phase 14: Scientific Engine Hardening -- IN PROGRESS (parallel with 13A-2) + 14A (Orchestrator Split) ──┬── 14B (Route Cleanup) ──┬── 14D (Tool Enrichment) + ├── 14C (Prompt Split) ├── 14E (Tree Search) + └───────────────────────── └── 14F (Paper Generator) + | + 14G (Validation Runs) + | Demo + Video -- TODO | Submission (Feb 16, 3PM ET) -- TODO @@ -1295,11 +1302,143 @@ Users see exactly what each investigation costs. --- +## Phase 14: Scientific Engine Hardening (Feb 14-15) -- IN PROGRESS + +Architecture cleanup, scientific depth improvements, agentic tree search, and paper generation. Runs in parallel with Phase 13A-2 (upload). Addresses audit findings from codebase review. + +### Prior Art + +Ehrlich sits alongside three contemporary AI scientific discovery systems: + +- **Sakana AI Scientist v2** (Apr 2025): End-to-end paper generation via agentic tree search. First AI-generated peer-reviewed workshop paper. ML-only. [arXiv:2504.08066](https://arxiv.org/abs/2504.08066) +- **FutureHouse Robin** (May 2025): Multi-agent system that discovered ripasudil as therapeutic candidate for AMD in 2.5 months. Biology-only, lab-in-the-loop. [arXiv:2505.13400](https://arxiv.org/abs/2505.13400) +- **Ai2 AutoDiscovery** (Feb 2026): Bayesian surprise-guided hypothesis exploration over large search spaces. [allenai.org/blog/autodiscovery](https://allenai.org/blog/autodiscovery) + +Ehrlich differentiates by: domain-agnostic architecture (4 domains vs single-domain competitors), multi-model orchestration (Director/Researcher/Summarizer), real-time streaming UI, 18 external data sources, and computational laboratory (ML pipelines + causal inference as experimentation). + +### 14A: Orchestrator Decomposition -- TODO + +Split `MultiModelOrchestrator` (1,788 lines, 11 concerns) into focused modules following Clean Architecture. No behavior changes -- pure structural refactoring. + +- [x] Extract `diagram_builder.py` -- Excalidraw element building + MCP bridge export (127 lines) +- [x] Extract `tool_dispatcher.py` -- tool execution + caching + `search_prior_research` / `query_uploaded_data` interception (34 lines) +- [x] Extract `researcher_executor.py` -- single experiment runner with tool loop (216 lines) +- [x] Extract `batch_executor.py` -- parallel experiment execution via asyncio.Queue (56 lines) +- [x] Merged uploaded_data_query into tool_dispatcher.py +- [x] Kept run() in multi_orchestrator.py as main coordination point +- [ ] Orchestrator reduced to ~400 lines: init, run (delegates to phase_manager), approve + +Result: `multi_orchestrator.py` from 1,788 to 1,216 lines (4 modules extracted: 718 lines total) + +### 14B: Route File Cleanup -- TODO + +Move infrastructure concerns out of `api/routes/investigation.py` (828 lines) into application layer. + +- [ ] Extract `application/registry_factory.py` -- `build_tool_registry()` + `build_domain_registry()` (128 lines of tool registration) +- [ ] Extract `application/orchestrator_factory.py` -- `create_orchestrator()` with all adapter wiring +- [ ] Move Pydantic DTOs to `api/schemas/investigation.py` (6 models) +- [ ] Lazy-initialize tool registry (singleton, built once instead of per-investigation) + +Target: route file from 828 to ~400 lines. + +### 14C: Prompt Module Split -- TODO + +Split `prompts.py` (1,512 lines) into focused modules. + +- [ ] `prompts/__init__.py` -- re-exports for backward compatibility +- [ ] `prompts/constants.py` -- `SCIENTIST_SYSTEM_PROMPT`, `SUMMARIZER_PROMPT`, `RESEARCHER_EXPERIMENT_PROMPT` +- [ ] `prompts/director.py` -- `DIRECTOR_FORMULATION_PROMPT`, `DIRECTOR_EXPERIMENT_PROMPT`, `DIRECTOR_EVALUATION_PROMPT`, `DIRECTOR_SYNTHESIS_PROMPT` +- [ ] `prompts/builders.py` -- `build_pico_and_classification_prompt`, `build_literature_survey_prompt`, `build_literature_assessment_prompt`, `build_formulation_prompt`, `build_experiment_prompt`, `build_synthesis_prompt`, `build_researcher_prompt`, `build_multi_investigation_context`, `build_uploaded_data_context`, `_build_prior_context` + +### 14D: Domain Tool Enrichment -- TODO + +Upgrade 4 composite tools from thin pass-throughs to real domain-specific logic. + +- [ ] `search_training_literature` -- MeSH term expansion, study type ranking (SR > RCT > cohort), date recency weighting, non-human study filtering +- [ ] `search_supplement_evidence` -- GRADE-style quality filtering, RCT/meta-analysis prioritization, retracted paper exclusion +- [ ] `compare_nutrients` -- per-nutrient delta computation, winner highlighting per category, overall MAR score comparison +- [ ] Fix `TrainingService` I-squared reimplementation -- delegate to `StatisticsService` (training_service.py:79-85) + +### 14E: Agentic Hypothesis Tree Search -- TODO + +Replace linear hypothesis testing with tree-based exploration inspired by AI Scientist v2's agentic tree search. The current parallel batch system (2 experiments per hypothesis) becomes a tree where hypotheses can branch, deepen, or be pruned based on evidence. + +- [ ] `HypothesisNode` dataclass -- extends Hypothesis with `depth`, `branch_score`, `children: list[str]` +- [ ] `TreeManager` in `application/tree_manager.py` -- decides which branches to explore based on confidence scores, prunes low-confidence branches (<0.4 threshold), tracks exploration budget +- [ ] Director formulation prompt update -- "given current evidence tree, propose sub-hypotheses for the most promising branch OR prune low-confidence branches" +- [ ] Director evaluation prompt update -- after evaluating, decide: (a) deepen (spawn sub-hypotheses), (b) prune (mark branch as dead), (c) branch (revise into alternative direction) +- [ ] `max_depth` parameter (default: 3) to prevent infinite recursion +- [ ] Budget-aware exploration -- track API cost per branch, prefer deepening high-confidence/low-cost branches +- [ ] `HypothesisTreeEvent` SSE event for frontend tree rendering +- [ ] Console: `InvestigationDiagram.tsx` already renders React Flow graph -- tree structure renders naturally as growing/pruning graph + +### 14F: Scientific Paper Generator -- TODO + +Generate structured scientific papers from completed investigations. Not AI-generated prose -- structured assembly of existing investigation data into publication format. + +- [ ] `application/paper_generator.py` -- takes `Investigation` entity, produces structured sections: + - Title (from research question) + - Abstract (Director synthesis summary) + - Introduction (PICO decomposition + literature survey context) + - Methods (experiment designs, tool descriptions, statistical approach) + - Results (findings with statistical tests, visualizations as base64 SVG) + - Discussion (hypothesis assessments, GRADE certainty, limitations, knowledge gaps) + - References (collected DOIs + citations) + - Supplementary (cost breakdown, model info, negative controls, tool call trace) +- [ ] `GET /investigate/{id}/paper` endpoint -- returns structured paper as JSON or Markdown +- [ ] `GET /investigate/{id}/paper?format=pdf` -- server-side PDF generation (optional, `weasyprint` or `reportlab`) +- [ ] Console: "Export Paper" button on completed investigation page +- [ ] Console: paper preview with section navigation + +### 14G: Validation Runs -- TODO + +Run real investigations to validate the engine produces meaningful scientific results. Compare Ehrlich vs raw Claude Opus 4.6 on identical prompts. + +- [ ] Molecular science validation: known drug discovery question with verifiable answer +- [ ] Training science validation: exercise physiology question with literature-backed answer +- [ ] Impact evaluation validation: real program evaluation data (CODESON) compared against professional auditors and raw Claude +- [ ] Document comparison results: where Ehrlich agrees/disagrees with baseline, where tree search found non-obvious paths +- [ ] Capture investigation traces for demo video + +### Dependency Graph + +``` +Phase 14A (Orchestrator Decomposition) + | +Phase 14B (Route File Cleanup) -- parallel with 14A + | +Phase 14C (Prompt Module Split) -- parallel with 14A/B + | +Phase 14D (Domain Tool Enrichment) -- after 14A (touches tool layer) + | +Phase 14E (Agentic Tree Search) -- after 14A (modifies orchestrator) + | +Phase 14F (Paper Generator) -- after 14A (reads investigation data) + | +Phase 14G (Validation Runs) -- after 14E + 14F (needs tree search + paper gen) +``` + +14A, 14B, 14C can run in parallel (independent refactorings). +14D, 14E, 14F depend on 14A (clean orchestrator). +14G depends on everything else being done. + +**Verification:** All existing tests pass after each sub-phase. New tests for tree search, paper generator, and enriched tools. Quality gates: ruff 0, mypy 0, pytest pass, tsc 0, vitest pass. + +--- + ## Backlog (Post-Hackathon) +- Phase 11: Investigation Comparison (side-by-side analysis) +- Phase 13C: Mexico Integration (INEGI, datos.gob.mx, Transparencia, Banxico) +- Phase 13D: US Integration (Census, BLS, USAspending, CDC WONDER) +- Phase 13E: MCP Bridge Self-Service +- Competitive Sports Domain - Batch screening mode (score entire ZINC subsets) - Automated synthesis lab integration (Emerald Cloud Lab, Strateos) - Community contributions: new domains, new tools, new data sources - Rewarded ads for free credits (AdMob/AdSense integration) - Stripe integration for paid credit packs - Admin dashboard for usage analytics +- Planner agent (Sonnet 4.5) for experiment design between Director and Researcher +- Reviewer agent (Sonnet 4.5) for findings audit between Researcher and Director +- Full Bayesian surprise scoring for tree search branch selection diff --git a/server/src/ehrlich/investigation/application/researcher_executor.py b/server/src/ehrlich/investigation/application/researcher_executor.py index 77268ad..abedf1b 100644 --- a/server/src/ehrlich/investigation/application/researcher_executor.py +++ b/server/src/ehrlich/investigation/application/researcher_executor.py @@ -33,12 +33,27 @@ logger = logging.getLogger(__name__) +_COMPACT_SCHEMAS: dict[str, list[str]] = { + "compute_descriptors": ["molecular_weight", "logp", "tpsa", "hbd", "hba", "qed", "num_rings"], + "compute_fingerprint": ["fingerprint_type", "num_bits"], + "validate_smiles": ["valid", "canonical_smiles"], + "explore_dataset": ["name", "target", "size", "active_count"], + "search_bioactivity": ["target", "size", "active_count"], + "search_protein_targets": ["query", "count", "targets"], + "tanimoto_similarity": ["similarity"], +} -def _compact_result(tool_name: str, result: str) -> str: - """Import and delegate to the module-level _compact_result in multi_orchestrator.""" - from ehrlich.investigation.application.multi_orchestrator import _compact_result as _cr - return _cr(tool_name, result) +def _compact_result(tool_name: str, result: str) -> str: + schema = _COMPACT_SCHEMAS.get(tool_name) + if not schema: + return result + try: + data = json.loads(result) + compacted = {k: data[k] for k in schema if k in data} + return json.dumps(compacted) + except (json.JSONDecodeError, TypeError): + return result async def summarize_output( From 5018315ac65f35502e3be6e8e5b854560b5215bf Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 07:25:59 -0700 Subject: [PATCH 08/35] feat(investigation): add document upload for CSV/XLSX/PDF (Phase 13A-2) Upload-first flow: POST /upload returns preview, POST /investigate references file_ids. UploadedFile domain entity with TabularData and DocumentData frozen dataclasses. FileProcessor parses CSV (pandas), XLSX (openpyxl), PDF (pymupdf). PostgreSQL uploaded_files table with JSONB. Uploaded data injected into Director/Researcher prompts as XML block. query_uploaded_data tool intercepted by ToolDispatcher for pandas-based filtering. Frontend drag-and-drop FileUpload, DataPreview cards, use-upload mutation hook. 85 tools total. --- CLAUDE.md | 27 +- README.md | 6 +- .../components/DataPreview.test.tsx | 167 ++++++++++++ .../investigation/components/DataPreview.tsx | 97 +++++++ .../components/FileUpload.test.tsx | 120 +++++++++ .../investigation/components/FileUpload.tsx | 150 +++++++++++ .../components/PromptInput.test.tsx | 12 + .../investigation/components/PromptInput.tsx | 12 +- .../investigation/hooks/use-upload.ts | 13 + console/src/features/investigation/types.ts | 20 ++ console/src/shared/lib/api.ts | 27 ++ docs/architecture.md | 10 +- docs/impact-evaluation-domain.md | 16 +- docs/roadmap.md | 28 +- server/pyproject.toml | 2 + server/src/ehrlich/api/app.py | 2 + .../src/ehrlich/api/routes/investigation.py | 15 ++ server/src/ehrlich/api/routes/upload.py | 82 ++++++ .../application/file_processor.py | 102 +++++++ .../investigation/application/prompts.py | 67 ++++- .../investigation/domain/investigation.py | 2 + .../investigation/domain/repository.py | 7 + .../investigation/domain/uploaded_file.py | 42 +++ .../infrastructure/repository.py | 81 ++++++ server/src/ehrlich/investigation/tools.py | 25 ++ server/tests/api/test_methodology.py | 2 +- server/tests/api/test_stats.py | 2 +- .../investigation/test_file_processor.py | 175 ++++++++++++ .../tests/investigation/test_uploaded_file.py | 249 ++++++++++++++++++ web/src/lib/constants.ts | 2 +- 30 files changed, 1519 insertions(+), 43 deletions(-) create mode 100644 console/src/features/investigation/components/DataPreview.test.tsx create mode 100644 console/src/features/investigation/components/DataPreview.tsx create mode 100644 console/src/features/investigation/components/FileUpload.test.tsx create mode 100644 console/src/features/investigation/components/FileUpload.tsx create mode 100644 console/src/features/investigation/hooks/use-upload.ts create mode 100644 server/src/ehrlich/api/routes/upload.py create mode 100644 server/src/ehrlich/investigation/application/file_processor.py create mode 100644 server/src/ehrlich/investigation/domain/uploaded_file.py create mode 100644 server/tests/investigation/test_file_processor.py create mode 100644 server/tests/investigation/test_uploaded_file.py diff --git a/CLAUDE.md b/CLAUDE.md index 53bcc50..91848a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ DDD monorepo: `server/` (Python 3.12) + `console/` (React 19 / TypeScript / Bun) ``` Opus 4.6 (Director) -- Formulates hypotheses, designs experiments, evaluates evidence, synthesizes (NO tools) -Sonnet 4.5 (Researcher) -- Executes experiments with 79 tools (parallel: 2 experiments per batch) +Sonnet 4.5 (Researcher) -- Executes experiments with 80 tools (parallel: 2 experiments per batch) Haiku 4.5 (Summarizer) -- Compresses large tool outputs (>2000 chars), PICO decomposition, evidence grading ``` @@ -141,7 +141,7 @@ Types: `feat`, `fix`, `refactor`, `chore`, `docs`, `test` Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, training, nutrition, impact, investigation, api, console, mol, data, ci, docs, infra -## Tools (84 Total) +## Tools (85 Total) ### Chemistry (6) -- RDKit cheminformatics, domain-agnostic - `validate_smiles` -- Validate SMILES string @@ -240,13 +240,14 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - `run_statistical_test` -- Compare two numeric groups (auto-selects t-test/Welch/Mann-Whitney, returns p-value, Cohen's d, 95% CI) - `run_categorical_test` -- Test contingency tables (auto-selects Fisher's exact/chi-squared, returns p-value, odds ratio/Cramer's V) -### Investigation (7) -- Hypothesis lifecycle + self-referential +### Investigation (8) -- Hypothesis lifecycle + self-referential + data upload - `propose_hypothesis` -- Register testable hypothesis - `design_experiment` -- Plan experiment with tool sequence - `evaluate_hypothesis` -- Assess outcome (supported/refuted/revised) - `record_finding` -- Record finding linked to hypothesis + evidence_type - `record_negative_control` -- Validate model with known-inactive compounds - `search_prior_research` -- Search Ehrlich's own past findings via full-text search +- `query_uploaded_data` -- Query user-uploaded files (CSV/Excel filtering, PDF text retrieval) - `conclude_investigation` -- Final summary with ranked candidates ## Key Patterns @@ -281,6 +282,7 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - **Authentication**: WorkOS JWT middleware (`api/auth.py`); JWKS verification via `PyJWKClient`; `get_current_user` (header) and `get_current_user_sse` (header or `?token=` query param for EventSource); `get_optional_user` for public routes - **Credit system**: director tier selection (haiku=1cr, sonnet=3cr, opus=5cr); credits deducted on investigation start; refunded on failure; `credit_transactions` table for audit trail; `GET /credits/balance` endpoint - **BYOK (Bring Your Own Key)**: `X-Anthropic-Key` header pass-through; bypasses credit system; API key forwarded to `AnthropicClientAdapter`; checked on both `POST /investigate` and `GET /investigate/{id}/stream` +- **Document upload**: CSV/XLSX/PDF upload via `POST /upload` (multipart/form-data); `FileProcessor` parses to `UploadedFile` domain entity; tabular files get summary stats + sample rows; PDFs get text extraction (pymupdf); uploaded data injected into Director/Researcher prompts as `` XML block; `query_uploaded_data` tool intercepted by `ToolDispatcher` for pandas-based filtering ### Integration Patterns @@ -423,15 +425,16 @@ All paths relative to `server/src/ehrlich/`. | File | Purpose | |------|---------| -| `application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop) | +| `application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop, upload context injection) | | `application/diagram_builder.py` | Excalidraw evidence synthesis diagram generation | -| `application/tool_dispatcher.py` | Tool execution dispatcher with caching and special handlers | +| `application/tool_dispatcher.py` | Tool execution dispatcher with caching, `search_prior_research` and `query_uploaded_data` interception | | `application/researcher_executor.py` | Single researcher experiment executor with tool loop | | `application/batch_executor.py` | Parallel batch experiment executor (2 concurrent researchers) | | `application/cost_tracker.py` | Per-model cost tracking with tiered pricing | | `application/tool_cache.py` | In-memory TTL cache for tool results | | `application/tool_registry.py` | `ToolRegistry` with domain tag filtering | -| `application/prompts.py` | Domain-adaptive prompts (PICO, literature survey, director, researcher, summarizer) | +| `application/file_processor.py` | CSV/XLSX/PDF parser producing `UploadedFile` domain entities | +| `application/prompts.py` | Domain-adaptive prompts (PICO, literature survey, director, researcher, summarizer, uploaded data context) | | `domain/schemas.py` | JSON schemas for structured output responses (6 schemas: PICO, Formulation, Experiment Design, Evaluation, Synthesis, Literature Grading) | | `domain/hypothesis.py` | Hypothesis entity + HypothesisStatus + HypothesisType enums | | `domain/experiment.py` | Experiment entity + ExperimentStatus enum | @@ -448,9 +451,11 @@ All paths relative to `server/src/ehrlich/`. | `domain/domains/training.py` | `TRAINING_SCIENCE` config | | `domain/domains/nutrition.py` | `NUTRITION_SCIENCE` config | | `domain/domains/impact.py` | `IMPACT_EVALUATION` config | +| `domain/uploaded_file.py` | `UploadedFile`, `TabularData`, `DocumentData` frozen dataclasses | | `domain/mcp_config.py` | `MCPServerConfig` frozen dataclass | +| `tools.py` | `query_uploaded_data` tool (intercepted by ToolDispatcher) | | `tools_viz.py` | 17 visualization tools | -| `infrastructure/repository.py` | PostgreSQL persistence (asyncpg) + tsvector findings search + user/credit management | +| `infrastructure/repository.py` | PostgreSQL persistence (asyncpg) + tsvector findings search + user/credit management + uploaded files | | `infrastructure/anthropic_client.py` | Anthropic API adapter with retry, streaming, and structured outputs | | `infrastructure/mcp_bridge.py` | MCP client bridge (stdio/SSE/streamable_http) | @@ -461,7 +466,8 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| | `auth.py` | WorkOS JWT middleware (JWKS verification, header + query param auth for SSE) | -| `routes/investigation.py` | REST + SSE endpoints, 84-tool registry, domain registry, MCP bridge, credit system, BYOK | +| `routes/investigation.py` | REST + SSE endpoints, 85-tool registry, domain registry, MCP bridge, credit system, BYOK | +| `routes/upload.py` | `POST /upload` multipart file upload (CSV/XLSX/PDF), preview response, pending upload store | | `routes/methodology.py` | GET /methodology: phases, domains, tools, data sources, models | | `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets endpoints | | `routes/stats.py` | GET /stats: aggregate counts | @@ -490,7 +496,9 @@ All paths relative to `console/src/`. | `features/investigation/components/NegativeControlPanel.tsx` | Negative control validation table | | `features/investigation/components/TemplateCards.tsx` | 9 cross-domain research prompt templates | | `features/investigation/components/ThreatAssessment.tsx` | Validity threat assessment panel with severity badges | -| `features/investigation/components/PromptInput.tsx` | Controlled prompt input with director tier selector pills | +| `features/investigation/components/FileUpload.tsx` | Drag-and-drop file upload (CSV/XLSX/PDF) with progress and preview | +| `features/investigation/components/DataPreview.tsx` | Compact inline preview cards for uploaded tabular/document data | +| `features/investigation/components/PromptInput.tsx` | Controlled prompt input with director tier selector pills and file upload | | `features/investigation/components/BYOKSettings.tsx` | API key management for BYOK users | ### Investigation lib + hooks @@ -502,6 +510,7 @@ All paths relative to `console/src/`. | `features/investigation/lib/export-markdown.ts` | Client-side markdown report generation | | `features/investigation/hooks/use-methodology.ts` | React hook for methodology data | | `features/investigation/hooks/use-credits.ts` | Credit balance hook (TanStack Query) | +| `features/investigation/hooks/use-upload.ts` | File upload mutation hook (TanStack Query) | ### Molecule components diff --git a/README.md b/README.md index 75b6781..5c96f44 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga ``` Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) -Sonnet 4.5 (Researcher) -- Executes experiments with 84 tools (10-20 calls) +Sonnet 4.5 (Researcher) -- Executes experiments with 85 tools (10-20 calls) Haiku 4.5 (Summarizer) -- Compresses large outputs, classifies domains (5-10 calls) ``` @@ -88,7 +88,7 @@ Cost: ~$3-4 per investigation (vs ~$11 with all-Opus). All data sources are free and open-access. -## 84 Tools +## 85 Tools | Context | Tool | Description | |---------|------|-------------| @@ -175,6 +175,7 @@ All data sources are free and open-access. | Investigation | `record_finding` | Record finding linked to hypothesis | | Investigation | `record_negative_control` | Validate model with known-inactive compounds | | Investigation | `search_prior_research` | Search past investigation findings via full-text search | +| Investigation | `query_uploaded_data` | Query user-uploaded files (CSV/Excel filtering, PDF text) | | Investigation | `conclude_investigation` | Final summary with ranked candidates | ## Tech Stack @@ -299,6 +300,7 @@ Server at :8000, Console at :3000. | POST | `/api/v1/investigate` | Create new investigation (`director_tier`: haiku/sonnet/opus) | | GET | `/api/v1/investigate/{id}/stream` | SSE stream of investigation events (supports `?token=` for EventSource) | | POST | `/api/v1/investigate/{id}/approve` | Approve/reject formulated hypotheses | +| POST | `/api/v1/upload` | Upload file (CSV/XLSX/PDF) for investigation data, returns preview | | GET | `/api/v1/credits/balance` | Current credit balance + BYOK status | ### SSE Event Types diff --git a/console/src/features/investigation/components/DataPreview.test.tsx b/console/src/features/investigation/components/DataPreview.test.tsx new file mode 100644 index 0000000..1088956 --- /dev/null +++ b/console/src/features/investigation/components/DataPreview.test.tsx @@ -0,0 +1,167 @@ +import { render, fireEvent, cleanup } from "@testing-library/react"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import { DataPreview } from "./DataPreview"; +import type { TabularPreview, DocumentPreview } from "../types"; + +const tabularPreview: TabularPreview = { + columns: ["name", "age", "score"], + dtypes: ["str", "int", "float"], + row_count: 150, + sample_rows: [ + ["Alice", "30", "95.5"], + ["Bob", "25", "88.0"], + ], +}; + +const documentPreview: DocumentPreview = { + text: "This is a sample PDF document with some content for testing purposes.", + page_count: 3, +}; + +describe("DataPreview", () => { + afterEach(() => { + cleanup(); + }); + + it("renders filename", () => { + const { container } = render( + , + ); + expect(container.textContent).toContain("data.csv"); + }); + + it("renders tabular preview with column names", () => { + const { container } = render( + , + ); + const text = container.textContent ?? ""; + expect(text).toContain("name"); + expect(text).toContain("age"); + expect(text).toContain("score"); + }); + + it("renders tabular preview with sample rows", () => { + const { container } = render( + , + ); + const text = container.textContent ?? ""; + expect(text).toContain("Alice"); + expect(text).toContain("Bob"); + expect(text).toContain("95.5"); + }); + + it("renders tabular row count badge", () => { + const { container } = render( + , + ); + const text = container.textContent ?? ""; + expect(text).toContain("150 rows"); + expect(text).toContain("3 columns"); + }); + + it("renders document preview with text excerpt", () => { + const { container } = render( + , + ); + expect(container.textContent).toContain("sample PDF document"); + }); + + it("renders document page count badge", () => { + const { container } = render( + , + ); + expect(container.textContent).toContain("3 pages"); + }); + + it("renders singular page label for single page", () => { + const singlePage: DocumentPreview = { text: "Content", page_count: 1 }; + const { container } = render( + , + ); + expect(container.textContent).toContain("1 page"); + expect(container.textContent).not.toContain("1 pages"); + }); + + it("calls onRemove when remove button is clicked", () => { + const onRemove = vi.fn(); + const { container } = render( + , + ); + const removeBtn = container.querySelector("button")!; + fireEvent.click(removeBtn); + expect(onRemove).toHaveBeenCalledOnce(); + }); + + it("truncates long document text at 300 chars", () => { + const longText: DocumentPreview = { + text: "A".repeat(400), + page_count: 5, + }; + const { container } = render( + , + ); + const text = container.textContent ?? ""; + expect(text).toContain("..."); + expect(text).not.toContain("A".repeat(400)); + }); + + it("renders dtypes next to column names", () => { + const { container } = render( + , + ); + const text = container.textContent ?? ""; + expect(text).toContain("str"); + expect(text).toContain("int"); + expect(text).toContain("float"); + }); +}); diff --git a/console/src/features/investigation/components/DataPreview.tsx b/console/src/features/investigation/components/DataPreview.tsx new file mode 100644 index 0000000..618d5e9 --- /dev/null +++ b/console/src/features/investigation/components/DataPreview.tsx @@ -0,0 +1,97 @@ +import { X } from "lucide-react"; +import type { TabularPreview, DocumentPreview } from "../types"; + +interface DataPreviewProps { + filename: string; + contentType: string; + preview: TabularPreview | DocumentPreview; + onRemove: () => void; +} + +function isTabular(preview: TabularPreview | DocumentPreview): preview is TabularPreview { + return "columns" in preview; +} + +export function DataPreview({ filename, preview, onRemove }: DataPreviewProps) { + return ( +
+ + +

+ {filename} +

+ + {isTabular(preview) ? ( +
+
+ {preview.columns.map((col, i) => ( + + {col} + {preview.dtypes[i]} + + ))} +
+
+ + + + {preview.columns.map((col) => ( + + ))} + + + + {preview.sample_rows.slice(0, 5).map((row, ri) => ( + + {row.map((cell, ci) => ( + + ))} + + ))} + +
+ {col} +
+ {cell} +
+
+
+ + {preview.row_count.toLocaleString()} rows + + + {preview.columns.length} columns + +
+
+ ) : ( +
+

+ {preview.text.length > 300 + ? preview.text.slice(0, 300) + "..." + : preview.text} +

+ + {preview.page_count} {preview.page_count === 1 ? "page" : "pages"} + +
+ )} +
+ ); +} diff --git a/console/src/features/investigation/components/FileUpload.test.tsx b/console/src/features/investigation/components/FileUpload.test.tsx new file mode 100644 index 0000000..b35c220 --- /dev/null +++ b/console/src/features/investigation/components/FileUpload.test.tsx @@ -0,0 +1,120 @@ +import { render, fireEvent, cleanup } from "@testing-library/react"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { UploadResponse } from "../types"; + +const mockMutateAsync = vi.fn(); +const mockUploadState = { mutateAsync: mockMutateAsync, isError: false }; + +vi.mock("../hooks/use-upload", () => ({ + useUpload: () => mockUploadState, +})); + +import { FileUpload } from "./FileUpload"; + +function renderWithProvider( + props: { onFilesChanged: (files: { file_id: string; filename: string }[]) => void }, +) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + , + ); +} + +describe("FileUpload", () => { + afterEach(() => { + cleanup(); + vi.clearAllMocks(); + mockUploadState.isError = false; + }); + + it("renders drop zone with instruction text", () => { + const { container } = renderWithProvider({ onFilesChanged: vi.fn() }); + const text = container.textContent ?? ""; + expect(text).toContain("Drop CSV, Excel, or PDF files here"); + }); + + it("renders hidden file input with accepted types", () => { + const { container } = renderWithProvider({ onFilesChanged: vi.fn() }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + expect(input).not.toBeNull(); + expect(input.accept).toBe(".csv,.xlsx,.pdf"); + expect(input.multiple).toBe(true); + }); + + it("shows uploaded files via DataPreview after upload", async () => { + const uploadResult: UploadResponse = { + file_id: "abc-123", + filename: "test.csv", + content_type: "text/csv", + preview: { + columns: ["col1", "col2"], + dtypes: ["str", "int"], + row_count: 10, + sample_rows: [["a", "1"]], + }, + }; + mockMutateAsync.mockResolvedValueOnce(uploadResult); + + const onFilesChanged = vi.fn(); + const { container } = renderWithProvider({ onFilesChanged }); + + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + const file = new File(["data"], "test.csv", { type: "text/csv" }); + fireEvent.change(input, { target: { files: [file] } }); + + // Wait for async upload + await vi.waitFor(() => { + expect(container.textContent).toContain("test.csv"); + }); + + expect(onFilesChanged).toHaveBeenCalledWith([ + { file_id: "abc-123", filename: "test.csv" }, + ]); + }); + + it("removes file when remove button is clicked", async () => { + const uploadResult: UploadResponse = { + file_id: "abc-123", + filename: "test.csv", + content_type: "text/csv", + preview: { + columns: ["col1"], + dtypes: ["str"], + row_count: 5, + sample_rows: [["x"]], + }, + }; + mockMutateAsync.mockResolvedValueOnce(uploadResult); + + const onFilesChanged = vi.fn(); + const { container } = renderWithProvider({ onFilesChanged }); + + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + const file = new File(["data"], "test.csv", { type: "text/csv" }); + fireEvent.change(input, { target: { files: [file] } }); + + await vi.waitFor(() => { + expect(container.textContent).toContain("test.csv"); + }); + + // Click remove button (the X button inside DataPreview) + const removeBtn = container.querySelector( + ".relative button", + ) as HTMLButtonElement; + fireEvent.click(removeBtn); + + expect(container.textContent).not.toContain("test.csv"); + expect(onFilesChanged).toHaveBeenLastCalledWith([]); + }); + + it("shows error message when upload fails", () => { + mockUploadState.isError = true; + const { container } = renderWithProvider({ onFilesChanged: vi.fn() }); + expect(container.textContent).toContain("Upload failed"); + }); +}); diff --git a/console/src/features/investigation/components/FileUpload.tsx b/console/src/features/investigation/components/FileUpload.tsx new file mode 100644 index 0000000..109d62a --- /dev/null +++ b/console/src/features/investigation/components/FileUpload.tsx @@ -0,0 +1,150 @@ +import { useCallback, useRef, useState } from "react"; +import { Upload, Loader2 } from "lucide-react"; +import { useUpload } from "../hooks/use-upload"; +import { DataPreview } from "./DataPreview"; +import type { UploadResponse } from "../types"; + +const ACCEPTED_TYPES = ".csv,.xlsx,.pdf"; +const MAX_FILES = 10; + +interface UploadedFile extends UploadResponse { + // UploadResponse already has file_id, filename, content_type, preview +} + +interface FileUploadProps { + onFilesChanged: (files: { file_id: string; filename: string }[]) => void; +} + +export function FileUpload({ onFilesChanged }: FileUploadProps) { + const [files, setFiles] = useState([]); + const [dragOver, setDragOver] = useState(false); + const [pendingCount, setPendingCount] = useState(0); + const inputRef = useRef(null); + const upload = useUpload(); + + const updateParent = useCallback( + (updated: UploadedFile[]) => { + onFilesChanged( + updated.map((f) => ({ file_id: f.file_id, filename: f.filename })), + ); + }, + [onFilesChanged], + ); + + function handleRemove(fileId: string) { + setFiles((prev) => { + const updated = prev.filter((f) => f.file_id !== fileId); + updateParent(updated); + return updated; + }); + } + + async function processFile(file: File) { + setPendingCount((c) => c + 1); + try { + const result = await upload.mutateAsync(file); + setFiles((prev) => { + const updated = [...prev, result]; + updateParent(updated); + return updated; + }); + } finally { + setPendingCount((c) => c - 1); + } + } + + function handleFiles(fileList: FileList | File[]) { + const incoming = Array.from(fileList); + const remaining = MAX_FILES - files.length; + const batch = incoming.slice(0, remaining); + for (const file of batch) { + void processFile(file); + } + } + + function handleDrop(e: React.DragEvent) { + e.preventDefault(); + setDragOver(false); + if (e.dataTransfer.files.length > 0) { + handleFiles(e.dataTransfer.files); + } + } + + function handleDragOver(e: React.DragEvent) { + e.preventDefault(); + setDragOver(true); + } + + function handleDragLeave(e: React.DragEvent) { + e.preventDefault(); + setDragOver(false); + } + + function handleInputChange(e: React.ChangeEvent) { + if (e.target.files && e.target.files.length > 0) { + handleFiles(e.target.files); + e.target.value = ""; + } + } + + const atCapacity = files.length >= MAX_FILES; + + return ( +
+ {!atCapacity && ( + + )} + + + + {upload.isError && ( +

+ Upload failed. Check file type and try again. +

+ )} + + {files.length > 0 && ( +
+ {files.map((f) => ( + handleRemove(f.file_id)} + /> + ))} +
+ )} +
+ ); +} diff --git a/console/src/features/investigation/components/PromptInput.test.tsx b/console/src/features/investigation/components/PromptInput.test.tsx index 4d53f5e..6e34760 100644 --- a/console/src/features/investigation/components/PromptInput.test.tsx +++ b/console/src/features/investigation/components/PromptInput.test.tsx @@ -21,6 +21,13 @@ vi.mock("../hooks/use-credits", () => ({ }), })); +vi.mock("../hooks/use-upload", () => ({ + useUpload: () => ({ + mutateAsync: vi.fn(), + isError: false, + }), +})); + import { PromptInput } from "./PromptInput"; function renderWithProvider(props: { value: string; onChange: (v: string) => void }) { @@ -80,4 +87,9 @@ describe("PromptInput", () => { const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); expect(container.textContent).toContain("Director"); }); + + it("renders FileUpload drop zone", () => { + const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); + expect(container.textContent).toContain("Drop CSV, Excel, or PDF"); + }); }); diff --git a/console/src/features/investigation/components/PromptInput.tsx b/console/src/features/investigation/components/PromptInput.tsx index 969b308..73ece1c 100644 --- a/console/src/features/investigation/components/PromptInput.tsx +++ b/console/src/features/investigation/components/PromptInput.tsx @@ -2,6 +2,7 @@ import { useState } from "react"; import { useNavigate } from "@tanstack/react-router"; import { useInvestigation } from "../hooks/use-investigation"; import { useCredits } from "../hooks/use-credits"; +import { FileUpload } from "./FileUpload"; import type { DirectorTier } from "../types"; const TIERS: { value: DirectorTier; label: string; credits: number }[] = [ @@ -20,6 +21,9 @@ export function PromptInput({ value, onChange }: PromptInputProps) { const mutation = useInvestigation(); const { data: creditData } = useCredits(); const [tier, setTier] = useState("opus"); + const [uploadedFiles, setUploadedFiles] = useState< + { file_id: string; filename: string }[] + >([]); const selectedTier = TIERS.find((t) => t.value === tier)!; const isByok = creditData?.is_byok ?? false; @@ -30,8 +34,13 @@ export function PromptInput({ value, onChange }: PromptInputProps) { e.preventDefault(); if (!value.trim() || mutation.isPending || !hasEnoughCredits) return; + const fileIds = uploadedFiles.map((f) => f.file_id); mutation.mutate( - { prompt: value.trim(), director_tier: tier }, + { + prompt: value.trim(), + director_tier: tier, + ...(fileIds.length > 0 && { file_ids: fileIds }), + }, { onSuccess: (data) => { void navigate({ to: "/investigation/$id", params: { id: data.id } }); @@ -49,6 +58,7 @@ export function PromptInput({ value, onChange }: PromptInputProps) { className="w-full rounded-sm border border-border bg-surface p-4 text-sm leading-relaxed placeholder:text-muted-foreground/50 focus:outline-none focus:ring-2 focus:ring-primary/40" rows={5} /> +
diff --git a/console/src/features/investigation/hooks/use-upload.ts b/console/src/features/investigation/hooks/use-upload.ts new file mode 100644 index 0000000..457f951 --- /dev/null +++ b/console/src/features/investigation/hooks/use-upload.ts @@ -0,0 +1,13 @@ +import { useMutation } from "@tanstack/react-query"; +import { apiUpload } from "@/shared/lib/api"; +import type { UploadResponse } from "../types"; + +export function useUpload() { + return useMutation({ + mutationFn: async (file: File): Promise => { + const form = new FormData(); + form.append("file", file); + return apiUpload("/upload", form); + }, + }); +} diff --git a/console/src/features/investigation/types.ts b/console/src/features/investigation/types.ts index 28cc592..461581c 100644 --- a/console/src/features/investigation/types.ts +++ b/console/src/features/investigation/types.ts @@ -236,9 +236,29 @@ export interface ValidationMetricsData { export type DirectorTier = "haiku" | "sonnet" | "opus"; +export interface TabularPreview { + columns: string[]; + dtypes: string[]; + row_count: number; + sample_rows: string[][]; +} + +export interface DocumentPreview { + text: string; + page_count: number; +} + +export interface UploadResponse { + file_id: string; + filename: string; + content_type: string; + preview: TabularPreview | DocumentPreview; +} + export interface InvestigationRequest { prompt: string; director_tier?: DirectorTier; + file_ids?: string[]; } export interface InvestigationResponse { diff --git a/console/src/shared/lib/api.ts b/console/src/shared/lib/api.ts index a202928..9e500c6 100644 --- a/console/src/shared/lib/api.ts +++ b/console/src/shared/lib/api.ts @@ -63,6 +63,33 @@ export async function apiFetch( return response.json() as Promise; } +/** + * Upload a file via multipart/form-data with proper auth headers. + * Does NOT set Content-Type -- browser sets it with boundary for FormData. + */ +export async function apiUpload( + path: string, + body: FormData, +): Promise { + const authHeaders = await getAuthHeaders(); + + const response = await fetch(`${BASE_URL}${path}`, { + method: "POST", + headers: authHeaders, + body, + }); + + if (!response.ok) { + const error: ApiError = { + status: response.status, + message: await response.text(), + }; + throw error; + } + + return response.json() as Promise; +} + /** * Build an SSE stream URL with auth token as query parameter. * EventSource does not support custom headers, so we pass the token as ?token=xxx. diff --git a/docs/architecture.md b/docs/architecture.md index 1788be3..c392588 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,10 +34,10 @@ Exercise physiology and sports medicine research: evidence-based training analys Nutrition science research: supplement evidence analysis, supplement label lookup (NIH DSLD), nutrient data (USDA FoodData), supplement safety monitoring (OpenFDA CAERS), drug interaction screening (RxNav), DRI-based nutrient adequacy assessment, nutrient ratio analysis, and inflammatory index scoring. Uses Semantic Scholar for literature search. ### Impact -Social program evaluation. Provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities, repository ABCs, and infrastructure clients for three external APIs. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools. Future phases will add Mexico/US-specific data sources and document upload. See `docs/impact-evaluation-domain.md` for full design. +Social program evaluation. Provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities, repository ABCs, and infrastructure clients for three external APIs. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools. Future phases will add Mexico/US-specific data sources. See `docs/impact-evaluation-domain.md` for full design. ### Investigation -Hypothesis-driven agent orchestration. Manages the Claude-driven research loop: literature survey, hypothesis formulation (with predictions, criteria, scope), parallel experiment execution, criteria-based evaluation, negative controls, and synthesis. Uses multi-model architecture (Director/Researcher/Summarizer) with user-guided steering, domain classification, and multi-investigation memory. Includes domain configuration system (`DomainConfig` + `DomainRegistry`) for pluggable scientific domains with tool tagging, score definitions, prompt adaptation, and visualization control. Each domain config includes `tool_examples` in `experiment_examples` with realistic tool chaining patterns for complete tool coverage. Optional MCP bridge (`MCPBridge`) connects to external MCP servers for extensibility (e.g. Excalidraw for visual summaries). +Hypothesis-driven agent orchestration. Manages the Claude-driven research loop: literature survey, hypothesis formulation (with predictions, criteria, scope), parallel experiment execution, criteria-based evaluation, negative controls, and synthesis. Uses multi-model architecture (Director/Researcher/Summarizer) with user-guided steering, domain classification, and multi-investigation memory. Supports document upload (CSV/XLSX/PDF) with prompt injection and queryable data via `query_uploaded_data` tool. Includes domain configuration system (`DomainConfig` + `DomainRegistry`) for pluggable scientific domains with tool tagging, score definitions, prompt adaptation, and visualization control. Each domain config includes `tool_examples` in `experiment_examples` with realistic tool chaining patterns for complete tool coverage. Optional MCP bridge (`MCPBridge`) connects to external MCP servers for extensibility (e.g. Excalidraw for visual summaries). ## Multi-Model Architecture @@ -47,7 +47,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) │ NO tool access, structured JSON responses only │ - ├── Sonnet 4.5 (Researcher) -- Executes experiments with 84 domain-filtered tools (10-20 calls, parallel x2) + ├── Sonnet 4.5 (Researcher) -- Executes experiments with 85 domain-filtered tools (10-20 calls, parallel x2) │ Tool-calling loop with max_iterations_per_experiment guard │ └── Haiku 4.5 (Summarizer) -- Compresses large tool outputs >2000 chars, PICO+classification, evidence grading @@ -233,8 +233,8 @@ api/ -> investigation/application/ only ### Multi-Model (Default) -1. User submits research prompt via Console (or selects a template) -2. API creates Investigation, persists to PostgreSQL, starts MultiModelOrchestrator +1. User submits research prompt via Console (or selects a template), optionally with uploaded files (CSV/XLSX/PDF via `POST /upload`) +2. API creates Investigation, links any uploaded files, persists to PostgreSQL, starts MultiModelOrchestrator (with uploaded data context injected into prompts) 3. **Haiku** decomposes prompt via PICO framework (Population, Intervention, Comparison, Outcome) and classifies domain in a single call; queries past completed investigations in same domain 4. **Domain detection** -- `DomainRegistry.detect()` returns `list[DomainConfig]` (one or more); `merge_domain_configs()` creates merged config for cross-domain; emits `DomainDetected` SSE event with display config (includes `domains` sub-list for multi-domain); researcher tool list filtered to merged domain-relevant tools 5. **Researcher** (Sonnet) conducts structured literature survey with domain-filtered tools, citation chasing, evidence-level grading; **Haiku** grades body-of-evidence (GRADE-adapted) and self-assesses quality (AMSTAR-2); emits `LiteratureSurveyCompleted` event diff --git a/docs/impact-evaluation-domain.md b/docs/impact-evaluation-domain.md index f0c1590..ad10564 100644 --- a/docs/impact-evaluation-domain.md +++ b/docs/impact-evaluation-domain.md @@ -474,12 +474,16 @@ File upload ## 10. Implementation Phases -### Phase 1: Foundation (document upload + universal APIs) -- File upload API (CSV, PDF, Excel) -- `extract_program_data` tool -- World Bank, FRED, WHO API clients -- Basic `compare_programs` tool using existing `run_statistical_test` -- 2 new viz tools: `render_program_dashboard`, `render_geographic_comparison` +### Phase 1: Foundation (document upload + universal APIs) -- DONE +- [x] File upload API (`POST /upload`, CSV/XLSX/PDF, 50MB limit, multipart/form-data) +- [x] `FileProcessor` service + `UploadedFile` domain entity (TabularData/DocumentData) +- [x] `query_uploaded_data` tool (intercepted by ToolDispatcher, pandas filtering) +- [x] PostgreSQL `uploaded_files` table with JSONB parsed_data +- [x] Uploaded data injected into Director/Researcher prompts as `` XML +- [x] Frontend: `FileUpload.tsx` drag-and-drop + `DataPreview.tsx` + `use-upload.ts` hook +- [x] World Bank, FRED, WHO API clients (done in Phase 13A) +- [x] `compare_programs` tool using existing `run_statistical_test` (done in Phase 13A) +- [x] 2 new viz tools: `render_program_dashboard`, `render_geographic_comparison` (done in Phase 13B) ### Phase 2: Causal Inference Engine -- DONE (lives in `analysis/`) - `estimate_did`, `estimate_psm`, `estimate_rdd`, `estimate_synthetic_control` tools (in `analysis/tools.py`) diff --git a/docs/roadmap.md b/docs/roadmap.md index 30ece61..5489f64 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -471,7 +471,7 @@ Enhancement Enhancement Sports | Phase 13: Impact Evaluation Domain -- IN PROGRESS (Causal inference + Document upload + MX/US APIs) - 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) -> 13C (Mexico) -> 13D (US) -> 13E (MCP) + 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) DONE -> 13C (Mexico) -> 13D (US) -> 13E (MCP) | Phase 14: Scientific Engine Hardening -- IN PROGRESS (parallel with 13A-2) 14A (Orchestrator Split) ──┬── 14B (Route Cleanup) ──┬── 14D (Tool Enrichment) @@ -1061,15 +1061,23 @@ Refactored causal inference from impact-specific to domain-agnostic. All causal **Counts:** 78 -> 84 tools (6 causal + 1 cost-effectiveness - 2 moved + 2 viz), 15 -> 17 viz tools, 5 -> 3 impact tools, 0 -> 6 causal tools (analysis/), 7 -> 9 templates. -### Phase 13A-2: Document Upload (TODO -- deferred post-hackathon) - -- [ ] File upload API (CSV, PDF, Excel) via multipart/form-data on `POST /investigate` -- [ ] PDF text extraction (`pymupdf`) + Haiku summarization for long documents -- [ ] CSV/Excel parsing (`pandas` + `openpyxl`) with schema detection and summary statistics -- [ ] Inject uploaded data into Director prompt as `` XML block -- [ ] `extract_program_data` tool -- parse uploaded files into structured program data -- [ ] Frontend: `FileUpload.tsx` drag-and-drop component in `PromptInput` -- [ ] Frontend: `DataPreview.tsx` for uploaded dataset preview +### Phase 13A-2: Document Upload -- DONE + +- [x] File upload API (`POST /upload`) with multipart/form-data, 50MB limit, WorkOS auth +- [x] PDF text extraction (`pymupdf`) with page-level parsing and 8000-char truncation +- [x] CSV/Excel parsing (`pandas` + `openpyxl`) with summary statistics and sample rows +- [x] `UploadedFile` domain entity with `TabularData` / `DocumentData` frozen dataclasses +- [x] `FileProcessor` application service (zero external deps in domain layer) +- [x] PostgreSQL `uploaded_files` table with JSONB `parsed_data` column +- [x] Inject uploaded data into Director/Researcher prompts as `` XML block +- [x] `query_uploaded_data` tool -- intercepted by `ToolDispatcher`, pandas-based filtering (eq/gt/lt/gte/lte/contains) +- [x] Upload-first flow: `POST /upload` returns preview, `POST /investigate` references `file_ids` +- [x] Frontend: `FileUpload.tsx` drag-and-drop component with progress spinner +- [x] Frontend: `DataPreview.tsx` compact inline cards for tabular/document previews +- [x] Frontend: `use-upload.ts` TanStack Query mutation hook +- [x] Tests: 16 server tests (FileProcessor + domain entities + query handler) + 16 console tests + +**Counts:** 84 -> 85 tools (added `query_uploaded_data`). Investigation tools: 7 -> 8. ### Phase 13C: Mexico Integration diff --git a/server/pyproject.toml b/server/pyproject.toml index c3144b7..8ef00e8 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -18,6 +18,8 @@ dependencies = [ "scikit-learn>=1.6,<2", "pandas>=2,<3", "numpy>=2,<3", + "openpyxl>=3.1,<4", + "pymupdf>=1.25,<2", "scipy>=1.14,<2", "requests>=2.32,<3", "joblib>=1.4,<2", diff --git a/server/src/ehrlich/api/app.py b/server/src/ehrlich/api/app.py index 668ff07..5dc10c2 100644 --- a/server/src/ehrlich/api/app.py +++ b/server/src/ehrlich/api/app.py @@ -12,6 +12,7 @@ from ehrlich.api.routes.methodology import router as methodology_router from ehrlich.api.routes.molecule import router as molecule_router from ehrlich.api.routes.stats import router as stats_router +from ehrlich.api.routes.upload import router as upload_router from ehrlich.config import get_settings logger = logging.getLogger(__name__) @@ -87,5 +88,6 @@ def create_app() -> FastAPI: app.include_router(methodology_router, prefix="/api/v1") app.include_router(molecule_router, prefix="/api/v1") app.include_router(stats_router, prefix="/api/v1") + app.include_router(upload_router, prefix="/api/v1") return app diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index 26f06b4..415f6cc 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -58,6 +58,7 @@ design_experiment, evaluate_hypothesis, propose_hypothesis, + query_uploaded_data, record_finding, record_negative_control, search_prior_research, @@ -158,6 +159,7 @@ class InvestigateRequest(BaseModel): prompt: str director_tier: str = "opus" + file_ids: list[str] = [] class CreditBalanceResponse(BaseModel): @@ -335,6 +337,7 @@ def _build_registry() -> ToolRegistry: ("evaluate_hypothesis", evaluate_hypothesis, None), ("record_negative_control", record_negative_control, None), ("search_prior_research", search_prior_research, None), + ("query_uploaded_data", query_uploaded_data, None), ] for name, func, tags in tagged_tools: registry.register(name, func, tags) @@ -441,6 +444,16 @@ async def start_investigation( # Save investigation first (credit_transactions FK references investigations) await repo.save(investigation, user_id=str(db_user["id"])) + # Attach uploaded files + if request.file_ids: + from ehrlich.api.routes.upload import get_pending_upload + + for fid in request.file_ids: + uploaded = get_pending_upload(fid) + if uploaded: + investigation.uploaded_files.append(uploaded) + await repo.save_uploaded_file(investigation.id, uploaded) + if not is_byok: deducted = await repo.deduct_credits(user["workos_id"], credit_cost, investigation.id) if not deducted: @@ -484,6 +497,8 @@ async def stream_investigation( investigation = _active_investigations.get(investigation_id) if investigation is None: investigation = await repo.get_by_id(investigation_id) + if investigation is not None: + investigation.uploaded_files = await repo.get_uploaded_files(investigation.id) if investigation is None: raise HTTPException(status_code=404, detail="Investigation not found") diff --git a/server/src/ehrlich/api/routes/upload.py b/server/src/ehrlich/api/routes/upload.py new file mode 100644 index 0000000..3a37b66 --- /dev/null +++ b/server/src/ehrlich/api/routes/upload.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from fastapi import APIRouter, Depends, HTTPException, UploadFile +from pydantic import BaseModel + +from ehrlich.api.auth import get_current_user +from ehrlich.investigation.application.file_processor import FileProcessor + +if TYPE_CHECKING: + from ehrlich.investigation.domain.uploaded_file import UploadedFile as UploadedFileEntity + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["upload"]) + +_require_user = Depends(get_current_user) +_processor = FileProcessor() + +_MAX_TOTAL_SIZE = 50 * 1024 * 1024 # 50 MB + +# Pending uploads waiting to be linked to an investigation. +# Keyed by file_id, removed when claimed by POST /investigate. +_pending_uploads: dict[str, UploadedFileEntity] = {} + + +class UploadResponse(BaseModel): + file_id: str + filename: str + content_type: str + preview: dict[str, Any] + + +@router.post("/upload", response_model=UploadResponse) +async def upload_file( + file: UploadFile, + user: dict[str, Any] = _require_user, +) -> UploadResponse: + """Upload a CSV, Excel, or PDF file for use in investigations.""" + if not file.filename: + raise HTTPException(status_code=400, detail="Filename is required") + + content = await file.read() + if len(content) > _MAX_TOTAL_SIZE: + raise HTTPException(status_code=413, detail="File exceeds 50MB limit") + + try: + uploaded = _processor.process(file.filename, content) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + _pending_uploads[uploaded.file_id] = uploaded + + preview: dict[str, Any] = {} + if uploaded.tabular: + preview = { + "type": "tabular", + "columns": list(uploaded.tabular.columns), + "dtypes": list(uploaded.tabular.dtypes), + "row_count": uploaded.tabular.row_count, + "sample_rows": [list(r) for r in uploaded.tabular.sample_rows[:5]], + } + elif uploaded.document: + preview = { + "type": "document", + "text": uploaded.document.text[:500], + "page_count": uploaded.document.page_count, + } + + return UploadResponse( + file_id=uploaded.file_id, + filename=uploaded.filename, + content_type=uploaded.content_type, + preview=preview, + ) + + +def get_pending_upload(file_id: str) -> UploadedFileEntity | None: + """Retrieve and remove a pending upload by ID. Called from investigation routes.""" + return _pending_uploads.pop(file_id, None) diff --git a/server/src/ehrlich/investigation/application/file_processor.py b/server/src/ehrlich/investigation/application/file_processor.py new file mode 100644 index 0000000..952bae4 --- /dev/null +++ b/server/src/ehrlich/investigation/application/file_processor.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import io + +import pandas as pd + +from ehrlich.investigation.domain.uploaded_file import ( + DocumentData, + TabularData, + UploadedFile, +) + +_MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB +_MAX_SAMPLE_ROWS = 10 +_MAX_PDF_TEXT_LENGTH = 8000 +_ALLOWED_EXTENSIONS = frozenset({"csv", "xlsx", "pdf"}) + + +class FileProcessor: + """Parses uploaded files into domain entities.""" + + def process(self, filename: str, content: bytes) -> UploadedFile: + if not content: + msg = "File is empty" + raise ValueError(msg) + if len(content) > _MAX_FILE_SIZE: + msg = f"File exceeds {_MAX_FILE_SIZE // (1024 * 1024)}MB limit" + raise ValueError(msg) + + ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if ext not in _ALLOWED_EXTENSIONS: + msg = ( + f"Unsupported file type: .{ext}. Allowed: {', '.join(sorted(_ALLOWED_EXTENSIONS))}" + ) + raise ValueError(msg) + + if ext == "csv": + return self._process_csv(filename, content) + if ext == "xlsx": + return self._process_xlsx(filename, content) + return self._process_pdf(filename, content) + + def _process_csv(self, filename: str, content: bytes) -> UploadedFile: + df = pd.read_csv(io.BytesIO(content)) + return self._build_tabular(filename, "text/csv", df) + + def _process_xlsx(self, filename: str, content: bytes) -> UploadedFile: + df = pd.read_excel(io.BytesIO(content), engine="openpyxl") + return self._build_tabular( + filename, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + df, + ) + + def _process_pdf(self, filename: str, content: bytes) -> UploadedFile: + import pymupdf # lazy import -- only needed for PDF + + doc: pymupdf.Document = pymupdf.open(stream=content, filetype="pdf") # type: ignore[no-untyped-call] + pages: list[str] = [] + for i in range(len(doc)): + page = doc[i] + text: str = page.get_text() # type: ignore[attr-defined] + if text.strip(): + pages.append(text) + doc.close() # type: ignore[no-untyped-call] + + full_text = "\n\n".join(pages) + if len(full_text) > _MAX_PDF_TEXT_LENGTH: + full_text = full_text[:_MAX_PDF_TEXT_LENGTH] + "..." + + return UploadedFile( + filename=filename, + content_type="application/pdf", + document=DocumentData(text=full_text, page_count=len(pages)), + ) + + def _build_tabular(self, filename: str, content_type: str, df: pd.DataFrame) -> UploadedFile: + columns = tuple(str(c) for c in df.columns) + dtypes = tuple(str(d) for d in df.dtypes) + row_count = len(df) + + # Summary statistics for numeric columns + desc = df.describe() + summary_stats: dict[str, dict[str, float]] = {} + for col in desc.columns: + summary_stats[str(col)] = {str(k): round(float(v), 4) for k, v in desc[col].items()} + + # Sample rows (first N, all values as strings) + sample = df.head(_MAX_SAMPLE_ROWS) + sample_rows = tuple(tuple(str(v) for v in row) for row in sample.values) + + return UploadedFile( + filename=filename, + content_type=content_type, + tabular=TabularData( + columns=columns, + dtypes=dtypes, + row_count=row_count, + summary_stats=summary_stats, + sample_rows=sample_rows, + ), + ) diff --git a/server/src/ehrlich/investigation/application/prompts.py b/server/src/ehrlich/investigation/application/prompts.py index 97efc50..8ae6452 100644 --- a/server/src/ehrlich/investigation/application/prompts.py +++ b/server/src/ehrlich/investigation/application/prompts.py @@ -5,6 +5,7 @@ if TYPE_CHECKING: from ehrlich.investigation.domain.domain_config import DomainConfig from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.domain.uploaded_file import UploadedFile SCIENTIST_SYSTEM_PROMPT = """\ You are Ehrlich, an AI molecular discovery scientist named after \ @@ -900,6 +901,7 @@ def build_pico_and_classification_prompt( categories: frozenset[str] | None = None, + uploaded_data_context: str = "", ) -> str: """Build a combined domain classification + PICO decomposition prompt. @@ -937,7 +939,7 @@ def build_pico_and_classification_prompt( ' "outcome": "measurable endpoint",\n' ' "search_terms": ["broad query 1", "broad query 2", "broad query 3"]\n' "}\n" - "" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") ) @@ -1075,7 +1077,10 @@ def build_multi_investigation_context( return "\n".join(parts) -def build_formulation_prompt(config: DomainConfig) -> str: +def build_formulation_prompt( + config: DomainConfig, + uploaded_data_context: str = "", +) -> str: """Build Director formulation prompt adapted to the domain config.""" hyp_types = "|".join(config.hypothesis_types) if config.hypothesis_types else "other" examples = config.director_examples or "" @@ -1142,11 +1147,14 @@ def build_formulation_prompt(config: DomainConfig) -> str: " }\n" " ]\n" "}\n" - "" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") ) -def build_experiment_prompt(config: DomainConfig) -> str: +def build_experiment_prompt( + config: DomainConfig, + uploaded_data_context: str = "", +) -> str: """Build Director experiment design prompt adapted to the domain config.""" examples = config.experiment_examples or "" return ( @@ -1195,7 +1203,7 @@ def build_experiment_prompt(config: DomainConfig) -> str: ' "success_criteria": "What result would support the hypothesis",\n' ' "failure_criteria": "What result would refute the hypothesis"\n' "}\n" - "" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") ) @@ -1366,7 +1374,10 @@ def build_synthesis_prompt(config: DomainConfig) -> str: ) -def build_researcher_prompt(config: DomainConfig) -> str: +def build_researcher_prompt( + config: DomainConfig, + uploaded_data_context: str = "", +) -> str: """Build researcher experiment prompt adapted to the domain config.""" examples = config.experiment_examples or "" return ( @@ -1425,7 +1436,7 @@ def build_researcher_prompt(config: DomainConfig) -> str: "4. If a tool returns an error, try an alternative approach.\n" "5. Be quantitative: report exact numbers and scores.\n" "6. Use at least 3 tool calls in this experiment.\n" - "" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") ) @@ -1449,3 +1460,45 @@ def _build_prior_context(investigation: Investigation) -> str: parts.append(" ") parts.append("") return "\n".join(parts) + + +def build_uploaded_data_context(files: list[UploadedFile]) -> str: + """Build XML context block describing user-uploaded files. + + Injected into Director and Researcher prompts so the model is aware + of available datasets and can reference them via ``query_uploaded_data``. + """ + if not files: + return "" + parts: list[str] = [""] + for f in files: + if f.tabular: + t = f.tabular + cols = ", ".join(t.columns) + dtypes = ", ".join(t.dtypes) + parts.append(f' ') + parts.append(f" Columns ({len(t.columns)}): {cols}") + parts.append(f" Dtypes: {dtypes}") + parts.append(f" Rows: {t.row_count}") + if t.summary_stats: + stats_lines = [] + for col_name, stats in list(t.summary_stats.items())[:6]: + mean = stats.get("mean", "?") + std = stats.get("std", "?") + stats_lines.append(f"{col_name}: mean={mean}, std={std}") + parts.append(f" Stats: {'; '.join(stats_lines)}") + if t.sample_rows: + header = " | ".join(t.columns) + parts.append(f" Sample: {header}") + for row in t.sample_rows[:3]: + parts.append(f" {' | '.join(row)}") + parts.append(" ") + elif f.document: + d = f.document + excerpt = d.text[:500] + "..." if len(d.text) > 500 else d.text + parts.append(f' ') + parts.append(f" Pages: {d.page_count}") + parts.append(f" Excerpt: {excerpt}") + parts.append(" ") + parts.append("") + return "\n".join(parts) diff --git a/server/src/ehrlich/investigation/domain/investigation.py b/server/src/ehrlich/investigation/domain/investigation.py index 8a7642c..00587d8 100644 --- a/server/src/ehrlich/investigation/domain/investigation.py +++ b/server/src/ehrlich/investigation/domain/investigation.py @@ -13,6 +13,7 @@ from ehrlich.investigation.domain.hypothesis import Hypothesis from ehrlich.investigation.domain.negative_control import NegativeControl from ehrlich.investigation.domain.positive_control import PositiveControl + from ehrlich.investigation.domain.uploaded_file import UploadedFile class InvestigationStatus(StrEnum): @@ -43,6 +44,7 @@ class Investigation: created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) trained_model_ids: list[str] = field(default_factory=list) cost_data: dict[str, object] = field(default_factory=dict) + uploaded_files: list[UploadedFile] = field(default_factory=list) def record_finding(self, finding: Finding) -> None: self.findings.append(finding) diff --git a/server/src/ehrlich/investigation/domain/repository.py b/server/src/ehrlich/investigation/domain/repository.py index 1678347..210d100 100644 --- a/server/src/ehrlich/investigation/domain/repository.py +++ b/server/src/ehrlich/investigation/domain/repository.py @@ -5,6 +5,7 @@ if TYPE_CHECKING: from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.domain.uploaded_file import UploadedFile class InvestigationRepository(ABC): @@ -28,3 +29,9 @@ async def get_events(self, investigation_id: str) -> list[dict[str, str]]: ... @abstractmethod async def search_findings(self, query: str, limit: int = 20) -> list[dict[str, Any]]: ... + + @abstractmethod + async def save_uploaded_file(self, investigation_id: str, file: UploadedFile) -> None: ... + + @abstractmethod + async def get_uploaded_files(self, investigation_id: str) -> list[UploadedFile]: ... diff --git a/server/src/ehrlich/investigation/domain/uploaded_file.py b/server/src/ehrlich/investigation/domain/uploaded_file.py new file mode 100644 index 0000000..d13734b --- /dev/null +++ b/server/src/ehrlich/investigation/domain/uploaded_file.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class TabularData: + """Parsed tabular file data (CSV or Excel).""" + + columns: tuple[str, ...] + dtypes: tuple[str, ...] + row_count: int + summary_stats: dict[str, dict[str, float]] + sample_rows: tuple[tuple[str, ...], ...] + + +@dataclass(frozen=True) +class DocumentData: + """Parsed document data (PDF).""" + + text: str + page_count: int + + +@dataclass(frozen=True) +class UploadedFile: + """A user-uploaded file with parsed content.""" + + filename: str + content_type: str + file_id: str = field(default_factory=lambda: str(uuid.uuid4())) + tabular: TabularData | None = None + document: DocumentData | None = None + + def __post_init__(self) -> None: + if not self.filename: + msg = "filename is required" + raise ValueError(msg) + if self.tabular is None and self.document is None: + msg = "UploadedFile must have either tabular or document data" + raise ValueError(msg) diff --git a/server/src/ehrlich/investigation/infrastructure/repository.py b/server/src/ehrlich/investigation/infrastructure/repository.py index 1302700..0fceab6 100644 --- a/server/src/ehrlich/investigation/infrastructure/repository.py +++ b/server/src/ehrlich/investigation/infrastructure/repository.py @@ -2,6 +2,7 @@ import json import logging +import uuid from datetime import UTC, datetime from typing import Any @@ -14,6 +15,7 @@ from ehrlich.investigation.domain.investigation import Investigation, InvestigationStatus from ehrlich.investigation.domain.negative_control import NegativeControl from ehrlich.investigation.domain.repository import InvestigationRepository as _Base +from ehrlich.investigation.domain.uploaded_file import DocumentData, TabularData, UploadedFile logger = logging.getLogger(__name__) @@ -74,6 +76,17 @@ CREATE INDEX IF NOT EXISTS idx_investigations_user ON investigations(user_id); CREATE INDEX IF NOT EXISTS idx_events_investigation ON events(investigation_id); CREATE INDEX IF NOT EXISTS idx_credit_transactions_user ON credit_transactions(user_id); + +CREATE TABLE IF NOT EXISTS uploaded_files ( + id UUID PRIMARY KEY, + investigation_id TEXT NOT NULL REFERENCES investigations(id) ON DELETE CASCADE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + parsed_data JSONB NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_uploaded_files_investigation ON uploaded_files(investigation_id); """ @@ -280,6 +293,74 @@ async def search_findings(self, query: str, limit: int = 20) -> list[dict[str, A ) return results[:limit] + async def save_uploaded_file(self, investigation_id: str, file: UploadedFile) -> None: + pool = self._get_pool() + parsed_data: dict[str, Any] = {} + if file.tabular: + parsed_data["type"] = "tabular" + parsed_data["columns"] = list(file.tabular.columns) + parsed_data["dtypes"] = list(file.tabular.dtypes) + parsed_data["row_count"] = file.tabular.row_count + parsed_data["summary_stats"] = file.tabular.summary_stats + parsed_data["sample_rows"] = [list(r) for r in file.tabular.sample_rows] + elif file.document: + parsed_data["type"] = "document" + parsed_data["text"] = file.document.text + parsed_data["page_count"] = file.document.page_count + + async with pool.acquire() as conn: + await conn.execute( + """INSERT INTO uploaded_files + (id, investigation_id, filename, content_type, parsed_data) + VALUES ($1, $2, $3, $4, $5)""", + uuid.UUID(file.file_id), + investigation_id, + file.filename, + file.content_type, + json.dumps(parsed_data), + ) + + async def get_uploaded_files(self, investigation_id: str) -> list[UploadedFile]: + pool = self._get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch( + "SELECT id, filename, content_type, parsed_data " + "FROM uploaded_files WHERE investigation_id = $1", + investigation_id, + ) + files: list[UploadedFile] = [] + for row in rows: + data = ( + json.loads(row["parsed_data"]) + if isinstance(row["parsed_data"], str) + else row["parsed_data"] + ) + tabular = None + document = None + if data.get("type") == "tabular": + tabular = TabularData( + columns=tuple(data["columns"]), + dtypes=tuple(data["dtypes"]), + row_count=data["row_count"], + summary_stats=data["summary_stats"], + sample_rows=tuple(tuple(r) for r in data["sample_rows"]), + ) + elif data.get("type") == "document": + document = DocumentData( + text=data["text"], + page_count=data["page_count"], + ) + files.append( + UploadedFile( + file_id=str(row["id"]), + filename=row["filename"], + content_type=row["content_type"], + tabular=tabular, + document=document, + ) + ) + return files + # -- User management (PostgreSQL-specific, not on ABC) ------------------------- async def get_or_create_user(self, workos_id: str, email: str) -> dict[str, Any]: diff --git a/server/src/ehrlich/investigation/tools.py b/server/src/ehrlich/investigation/tools.py index 3e89d2a..830ec4e 100644 --- a/server/src/ehrlich/investigation/tools.py +++ b/server/src/ehrlich/investigation/tools.py @@ -3,6 +3,31 @@ import json +async def query_uploaded_data( + file_id: str, + columns: str | None = None, + filter_column: str | None = None, + filter_op: str | None = None, + filter_value: str | None = None, + head: int = 50, +) -> str: + """Query data from a user-uploaded file (CSV, Excel, or PDF). + + For tabular files: select columns, filter rows, and return JSON results. + For PDF files: return the extracted text content. + + Args: + file_id: The ID of the uploaded file to query + columns: Comma-separated column names to select (omit for all columns) + filter_column: Column name to filter on + filter_op: Filter operation: eq, gt, lt, gte, lte, contains + filter_value: Value to filter against + head: Maximum number of rows to return (default 50) + """ + # Stub -- intercepted by orchestrator._dispatch_tool() + return json.dumps({"status": "query_dispatched", "file_id": file_id}) + + async def search_prior_research(query: str, limit: int = 10) -> str: """Search Ehrlich's own past investigation findings for prior knowledge. diff --git a/server/tests/api/test_methodology.py b/server/tests/api/test_methodology.py index 3767c97..ec0c391 100644 --- a/server/tests/api/test_methodology.py +++ b/server/tests/api/test_methodology.py @@ -72,7 +72,7 @@ def test_tools_grouped_by_context(self, client: TestClient) -> None: def test_total_tool_count(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() total = sum(len(g["tools"]) for g in data["tools"]) - assert total == 73 + assert total == 85 def test_tool_has_name_and_description(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() diff --git a/server/tests/api/test_stats.py b/server/tests/api/test_stats.py index 2cc400f..d521fa6 100644 --- a/server/tests/api/test_stats.py +++ b/server/tests/api/test_stats.py @@ -24,7 +24,7 @@ def test_returns_all_fields(self, client: TestClient) -> None: def test_tool_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["tool_count"] == 73 + assert data["tool_count"] == 85 def test_domain_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") diff --git a/server/tests/investigation/test_file_processor.py b/server/tests/investigation/test_file_processor.py new file mode 100644 index 0000000..f1de5b3 --- /dev/null +++ b/server/tests/investigation/test_file_processor.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import io + +import pytest + +from ehrlich.investigation.application.file_processor import FileProcessor + + +@pytest.fixture +def processor() -> FileProcessor: + return FileProcessor() + + +class TestCSVProcessing: + def test_valid_csv(self, processor: FileProcessor) -> None: + content = b"name,age,score\nAlice,30,85.5\nBob,25,92.0\nCharlie,35,78.3" + result = processor.process("data.csv", content) + assert result.filename == "data.csv" + assert result.content_type == "text/csv" + assert result.tabular is not None + assert result.document is None + assert result.tabular.columns == ("name", "age", "score") + assert result.tabular.row_count == 3 + assert len(result.tabular.sample_rows) == 3 + + def test_csv_dtypes(self, processor: FileProcessor) -> None: + content = b"x,y\n1,hello\n2,world" + result = processor.process("types.csv", content) + assert result.tabular is not None + assert len(result.tabular.dtypes) == 2 + assert "int" in result.tabular.dtypes[0] + assert "object" in result.tabular.dtypes[1] + + def test_csv_summary_stats(self, processor: FileProcessor) -> None: + content = b"value\n1\n2\n3\n4\n5" + result = processor.process("stats.csv", content) + assert result.tabular is not None + stats = result.tabular.summary_stats + assert "value" in stats + assert stats["value"]["mean"] == 3.0 + assert stats["value"]["count"] == 5.0 + + def test_csv_sample_rows_limited(self, processor: FileProcessor) -> None: + lines = ["x"] + [str(i) for i in range(20)] + content = "\n".join(lines).encode() + result = processor.process("big.csv", content) + assert result.tabular is not None + assert len(result.tabular.sample_rows) <= 10 + + def test_csv_sample_rows_as_strings(self, processor: FileProcessor) -> None: + content = b"a,b\n1,2.5\n3,4.0" + result = processor.process("nums.csv", content) + assert result.tabular is not None + for row in result.tabular.sample_rows: + for val in row: + assert isinstance(val, str) + + def test_csv_file_id_generated(self, processor: FileProcessor) -> None: + content = b"col\n1" + result = processor.process("id.csv", content) + assert result.file_id + assert len(result.file_id) == 36 # UUID format + + +class TestXLSXProcessing: + def test_valid_xlsx(self, processor: FileProcessor) -> None: + import openpyxl + + wb = openpyxl.Workbook() + ws = wb.active + assert ws is not None + ws.append(["name", "value"]) + ws.append(["A", 1]) + ws.append(["B", 2]) + buf = io.BytesIO() + wb.save(buf) + content = buf.getvalue() + + result = processor.process("data.xlsx", content) + assert result.filename == "data.xlsx" + assert result.tabular is not None + assert result.tabular.columns == ("name", "value") + assert result.tabular.row_count == 2 + + def test_xlsx_content_type(self, processor: FileProcessor) -> None: + import openpyxl + + wb = openpyxl.Workbook() + ws = wb.active + assert ws is not None + ws.append(["x"]) + ws.append([1]) + buf = io.BytesIO() + wb.save(buf) + + result = processor.process("test.xlsx", buf.getvalue()) + assert "spreadsheetml" in result.content_type + + +class TestPDFProcessing: + def test_valid_pdf(self, processor: FileProcessor) -> None: + import pymupdf + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((50, 50), "Hello world test document") + content = doc.tobytes() + doc.close() + + result = processor.process("report.pdf", content) + assert result.filename == "report.pdf" + assert result.content_type == "application/pdf" + assert result.document is not None + assert result.tabular is None + assert "Hello world" in result.document.text + assert result.document.page_count >= 1 + + def test_pdf_text_truncation(self, processor: FileProcessor) -> None: + import pymupdf + + doc = pymupdf.open() + page = doc.new_page() + # Insert a lot of text + long_text = "A" * 500 + for y in range(50, 800, 15): + page.insert_text((50, y), long_text) + # Add more pages + for _ in range(5): + p = doc.new_page() + for y in range(50, 800, 15): + p.insert_text((50, y), long_text) + content = doc.tobytes() + doc.close() + + result = processor.process("long.pdf", content) + assert result.document is not None + assert len(result.document.text) <= 8001 + 3 # 8000 + "..." + + def test_pdf_multiple_pages(self, processor: FileProcessor) -> None: + import pymupdf + + doc = pymupdf.open() + for i in range(3): + page = doc.new_page() + page.insert_text((50, 50), f"Page {i + 1} content") + content = doc.tobytes() + doc.close() + + result = processor.process("multi.pdf", content) + assert result.document is not None + assert result.document.page_count == 3 + + +class TestValidation: + def test_empty_file_raises(self, processor: FileProcessor) -> None: + with pytest.raises(ValueError, match="empty"): + processor.process("empty.csv", b"") + + def test_unsupported_extension_raises(self, processor: FileProcessor) -> None: + with pytest.raises(ValueError, match="Unsupported"): + processor.process("data.json", b'{"key": "value"}') + + def test_no_extension_raises(self, processor: FileProcessor) -> None: + with pytest.raises(ValueError, match="Unsupported"): + processor.process("noext", b"some data") + + def test_oversized_file_raises(self, processor: FileProcessor) -> None: + huge = b"x" * (51 * 1024 * 1024) + with pytest.raises(ValueError, match="50MB"): + processor.process("huge.csv", huge) + + def test_txt_extension_raises(self, processor: FileProcessor) -> None: + with pytest.raises(ValueError, match="Unsupported"): + processor.process("notes.txt", b"some text") diff --git a/server/tests/investigation/test_uploaded_file.py b/server/tests/investigation/test_uploaded_file.py new file mode 100644 index 0000000..b7fedbe --- /dev/null +++ b/server/tests/investigation/test_uploaded_file.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import json + +import pytest + +from ehrlich.investigation.domain.uploaded_file import ( + DocumentData, + TabularData, + UploadedFile, +) + + +class TestUploadedFile: + def test_tabular_file(self) -> None: + tab = TabularData( + columns=("a", "b"), + dtypes=("int64", "float64"), + row_count=2, + summary_stats={"a": {"mean": 1.5}}, + sample_rows=(("1", "2.0"), ("2", "3.0")), + ) + f = UploadedFile(filename="data.csv", content_type="text/csv", tabular=tab) + assert f.filename == "data.csv" + assert f.tabular is not None + assert f.document is None + assert len(f.file_id) == 36 + + def test_document_file(self) -> None: + doc = DocumentData(text="Hello world", page_count=1) + f = UploadedFile(filename="report.pdf", content_type="application/pdf", document=doc) + assert f.document is not None + assert f.tabular is None + assert f.document.page_count == 1 + + def test_must_have_data(self) -> None: + with pytest.raises(ValueError, match="either tabular or document"): + UploadedFile(filename="empty.csv", content_type="text/csv") + + def test_filename_required(self) -> None: + tab = TabularData( + columns=("x",), + dtypes=("int64",), + row_count=1, + summary_stats={}, + sample_rows=(("1",),), + ) + with pytest.raises(ValueError, match="filename"): + UploadedFile(filename="", content_type="text/csv", tabular=tab) + + def test_frozen(self) -> None: + tab = TabularData( + columns=("x",), + dtypes=("int64",), + row_count=1, + summary_stats={}, + sample_rows=(("1",),), + ) + f = UploadedFile(filename="f.csv", content_type="text/csv", tabular=tab) + with pytest.raises(AttributeError): + f.filename = "other.csv" # type: ignore[misc] + + +class TestQueryUploadedDataHandler: + """Test the orchestrator's _handle_query_uploaded_data method.""" + + def _make_orchestrator(self) -> object: + """Create a minimal orchestrator with uploaded files.""" + from unittest.mock import MagicMock + + from ehrlich.investigation.application.multi_orchestrator import ( + MultiModelOrchestrator, + ) + + orch = MultiModelOrchestrator( + director=MagicMock(), + researcher=MagicMock(), + summarizer=MagicMock(), + registry=MagicMock(), + ) + + tab = TabularData( + columns=("name", "age", "score"), + dtypes=("object", "int64", "float64"), + row_count=3, + summary_stats={"age": {"mean": 30.0}, "score": {"mean": 85.0}}, + sample_rows=( + ("Alice", "30", "85.5"), + ("Bob", "25", "92.0"), + ("Charlie", "35", "78.3"), + ), + ) + tabular_file = UploadedFile( + file_id="tab-123", + filename="data.csv", + content_type="text/csv", + tabular=tab, + ) + + doc = DocumentData(text="This is a test document about education policy.", page_count=2) + doc_file = UploadedFile( + file_id="doc-456", + filename="report.pdf", + content_type="application/pdf", + document=doc, + ) + + orch._uploaded_files = {"tab-123": tabular_file, "doc-456": doc_file} + return orch + + def test_query_all_columns(self) -> None: + orch = self._make_orchestrator() + result = json.loads(orch._handle_query_uploaded_data({"file_id": "tab-123"})) # type: ignore[attr-defined] + assert result["type"] == "tabular" + assert result["returned_rows"] == 3 + assert result["columns"] == ["name", "age", "score"] + + def test_query_specific_columns(self) -> None: + orch = self._make_orchestrator() + result = json.loads( + orch._handle_query_uploaded_data( + { # type: ignore[attr-defined] + "file_id": "tab-123", + "columns": "name,score", + } + ) + ) + assert result["columns"] == ["name", "score"] + + def test_query_filter_eq(self) -> None: + orch = self._make_orchestrator() + result = json.loads( + orch._handle_query_uploaded_data( + { # type: ignore[attr-defined] + "file_id": "tab-123", + "filter_column": "name", + "filter_op": "eq", + "filter_value": "Alice", + } + ) + ) + assert result["returned_rows"] == 1 + + def test_query_filter_gt(self) -> None: + orch = self._make_orchestrator() + result = json.loads( + orch._handle_query_uploaded_data( + { # type: ignore[attr-defined] + "file_id": "tab-123", + "filter_column": "age", + "filter_op": "gt", + "filter_value": "28", + } + ) + ) + assert result["returned_rows"] == 2 # Alice (30) and Charlie (35) + + def test_query_head_limit(self) -> None: + orch = self._make_orchestrator() + result = json.loads( + orch._handle_query_uploaded_data( + { # type: ignore[attr-defined] + "file_id": "tab-123", + "head": 2, + } + ) + ) + assert result["returned_rows"] == 2 + + def test_query_not_found(self) -> None: + orch = self._make_orchestrator() + result = json.loads(orch._handle_query_uploaded_data({"file_id": "missing"})) # type: ignore[attr-defined] + assert "error" in result + + def test_query_document(self) -> None: + orch = self._make_orchestrator() + result = json.loads(orch._handle_query_uploaded_data({"file_id": "doc-456"})) # type: ignore[attr-defined] + assert result["type"] == "document" + assert "education policy" in result["text"] + assert result["page_count"] == 2 + + def test_query_document_keyword_filter(self) -> None: + orch = self._make_orchestrator() + result = json.loads( + orch._handle_query_uploaded_data( + { # type: ignore[attr-defined] + "file_id": "doc-456", + "filter_value": "education", + } + ) + ) + assert "education" in result["text"].lower() + + def test_query_filter_contains(self) -> None: + orch = self._make_orchestrator() + result = json.loads( + orch._handle_query_uploaded_data( + { # type: ignore[attr-defined] + "file_id": "tab-123", + "filter_column": "name", + "filter_op": "contains", + "filter_value": "li", + } + ) + ) + # Alice and Charlie contain "li" + assert result["returned_rows"] == 2 + + +class TestPromptInjection: + def test_empty_files(self) -> None: + from ehrlich.investigation.application.prompts import ( + build_uploaded_data_context, + ) + + assert build_uploaded_data_context([]) == "" + + def test_tabular_context(self) -> None: + from ehrlich.investigation.application.prompts import ( + build_uploaded_data_context, + ) + + tab = TabularData( + columns=("x", "y"), + dtypes=("int64", "float64"), + row_count=10, + summary_stats={"x": {"mean": 5.0, "std": 2.0}}, + sample_rows=(("1", "2.0"), ("3", "4.0")), + ) + f = UploadedFile(filename="data.csv", content_type="text/csv", tabular=tab) + ctx = build_uploaded_data_context([f]) + assert "" in ctx + assert "data.csv" in ctx + assert "x, y" in ctx or "x" in ctx + assert "10" in ctx + assert "" in ctx + + def test_document_context(self) -> None: + from ehrlich.investigation.application.prompts import ( + build_uploaded_data_context, + ) + + doc = DocumentData(text="Test content here", page_count=3) + f = UploadedFile(filename="report.pdf", content_type="application/pdf", document=doc) + ctx = build_uploaded_data_context([f]) + assert "" in ctx + assert "report.pdf" in ctx + assert "Test content" in ctx + assert "3" in ctx diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 7362f24..dddd343 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -1,5 +1,5 @@ export const STATS = { - tools: 84, + tools: 85, dataSources: 19, domains: 4, models: 3, From e2732318b0c1f2461f749a062c7bb172cfa3ade1 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 07:58:05 -0700 Subject: [PATCH 09/35] refactor(docs): reorganize docs into roadmap/, research/, and adr/ folders Split monolithic roadmap.md (1,453 lines) into docs/roadmap/ with 20 phase-numbered files and an index. Moved research/ into docs/research/ with new impact/ subdirectory. Created docs/adr/ for architecture decision records (business-strategy, domain-agnostic, impact-evaluation). Renamed unnumbered phases to phase-10b through phase-10f for consistency. Updated all cross-references across CLAUDE.md, CONTRIBUTING.md, architecture.md, scientific-methodology.md, and internal links. --- CLAUDE.md | 2 +- CONTRIBUTING.md | 10 +- docs/{ => adr}/business-strategy.md | 2 +- docs/{ => adr}/domain-agnostic.md | 2 +- docs/{ => adr}/impact-evaluation-domain.md | 2 +- docs/architecture.md | 2 +- {research => docs/research}/README.md | 2 +- .../competitive-landscape-social-programs.md | 0 .../impact}/us-public-data-apis.md | 0 .../methodology/evidence-evaluation.md | 0 .../methodology/experiment-design.md | 0 .../methodology/literature-survey.md | 0 .../methodology/negative-controls.md | 0 .../research}/methodology/synthesis.md | 0 .../research}/molecular/amr-crisis.md | 0 .../research}/molecular/antimicrobial-ml.md | 0 .../molecular/cheminformatics-rdkit.md | 0 .../research}/molecular/data-sources.md | 0 .../molecular/simulation-docking-admet.md | 0 .../research}/platform/agent-patterns.md | 0 .../research}/platform/literature-api.md | 0 .../research}/platform/mcp-server.md | 0 docs/roadmap.md | 1452 ----------------- docs/roadmap/README.md | 100 ++ docs/roadmap/backlog.md | 118 ++ docs/roadmap/phase-00-infrastructure.md | 16 + docs/roadmap/phase-01-chemistry.md | 31 + docs/roadmap/phase-02-literature-analysis.md | 46 + docs/roadmap/phase-03-prediction.md | 40 + docs/roadmap/phase-04-simulation.md | 43 + docs/roadmap/phase-05-investigation-agent.md | 64 + docs/roadmap/phase-06-console.md | 35 + docs/roadmap/phase-07-integration-demo.md | 24 + docs/roadmap/phase-08-multi-model.md | 59 + .../phase-09-molecule-visualization.md | 41 + docs/roadmap/phase-10a-hypothesis-engine.md | 59 + docs/roadmap/phase-10b-domain-agnostic.md | 144 ++ docs/roadmap/phase-10c-sdk-optimization.md | 118 ++ docs/roadmap/phase-10d-landing-site.md | 121 ++ .../roadmap/phase-10e-training-enhancement.md | 35 + .../phase-10f-nutrition-enhancement.md | 30 + docs/roadmap/phase-12-saas-infrastructure.md | 70 + docs/roadmap/phase-13-impact-evaluation.md | 140 ++ docs/roadmap/phase-14-engine-hardening.md | 123 ++ docs/scientific-methodology.md | 8 +- 45 files changed, 1472 insertions(+), 1467 deletions(-) rename docs/{ => adr}/business-strategy.md (98%) rename docs/{ => adr}/domain-agnostic.md (99%) rename docs/{ => adr}/impact-evaluation-domain.md (99%) rename {research => docs/research}/README.md (95%) rename docs/{ => research/impact}/competitive-landscape-social-programs.md (100%) rename docs/{ => research/impact}/us-public-data-apis.md (100%) rename {research => docs/research}/methodology/evidence-evaluation.md (100%) rename {research => docs/research}/methodology/experiment-design.md (100%) rename {research => docs/research}/methodology/literature-survey.md (100%) rename {research => docs/research}/methodology/negative-controls.md (100%) rename {research => docs/research}/methodology/synthesis.md (100%) rename {research => docs/research}/molecular/amr-crisis.md (100%) rename {research => docs/research}/molecular/antimicrobial-ml.md (100%) rename {research => docs/research}/molecular/cheminformatics-rdkit.md (100%) rename {research => docs/research}/molecular/data-sources.md (100%) rename {research => docs/research}/molecular/simulation-docking-admet.md (100%) rename {research => docs/research}/platform/agent-patterns.md (100%) rename {research => docs/research}/platform/literature-api.md (100%) rename {research => docs/research}/platform/mcp-server.md (100%) delete mode 100644 docs/roadmap.md create mode 100644 docs/roadmap/README.md create mode 100644 docs/roadmap/backlog.md create mode 100644 docs/roadmap/phase-00-infrastructure.md create mode 100644 docs/roadmap/phase-01-chemistry.md create mode 100644 docs/roadmap/phase-02-literature-analysis.md create mode 100644 docs/roadmap/phase-03-prediction.md create mode 100644 docs/roadmap/phase-04-simulation.md create mode 100644 docs/roadmap/phase-05-investigation-agent.md create mode 100644 docs/roadmap/phase-06-console.md create mode 100644 docs/roadmap/phase-07-integration-demo.md create mode 100644 docs/roadmap/phase-08-multi-model.md create mode 100644 docs/roadmap/phase-09-molecule-visualization.md create mode 100644 docs/roadmap/phase-10a-hypothesis-engine.md create mode 100644 docs/roadmap/phase-10b-domain-agnostic.md create mode 100644 docs/roadmap/phase-10c-sdk-optimization.md create mode 100644 docs/roadmap/phase-10d-landing-site.md create mode 100644 docs/roadmap/phase-10e-training-enhancement.md create mode 100644 docs/roadmap/phase-10f-nutrition-enhancement.md create mode 100644 docs/roadmap/phase-12-saas-infrastructure.md create mode 100644 docs/roadmap/phase-13-impact-evaluation.md create mode 100644 docs/roadmap/phase-14-engine-hardening.md diff --git a/CLAUDE.md b/CLAUDE.md index 91848a5..783a01a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ The engine is domain-agnostic: a pluggable `DomainConfig` + `DomainRegistry` sys - **Molecular Science** -- antimicrobial resistance, drug discovery, toxicology, agricultural biocontrol - **Training Science** -- exercise physiology, protocol optimization, injury risk assessment, clinical trials - **Nutrition Science** -- supplement evidence analysis, dietary supplement safety, nutrient profiling, DRI adequacy assessment, drug-nutrient interactions, inflammatory index scoring -- **Impact Evaluation** -- causal analysis of social programs (education, health, sports, employment, housing). Economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmark fetching. See `docs/impact-evaluation-domain.md` +- **Impact Evaluation** -- causal analysis of social programs (education, health, sports, employment, housing). Economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmark fetching. See `docs/adr/impact-evaluation-domain.md` ## Architecture diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 216e187..0dc5dfc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ Ehrlich is an AI-powered scientific discovery engine that uses Claude as a hypothesis-driven reasoning engine. The core engine is domain-agnostic -- any scientific domain can plug in with its own tools, scoring, and visualization. -See [docs/domain-agnostic.md](docs/domain-agnostic.md) for the full story of how and why the engine was generalized. +See [docs/adr/domain-agnostic.md](docs/adr/domain-agnostic.md) for the full story of how and why the engine was generalized. ## Contributor License Agreement @@ -152,16 +152,16 @@ Before writing any code, research the domain deeply. This upfront work determine 5. **Visualization needs** -- How should results be displayed? Charts, 3D models, diagrams, maps, tables? 6. **Domain vocabulary** -- Key terms, units, standard notations that Claude needs in its prompts -**Document your research** in `research/{yourdomain}/`: +**Document your research** in `docs/research/{yourdomain}/`: ``` -research/yourdomain/ +docs/research/yourdomain/ data-sources.md # APIs, databases, auth, rate limits domain-model.md # Entities, scoring criteria, thresholds prior-work.md # Existing tools and approaches in this field ``` -See `research/molecular/`, `research/methodology/`, and the `training/` and `nutrition/` bounded contexts for examples of thorough domain research and implementation. +See `docs/research/molecular/`, `docs/research/methodology/`, and the `training/` and `nutrition/` bounded contexts for examples of thorough domain research and implementation. ### Step 1: Create the Bounded Context @@ -406,7 +406,7 @@ Update these files (in the same commit as the code): | `CLAUDE.md` | Tool count, bounded contexts table, tools section, scopes list | | `README.md` | Tool count, bounded contexts table, tools table, "What Can Ehrlich Investigate" | | `docs/architecture.md` | Bounded contexts, tool count, data flow | -| `docs/roadmap.md` | Add completion entry | +| `docs/roadmap/` | Add completion entry to appropriate phase file | | `web/src/lib/constants.ts` | STATS counts, DOMAINS array (toolCount, capabilities, sources, vizTools), DATA_SOURCES array | | `console/` | Template prompts, domain badges, tool references where applicable | diff --git a/docs/business-strategy.md b/docs/adr/business-strategy.md similarity index 98% rename from docs/business-strategy.md rename to docs/adr/business-strategy.md index 1a2adda..90127e8 100644 --- a/docs/business-strategy.md +++ b/docs/adr/business-strategy.md @@ -1,7 +1,7 @@ # Ehrlich Business Strategy > Public strategy document. Ehrlich is AGPL-3.0 open source. -> For technical implementation, see [roadmap.md](roadmap.md) Phase 12. +> For technical implementation, see [Phase 12: SaaS Infrastructure](roadmap/phase-12-saas-infrastructure.md). ## Market Position diff --git a/docs/domain-agnostic.md b/docs/adr/domain-agnostic.md similarity index 99% rename from docs/domain-agnostic.md rename to docs/adr/domain-agnostic.md index 386b2af..cb8461f 100644 --- a/docs/domain-agnostic.md +++ b/docs/adr/domain-agnostic.md @@ -151,7 +151,7 @@ The fourth domain proves the architecture handles non-scientific domains too. ** **Why it matters:** Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evaluations (0.4%). No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. Ehrlich fills this gap. -**What's new:** Document/CSV upload (user-provided program data), causal inference tools (DiD, PSM, RDD, synthetic control), Mexico + US public data API clients (INEGI, Census, World Bank, FRED), evaluation report templates (CONEVAL ECR, WWC-compliant). See `docs/impact-evaluation-domain.md` for full design. +**What's new:** Document/CSV upload (user-provided program data), causal inference tools (DiD, PSM, RDD, synthetic control), Mexico + US public data API clients (INEGI, Census, World Bank, FRED), evaluation report templates (CONEVAL ECR, WWC-compliant). See `impact-evaluation-domain.md` for full design. **What stays the same:** The DomainConfig, tool tagging, multi-model orchestration, hypothesis framework, evidence grading, and visualization pipeline are all reused. Zero changes to existing code paths. diff --git a/docs/impact-evaluation-domain.md b/docs/adr/impact-evaluation-domain.md similarity index 99% rename from docs/impact-evaluation-domain.md rename to docs/adr/impact-evaluation-domain.md index ad10564..e06aa93 100644 --- a/docs/impact-evaluation-domain.md +++ b/docs/adr/impact-evaluation-domain.md @@ -527,4 +527,4 @@ File upload - [World Bank API](https://api.worldbank.org/v2/) - [FRED API](https://api.stlouisfed.org/fred/) - [CGD: Only 11 Impact Evaluations out of 2,800](https://pubs.cgdev.org/evidence-to-impact/1-progress-and-challenges/challenges/index.html) -- See also: `docs/competitive-landscape-social-programs.md`, `docs/us-public-data-apis.md` +- See also: `../research/impact/competitive-landscape-social-programs.md`, `../research/impact/us-public-data-apis.md` diff --git a/docs/architecture.md b/docs/architecture.md index c392588..2e34fca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,7 +34,7 @@ Exercise physiology and sports medicine research: evidence-based training analys Nutrition science research: supplement evidence analysis, supplement label lookup (NIH DSLD), nutrient data (USDA FoodData), supplement safety monitoring (OpenFDA CAERS), drug interaction screening (RxNav), DRI-based nutrient adequacy assessment, nutrient ratio analysis, and inflammatory index scoring. Uses Semantic Scholar for literature search. ### Impact -Social program evaluation. Provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities, repository ABCs, and infrastructure clients for three external APIs. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools. Future phases will add Mexico/US-specific data sources. See `docs/impact-evaluation-domain.md` for full design. +Social program evaluation. Provides economic indicator search (World Bank, WHO GHO, FRED), international benchmarking, and cross-program comparison using existing statistical tests. Hypothesis-driven analysis of any program type (education, health, sports, employment, housing) in any country. Full DDD structure with domain entities, repository ABCs, and infrastructure clients for three external APIs. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools. Future phases will add Mexico/US-specific data sources. See `docs/adr/impact-evaluation-domain.md` for full design. ### Investigation Hypothesis-driven agent orchestration. Manages the Claude-driven research loop: literature survey, hypothesis formulation (with predictions, criteria, scope), parallel experiment execution, criteria-based evaluation, negative controls, and synthesis. Uses multi-model architecture (Director/Researcher/Summarizer) with user-guided steering, domain classification, and multi-investigation memory. Supports document upload (CSV/XLSX/PDF) with prompt injection and queryable data via `query_uploaded_data` tool. Includes domain configuration system (`DomainConfig` + `DomainRegistry`) for pluggable scientific domains with tool tagging, score definitions, prompt adaptation, and visualization control. Each domain config includes `tool_examples` in `experiment_examples` with realistic tool chaining patterns for complete tool coverage. Optional MCP bridge (`MCPBridge`) connects to external MCP servers for extensibility (e.g. Excalidraw for visual summaries). diff --git a/research/README.md b/docs/research/README.md similarity index 95% rename from research/README.md rename to docs/research/README.md index 1b15ac8..83e6f37 100644 --- a/research/README.md +++ b/docs/research/README.md @@ -14,7 +14,7 @@ Architecture and infrastructure decisions made before and during initial develop ## Methodology -Scientific methodology research (phases 2-6). Each file contains verified sources, implementation notes, and design rationale. See [docs/scientific-methodology.md](../docs/scientific-methodology.md) for the implemented specification. +Scientific methodology research (phases 2-6). Each file contains verified sources, implementation notes, and design rationale. See [scientific-methodology.md](../scientific-methodology.md) for the implemented specification. | File | Phase | Sources | |------|-------|---------| diff --git a/docs/competitive-landscape-social-programs.md b/docs/research/impact/competitive-landscape-social-programs.md similarity index 100% rename from docs/competitive-landscape-social-programs.md rename to docs/research/impact/competitive-landscape-social-programs.md diff --git a/docs/us-public-data-apis.md b/docs/research/impact/us-public-data-apis.md similarity index 100% rename from docs/us-public-data-apis.md rename to docs/research/impact/us-public-data-apis.md diff --git a/research/methodology/evidence-evaluation.md b/docs/research/methodology/evidence-evaluation.md similarity index 100% rename from research/methodology/evidence-evaluation.md rename to docs/research/methodology/evidence-evaluation.md diff --git a/research/methodology/experiment-design.md b/docs/research/methodology/experiment-design.md similarity index 100% rename from research/methodology/experiment-design.md rename to docs/research/methodology/experiment-design.md diff --git a/research/methodology/literature-survey.md b/docs/research/methodology/literature-survey.md similarity index 100% rename from research/methodology/literature-survey.md rename to docs/research/methodology/literature-survey.md diff --git a/research/methodology/negative-controls.md b/docs/research/methodology/negative-controls.md similarity index 100% rename from research/methodology/negative-controls.md rename to docs/research/methodology/negative-controls.md diff --git a/research/methodology/synthesis.md b/docs/research/methodology/synthesis.md similarity index 100% rename from research/methodology/synthesis.md rename to docs/research/methodology/synthesis.md diff --git a/research/molecular/amr-crisis.md b/docs/research/molecular/amr-crisis.md similarity index 100% rename from research/molecular/amr-crisis.md rename to docs/research/molecular/amr-crisis.md diff --git a/research/molecular/antimicrobial-ml.md b/docs/research/molecular/antimicrobial-ml.md similarity index 100% rename from research/molecular/antimicrobial-ml.md rename to docs/research/molecular/antimicrobial-ml.md diff --git a/research/molecular/cheminformatics-rdkit.md b/docs/research/molecular/cheminformatics-rdkit.md similarity index 100% rename from research/molecular/cheminformatics-rdkit.md rename to docs/research/molecular/cheminformatics-rdkit.md diff --git a/research/molecular/data-sources.md b/docs/research/molecular/data-sources.md similarity index 100% rename from research/molecular/data-sources.md rename to docs/research/molecular/data-sources.md diff --git a/research/molecular/simulation-docking-admet.md b/docs/research/molecular/simulation-docking-admet.md similarity index 100% rename from research/molecular/simulation-docking-admet.md rename to docs/research/molecular/simulation-docking-admet.md diff --git a/research/platform/agent-patterns.md b/docs/research/platform/agent-patterns.md similarity index 100% rename from research/platform/agent-patterns.md rename to docs/research/platform/agent-patterns.md diff --git a/research/platform/literature-api.md b/docs/research/platform/literature-api.md similarity index 100% rename from research/platform/literature-api.md rename to docs/research/platform/literature-api.md diff --git a/research/platform/mcp-server.md b/docs/research/platform/mcp-server.md similarity index 100% rename from research/platform/mcp-server.md rename to docs/research/platform/mcp-server.md diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 5489f64..0000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,1452 +0,0 @@ -# Ehrlich Roadmap - -Hackathon: Feb 10-16, 2026. Each phase ends with something testable. - ---- - -## Phase 0: Infrastructure (Feb 10) -- DONE - -- [x] Git init, LICENSE, README, CLAUDE.md, CHANGELOG -- [x] Server scaffolding: pyproject.toml, uv, all bounded contexts (domain/application/infrastructure/tools) -- [x] Kernel: SMILES/InChIKey/MolBlock types, Molecule value object, exception hierarchy -- [x] API: FastAPI factory, CORS, health endpoint -- [x] Console scaffolding: React 19, TypeScript, Bun, Vite, TanStack Router -- [x] Console features: investigation + molecule component stubs -- [x] CI/CD: GitHub Actions (ruff, mypy, pytest, tsc, vite build) -- [x] Docker: multi-stage builds (server + console) -- [x] Pre-commit: ruff lint + format -- [x] All quality gates passing (ruff 0, mypy 0, pytest 5/5, tsc 0) - -**Verification:** `uv run pytest` passes, `GET /api/v1/health` returns 200, `bun run build` succeeds. - ---- - -## Phase 1: Chemistry Context (Feb 10) -- DONE - -Foundation for the Molecular Science domain. Provides RDKit cheminformatics for SMILES processing, descriptors, fingerprints, and 3D conformers. - -### 1A. RDKit Adapter -- Core Operations -- [x] `validate_smiles(smiles)` -- `Chem.MolFromSmiles`, null-check, return bool -- [x] `canonicalize(smiles)` -- canonical SMILES via RDKit -- [x] `to_inchikey(smiles)` -- SMILES -> InChI -> InChIKey -- [x] Tests: valid/invalid SMILES, canonicalization consistency (24 tests) - -### 1B. RDKit Adapter -- Descriptors & Fingerprints -- [x] `compute_descriptors(smiles)` -- MW, LogP, TPSA, HBD, HBA, RotatableBonds, QED, NumRings -- [x] `compute_fingerprint(smiles, fp_type)` -- Morgan/ECFP (radius=2, 2048-bit) + MACCS (166-bit) -- [x] `tanimoto_similarity(fp1, fp2)` -- similarity score (0.0-1.0) -- [x] Tests: aspirin descriptors match known values, fingerprint bit counts - -### 1C. RDKit Adapter -- 3D & Substructure -- [x] `generate_conformer(smiles)` -- AddHs, EmbedMolecule(ETKDGv3), MMFFOptimize, MolToMolBlock -- [x] `substructure_match(smiles, pattern)` -- returns bool + matching atom indices -- [x] Tests: 3D conformer has coordinates, substructure match on known patterns - -### 1D. Chemistry Service + Tools -- [x] Wire `ChemistryService` to `RDKitAdapter` via dependency injection -- [x] Implement `generate_3d` tool -- SMILES -> JSON with MolBlock + energy -- [x] Implement `substructure_match` tool -- SMILES + pattern -> JSON with match + atoms -- [x] Added `validate_smiles`, `compute_descriptors`, `compute_fingerprint`, `tanimoto_similarity` tools -- [x] Tests: service integration tests (9), tool JSON output validation (11) - -**Verification:** `uv run pytest tests/chemistry/ -v` -- 44 passed, mypy clean. - ---- - -## Phase 2: Literature + Analysis (Feb 10) -- DONE - -Two independent contexts. Can be built in parallel. - -### 2A. Semantic Scholar Client -- [x] HTTP client with `httpx` -- search endpoint (`/graph/v1/paper/search`) -- [x] Parse response: title, authors, year, abstract, DOI, citationCount -- [x] Rate limiting: respect 100 req/sec unauthenticated limit -- [x] Error handling: timeout, 429 rate limit, malformed response -- [x] Tests: Paper entity construction from API response (3 tests) - -### 2B. Core Reference Set -- [x] Load `data/references/core_references.json` into `CoreReferenceSet` -- [x] Lookup by key: halicin, abaucin, who_bppl_2024, chemprop, pkcsm, amr_crisis -- [x] Expanded JSON with full metadata (key findings, training size, hit rates) -- [x] Tests: load fixture, find by key, all 6 keys verified - -### 2C. Literature Service + Tools -- [x] `search_papers(query, limit)` -- Semantic Scholar -- [x] `get_reference(key)` -- core reference lookup + DOI fallback -- [x] `format_citation(paper)` -- APA-style string -- [x] Implement `search_literature` tool -- query -> JSON with papers -- [x] Implement `get_reference` tool -- key -> JSON with full citation -- [x] Tests: service with mock repos (8), tool JSON output (3) - -### 2D. ChEMBL Loader -- [x] HTTP client for ChEMBL REST API (direct httpx) -- [x] Filter by: target_organism, standard_type=MIC/IC50, standard_relation="=" -- [x] Deduplicate: one entry per SMILES (median activity if duplicates) -- [x] Compute pActivity: -log10(standard_value * 1e-6) -- [x] Parquet caching for downloaded datasets -- [x] Build `Dataset` entity: smiles_list, activities, metadata -- [ ] Tests: mock API (deferred -- uses mock service in tools tests) - -### 2E. Analysis Service + Tools -- [x] `explore(target, threshold)` -- load dataset, return stats -- [x] `analyze_substructures(dataset)` -- chi-squared enrichment on 10 known substructures -- [x] `compute_properties(dataset)` -- descriptor distributions (active vs inactive) -- [x] Implement `explore_dataset` tool -- target -> JSON with stats -- [x] Implement `analyze_substructures` tool -- target -> JSON with enriched substructures -- [x] Implement `compute_properties` tool -- target -> JSON with property distributions -- [x] Tests: mock datasets (6), enrichment logic, property computation, tool output (3) - -**Verification:** `uv run pytest tests/literature/ tests/analysis/ -v` -- 23 passed. - ---- - -## Phase 3: Prediction Context (Feb 10) -- DONE - -ML models for activity/outcome prediction (domain-agnostic scoring). - -### 3A. XGBoost Adapter -- [x] Train: Morgan fingerprints (2048-bit) + activities -> XGBoost classifier -- [x] Scaffold split: train/val/test by Murcko scaffold (prevents data leakage) -- [x] Metrics: AUROC, AUPRC, accuracy, F1, confusion matrix -- [x] Feature importance: top fingerprint bits -> interpret as substructures -- [x] Predict: fingerprints -> probabilities -- [x] Tests: small fixture dataset, verify model trains and predicts (5 tests) - -### 3B. Model Store -- [x] Save: `TrainedModel` metadata + joblib artifact -> disk -- [x] Load: model_id -> `TrainedModel` + artifact -- [x] List: all saved models with metrics -- [x] Tests: save/load roundtrip (4 tests) - -### 3C. Prediction Service -- Train & Predict -- [x] `train(target, model_type)` -- load dataset, compute fingerprints, scaffold split, train, save -- [x] `predict(smiles_list, model_id)` -- load model, compute fingerprints, predict, rank -- [x] `cluster(smiles_list, n_clusters)` -- Butina clustering on Tanimoto distances -- [x] Tests: full train -> predict pipeline with fixture data (8 tests) - -### 3D. Chemprop Adapter (Optional Extra) -- [x] Train: SMILES + activities -> Chemprop D-MPNN (requires `deeplearning` extra) -- [x] Predict: SMILES -> probabilities -- [x] Guard: graceful skip if chemprop not installed -- [x] Tests: mock or skip if extra not available - -### 3E. Ensemble & Tools -- [x] `ensemble(smiles_list)` -- average XGBoost + Chemprop (0.5 each), fallback to single -- [x] Implement `train_model` tool -- target + model_type -> JSON with metrics -- [x] Implement `predict_candidates` tool -- smiles_list + model_id -> JSON with ranked predictions -- [x] Implement `cluster_compounds` tool -- smiles_list -> JSON with clusters -- [x] Tests: ensemble logic, tool JSON output (3 tests) - -**Verification:** `uv run pytest tests/prediction/ -v` -- 20 passed, mypy clean. - ---- - -## Phase 4: Simulation Context (Feb 10) -- DONE - -Molecular docking, ADMET prediction, resistance assessment. - -### 4A. Protein Store -- [x] Manage PDBQT files in `data/proteins/` -- [x] Target registry: PDB ID -> name, organism, center coordinates, box size -- [x] Pre-configure 5 MRSA targets: PBP2a (1VQQ), DHPS (1AD4), DNA Gyrase (2XCT), MurA (1UAE), NDM-1 (3SPU) -- [x] `get_pdbqt(pdb_id)` -- return file path or download -- [x] Tests: target lookup, file existence check (6 tests) - -### 4B. Vina Adapter (Optional Extra) -- [x] Dock: SMILES -> conformer (RDKit) -> Meeko prep -> Vina dock -> energy + pose -- [x] Parse results: binding energy (kcal/mol), best pose PDBQT, RMSD -- [x] Interpret energy: excellent (<= -10), strong (-8 to -10), moderate (-6 to -8), weak (> -6) -- [x] Guard: graceful skip if vina/meeko not installed -- [x] Tests: energy interpretation (6 tests in protein_store) - -### 4C. ADMET Client -- [x] RDKit-based ADMET: Lipinski violations, mutagenic alerts (SMARTS), hepatotoxicity, hERG, BBB -- [x] Build `ADMETProfile` with absorption, distribution_vd, metabolism, excretion, toxicity -- [x] Toxicity flags: Ames (nitro/azide/quinone alerts), hERG (LogP + MW), hepatotoxicity (LogP + MW + acyl chloride/thioester) -- [x] Tests: aspirin profile, ethanol BBB, nitroaromatic mutagenic, hepatotoxic compound (5 tests) - -### 4D. Resistance Assessment -- [x] Knowledge-based mutation risk with compound class pattern matching -- [x] Known mutations: PBP2a S403A/N146K, DHPS F17L, DNA Gyrase S84L, MurA C115D, NDM-1 V73_ins/M154L -- [x] Compound class patterns: beta-lactam ring, fluoroquinolone, sulfonamide -> affected targets -- [x] Build `ResistanceAssessment` with per-mutation `MutationRisk` and overall risk level -- [x] Tests: known target mutations, DNA gyrase, mutations dict (3 tests in simulation_service) - -### 4E. Simulation Service + Tools -- [x] `dock(smiles, target_id)` -- Vina with RDKit descriptor-based estimate fallback -- [x] `predict_admet(smiles)` -- RDKit-based via PkCSMClient -- [x] `assess_resistance(smiles, target_id)` -- knowledge-based with compound class risk -- [x] Implement `dock_against_target` tool -- smiles + target -> JSON with energy + interpretation -- [x] Implement `predict_admet` tool -- smiles -> JSON with ADMET profile + toxicity flags -- [x] Implement `assess_resistance` tool -- smiles + target -> JSON with mutation details -- [x] Tests: service integration (6), tool JSON output (3) - -**Verification:** `uv run pytest tests/simulation/ -v` -- 20 passed, mypy clean. - ---- - -## Phase 5: Investigation Agent (Feb 10) -- DONE - -The core: Claude as an autonomous scientist. - -### 5A. Anthropic Client Adapter -- [x] Wrap `anthropic.AsyncAnthropic().messages.create()` -- isolate SDK dependency -- [x] Handle: system prompt, messages array, tools list, max_tokens -- [x] Parse response: content blocks (text, tool_use), stop_reason, usage -- [x] Error handling: rate limits, API errors, timeout -- [x] Tests: mock API, verify request/response handling - -### 5B. Tool Registry -- [x] Register all tools from all contexts (6 chemistry, 3 literature, 12 analysis, 3 prediction, 7 simulation, 11 training, 10 nutrition, 7 investigation control -- 38 at time of Phase 5; now 79 with API tools + visualization tools + statistics tools + generic ML tools + impact tools + causal tools) -- [x] Auto-generate JSON Schema from Python type hints + docstrings -- [x] `get(name)` -> callable, `list_tools()` -> all registered tools, `list_schemas()` -> Anthropic-compatible schemas -- [x] Schema format matches Anthropic tool_use specification -- [x] Tests: register tool, verify schema generation, lookup, list params, defaults (8 tests) - -### 5C. Cost Tracker -- [x] Track per-run: input_tokens, output_tokens, tool_calls count -- [x] Compute cost: Sonnet 4.5 pricing ($3/M input, $15/M output) -- [x] Running totals across iterations, `to_dict()` for serialization -- [x] Tests: add_usage, verify total_cost calculation (6 tests) - -### 5D. System Prompt -- [x] Scientist persona: Paul Ehrlich, methodology, phases, rules -- [x] 7 research phases with specific tool guidance per phase -- [x] Constraints: minimum 3 tools per phase, always cite references, record findings -- [x] Output format: structured findings + ranked candidates + citations - -### 5E. Orchestrator -- Agentic Loop -- [x] Create `Investigation` entity, set status to RUNNING -- [x] Build messages array: system prompt + user research question -- [x] Loop: call Claude -> check stop_reason -> dispatch tool_use -> collect results -> repeat -- [x] Max iteration guard (configurable, default 50) -- [x] Emit domain events: HypothesisFormulated, ExperimentStarted, ExperimentCompleted, HypothesisEvaluated, NegativeControlRecorded, ToolCalled, ToolResultEvent, FindingRecorded, Thinking, InvestigationCompleted, InvestigationError -- [x] Handle: parallel tool calls (Claude can request multiple), tool errors (graceful JSON error return) -- [x] End condition: stop_reason == "end_turn", conclude_investigation called, or max iterations -- [x] Phase auto-detection from tool names -- [x] Tests: mock Claude responses with tool_use, verify dispatch + event emission (9 tests) - -### 5F. SSE Streaming -- [x] Convert domain events to SSE events via `domain_event_to_sse()` mapper -- [x] Wire orchestrator async generator to `sse-starlette` EventSourceResponse -- [x] 20 event types (see README for full list): hypothesis lifecycle, experiment lifecycle, tool calls, findings, thinking, controls, validation, phases, cost, domain detection, literature survey, visualization, completion, error -- [x] Include cost tracker data in completed event -- [x] Tests: event type conversions + JSON format - -### 5G. Investigation API Routes -- [x] `POST /api/v1/investigate` -- accept prompt, create investigation, return ID -- [x] `GET /api/v1/investigate/{id}/stream` -- SSE stream of orchestrator events -- [x] Request/response DTOs with Pydantic models (InvestigateRequest, InvestigateResponse) -- [x] Error handling: 404 invalid ID, 409 investigation already running -- [x] Tests: FastAPI TestClient (4 tests) - -### 5H. Control Tools -- [x] `record_finding(title, detail, hypothesis_id, evidence_type)` -- orchestrator intercepts and stores in Investigation entity -- [x] `conclude_investigation(summary, candidates, citations)` -- orchestrator intercepts, sets candidates/citations, ends loop -- [x] `propose_hypothesis`, `design_experiment`, `evaluate_hypothesis`, `record_negative_control` -- hypothesis control tools -- [x] Tests: verify finding storage, conclusion structure, hypothesis tools (10 tests) - -**Verification:** `uv run pytest tests/investigation/ tests/api/ -v` -- 41 passed. Full suite 153 passed, 82% coverage, ruff 0, mypy 0. - ---- - -## Phase 6: Console Integration (Feb 10) -- DONE - -Visualization and real-time streaming UI. - -### 6A. ~~RDKit.js WASM Integration~~ (Replaced by server-side SVG) -- [x] Replaced with server-side RDKit `rdMolDraw2D` SVG depiction (higher quality, simpler) -- [x] Deleted `useRDKit` hook (dead code) - -### 6B. Molecule Viewers -- [x] `MolViewer2D`: server-side SVG via ``, lazy loading, error fallback -- [x] `MolViewer3D`: 3Dmol.js WebGL viewer with stick style, Jmol coloring, dynamic import -- [x] `DockingViewer`: 3Dmol.js protein cartoon (spectrum) + ligand stick (green carbon) overlay -- [x] `PropertyCard`: display descriptors with generic property grid - -### 6C. Investigation Flow -- [x] Wire `PromptInput` to `useInvestigation` mutation (POST /investigate) -- [x] Navigate to `/investigation/$id` on success -- [x] `useSSE` hook: named event listeners for all 7 SSE event types, parsed state (currentPhase, findings, summary, cost, error) -- [x] `Timeline`: rich rendering per event type with phase icons, tool call previews, finding highlights -- [x] `FindingsPanel`: sidebar panel showing accumulated findings with phase badges -- [x] `CandidateTable`: populate from conclusion event data -- [x] `ReportViewer`: render markdown summary from conclude event -- [x] `CostBadge`: live token/cost from SSE completed event - -### 6D. Polish -- [x] Loading states: spinner on connecting, "Starting..." on submit -- [x] Error states: connection lost, API errors, inline error messages -- [x] Responsive layout: 3-column grid on desktop (timeline + report | findings + candidates) -- [x] Status indicator: connecting / running (animated) / completed / error -- [x] Auto-scroll timeline to latest event -- [x] Tests: component rendering with mock data (14 tests across 4 files) - -**Verification:** `npx vitest run` -- 19 passed. `bun run build` + `tsc --noEmit` -- zero errors. - ---- - -## Phase 7: Integration + Demo (Feb 10) -- DONE - -### 7A. End-to-End Validation -- [x] E2E smoke test exercising full pipeline (tool registry, orchestrator dispatch, SSE events) -- [x] API key wired from Settings to AnthropicClientAdapter -- [x] Exponential backoff retry (3 attempts) for rate-limit and timeout errors - -### 7B. Data Preparation -- [x] Data preparation script (`data/scripts/prepare_data.py`) for ChEMBL + PDB downloads -- [x] Parquet caching in `data/datasets/` - -### 7C. Error Handling Sweep -- [x] Graceful degradation: skip docking if vina not installed -- [x] Graceful degradation: skip Chemprop if torch not installed -- [x] Improved error handling in explore_dataset, train_model, predict_candidates, dock_against_target, assess_resistance -- [x] Agent loop: handle tool errors without crashing investigation - -### 7D. Documentation -- [x] README: environment variables, demo instructions, API endpoints -- [x] Architecture and roadmap docs - -**Verification:** 160 tests, 81.67% coverage, all quality gates green. - ---- - -## Phase 8: Multi-Model Architecture + Polish (Feb 10) -- DONE - -Cost-efficient multi-model orchestration, persistence, and UI polish. - -### 8A. Domain Foundation + Config -- [x] Domain events: `OutputSummarized` (kept), hypothesis/experiment events (added in Phase 10A) -- [x] `created_at` and `cost_data` fields on Investigation entity -- [x] Per-model config settings: director, researcher, summarizer models -- [x] `summarizer_threshold`, `max_iterations_per_experiment`, `db_path` settings - -### 8B. Multi-Model Cost Tracker -- [x] Per-model usage tracking with model-specific pricing -- [x] Pricing dict: Opus $15/$75, Sonnet $3/$15, Haiku $0.80/$4 per M tokens -- [x] `to_dict()` with `by_model` breakdown (director/researcher/summarizer tiers) -- [x] `AnthropicClientAdapter.model` property for cost attribution - -### 8C. Multi-Model Orchestrator -- [x] `MultiModelOrchestrator` with hypothesis-driven Director-Worker-Summarizer pattern -- [x] Director (Opus) formulates hypotheses, designs experiments, evaluates evidence, synthesizes -- NO tool access -- [x] Researcher (Sonnet) executes experiments with 73 domain-filtered tools (max 10 iterations per experiment) -- [x] Summarizer (Haiku) compresses large outputs exceeding threshold -- [x] 7 prompts: director formulation/experiment/evaluation/synthesis, researcher experiment, scientist, summarizer -- [x] Auto-fallback to single-model Orchestrator when researcher == director -- [x] Tests: 12 tests across 8 classes (formulation, execution, evaluation, negative controls, compression, synthesis, full flow, errors) - -### 8D. SQLite Persistence -- [x] `InvestigationRepository` ABC in domain layer -- [x] `SqliteInvestigationRepository` with WAL mode via `aiosqlite` -- [x] Single `investigations` table with JSON serialization for complex fields -- [x] Tests: 8 tests (save, retrieve, list, update, cost data) - -### 8E. API Wiring -- [x] `GET /investigate` -- list all investigations (most recent first) -- [x] `GET /investigate/{id}` -- full investigation detail -- [x] SQLite repository initialization in app lifespan -- [x] Automatic orchestrator selection (multi-model vs single-model) -- [x] SSE event types: `hypothesis_formulated`, `experiment_started`, `experiment_completed`, `hypothesis_evaluated`, `negative_control`, `output_summarized` -- [x] `completed` event includes candidates and cost data - -### 8F. SSE Reconnection -- [x] Exponential backoff (1s, 2s, 4s) with max 3 retries -- [x] `reconnecting` state with amber WiFi-off indicator -- [x] Reset attempt counter on successful reconnect -- [x] Track `experiments`, `hypotheses`, `negativeControls` and `toolCallCount` from events - -### 8G. Investigation History -- [x] `useInvestigations` hook with TanStack Query (10s refetch) -- [x] `InvestigationList` component with status badges and candidate counts -- [x] History section on home page below prompt input - -### 8H. UI Feedback -- [x] `HypothesisBoard` kanban-style grid with status columns and confidence bars -- [x] Hypothesis/experiment event rendering in Timeline -- [x] Candidates wired to `CandidateTable` from SSE completed event -- [x] `StatusIndicator` handles reconnecting state - -**Verification:** 185 tests, 82.28% coverage, all quality gates green (ruff, mypy, tsc, vitest). - ---- - -## Phase 9: Molecule Visualization Suite (Feb 10) -- DONE - -Full visualization: server-side 2D SVG depiction, 3Dmol.js for 3D/docking views, expandable candidate detail panel. - -### 9A. Server-Side 2D Depiction -- [x] `RDKitAdapter.depict_2d(smiles, width, height)` -- RDKit `rdMolDraw2D.MolDraw2DSVG` -- [x] `ChemistryService.depict_2d()` thin wrapper -- [x] Tests: returns SVG, custom dimensions, invalid SMILES raises (3 tests) - -### 9B. Molecule API Routes -- [x] `GET /molecule/depict?smiles=&w=&h=` -- SVG response, 24h cache, error SVG for invalid SMILES -- [x] `GET /molecule/conformer?smiles=` -- JSON with mol_block, energy, num_atoms -- [x] `GET /molecule/descriptors?smiles=` -- JSON descriptors + passes_lipinski -- [x] `GET /targets` -- JSON list of protein targets (5 pre-configured) -- [x] Router registered in app.py -- [x] Tests: 10 API tests (depict, conformer, descriptors, targets) - -### 9C. 3Dmol.js Integration -- [x] Added `3dmol` package to console dependencies -- [x] TypeScript type stubs (`console/src/types/3dmol.d.ts`) -- [x] Dynamic import for code splitting (~575KB separate chunk) - -### 9D. Molecule Viewer Components -- [x] `MolViewer2D`: `` tag with server-side SVG, lazy loading, error fallback to SMILES text -- [x] `MolViewer3D`: 3Dmol.js WebGL viewer, stick style, Jmol coloring, cleanup on unmount -- [x] `DockingViewer`: 3Dmol.js protein cartoon (spectrum) + ligand stick (green carbon), zoom to ligand -- [x] Tests: 4 MolViewer2D component tests (src, encoding, dimensions, error fallback) - -### 9E. CandidateDetail + CandidateTable -- [x] `CandidateDetail` panel: parallel fetch of conformer + descriptors, 3-column grid (2D | 3D + energy | properties + Lipinski badge) -- [x] `CandidateTable` rewrite: 2D thumbnails (80x60), chevron expand/collapse, click-to-expand detail panel -- [x] Removed raw SMILES column (SMILES in img alt text) -- [x] Updated tests for new table structure - -### 9F. Cleanup -- [x] Deleted `useRDKit` hook (dead code) -- [x] Added `CandidateDetail` to barrel export (`index.ts`) - -**Verification:** 198 tests, 82.09% coverage, 19 console tests. All quality gates green (ruff 0, mypy 0, tsc 0, vitest 19/19). - ---- - -## Dependency Graph - -``` -Phase 0 (Infrastructure) -- DONE - | -Phase 1 (Chemistry) -- DONE - | \ -Phase 2A-C Phase 2D-E -(Literature) (Analysis) -- DONE - | | - +----- + ------+ - | - Phase 3 (Prediction) -- DONE - | - Phase 4 (Simulation) -- DONE - | - Phase 5 (Agent Loop) -- DONE - | - Phase 6 (Console) -- DONE - | - Phase 7 (Integration + Demo) -- DONE - | - Phase 8 (Multi-Model + Polish) -- DONE - | - Phase 9 (Molecule Visualization) -- DONE - | - Phase 10A (Hypothesis-Driven Engine) -- DONE - | - Scientific Methodology Upgrade (cross-cutting) -- All 6 Phases DONE - | - Domain-Agnostic Generalization -- DONE - | - Multi-Domain Investigations -- DONE - | - Self-Referential Research -- DONE - | - Shared Context + MCP Bridge + Training/Nutrition APIs -- DONE - | - Domain-Specific Visualization System -- DONE - | - Claude SDK Optimization -- DONE - | - Landing Site (web/) -- DONE - (TanStack Start + SSR) - | - ┌─────┼──────────────┐ - │ │ │ -Training Nutrition Competitive -Enhancement Enhancement Sports - (DONE) (DONE) Domain (TODO) - │ │ │ - └─────┼──────────────┘ - | - Phase 12: SaaS Infrastructure -- DONE - (PostgreSQL + WorkOS Auth + BYOK + Credits) - | - Phase 13: Impact Evaluation Domain -- IN PROGRESS - (Causal inference + Document upload + MX/US APIs) - 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) DONE -> 13C (Mexico) -> 13D (US) -> 13E (MCP) - | - Phase 14: Scientific Engine Hardening -- IN PROGRESS (parallel with 13A-2) - 14A (Orchestrator Split) ──┬── 14B (Route Cleanup) ──┬── 14D (Tool Enrichment) - ├── 14C (Prompt Split) ├── 14E (Tree Search) - └───────────────────────── └── 14F (Paper Generator) - | - 14G (Validation Runs) - | - Demo + Video -- TODO - | - Submission (Feb 16, 3PM ET) -- TODO -``` - -## Scientific Methodology Upgrade (Cross-Cutting) - -Grounding every phase of the investigation workflow in established scientific methodology. Each phase gets the treatment that Phase 1 (Hypothesis) received: deep research, universal components, entity upgrades, prompt updates. See [`docs/scientific-methodology.md`](scientific-methodology.md) for full details. - -| # | Phase | Status | -|---|-------|--------| -| 1 | Hypothesis Formulation (Popper, Platt, Feynman, Bayesian) | DONE | -| 2 | Literature Survey (PICO, citation chasing, GRADE, AMSTAR-2) | DONE | -| 3 | Experiment Design (Fisher, controls, sensitivity, AD) | DONE | -| 4 | Evidence Evaluation (evidence hierarchy, GRADE, effect sizes) | DONE | -| 5 | Negative Controls (Z'-factor, permutation significance, scaffold-split) | DONE | -| 6 | Synthesis (GRADE certainty, priority tiers, knowledge gaps) | DONE | - ---- - -## Phase 10A: Hypothesis-Driven Investigation Engine (Feb 10) -- DONE - -Replaced the linear 7-phase pipeline with a hypothesis-driven scientific method loop. - -### Architecture Change -- **Old:** Literature -> Data -> Model -> Screen -> Structure -> Resistance -> Conclude (linear recipe) -- **New:** Literature Survey -> Formulate Hypotheses -> For each: Design Experiment -> Execute -> Evaluate -> Negative Controls -> Synthesize - -### Domain Layer -- [x] `Hypothesis` entity: statement, rationale, status (proposed/testing/supported/refuted/revised), confidence, evidence lists -- [x] `Experiment` entity: hypothesis_id, description, tool_plan, status, result_summary -- [x] `NegativeControl` frozen dataclass: identifier, identifier_type, score, threshold, correctly_classified property (generalized in Domain-Agnostic phase) -- [x] `Finding` modified: `hypothesis_id` + `evidence_type` replace `phase` -- [x] `Investigation` modified: hypotheses, experiments, negative_controls replace phases - -### Events (12 total, was 11) -- [x] Removed: `PhaseStarted`, `PhaseCompleted`, `DirectorPlanning`, `DirectorDecision` -- [x] Added: `HypothesisFormulated`, `ExperimentStarted`, `ExperimentCompleted`, `HypothesisEvaluated`, `NegativeControlRecorded` -- [x] Modified: `FindingRecorded` (hypothesis_id + evidence_type), `InvestigationCompleted` (hypotheses + negative_controls) - -### Tools (23 total, was 19) -- [x] 4 new control tools: `propose_hypothesis`, `design_experiment`, `evaluate_hypothesis`, `record_negative_control` -- [x] Modified: `record_finding` (hypothesis_id + evidence_type), `conclude_investigation` (hypothesis assessments) - -### Orchestrators -- [x] Single-model `Orchestrator`: hypothesis control tool dispatch with entity creation -- [x] `MultiModelOrchestrator`: hypothesis-driven loop (formulate -> design -> execute -> evaluate per hypothesis) -- [x] 7 hypothesis-driven prompts replacing 6 phase-based prompts - -### Console -- [x] `HypothesisBoard` + `HypothesisCard`: kanban-style status grid with confidence bars -- [x] `ActiveExperimentCard`: replaces `ActivePhaseCard` with experiment context -- [x] `NegativeControlPanel`: validation table with pass/fail indicators -- [x] `FindingsPanel`: evidence type badges + hypothesis grouping -- [x] `Timeline`: hypothesis/experiment/evaluation event rendering -- [x] Deleted: `PhaseProgress.tsx`, `ActivePhaseCard.tsx` - -### Persistence -- [x] SQLite schema: hypotheses, experiments, negative_controls columns (JSON serialized) -- [x] Removed: phases, current_phase columns - -**Verification:** 212 tests passed, mypy 0 errors, ruff 0 violations, tsc 0 errors, bun build success. - ---- - -## Domain-Agnostic Generalization (Feb 11-12) -- DONE - -Generalized the entire engine from molecular-science-specific to domain-agnostic. Proved the architecture works by adding Training Science and Nutrition Science as additional domains. - -### Backend Entity Generalization -- [x] `Candidate` entity: generic `identifier`/`identifier_type`/`scores`/`attributes` (replaces `smiles`/`prediction_score`/`docking_score`/`admet_score`/`resistance_risk`) -- [x] `NegativeControl` entity: generic `identifier`/`identifier_type`/`score`/`threshold` (replaces `smiles`/`prediction_score`) -- [x] `NegativeControlRecorded` event: generic fields -- [x] SQLite serialization updated for new field names -- [x] All 274 existing tests pass after entity changes (274 at time of this phase) - -### Domain Configuration + Tool Tagging -- [x] `DomainConfig` + `ScoreDefinition` frozen dataclasses -- [x] `DomainRegistry`: register, detect by classified category, list template prompts -- [x] `MOLECULAR_SCIENCE` config (6 tool tags, 3 scores, 4 templates, multishot examples) -- [x] `TRAINING_SCIENCE` config (3 tool tags, 3 scores, 2 templates, multishot examples) -- [x] `NUTRITION_SCIENCE` config (3 tool tags, 3 scores, 1 template, multishot examples) -- [x] `ToolRegistry` with domain tag filtering (`list_schemas_for_domain`, `list_tools_for_domain`) -- [x] Tools tagged: chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, literature; investigation control universal -- [x] `DomainDetected` SSE event (16th event type) sends display config to frontend - -### Prompt Template Generalization -- [x] Builder functions: `build_scientist_prompt()`, `build_formulation_prompt()`, `build_experiment_prompt()`, `build_synthesis_prompt()` -- [x] Dynamic domain classification from all registered domain categories -- [x] HypothesisType enum expanded: PHYSIOLOGICAL, PERFORMANCE, EPIDEMIOLOGICAL - -### Frontend Generalization -- [x] Dynamic score columns from `DomainDisplayConfig` (replaces hardcoded Pred/Dock/ADMET/Resist) -- [x] Conditional MolViewer2D: only when `identifier_type === "smiles"` -- [x] Unified reactive VisualizationPanel: LiveLabViewer auto-appears for molecular tool events, charts render inline -- [x] 7 template cards (4 molecular + 2 training + 1 nutrition) with domain badges -- [x] Generic candidate comparison, negative control panel, markdown export - -### Training Science Bounded Context (6 tools) -- [x] `search_training_literature` -- Semantic Scholar with training science context -- [x] `analyze_training_evidence` -- effect sizes, heterogeneity, evidence grading (A-D) -- [x] `compare_protocols` -- evidence-weighted composite scoring -- [x] `assess_injury_risk` -- knowledge-based multi-factor risk scoring -- [x] `compute_training_metrics` -- ACWR, monotony, strain, RPE load -- [x] `search_clinical_trials` -- ClinicalTrials.gov exercise/training RCT search - -### Nutrition Science Bounded Context (10 tools) -- [x] `search_supplement_evidence` -- supplement meta-analysis search -- [x] `search_supplement_labels` -- NIH DSLD supplement product ingredient lookup -- [x] `search_nutrient_data` -- USDA FoodData Central nutrient profiles -- [x] `search_supplement_safety` -- OpenFDA CAERS adverse event reports for supplements -- [x] `compare_nutrients` -- side-by-side nutrient comparison (added in Nutrition Enhancement) -- [x] `assess_nutrient_adequacy` -- DRI-based adequacy assessment (added in Nutrition Enhancement) -- [x] `check_intake_safety` -- UL safety screening (added in Nutrition Enhancement) -- [x] `check_interactions` -- RxNav drug interactions (added in Nutrition Enhancement) -- [x] `analyze_nutrient_ratios` -- nutrient ratio analysis (added in Nutrition Enhancement) -- [x] `compute_inflammatory_index` -- DII scoring (added in Nutrition Enhancement) -- [x] Tests: training (8 tests) + nutrition (10 tests initially, expanded in Enhancement), all passing - -**Verification:** 288 server tests, 107 console tests. All quality gates green: ruff 0, mypy 0 (117 files), tsc 0, vitest 107/107. - ---- - -## Multi-Domain Investigations (Feb 12) -- DONE - -Cross-domain research questions that span multiple scientific domains (e.g., molecular + nutrition, training + nutrition). - -### Changes - -- [x] `DomainRegistry.detect()` returns `list[DomainConfig]` (multi-domain detection, deduplicates) -- [x] Haiku classifier outputs JSON array of domain categories -- [x] `merge_domain_configs()` creates synthetic merged config (union tool_tags, concatenated scores, joined examples) -- [x] Tool filtering uses merged config's union of domain tags -- [x] Prompt examples merged from all relevant domains -- [x] `DomainDetected` SSE event carries merged display config with `domains` sub-list -- [x] Frontend `DomainDisplayConfig.domains` for sub-domain rendering -- [x] Tests: 16 tests (multi-detect, deduplication, fallback, merge, display dict) - ---- - -## Self-Referential Research (Feb 12) -- DONE - -Ehrlich builds institutional knowledge by querying its own past investigations during new research. - -### Changes - -- [x] SQLite FTS5 virtual table (`findings_fts`) on finding title, detail, evidence_type, hypothesis statement/status, source provenance -- [x] `search_prior_research` tool -- intercepted in orchestrator, routed to FTS5 via repository -- [x] BM25-ranked full-text search with investigation prompt context -- [x] FTS5 query sanitization (quoted tokens for literal hyphens/operators) -- [x] FTS5 index rebuilt on investigation completion via `_rebuild_fts()` -- [x] Provenance: `source_type: "ehrlich"`, `source_id: "{investigation_id}"` -- [x] Available during Phase 2 (Literature Survey) alongside external search tools -- [x] Frontend: Ehrlich-branded source badges with internal navigation -- [x] Tests: 6 FTS5 tests (empty, match, prompt context, limit, completion-gating, cross-investigation) - ---- - -## Shared Context + MCP Bridge + Training/Nutrition APIs (Feb 12) -- DONE - -DDD cleanup, shared bounded context, MCP bridge for external tools, 4 new data source clients (training + nutrition), and methodology page. - -### Shared Bounded Context -- [x] `shared/` context: cross-cutting ports and value objects (`ChemistryPort` ABC, `Fingerprint`, `MolecularDescriptors`, `Conformer3D`) -- [x] `ChemistryPort` ABC decouples chemistry operations from RDKit infrastructure -- [x] Moved value objects from `chemistry/domain/` to `shared/` for cross-context use - -### MCP Bridge -- [x] `MCPBridge` connects to external MCP servers (stdio/SSE/streamable_http transports) -- [x] Tools registered dynamically via `ToolRegistry.register_mcp_tools()` with domain tags -- [x] Lifecycle managed by orchestrator (connect on start, disconnect on completion) -- [x] Enabled via `EHRLICH_MCP_EXCALIDRAW=true` env var - -### Data Source API Tools (4 tools across training + nutrition) -- [x] `search_clinical_trials` (training) -- ClinicalTrials.gov v2 API for exercise/training RCTs -- [x] `search_supplement_labels` (nutrition) -- NIH DSLD supplement label database -- [x] `search_nutrient_data` (nutrition) -- USDA FoodData Central nutrient profiles -- [x] `search_supplement_safety` (nutrition) -- OpenFDA CAERS adverse event reports -- [x] Full DDD: domain entities + repository ABCs + infrastructure clients in respective bounded contexts - -### Methodology Page -- [x] `GET /methodology` endpoint: phases, domains, tools, data sources, models -- [x] `GET /stats` endpoint: aggregate counts -- [x] Console methodology page with phases, models, domains, tools, data sources - -**Verification:** 527 server tests, 107 console tests. All quality gates green. - ---- - -## Domain-Specific Visualization System (Feb 12) -- DONE - -6 visualization tools with structured payloads, orchestrator interception, and lazy-loaded React components. - -### Backend -- [x] `VisualizationPayload` frozen dataclass (viz_type, title, data, config, domain) -- [x] 6 viz tools: `render_binding_scatter`, `render_admet_radar`, `render_training_timeline`, `render_muscle_heatmap`, `render_forest_plot`, `render_evidence_matrix` -- [x] Orchestrator intercepts viz tool results via `_maybe_viz_event()`, emits `VisualizationRendered` SSE event (20th event type) - -### Frontend -- [x] `VizRegistry` maps viz_type to lazy-loaded React component -- [x] `VisualizationPanel` renders multiple charts in grid layout -- [x] Recharts: scatter (compound affinities), radar (ADMET), timeline (training load + ACWR) -- [x] Visx: forest plot (meta-analysis), evidence matrix (heatmap) -- [x] Custom SVG: anatomical body diagram with muscle activation/risk heatmap -- [x] OKLCH color tokens for consistent chart theming - -**Verification:** 527 server tests, 107 console tests. All quality gates green. - ---- - -## Claude SDK Optimization -- DONE - -Upgraded the Anthropic SDK integration to use all available features for better reasoning quality, lower costs, and improved UX. - -### SDK-1: Fix Pricing + Cache-Aware Cost Tracking -- DONE - -Fixed incorrect hardcoded pricing and added cache hit/miss token tracking for accurate cost reporting. - -- [x] Fix Opus pricing: `$15/$75` -> `$5/$25` per M tokens (Opus 4.5/4.6 pricing) -- [x] Fix Haiku pricing: `$0.80/$4.0` -> `$1/$5` per M tokens (Haiku 4.5 pricing) -- [x] Track `cache_creation_input_tokens` and `cache_read_input_tokens` from `response.usage` -- [x] Compute cache-aware cost: cache writes at 1.25x, cache reads at 0.1x base input rate -- [x] Add `cache_read_tokens` and `cache_write_tokens` to `CostTracker.to_dict()` -- [x] Update `CostUpdate` SSE event with cache breakdown -- [x] Update `MessageResponse` dataclass with cache token fields -- [x] Tests: cost calculation with cache hits, cache misses, mixed scenarios - -**Files:** `cost_tracker.py`, `anthropic_client.py`, `multi_orchestrator.py`, `events.py`, `sse.py`, `test_cost_tracker.py` - -### SDK-2: Prompt Caching on Tools Array -- DONE - -Cache the 73-tool schema array that repeats on every researcher API call. - -- [x] Add `cache_control: {"type": "ephemeral"}` to the last tool in the tools array before passing to `messages.create` -- [x] Only apply when tools list is non-empty -- [x] No changes to tool registry -- caching applied at the adapter level - -**Files:** `anthropic_client.py` - -### SDK-3: Effort Parameter -- DONE - -Use `effort` to control token spend. Only supported by Opus models (4.5+); Sonnet and Haiku do not support it. - -- [x] Add `effort: str | None = None` parameter to `AnthropicClientAdapter.__init__` -- [x] Pass `effort` via `output_config` dict to `messages.create()` (SDK 0.79+ requires it nested, not top-level) -- [x] Configure Director effort in `Settings`: `effort="high"` (default), gated on Opus model name -- [x] Add `EHRLICH_DIRECTOR_EFFORT` env var (only applied when Director model contains "opus") -- [x] Researcher (Sonnet) and Summarizer (Haiku) never receive effort -- unsupported by those models -- [x] Wire in API route when creating adapters - -**Files:** `config.py`, `anthropic_client.py`, `routes/investigation.py` - -### SDK-4: Extended Thinking for Director -- DONE - -Enabled extended thinking on Opus 4.6 Director for deeper scientific reasoning. - -- [x] Add `thinking: dict | None = None` parameter to `AnthropicClientAdapter.__init__` -- [x] Pass `thinking` to `messages.create()` when set -- [x] Default for Director: `{"type": "enabled", "budget_tokens": 10000}` -- [x] Parse `thinking` content blocks in `_parse_content_blocks()` -- extract `block.thinking` text -- [x] Emit thinking text via existing `Thinking` SSE event (already supported in frontend) -- [x] `max_tokens` increased to 32768 for Director to accommodate thinking budget -- [x] Add `EHRLICH_DIRECTOR_THINKING` env var (enabled/disabled) -- [x] Track thinking tokens separately in `MessageResponse` (thinking tokens billed as output) - -**Files:** `config.py`, `anthropic_client.py`, `multi_orchestrator.py`, `routes/investigation.py` - -### SDK-5: Structured Outputs for Director (Medium Impact) -- DONE - -Guarantee valid JSON from Director calls (hypothesis formulation, experiment design, evaluation, synthesis). - -- [x] Add `output_config: dict | None = None` parameter to `AnthropicClientAdapter.create_message()` and `stream_message()` -- [x] Pass `output_config` to `messages.create()` / `messages.stream()` when set -- [x] Define JSON schemas for Director outputs (6 schemas in `domain/schemas.py`): - - `PICO_SCHEMA` -- PICO decomposition for literature survey - - `FORMULATION_SCHEMA` -- array of hypothesis objects - - `EXPERIMENT_DESIGN_SCHEMA` -- experiment plan object - - `EVALUATION_SCHEMA` -- evaluation result object - - `SYNTHESIS_SCHEMA` -- synthesis result object - - `LITERATURE_GRADING_SCHEMA` -- evidence grading result -- [x] Update Director call sites in `MultiModelOrchestrator` to pass schemas via `_build_output_config()` -- [x] Remove `_parse_json()` fallback -- structured outputs guarantee valid JSON -- [x] Structured outputs work alongside streaming (`stream_message` with `output_config`) - -**Files:** `anthropic_client.py`, `multi_orchestrator.py`, `investigation/domain/schemas.py` (new) - -### SDK-6: tool_choice Control -- DONE - -Control how the researcher uses tools. - -- [x] Add `tool_choice: dict | None = None` parameter to `AnthropicClientAdapter.create_message()` -- [x] Pass `tool_choice` to `messages.create()` when set -- [x] Researcher first turn: `tool_choice={"type": "any"}` to force tool use (researcher should always start by calling tools) -- [x] Researcher subsequent turns: `tool_choice=None` (default, let model decide) -- [x] Literature survey first turn: `tool_choice={"type": "any"}` to force tool use - -**Files:** `anthropic_client.py`, `multi_orchestrator.py` - -### SDK-7: Streaming API (High Impact, UX) -- DONE (Director only) - -Replace non-streaming `messages.create()` with streaming for real-time Director token display. - -- [x] Add `stream_message()` async generator method to `AnthropicClientAdapter` -- [x] Use `client.messages.stream()` (high-level SDK helper) for async context manager -- [x] Yield intermediate events: `thinking` and `text` deltas, plus `result` with final `MessageResponse` -- [x] Accumulate final message via `stream.get_final_message()` for usage tracking -- [x] Update `MultiModelOrchestrator._director_call()` to use streaming -- yields `Thinking` events in real time -- [x] Emit real-time `Thinking` events as thinking tokens stream in (no buffering) -- [x] Keep non-streaming `create_message()` for Researcher and Summarizer (tool dispatch loop is simpler without streaming) -- [x] Retry with exponential backoff on `RateLimitError` / `APITimeoutError` (same as `create_message`) - -**Files:** `anthropic_client.py`, `multi_orchestrator.py` - -### Implementation Summary - -SDK-1 through SDK-7 all implemented. SDK-1 through SDK-4 and SDK-6 done in parallel by 3 agents (`sdk-cost`, `sdk-adapter`, `sdk-wiring`). SDK-5 (Structured Outputs) and SDK-7 (Director Streaming) wired in subsequently. - -### Verification - -**Completed:** 540 server tests passing, 107 console tests passing. All quality gates green. - -- `uv run pytest` -- 540 passed (13 new tests for SDK features) -- `uv run ruff check src/ tests/` -- zero violations -- `uv run mypy src/ehrlich/` -- zero errors (140 source files) -- `bun run build && bun run typecheck` -- zero errors -- `bunx vitest run` -- 107 passed - ---- - -## Landing Site (`web/`) -- DONE - -Separate TanStack Start project for the public-facing landing page. SSR/SSG for SEO, same React + TypeScript + Bun stack as console, independent build and deployment. - -### Why Separate from Console - -- **Different concerns**: landing page is marketing/SEO content; console is the authenticated SPA -- **Different optimization**: SSR/SSG with zero-JS static pages vs client-side SPA with heavy WebGL/charting -- **Independent deployment**: CDN-friendly static output vs API-connected app server -- **Clean separation**: no marketing code in the investigation UI, no app dependencies in the landing page - -### Tech Stack - -| Layer | Choice | Rationale | -|-------|--------|-----------| -| Framework | TanStack Start | Same TanStack ecosystem as console, SSR/SSG built-in, Nitro server | -| Runtime | Bun | Same as console | -| Styling | Tailwind CSS 4 | Same as console, shared design tokens (OKLCH) | -| Fonts | Space Grotesk + JetBrains Mono | Same as console (Lab Protocol identity) | -| Icons | Lucide React | Same as console | -| Build | Vite 7 (via Nitro) | Same toolchain | - -### Project Structure - -``` -web/ -├── src/ -│ ├── routes/ -│ │ ├── __root.tsx # Root layout (shell, SEO meta, font loading) -│ │ └── index.tsx # Landing page (all sections wired) -│ ├── components/ -│ │ ├── Nav.tsx # Fixed navbar with scroll progress bar + mobile menu -│ │ ├── Footer.tsx # Minimal footer (links, license, year) -│ │ ├── SectionHeader.tsx # Mono label with left border accent -│ │ ├── Hero.tsx # Bottom-third hero with MolecularNetwork, stats badges, CTAs -│ │ ├── MolecularNetwork.tsx # 3D rotating node graph (Canvas 2D, mouse repulsion, CSS mask) -│ │ ├── HowItWorks.tsx # 6-phase methodology timeline with vertical line -│ │ ├── ConsolePreview.tsx # Browser-frame mockups (timeline, hypotheses, candidates) -│ │ ├── Architecture.tsx # Director-Worker-Summarizer model cards, dot grid bg -│ │ ├── Domains.tsx # 3 domain cards with tool counts + multi-domain callout -│ │ ├── Visualizations.tsx # 4 visualization category cards -│ │ ├── DataSources.tsx # 16 source cards with teal glow -│ │ ├── WhoItsFor.tsx # 3 persona cards (Student, Academic, Industry) -│ │ ├── Differentiators.tsx # 3 differentiator cards with capabilities -│ │ ├── OpenSource.tsx # Code snippet + licensing section -│ │ ├── Roadmap.tsx # Planned domains + platform features -│ │ └── CTA.tsx # Pricing tiers, terminal quickstart, primary glow -│ ├── styles/ -│ │ └── app.css # Tailwind 4 entry + OKLCH tokens + animations -│ ├── lib/ -│ │ ├── constants.ts # Stats, links, domain data, methodology phases -│ │ ├── use-reveal.ts # IntersectionObserver scroll reveal hook -│ │ └── use-scroll-progress.ts # Scroll progress fraction hook -│ ├── router.tsx # TanStack Router factory (getRouter) -│ └── routeTree.gen.ts # Auto-generated route tree -├── public/ -│ └── favicon.svg # Green "E" favicon -├── package.json # ehrlich-web, Bun scripts -├── vite.config.ts # TanStack Start + Nitro + Tailwind -└── tsconfig.json # src/ paths, strict mode -``` - -### WEB-1: Project Scaffolding -- DONE - -- [x] TanStack Start project in `web/` with Bun -- [x] `vite.config.ts` with tanstackStart + nitro + tailwindcss + viteReact + tsConfigPaths -- [x] Tailwind CSS 4 with OKLCH design tokens matching console's Lab Protocol identity -- [x] Space Grotesk + JetBrains Mono font loading via Google Fonts -- [x] Root layout with `shellComponent`, `HeadContent`, CSS `?url` import -- [x] Favicon + meta tags (title, description, OG, Twitter card, theme-color) -- [x] `bun run build` produces `.output/` (client + SSR + Nitro server) -- [x] `bun run typecheck` -- zero errors - -### WEB-2: Landing Page Sections -- DONE - -- [x] **Hero**: bottom-third placement, MolecularNetwork 3D canvas (CSS mask edge fade), stats badges, CTA buttons -- [x] **HowItWorks**: 6-phase methodology timeline with vertical connecting line, hypothesis/experiment structure callouts -- [x] **ConsolePreview**: browser-frame mockups (investigation timeline, hypothesis board, candidate table, ADMET radar) -- [x] **Architecture**: Director-Worker-Summarizer model cards with data pipes, dot grid bg, amber radial glow -- [x] **Domains**: 3 domain cards with tool counts, multi-domain callout, extensibility card -- [x] **Visualizations**: 4 visualization category cards with tech labels, extensibility callout -- [x] **DataSources**: 16 source cards (2-col grid), surface bg with teal radial glow, self-referential research callout -- [x] **WhoItsFor**: 3 persona cards (Student, Academic, Industry) -- [x] **Differentiators**: 3 differentiator cards with capabilities lists -- [x] **OpenSource**: code snippet + dual licensing (AGPL-3.0 + Commercial), GitHub link -- [x] **Roadmap**: planned domains (3 cards) + platform features (dashed border cards) -- [x] **CTA**: pricing tiers (4 cards), terminal quickstart with copy, primary radial glow - -### WEB-3: Navigation + Footer + Meta -- DONE - -- [x] **Nav**: fixed top bar, scroll progress indicator, desktop links, mobile hamburger menu -- [x] **Footer**: AGPL-3.0 branding, mapped footer links, responsive layout -- [x] **SectionHeader**: mono label with left border accent (reused across sections) -- [x] **SEO meta**: full OG tags, Twitter card, theme-color, description -- [x] **Smooth scroll**: anchor links (`#architecture`, `#methodology`, `#domains`, `#data-sources`) -- [x] Responsive: mobile hamburger nav, stacked sections - -### WEB-5: Visual Polish + Animations -- DONE - -- [x] Motion scroll-triggered reveals (whileInView, once: true) -- [x] Staggered children animation (80-120ms delay cascade) -- [x] Architecture data pipe animation (flowing dots between model tiers) -- [x] Domain/differentiator/persona card hover effects (border-primary, y-shift) -- [x] MolecularNetwork 3D canvas: 90 nodes, mouse repulsion, shimmer connections, CSS mask edge fade -- [x] Section zoning: alternating bg-surface/30 + strategic radial glows (amber/teal/primary) -- [x] CSS dot grid on Architecture (32px, static, graph-paper aesthetic) -- [x] Scroll progress bar in navbar -- [x] WCAG AA color contrast (muted-foreground 0.60, secondary 0.60, min /60 opacity) -- [x] Sequential heading hierarchy (h1 -> h2 -> h3, no skips) -- [x] Font preloading for Google Fonts (preload + stylesheet) -- [x] All OKLCH tokens (zero hardcoded gray-* classes) -- [x] Lighthouse: Performance 67, Accessibility 95, Best Practices 100, SEO 100 - -### Verification - -- [x] `cd web && bun run build` -- zero errors -- [x] `cd web && bun run typecheck` -- zero TypeScript errors -- [x] Visual: Lab Protocol identity consistent with console (OKLCH tokens, fonts, dark theme) -- [x] All docs updated: `CLAUDE.md`, `README.md`, `docs/architecture.md`, `docs/roadmap.md` - ---- - -## Training Science Enhancement (Feb 12) -- DONE - -Deepened the training bounded context with 2 new data sources, 4 new tools, and 3 new visualization tools. - -### New Data Sources -- [x] PubMed API (`https://eutils.ncbi.nlm.nih.gov/entrez/eutils/`) -- exercise physiology literature via E-utilities (ESearch + EFetch) -- [x] wger API (`https://wger.de/api/v2`) -- exercise database with muscles, equipment, categories - -### New Tools -- [x] `search_pubmed_training` -- PubMed E-utilities search for exercise/training papers (MeSH terms, structured abstracts) -- [x] `search_exercise_database` -- exercise lookup by muscle group, movement pattern, equipment -- [x] `compute_performance_model` -- Banister fitness-fatigue model (CTL/ATL/TSB via EWMA) -- [x] `compute_dose_response` -- dose-response curve from dose-effect data points - -### New Visualization Tools -- [x] `render_performance_chart` -- Recharts ComposedChart with fitness/fatigue/performance curves -- [x] `render_funnel_plot` -- Visx scatter plot for publication bias detection -- [x] `render_dose_response` -- Visx area+line chart for dose-response curves - -### Infrastructure -- [x] `PubMedRepository` ABC + `PubMedClient` adapter (ESearch JSON + EFetch XML parsing) -- [x] `ExerciseRepository` ABC + `WgerClient` adapter (REST with pagination) -- [x] 4 new domain entities: `PubMedArticle`, `Exercise`, `PerformanceModelPoint`, `DoseResponsePoint` -- [x] Frontend: 3 lazy-loaded chart components registered in `VizRegistry` - -### Phase 2 (Deferred Items -- Now Complete) -- [x] PubMed integration in literature survey (domain-specific prompt guidance + search stats tracking) -- [x] Richer injury risk model (epidemiological context from PubMed systematic reviews) -- [x] `plan_periodization` tool (linear/undulating/block models with evidence-based prescriptions) -- [x] 2 new domain entities: `PeriodizationBlock`, `PeriodizationPlan` -- [x] `SEARCH_TOOLS` module-level frozenset for search stats tracking - -**Counts:** 48 → 56 tools, 13 → 15 data sources, 6 → 9 viz tools. 577 server tests, 107 console tests. - ---- - -## Nutrition Science Enhancement (Feb 12) -- DONE - -Deepened the nutrition bounded context with 1 new data source, 6 new tools, and 3 new visualization tools. - -### New Data Source -- [x] RxNav API (`https://rxnav.nlm.nih.gov/REST`) -- Drug interaction screening via RxCUI resolution - -### New Tools -- [x] `compare_nutrients` -- side-by-side nutrient comparison between 2+ foods/supplements -- [x] `assess_nutrient_adequacy` -- DRI-based nutrient adequacy assessment (EAR, RDA, AI, UL) -- [x] `check_intake_safety` -- Tolerable Upper Intake Level safety screening -- [x] `check_interactions` -- Drug-supplement interaction screening via RxNav -- [x] `analyze_nutrient_ratios` -- Key nutrient ratio analysis (omega-6:3, Ca:Mg, Na:K, Zn:Cu, Ca:P, Fe:Cu) -- [x] `compute_inflammatory_index` -- Simplified Dietary Inflammatory Index (DII) scoring - -### New Visualization Tools -- [x] `render_nutrient_comparison` -- Recharts grouped BarChart comparing nutrient profiles -- [x] `render_nutrient_adequacy` -- Recharts horizontal BarChart with MAR score and DRI zones -- [x] `render_therapeutic_window` -- Visx custom range chart with EAR/RDA/AI/UL therapeutic zones - -### Infrastructure -- [x] `InteractionRepository` ABC + `RxNavClient` adapter (RxCUI resolution + interaction lookup) -- [x] `dri.py` DRI reference module (~30 nutrients, EAR/RDA/AI/UL by age_group/sex) -- [x] 3 new domain entities: `DrugInteraction`, `NutrientRatio`, `AdequacyResult` -- [x] `NutritionService` expanded with 6 new methods + interaction repository injection -- [x] Frontend: 3 lazy-loaded chart components registered in `VizRegistry` - -**Counts:** 56 -> 67 -> 70 tools (3 generic ML), 15 -> 16 data sources, 9 -> 12 viz tools. 692 server tests, 120 console tests. - ---- - -## Phase 13: Impact Evaluation Domain -- IN PROGRESS - -New bounded context for hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools usable by any domain. No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. See `docs/impact-evaluation-domain.md` for full design. - -### Market Gap - -Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evaluations (0.4%). No competitor combines: autonomous hypothesis formulation + causal inference + public API connectors + evidence grading + multi-model orchestration. - -### Phase 13A: Foundation (Universal APIs + Domain Config) -- DONE - -- [x] World Bank API client (`https://api.worldbank.org/v2/`) -- development indicators by country -- [x] WHO GHO API client (`https://ghoapi.azureedge.net/api/`) -- health statistics -- [x] FRED API client (`https://api.stlouisfed.org/fred/`) -- 800K+ economic time series -- [x] `search_economic_indicators` tool -- query FRED/World Bank/WHO GHO time series -- [x] `fetch_benchmark` tool -- get comparison values from international sources -- [x] `compare_programs` tool -- cross-program comparison using existing statistical tests -- [x] `IMPACT_EVALUATION` domain config with detection keywords and experiment examples -- [x] Full DDD: domain entities + repository ABCs + infrastructure clients -- [x] Tests: impact bounded context (unit + integration) - -**Counts:** 70 -> 73 tools, 16 -> 19 data sources (18 external + 1 internal), 10 -> 11 bounded contexts, 3 -> 4 domains. - -### Phase 13B (partial): Causal Inference + Impact Viz -- DONE - -- [x] `CausalEstimator` port ABC in `impact/domain/ports.py` (initial placement) -- [x] `DiDEstimator` infrastructure with parallel trends test and automated threat assessment -- [x] `estimate_did` tool -- difference-in-differences with effect size, SE, p-value, threats, evidence tier -- [x] `assess_threats` tool -- knowledge-based validity threat assessment (DiD, PSM, RDD, RCT, IV) -- [x] `render_program_dashboard` viz tool -- multi-indicator KPI dashboard with target tracking -- [x] `render_geographic_comparison` viz tool -- region bar chart with benchmark reference line -- [x] `render_parallel_trends` viz tool -- DiD treatment vs control over time -- [x] Console: 3 new React chart components (ProgramDashboard, GeographicComparison, ParallelTrends) -- [x] Console: 2 Impact Evaluation template cards (CCT School Enrollment, Health Worker Programs) -- [x] Tests: DiD estimator + tool tests (35+ new tests) - -**Counts:** 73 -> 78 tools, 12 -> 15 viz tools, 3 -> 5 impact tools. - -### Phase 13B: Domain-Agnostic Causal Inference -- DONE - -Refactored causal inference from impact-specific to domain-agnostic. All causal estimation methods now live in `analysis/` (alongside statistics), usable by any domain. - -- [x] Moved `ThreatToValidity`, `CausalEstimate` entities from `impact/domain/` to `analysis/domain/causal.py` -- [x] Moved `evaluation_standards.py` (WWC tiers, CONEVAL, CREMAA) from `impact/domain/` to `analysis/domain/evidence_standards.py` -- [x] Created `analysis/domain/causal_ports.py` -- 4 estimator port ABCs: `DiDEstimatorPort`, `PSMEstimatorPort`, `RDDEstimatorPort`, `SyntheticControlPort` -- [x] Moved `DiDEstimator` from `impact/infrastructure/` to `analysis/infrastructure/did_estimator.py` -- [x] New `analysis/infrastructure/psm_estimator.py` -- propensity score matching with balance diagnostics -- [x] New `analysis/infrastructure/rdd_estimator.py` -- regression discontinuity (sharp/fuzzy) -- [x] New `analysis/infrastructure/synthetic_control_estimator.py` -- synthetic control method -- [x] New `analysis/application/causal_service.py` -- orchestrates all 4 estimators + threat assessment + cost-effectiveness -- [x] New `compute_cost_effectiveness` tool -- cost per unit outcome, ICER -- [x] Deleted `impact/domain/ports.py` -- causal estimator port moved to analysis/ -- [x] Trimmed `ImpactService` -- removed DiD/threat methods (delegated to CausalService) -- [x] Impact tools reduced from 5 to 3 (estimate_did + assess_threats moved to analysis/) -- [x] 6 causal tools tagged `"causal"` in analysis/tools.py -- [x] New `render_rdd_plot` viz tool -- regression discontinuity scatter with cutoff (Visx) -- [x] New `render_causal_diagram` viz tool -- DAG showing treatment/outcome/confounders (Visx) -- [x] Console: `RDDPlot.tsx`, `CausalDiagram.tsx` chart components -- [x] Console: `ThreatAssessment.tsx` panel with severity badges -- [x] Console: 2 new template cards (Education Policy, Health Policy) -- 9 total -- [x] VizRegistry updated with 2 new viz types - -**Counts:** 78 -> 84 tools (6 causal + 1 cost-effectiveness - 2 moved + 2 viz), 15 -> 17 viz tools, 5 -> 3 impact tools, 0 -> 6 causal tools (analysis/), 7 -> 9 templates. - -### Phase 13A-2: Document Upload -- DONE - -- [x] File upload API (`POST /upload`) with multipart/form-data, 50MB limit, WorkOS auth -- [x] PDF text extraction (`pymupdf`) with page-level parsing and 8000-char truncation -- [x] CSV/Excel parsing (`pandas` + `openpyxl`) with summary statistics and sample rows -- [x] `UploadedFile` domain entity with `TabularData` / `DocumentData` frozen dataclasses -- [x] `FileProcessor` application service (zero external deps in domain layer) -- [x] PostgreSQL `uploaded_files` table with JSONB `parsed_data` column -- [x] Inject uploaded data into Director/Researcher prompts as `` XML block -- [x] `query_uploaded_data` tool -- intercepted by `ToolDispatcher`, pandas-based filtering (eq/gt/lt/gte/lte/contains) -- [x] Upload-first flow: `POST /upload` returns preview, `POST /investigate` references `file_ids` -- [x] Frontend: `FileUpload.tsx` drag-and-drop component with progress spinner -- [x] Frontend: `DataPreview.tsx` compact inline cards for tabular/document previews -- [x] Frontend: `use-upload.ts` TanStack Query mutation hook -- [x] Tests: 16 server tests (FileProcessor + domain entities + query handler) + 16 console tests - -**Counts:** 84 -> 85 tools (added `query_uploaded_data`). Investigation tools: 7 -> 8. - -### Phase 13C: Mexico Integration - -- [ ] INEGI Indicadores API client (`https://www.inegi.org.mx/app/api/indicadores/`) -- census, demographics -- [ ] INEGI DENUE API client (`https://www.inegi.org.mx/app/api/denue/v1/`) -- 5M+ businesses -- [ ] datos.gob.mx CKAN client (`https://datos.gob.mx/api/3/action/`) -- 1000+ federal datasets -- [ ] Transparencia Presupuestaria client (`https://nptp.hacienda.gob.mx/`) -- budget execution -- [ ] Banxico SIE client (`https://www.banxico.org.mx/SieAPIRest/service/v1/`) -- economic indicators -- [ ] `search_census_data` tool -- query INEGI/Census by indicator/geography/period -- [ ] `search_budget_data` tool -- query budget execution and social program spending -- [ ] `extract_mir` tool -- extract MIR (Matriz de Indicadores) from evaluation documents -- [ ] `analyze_program_indicators` tool -- MIR/logical framework indicator analysis with CREMAA validation -- [ ] Frontend: `MIRTable.tsx` logical framework matrix -- [ ] CONEVAL ECR report template for `generate_evaluation_report` - -### Phase 13D: US Integration - -- [ ] Census Bureau API client (`https://api.census.gov/data/`) -- ACS, demographics -- [ ] BLS API client (`https://api.bls.gov/publicAPI/v2/`) -- employment, wages, CPI -- [ ] USAspending API client (`https://api.usaspending.gov/api/v2/`) -- federal grants/contracts -- [ ] College Scorecard API client (`https://api.data.gov/ed/collegescorecard/v1/`) -- education outcomes -- [ ] HUD API client (`https://www.huduser.gov/hudapi/public/`) -- housing data -- [ ] CDC WONDER client (`https://wonder.cdc.gov/`) -- public health data -- [ ] `search_health_indicators` tool -- query WHO GHO or CDC WONDER -- [ ] `search_open_data` tool -- query datos.gob.mx or data.gov CKAN portals -- [ ] WWC evidence tier classification in evaluation reports -- [ ] US-formatted evaluation report template - -### Phase 13E: MCP Bridge Self-Service - -- [ ] User MCP server registration UI (`/settings/data-sources`) -- [ ] Connection testing and tool auto-discovery -- [ ] Per-organization data source management -- [ ] Tool auto-tagging with domain tags for discovery - -### Data Sources (Planned: 8+ new, 3 implemented in 13A) - -| Source | API | Purpose | Auth | Phase | -|--------|-----|---------|------|-------| -| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country | None | 13A | -| WHO GHO | `https://ghoapi.azureedge.net/api/` | Health statistics by country | None | 13A | -| FRED | `https://api.stlouisfed.org/fred/` | 800K+ economic time series | API key | 13A | -| INEGI Indicadores | `https://www.inegi.org.mx/app/api/indicadores/` | Mexico census, demographics | Token | 13C | -| INEGI DENUE | `https://www.inegi.org.mx/app/api/denue/v1/` | Mexico business directory (5M+) | Token | 13C | -| datos.gob.mx | `https://datos.gob.mx/api/3/action/` | Mexico open data (CKAN, 1000+ datasets) | Token (optional) | 13C | -| Transparencia Presupuestaria | `https://nptp.hacienda.gob.mx/` | Mexico budget execution | None | 13C | -| Banxico SIE | `https://www.banxico.org.mx/SieAPIRest/service/v1/` | Mexico economic indicators | Token | 13C | -| US Census Bureau | `https://api.census.gov/data/` | ACS, demographics | API key (optional) | 13D | -| BLS | `https://api.bls.gov/publicAPI/v2/` | Employment, wages, CPI | API key | 13D | -| USAspending | `https://api.usaspending.gov/api/v2/` | Federal grants, contracts | None | 13D | - -### Tools (Planned: ~18 new tools + ~6 viz tools) - -**Data Access:** `search_census_data`, `search_economic_indicators`, `search_budget_data`, `search_health_indicators`, `search_open_data`, `fetch_benchmark` -**Causal Inference:** `estimate_did`, `estimate_psm`, `estimate_rdd`, `estimate_synthetic_control`, `assess_threats` -**Program Analysis:** `analyze_program_indicators`, `compute_cost_effectiveness`, `compare_programs`, `analyze_beneficiary_data` -**Document Processing:** `extract_program_data`, `extract_mir`, `generate_evaluation_report` -**Visualization:** `render_causal_diagram`, `render_parallel_trends`, `render_rdd_plot`, `render_program_dashboard`, `render_cost_effectiveness`, `render_geographic_comparison` - ---- - -## Competitive Sports Domain -- TODO - -New bounded context for actual competitive sports analytics: game statistics, player performance, team analysis, and sports-specific strategy research. - -### Bounded Context: `sports/` - -``` -server/src/ehrlich/sports/ -├── domain/ -│ ├── entities.py # Player, Team, GameStats, SeasonStats, PerformanceMetric -│ └── repository.py # SportsDataRepository ABC -├── application/ -│ └── sports_service.py # SportsService (stats analysis, performance comparison) -├── infrastructure/ -│ └── *_client.py # API clients for sports data providers -└── tools.py # Tools for Claude -``` - -### Data Sources (Research Needed) -- [ ] Basketball: NBA API / Basketball Reference (player stats, advanced metrics, play-by-play) -- [ ] Soccer/Football: football-data.org or API-Football (match results, player stats, league standings) -- [ ] American Football: NFL data feeds (player stats, play-by-play, combine data) -- [ ] General: ESPN API or Sports Reference family (multi-sport coverage) - -### Planned Tools -- [ ] `search_player_stats` -- player statistics by name/team/season (points, assists, rebounds, goals, etc.) -- [ ] `compare_players` -- side-by-side player comparison with advanced metrics -- [ ] `analyze_team_performance` -- team-level analytics (win rate, offensive/defensive efficiency) -- [ ] `search_sports_literature` -- Semantic Scholar + PubMed for sports-specific research (tactics, biomechanics, game theory) -- [ ] `compute_advanced_metrics` -- sport-specific advanced stats (PER, WAR, xG, passer rating, etc.) -- [ ] `analyze_matchup` -- head-to-head matchup analysis between players or teams - -### Domain Config: `COMPETITIVE_SPORTS` -- tool_tags: `{"sports", "literature"}` -- valid_domain_categories: `("competitive_sports", "basketball", "soccer", "football", "baseball", "tennis")` -- hypothesis_types: `("performance", "tactical", "statistical", "predictive")` -- score_definitions: performance_score, statistical_significance, sample_size -- identifier_type: `"player"` or `"team"` - -### Visualization -- [ ] `render_player_radar` -- radar chart comparing player attributes (Recharts) -- [ ] `render_season_timeline` -- performance trends over a season (Recharts) -- [ ] `render_shot_chart` -- spatial shot/play visualization (Custom SVG, basketball/soccer) - -### Cross-Domain Potential -- Sports + Training: "What training protocols produce the best NBA pre-season conditioning results?" -- Sports + Nutrition: "What supplements do elite soccer players use and what's the evidence?" -- Sports + Molecular: "What are the pharmacological mechanisms behind caffeine's effect on sprint performance?" - ---- - -## Additional Domains -- BACKLOG - -The engine is domain-agnostic via `DomainConfig` + `DomainRegistry`. Adding a new domain requires zero changes to existing code (see `CONTRIBUTING.md`). Potential future domains: - -- [ ] Genomics / bioinformatics (variant interpretation, gene expression analysis) -- [ ] Environmental science (pollutant fate, ecosystem modeling) -- [ ] Materials science (property prediction, synthesis planning) -- [ ] Clinical pharmacology (drug interactions, pharmacokinetics) - -For molecular science specifically: -- [ ] Expanded organism coverage (E. coli, P. aeruginosa, A. baumannii, M. tuberculosis) -- [ ] Per-organism resistance knowledge base with literature references -- [ ] Organism-aware prompt guidance (Gram-neg vs mycobacteria screening strategies) - ---- - -## Phase 11: Investigation Comparison -- BACKLOG - -Side-by-side comparison of investigation runs for reproducibility and consensus analysis. - -### 11A. Comparison Domain -- [ ] `Comparison` entity: list of investigation IDs, consensus candidates, overlap metrics -- [ ] Candidate overlap calculation (by identifier + domain-specific similarity) -- [ ] Finding overlap detection (by hypothesis + title similarity) -- [ ] Score aggregation across runs (mean, std, min, max) - -### 11B. Comparison API -- [ ] `POST /compare` -- accept list of investigation IDs, return comparison -- [ ] `GET /compare/{id}` -- retrieve saved comparison - -### 11C. Comparison Console -- [ ] `/compare` page: pick 2+ completed investigations -- [ ] Side-by-side candidate table with overlap highlighting -- [ ] Consensus candidates panel (appear in N/M runs) -- [ ] Score distribution visualization (per candidate across runs) -- [ ] Findings diff view (shared vs unique per run) - -**Verification:** Compare 2 completed investigations, verify overlap metrics and consensus candidates render correctly. - ---- - -## ~~MCP Server~~ -- REJECTED - -Exposing Ehrlich's 84 tools as an MCP server was considered and rejected. The tools alone (ChEMBL queries, RDKit computations, etc.) are commodity API wrappers -- the value is the multi-model orchestration (Director-Worker-Summarizer), hypothesis-driven methodology, and parallel experiment execution. An MCP client consuming Ehrlich's tools would lose the scientific rigor that the orchestration guarantees. - -Ehrlich remains an MCP **consumer** via `MCPBridge` (connecting to external servers like Excalidraw), which adds value by extending the Researcher's toolkit. - ---- - -## Phase 12: SaaS Infrastructure (Feb 13) -- DONE - -Production-ready authentication, credit-based billing, model-tiered investigations, and PostgreSQL persistence. - -> Business rationale and pricing research: see [business-strategy.md](business-strategy.md) - -### 12A. PostgreSQL Migration -- DONE - -Replaced SQLite with PostgreSQL for multi-tenant persistence. - -- [x] `InvestigationRepository` implementing existing `InvestigationRepository` ABC -- [x] `asyncpg` connection pooling (replaced `aiosqlite`) -- [x] `tsvector` + GIN index for self-referential research (replaces FTS5) -- [x] `JSONB` columns for hypotheses, experiments, findings, candidates -- [x] `users` table: id (UUID), workos_id, email, credits, encrypted_api_key, created_at -- [x] `credit_transactions` table: user_id, amount, type, investigation_id, created_at -- [x] `EHRLICH_DATABASE_URL` env var (replaces `EHRLICH_DB_PATH`) -- [x] `sqlite_repository.py` deleted, `aiosqlite` removed from dependencies - -### 12B. Authentication (WorkOS AuthKit) -- DONE - -User identity via WorkOS (1M MAU free tier). JWT-based. - -- [x] `api/auth.py` with JWKS verification via `PyJWKClient` -- [x] Frontend: `@workos-inc/authkit-react` provider + `` -- [x] Backend: `get_current_user`, `get_current_user_sse`, `get_optional_user` dependencies -- [x] `user_id` extraction from JWT, linked to `users` table via `get_or_create_user()` -- [x] Public routes: `GET /health`, `GET /methodology`, `GET /stats`, `GET /molecule/*`, `GET /investigate/{id}` -- [x] Protected routes: `POST /investigate`, `GET /investigate`, `GET /investigate/{id}/stream`, `POST /investigate/{id}/approve`, `GET /credits/balance` -- [x] Env vars: `EHRLICH_WORKOS_API_KEY`, `EHRLICH_WORKOS_CLIENT_ID` -- [x] SSE auth via `?token=` query param fallback (EventSource does not support headers) - -### 12C. Universal Credit System -- DONE - -One currency. User decides model quality per investigation. - -- [x] Credit spending: Haiku Director = 1 credit, Sonnet Director = 3 credits, Opus Director = 5 credits -- [x] Researcher always Sonnet 4.5, Summarizer always Haiku 4.5 (regardless of tier) -- [x] `GET /credits/balance` -- current balance + BYOK status -- [x] `POST /investigate` accepts `director_tier: "haiku" | "sonnet" | "opus"` parameter -- [x] Credit deduction on investigation start (not completion -- prevents abuse) -- [x] Credit refund on investigation failure -- [x] Default 5 credits for new users - -### 12D. BYOK (Bring Your Own Key) -- DONE - -Users provide their own Anthropic API key. Bypasses credit system. - -- [x] Frontend: `BYOKSettings.tsx` for API key management, stored in `localStorage` -- [x] `X-Anthropic-Key` header sent with requests -- [x] Backend: `api_key_override` forwarded to `AnthropicClientAdapter` -- [x] BYOK users bypass credit system -- pay Anthropic directly - -### 12E. Cost Transparency -- DONE - -Users see exactly what each investigation costs. - -- [x] `CompletionSummaryCard` with expandable cost breakdown -- [x] Tokens (input/output/cache read/cache write) by model role (Director/Researcher/Summarizer) -- [x] Cost data in real-time during SSE stream via `CostUpdate` events - -### 12F-H. Deferred to Post-Hackathon - -- [ ] Ad-funded bonus credits (12F) -- [ ] AGPL dual licensing docs (12G) -- [ ] Production deployment (12H) - -**Verification:** Auth flow, credit deduction/refund, BYOK pass-through, tier selection, cost breakdown all working. - ---- - -## Phase 14: Scientific Engine Hardening (Feb 14-15) -- IN PROGRESS - -Architecture cleanup, scientific depth improvements, agentic tree search, and paper generation. Runs in parallel with Phase 13A-2 (upload). Addresses audit findings from codebase review. - -### Prior Art - -Ehrlich sits alongside three contemporary AI scientific discovery systems: - -- **Sakana AI Scientist v2** (Apr 2025): End-to-end paper generation via agentic tree search. First AI-generated peer-reviewed workshop paper. ML-only. [arXiv:2504.08066](https://arxiv.org/abs/2504.08066) -- **FutureHouse Robin** (May 2025): Multi-agent system that discovered ripasudil as therapeutic candidate for AMD in 2.5 months. Biology-only, lab-in-the-loop. [arXiv:2505.13400](https://arxiv.org/abs/2505.13400) -- **Ai2 AutoDiscovery** (Feb 2026): Bayesian surprise-guided hypothesis exploration over large search spaces. [allenai.org/blog/autodiscovery](https://allenai.org/blog/autodiscovery) - -Ehrlich differentiates by: domain-agnostic architecture (4 domains vs single-domain competitors), multi-model orchestration (Director/Researcher/Summarizer), real-time streaming UI, 18 external data sources, and computational laboratory (ML pipelines + causal inference as experimentation). - -### 14A: Orchestrator Decomposition -- TODO - -Split `MultiModelOrchestrator` (1,788 lines, 11 concerns) into focused modules following Clean Architecture. No behavior changes -- pure structural refactoring. - -- [x] Extract `diagram_builder.py` -- Excalidraw element building + MCP bridge export (127 lines) -- [x] Extract `tool_dispatcher.py` -- tool execution + caching + `search_prior_research` / `query_uploaded_data` interception (34 lines) -- [x] Extract `researcher_executor.py` -- single experiment runner with tool loop (216 lines) -- [x] Extract `batch_executor.py` -- parallel experiment execution via asyncio.Queue (56 lines) -- [x] Merged uploaded_data_query into tool_dispatcher.py -- [x] Kept run() in multi_orchestrator.py as main coordination point -- [ ] Orchestrator reduced to ~400 lines: init, run (delegates to phase_manager), approve - -Result: `multi_orchestrator.py` from 1,788 to 1,216 lines (4 modules extracted: 718 lines total) - -### 14B: Route File Cleanup -- TODO - -Move infrastructure concerns out of `api/routes/investigation.py` (828 lines) into application layer. - -- [ ] Extract `application/registry_factory.py` -- `build_tool_registry()` + `build_domain_registry()` (128 lines of tool registration) -- [ ] Extract `application/orchestrator_factory.py` -- `create_orchestrator()` with all adapter wiring -- [ ] Move Pydantic DTOs to `api/schemas/investigation.py` (6 models) -- [ ] Lazy-initialize tool registry (singleton, built once instead of per-investigation) - -Target: route file from 828 to ~400 lines. - -### 14C: Prompt Module Split -- TODO - -Split `prompts.py` (1,512 lines) into focused modules. - -- [ ] `prompts/__init__.py` -- re-exports for backward compatibility -- [ ] `prompts/constants.py` -- `SCIENTIST_SYSTEM_PROMPT`, `SUMMARIZER_PROMPT`, `RESEARCHER_EXPERIMENT_PROMPT` -- [ ] `prompts/director.py` -- `DIRECTOR_FORMULATION_PROMPT`, `DIRECTOR_EXPERIMENT_PROMPT`, `DIRECTOR_EVALUATION_PROMPT`, `DIRECTOR_SYNTHESIS_PROMPT` -- [ ] `prompts/builders.py` -- `build_pico_and_classification_prompt`, `build_literature_survey_prompt`, `build_literature_assessment_prompt`, `build_formulation_prompt`, `build_experiment_prompt`, `build_synthesis_prompt`, `build_researcher_prompt`, `build_multi_investigation_context`, `build_uploaded_data_context`, `_build_prior_context` - -### 14D: Domain Tool Enrichment -- TODO - -Upgrade 4 composite tools from thin pass-throughs to real domain-specific logic. - -- [ ] `search_training_literature` -- MeSH term expansion, study type ranking (SR > RCT > cohort), date recency weighting, non-human study filtering -- [ ] `search_supplement_evidence` -- GRADE-style quality filtering, RCT/meta-analysis prioritization, retracted paper exclusion -- [ ] `compare_nutrients` -- per-nutrient delta computation, winner highlighting per category, overall MAR score comparison -- [ ] Fix `TrainingService` I-squared reimplementation -- delegate to `StatisticsService` (training_service.py:79-85) - -### 14E: Agentic Hypothesis Tree Search -- TODO - -Replace linear hypothesis testing with tree-based exploration inspired by AI Scientist v2's agentic tree search. The current parallel batch system (2 experiments per hypothesis) becomes a tree where hypotheses can branch, deepen, or be pruned based on evidence. - -- [ ] `HypothesisNode` dataclass -- extends Hypothesis with `depth`, `branch_score`, `children: list[str]` -- [ ] `TreeManager` in `application/tree_manager.py` -- decides which branches to explore based on confidence scores, prunes low-confidence branches (<0.4 threshold), tracks exploration budget -- [ ] Director formulation prompt update -- "given current evidence tree, propose sub-hypotheses for the most promising branch OR prune low-confidence branches" -- [ ] Director evaluation prompt update -- after evaluating, decide: (a) deepen (spawn sub-hypotheses), (b) prune (mark branch as dead), (c) branch (revise into alternative direction) -- [ ] `max_depth` parameter (default: 3) to prevent infinite recursion -- [ ] Budget-aware exploration -- track API cost per branch, prefer deepening high-confidence/low-cost branches -- [ ] `HypothesisTreeEvent` SSE event for frontend tree rendering -- [ ] Console: `InvestigationDiagram.tsx` already renders React Flow graph -- tree structure renders naturally as growing/pruning graph - -### 14F: Scientific Paper Generator -- TODO - -Generate structured scientific papers from completed investigations. Not AI-generated prose -- structured assembly of existing investigation data into publication format. - -- [ ] `application/paper_generator.py` -- takes `Investigation` entity, produces structured sections: - - Title (from research question) - - Abstract (Director synthesis summary) - - Introduction (PICO decomposition + literature survey context) - - Methods (experiment designs, tool descriptions, statistical approach) - - Results (findings with statistical tests, visualizations as base64 SVG) - - Discussion (hypothesis assessments, GRADE certainty, limitations, knowledge gaps) - - References (collected DOIs + citations) - - Supplementary (cost breakdown, model info, negative controls, tool call trace) -- [ ] `GET /investigate/{id}/paper` endpoint -- returns structured paper as JSON or Markdown -- [ ] `GET /investigate/{id}/paper?format=pdf` -- server-side PDF generation (optional, `weasyprint` or `reportlab`) -- [ ] Console: "Export Paper" button on completed investigation page -- [ ] Console: paper preview with section navigation - -### 14G: Validation Runs -- TODO - -Run real investigations to validate the engine produces meaningful scientific results. Compare Ehrlich vs raw Claude Opus 4.6 on identical prompts. - -- [ ] Molecular science validation: known drug discovery question with verifiable answer -- [ ] Training science validation: exercise physiology question with literature-backed answer -- [ ] Impact evaluation validation: real program evaluation data (CODESON) compared against professional auditors and raw Claude -- [ ] Document comparison results: where Ehrlich agrees/disagrees with baseline, where tree search found non-obvious paths -- [ ] Capture investigation traces for demo video - -### Dependency Graph - -``` -Phase 14A (Orchestrator Decomposition) - | -Phase 14B (Route File Cleanup) -- parallel with 14A - | -Phase 14C (Prompt Module Split) -- parallel with 14A/B - | -Phase 14D (Domain Tool Enrichment) -- after 14A (touches tool layer) - | -Phase 14E (Agentic Tree Search) -- after 14A (modifies orchestrator) - | -Phase 14F (Paper Generator) -- after 14A (reads investigation data) - | -Phase 14G (Validation Runs) -- after 14E + 14F (needs tree search + paper gen) -``` - -14A, 14B, 14C can run in parallel (independent refactorings). -14D, 14E, 14F depend on 14A (clean orchestrator). -14G depends on everything else being done. - -**Verification:** All existing tests pass after each sub-phase. New tests for tree search, paper generator, and enriched tools. Quality gates: ruff 0, mypy 0, pytest pass, tsc 0, vitest pass. - ---- - -## Backlog (Post-Hackathon) - -- Phase 11: Investigation Comparison (side-by-side analysis) -- Phase 13C: Mexico Integration (INEGI, datos.gob.mx, Transparencia, Banxico) -- Phase 13D: US Integration (Census, BLS, USAspending, CDC WONDER) -- Phase 13E: MCP Bridge Self-Service -- Competitive Sports Domain -- Batch screening mode (score entire ZINC subsets) -- Automated synthesis lab integration (Emerald Cloud Lab, Strateos) -- Community contributions: new domains, new tools, new data sources -- Rewarded ads for free credits (AdMob/AdSense integration) -- Stripe integration for paid credit packs -- Admin dashboard for usage analytics -- Planner agent (Sonnet 4.5) for experiment design between Director and Researcher -- Reviewer agent (Sonnet 4.5) for findings audit between Researcher and Director -- Full Bayesian surprise scoring for tree search branch selection diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md new file mode 100644 index 0000000..4f24593 --- /dev/null +++ b/docs/roadmap/README.md @@ -0,0 +1,100 @@ +# Ehrlich Roadmap + +Hackathon: Feb 10-16, 2026. Each phase ends with something testable. + +## Phases + +| Phase | Status | File | +|-------|--------|------| +| Phase 0: Infrastructure | DONE | [phase-00-infrastructure.md](phase-00-infrastructure.md) | +| Phase 1: Chemistry Context | DONE | [phase-01-chemistry.md](phase-01-chemistry.md) | +| Phase 2: Literature + Analysis | DONE | [phase-02-literature-analysis.md](phase-02-literature-analysis.md) | +| Phase 3: Prediction Context | DONE | [phase-03-prediction.md](phase-03-prediction.md) | +| Phase 4: Simulation Context | DONE | [phase-04-simulation.md](phase-04-simulation.md) | +| Phase 5: Investigation Agent | DONE | [phase-05-investigation-agent.md](phase-05-investigation-agent.md) | +| Phase 6: Console Integration | DONE | [phase-06-console.md](phase-06-console.md) | +| Phase 7: Integration + Demo | DONE | [phase-07-integration-demo.md](phase-07-integration-demo.md) | +| Phase 8: Multi-Model Architecture + Polish | DONE | [phase-08-multi-model.md](phase-08-multi-model.md) | +| Phase 9: Molecule Visualization Suite | DONE | [phase-09-molecule-visualization.md](phase-09-molecule-visualization.md) | +| Phase 10A: Hypothesis-Driven Investigation Engine | DONE | [phase-10a-hypothesis-engine.md](phase-10a-hypothesis-engine.md) | +| Phase 10B: Domain-Agnostic Generalization | DONE | [phase-10b-domain-agnostic.md](phase-10b-domain-agnostic.md) | +| Phase 10C: Claude SDK Optimization | DONE | [phase-10c-sdk-optimization.md](phase-10c-sdk-optimization.md) | +| Phase 10D: Landing Site (web/) | DONE | [phase-10d-landing-site.md](phase-10d-landing-site.md) | +| Phase 10E: Training Science Enhancement | DONE | [phase-10e-training-enhancement.md](phase-10e-training-enhancement.md) | +| Phase 10F: Nutrition Science Enhancement | DONE | [phase-10f-nutrition-enhancement.md](phase-10f-nutrition-enhancement.md) | +| Phase 12: SaaS Infrastructure | DONE | [phase-12-saas-infrastructure.md](phase-12-saas-infrastructure.md) | +| Phase 13: Impact Evaluation Domain | IN PROGRESS | [phase-13-impact-evaluation.md](phase-13-impact-evaluation.md) | +| Phase 14: Scientific Engine Hardening | IN PROGRESS | [phase-14-engine-hardening.md](phase-14-engine-hardening.md) | +| Backlog | TODO | [backlog.md](backlog.md) | + +## Dependency Graph + +``` +Phase 0 (Infrastructure) -- DONE + | +Phase 1 (Chemistry) -- DONE + | \ +Phase 2A-C Phase 2D-E +(Literature) (Analysis) -- DONE + | | + +----- + ------+ + | + Phase 3 (Prediction) -- DONE + | + Phase 4 (Simulation) -- DONE + | + Phase 5 (Agent Loop) -- DONE + | + Phase 6 (Console) -- DONE + | + Phase 7 (Integration + Demo) -- DONE + | + Phase 8 (Multi-Model + Polish) -- DONE + | + Phase 9 (Molecule Visualization) -- DONE + | + Phase 10A (Hypothesis-Driven Engine) -- DONE + | + Scientific Methodology Upgrade (cross-cutting) -- All 6 Phases DONE + | + Domain-Agnostic Generalization -- DONE + | + Multi-Domain Investigations -- DONE + | + Self-Referential Research -- DONE + | + Shared Context + MCP Bridge + Training/Nutrition APIs -- DONE + | + Domain-Specific Visualization System -- DONE + | + Claude SDK Optimization -- DONE + | + Landing Site (web/) -- DONE + (TanStack Start + SSR) + | + ┌─────┼──────────────┐ + │ │ │ +Training Nutrition Competitive +Enhancement Enhancement Sports + (DONE) (DONE) Domain (TODO) + │ │ │ + └─────┼──────────────┘ + | + Phase 12: SaaS Infrastructure -- DONE + (PostgreSQL + WorkOS Auth + BYOK + Credits) + | + Phase 13: Impact Evaluation Domain -- IN PROGRESS + (Causal inference + Document upload + MX/US APIs) + 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) DONE -> 13C (Mexico) -> 13D (US) -> 13E (MCP) + | + Phase 14: Scientific Engine Hardening -- IN PROGRESS (parallel with 13A-2) + 14A (Orchestrator Split) ──┬── 14B (Route Cleanup) ──┬── 14D (Tool Enrichment) + ├── 14C (Prompt Split) ├── 14E (Tree Search) + └───────────────────────── └── 14F (Paper Generator) + | + 14G (Validation Runs) + | + Demo + Video -- TODO + | + Submission (Feb 16, 3PM ET) -- TODO +``` diff --git a/docs/roadmap/backlog.md b/docs/roadmap/backlog.md new file mode 100644 index 0000000..110877f --- /dev/null +++ b/docs/roadmap/backlog.md @@ -0,0 +1,118 @@ +Back to [Roadmap Index](README.md) + +# Backlog (Post-Hackathon) + +- Phase 11: Investigation Comparison (side-by-side analysis) +- Phase 13C: Mexico Integration (INEGI, datos.gob.mx, Transparencia, Banxico) +- Phase 13D: US Integration (Census, BLS, USAspending, CDC WONDER) +- Phase 13E: MCP Bridge Self-Service +- Competitive Sports Domain +- Batch screening mode (score entire ZINC subsets) +- Automated synthesis lab integration (Emerald Cloud Lab, Strateos) +- Community contributions: new domains, new tools, new data sources +- Rewarded ads for free credits (AdMob/AdSense integration) +- Stripe integration for paid credit packs +- Admin dashboard for usage analytics +- Planner agent (Sonnet 4.5) for experiment design between Director and Researcher +- Reviewer agent (Sonnet 4.5) for findings audit between Researcher and Director +- Full Bayesian surprise scoring for tree search branch selection + +--- + +# Competitive Sports Domain -- TODO + +New bounded context for actual competitive sports analytics: game statistics, player performance, team analysis, and sports-specific strategy research. + +## Bounded Context: `sports/` + +``` +server/src/ehrlich/sports/ +├── domain/ +│ ├── entities.py # Player, Team, GameStats, SeasonStats, PerformanceMetric +│ └── repository.py # SportsDataRepository ABC +├── application/ +│ └── sports_service.py # SportsService (stats analysis, performance comparison) +├── infrastructure/ +│ └── *_client.py # API clients for sports data providers +└── tools.py # Tools for Claude +``` + +## Data Sources (Research Needed) +- [ ] Basketball: NBA API / Basketball Reference (player stats, advanced metrics, play-by-play) +- [ ] Soccer/Football: football-data.org or API-Football (match results, player stats, league standings) +- [ ] American Football: NFL data feeds (player stats, play-by-play, combine data) +- [ ] General: ESPN API or Sports Reference family (multi-sport coverage) + +## Planned Tools +- [ ] `search_player_stats` -- player statistics by name/team/season (points, assists, rebounds, goals, etc.) +- [ ] `compare_players` -- side-by-side player comparison with advanced metrics +- [ ] `analyze_team_performance` -- team-level analytics (win rate, offensive/defensive efficiency) +- [ ] `search_sports_literature` -- Semantic Scholar + PubMed for sports-specific research (tactics, biomechanics, game theory) +- [ ] `compute_advanced_metrics` -- sport-specific advanced stats (PER, WAR, xG, passer rating, etc.) +- [ ] `analyze_matchup` -- head-to-head matchup analysis between players or teams + +## Domain Config: `COMPETITIVE_SPORTS` +- tool_tags: `{"sports", "literature"}` +- valid_domain_categories: `("competitive_sports", "basketball", "soccer", "football", "baseball", "tennis")` +- hypothesis_types: `("performance", "tactical", "statistical", "predictive")` +- score_definitions: performance_score, statistical_significance, sample_size +- identifier_type: `"player"` or `"team"` + +## Visualization +- [ ] `render_player_radar` -- radar chart comparing player attributes (Recharts) +- [ ] `render_season_timeline` -- performance trends over a season (Recharts) +- [ ] `render_shot_chart` -- spatial shot/play visualization (Custom SVG, basketball/soccer) + +## Cross-Domain Potential +- Sports + Training: "What training protocols produce the best NBA pre-season conditioning results?" +- Sports + Nutrition: "What supplements do elite soccer players use and what's the evidence?" +- Sports + Molecular: "What are the pharmacological mechanisms behind caffeine's effect on sprint performance?" + +--- + +# Additional Domains -- BACKLOG + +The engine is domain-agnostic via `DomainConfig` + `DomainRegistry`. Adding a new domain requires zero changes to existing code (see `CONTRIBUTING.md`). Potential future domains: + +- [ ] Genomics / bioinformatics (variant interpretation, gene expression analysis) +- [ ] Environmental science (pollutant fate, ecosystem modeling) +- [ ] Materials science (property prediction, synthesis planning) +- [ ] Clinical pharmacology (drug interactions, pharmacokinetics) + +For molecular science specifically: +- [ ] Expanded organism coverage (E. coli, P. aeruginosa, A. baumannii, M. tuberculosis) +- [ ] Per-organism resistance knowledge base with literature references +- [ ] Organism-aware prompt guidance (Gram-neg vs mycobacteria screening strategies) + +--- + +# Phase 11: Investigation Comparison -- BACKLOG + +Side-by-side comparison of investigation runs for reproducibility and consensus analysis. + +## 11A. Comparison Domain +- [ ] `Comparison` entity: list of investigation IDs, consensus candidates, overlap metrics +- [ ] Candidate overlap calculation (by identifier + domain-specific similarity) +- [ ] Finding overlap detection (by hypothesis + title similarity) +- [ ] Score aggregation across runs (mean, std, min, max) + +## 11B. Comparison API +- [ ] `POST /compare` -- accept list of investigation IDs, return comparison +- [ ] `GET /compare/{id}` -- retrieve saved comparison + +## 11C. Comparison Console +- [ ] `/compare` page: pick 2+ completed investigations +- [ ] Side-by-side candidate table with overlap highlighting +- [ ] Consensus candidates panel (appear in N/M runs) +- [ ] Score distribution visualization (per candidate across runs) +- [ ] Findings diff view (shared vs unique per run) + +**Verification:** Compare 2 completed investigations, verify overlap metrics and consensus candidates render correctly. + +--- + +# ~~MCP Server~~ -- REJECTED + +Exposing Ehrlich's 84 tools as an MCP server was considered and rejected. The tools alone (ChEMBL queries, RDKit computations, etc.) are commodity API wrappers -- the value is the multi-model orchestration (Director-Worker-Summarizer), hypothesis-driven methodology, and parallel experiment execution. An MCP client consuming Ehrlich's tools would lose the scientific rigor that the orchestration guarantees. + +Ehrlich remains an MCP **consumer** via `MCPBridge` (connecting to external servers like Excalidraw), which adds value by extending the Researcher's toolkit. diff --git a/docs/roadmap/phase-00-infrastructure.md b/docs/roadmap/phase-00-infrastructure.md new file mode 100644 index 0000000..f36a242 --- /dev/null +++ b/docs/roadmap/phase-00-infrastructure.md @@ -0,0 +1,16 @@ +Back to [Roadmap Index](README.md) + +# Phase 0: Infrastructure (Feb 10) -- DONE + +- [x] Git init, LICENSE, README, CLAUDE.md, CHANGELOG +- [x] Server scaffolding: pyproject.toml, uv, all bounded contexts (domain/application/infrastructure/tools) +- [x] Kernel: SMILES/InChIKey/MolBlock types, Molecule value object, exception hierarchy +- [x] API: FastAPI factory, CORS, health endpoint +- [x] Console scaffolding: React 19, TypeScript, Bun, Vite, TanStack Router +- [x] Console features: investigation + molecule component stubs +- [x] CI/CD: GitHub Actions (ruff, mypy, pytest, tsc, vite build) +- [x] Docker: multi-stage builds (server + console) +- [x] Pre-commit: ruff lint + format +- [x] All quality gates passing (ruff 0, mypy 0, pytest 5/5, tsc 0) + +**Verification:** `uv run pytest` passes, `GET /api/v1/health` returns 200, `bun run build` succeeds. diff --git a/docs/roadmap/phase-01-chemistry.md b/docs/roadmap/phase-01-chemistry.md new file mode 100644 index 0000000..c6ed879 --- /dev/null +++ b/docs/roadmap/phase-01-chemistry.md @@ -0,0 +1,31 @@ +Back to [Roadmap Index](README.md) + +# Phase 1: Chemistry Context (Feb 10) -- DONE + +Foundation for the Molecular Science domain. Provides RDKit cheminformatics for SMILES processing, descriptors, fingerprints, and 3D conformers. + +## 1A. RDKit Adapter -- Core Operations +- [x] `validate_smiles(smiles)` -- `Chem.MolFromSmiles`, null-check, return bool +- [x] `canonicalize(smiles)` -- canonical SMILES via RDKit +- [x] `to_inchikey(smiles)` -- SMILES -> InChI -> InChIKey +- [x] Tests: valid/invalid SMILES, canonicalization consistency (24 tests) + +## 1B. RDKit Adapter -- Descriptors & Fingerprints +- [x] `compute_descriptors(smiles)` -- MW, LogP, TPSA, HBD, HBA, RotatableBonds, QED, NumRings +- [x] `compute_fingerprint(smiles, fp_type)` -- Morgan/ECFP (radius=2, 2048-bit) + MACCS (166-bit) +- [x] `tanimoto_similarity(fp1, fp2)` -- similarity score (0.0-1.0) +- [x] Tests: aspirin descriptors match known values, fingerprint bit counts + +## 1C. RDKit Adapter -- 3D & Substructure +- [x] `generate_conformer(smiles)` -- AddHs, EmbedMolecule(ETKDGv3), MMFFOptimize, MolToMolBlock +- [x] `substructure_match(smiles, pattern)` -- returns bool + matching atom indices +- [x] Tests: 3D conformer has coordinates, substructure match on known patterns + +## 1D. Chemistry Service + Tools +- [x] Wire `ChemistryService` to `RDKitAdapter` via dependency injection +- [x] Implement `generate_3d` tool -- SMILES -> JSON with MolBlock + energy +- [x] Implement `substructure_match` tool -- SMILES + pattern -> JSON with match + atoms +- [x] Added `validate_smiles`, `compute_descriptors`, `compute_fingerprint`, `tanimoto_similarity` tools +- [x] Tests: service integration tests (9), tool JSON output validation (11) + +**Verification:** `uv run pytest tests/chemistry/ -v` -- 44 passed, mypy clean. diff --git a/docs/roadmap/phase-02-literature-analysis.md b/docs/roadmap/phase-02-literature-analysis.md new file mode 100644 index 0000000..5d0019c --- /dev/null +++ b/docs/roadmap/phase-02-literature-analysis.md @@ -0,0 +1,46 @@ +Back to [Roadmap Index](README.md) + +# Phase 2: Literature + Analysis (Feb 10) -- DONE + +Two independent contexts. Can be built in parallel. + +## 2A. Semantic Scholar Client +- [x] HTTP client with `httpx` -- search endpoint (`/graph/v1/paper/search`) +- [x] Parse response: title, authors, year, abstract, DOI, citationCount +- [x] Rate limiting: respect 100 req/sec unauthenticated limit +- [x] Error handling: timeout, 429 rate limit, malformed response +- [x] Tests: Paper entity construction from API response (3 tests) + +## 2B. Core Reference Set +- [x] Load `data/references/core_references.json` into `CoreReferenceSet` +- [x] Lookup by key: halicin, abaucin, who_bppl_2024, chemprop, pkcsm, amr_crisis +- [x] Expanded JSON with full metadata (key findings, training size, hit rates) +- [x] Tests: load fixture, find by key, all 6 keys verified + +## 2C. Literature Service + Tools +- [x] `search_papers(query, limit)` -- Semantic Scholar +- [x] `get_reference(key)` -- core reference lookup + DOI fallback +- [x] `format_citation(paper)` -- APA-style string +- [x] Implement `search_literature` tool -- query -> JSON with papers +- [x] Implement `get_reference` tool -- key -> JSON with full citation +- [x] Tests: service with mock repos (8), tool JSON output (3) + +## 2D. ChEMBL Loader +- [x] HTTP client for ChEMBL REST API (direct httpx) +- [x] Filter by: target_organism, standard_type=MIC/IC50, standard_relation="=" +- [x] Deduplicate: one entry per SMILES (median activity if duplicates) +- [x] Compute pActivity: -log10(standard_value * 1e-6) +- [x] Parquet caching for downloaded datasets +- [x] Build `Dataset` entity: smiles_list, activities, metadata +- [ ] Tests: mock API (deferred -- uses mock service in tools tests) + +## 2E. Analysis Service + Tools +- [x] `explore(target, threshold)` -- load dataset, return stats +- [x] `analyze_substructures(dataset)` -- chi-squared enrichment on 10 known substructures +- [x] `compute_properties(dataset)` -- descriptor distributions (active vs inactive) +- [x] Implement `explore_dataset` tool -- target -> JSON with stats +- [x] Implement `analyze_substructures` tool -- target -> JSON with enriched substructures +- [x] Implement `compute_properties` tool -- target -> JSON with property distributions +- [x] Tests: mock datasets (6), enrichment logic, property computation, tool output (3) + +**Verification:** `uv run pytest tests/literature/ tests/analysis/ -v` -- 23 passed. diff --git a/docs/roadmap/phase-03-prediction.md b/docs/roadmap/phase-03-prediction.md new file mode 100644 index 0000000..bd2c97a --- /dev/null +++ b/docs/roadmap/phase-03-prediction.md @@ -0,0 +1,40 @@ +Back to [Roadmap Index](README.md) + +# Phase 3: Prediction Context (Feb 10) -- DONE + +ML models for activity/outcome prediction (domain-agnostic scoring). + +## 3A. XGBoost Adapter +- [x] Train: Morgan fingerprints (2048-bit) + activities -> XGBoost classifier +- [x] Scaffold split: train/val/test by Murcko scaffold (prevents data leakage) +- [x] Metrics: AUROC, AUPRC, accuracy, F1, confusion matrix +- [x] Feature importance: top fingerprint bits -> interpret as substructures +- [x] Predict: fingerprints -> probabilities +- [x] Tests: small fixture dataset, verify model trains and predicts (5 tests) + +## 3B. Model Store +- [x] Save: `TrainedModel` metadata + joblib artifact -> disk +- [x] Load: model_id -> `TrainedModel` + artifact +- [x] List: all saved models with metrics +- [x] Tests: save/load roundtrip (4 tests) + +## 3C. Prediction Service -- Train & Predict +- [x] `train(target, model_type)` -- load dataset, compute fingerprints, scaffold split, train, save +- [x] `predict(smiles_list, model_id)` -- load model, compute fingerprints, predict, rank +- [x] `cluster(smiles_list, n_clusters)` -- Butina clustering on Tanimoto distances +- [x] Tests: full train -> predict pipeline with fixture data (8 tests) + +## 3D. Chemprop Adapter (Optional Extra) +- [x] Train: SMILES + activities -> Chemprop D-MPNN (requires `deeplearning` extra) +- [x] Predict: SMILES -> probabilities +- [x] Guard: graceful skip if chemprop not installed +- [x] Tests: mock or skip if extra not available + +## 3E. Ensemble & Tools +- [x] `ensemble(smiles_list)` -- average XGBoost + Chemprop (0.5 each), fallback to single +- [x] Implement `train_model` tool -- target + model_type -> JSON with metrics +- [x] Implement `predict_candidates` tool -- smiles_list + model_id -> JSON with ranked predictions +- [x] Implement `cluster_compounds` tool -- smiles_list -> JSON with clusters +- [x] Tests: ensemble logic, tool JSON output (3 tests) + +**Verification:** `uv run pytest tests/prediction/ -v` -- 20 passed, mypy clean. diff --git a/docs/roadmap/phase-04-simulation.md b/docs/roadmap/phase-04-simulation.md new file mode 100644 index 0000000..28c4a89 --- /dev/null +++ b/docs/roadmap/phase-04-simulation.md @@ -0,0 +1,43 @@ +Back to [Roadmap Index](README.md) + +# Phase 4: Simulation Context (Feb 10) -- DONE + +Molecular docking, ADMET prediction, resistance assessment. + +## 4A. Protein Store +- [x] Manage PDBQT files in `data/proteins/` +- [x] Target registry: PDB ID -> name, organism, center coordinates, box size +- [x] Pre-configure 5 MRSA targets: PBP2a (1VQQ), DHPS (1AD4), DNA Gyrase (2XCT), MurA (1UAE), NDM-1 (3SPU) +- [x] `get_pdbqt(pdb_id)` -- return file path or download +- [x] Tests: target lookup, file existence check (6 tests) + +## 4B. Vina Adapter (Optional Extra) +- [x] Dock: SMILES -> conformer (RDKit) -> Meeko prep -> Vina dock -> energy + pose +- [x] Parse results: binding energy (kcal/mol), best pose PDBQT, RMSD +- [x] Interpret energy: excellent (<= -10), strong (-8 to -10), moderate (-6 to -8), weak (> -6) +- [x] Guard: graceful skip if vina/meeko not installed +- [x] Tests: energy interpretation (6 tests in protein_store) + +## 4C. ADMET Client +- [x] RDKit-based ADMET: Lipinski violations, mutagenic alerts (SMARTS), hepatotoxicity, hERG, BBB +- [x] Build `ADMETProfile` with absorption, distribution_vd, metabolism, excretion, toxicity +- [x] Toxicity flags: Ames (nitro/azide/quinone alerts), hERG (LogP + MW), hepatotoxicity (LogP + MW + acyl chloride/thioester) +- [x] Tests: aspirin profile, ethanol BBB, nitroaromatic mutagenic, hepatotoxic compound (5 tests) + +## 4D. Resistance Assessment +- [x] Knowledge-based mutation risk with compound class pattern matching +- [x] Known mutations: PBP2a S403A/N146K, DHPS F17L, DNA Gyrase S84L, MurA C115D, NDM-1 V73_ins/M154L +- [x] Compound class patterns: beta-lactam ring, fluoroquinolone, sulfonamide -> affected targets +- [x] Build `ResistanceAssessment` with per-mutation `MutationRisk` and overall risk level +- [x] Tests: known target mutations, DNA gyrase, mutations dict (3 tests in simulation_service) + +## 4E. Simulation Service + Tools +- [x] `dock(smiles, target_id)` -- Vina with RDKit descriptor-based estimate fallback +- [x] `predict_admet(smiles)` -- RDKit-based via PkCSMClient +- [x] `assess_resistance(smiles, target_id)` -- knowledge-based with compound class risk +- [x] Implement `dock_against_target` tool -- smiles + target -> JSON with energy + interpretation +- [x] Implement `predict_admet` tool -- smiles -> JSON with ADMET profile + toxicity flags +- [x] Implement `assess_resistance` tool -- smiles + target -> JSON with mutation details +- [x] Tests: service integration (6), tool JSON output (3) + +**Verification:** `uv run pytest tests/simulation/ -v` -- 20 passed, mypy clean. diff --git a/docs/roadmap/phase-05-investigation-agent.md b/docs/roadmap/phase-05-investigation-agent.md new file mode 100644 index 0000000..bb915c5 --- /dev/null +++ b/docs/roadmap/phase-05-investigation-agent.md @@ -0,0 +1,64 @@ +Back to [Roadmap Index](README.md) + +# Phase 5: Investigation Agent (Feb 10) -- DONE + +The core: Claude as an autonomous scientist. + +## 5A. Anthropic Client Adapter +- [x] Wrap `anthropic.AsyncAnthropic().messages.create()` -- isolate SDK dependency +- [x] Handle: system prompt, messages array, tools list, max_tokens +- [x] Parse response: content blocks (text, tool_use), stop_reason, usage +- [x] Error handling: rate limits, API errors, timeout +- [x] Tests: mock API, verify request/response handling + +## 5B. Tool Registry +- [x] Register all tools from all contexts (6 chemistry, 3 literature, 12 analysis, 3 prediction, 7 simulation, 11 training, 10 nutrition, 7 investigation control -- 38 at time of Phase 5; now 79 with API tools + visualization tools + statistics tools + generic ML tools + impact tools + causal tools) +- [x] Auto-generate JSON Schema from Python type hints + docstrings +- [x] `get(name)` -> callable, `list_tools()` -> all registered tools, `list_schemas()` -> Anthropic-compatible schemas +- [x] Schema format matches Anthropic tool_use specification +- [x] Tests: register tool, verify schema generation, lookup, list params, defaults (8 tests) + +## 5C. Cost Tracker +- [x] Track per-run: input_tokens, output_tokens, tool_calls count +- [x] Compute cost: Sonnet 4.5 pricing ($3/M input, $15/M output) +- [x] Running totals across iterations, `to_dict()` for serialization +- [x] Tests: add_usage, verify total_cost calculation (6 tests) + +## 5D. System Prompt +- [x] Scientist persona: Paul Ehrlich, methodology, phases, rules +- [x] 7 research phases with specific tool guidance per phase +- [x] Constraints: minimum 3 tools per phase, always cite references, record findings +- [x] Output format: structured findings + ranked candidates + citations + +## 5E. Orchestrator -- Agentic Loop +- [x] Create `Investigation` entity, set status to RUNNING +- [x] Build messages array: system prompt + user research question +- [x] Loop: call Claude -> check stop_reason -> dispatch tool_use -> collect results -> repeat +- [x] Max iteration guard (configurable, default 50) +- [x] Emit domain events: HypothesisFormulated, ExperimentStarted, ExperimentCompleted, HypothesisEvaluated, NegativeControlRecorded, ToolCalled, ToolResultEvent, FindingRecorded, Thinking, InvestigationCompleted, InvestigationError +- [x] Handle: parallel tool calls (Claude can request multiple), tool errors (graceful JSON error return) +- [x] End condition: stop_reason == "end_turn", conclude_investigation called, or max iterations +- [x] Phase auto-detection from tool names +- [x] Tests: mock Claude responses with tool_use, verify dispatch + event emission (9 tests) + +## 5F. SSE Streaming +- [x] Convert domain events to SSE events via `domain_event_to_sse()` mapper +- [x] Wire orchestrator async generator to `sse-starlette` EventSourceResponse +- [x] 20 event types (see README for full list): hypothesis lifecycle, experiment lifecycle, tool calls, findings, thinking, controls, validation, phases, cost, domain detection, literature survey, visualization, completion, error +- [x] Include cost tracker data in completed event +- [x] Tests: event type conversions + JSON format + +## 5G. Investigation API Routes +- [x] `POST /api/v1/investigate` -- accept prompt, create investigation, return ID +- [x] `GET /api/v1/investigate/{id}/stream` -- SSE stream of orchestrator events +- [x] Request/response DTOs with Pydantic models (InvestigateRequest, InvestigateResponse) +- [x] Error handling: 404 invalid ID, 409 investigation already running +- [x] Tests: FastAPI TestClient (4 tests) + +## 5H. Control Tools +- [x] `record_finding(title, detail, hypothesis_id, evidence_type)` -- orchestrator intercepts and stores in Investigation entity +- [x] `conclude_investigation(summary, candidates, citations)` -- orchestrator intercepts, sets candidates/citations, ends loop +- [x] `propose_hypothesis`, `design_experiment`, `evaluate_hypothesis`, `record_negative_control` -- hypothesis control tools +- [x] Tests: verify finding storage, conclusion structure, hypothesis tools (10 tests) + +**Verification:** `uv run pytest tests/investigation/ tests/api/ -v` -- 41 passed. Full suite 153 passed, 82% coverage, ruff 0, mypy 0. diff --git a/docs/roadmap/phase-06-console.md b/docs/roadmap/phase-06-console.md new file mode 100644 index 0000000..5e4ba38 --- /dev/null +++ b/docs/roadmap/phase-06-console.md @@ -0,0 +1,35 @@ +Back to [Roadmap Index](README.md) + +# Phase 6: Console Integration (Feb 10) -- DONE + +Visualization and real-time streaming UI. + +## 6A. ~~RDKit.js WASM Integration~~ (Replaced by server-side SVG) +- [x] Replaced with server-side RDKit `rdMolDraw2D` SVG depiction (higher quality, simpler) +- [x] Deleted `useRDKit` hook (dead code) + +## 6B. Molecule Viewers +- [x] `MolViewer2D`: server-side SVG via ``, lazy loading, error fallback +- [x] `MolViewer3D`: 3Dmol.js WebGL viewer with stick style, Jmol coloring, dynamic import +- [x] `DockingViewer`: 3Dmol.js protein cartoon (spectrum) + ligand stick (green carbon) overlay +- [x] `PropertyCard`: display descriptors with generic property grid + +## 6C. Investigation Flow +- [x] Wire `PromptInput` to `useInvestigation` mutation (POST /investigate) +- [x] Navigate to `/investigation/$id` on success +- [x] `useSSE` hook: named event listeners for all 7 SSE event types, parsed state (currentPhase, findings, summary, cost, error) +- [x] `Timeline`: rich rendering per event type with phase icons, tool call previews, finding highlights +- [x] `FindingsPanel`: sidebar panel showing accumulated findings with phase badges +- [x] `CandidateTable`: populate from conclusion event data +- [x] `ReportViewer`: render markdown summary from conclude event +- [x] `CostBadge`: live token/cost from SSE completed event + +## 6D. Polish +- [x] Loading states: spinner on connecting, "Starting..." on submit +- [x] Error states: connection lost, API errors, inline error messages +- [x] Responsive layout: 3-column grid on desktop (timeline + report | findings + candidates) +- [x] Status indicator: connecting / running (animated) / completed / error +- [x] Auto-scroll timeline to latest event +- [x] Tests: component rendering with mock data (14 tests across 4 files) + +**Verification:** `npx vitest run` -- 19 passed. `bun run build` + `tsc --noEmit` -- zero errors. diff --git a/docs/roadmap/phase-07-integration-demo.md b/docs/roadmap/phase-07-integration-demo.md new file mode 100644 index 0000000..e84cf52 --- /dev/null +++ b/docs/roadmap/phase-07-integration-demo.md @@ -0,0 +1,24 @@ +Back to [Roadmap Index](README.md) + +# Phase 7: Integration + Demo (Feb 10) -- DONE + +## 7A. End-to-End Validation +- [x] E2E smoke test exercising full pipeline (tool registry, orchestrator dispatch, SSE events) +- [x] API key wired from Settings to AnthropicClientAdapter +- [x] Exponential backoff retry (3 attempts) for rate-limit and timeout errors + +## 7B. Data Preparation +- [x] Data preparation script (`data/scripts/prepare_data.py`) for ChEMBL + PDB downloads +- [x] Parquet caching in `data/datasets/` + +## 7C. Error Handling Sweep +- [x] Graceful degradation: skip docking if vina not installed +- [x] Graceful degradation: skip Chemprop if torch not installed +- [x] Improved error handling in explore_dataset, train_model, predict_candidates, dock_against_target, assess_resistance +- [x] Agent loop: handle tool errors without crashing investigation + +## 7D. Documentation +- [x] README: environment variables, demo instructions, API endpoints +- [x] Architecture and roadmap docs + +**Verification:** 160 tests, 81.67% coverage, all quality gates green. diff --git a/docs/roadmap/phase-08-multi-model.md b/docs/roadmap/phase-08-multi-model.md new file mode 100644 index 0000000..bc21a6e --- /dev/null +++ b/docs/roadmap/phase-08-multi-model.md @@ -0,0 +1,59 @@ +Back to [Roadmap Index](README.md) + +# Phase 8: Multi-Model Architecture + Polish (Feb 10) -- DONE + +Cost-efficient multi-model orchestration, persistence, and UI polish. + +## 8A. Domain Foundation + Config +- [x] Domain events: `OutputSummarized` (kept), hypothesis/experiment events (added in Phase 10A) +- [x] `created_at` and `cost_data` fields on Investigation entity +- [x] Per-model config settings: director, researcher, summarizer models +- [x] `summarizer_threshold`, `max_iterations_per_experiment`, `db_path` settings + +## 8B. Multi-Model Cost Tracker +- [x] Per-model usage tracking with model-specific pricing +- [x] Pricing dict: Opus $15/$75, Sonnet $3/$15, Haiku $0.80/$4 per M tokens +- [x] `to_dict()` with `by_model` breakdown (director/researcher/summarizer tiers) +- [x] `AnthropicClientAdapter.model` property for cost attribution + +## 8C. Multi-Model Orchestrator +- [x] `MultiModelOrchestrator` with hypothesis-driven Director-Worker-Summarizer pattern +- [x] Director (Opus) formulates hypotheses, designs experiments, evaluates evidence, synthesizes -- NO tool access +- [x] Researcher (Sonnet) executes experiments with 73 domain-filtered tools (max 10 iterations per experiment) +- [x] Summarizer (Haiku) compresses large outputs exceeding threshold +- [x] 7 prompts: director formulation/experiment/evaluation/synthesis, researcher experiment, scientist, summarizer +- [x] Auto-fallback to single-model Orchestrator when researcher == director +- [x] Tests: 12 tests across 8 classes (formulation, execution, evaluation, negative controls, compression, synthesis, full flow, errors) + +## 8D. SQLite Persistence +- [x] `InvestigationRepository` ABC in domain layer +- [x] `SqliteInvestigationRepository` with WAL mode via `aiosqlite` +- [x] Single `investigations` table with JSON serialization for complex fields +- [x] Tests: 8 tests (save, retrieve, list, update, cost data) + +## 8E. API Wiring +- [x] `GET /investigate` -- list all investigations (most recent first) +- [x] `GET /investigate/{id}` -- full investigation detail +- [x] SQLite repository initialization in app lifespan +- [x] Automatic orchestrator selection (multi-model vs single-model) +- [x] SSE event types: `hypothesis_formulated`, `experiment_started`, `experiment_completed`, `hypothesis_evaluated`, `negative_control`, `output_summarized` +- [x] `completed` event includes candidates and cost data + +## 8F. SSE Reconnection +- [x] Exponential backoff (1s, 2s, 4s) with max 3 retries +- [x] `reconnecting` state with amber WiFi-off indicator +- [x] Reset attempt counter on successful reconnect +- [x] Track `experiments`, `hypotheses`, `negativeControls` and `toolCallCount` from events + +## 8G. Investigation History +- [x] `useInvestigations` hook with TanStack Query (10s refetch) +- [x] `InvestigationList` component with status badges and candidate counts +- [x] History section on home page below prompt input + +## 8H. UI Feedback +- [x] `HypothesisBoard` kanban-style grid with status columns and confidence bars +- [x] Hypothesis/experiment event rendering in Timeline +- [x] Candidates wired to `CandidateTable` from SSE completed event +- [x] `StatusIndicator` handles reconnecting state + +**Verification:** 185 tests, 82.28% coverage, all quality gates green (ruff, mypy, tsc, vitest). diff --git a/docs/roadmap/phase-09-molecule-visualization.md b/docs/roadmap/phase-09-molecule-visualization.md new file mode 100644 index 0000000..8555758 --- /dev/null +++ b/docs/roadmap/phase-09-molecule-visualization.md @@ -0,0 +1,41 @@ +Back to [Roadmap Index](README.md) + +# Phase 9: Molecule Visualization Suite (Feb 10) -- DONE + +Full visualization: server-side 2D SVG depiction, 3Dmol.js for 3D/docking views, expandable candidate detail panel. + +## 9A. Server-Side 2D Depiction +- [x] `RDKitAdapter.depict_2d(smiles, width, height)` -- RDKit `rdMolDraw2D.MolDraw2DSVG` +- [x] `ChemistryService.depict_2d()` thin wrapper +- [x] Tests: returns SVG, custom dimensions, invalid SMILES raises (3 tests) + +## 9B. Molecule API Routes +- [x] `GET /molecule/depict?smiles=&w=&h=` -- SVG response, 24h cache, error SVG for invalid SMILES +- [x] `GET /molecule/conformer?smiles=` -- JSON with mol_block, energy, num_atoms +- [x] `GET /molecule/descriptors?smiles=` -- JSON descriptors + passes_lipinski +- [x] `GET /targets` -- JSON list of protein targets (5 pre-configured) +- [x] Router registered in app.py +- [x] Tests: 10 API tests (depict, conformer, descriptors, targets) + +## 9C. 3Dmol.js Integration +- [x] Added `3dmol` package to console dependencies +- [x] TypeScript type stubs (`console/src/types/3dmol.d.ts`) +- [x] Dynamic import for code splitting (~575KB separate chunk) + +## 9D. Molecule Viewer Components +- [x] `MolViewer2D`: `` tag with server-side SVG, lazy loading, error fallback to SMILES text +- [x] `MolViewer3D`: 3Dmol.js WebGL viewer, stick style, Jmol coloring, cleanup on unmount +- [x] `DockingViewer`: 3Dmol.js protein cartoon (spectrum) + ligand stick (green carbon), zoom to ligand +- [x] Tests: 4 MolViewer2D component tests (src, encoding, dimensions, error fallback) + +## 9E. CandidateDetail + CandidateTable +- [x] `CandidateDetail` panel: parallel fetch of conformer + descriptors, 3-column grid (2D | 3D + energy | properties + Lipinski badge) +- [x] `CandidateTable` rewrite: 2D thumbnails (80x60), chevron expand/collapse, click-to-expand detail panel +- [x] Removed raw SMILES column (SMILES in img alt text) +- [x] Updated tests for new table structure + +## 9F. Cleanup +- [x] Deleted `useRDKit` hook (dead code) +- [x] Added `CandidateDetail` to barrel export (`index.ts`) + +**Verification:** 198 tests, 82.09% coverage, 19 console tests. All quality gates green (ruff 0, mypy 0, tsc 0, vitest 19/19). diff --git a/docs/roadmap/phase-10a-hypothesis-engine.md b/docs/roadmap/phase-10a-hypothesis-engine.md new file mode 100644 index 0000000..5cd5123 --- /dev/null +++ b/docs/roadmap/phase-10a-hypothesis-engine.md @@ -0,0 +1,59 @@ +Back to [Roadmap Index](README.md) + +# Phase 10A: Hypothesis-Driven Investigation Engine (Feb 10) -- DONE + +Replaced the linear 7-phase pipeline with a hypothesis-driven scientific method loop. + +## Architecture Change +- **Old:** Literature -> Data -> Model -> Screen -> Structure -> Resistance -> Conclude (linear recipe) +- **New:** Literature Survey -> Formulate Hypotheses -> For each: Design Experiment -> Execute -> Evaluate -> Negative Controls -> Synthesize + +## Domain Layer +- [x] `Hypothesis` entity: statement, rationale, status (proposed/testing/supported/refuted/revised), confidence, evidence lists +- [x] `Experiment` entity: hypothesis_id, description, tool_plan, status, result_summary +- [x] `NegativeControl` frozen dataclass: identifier, identifier_type, score, threshold, correctly_classified property (generalized in Domain-Agnostic phase) +- [x] `Finding` modified: `hypothesis_id` + `evidence_type` replace `phase` +- [x] `Investigation` modified: hypotheses, experiments, negative_controls replace phases + +## Events (12 total, was 11) +- [x] Removed: `PhaseStarted`, `PhaseCompleted`, `DirectorPlanning`, `DirectorDecision` +- [x] Added: `HypothesisFormulated`, `ExperimentStarted`, `ExperimentCompleted`, `HypothesisEvaluated`, `NegativeControlRecorded` +- [x] Modified: `FindingRecorded` (hypothesis_id + evidence_type), `InvestigationCompleted` (hypotheses + negative_controls) + +## Tools (23 total, was 19) +- [x] 4 new control tools: `propose_hypothesis`, `design_experiment`, `evaluate_hypothesis`, `record_negative_control` +- [x] Modified: `record_finding` (hypothesis_id + evidence_type), `conclude_investigation` (hypothesis assessments) + +## Orchestrators +- [x] Single-model `Orchestrator`: hypothesis control tool dispatch with entity creation +- [x] `MultiModelOrchestrator`: hypothesis-driven loop (formulate -> design -> execute -> evaluate per hypothesis) +- [x] 7 hypothesis-driven prompts replacing 6 phase-based prompts + +## Console +- [x] `HypothesisBoard` + `HypothesisCard`: kanban-style status grid with confidence bars +- [x] `ActiveExperimentCard`: replaces `ActivePhaseCard` with experiment context +- [x] `NegativeControlPanel`: validation table with pass/fail indicators +- [x] `FindingsPanel`: evidence type badges + hypothesis grouping +- [x] `Timeline`: hypothesis/experiment/evaluation event rendering +- [x] Deleted: `PhaseProgress.tsx`, `ActivePhaseCard.tsx` + +## Persistence +- [x] SQLite schema: hypotheses, experiments, negative_controls columns (JSON serialized) +- [x] Removed: phases, current_phase columns + +**Verification:** 212 tests passed, mypy 0 errors, ruff 0 violations, tsc 0 errors, bun build success. + +--- + +## Scientific Methodology Upgrade (Cross-Cutting) + +Grounding every phase of the investigation workflow in established scientific methodology. Each phase gets the treatment that Phase 1 (Hypothesis) received: deep research, universal components, entity upgrades, prompt updates. See [`docs/scientific-methodology.md`](../scientific-methodology.md) for full details. + +| # | Phase | Status | +|---|-------|--------| +| 1 | Hypothesis Formulation (Popper, Platt, Feynman, Bayesian) | DONE | +| 2 | Literature Survey (PICO, citation chasing, GRADE, AMSTAR-2) | DONE | +| 3 | Experiment Design (Fisher, controls, sensitivity, AD) | DONE | +| 4 | Evidence Evaluation (evidence hierarchy, GRADE, effect sizes) | DONE | +| 5 | Negative Controls (Z'-factor, permutation significance, scaffold-split) | DONE | +| 6 | Synthesis (GRADE certainty, priority tiers, knowledge gaps) | DONE | diff --git a/docs/roadmap/phase-10b-domain-agnostic.md b/docs/roadmap/phase-10b-domain-agnostic.md new file mode 100644 index 0000000..f9add81 --- /dev/null +++ b/docs/roadmap/phase-10b-domain-agnostic.md @@ -0,0 +1,144 @@ +Back to [Roadmap Index](README.md) + +# Phase 10B: Domain-Agnostic Generalization (Feb 11-12) -- DONE + +Generalized the entire engine from molecular-science-specific to domain-agnostic. Proved the architecture works by adding Training Science and Nutrition Science as additional domains. + +## Backend Entity Generalization +- [x] `Candidate` entity: generic `identifier`/`identifier_type`/`scores`/`attributes` (replaces `smiles`/`prediction_score`/`docking_score`/`admet_score`/`resistance_risk`) +- [x] `NegativeControl` entity: generic `identifier`/`identifier_type`/`score`/`threshold` (replaces `smiles`/`prediction_score`) +- [x] `NegativeControlRecorded` event: generic fields +- [x] SQLite serialization updated for new field names +- [x] All 274 existing tests pass after entity changes (274 at time of this phase) + +## Domain Configuration + Tool Tagging +- [x] `DomainConfig` + `ScoreDefinition` frozen dataclasses +- [x] `DomainRegistry`: register, detect by classified category, list template prompts +- [x] `MOLECULAR_SCIENCE` config (6 tool tags, 3 scores, 4 templates, multishot examples) +- [x] `TRAINING_SCIENCE` config (3 tool tags, 3 scores, 2 templates, multishot examples) +- [x] `NUTRITION_SCIENCE` config (3 tool tags, 3 scores, 1 template, multishot examples) +- [x] `ToolRegistry` with domain tag filtering (`list_schemas_for_domain`, `list_tools_for_domain`) +- [x] Tools tagged: chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, literature; investigation control universal +- [x] `DomainDetected` SSE event (16th event type) sends display config to frontend + +## Prompt Template Generalization +- [x] Builder functions: `build_scientist_prompt()`, `build_formulation_prompt()`, `build_experiment_prompt()`, `build_synthesis_prompt()` +- [x] Dynamic domain classification from all registered domain categories +- [x] HypothesisType enum expanded: PHYSIOLOGICAL, PERFORMANCE, EPIDEMIOLOGICAL + +## Frontend Generalization +- [x] Dynamic score columns from `DomainDisplayConfig` (replaces hardcoded Pred/Dock/ADMET/Resist) +- [x] Conditional MolViewer2D: only when `identifier_type === "smiles"` +- [x] Unified reactive VisualizationPanel: LiveLabViewer auto-appears for molecular tool events, charts render inline +- [x] 7 template cards (4 molecular + 2 training + 1 nutrition) with domain badges +- [x] Generic candidate comparison, negative control panel, markdown export + +## Training Science Bounded Context (6 tools) +- [x] `search_training_literature` -- Semantic Scholar with training science context +- [x] `analyze_training_evidence` -- effect sizes, heterogeneity, evidence grading (A-D) +- [x] `compare_protocols` -- evidence-weighted composite scoring +- [x] `assess_injury_risk` -- knowledge-based multi-factor risk scoring +- [x] `compute_training_metrics` -- ACWR, monotony, strain, RPE load +- [x] `search_clinical_trials` -- ClinicalTrials.gov exercise/training RCT search + +## Nutrition Science Bounded Context (10 tools) +- [x] `search_supplement_evidence` -- supplement meta-analysis search +- [x] `search_supplement_labels` -- NIH DSLD supplement product ingredient lookup +- [x] `search_nutrient_data` -- USDA FoodData Central nutrient profiles +- [x] `search_supplement_safety` -- OpenFDA CAERS adverse event reports for supplements +- [x] `compare_nutrients` -- side-by-side nutrient comparison (added in Nutrition Enhancement) +- [x] `assess_nutrient_adequacy` -- DRI-based adequacy assessment (added in Nutrition Enhancement) +- [x] `check_intake_safety` -- UL safety screening (added in Nutrition Enhancement) +- [x] `check_interactions` -- RxNav drug interactions (added in Nutrition Enhancement) +- [x] `analyze_nutrient_ratios` -- nutrient ratio analysis (added in Nutrition Enhancement) +- [x] `compute_inflammatory_index` -- DII scoring (added in Nutrition Enhancement) +- [x] Tests: training (8 tests) + nutrition (10 tests initially, expanded in Enhancement), all passing + +**Verification:** 288 server tests, 107 console tests. All quality gates green: ruff 0, mypy 0 (117 files), tsc 0, vitest 107/107. + +--- + +# Multi-Domain Investigations (Feb 12) -- DONE + +Cross-domain research questions that span multiple scientific domains (e.g., molecular + nutrition, training + nutrition). + +## Changes + +- [x] `DomainRegistry.detect()` returns `list[DomainConfig]` (multi-domain detection, deduplicates) +- [x] Haiku classifier outputs JSON array of domain categories +- [x] `merge_domain_configs()` creates synthetic merged config (union tool_tags, concatenated scores, joined examples) +- [x] Tool filtering uses merged config's union of domain tags +- [x] Prompt examples merged from all relevant domains +- [x] `DomainDetected` SSE event carries merged display config with `domains` sub-list +- [x] Frontend `DomainDisplayConfig.domains` for sub-domain rendering +- [x] Tests: 16 tests (multi-detect, deduplication, fallback, merge, display dict) + +--- + +# Self-Referential Research (Feb 12) -- DONE + +Ehrlich builds institutional knowledge by querying its own past investigations during new research. + +## Changes + +- [x] SQLite FTS5 virtual table (`findings_fts`) on finding title, detail, evidence_type, hypothesis statement/status, source provenance +- [x] `search_prior_research` tool -- intercepted in orchestrator, routed to FTS5 via repository +- [x] BM25-ranked full-text search with investigation prompt context +- [x] FTS5 query sanitization (quoted tokens for literal hyphens/operators) +- [x] FTS5 index rebuilt on investigation completion via `_rebuild_fts()` +- [x] Provenance: `source_type: "ehrlich"`, `source_id: "{investigation_id}"` +- [x] Available during Phase 2 (Literature Survey) alongside external search tools +- [x] Frontend: Ehrlich-branded source badges with internal navigation +- [x] Tests: 6 FTS5 tests (empty, match, prompt context, limit, completion-gating, cross-investigation) + +--- + +# Shared Context + MCP Bridge + Training/Nutrition APIs (Feb 12) -- DONE + +DDD cleanup, shared bounded context, MCP bridge for external tools, 4 new data source clients (training + nutrition), and methodology page. + +## Shared Bounded Context +- [x] `shared/` context: cross-cutting ports and value objects (`ChemistryPort` ABC, `Fingerprint`, `MolecularDescriptors`, `Conformer3D`) +- [x] `ChemistryPort` ABC decouples chemistry operations from RDKit infrastructure +- [x] Moved value objects from `chemistry/domain/` to `shared/` for cross-context use + +## MCP Bridge +- [x] `MCPBridge` connects to external MCP servers (stdio/SSE/streamable_http transports) +- [x] Tools registered dynamically via `ToolRegistry.register_mcp_tools()` with domain tags +- [x] Lifecycle managed by orchestrator (connect on start, disconnect on completion) +- [x] Enabled via `EHRLICH_MCP_EXCALIDRAW=true` env var + +## Data Source API Tools (4 tools across training + nutrition) +- [x] `search_clinical_trials` (training) -- ClinicalTrials.gov v2 API for exercise/training RCTs +- [x] `search_supplement_labels` (nutrition) -- NIH DSLD supplement label database +- [x] `search_nutrient_data` (nutrition) -- USDA FoodData Central nutrient profiles +- [x] `search_supplement_safety` (nutrition) -- OpenFDA CAERS adverse event reports +- [x] Full DDD: domain entities + repository ABCs + infrastructure clients in respective bounded contexts + +## Methodology Page +- [x] `GET /methodology` endpoint: phases, domains, tools, data sources, models +- [x] `GET /stats` endpoint: aggregate counts +- [x] Console methodology page with phases, models, domains, tools, data sources + +**Verification:** 527 server tests, 107 console tests. All quality gates green. + +--- + +# Domain-Specific Visualization System (Feb 12) -- DONE + +6 visualization tools with structured payloads, orchestrator interception, and lazy-loaded React components. + +## Backend +- [x] `VisualizationPayload` frozen dataclass (viz_type, title, data, config, domain) +- [x] 6 viz tools: `render_binding_scatter`, `render_admet_radar`, `render_training_timeline`, `render_muscle_heatmap`, `render_forest_plot`, `render_evidence_matrix` +- [x] Orchestrator intercepts viz tool results via `_maybe_viz_event()`, emits `VisualizationRendered` SSE event (20th event type) + +## Frontend +- [x] `VizRegistry` maps viz_type to lazy-loaded React component +- [x] `VisualizationPanel` renders multiple charts in grid layout +- [x] Recharts: scatter (compound affinities), radar (ADMET), timeline (training load + ACWR) +- [x] Visx: forest plot (meta-analysis), evidence matrix (heatmap) +- [x] Custom SVG: anatomical body diagram with muscle activation/risk heatmap +- [x] OKLCH color tokens for consistent chart theming + +**Verification:** 527 server tests, 107 console tests. All quality gates green. diff --git a/docs/roadmap/phase-10c-sdk-optimization.md b/docs/roadmap/phase-10c-sdk-optimization.md new file mode 100644 index 0000000..82315d5 --- /dev/null +++ b/docs/roadmap/phase-10c-sdk-optimization.md @@ -0,0 +1,118 @@ +Back to [Roadmap Index](README.md) + +# Phase 10C: Claude SDK Optimization -- DONE + +Upgraded the Anthropic SDK integration to use all available features for better reasoning quality, lower costs, and improved UX. + +## SDK-1: Fix Pricing + Cache-Aware Cost Tracking -- DONE + +Fixed incorrect hardcoded pricing and added cache hit/miss token tracking for accurate cost reporting. + +- [x] Fix Opus pricing: `$15/$75` -> `$5/$25` per M tokens (Opus 4.5/4.6 pricing) +- [x] Fix Haiku pricing: `$0.80/$4.0` -> `$1/$5` per M tokens (Haiku 4.5 pricing) +- [x] Track `cache_creation_input_tokens` and `cache_read_input_tokens` from `response.usage` +- [x] Compute cache-aware cost: cache writes at 1.25x, cache reads at 0.1x base input rate +- [x] Add `cache_read_tokens` and `cache_write_tokens` to `CostTracker.to_dict()` +- [x] Update `CostUpdate` SSE event with cache breakdown +- [x] Update `MessageResponse` dataclass with cache token fields +- [x] Tests: cost calculation with cache hits, cache misses, mixed scenarios + +**Files:** `cost_tracker.py`, `anthropic_client.py`, `multi_orchestrator.py`, `events.py`, `sse.py`, `test_cost_tracker.py` + +## SDK-2: Prompt Caching on Tools Array -- DONE + +Cache the 73-tool schema array that repeats on every researcher API call. + +- [x] Add `cache_control: {"type": "ephemeral"}` to the last tool in the tools array before passing to `messages.create` +- [x] Only apply when tools list is non-empty +- [x] No changes to tool registry -- caching applied at the adapter level + +**Files:** `anthropic_client.py` + +## SDK-3: Effort Parameter -- DONE + +Use `effort` to control token spend. Only supported by Opus models (4.5+); Sonnet and Haiku do not support it. + +- [x] Add `effort: str | None = None` parameter to `AnthropicClientAdapter.__init__` +- [x] Pass `effort` via `output_config` dict to `messages.create()` (SDK 0.79+ requires it nested, not top-level) +- [x] Configure Director effort in `Settings`: `effort="high"` (default), gated on Opus model name +- [x] Add `EHRLICH_DIRECTOR_EFFORT` env var (only applied when Director model contains "opus") +- [x] Researcher (Sonnet) and Summarizer (Haiku) never receive effort -- unsupported by those models +- [x] Wire in API route when creating adapters + +**Files:** `config.py`, `anthropic_client.py`, `routes/investigation.py` + +## SDK-4: Extended Thinking for Director -- DONE + +Enabled extended thinking on Opus 4.6 Director for deeper scientific reasoning. + +- [x] Add `thinking: dict | None = None` parameter to `AnthropicClientAdapter.__init__` +- [x] Pass `thinking` to `messages.create()` when set +- [x] Default for Director: `{"type": "enabled", "budget_tokens": 10000}` +- [x] Parse `thinking` content blocks in `_parse_content_blocks()` -- extract `block.thinking` text +- [x] Emit thinking text via existing `Thinking` SSE event (already supported in frontend) +- [x] `max_tokens` increased to 32768 for Director to accommodate thinking budget +- [x] Add `EHRLICH_DIRECTOR_THINKING` env var (enabled/disabled) +- [x] Track thinking tokens separately in `MessageResponse` (thinking tokens billed as output) + +**Files:** `config.py`, `anthropic_client.py`, `multi_orchestrator.py`, `routes/investigation.py` + +## SDK-5: Structured Outputs for Director (Medium Impact) -- DONE + +Guarantee valid JSON from Director calls (hypothesis formulation, experiment design, evaluation, synthesis). + +- [x] Add `output_config: dict | None = None` parameter to `AnthropicClientAdapter.create_message()` and `stream_message()` +- [x] Pass `output_config` to `messages.create()` / `messages.stream()` when set +- [x] Define JSON schemas for Director outputs (6 schemas in `domain/schemas.py`): + - `PICO_SCHEMA` -- PICO decomposition for literature survey + - `FORMULATION_SCHEMA` -- array of hypothesis objects + - `EXPERIMENT_DESIGN_SCHEMA` -- experiment plan object + - `EVALUATION_SCHEMA` -- evaluation result object + - `SYNTHESIS_SCHEMA` -- synthesis result object + - `LITERATURE_GRADING_SCHEMA` -- evidence grading result +- [x] Update Director call sites in `MultiModelOrchestrator` to pass schemas via `_build_output_config()` +- [x] Remove `_parse_json()` fallback -- structured outputs guarantee valid JSON +- [x] Structured outputs work alongside streaming (`stream_message` with `output_config`) + +**Files:** `anthropic_client.py`, `multi_orchestrator.py`, `investigation/domain/schemas.py` (new) + +## SDK-6: tool_choice Control -- DONE + +Control how the researcher uses tools. + +- [x] Add `tool_choice: dict | None = None` parameter to `AnthropicClientAdapter.create_message()` +- [x] Pass `tool_choice` to `messages.create()` when set +- [x] Researcher first turn: `tool_choice={"type": "any"}` to force tool use (researcher should always start by calling tools) +- [x] Researcher subsequent turns: `tool_choice=None` (default, let model decide) +- [x] Literature survey first turn: `tool_choice={"type": "any"}` to force tool use + +**Files:** `anthropic_client.py`, `multi_orchestrator.py` + +## SDK-7: Streaming API (High Impact, UX) -- DONE (Director only) + +Replace non-streaming `messages.create()` with streaming for real-time Director token display. + +- [x] Add `stream_message()` async generator method to `AnthropicClientAdapter` +- [x] Use `client.messages.stream()` (high-level SDK helper) for async context manager +- [x] Yield intermediate events: `thinking` and `text` deltas, plus `result` with final `MessageResponse` +- [x] Accumulate final message via `stream.get_final_message()` for usage tracking +- [x] Update `MultiModelOrchestrator._director_call()` to use streaming -- yields `Thinking` events in real time +- [x] Emit real-time `Thinking` events as thinking tokens stream in (no buffering) +- [x] Keep non-streaming `create_message()` for Researcher and Summarizer (tool dispatch loop is simpler without streaming) +- [x] Retry with exponential backoff on `RateLimitError` / `APITimeoutError` (same as `create_message`) + +**Files:** `anthropic_client.py`, `multi_orchestrator.py` + +## Implementation Summary + +SDK-1 through SDK-7 all implemented. SDK-1 through SDK-4 and SDK-6 done in parallel by 3 agents (`sdk-cost`, `sdk-adapter`, `sdk-wiring`). SDK-5 (Structured Outputs) and SDK-7 (Director Streaming) wired in subsequently. + +## Verification + +**Completed:** 540 server tests passing, 107 console tests passing. All quality gates green. + +- `uv run pytest` -- 540 passed (13 new tests for SDK features) +- `uv run ruff check src/ tests/` -- zero violations +- `uv run mypy src/ehrlich/` -- zero errors (140 source files) +- `bun run build && bun run typecheck` -- zero errors +- `bunx vitest run` -- 107 passed diff --git a/docs/roadmap/phase-10d-landing-site.md b/docs/roadmap/phase-10d-landing-site.md new file mode 100644 index 0000000..905562c --- /dev/null +++ b/docs/roadmap/phase-10d-landing-site.md @@ -0,0 +1,121 @@ +Back to [Roadmap Index](README.md) + +# Phase 10D: Landing Site (`web/`) -- DONE + +Separate TanStack Start project for the public-facing landing page. SSR/SSG for SEO, same React + TypeScript + Bun stack as console, independent build and deployment. + +## Why Separate from Console + +- **Different concerns**: landing page is marketing/SEO content; console is the authenticated SPA +- **Different optimization**: SSR/SSG with zero-JS static pages vs client-side SPA with heavy WebGL/charting +- **Independent deployment**: CDN-friendly static output vs API-connected app server +- **Clean separation**: no marketing code in the investigation UI, no app dependencies in the landing page + +## Tech Stack + +| Layer | Choice | Rationale | +|-------|--------|-----------| +| Framework | TanStack Start | Same TanStack ecosystem as console, SSR/SSG built-in, Nitro server | +| Runtime | Bun | Same as console | +| Styling | Tailwind CSS 4 | Same as console, shared design tokens (OKLCH) | +| Fonts | Space Grotesk + JetBrains Mono | Same as console (Lab Protocol identity) | +| Icons | Lucide React | Same as console | +| Build | Vite 7 (via Nitro) | Same toolchain | + +## Project Structure + +``` +web/ +├── src/ +│ ├── routes/ +│ │ ├── __root.tsx # Root layout (shell, SEO meta, font loading) +│ │ └── index.tsx # Landing page (all sections wired) +│ ├── components/ +│ │ ├── Nav.tsx # Fixed navbar with scroll progress bar + mobile menu +│ │ ├── Footer.tsx # Minimal footer (links, license, year) +│ │ ├── SectionHeader.tsx # Mono label with left border accent +│ │ ├── Hero.tsx # Bottom-third hero with MolecularNetwork, stats badges, CTAs +│ │ ├── MolecularNetwork.tsx # 3D rotating node graph (Canvas 2D, mouse repulsion, CSS mask) +│ │ ├── HowItWorks.tsx # 6-phase methodology timeline with vertical line +│ │ ├── ConsolePreview.tsx # Browser-frame mockups (timeline, hypotheses, candidates) +│ │ ├── Architecture.tsx # Director-Worker-Summarizer model cards, dot grid bg +│ │ ├── Domains.tsx # 3 domain cards with tool counts + multi-domain callout +│ │ ├── Visualizations.tsx # 4 visualization category cards +│ │ ├── DataSources.tsx # 16 source cards with teal glow +│ │ ├── WhoItsFor.tsx # 3 persona cards (Student, Academic, Industry) +│ │ ├── Differentiators.tsx # 3 differentiator cards with capabilities +│ │ ├── OpenSource.tsx # Code snippet + licensing section +│ │ ├── Roadmap.tsx # Planned domains + platform features +│ │ └── CTA.tsx # Pricing tiers, terminal quickstart, primary glow +│ ├── styles/ +│ │ └── app.css # Tailwind 4 entry + OKLCH tokens + animations +│ ├── lib/ +│ │ ├── constants.ts # Stats, links, domain data, methodology phases +│ │ ├── use-reveal.ts # IntersectionObserver scroll reveal hook +│ │ └── use-scroll-progress.ts # Scroll progress fraction hook +│ ├── router.tsx # TanStack Router factory (getRouter) +│ └── routeTree.gen.ts # Auto-generated route tree +├── public/ +│ └── favicon.svg # Green "E" favicon +├── package.json # ehrlich-web, Bun scripts +├── vite.config.ts # TanStack Start + Nitro + Tailwind +└── tsconfig.json # src/ paths, strict mode +``` + +## WEB-1: Project Scaffolding -- DONE + +- [x] TanStack Start project in `web/` with Bun +- [x] `vite.config.ts` with tanstackStart + nitro + tailwindcss + viteReact + tsConfigPaths +- [x] Tailwind CSS 4 with OKLCH design tokens matching console's Lab Protocol identity +- [x] Space Grotesk + JetBrains Mono font loading via Google Fonts +- [x] Root layout with `shellComponent`, `HeadContent`, CSS `?url` import +- [x] Favicon + meta tags (title, description, OG, Twitter card, theme-color) +- [x] `bun run build` produces `.output/` (client + SSR + Nitro server) +- [x] `bun run typecheck` -- zero errors + +## WEB-2: Landing Page Sections -- DONE + +- [x] **Hero**: bottom-third placement, MolecularNetwork 3D canvas (CSS mask edge fade), stats badges, CTA buttons +- [x] **HowItWorks**: 6-phase methodology timeline with vertical connecting line, hypothesis/experiment structure callouts +- [x] **ConsolePreview**: browser-frame mockups (investigation timeline, hypothesis board, candidate table, ADMET radar) +- [x] **Architecture**: Director-Worker-Summarizer model cards with data pipes, dot grid bg, amber radial glow +- [x] **Domains**: 3 domain cards with tool counts, multi-domain callout, extensibility card +- [x] **Visualizations**: 4 visualization category cards with tech labels, extensibility callout +- [x] **DataSources**: 16 source cards (2-col grid), surface bg with teal radial glow, self-referential research callout +- [x] **WhoItsFor**: 3 persona cards (Student, Academic, Industry) +- [x] **Differentiators**: 3 differentiator cards with capabilities lists +- [x] **OpenSource**: code snippet + dual licensing (AGPL-3.0 + Commercial), GitHub link +- [x] **Roadmap**: planned domains (3 cards) + platform features (dashed border cards) +- [x] **CTA**: pricing tiers (4 cards), terminal quickstart with copy, primary radial glow + +## WEB-3: Navigation + Footer + Meta -- DONE + +- [x] **Nav**: fixed top bar, scroll progress indicator, desktop links, mobile hamburger menu +- [x] **Footer**: AGPL-3.0 branding, mapped footer links, responsive layout +- [x] **SectionHeader**: mono label with left border accent (reused across sections) +- [x] **SEO meta**: full OG tags, Twitter card, theme-color, description +- [x] **Smooth scroll**: anchor links (`#architecture`, `#methodology`, `#domains`, `#data-sources`) +- [x] Responsive: mobile hamburger nav, stacked sections + +## WEB-5: Visual Polish + Animations -- DONE + +- [x] Motion scroll-triggered reveals (whileInView, once: true) +- [x] Staggered children animation (80-120ms delay cascade) +- [x] Architecture data pipe animation (flowing dots between model tiers) +- [x] Domain/differentiator/persona card hover effects (border-primary, y-shift) +- [x] MolecularNetwork 3D canvas: 90 nodes, mouse repulsion, shimmer connections, CSS mask edge fade +- [x] Section zoning: alternating bg-surface/30 + strategic radial glows (amber/teal/primary) +- [x] CSS dot grid on Architecture (32px, static, graph-paper aesthetic) +- [x] Scroll progress bar in navbar +- [x] WCAG AA color contrast (muted-foreground 0.60, secondary 0.60, min /60 opacity) +- [x] Sequential heading hierarchy (h1 -> h2 -> h3, no skips) +- [x] Font preloading for Google Fonts (preload + stylesheet) +- [x] All OKLCH tokens (zero hardcoded gray-* classes) +- [x] Lighthouse: Performance 67, Accessibility 95, Best Practices 100, SEO 100 + +## Verification + +- [x] `cd web && bun run build` -- zero errors +- [x] `cd web && bun run typecheck` -- zero TypeScript errors +- [x] Visual: Lab Protocol identity consistent with console (OKLCH tokens, fonts, dark theme) +- [x] All docs updated: `CLAUDE.md`, `README.md`, `docs/architecture.md`, `docs/roadmap.md` diff --git a/docs/roadmap/phase-10e-training-enhancement.md b/docs/roadmap/phase-10e-training-enhancement.md new file mode 100644 index 0000000..6b7e899 --- /dev/null +++ b/docs/roadmap/phase-10e-training-enhancement.md @@ -0,0 +1,35 @@ +Back to [Roadmap Index](README.md) + +# Phase 10E: Training Science Enhancement (Feb 12) -- DONE + +Deepened the training bounded context with 2 new data sources, 4 new tools, and 3 new visualization tools. + +## New Data Sources +- [x] PubMed API (`https://eutils.ncbi.nlm.nih.gov/entrez/eutils/`) -- exercise physiology literature via E-utilities (ESearch + EFetch) +- [x] wger API (`https://wger.de/api/v2`) -- exercise database with muscles, equipment, categories + +## New Tools +- [x] `search_pubmed_training` -- PubMed E-utilities search for exercise/training papers (MeSH terms, structured abstracts) +- [x] `search_exercise_database` -- exercise lookup by muscle group, movement pattern, equipment +- [x] `compute_performance_model` -- Banister fitness-fatigue model (CTL/ATL/TSB via EWMA) +- [x] `compute_dose_response` -- dose-response curve from dose-effect data points + +## New Visualization Tools +- [x] `render_performance_chart` -- Recharts ComposedChart with fitness/fatigue/performance curves +- [x] `render_funnel_plot` -- Visx scatter plot for publication bias detection +- [x] `render_dose_response` -- Visx area+line chart for dose-response curves + +## Infrastructure +- [x] `PubMedRepository` ABC + `PubMedClient` adapter (ESearch JSON + EFetch XML parsing) +- [x] `ExerciseRepository` ABC + `WgerClient` adapter (REST with pagination) +- [x] 4 new domain entities: `PubMedArticle`, `Exercise`, `PerformanceModelPoint`, `DoseResponsePoint` +- [x] Frontend: 3 lazy-loaded chart components registered in `VizRegistry` + +## Phase 2 (Deferred Items -- Now Complete) +- [x] PubMed integration in literature survey (domain-specific prompt guidance + search stats tracking) +- [x] Richer injury risk model (epidemiological context from PubMed systematic reviews) +- [x] `plan_periodization` tool (linear/undulating/block models with evidence-based prescriptions) +- [x] 2 new domain entities: `PeriodizationBlock`, `PeriodizationPlan` +- [x] `SEARCH_TOOLS` module-level frozenset for search stats tracking + +**Counts:** 48 → 56 tools, 13 → 15 data sources, 6 → 9 viz tools. 577 server tests, 107 console tests. diff --git a/docs/roadmap/phase-10f-nutrition-enhancement.md b/docs/roadmap/phase-10f-nutrition-enhancement.md new file mode 100644 index 0000000..2bddd23 --- /dev/null +++ b/docs/roadmap/phase-10f-nutrition-enhancement.md @@ -0,0 +1,30 @@ +Back to [Roadmap Index](README.md) + +# Phase 10F: Nutrition Science Enhancement (Feb 12) -- DONE + +Deepened the nutrition bounded context with 1 new data source, 6 new tools, and 3 new visualization tools. + +## New Data Source +- [x] RxNav API (`https://rxnav.nlm.nih.gov/REST`) -- Drug interaction screening via RxCUI resolution + +## New Tools +- [x] `compare_nutrients` -- side-by-side nutrient comparison between 2+ foods/supplements +- [x] `assess_nutrient_adequacy` -- DRI-based nutrient adequacy assessment (EAR, RDA, AI, UL) +- [x] `check_intake_safety` -- Tolerable Upper Intake Level safety screening +- [x] `check_interactions` -- Drug-supplement interaction screening via RxNav +- [x] `analyze_nutrient_ratios` -- Key nutrient ratio analysis (omega-6:3, Ca:Mg, Na:K, Zn:Cu, Ca:P, Fe:Cu) +- [x] `compute_inflammatory_index` -- Simplified Dietary Inflammatory Index (DII) scoring + +## New Visualization Tools +- [x] `render_nutrient_comparison` -- Recharts grouped BarChart comparing nutrient profiles +- [x] `render_nutrient_adequacy` -- Recharts horizontal BarChart with MAR score and DRI zones +- [x] `render_therapeutic_window` -- Visx custom range chart with EAR/RDA/AI/UL therapeutic zones + +## Infrastructure +- [x] `InteractionRepository` ABC + `RxNavClient` adapter (RxCUI resolution + interaction lookup) +- [x] `dri.py` DRI reference module (~30 nutrients, EAR/RDA/AI/UL by age_group/sex) +- [x] 3 new domain entities: `DrugInteraction`, `NutrientRatio`, `AdequacyResult` +- [x] `NutritionService` expanded with 6 new methods + interaction repository injection +- [x] Frontend: 3 lazy-loaded chart components registered in `VizRegistry` + +**Counts:** 56 -> 67 -> 70 tools (3 generic ML), 15 -> 16 data sources, 9 -> 12 viz tools. 692 server tests, 120 console tests. diff --git a/docs/roadmap/phase-12-saas-infrastructure.md b/docs/roadmap/phase-12-saas-infrastructure.md new file mode 100644 index 0000000..e2da9fc --- /dev/null +++ b/docs/roadmap/phase-12-saas-infrastructure.md @@ -0,0 +1,70 @@ +Back to [Roadmap Index](README.md) + +# Phase 12: SaaS Infrastructure (Feb 13) -- DONE + +Production-ready authentication, credit-based billing, model-tiered investigations, and PostgreSQL persistence. + +> Business rationale and pricing research: see [business-strategy.md](../adr/business-strategy.md) + +## 12A. PostgreSQL Migration -- DONE + +Replaced SQLite with PostgreSQL for multi-tenant persistence. + +- [x] `InvestigationRepository` implementing existing `InvestigationRepository` ABC +- [x] `asyncpg` connection pooling (replaced `aiosqlite`) +- [x] `tsvector` + GIN index for self-referential research (replaces FTS5) +- [x] `JSONB` columns for hypotheses, experiments, findings, candidates +- [x] `users` table: id (UUID), workos_id, email, credits, encrypted_api_key, created_at +- [x] `credit_transactions` table: user_id, amount, type, investigation_id, created_at +- [x] `EHRLICH_DATABASE_URL` env var (replaces `EHRLICH_DB_PATH`) +- [x] `sqlite_repository.py` deleted, `aiosqlite` removed from dependencies + +## 12B. Authentication (WorkOS AuthKit) -- DONE + +User identity via WorkOS (1M MAU free tier). JWT-based. + +- [x] `api/auth.py` with JWKS verification via `PyJWKClient` +- [x] Frontend: `@workos-inc/authkit-react` provider + `` +- [x] Backend: `get_current_user`, `get_current_user_sse`, `get_optional_user` dependencies +- [x] `user_id` extraction from JWT, linked to `users` table via `get_or_create_user()` +- [x] Public routes: `GET /health`, `GET /methodology`, `GET /stats`, `GET /molecule/*`, `GET /investigate/{id}` +- [x] Protected routes: `POST /investigate`, `GET /investigate`, `GET /investigate/{id}/stream`, `POST /investigate/{id}/approve`, `GET /credits/balance` +- [x] Env vars: `EHRLICH_WORKOS_API_KEY`, `EHRLICH_WORKOS_CLIENT_ID` +- [x] SSE auth via `?token=` query param fallback (EventSource does not support headers) + +## 12C. Universal Credit System -- DONE + +One currency. User decides model quality per investigation. + +- [x] Credit spending: Haiku Director = 1 credit, Sonnet Director = 3 credits, Opus Director = 5 credits +- [x] Researcher always Sonnet 4.5, Summarizer always Haiku 4.5 (regardless of tier) +- [x] `GET /credits/balance` -- current balance + BYOK status +- [x] `POST /investigate` accepts `director_tier: "haiku" | "sonnet" | "opus"` parameter +- [x] Credit deduction on investigation start (not completion -- prevents abuse) +- [x] Credit refund on investigation failure +- [x] Default 5 credits for new users + +## 12D. BYOK (Bring Your Own Key) -- DONE + +Users provide their own Anthropic API key. Bypasses credit system. + +- [x] Frontend: `BYOKSettings.tsx` for API key management, stored in `localStorage` +- [x] `X-Anthropic-Key` header sent with requests +- [x] Backend: `api_key_override` forwarded to `AnthropicClientAdapter` +- [x] BYOK users bypass credit system -- pay Anthropic directly + +## 12E. Cost Transparency -- DONE + +Users see exactly what each investigation costs. + +- [x] `CompletionSummaryCard` with expandable cost breakdown +- [x] Tokens (input/output/cache read/cache write) by model role (Director/Researcher/Summarizer) +- [x] Cost data in real-time during SSE stream via `CostUpdate` events + +## 12F-H. Deferred to Post-Hackathon + +- [ ] Ad-funded bonus credits (12F) +- [ ] AGPL dual licensing docs (12G) +- [ ] Production deployment (12H) + +**Verification:** Auth flow, credit deduction/refund, BYOK pass-through, tier selection, cost breakdown all working. diff --git a/docs/roadmap/phase-13-impact-evaluation.md b/docs/roadmap/phase-13-impact-evaluation.md new file mode 100644 index 0000000..02443db --- /dev/null +++ b/docs/roadmap/phase-13-impact-evaluation.md @@ -0,0 +1,140 @@ +Back to [Roadmap Index](README.md) + +# Phase 13: Impact Evaluation Domain -- IN PROGRESS + +New bounded context for hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools usable by any domain. No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. See `docs/adr/impact-evaluation-domain.md` for full design. + +## Market Gap + +Of 2,800 evaluations commissioned by CONEVAL in Mexico, only 11 are impact evaluations (0.4%). No competitor combines: autonomous hypothesis formulation + causal inference + public API connectors + evidence grading + multi-model orchestration. + +## Phase 13A: Foundation (Universal APIs + Domain Config) -- DONE + +- [x] World Bank API client (`https://api.worldbank.org/v2/`) -- development indicators by country +- [x] WHO GHO API client (`https://ghoapi.azureedge.net/api/`) -- health statistics +- [x] FRED API client (`https://api.stlouisfed.org/fred/`) -- 800K+ economic time series +- [x] `search_economic_indicators` tool -- query FRED/World Bank/WHO GHO time series +- [x] `fetch_benchmark` tool -- get comparison values from international sources +- [x] `compare_programs` tool -- cross-program comparison using existing statistical tests +- [x] `IMPACT_EVALUATION` domain config with detection keywords and experiment examples +- [x] Full DDD: domain entities + repository ABCs + infrastructure clients +- [x] Tests: impact bounded context (unit + integration) + +**Counts:** 70 -> 73 tools, 16 -> 19 data sources (18 external + 1 internal), 10 -> 11 bounded contexts, 3 -> 4 domains. + +## Phase 13B (partial): Causal Inference + Impact Viz -- DONE + +- [x] `CausalEstimator` port ABC in `impact/domain/ports.py` (initial placement) +- [x] `DiDEstimator` infrastructure with parallel trends test and automated threat assessment +- [x] `estimate_did` tool -- difference-in-differences with effect size, SE, p-value, threats, evidence tier +- [x] `assess_threats` tool -- knowledge-based validity threat assessment (DiD, PSM, RDD, RCT, IV) +- [x] `render_program_dashboard` viz tool -- multi-indicator KPI dashboard with target tracking +- [x] `render_geographic_comparison` viz tool -- region bar chart with benchmark reference line +- [x] `render_parallel_trends` viz tool -- DiD treatment vs control over time +- [x] Console: 3 new React chart components (ProgramDashboard, GeographicComparison, ParallelTrends) +- [x] Console: 2 Impact Evaluation template cards (CCT School Enrollment, Health Worker Programs) +- [x] Tests: DiD estimator + tool tests (35+ new tests) + +**Counts:** 73 -> 78 tools, 12 -> 15 viz tools, 3 -> 5 impact tools. + +## Phase 13B: Domain-Agnostic Causal Inference -- DONE + +Refactored causal inference from impact-specific to domain-agnostic. All causal estimation methods now live in `analysis/` (alongside statistics), usable by any domain. + +- [x] Moved `ThreatToValidity`, `CausalEstimate` entities from `impact/domain/` to `analysis/domain/causal.py` +- [x] Moved `evaluation_standards.py` (WWC tiers, CONEVAL, CREMAA) from `impact/domain/` to `analysis/domain/evidence_standards.py` +- [x] Created `analysis/domain/causal_ports.py` -- 4 estimator port ABCs: `DiDEstimatorPort`, `PSMEstimatorPort`, `RDDEstimatorPort`, `SyntheticControlPort` +- [x] Moved `DiDEstimator` from `impact/infrastructure/` to `analysis/infrastructure/did_estimator.py` +- [x] New `analysis/infrastructure/psm_estimator.py` -- propensity score matching with balance diagnostics +- [x] New `analysis/infrastructure/rdd_estimator.py` -- regression discontinuity (sharp/fuzzy) +- [x] New `analysis/infrastructure/synthetic_control_estimator.py` -- synthetic control method +- [x] New `analysis/application/causal_service.py` -- orchestrates all 4 estimators + threat assessment + cost-effectiveness +- [x] New `compute_cost_effectiveness` tool -- cost per unit outcome, ICER +- [x] Deleted `impact/domain/ports.py` -- causal estimator port moved to analysis/ +- [x] Trimmed `ImpactService` -- removed DiD/threat methods (delegated to CausalService) +- [x] Impact tools reduced from 5 to 3 (estimate_did + assess_threats moved to analysis/) +- [x] 6 causal tools tagged `"causal"` in analysis/tools.py +- [x] New `render_rdd_plot` viz tool -- regression discontinuity scatter with cutoff (Visx) +- [x] New `render_causal_diagram` viz tool -- DAG showing treatment/outcome/confounders (Visx) +- [x] Console: `RDDPlot.tsx`, `CausalDiagram.tsx` chart components +- [x] Console: `ThreatAssessment.tsx` panel with severity badges +- [x] Console: 2 new template cards (Education Policy, Health Policy) -- 9 total +- [x] VizRegistry updated with 2 new viz types + +**Counts:** 78 -> 84 tools (6 causal + 1 cost-effectiveness - 2 moved + 2 viz), 15 -> 17 viz tools, 5 -> 3 impact tools, 0 -> 6 causal tools (analysis/), 7 -> 9 templates. + +## Phase 13A-2: Document Upload -- DONE + +- [x] File upload API (`POST /upload`) with multipart/form-data, 50MB limit, WorkOS auth +- [x] PDF text extraction (`pymupdf`) with page-level parsing and 8000-char truncation +- [x] CSV/Excel parsing (`pandas` + `openpyxl`) with summary statistics and sample rows +- [x] `UploadedFile` domain entity with `TabularData` / `DocumentData` frozen dataclasses +- [x] `FileProcessor` application service (zero external deps in domain layer) +- [x] PostgreSQL `uploaded_files` table with JSONB `parsed_data` column +- [x] Inject uploaded data into Director/Researcher prompts as `` XML block +- [x] `query_uploaded_data` tool -- intercepted by `ToolDispatcher`, pandas-based filtering (eq/gt/lt/gte/lte/contains) +- [x] Upload-first flow: `POST /upload` returns preview, `POST /investigate` references `file_ids` +- [x] Frontend: `FileUpload.tsx` drag-and-drop component with progress spinner +- [x] Frontend: `DataPreview.tsx` compact inline cards for tabular/document previews +- [x] Frontend: `use-upload.ts` TanStack Query mutation hook +- [x] Tests: 16 server tests (FileProcessor + domain entities + query handler) + 16 console tests + +**Counts:** 84 -> 85 tools (added `query_uploaded_data`). Investigation tools: 7 -> 8. + +## Phase 13C: Mexico Integration + +- [ ] INEGI Indicadores API client (`https://www.inegi.org.mx/app/api/indicadores/`) -- census, demographics +- [ ] INEGI DENUE API client (`https://www.inegi.org.mx/app/api/denue/v1/`) -- 5M+ businesses +- [ ] datos.gob.mx CKAN client (`https://datos.gob.mx/api/3/action/`) -- 1000+ federal datasets +- [ ] Transparencia Presupuestaria client (`https://nptp.hacienda.gob.mx/`) -- budget execution +- [ ] Banxico SIE client (`https://www.banxico.org.mx/SieAPIRest/service/v1/`) -- economic indicators +- [ ] `search_census_data` tool -- query INEGI/Census by indicator/geography/period +- [ ] `search_budget_data` tool -- query budget execution and social program spending +- [ ] `extract_mir` tool -- extract MIR (Matriz de Indicadores) from evaluation documents +- [ ] `analyze_program_indicators` tool -- MIR/logical framework indicator analysis with CREMAA validation +- [ ] Frontend: `MIRTable.tsx` logical framework matrix +- [ ] CONEVAL ECR report template for `generate_evaluation_report` + +## Phase 13D: US Integration + +- [ ] Census Bureau API client (`https://api.census.gov/data/`) -- ACS, demographics +- [ ] BLS API client (`https://api.bls.gov/publicAPI/v2/`) -- employment, wages, CPI +- [ ] USAspending API client (`https://api.usaspending.gov/api/v2/`) -- federal grants/contracts +- [ ] College Scorecard API client (`https://api.data.gov/ed/collegescorecard/v1/`) -- education outcomes +- [ ] HUD API client (`https://www.huduser.gov/hudapi/public/`) -- housing data +- [ ] CDC WONDER client (`https://wonder.cdc.gov/`) -- public health data +- [ ] `search_health_indicators` tool -- query WHO GHO or CDC WONDER +- [ ] `search_open_data` tool -- query datos.gob.mx or data.gov CKAN portals +- [ ] WWC evidence tier classification in evaluation reports +- [ ] US-formatted evaluation report template + +## Phase 13E: MCP Bridge Self-Service + +- [ ] User MCP server registration UI (`/settings/data-sources`) +- [ ] Connection testing and tool auto-discovery +- [ ] Per-organization data source management +- [ ] Tool auto-tagging with domain tags for discovery + +## Data Sources (Planned: 8+ new, 3 implemented in 13A) + +| Source | API | Purpose | Auth | Phase | +|--------|-----|---------|------|-------| +| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country | None | 13A | +| WHO GHO | `https://ghoapi.azureedge.net/api/` | Health statistics by country | None | 13A | +| FRED | `https://api.stlouisfed.org/fred/` | 800K+ economic time series | API key | 13A | +| INEGI Indicadores | `https://www.inegi.org.mx/app/api/indicadores/` | Mexico census, demographics | Token | 13C | +| INEGI DENUE | `https://www.inegi.org.mx/app/api/denue/v1/` | Mexico business directory (5M+) | Token | 13C | +| datos.gob.mx | `https://datos.gob.mx/api/3/action/` | Mexico open data (CKAN, 1000+ datasets) | Token (optional) | 13C | +| Transparencia Presupuestaria | `https://nptp.hacienda.gob.mx/` | Mexico budget execution | None | 13C | +| Banxico SIE | `https://www.banxico.org.mx/SieAPIRest/service/v1/` | Mexico economic indicators | Token | 13C | +| US Census Bureau | `https://api.census.gov/data/` | ACS, demographics | API key (optional) | 13D | +| BLS | `https://api.bls.gov/publicAPI/v2/` | Employment, wages, CPI | API key | 13D | +| USAspending | `https://api.usaspending.gov/api/v2/` | Federal grants, contracts | None | 13D | + +## Tools (Planned: ~18 new tools + ~6 viz tools) + +**Data Access:** `search_census_data`, `search_economic_indicators`, `search_budget_data`, `search_health_indicators`, `search_open_data`, `fetch_benchmark` +**Causal Inference:** `estimate_did`, `estimate_psm`, `estimate_rdd`, `estimate_synthetic_control`, `assess_threats` +**Program Analysis:** `analyze_program_indicators`, `compute_cost_effectiveness`, `compare_programs`, `analyze_beneficiary_data` +**Document Processing:** `extract_program_data`, `extract_mir`, `generate_evaluation_report` +**Visualization:** `render_causal_diagram`, `render_parallel_trends`, `render_rdd_plot`, `render_program_dashboard`, `render_cost_effectiveness`, `render_geographic_comparison` diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md new file mode 100644 index 0000000..3e38253 --- /dev/null +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -0,0 +1,123 @@ +Back to [Roadmap Index](README.md) + +# Phase 14: Scientific Engine Hardening (Feb 14-15) -- IN PROGRESS + +Architecture cleanup, scientific depth improvements, agentic tree search, and paper generation. Runs in parallel with Phase 13A-2 (upload). Addresses audit findings from codebase review. + +## Prior Art + +Ehrlich sits alongside three contemporary AI scientific discovery systems: + +- **Sakana AI Scientist v2** (Apr 2025): End-to-end paper generation via agentic tree search. First AI-generated peer-reviewed workshop paper. ML-only. [arXiv:2504.08066](https://arxiv.org/abs/2504.08066) +- **FutureHouse Robin** (May 2025): Multi-agent system that discovered ripasudil as therapeutic candidate for AMD in 2.5 months. Biology-only, lab-in-the-loop. [arXiv:2505.13400](https://arxiv.org/abs/2505.13400) +- **Ai2 AutoDiscovery** (Feb 2026): Bayesian surprise-guided hypothesis exploration over large search spaces. [allenai.org/blog/autodiscovery](https://allenai.org/blog/autodiscovery) + +Ehrlich differentiates by: domain-agnostic architecture (4 domains vs single-domain competitors), multi-model orchestration (Director/Researcher/Summarizer), real-time streaming UI, 18 external data sources, and computational laboratory (ML pipelines + causal inference as experimentation). + +## 14A: Orchestrator Decomposition -- TODO + +Split `MultiModelOrchestrator` (1,788 lines, 11 concerns) into focused modules following Clean Architecture. No behavior changes -- pure structural refactoring. + +- [x] Extract `diagram_builder.py` -- Excalidraw element building + MCP bridge export (127 lines) +- [x] Extract `tool_dispatcher.py` -- tool execution + caching + `search_prior_research` / `query_uploaded_data` interception (34 lines) +- [x] Extract `researcher_executor.py` -- single experiment runner with tool loop (216 lines) +- [x] Extract `batch_executor.py` -- parallel experiment execution via asyncio.Queue (56 lines) +- [x] Merged uploaded_data_query into tool_dispatcher.py +- [x] Kept run() in multi_orchestrator.py as main coordination point +- [ ] Orchestrator reduced to ~400 lines: init, run (delegates to phase_manager), approve + +Result: `multi_orchestrator.py` from 1,788 to 1,216 lines (4 modules extracted: 718 lines total) + +## 14B: Route File Cleanup -- TODO + +Move infrastructure concerns out of `api/routes/investigation.py` (828 lines) into application layer. + +- [ ] Extract `application/registry_factory.py` -- `build_tool_registry()` + `build_domain_registry()` (128 lines of tool registration) +- [ ] Extract `application/orchestrator_factory.py` -- `create_orchestrator()` with all adapter wiring +- [ ] Move Pydantic DTOs to `api/schemas/investigation.py` (6 models) +- [ ] Lazy-initialize tool registry (singleton, built once instead of per-investigation) + +Target: route file from 828 to ~400 lines. + +## 14C: Prompt Module Split -- TODO + +Split `prompts.py` (1,512 lines) into focused modules. + +- [ ] `prompts/__init__.py` -- re-exports for backward compatibility +- [ ] `prompts/constants.py` -- `SCIENTIST_SYSTEM_PROMPT`, `SUMMARIZER_PROMPT`, `RESEARCHER_EXPERIMENT_PROMPT` +- [ ] `prompts/director.py` -- `DIRECTOR_FORMULATION_PROMPT`, `DIRECTOR_EXPERIMENT_PROMPT`, `DIRECTOR_EVALUATION_PROMPT`, `DIRECTOR_SYNTHESIS_PROMPT` +- [ ] `prompts/builders.py` -- `build_pico_and_classification_prompt`, `build_literature_survey_prompt`, `build_literature_assessment_prompt`, `build_formulation_prompt`, `build_experiment_prompt`, `build_synthesis_prompt`, `build_researcher_prompt`, `build_multi_investigation_context`, `build_uploaded_data_context`, `_build_prior_context` + +## 14D: Domain Tool Enrichment -- TODO + +Upgrade 4 composite tools from thin pass-throughs to real domain-specific logic. + +- [ ] `search_training_literature` -- MeSH term expansion, study type ranking (SR > RCT > cohort), date recency weighting, non-human study filtering +- [ ] `search_supplement_evidence` -- GRADE-style quality filtering, RCT/meta-analysis prioritization, retracted paper exclusion +- [ ] `compare_nutrients` -- per-nutrient delta computation, winner highlighting per category, overall MAR score comparison +- [ ] Fix `TrainingService` I-squared reimplementation -- delegate to `StatisticsService` (training_service.py:79-85) + +## 14E: Agentic Hypothesis Tree Search -- TODO + +Replace linear hypothesis testing with tree-based exploration inspired by AI Scientist v2's agentic tree search. The current parallel batch system (2 experiments per hypothesis) becomes a tree where hypotheses can branch, deepen, or be pruned based on evidence. + +- [ ] `HypothesisNode` dataclass -- extends Hypothesis with `depth`, `branch_score`, `children: list[str]` +- [ ] `TreeManager` in `application/tree_manager.py` -- decides which branches to explore based on confidence scores, prunes low-confidence branches (<0.4 threshold), tracks exploration budget +- [ ] Director formulation prompt update -- "given current evidence tree, propose sub-hypotheses for the most promising branch OR prune low-confidence branches" +- [ ] Director evaluation prompt update -- after evaluating, decide: (a) deepen (spawn sub-hypotheses), (b) prune (mark branch as dead), (c) branch (revise into alternative direction) +- [ ] `max_depth` parameter (default: 3) to prevent infinite recursion +- [ ] Budget-aware exploration -- track API cost per branch, prefer deepening high-confidence/low-cost branches +- [ ] `HypothesisTreeEvent` SSE event for frontend tree rendering +- [ ] Console: `InvestigationDiagram.tsx` already renders React Flow graph -- tree structure renders naturally as growing/pruning graph + +## 14F: Scientific Paper Generator -- TODO + +Generate structured scientific papers from completed investigations. Not AI-generated prose -- structured assembly of existing investigation data into publication format. + +- [ ] `application/paper_generator.py` -- takes `Investigation` entity, produces structured sections: + - Title (from research question) + - Abstract (Director synthesis summary) + - Introduction (PICO decomposition + literature survey context) + - Methods (experiment designs, tool descriptions, statistical approach) + - Results (findings with statistical tests, visualizations as base64 SVG) + - Discussion (hypothesis assessments, GRADE certainty, limitations, knowledge gaps) + - References (collected DOIs + citations) + - Supplementary (cost breakdown, model info, negative controls, tool call trace) +- [ ] `GET /investigate/{id}/paper` endpoint -- returns structured paper as JSON or Markdown +- [ ] `GET /investigate/{id}/paper?format=pdf` -- server-side PDF generation (optional, `weasyprint` or `reportlab`) +- [ ] Console: "Export Paper" button on completed investigation page +- [ ] Console: paper preview with section navigation + +## 14G: Validation Runs -- TODO + +Run real investigations to validate the engine produces meaningful scientific results. Compare Ehrlich vs raw Claude Opus 4.6 on identical prompts. + +- [ ] Molecular science validation: known drug discovery question with verifiable answer +- [ ] Training science validation: exercise physiology question with literature-backed answer +- [ ] Impact evaluation validation: real program evaluation data (CODESON) compared against professional auditors and raw Claude +- [ ] Document comparison results: where Ehrlich agrees/disagrees with baseline, where tree search found non-obvious paths +- [ ] Capture investigation traces for demo video + +## Dependency Graph + +``` +Phase 14A (Orchestrator Decomposition) + | +Phase 14B (Route File Cleanup) -- parallel with 14A + | +Phase 14C (Prompt Module Split) -- parallel with 14A/B + | +Phase 14D (Domain Tool Enrichment) -- after 14A (touches tool layer) + | +Phase 14E (Agentic Tree Search) -- after 14A (modifies orchestrator) + | +Phase 14F (Paper Generator) -- after 14A (reads investigation data) + | +Phase 14G (Validation Runs) -- after 14E + 14F (needs tree search + paper gen) +``` + +14A, 14B, 14C can run in parallel (independent refactorings). +14D, 14E, 14F depend on 14A (clean orchestrator). +14G depends on everything else being done. + +**Verification:** All existing tests pass after each sub-phase. New tests for tree search, paper generator, and enriched tools. Quality gates: ruff 0, mypy 0, pytest pass, tsc 0, vitest pass. diff --git a/docs/scientific-methodology.md b/docs/scientific-methodology.md index 2d099b5..4d11671 100644 --- a/docs/scientific-methodology.md +++ b/docs/scientific-methodology.md @@ -1,6 +1,6 @@ # Scientific Methodology -> Part of the [development roadmap](roadmap.md). This is a cross-cutting upgrade that applies to all investigation phases. +> Part of the [development roadmap](roadmap/README.md). This is a cross-cutting upgrade that applies to all investigation phases. Ehrlich grounds every phase of its investigation workflow in established scientific methodology. Each phase is backed by research from the world's best scientists and epistemological frameworks. @@ -208,7 +208,7 @@ Hypotheses are tested in parallel batches of 2. Without differentiation, paralle ### Phase 4: Evidence Evaluation -- UPGRADED -**Status:** Complete. Grounded in 47 verified sources spanning evidence hierarchies (Burns & Chung), effect sizes (Cohen), Bayesian updating (Kass & Raftery, Jeffreys), contradiction resolution (Lakatos, Hunter & Schmidt), and convergence of evidence (Munafo et al.). See [evidence-evaluation.md](../research/methodology/evidence-evaluation.md). +**Status:** Complete. Grounded in 47 verified sources spanning evidence hierarchies (Burns & Chung), effect sizes (Cohen), Bayesian updating (Kass & Raftery, Jeffreys), contradiction resolution (Lakatos, Hunter & Schmidt), and convergence of evidence (Munafo et al.). See [evidence-evaluation.md](research/methodology/evidence-evaluation.md). **Sources:** - Burns & Chung (2011) -- Evidence hierarchy (8-tier ranking by directness, reproducibility, controls) @@ -274,7 +274,7 @@ Orchestrator wiring: ### Phase 5: Negative Controls & Model Validation -- UPGRADED -**Status:** Complete. Grounded in 33 verified sources. See [negative-controls.md](../research/methodology/negative-controls.md). +**Status:** Complete. Grounded in 33 verified sources. See [negative-controls.md](research/methodology/negative-controls.md). **Sources:** - Zhang et al. (1999) -- Z-factor / Z' statistic for assay quality separation @@ -361,7 +361,7 @@ PositiveControlRecorded domain event → positive_control SSE event type ### Phase 6: Synthesis -- UPGRADED -**Status:** Complete. Grounded in 29 verified sources. See [synthesis.md](../research/methodology/synthesis.md). +**Status:** Complete. Grounded in 29 verified sources. See [synthesis.md](research/methodology/synthesis.md). **Sources:** - Guyatt et al. (2008) -- GRADE: certainty of evidence (high/moderate/low/very low) with 5 downgrading + 3 upgrading domains From f483de2845c2e5a1091919077e6b0fad6b644552 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 08:08:21 -0700 Subject: [PATCH 10/35] refactor(docs): deduplicate CLAUDE.md, trim CONTRIBUTING.md, replace stale CHANGELOG CLAUDE.md: 591 -> 231 lines. Removed duplicated tools catalog, data sources table, and domain descriptions (all in README.md). Kept AI-specific content: key patterns, key files, dependency rules, commands. CONTRIBUTING.md: 524 -> 307 lines. Removed duplicated setup/quality gates/ architecture sections, fixed stale data (10->11 contexts, 70->85 tools, SQLite->PostgreSQL). Kept unique contribution guide. CHANGELOG.md: replaced stale unversioned content with pointer to docs/roadmap/README.md which tracks all 18 phases. --- CHANGELOG.md | 283 +------------------------ CLAUDE.md | 540 ++++++++---------------------------------------- CONTRIBUTING.md | 140 ++----------- 3 files changed, 111 insertions(+), 852 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7936d..48f2f11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,284 +1,5 @@ # Changelog -All notable changes to Ehrlich will be documented in this file. +Project history is tracked via phase documents in the roadmap. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - -## [Unreleased] - -### Added - -- Token optimization: prompt caching via `cache_control: ephemeral` on system messages (90% cheaper cached tokens across 20-40 calls) -- Token optimization: phase-based tool filtering -- researcher gets only planned tools per experiment (5-10 instead of 23) -- Token optimization: structured extraction (`_compact_result`) strips verbose fields before summarizer check, eliminating 50-80% of Haiku calls -- Token optimization: in-memory TTL tool cache (`ToolCache`) -- deterministic tools cached forever, API results cached 24h-7d -- Per-experiment Lab View: `experiment_id` field on `ToolCalled` and `ToolResultEvent` domain events -- Per-experiment Lab View: experiment selector dropdown in `LiveLabViewer` for filtering 3D scene by experiment -- Per-experiment Lab View: `eventsByExperiment` state in `use-sse.ts` for grouping SSE events -- Data source: UniProt REST client (`uniprot_client.py`) -- protein function, disease associations, GO terms, PDB cross-refs -- Data source: Open Targets GraphQL client (`opentargets_client.py`) -- disease-target associations with scored evidence -- Data source: GtoPdb REST client (`gtopdb_client.py`) -- expert-curated receptor/ligand interactions with affinities -- Tools: `get_protein_annotation` -- UniProt protein annotations (tool #28) -- Tools: `search_disease_targets` -- Open Targets disease-target associations (tool #29) -- Tools: `search_pharmacology` -- GtoPdb curated pharmacology data (tool #30) -- Domain entities: `ProteinAnnotation`, `TargetAssociation`, `PharmacologyEntry` -- Repository ABCs: `ProteinAnnotationRepository`, `TargetAssociationRepository`, `PharmacologyRepository` -- Tests: UniProt, Open Targets, GtoPdb client tests with respx mocking -- Parallel researchers: `_run_experiment_batch()` runs 2 experiments concurrently via `asyncio.Queue` event merging -- Parallel researchers: `asyncio.Lock` protects shared `Investigation` state and `CostTracker` during concurrent execution -- Prompt engineering: XML-tagged instructions (``, ``, ``) in all Director and Researcher prompts -- Prompt engineering: multishot examples in `DIRECTOR_FORMULATION_PROMPT` (MRSA + Alzheimer's domains) and `DIRECTOR_EXPERIMENT_PROMPT` -- Prompt engineering: tool usage examples in `RESEARCHER_EXPERIMENT_PROMPT` (docking, model training, disease-target search) -- Prompt engineering: search strategy guidance ("start broad, then narrow") from Anthropic multi-agent research patterns -- Context compaction: `_build_prior_context()` compresses completed hypotheses into XML summary for Director between batches - -### Changed - -- Always uses `MultiModelOrchestrator` (single-model `Orchestrator` removed) -- Hypothesis loop runs batched parallel execution (up to 2 hypotheses per batch) -- Tool count: 27 -> 30 (3 new data source tools) -- All prompts rewritten with XML structure, multishot examples, and Anthropic best practices -- E2E tests: rewritten to use `MultiModelOrchestrator` -- Server tests: 267 -> 271 passing (80.2% coverage) - -### Removed - -- `Orchestrator` class (`orchestrator.py`) -- single-model fallback path eliminated -- `test_orchestrator.py` -- tests for removed class - ---- - -### Added - -- Auditability: SSE event persistence in SQLite `events` table -- all events stored during investigation, replayed on page reload -- Auditability: `save_event()` / `get_events()` methods on `InvestigationRepository` interface and SQLite implementation -- Auditability: `prompt` field included in `completed` SSE event during replay (research question visible in report) -- Console: `InvestigationReport` -- structured 8-section report replacing plain-text ReportViewer (Research Question, Executive Summary, Hypotheses & Outcomes, Methodology, Key Findings, Candidate Molecules, Model Validation, Cost & Performance) -- Console: `DiagramRenderer` -- React Flow (`@xyflow/react`) investigation diagram with custom node types, minimap, controls -- Console: Dark-friendly diagram palette with dark fills and light text -- Console: Diagram zoom, pan, and minimap via React Flow built-ins -- Dependencies: `@xyflow/react` for node-graph diagrams - -### Changed - -- Console: `InvestigationDiagram` uses React Flow instead of Excalidraw (lazy-loaded, code-split ~188KB chunk) -- Console: `diagram-builder.ts` outputs React Flow `Node[]` + `Edge[]` with `smoothstep` routing instead of Excalidraw skeletons -- Console: Investigation diagram is always read-only (`nodesDraggable={false}`) -- Console: Investigation page shows `FindingsPanel` during live investigation, full `InvestigationReport` after completion -- Console: Completed investigation replay includes full timeline (all SSE events) instead of just final status -- Server: `_replay_final()` yields stored events before completed event for full timeline reconstruction - -### Removed - -- Console: `ExcalidrawWrapper` component (replaced by `DiagramRenderer`) -- Console: `ReportViewer` component (replaced by `InvestigationReport`) -- Console: `@excalidraw/excalidraw` dependency removed from `package.json` - ---- - -### Added - -- Domain-agnostic engine: RCSB PDB client for dynamic protein target discovery by organism/function/keyword -- Domain-agnostic engine: PubChem PUG REST client for compound search by name/SMILES/similarity -- Domain-agnostic engine: EPA CompTox CTX API client for environmental toxicity profiles (LD50, LC50, BCF, mutagenicity) -- Domain-agnostic engine: Flexible ChEMBL bioactivity search supporting any assay type (Ki, EC50, Kd, MIC, IC50) -- Tools: `search_protein_targets` -- RCSB PDB search by organism/function (tool #24) -- Tools: `search_compounds` -- PubChem compound search by name/SMILES/similarity (tool #25) -- Tools: `search_bioactivity` -- flexible ChEMBL query with configurable assay types (tool #26) -- Tools: `fetch_toxicity_profile` -- EPA CompTox environmental toxicity data (tool #27) -- Domain entities: `ToxicityProfile`, `CompoundSearchResult` -- Repository ABCs: `ProteinTargetRepository`, `ToxicityRepository`, `CompoundSearchRepository` -- Data: `data/targets/default.yaml` -- YAML-configured protein targets (replaces hardcoded dict) -- Data: `data/resistance/default.yaml` -- YAML-configured resistance mutations and SMARTS patterns -- Config: `EHRLICH_COMPTOX_API_KEY` setting for EPA CompTox API access -- Console: `LiveLabViewer` -- 3Dmol.js WebGL scene driven by SSE events (proteins load, ligands dock, candidates color by score) -- Console: `InvestigationDiagram` -- Excalidraw hypothesis map with status-colored nodes and evidence-chain arrows -- Console: `ExcalidrawWrapper` -- lazy-loaded Excalidraw component with dark theme -- Console: `scene-builder.ts` -- maps SSE events to 3Dmol.js operations (addProtein, addLigand, colorByScore, highlight) -- Console: `diagram-builder.ts` -- converts investigation data to Excalidraw elements (nodes, arrows, labels) -- Console: Tab system on investigation page (Timeline | Lab View | Diagram) -- Tests: 45+ new tests for RCSB, PubChem, CompTox clients, resistance YAML loader, new tool functions -- Dependencies: `pyyaml`, `respx` (dev), `@excalidraw/excalidraw` - -### Changed - -- Engine generalized from antimicrobial-only to domain-agnostic molecular discovery (23 -> 27 tools) -- Prompts: all "antimicrobial scientist" language replaced with "molecular discovery scientist" -- Prompts: added domain-adaptive instruction ("The user's research question defines the domain") -- Prompts: tool references updated to include 4 new data source tools -- Prediction docstrings: "antimicrobial activity" -> "bioactivity" -- `ProteinStore`: hardcoded 5-target dict replaced with YAML loader + RCSB PDB fallback search -- `SimulationService`: hardcoded resistance mutations/patterns replaced with YAML-loaded data -- `ChEMBLLoader`: parameterized `assay_types` (was hardcoded `MIC,IC50`), type-aware cache keys -- `AnalysisService`: added `compound_repo` for PubChem integration -- Tool registry: 23 -> 27 tools registered in investigation routes -- Server tests: 212 -> 257 passing (80.6% coverage) - ---- - -### Added - -- Investigation: hypothesis-driven scientific method replacing linear pipeline -- Investigation: `propose_hypothesis` tool -- register testable hypotheses with rationale -- Investigation: `design_experiment` tool -- plan experiments with tool sequences -- Investigation: `evaluate_hypothesis` tool -- assess outcomes (supported/refuted/revised) with confidence -- Investigation: `record_finding` tool -- findings linked to hypothesis + evidence type -- Investigation: `record_negative_control` tool -- validate models with known-inactive compounds -- Investigation: `conclude_investigation` tool -- final summary with ranked candidates -- Domain: `Hypothesis` entity with status enum (proposed/testing/supported/refuted/revised) -- Domain: `Experiment` entity with status enum (planned/running/completed/failed) -- Domain: `NegativeControl` frozen dataclass -- Domain: 5 new events (HypothesisFormulated, HypothesisEvaluated, ExperimentStarted, ExperimentCompleted, NegativeControlRecorded) -- SSE: 5 new event types (hypothesis_formulated, hypothesis_evaluated, experiment_started, experiment_completed, negative_control) -- Console: `HypothesisBoard` -- kanban-style hypothesis status grid -- Console: `HypothesisCard` -- expandable card with confidence bar -- Console: `ActiveExperimentCard` -- live experiment activity card -- Console: `NegativeControlPanel` -- validation results table -- Console: `CompletionSummaryCard` -- post-completion counts (candidates, findings, hypotheses) -- Console: Findings replay from `InvestigationCompleted` event on page reload -- Console: Toast notifications via `sonner` for completion + error events -- Console: Custom scrollbar CSS (8px webkit + Firefox thin) with OKLCH theme - -### Changed - -- `MultiModelOrchestrator`: hypothesis-driven loop (formulate -> design -> execute -> evaluate per hypothesis) -- Director prompts: 4 phases (formulation, experiment, evaluation, synthesis) with domain-adaptive language -- Tool count: 19 -> 23 (6 investigation control tools added) -- SQLite repository: serializes hypothesis/experiment/negative_control data - ---- - -### Added - -- Branding: "Lab Protocol" dark-only visual identity (Industrial Scientific + Editorial + Cyberpunk Lab) -- Branding: OKLCH color system with Molecular Green primary (`oklch(0.72 0.19 155)`) -- Branding: Space Grotesk display/body font + JetBrains Mono data/label font via Google Fonts -- Branding: `pulse-glow` keyframe animation for active/running states -- Branding: Dark prose overrides for react-markdown content -- Console: Molecular bond phase progress (node-and-bond visualization replacing flat bars) -- Console: Section headers with monospace uppercase + left green border accent -- Data pipeline: `evidence` field threaded from Finding entity through FindingRecorded event to SSE to frontend -- Data pipeline: Candidate scoring fields (`prediction_score`, `docking_score`, `admet_score`, `resistance_risk`) populated from orchestrator prompts and serialized through SSE -- Console: Expandable Timeline events -- tool results, thinking, director decisions expand on click -- Console: Rich Director decision cards -- planning shows phase goals, review shows quality/gaps/guidance, synthesis shows confidence/limitations -- Console: FindingsPanel grid layout (2-3 columns) with evidence section -- Console: CandidateTable scoring columns (Prediction, Docking, ADMET, Resistance) with color-coded values -- Console: CostBadge per-model breakdown popover (Director, Researcher, Summarizer costs) -- Molecule visualization: server-side 2D SVG depiction via RDKit `rdMolDraw2D` (`depict_2d` on RDKitAdapter and ChemistryService) -- Molecule API: `GET /molecule/depict` returns `image/svg+xml` with 24h cache, error SVG for invalid SMILES -- Molecule API: `GET /molecule/conformer` returns 3D conformer JSON (mol_block, energy, num_atoms) -- Molecule API: `GET /molecule/descriptors` returns molecular descriptors + Lipinski pass/fail -- Molecule API: `GET /targets` returns list of protein targets (pdb_id, name, organism) -- Console: `MolViewer2D` component -- server-side SVG via `` tag with lazy loading and error fallback -- Console: `MolViewer3D` component -- 3Dmol.js WebGL viewer with stick style and Jmol coloring -- Console: `DockingViewer` component -- 3Dmol.js protein (cartoon/spectrum) + ligand (stick/green) overlay -- Console: `CandidateDetail` expandable panel with 2D/3D views, property card, and Lipinski badge -- Console: `CandidateTable` rewrite with 2D structure thumbnails, expand/collapse rows, chevron indicators -- Console: 3Dmol.js dependency for 3D molecular visualization -- Tests: 10 molecule API tests, 3 RDKit depict tests, 4 MolViewer2D component tests - -### Removed - -- Console: `useRDKit` hook (dead code, replaced by server-side SVG rendering) - -### Changed - -- Console: Complete dark theme overhaul — all 15 components restyled for dark background -- Console: AppLayout header redesigned with green bar accent, monospace alpha badge, right-aligned label -- Console: Home page left-aligned (anti-AI pattern), mono stat line "19 tools · 7 phases · multi-model" -- Console: Investigation page redesigned as full-width vertical stack (was 2/3+1/3 grid) -- Console: Timeline events restyled — green findings, amber director events, mono tool names -- Console: CostBadge uses monospace with green-highlighted cost amount, clickable for model breakdown -- Console: PromptInput dark textarea with green focus ring and glow hover on button -- Console: InvestigationList dark cards with green border hover effect -- Console: FindingsPanel dark surface cards with green file icon -- Console: ReportViewer uses `prose-invert` with custom dark prose colors -- Console: Lipinski badges use `bg-primary/20` (pass) and `bg-destructive/20` (fail) -- Molecule: 3Dmol.js viewers use dark background (`#1a1e1a`) instead of white -- Molecule: Error SVG uses dark background matching surface color -- Molecule: MolViewer2D error fallback uses `bg-surface` instead of `bg-white` -- `CandidateTable` replaces raw SMILES text column with inline 2D structure thumbnails (100x75) -- `CandidateTable` rows are now clickable to expand `CandidateDetail` panel -- `FindingsPanel` shows full detail text (removed 200-char truncation) -- Tool result preview increased from 500 to 1500 chars in both orchestrators -- Prompts: `DIRECTOR_SYNTHESIS_PROMPT` and `SCIENTIST_SYSTEM_PROMPT` request candidate scoring fields - ---- - -### Added - -- Multi-model: Director-Worker-Summarizer architecture (Opus 4.6 plans, Sonnet 4.5 executes tools, Haiku 4.5 compresses large outputs) -- Multi-model: `MultiModelOrchestrator` with phase-based execution, director review, and final synthesis -- Multi-model: Per-model cost tracking with tiered pricing (Opus $15/$75, Sonnet $3/$15, Haiku $0.80/$4 per M tokens) -- Multi-model: 5 new prompts -- director planning, review, synthesis, researcher phase, summarizer -- Multi-model: 3 new domain events -- `DirectorPlanning`, `DirectorDecision`, `OutputSummarized` -- Multi-model: 3 new SSE event types -- `director_planning`, `director_decision`, `output_summarized` -- Persistence: SQLite repository with WAL mode via `aiosqlite` for investigation storage -- Persistence: `InvestigationRepository` ABC in domain layer -- API: `GET /investigate` -- list all investigations (most recent first) -- API: `GET /investigate/{id}` -- full investigation detail with findings, candidates, cost -- API: Orchestrator creation from model settings (always multi-model) -- Console: SSE reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) -- Console: Investigation history list on home page with status badges and candidate counts -- Console: Phase progress bar (7-segment) with active/completed/inactive states -- Console: Director event rendering in Timeline (planning, decision, output summarized) -- Console: `useInvestigations` hook with TanStack Query (10s refetch interval) -- Investigation: `created_at` and `cost_data` fields on Investigation entity -- Config: Per-model settings (`EHRLICH_DIRECTOR_MODEL`, `EHRLICH_RESEARCHER_MODEL`, `EHRLICH_SUMMARIZER_MODEL`) -- Config: `EHRLICH_SUMMARIZER_THRESHOLD`, `EHRLICH_MAX_ITERATIONS_PER_PHASE`, `EHRLICH_DB_PATH` -- Tests: 25+ new tests for multi-model orchestrator, SQLite repository, updated API and cost tracker - -### Fixed - -- Added `pyarrow` to core dependencies -- required for ChEMBL parquet caching (`explore_dataset` crashed without it) - -### Changed - -- Cost tracker refactored to track per-model usage with model-specific pricing breakdown -- `AnthropicClientAdapter` exposes `model` property for cost attribution -- Orchestrator passes model to `cost.add_usage()` for accurate per-model tracking -- API routes use SQLite repository instead of in-memory dict -- `InvestigationCompleted` event includes `candidates` list -- SSE `completed` event includes candidates and cost data from investigation -- Integration: API key wired from `Settings` to `AnthropicClientAdapter` constructor -- Integration: Exponential backoff retry (3 attempts) for rate-limit and timeout errors -- Integration: Startup lifespan event with API key validation, tool count logging, optional dependency checks -- Integration: Data preparation script (`data/scripts/prepare_data.py`) using `ChEMBLLoader` for pre-downloading ChEMBL bioactivity data and PDB protein structures -- Integration: E2E smoke test exercising full pipeline (tool registry, orchestrator dispatch, SSE events) -- Integration: Environment variable documentation in README -- Integration: Demo instructions in README - -### Changed - -- Removed `modify_molecule` stub from tool registry (19 tools, down from 20) -- Improved error handling in `explore_dataset` (ChEMBL timeout, empty datasets) -- Improved error handling in `train_model` (dataset too small) -- Improved error handling in `predict_candidates` (model not found) -- Improved error handling in `dock_against_target` (invalid target, lists available targets) -- Improved error handling in `assess_resistance` (invalid target, invalid SMILES) -- Added `has_api_key` property to `Settings` for startup validation - -### Fixed - -- `AnthropicClientAdapter` no longer ignores `Settings.anthropic_api_key` (was reading only from env) - ---- - -- Initial project scaffolding with DDD architecture (6 bounded contexts) -- Server: Python 3.12 + FastAPI + uv with full domain/application/infrastructure layers -- Console: React 19 + TypeScript 5.7 + Bun + Vite 6 + TanStack Router -- Shared kernel: SMILES/InChIKey/MolBlock types, Molecule value object, exception hierarchy -- API layer: FastAPI factory with CORS, health endpoint, investigation route stub -- SSE event types for real-time investigation streaming -- Investigation feature: Timeline, PromptInput, CandidateTable, ReportViewer, CostBadge -- Molecule feature: MolViewer2D, MolViewer3D, DockingViewer, PropertyCard stubs -- CI/CD: GitHub Actions (server: ruff + mypy + pytest, console: build + typecheck + test) -- Docker: multi-stage builds for server (uv + python:3.12-slim) and console (bun + nginx) -- Data: core references JSON, data preparation script, protein directory -- Optional dependency groups: `docking` (vina, meeko), `deeplearning` (chemprop) -- Pre-commit hooks: ruff lint + format -- Chemistry context: full RDKit adapter (descriptors, fingerprints, conformers, scaffolds, Butina clustering, substructure matching), ChemistryService, 7 tools (44 tests) -- Literature context: Semantic Scholar client, PubMed stub, core reference set with 6 key papers, LiteratureService, 2 tools (11 tests) -- Analysis context: ChEMBL loader with parquet caching, Tox21 loader, substructure enrichment (chi-squared), property distributions, AnalysisService, 3 tools (12 tests) -- Prediction context: XGBoost adapter (train/predict with scale_pos_weight, AUROC/AUPRC/F1 metrics, feature importance), model store (joblib save/load/list), Chemprop adapter (guarded), scaffold split, Butina clustering, ensemble, PredictionService, 3 tools (20 tests) -- Simulation context: protein store (5 MRSA targets -- PBP2a, DHPS, DNA Gyrase, MurA, NDM-1), Vina adapter (guarded), RDKit-based ADMET prediction (Lipinski, mutagenic SMARTS alerts, hERG, BBB, hepatotoxicity), knowledge-based resistance assessment (7 mutations across 5 targets, compound class pattern matching), SimulationService, 3 tools (20 tests) +See [docs/roadmap/README.md](docs/roadmap/README.md) for the full phase-by-phase development history with dependency graph. diff --git a/CLAUDE.md b/CLAUDE.md index 783a01a..48085eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,26 +2,19 @@ AGPL-3.0 | Claude Code Hackathon (Feb 10-16, 2026) -## Project Overview +Domain-agnostic scientific discovery platform using Claude as a hypothesis-driven reasoning engine. Pluggable `DomainConfig` + `DomainRegistry` system. Four domains: Molecular Science, Training Science, Nutrition Science, Impact Evaluation. -Ehrlich is a domain-agnostic scientific discovery platform that uses Claude as a hypothesis-driven scientific reasoning engine. Named after Paul Ehrlich, the father of the "magic bullet" concept. - -The engine is domain-agnostic: a pluggable `DomainConfig` + `DomainRegistry` system lets any scientific domain plug in with its own tools, score definitions, and prompt examples. Currently supports four domains: - -- **Molecular Science** -- antimicrobial resistance, drug discovery, toxicology, agricultural biocontrol -- **Training Science** -- exercise physiology, protocol optimization, injury risk assessment, clinical trials -- **Nutrition Science** -- supplement evidence analysis, dietary supplement safety, nutrient profiling, DRI adequacy assessment, drug-nutrient interactions, inflammatory index scoring -- **Impact Evaluation** -- causal analysis of social programs (education, health, sports, employment, housing). Economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmark fetching. See `docs/adr/impact-evaluation-domain.md` +See `README.md` for setup, tools catalog (85 tools), data sources (18 APIs), API endpoints, and demo instructions. ## Architecture DDD monorepo: `server/` (Python 3.12) + `console/` (React 19 / TypeScript / Bun) + `web/` (TanStack Start landing page). -### Multi-Model Architecture (Director-Worker-Summarizer) +### Multi-Model (Director-Worker-Summarizer) ``` Opus 4.6 (Director) -- Formulates hypotheses, designs experiments, evaluates evidence, synthesizes (NO tools) -Sonnet 4.5 (Researcher) -- Executes experiments with 80 tools (parallel: 2 experiments per batch) +Sonnet 4.5 (Researcher) -- Executes experiments with 85 tools (parallel: 2 experiments per batch) Haiku 4.5 (Summarizer) -- Compresses large tool outputs (>2000 chars), PICO decomposition, evidence grading ``` @@ -32,41 +25,17 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 | Context | Location | Purpose | |---------|----------|---------| | kernel | `server/src/ehrlich/kernel/` | Shared primitives (SMILES, Molecule, exceptions) | -| shared | `server/src/ehrlich/shared/` | Cross-cutting ports and value objects (ChemistryPort, Fingerprint, Conformer3D, MolecularDescriptors) | +| shared | `server/src/ehrlich/shared/` | Cross-cutting ports and value objects (ChemistryPort, Fingerprint, Conformer3D) | | literature | `server/src/ehrlich/literature/` | Paper search (Semantic Scholar), references | | chemistry | `server/src/ehrlich/chemistry/` | RDKit cheminformatics | -| analysis | `server/src/ehrlich/analysis/` | Dataset exploration (ChEMBL, PubChem, GtoPdb), enrichment, domain-agnostic causal inference (DiD, PSM, RDD, Synthetic Control) | +| analysis | `server/src/ehrlich/analysis/` | Dataset exploration (ChEMBL, PubChem, GtoPdb), causal inference (DiD, PSM, RDD, SC) | | prediction | `server/src/ehrlich/prediction/` | ML models (Chemprop, XGBoost) | -| simulation | `server/src/ehrlich/simulation/` | Docking, ADMET, resistance, target discovery (RCSB PDB, UniProt, Open Targets), toxicity (EPA CompTox) | -| training | `server/src/ehrlich/training/` | Evidence analysis, protocol comparison, injury risk, training metrics, clinical trials (ClinicalTrials.gov), PubMed literature (E-utilities), exercise database (wger) | -| nutrition | `server/src/ehrlich/nutrition/` | Supplement evidence, supplement labels (NIH DSLD), nutrient data (USDA FoodData), supplement safety (OpenFDA CAERS), drug-nutrient interactions (RxNav), DRI adequacy, nutrient ratios, inflammatory index | -| impact | `server/src/ehrlich/impact/` | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking | +| simulation | `server/src/ehrlich/simulation/` | Docking, ADMET, resistance, targets (RCSB PDB, UniProt, Open Targets), toxicity (EPA CompTox) | +| training | `server/src/ehrlich/training/` | Evidence analysis, protocol comparison, injury risk, clinical trials, PubMed, exercises | +| nutrition | `server/src/ehrlich/nutrition/` | Supplement evidence, labels, nutrients, safety, interactions, adequacy, inflammatory index | +| impact | `server/src/ehrlich/impact/` | Economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking | | investigation | `server/src/ehrlich/investigation/` | Multi-model orchestration + PostgreSQL persistence + domain registry + MCP bridge | -### External Data Sources (18 external + 1 internal) - -| Source | API | Purpose | Auth | -|--------|-----|---------|------| -| ChEMBL | `https://www.ebi.ac.uk/chembl/api/data` | Bioactivity data (any assay type: MIC, IC50, Ki, EC50, Kd) | None | -| Semantic Scholar | `https://api.semanticscholar.org/graph/v1` | Scientific paper search | None | -| RCSB PDB | `https://search.rcsb.org` + `https://data.rcsb.org` | Dynamic protein target discovery by organism/function | None | -| PubChem | `https://pubchem.ncbi.nlm.nih.gov/rest/pug` | Compound search by target/activity/similarity | None | -| EPA CompTox | `https://api-ccte.epa.gov` | Environmental toxicity, bioaccumulation, fate (1M+ chemicals) | Free API key | -| UniProt | `https://rest.uniprot.org` | Protein function, disease associations, GO terms, PDB cross-refs | None | -| Open Targets | `https://api.platform.opentargets.org` | Disease-target associations (GraphQL, scored evidence) | None | -| GtoPdb | `https://www.guidetopharmacology.org/services` | Expert-curated pharmacology (pKi, pIC50, receptor classification) | None | -| ClinicalTrials.gov | `https://clinicaltrials.gov/api/v2` | Registered clinical trials for exercise/training RCTs | None | -| PubMed | `https://eutils.ncbi.nlm.nih.gov/entrez/eutils` | Biomedical literature search with MeSH terms (E-utilities) | None | -| wger | `https://wger.de/api/v2` | Exercise database (muscles, equipment, categories) | None | -| NIH DSLD | `https://api.ods.od.nih.gov/dsld/v9` | Dietary supplement label database (ingredients, amounts) | None | -| USDA FoodData | `https://api.nal.usda.gov/fdc/v1` | Nutrient profiles for foods and supplements | API key | -| OpenFDA CAERS | `https://api.fda.gov/food/event.json` | Supplement adverse event reports (safety monitoring) | None | -| RxNav | `https://rxnav.nlm.nih.gov/REST` | Drug-supplement and drug-nutrient interactions (RxNorm) | None | -| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country (GDP, poverty, education, health) | None | -| WHO GHO | `https://ghoapi.azureedge.net/api/` | Global health statistics by country (life expectancy, mortality, disease burden) | None | -| FRED | `https://api.stlouisfed.org/fred/` | 800K+ US economic time series (GDP, unemployment, CPI, interest rates) | API key | -| Ehrlich tsvector | internal | Full-text search of past investigation findings (PostgreSQL GIN index) | None | - ### Dependency Rules (STRICT) 1. `domain/` has ZERO external deps -- pure Python only (dataclasses, ABC, typing) @@ -85,7 +54,6 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 ```bash cd server uv sync --extra dev # Core + dev deps -# uv sync --extra all --extra dev # All deps (deep learning) uv run uvicorn ehrlich.api.app:create_app --factory --reload --port 8000 uv run pytest --cov=ehrlich --cov-report=term-missing # Tests uv run ruff check src/ tests/ # Lint @@ -93,10 +61,6 @@ uv run ruff format src/ tests/ # Format uv run mypy src/ehrlich/ # Type check ``` -Required env vars: `ANTHROPIC_API_KEY`, `EHRLICH_DATABASE_URL` (PostgreSQL), `EHRLICH_WORKOS_CLIENT_ID`, `EHRLICH_WORKOS_API_KEY`. - -Optional extras: `deeplearning` (chemprop), `all` (everything). - ### Console ```bash @@ -108,19 +72,14 @@ bun run build # vite build (generates route tree + bundles) bun run typecheck # tsc --noEmit (run after build for route types) ``` -Required env vars in `console/.env.local`: `VITE_API_URL` (e.g. `http://localhost:8000/api/v1`), `VITE_WORKOS_CLIENT_ID`, `VITE_WORKOS_REDIRECT_URI` (e.g. `http://localhost:5173/callback`). - -WorkOS dashboard must whitelist the console origin (e.g. `http://localhost:5173`) in **Authentication > Sessions > CORS**. - ### Web (Landing Page) ```bash cd web bun install -bun run dev # Dev server :3000 (vite dev) +bun run dev # Dev server :3000 bun run build # vite build (generates .output/) bun run typecheck # tsc --noEmit -bun run start # node .output/server/index.mjs ``` ## Quality Gates @@ -141,323 +100,86 @@ Types: `feat`, `fix`, `refactor`, `chore`, `docs`, `test` Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, training, nutrition, impact, investigation, api, console, mol, data, ci, docs, infra -## Tools (85 Total) - -### Chemistry (6) -- RDKit cheminformatics, domain-agnostic -- `validate_smiles` -- Validate SMILES string -- `compute_descriptors` -- MW, LogP, TPSA, HBD, HBA, QED, rings -- `compute_fingerprint` -- Morgan (2048-bit) or MACCS (166-bit) -- `tanimoto_similarity` -- Similarity between two molecules -- `generate_3d` -- 3D conformer with MMFF94 optimization -- `substructure_match` -- SMARTS/SMILES substructure search - -### Literature (3) -- Semantic Scholar -- `search_literature` -- Paper search (title, authors, year, DOI, abstract, citations) -- `search_citations` -- Citation chasing (snowballing) via references/citing papers -- `get_reference` -- Curated reference lookup by key or DOI - -### Analysis (12) -- ChEMBL + PubChem + GtoPdb + Domain-Agnostic Causal Inference -- `explore_dataset` -- Load ChEMBL bioactivity data for any organism/target -- `search_bioactivity` -- Flexible ChEMBL query (any assay type: MIC, Ki, EC50, IC50, Kd) -- `search_compounds` -- PubChem compound search by target/activity/similarity -- `search_pharmacology` -- GtoPdb curated receptor/ligand interactions with affinities -- `analyze_substructures` -- Chi-squared enrichment of SMARTS patterns -- `compute_properties` -- Property distributions (active vs inactive) -- `estimate_did` -- Difference-in-differences causal estimation with parallel trends test and threat assessment -- `estimate_psm` -- Propensity score matching with balance diagnostics -- `estimate_rdd` -- Regression discontinuity design (sharp/fuzzy) -- `estimate_synthetic_control` -- Synthetic control method for single treated unit -- `assess_threats` -- Knowledge-based validity threat assessment for causal methods (DiD, PSM, RDD, RCT, IV) -- `compute_cost_effectiveness` -- Cost per unit outcome, ICER calculation - -### Prediction (6) -- XGBoost, domain-agnostic ML -- `train_model` -- Train ML model on SMILES+activity dataset (scaffold-split + random-split metrics, permutation p-value) -- `predict_candidates` -- Score compounds with trained model -- `cluster_compounds` -- Butina structural clustering -- `train_classifier` -- Train binary classifier on tabular feature data (any domain) -- `predict_scores` -- Score samples with trained classifier (any domain) -- `cluster_data` -- Hierarchical clustering on tabular feature data (any domain) - -### Simulation (7) -- Docking, ADMET, targets, toxicity, annotations -- `search_protein_targets` -- RCSB PDB search by organism/function/keyword -- `get_protein_annotation` -- UniProt protein function, disease links, GO terms -- `search_disease_targets` -- Open Targets disease-target associations (scored) -- `dock_against_target` -- Descriptor-based binding energy estimation -- `predict_admet` -- Drug-likeness profiling (absorption, metabolism, toxicity) -- `fetch_toxicity_profile` -- EPA CompTox environmental toxicity data -- `assess_resistance` -- Knowledge-based resistance mutation scoring - -### Training Science (11) -- Exercise physiology and sports medicine -- `search_training_literature` -- Semantic Scholar with training science context -- `analyze_training_evidence` -- Pooled effect sizes, heterogeneity, evidence grading (A-D) -- `compare_protocols` -- Evidence-weighted protocol comparison with composite scoring -- `assess_injury_risk` -- Knowledge-based injury risk scoring (sport, load, history, age) -- `compute_training_metrics` -- ACWR, monotony, strain, session RPE load -- `search_clinical_trials` -- ClinicalTrials.gov exercise/training RCT search -- `search_pubmed_training` -- PubMed E-utilities with MeSH term support -- `search_exercise_database` -- wger exercise database by muscle/equipment/category -- `compute_performance_model` -- Banister fitness-fatigue model (CTL/ATL/TSB) -- `compute_dose_response` -- Dose-response curve from dose-effect data points -- `plan_periodization` -- Evidence-based periodization planning (linear/undulating/block) - -### Nutrition Science (10) -- Supplement evidence, safety, adequacy, interactions -- `search_supplement_evidence` -- Supplement efficacy literature search -- `search_supplement_labels` -- NIH DSLD supplement product ingredient lookup -- `search_nutrient_data` -- USDA FoodData Central nutrient profiles -- `search_supplement_safety` -- OpenFDA CAERS adverse event reports for supplements -- `compare_nutrients` -- Side-by-side nutrient comparison across multiple foods -- `assess_nutrient_adequacy` -- DRI-based nutrient adequacy assessment (EAR/RDA/AI/UL) -- `check_intake_safety` -- Tolerable Upper Intake Level (UL) safety screening -- `check_interactions` -- Drug-supplement and drug-nutrient interaction screening (RxNav) -- `analyze_nutrient_ratios` -- Clinically relevant nutrient ratios (omega-6:3, Ca:Mg, Na:K, Zn:Cu, Ca:P, Fe:Cu) -- `compute_inflammatory_index` -- Simplified Dietary Inflammatory Index (DII) scoring - -### Impact Evaluation (3) -- Social program analysis, economic indicators -- `search_economic_indicators` -- Query economic time series from FRED, World Bank, or WHO GHO -- `fetch_benchmark` -- Get comparison values from international sources (World Bank, WHO GHO) -- `compare_programs` -- Cross-program comparison using existing statistical tests - -### Visualization (17) -- Domain-specific interactive charts -- `render_binding_scatter` -- Scatter plot of compound binding affinities (Recharts) -- `render_admet_radar` -- Radar chart of ADMET/drug-likeness properties (Recharts) -- `render_training_timeline` -- Training load timeline with ACWR danger zones (Recharts) -- `render_muscle_heatmap` -- Anatomical body diagram with muscle activation/risk (Custom SVG) -- `render_forest_plot` -- Forest plot for meta-analysis results (Visx) -- `render_evidence_matrix` -- Hypothesis-by-evidence heatmap (Visx) -- `render_performance_chart` -- Banister fitness-fatigue performance chart (Recharts) -- `render_funnel_plot` -- Funnel plot for publication bias assessment (Visx) -- `render_dose_response` -- Dose-response curve with confidence intervals (Visx) -- `render_nutrient_comparison` -- Grouped bar chart comparing nutrient profiles across foods (Recharts) -- `render_nutrient_adequacy` -- Horizontal bar chart showing % RDA with MAR score (Recharts) -- `render_therapeutic_window` -- Range chart showing EAR/RDA/UL safety zones per nutrient (Visx) -- `render_program_dashboard` -- Multi-indicator KPI dashboard with target tracking (Recharts) -- `render_geographic_comparison` -- Region bar chart with benchmark reference line (Recharts) -- `render_parallel_trends` -- DiD parallel trends chart, treatment vs control over time (Recharts) -- `render_rdd_plot` -- Regression discontinuity scatter with cutoff line and fitted curves (Visx) -- `render_causal_diagram` -- DAG showing treatment, outcome, confounders, and mediators (Visx) - -### Statistics (2) -- Domain-agnostic hypothesis testing (scipy.stats) -- `run_statistical_test` -- Compare two numeric groups (auto-selects t-test/Welch/Mann-Whitney, returns p-value, Cohen's d, 95% CI) -- `run_categorical_test` -- Test contingency tables (auto-selects Fisher's exact/chi-squared, returns p-value, odds ratio/Cramer's V) - -### Investigation (8) -- Hypothesis lifecycle + self-referential + data upload -- `propose_hypothesis` -- Register testable hypothesis -- `design_experiment` -- Plan experiment with tool sequence -- `evaluate_hypothesis` -- Assess outcome (supported/refuted/revised) -- `record_finding` -- Record finding linked to hypothesis + evidence_type -- `record_negative_control` -- Validate model with known-inactive compounds -- `search_prior_research` -- Search Ehrlich's own past findings via full-text search -- `query_uploaded_data` -- Query user-uploaded files (CSV/Excel filtering, PDF text retrieval) -- `conclude_investigation` -- Final summary with ranked candidates - ## Key Patterns ### Scientific Methodology -- **Hypothesis-driven investigation loop**: formulate hypotheses (with predictions, criteria, scope, prior confidence) -> design structured experiment protocols -> execute tools with methodology guidance -> evaluate with scientific methodology -> revise/reject -> synthesize. See `docs/scientific-methodology.md` for full details -- **Hypothesis model**: Based on Popper (falsifiability), Platt (strong inference), Feynman (compute consequences), Bayesian updating. Each hypothesis carries: prediction, null_prediction, success_criteria, failure_criteria, scope, hypothesis_type, prior_confidence, certainty_of_evidence. Evaluation uses 8-tier evidence hierarchy, effect size thresholds, Bayesian prior-to-posterior updating, contradiction resolution, and convergence checks -- **Experiment protocols**: Based on Fisher (1935), Platt (1964), Cohen (1988), Saltelli (2008), OECD (2007), Tropsha (2010). Each experiment carries: independent_variable, dependent_variable, controls, confounders, analysis_plan, success_criteria, failure_criteria. Director designs with 5-principle methodology (VARIABLES, CONTROLS, CONFOUNDERS, ANALYSIS PLAN, SENSITIVITY). Researcher executes with 5-principle methodology (SENSITIVITY, APPLICABILITY DOMAIN, UNCERTAINTY, VERIFICATION, NEGATIVE RESULTS) -- **Statistical testing in experiments** -- Director plans statistical tests (test type, alpha, effect size) in experiment design via optional `statistical_test_plan`; Researcher executes `run_statistical_test` (continuous) or `run_categorical_test` (categorical) after gathering comparison data; results recorded as findings with evidence_type based on significance -- **Evidence-linked findings**: every finding references a `hypothesis_id` + `evidence_type` (supporting/contradicting/neutral) -- **Controls**: negative controls validate model predictions with known-inactive compounds (`NegativeControl` entity); positive controls validate pipeline with known-active compounds (`PositiveControl` entity, correctly_classified >= 0.5, quality: sufficient/marginal/insufficient) -- **Validation metrics**: Z'-factor assay quality (Zhang et al., 1999), permutation significance (Y-scrambling, p-value), scaffold-split vs random-split comparison; controls scored through trained models; `ValidationMetricsComputed` event -- **Literature survey**: PICO decomposition + domain classification merged into single Haiku call; structured search with citation chasing (snowballing); GRADE-adapted body-of-evidence grading via Haiku; AMSTAR-2-adapted self-assessment; `LiteratureSurveyCompleted` event carries PICO, search stats, grade -- **User-guided steering**: `HypothesisApprovalRequested` event pauses orchestrator after formulation; user approves/rejects via `POST /investigate/{id}/approve`; `REJECTED` status; 5-min auto-approve timeout -- **Synthesis**: GRADE certainty grading (5 downgrading + 3 upgrading domains), priority tiers (1-4), structured limitations taxonomy, knowledge gap analysis, follow-up experiment recommendations +- **Hypothesis-driven investigation loop**: formulate -> design experiment -> execute tools -> evaluate -> revise/reject -> synthesize. See `docs/scientific-methodology.md` +- **Hypothesis model**: Popper + Platt + Feynman + Bayesian updating. Fields: prediction, null_prediction, success/failure_criteria, scope, hypothesis_type, prior_confidence, certainty_of_evidence. Evaluation: 8-tier evidence hierarchy, effect sizes, Bayesian updating, convergence checks +- **Experiment protocols**: Fisher/Platt/Cohen/Saltelli/OECD/Tropsha. Fields: independent/dependent_variable, controls, confounders, analysis_plan, success/failure_criteria. Director designs with 5 principles (VARIABLES, CONTROLS, CONFOUNDERS, ANALYSIS PLAN, SENSITIVITY) +- **Statistical testing**: Director plans tests in `statistical_test_plan`; Researcher executes `run_statistical_test`/`run_categorical_test`; findings recorded with evidence_type based on significance +- **Controls**: negative controls (known-inactive compounds), positive controls (correctly_classified >= 0.5) +- **Validation**: Z'-factor assay quality, permutation significance (Y-scrambling), scaffold-split vs random-split +- **Literature survey**: PICO decomposition + domain classification (Haiku); citation chasing; GRADE-adapted grading; AMSTAR-2 self-assessment +- **User-guided steering**: `HypothesisApprovalRequested` pauses orchestrator; user approves/rejects via `POST /investigate/{id}/approve`; 5-min auto-approve timeout +- **Synthesis**: GRADE certainty grading, priority tiers (1-4), structured limitations, knowledge gap analysis ### System Patterns -- Each bounded context has: `domain/`, `application/`, `infrastructure/`, `tools.py` -- Domain entities use `@dataclass` with `__post_init__` validation; repository interfaces are ABCs in `domain/repository.py`; infrastructure adapters implement those ABCs -- Tool functions in `tools.py` are the boundary between Claude and application services -- **Parallel researchers**: `_run_experiment_batch()` uses `asyncio.Queue` to merge events from 2 concurrent experiments; three-layer differentiation prevents duplicate work: (1) orthogonal hypothesis formulation via Director prompt, (2) sibling-aware experiment design with `` context, (3) researcher sibling context via `` tag -- **Context compaction**: `_build_prior_context()` compresses completed hypotheses into XML summary for Director -- **Prompt engineering**: XML-tagged instructions (``, ``, ``, ``), multishot examples (2 per Director prompt), tool usage examples for Researcher -- **ToolCache**: in-memory TTL-based caching for deterministic and API tool results -- **Event persistence**: all SSE events stored in PostgreSQL `events` table; completed investigations replay full timeline on page reload -- **CostTracker**: per-model token usage with tiered pricing; `CostUpdate` event yields cost snapshots after each phase/batch -- **Structured outputs**: Director calls use `output_config` with JSON schemas (6 schemas in `domain/schemas.py`) to guarantee valid JSON; eliminates parsing fallbacks -- **Director streaming**: `_director_call()` is an async generator using `stream_message()`; yields `Thinking` events in real time as tokens arrive; Researcher and Summarizer remain non-streaming -- All external API clients follow same pattern: `httpx.AsyncClient`, retry with exponential backoff, structured error handling -- **Authentication**: WorkOS JWT middleware (`api/auth.py`); JWKS verification via `PyJWKClient`; `get_current_user` (header) and `get_current_user_sse` (header or `?token=` query param for EventSource); `get_optional_user` for public routes -- **Credit system**: director tier selection (haiku=1cr, sonnet=3cr, opus=5cr); credits deducted on investigation start; refunded on failure; `credit_transactions` table for audit trail; `GET /credits/balance` endpoint -- **BYOK (Bring Your Own Key)**: `X-Anthropic-Key` header pass-through; bypasses credit system; API key forwarded to `AnthropicClientAdapter`; checked on both `POST /investigate` and `GET /investigate/{id}/stream` -- **Document upload**: CSV/XLSX/PDF upload via `POST /upload` (multipart/form-data); `FileProcessor` parses to `UploadedFile` domain entity; tabular files get summary stats + sample rows; PDFs get text extraction (pymupdf); uploaded data injected into Director/Researcher prompts as `` XML block; `query_uploaded_data` tool intercepted by `ToolDispatcher` for pandas-based filtering +- Each bounded context: `domain/`, `application/`, `infrastructure/`, `tools.py` +- Domain entities: `@dataclass` with `__post_init__` validation; repositories: ABCs in `domain/repository.py` +- `tools.py` is the boundary between Claude and application services +- **Parallel researchers**: `_run_experiment_batch()` uses `asyncio.Queue` for 2 concurrent experiments; three-layer differentiation prevents duplicates +- **Context compaction**: `_build_prior_context()` compresses completed hypotheses for Director +- **Prompt engineering**: XML-tagged instructions, multishot examples (2 per Director prompt), tool usage examples for Researcher +- **ToolCache**: in-memory TTL cache (deterministic: forever, API: 24h-7d) +- **Event persistence**: PostgreSQL `events` table; completed investigations replay full timeline +- **Structured outputs**: Director uses `output_config` with 6 JSON schemas (`domain/schemas.py`) +- **Director streaming**: `_director_call()` async generator with `stream_message()`; Researcher/Summarizer non-streaming +- External API clients: `httpx.AsyncClient`, retry with exponential backoff +- **Auth**: WorkOS JWT middleware; JWKS verification; header + query param auth for SSE +- **Credits**: director tier (haiku=1, sonnet=3, opus=5); deducted on start, refunded on failure +- **BYOK**: `X-Anthropic-Key` header pass-through; bypasses credits +- **Document upload**: CSV/XLSX/PDF via `POST /upload`; `FileProcessor` -> `UploadedFile`; injected as `` XML ### Integration Patterns -- **Domain configuration**: `DomainConfig` defines per-domain tool tags, score definitions, prompt examples; `DomainRegistry` auto-detects domain; `DomainDetected` SSE event sends display config to frontend -- **Domain tool examples**: each `DomainConfig` includes `tool_examples` in `experiment_examples` showing realistic tool chaining patterns; ensures Researcher (Sonnet 4.5) has usage guidance for all domain-specific tools -- **Multi-domain investigations**: `DomainRegistry.detect()` returns `list[DomainConfig]`; `merge_domain_configs()` creates synthetic merged config with union of tool_tags, concatenated score_definitions, joined prompt examples -- **Tool tagging**: Tools tagged with frozenset domain tags (chemistry, analysis, prediction, simulation, training, clinical, nutrition, safety, impact, causal, literature, visualization); investigation control tools are universal (no tags); researcher sees only domain-relevant tools -- **Self-referential research**: `search_prior_research` queries PostgreSQL `tsvector` + GIN index of past findings; indexed on completion; intercepted in orchestrator `_dispatch_tool()` and routed to repository; "ehrlich" source type links to past investigations -- **MCP bridge**: Optional `MCPBridge` connects to external MCP servers (e.g. Excalidraw); tools registered dynamically via `ToolRegistry.register_mcp_tools()`; lifecycle managed by orchestrator; enabled via `EHRLICH_MCP_EXCALIDRAW=true` -- **SSE streaming**: 20 event types for real-time updates; reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) -- **Phase progress**: `PhaseChanged` event tracks 6 orchestrator phases (Classification & PICO -> Literature Survey -> Formulation -> Hypothesis Testing -> Negative Controls -> Synthesis); frontend renders 6-segment progress bar -- **Citation provenance**: `source_type` + `source_id` on findings, rendered as clickable badges linking to ChEMBL, PDB, DOI, PubChem, UniProt, Open Targets -- **Data configuration**: protein targets via YAML (`data/targets/*.yaml`) + dynamic RCSB PDB discovery; resistance data via YAML (`data/resistance/*.yaml`) -- **Flexible data loading**: ChEMBL queries accept any assay type (MIC, Ki, EC50, etc.), not hardcoded +- **Domain config**: `DomainConfig` with tool_tags, score_definitions, prompt examples; `DomainRegistry` auto-detects; `DomainDetected` SSE event +- **Multi-domain**: `merge_domain_configs()` creates synthetic config from multiple domains +- **Tool tagging**: frozenset domain tags; investigation tools are universal; researcher sees only domain-relevant tools +- **Self-referential**: `search_prior_research` queries PostgreSQL tsvector + GIN index +- **MCP bridge**: Optional external MCP servers; dynamic tool registration; `EHRLICH_MCP_EXCALIDRAW=true` +- **SSE**: 20 event types; reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) +- **Phase progress**: 6 phases (Classification & PICO -> Literature Survey -> Formulation -> Hypothesis Testing -> Negative Controls -> Synthesis) +- **Citation provenance**: `source_type` + `source_id` on findings (ChEMBL, PDB, DOI, PubChem, UniProt, Open Targets) ### UI Patterns -- **Visualization pipeline**: 17 viz tools return `VisualizationPayload` JSON; orchestrator intercepts via `_maybe_viz_event()` and emits `VisualizationRendered` SSE event; `VizRegistry` maps `viz_type` to lazy-loaded React component; `VisualizationPanel` renders all charts -- **LiveLabViewer**: 3Dmol.js scene driven by SSE events -- protein targets load, ligands dock, candidates color by score. SSE event mapping: `dock_against_target` -> ligand in binding pocket, `predict_candidates` -> color by probability, `completed` -> top candidates glow. Interactive rotate/zoom/click, split-pane with Timeline -- **React Flow diagrams**: `@xyflow/react` node graph with custom `InvestigationNode` and `AnnotationNode` types; status-colored nodes (proposed=gray, testing=blue, supported=green, refuted=red, revised=orange); dashed edges for revisions, solid smoothstep for links; read-only with zoom/pan/minimap; lazy-loaded (~188KB chunk) -- **Investigation report**: 8 sections (Research Question, Executive Summary, Hypotheses & Outcomes, Methodology, Key Findings, Candidate Molecules, Model Validation, Cost & Performance); shown post-completion with full audit trail; markdown export via client-side generation -- **Molecule visualization**: server-side 2D SVG (RDKit `rdMolDraw2D`), 3Dmol.js for 3D/docking; `CandidateTable` shows thumbnails with expandable detail (3D viewer + properties + Lipinski badge) -- **Molecule API**: `/molecule/depict` (SVG, cached 24h), `/molecule/conformer`, `/molecule/descriptors`, `/molecule/targets` -- **Investigation templates**: 9 cross-domain prompts (4 molecular + 2 training + 1 nutrition + 2 impact) on home page via `TemplateCards` with domain badges -- **Candidate comparison**: side-by-side scoring view for 2-4 candidates with best-in-group highlighting -- **HypothesisBoard**: kanban-style card grid showing hypothesis status with expandable confidence bars -- **Hypothesis approval**: `HypothesisApprovalCard` renders in main content area (not status bar) for maximum prominence during approval gate -- **Home page**: hero section with stats, collapsible BYOK toggle, SectionHeader-driven sections, empty state for new users -- **Design system**: OKLCH hue 155 (green) brand primary; `rounded-sm` buttons, `rounded-md` cards; `hover:opacity-90` (no glow); Space Grotesk + JetBrains Mono fonts; shared `SectionHeader` component for consistent section headings; VIZ_COLORS aligned to hue 155 -- **Layout**: conditional `hideHeader` on AppLayout for investigation/compare pages (own header); consistent `max-w-[1200px]` across all pages +- **Viz pipeline**: 17 viz tools -> `VisualizationPayload` -> `VisualizationRendered` SSE -> `VizRegistry` lazy-loads React component +- **LiveLabViewer**: 3Dmol.js SSE-driven scene (protein load, ligand dock, candidate score coloring) +- **React Flow**: `@xyflow/react` with custom node types; status-colored (proposed=gray, testing=blue, supported=green, refuted=red) +- **Design system**: OKLCH hue 155 (green); `rounded-sm` buttons, `rounded-md` cards; Space Grotesk + JetBrains Mono +- **Layout**: conditional `hideHeader` for investigation/compare pages; `max-w-[1200px]` - TanStack Router file-based routing; `ErrorBoundary` wraps LiveLabViewer and InvestigationDiagram -- Toast notifications via `sonner` (dark-themed OKLCH colors); custom scrollbar CSS (8px webkit + Firefox thin) -- `InvestigationCompleted` event carries `findings[]`, `hypotheses[]`, `negative_controls[]`, `prompt` for replay hydration ## Key Files (Server) All paths relative to `server/src/ehrlich/`. -### shared/ - -| File | Purpose | -|------|---------| -| `chemistry_port.py` | `ChemistryPort` ABC -- cross-cutting interface for chemistry operations | -| `fingerprint.py` | `Fingerprint` value object | -| `descriptors.py` | `MolecularDescriptors` value object | -| `conformer.py` | `Conformer3D` value object | - -### literature/ - -| File | Purpose | -|------|---------| -| `infrastructure/semantic_scholar.py` | Semantic Scholar paper search with retry | - -### chemistry/ - -| File | Purpose | -|------|---------| -| `infrastructure/rdkit_adapter.py` | RDKit adapter including `depict_2d` (SVG) | -| `application/chemistry_service.py` | Thin wrapper over adapter | - -### analysis/ - -| File | Purpose | -|------|---------| -| `domain/statistics.py` | `StatisticalResult` frozen dataclass | -| `domain/causal.py` | `ThreatToValidity`, `CausalEstimate`, `CostEffectivenessResult` frozen dataclasses | -| `domain/causal_ports.py` | `DiDEstimatorPort`, `PSMEstimatorPort`, `RDDEstimatorPort`, `SyntheticControlPort` ABCs | -| `domain/evidence_standards.py` | WWC evidence tiers, CONEVAL MIR levels, CREMAA criteria, `classify_evidence_tier()` | -| `application/statistics_service.py` | `StatisticsService` -- scipy.stats hypothesis testing (t-test, Welch, Mann-Whitney, chi-squared, Fisher) | -| `application/causal_service.py` | `CausalService` -- DiD, PSM, RDD, Synthetic Control estimation, threat assessment, cost-effectiveness | -| `infrastructure/chembl_loader.py` | ChEMBL bioactivity loader (flexible assay types) | -| `infrastructure/pubchem_client.py` | PubChem PUG REST compound search | -| `infrastructure/gtopdb_client.py` | GtoPdb REST client for curated pharmacology | -| `infrastructure/did_estimator.py` | DiD estimator: effect size, SE, p-value, parallel trends, automated threat assessment | -| `infrastructure/psm_estimator.py` | Propensity score matching with balance diagnostics | -| `infrastructure/rdd_estimator.py` | Regression discontinuity design (sharp/fuzzy) estimator | -| `infrastructure/synthetic_control_estimator.py` | Synthetic control method for single treated unit | - -### prediction/ - -| File | Purpose | -|------|---------| -| `domain/ports.py` | `FeatureExtractor`, `DataSplitter`, `Clusterer` ABCs (domain-agnostic ML ports) | -| `domain/prediction_result.py` | `PredictionResult` frozen dataclass (generic `identifier: str`) | -| `application/prediction_service.py` | Generic ML service: train, predict, cluster via injected ports | -| `infrastructure/molecular_adapters.py` | `MolecularFeatureExtractor`, `ScaffoldSplitter`, `MolecularClusterer` (RDKit) | -| `infrastructure/generic_adapters.py` | `RandomSplitter`, `DistanceClusterer` (scipy hierarchical) | -| `infrastructure/xgboost_adapter.py` | XGBoost training, prediction, permutation test | -| `tools.py` | 6 prediction tools (3 molecular + 3 generic) | - -### simulation/ - -| File | Purpose | -|------|---------| -| `infrastructure/rcsb_client.py` | RCSB PDB Search + Data API for target discovery | -| `infrastructure/comptox_client.py` | EPA CompTox CTX API for environmental toxicity | -| `infrastructure/protein_store.py` | YAML-based protein target store + RCSB PDB fallback | -| `infrastructure/uniprot_client.py` | UniProt REST client for protein annotations | -| `infrastructure/opentargets_client.py` | Open Targets GraphQL client for disease-target associations | - -### training/ - -| File | Purpose | -|------|---------| -| `tools.py` | 10 training science tools | -| `domain/entities.py` | EvidenceGrade, StudyResult, EvidenceAnalysis, TrainingProtocol, ProtocolComparison, InjuryRiskAssessment, WorkloadMetrics, ClinicalTrial, PubMedArticle, Exercise, PerformanceModelPoint, DoseResponsePoint | -| `domain/repository.py` | ClinicalTrialRepository, PubMedRepository, ExerciseRepository ABCs | -| `application/training_service.py` | TrainingService (evidence analysis, protocol comparison, injury risk, metrics, clinical trials, PubMed, exercises, performance model, dose-response) | -| `infrastructure/clinicaltrials_client.py` | ClinicalTrials.gov v2 API | -| `infrastructure/pubmed_client.py` | PubMed E-utilities API (ESearch + EFetch) | -| `infrastructure/wger_client.py` | wger exercise database REST API | - -### nutrition/ - | File | Purpose | |------|---------| -| `tools.py` | 10 nutrition science tools | -| `domain/entities.py` | IngredientEntry, SupplementLabel, NutrientEntry, NutrientProfile, AdverseEvent, DrugInteraction, NutrientRatio, AdequacyResult | -| `domain/repository.py` | SupplementRepository, NutrientRepository, AdverseEventRepository, InteractionRepository ABCs | -| `domain/dri.py` | Dietary Reference Intake (DRI) tables -- EAR, RDA, AI, UL for ~30 nutrients by age/sex | -| `application/nutrition_service.py` | NutritionService (supplement evidence, labels, nutrients, safety, comparison, adequacy, interactions, ratios, inflammatory index) | -| `infrastructure/dsld_client.py` | NIH DSLD supplement label database | -| `infrastructure/fooddata_client.py` | USDA FoodData Central nutrient profiles | -| `infrastructure/openfda_client.py` | OpenFDA CAERS adverse event reporting | -| `infrastructure/rxnav_client.py` | RxNav drug-nutrient interaction API | - -### impact/ - -| File | Purpose | -|------|---------| -| `tools.py` | 3 impact evaluation tools (economic indicators, benchmarks, program comparison) | -| `domain/entities.py` | DataPoint, Program, Indicator, Benchmark, EconomicSeries, HealthIndicator | -| `domain/repository.py` | EconomicDataRepository, HealthDataRepository, DevelopmentDataRepository ABCs | -| `application/impact_service.py` | ImpactService (economic indicators, benchmarks, program comparison) | -| `infrastructure/worldbank_client.py` | World Bank API v2 client for development indicators | -| `infrastructure/who_client.py` | WHO GHO API client for health statistics | -| `infrastructure/fred_client.py` | FRED API client for US economic time series | - -### investigation/ - -| File | Purpose | -|------|---------| -| `application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop, upload context injection) | -| `application/diagram_builder.py` | Excalidraw evidence synthesis diagram generation | -| `application/tool_dispatcher.py` | Tool execution dispatcher with caching, `search_prior_research` and `query_uploaded_data` interception | -| `application/researcher_executor.py` | Single researcher experiment executor with tool loop | -| `application/batch_executor.py` | Parallel batch experiment executor (2 concurrent researchers) | -| `application/cost_tracker.py` | Per-model cost tracking with tiered pricing | -| `application/tool_cache.py` | In-memory TTL cache for tool results | -| `application/tool_registry.py` | `ToolRegistry` with domain tag filtering | -| `application/file_processor.py` | CSV/XLSX/PDF parser producing `UploadedFile` domain entities | -| `application/prompts.py` | Domain-adaptive prompts (PICO, literature survey, director, researcher, summarizer, uploaded data context) | -| `domain/schemas.py` | JSON schemas for structured output responses (6 schemas: PICO, Formulation, Experiment Design, Evaluation, Synthesis, Literature Grading) | -| `domain/hypothesis.py` | Hypothesis entity + HypothesisStatus + HypothesisType enums | -| `domain/experiment.py` | Experiment entity + ExperimentStatus enum | -| `domain/candidate.py` | Candidate dataclass (identifier, scores, attributes) | -| `domain/negative_control.py` | NegativeControl frozen dataclass | -| `domain/positive_control.py` | PositiveControl frozen dataclass | -| `domain/validation.py` | Z'-factor assay quality metric | -| `domain/events.py` | 20 domain events | -| `domain/visualization.py` | `VisualizationPayload` frozen dataclass | -| `domain/repository.py` | InvestigationRepository ABC | -| `domain/domain_config.py` | `DomainConfig` + `ScoreDefinition` + `merge_domain_configs()` | -| `domain/domain_registry.py` | `DomainRegistry` (register, multi-detect, lookup) | -| `domain/domains/molecular.py` | `MOLECULAR_SCIENCE` config | -| `domain/domains/training.py` | `TRAINING_SCIENCE` config | -| `domain/domains/nutrition.py` | `NUTRITION_SCIENCE` config | -| `domain/domains/impact.py` | `IMPACT_EVALUATION` config | -| `domain/uploaded_file.py` | `UploadedFile`, `TabularData`, `DocumentData` frozen dataclasses | -| `domain/mcp_config.py` | `MCPServerConfig` frozen dataclass | -| `tools.py` | `query_uploaded_data` tool (intercepted by ToolDispatcher) | -| `tools_viz.py` | 17 visualization tools | -| `infrastructure/repository.py` | PostgreSQL persistence (asyncpg) + tsvector findings search + user/credit management + uploaded files | -| `infrastructure/anthropic_client.py` | Anthropic API adapter with retry, streaming, and structured outputs | -| `infrastructure/mcp_bridge.py` | MCP client bridge (stdio/SSE/streamable_http) | +| `shared/chemistry_port.py` | `ChemistryPort` ABC | +| `chemistry/infrastructure/rdkit_adapter.py` | RDKit adapter (only RDKit import point) | +| `analysis/application/causal_service.py` | DiD, PSM, RDD, Synthetic Control, threat assessment, cost-effectiveness | +| `analysis/domain/causal_ports.py` | Causal estimator port ABCs | +| `analysis/domain/evidence_standards.py` | WWC tiers, CONEVAL MIR, CREMAA criteria | +| `prediction/application/prediction_service.py` | Generic ML: train, predict, cluster via injected ports | +| `prediction/domain/ports.py` | `FeatureExtractor`, `DataSplitter`, `Clusterer` ABCs | +| `training/application/training_service.py` | Evidence, protocols, injury risk, metrics, clinical trials, PubMed, exercises | +| `nutrition/application/nutrition_service.py` | Supplements, labels, nutrients, safety, adequacy, interactions, ratios | +| `nutrition/domain/dri.py` | DRI tables (EAR, RDA, AI, UL by age/sex) | +| `impact/application/impact_service.py` | Economic indicators, benchmarks, program comparison | +| `investigation/application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop) | +| `investigation/application/tool_dispatcher.py` | Tool execution with caching, `search_prior_research` and `query_uploaded_data` interception | +| `investigation/application/researcher_executor.py` | Single researcher experiment executor | +| `investigation/application/batch_executor.py` | Parallel batch executor (2 concurrent) | +| `investigation/application/prompts.py` | All domain-adaptive prompts | +| `investigation/domain/schemas.py` | 6 JSON schemas for structured outputs | +| `investigation/domain/domain_config.py` | `DomainConfig` + `ScoreDefinition` + `merge_domain_configs()` | +| `investigation/domain/domain_registry.py` | `DomainRegistry` (register, multi-detect, lookup) | +| `investigation/domain/domains/*.py` | Per-domain configs (molecular, training, nutrition, impact) | +| `investigation/infrastructure/repository.py` | PostgreSQL persistence + tsvector search + credits + uploads | +| `investigation/infrastructure/anthropic_client.py` | Anthropic API adapter (retry, streaming, structured outputs) | ## Key Files (API) @@ -465,99 +187,34 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| -| `auth.py` | WorkOS JWT middleware (JWKS verification, header + query param auth for SSE) | -| `routes/investigation.py` | REST + SSE endpoints, 85-tool registry, domain registry, MCP bridge, credit system, BYOK | -| `routes/upload.py` | `POST /upload` multipart file upload (CSV/XLSX/PDF), preview response, pending upload store | -| `routes/methodology.py` | GET /methodology: phases, domains, tools, data sources, models | -| `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets endpoints | -| `routes/stats.py` | GET /stats: aggregate counts | +| `auth.py` | WorkOS JWT middleware (JWKS, header + query param) | +| `routes/investigation.py` | REST + SSE, 85-tool registry, domain registry, credits, BYOK | +| `routes/upload.py` | File upload (CSV/XLSX/PDF) | +| `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets | | `sse.py` | Domain event to SSE conversion (20 types) | ## Key Files (Console) All paths relative to `console/src/`. -### Investigation components - | File | Purpose | |------|---------| | `features/investigation/components/LiveLabViewer.tsx` | 3Dmol.js SSE-driven molecular scene | | `features/investigation/components/InvestigationDiagram.tsx` | Lazy-loaded React Flow wrapper | -| `features/investigation/components/DiagramRenderer.tsx` | React Flow renderer with custom node types, minimap, controls | | `features/investigation/components/InvestigationReport.tsx` | 8-section structured report | | `features/investigation/components/CandidateTable.tsx` | Thumbnail grid with expandable rows | -| `features/investigation/components/CandidateDetail.tsx` | 2D + 3D views + property card + Lipinski badge | -| `features/investigation/components/CandidateComparison.tsx` | Side-by-side scoring comparison | -| `features/investigation/components/HypothesisBoard.tsx` | Kanban-style hypothesis status grid | -| `features/investigation/components/HypothesisCard.tsx` | Expandable hypothesis card with confidence bar | +| `features/investigation/components/HypothesisBoard.tsx` | Kanban hypothesis status grid | | `features/investigation/components/HypothesisApprovalCard.tsx` | Approve/reject before testing | -| `features/investigation/components/ActiveExperimentCard.tsx` | Live experiment activity card | -| `features/investigation/components/CompletionSummaryCard.tsx` | Post-completion summary card with expandable cost breakdown | -| `features/investigation/components/NegativeControlPanel.tsx` | Negative control validation table | -| `features/investigation/components/TemplateCards.tsx` | 9 cross-domain research prompt templates | -| `features/investigation/components/ThreatAssessment.tsx` | Validity threat assessment panel with severity badges | -| `features/investigation/components/FileUpload.tsx` | Drag-and-drop file upload (CSV/XLSX/PDF) with progress and preview | -| `features/investigation/components/DataPreview.tsx` | Compact inline preview cards for uploaded tabular/document data | -| `features/investigation/components/PromptInput.tsx` | Controlled prompt input with director tier selector pills and file upload | -| `features/investigation/components/BYOKSettings.tsx` | API key management for BYOK users | - -### Investigation lib + hooks - -| File | Purpose | -|------|---------| +| `features/investigation/components/PromptInput.tsx` | Prompt input with director tier selector + file upload | +| `features/investigation/components/ThreatAssessment.tsx` | Validity threat panel with severity badges | +| `features/investigation/components/FileUpload.tsx` | Drag-and-drop upload (CSV/XLSX/PDF) | | `features/investigation/lib/scene-builder.ts` | SSE events to 3Dmol.js operations | -| `features/investigation/lib/diagram-builder.ts` | Hypotheses/experiments/findings to React Flow nodes + edges | -| `features/investigation/lib/export-markdown.ts` | Client-side markdown report generation | -| `features/investigation/hooks/use-methodology.ts` | React hook for methodology data | -| `features/investigation/hooks/use-credits.ts` | Credit balance hook (TanStack Query) | -| `features/investigation/hooks/use-upload.ts` | File upload mutation hook (TanStack Query) | - -### Molecule components - -| File | Purpose | -|------|---------| -| `features/molecule/components/MolViewer2D.tsx` | Server-side SVG viewer | -| `features/molecule/components/MolViewer3D.tsx` | 3Dmol.js WebGL conformer viewer | -| `features/molecule/components/DockingViewer.tsx` | 3Dmol.js protein+ligand overlay | - -### Visualization system - -| File | Purpose | -|------|---------| -| `features/visualization/VizRegistry.tsx` | Lazy-loaded component registry (viz_type to React component) | -| `features/visualization/VisualizationPanel.tsx` | Grid layout for multiple visualizations | -| `features/visualization/theme.ts` | OKLCH color tokens for charts | -| `features/visualization/charts/BindingScatter.tsx` | Recharts ScatterChart | -| `features/visualization/charts/ADMETRadar.tsx` | Recharts RadarChart | -| `features/visualization/charts/TrainingTimeline.tsx` | Recharts ComposedChart with ACWR + Brush | -| `features/visualization/charts/ForestPlot.tsx` | Visx custom forest plot | -| `features/visualization/charts/EvidenceMatrix.tsx` | Visx HeatmapRect with OKLCH scale | -| `features/visualization/charts/PerformanceChart.tsx` | Recharts ComposedChart for Banister fitness-fatigue model | -| `features/visualization/charts/FunnelPlot.tsx` | Visx funnel plot for publication bias assessment | -| `features/visualization/charts/DoseResponseCurve.tsx` | Visx dose-response curve with confidence band | -| `features/visualization/charts/NutrientComparison.tsx` | Recharts grouped BarChart for food nutrient comparison | -| `features/visualization/charts/NutrientAdequacy.tsx` | Recharts horizontal BarChart showing % RDA with MAR score | -| `features/visualization/charts/TherapeuticWindow.tsx` | Visx custom range chart with EAR/RDA/UL safety zones | -| `features/visualization/charts/ProgramDashboard.tsx` | Recharts horizontal BarChart with target tracking | -| `features/visualization/charts/GeographicComparison.tsx` | Recharts BarChart with benchmark ReferenceLine | -| `features/visualization/charts/ParallelTrends.tsx` | Recharts ComposedChart for DiD treatment vs control | -| `features/visualization/charts/RDDPlot.tsx` | Visx regression discontinuity scatter with cutoff and fitted curves | -| `features/visualization/charts/CausalDiagram.tsx` | Visx DAG showing treatment, outcome, confounders, mediators | -| `features/visualization/anatomy/BodyDiagram.tsx` | Custom SVG anatomical body diagram | -| `features/visualization/anatomy/body-paths.ts` | SVG path data for front/back views | -| `features/visualization/anatomy/color-scale.ts` | OKLCH intensity color interpolation | - -### Shared + routes - -| File | Purpose | -|------|---------| -| `shared/hooks/use-auth.ts` | WorkOS auth hook wrapper (AuthKitProvider integration) | -| `shared/lib/api.ts` | Authenticated fetch with BYOK `X-Anthropic-Key` header support | -| `shared/components/ErrorBoundary.tsx` | Error boundary for LiveLabViewer and InvestigationDiagram | -| `shared/components/ui/SectionHeader.tsx` | Shared section header with icon, title, description (border-l-2 accent) | -| `shared/components/ui/Toaster.tsx` | Sonner toast wrapper with dark OKLCH theme | -| `routes/methodology.tsx` | Methodology page | -| `routes/callback.tsx` | WorkOS OAuth callback (waits for auth, then redirects home) | +| `features/investigation/lib/diagram-builder.ts` | Hypotheses/experiments/findings to React Flow nodes | +| `features/visualization/VizRegistry.tsx` | Lazy-loaded chart component registry | +| `features/visualization/VisualizationPanel.tsx` | Grid layout for multiple charts | +| `features/visualization/theme.ts` | OKLCH color tokens | +| `shared/hooks/use-auth.ts` | WorkOS auth wrapper | +| `shared/lib/api.ts` | Authenticated fetch with BYOK support | ## Key Files (Web Landing Page) @@ -565,26 +222,9 @@ All paths relative to `web/src/`. | File | Purpose | |------|---------| -| `routes/__root.tsx` | Root layout with shellComponent, HeadContent, SEO meta, self-hosted font preloads | -| `routes/index.tsx` | Landing page with lazy-loaded components (React.lazy + Suspense) | -| `components/Nav.tsx` | Fixed navbar with scroll progress bar + mobile menu | -| `components/Footer.tsx` | Minimal footer with links and license | -| `components/SectionHeader.tsx` | Mono label with left border accent | -| `components/Hero.tsx` | Bottom-third hero with MolecularNetwork, stats badges, CTAs | -| `components/MolecularNetwork.tsx` | 3D rotating node graph (Canvas 2D, mouse repulsion, CSS mask edge fade) | -| `components/HowItWorks.tsx` | 6-phase methodology timeline with vertical connecting line | -| `components/ConsolePreview.tsx` | Browser-frame mockups (timeline, hypothesis board, candidates, radar) | -| `components/Architecture.tsx` | Director-Worker-Summarizer model cards, dot grid bg, amber glow | -| `components/Domains.tsx` | 4 domain cards with tool counts, multi-domain callout | -| `components/Visualizations.tsx` | 4 visualization category cards with tech labels | -| `components/DataSources.tsx` | 19 source cards, surface bg with teal glow | -| `components/WhoItsFor.tsx` | 3 persona cards (Student, Academic, Industry) | -| `components/Differentiators.tsx` | 3 differentiator cards with capabilities lists | -| `components/OpenSource.tsx` | Typography-driven section with code snippet + licensing | -| `components/Roadmap.tsx` | Planned domains + platform features (dashed border cards) | -| `components/CTA.tsx` | Pricing tiers, terminal quickstart, primary glow accent | -| `styles/app.css` | Tailwind 4 + OKLCH tokens + @font-face declarations + scroll/stagger/terminal animations | -| `public/fonts/*.woff2` | Self-hosted fonts (6 files: Space Grotesk 400-700, JetBrains Mono 400-500) | -| `lib/constants.ts` | Stats, nav/footer links, domains, data sources, methodology phases | -| `lib/use-reveal.ts` | IntersectionObserver scroll reveal hook | -| `lib/use-scroll-progress.ts` | Scroll progress fraction (0-1) hook | +| `routes/__root.tsx` | Root layout, SEO meta, self-hosted font preloads | +| `routes/index.tsx` | Landing page with lazy-loaded sections | +| `components/Hero.tsx` | Hero with MolecularNetwork, stats, CTAs | +| `components/MolecularNetwork.tsx` | 3D rotating node graph (Canvas 2D) | +| `styles/app.css` | Tailwind 4 + OKLCH tokens + animations | +| `lib/constants.ts` | Stats, links, domains, data sources | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0dc5dfc..5002853 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,8 @@ Ehrlich is an AI-powered scientific discovery engine that uses Claude as a hypothesis-driven reasoning engine. The core engine is domain-agnostic -- any scientific domain can plug in with its own tools, scoring, and visualization. -See [docs/adr/domain-agnostic.md](docs/adr/domain-agnostic.md) for the full story of how and why the engine was generalized. +See [README.md](README.md) for setup, architecture overview, and development commands. +See [CLAUDE.md](CLAUDE.md) for dependency rules, key patterns, and key files. ## Contributor License Agreement @@ -24,115 +25,16 @@ If you have questions about this CLA, open an issue before contributing. --- -## Prerequisites +## Code Standards (Non-Negotiable) -- **Python 3.12+** with [uv](https://docs.astral.sh/uv/) package manager -- **Bun** (or Node 20+) for the console -- **ANTHROPIC_API_KEY** environment variable set - -## Setup - -### Server - -```bash -cd server -uv sync --extra dev -uv run uvicorn ehrlich.api.app:create_app --factory --reload --port 8000 -``` - -### Console - -```bash -cd console -bun install -bun dev # Dev server at :5173 -``` - -## Development Workflow - -### Quality Gates (all must pass before commit) - -**Server:** -```bash -uv run ruff check src/ tests/ # Lint (zero violations) -uv run ruff format src/ tests/ --check # Format check -uv run mypy src/ehrlich/ # Type check (strict) -uv run pytest --cov=ehrlich # Tests (80%+ coverage) -``` - -**Console:** -```bash -bun run build # Vite build (generates route tree) -bun run typecheck # tsc --noEmit (run after build) -bunx vitest run # Tests -``` - -### Commit Format - -``` -type(scope): description -``` - -**Types:** `feat`, `fix`, `refactor`, `chore`, `docs`, `test` - -**Scopes:** `kernel`, `shared`, `literature`, `chemistry`, `analysis`, `prediction`, `simulation`, `training`, `nutrition`, `investigation`, `api`, `console`, `mol`, `data` - -### Code Standards (Non-Negotiable) - -Every contribution must respect: - -- **DDD + Clean Architecture** -- Respect bounded contexts. `domain/` is pure Python with zero external deps. `application/` depends on domain interfaces, never infrastructure. `infrastructure/` implements domain ABCs. No cross-context domain imports. -- **No redundancy** -- DRY across layers. If a pattern exists, reuse it. If a utility exists, find it before writing a new one. Three similar lines of code is better than a premature abstraction, but true duplication is never acceptable. -- **No dead code** -- No commented-out blocks, no unused imports, no orphaned functions. Delete what you don't need. -- **No boilerplate** -- If it can be derived (schemas from type hints, routes from file conventions), derive it. Don't write what the framework gives you. -- **No backward-compatibility hacks** -- No renamed `_unused` vars, no re-exports for removed code, no `# removed` comments. If something is unused, delete it. Exception: production hotfixes with a follow-up cleanup ticket. -- **Clean Code** -- Readability over cleverness. Explicit imports only (no wildcards). Fail fast at boundaries. Meaningful names. Small focused functions. -- **Documentation as code** -- Every code change updates relevant docs in the same commit. Tool counts, event counts, file tables, component lists. This is the #1 recurring failure -- never skip it. +- **DDD + Clean Architecture** -- Respect bounded contexts. See CLAUDE.md for the 8 strict dependency rules. +- **No redundancy** -- DRY across layers. Three similar lines is better than a premature abstraction, but true duplication is never acceptable. +- **No dead code** -- No commented-out blocks, no unused imports, no orphaned functions. +- **No backward-compatibility hacks** -- If something is unused, delete it. +- **Clean Code** -- Readability over cleverness. Explicit imports only (no wildcards). Fail fast at boundaries. +- **Documentation as code** -- Every code change updates relevant docs in the same commit. Tool counts, event counts, file tables, component lists. - **Test before claiming complete** -- All quality gates must pass. Never say "done" without running the full suite. -## Architecture - -DDD monorepo with 10 bounded contexts. Each context follows `domain/` -> `application/` -> `infrastructure/` layering. - -### Layer Rules (Strict) - -1. `domain/` has ZERO external dependencies -- pure Python only -2. `application/` depends on `domain/` interfaces, never on `infrastructure/` -3. `infrastructure/` implements `domain/` repository interfaces -4. `tools.py` calls `application/` services, returns JSON for Claude -5. No cross-context domain imports -- communicate via `kernel/` primitives or `shared/` ports -6. `shared/` contains cross-cutting ports (ABCs) and value objects -- no infrastructure deps -7. RDKit imports ONLY in `chemistry/infrastructure/rdkit_adapter.py` - -### Multi-Model Orchestrator - -``` -Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence (NO tools) -Sonnet 4.5 (Researcher) -- Executes experiments with 70 tools (parallel: 2 per batch) -Haiku 4.5 (Summarizer) -- Compresses large outputs, classifies domains -``` - -### Bounded Contexts - -| Context | Domain | Purpose | -|---------|--------|---------| -| kernel | All | Shared primitives (SMILES, Molecule, exceptions) | -| shared | All | Cross-cutting ports and value objects (ChemistryPort, Fingerprint, Conformer3D) | -| literature | All | Paper search (Semantic Scholar) | -| chemistry | Molecular | RDKit cheminformatics | -| analysis | Molecular | Dataset exploration (ChEMBL, PubChem, GtoPdb) | -| prediction | Molecular | ML models (XGBoost, Chemprop) | -| simulation | Molecular | Docking, ADMET, targets (RCSB PDB, UniProt, Open Targets, EPA CompTox) | -| training | Training | Exercise physiology (evidence analysis, protocol comparison, injury risk, training metrics, clinical trials) | -| nutrition | Nutrition | Nutrition science (supplement evidence, labels, nutrients, safety, interactions, adequacy, inflammatory index) | -| investigation | All | Multi-model orchestration + domain registry + SQLite persistence | - -## Testing - -- Server: `uv run pytest` -- unit + integration tests -- Console: `bunx vitest run` -- component + utility tests -- Coverage minimum: 80% (server), enforced by pytest-cov - --- ## Adding a New Scientific Domain @@ -145,9 +47,9 @@ Before writing any code, research the domain deeply. This upfront work determine **What to research:** -1. **Problem definition** -- What scientific questions should this domain answer? What does "discovery" mean here? (e.g., molecular: find candidate drugs; training: find optimal training protocols) +1. **Problem definition** -- What scientific questions should this domain answer? What does "discovery" mean here? 2. **Data sources** -- Where does real experimental data come from? Identify free APIs, databases, or datasets. For each source: base URL, auth requirements, rate limits, response format, coverage -3. **Candidate criteria** -- What makes a good candidate in this domain? What scores matter? What thresholds separate good from bad? (e.g., molecular: prediction_score > 0.7, docking < -7.0 kcal/mol; training: effect_size > 0.5, evidence_grade A-B) +3. **Candidate criteria** -- What makes a good candidate in this domain? What scores matter? What thresholds separate good from bad? 4. **Hypothesis types** -- What kinds of hypotheses does this domain test? (mechanistic, structural, correlational, physiological, etc.) 5. **Visualization needs** -- How should results be displayed? Charts, 3D models, diagrams, maps, tables? 6. **Domain vocabulary** -- Key terms, units, standard notations that Claude needs in its prompts @@ -161,7 +63,7 @@ docs/research/yourdomain/ prior-work.md # Existing tools and approaches in this field ``` -See `docs/research/molecular/`, `docs/research/methodology/`, and the `training/` and `nutrition/` bounded contexts for examples of thorough domain research and implementation. +See `docs/research/molecular/`, `docs/research/methodology/`, and the `training/` and `nutrition/` bounded contexts for examples. ### Step 1: Create the Bounded Context @@ -241,7 +143,6 @@ YOUR_DOMAIN = DomainConfig( ), attribute_keys=("category", "source"), # Non-numeric candidate attributes negative_control_threshold=0.5, # Score below = correctly classified - # visualization is reactive: LiveLabViewer auto-appears for molecular tools, hypothesis_types=( "mechanistic", # Valid hypothesis_type values for this domain "correlational", @@ -249,7 +150,7 @@ YOUR_DOMAIN = DomainConfig( ), valid_domain_categories=( "your_domain", # Categories the classifier might output - "related_field", # Map these to your domain + "related_field", ), template_prompts=( { @@ -265,9 +166,9 @@ YOUR_DOMAIN = DomainConfig( ``` Key decisions: -- **Visualization**: Visualization is reactive -- LiveLabViewer auto-appears when molecular tool events (docking, descriptors, etc.) are detected in the SSE stream. Chart visualizations render inline from `VisualizationRendered` events. No configuration needed. +- **Visualization**: Visualization is reactive -- LiveLabViewer auto-appears when molecular tool events are detected in the SSE stream. Chart visualizations render inline from `VisualizationRendered` events. No configuration needed. - **`tool_tags`**: This frozenset determines which tools the Researcher can see. Include `"literature"` if you want paper search. -- **`director_examples`**: These are the most impactful part. Good multishot examples make the Director formulate domain-appropriate hypotheses. Use `{{` and `}}` for literal braces in f-string-like contexts. +- **`director_examples`**: These are the most impactful part. Good multishot examples make the Director formulate domain-appropriate hypotheses. - **`score_definitions`**: These become the columns in the CandidateTable. The frontend auto-generates color-coded cells based on thresholds. ### Step 4: Register Tools and Domain @@ -403,12 +304,9 @@ Update these files (in the same commit as the code): | File | What to update | |------|---------------| -| `CLAUDE.md` | Tool count, bounded contexts table, tools section, scopes list | +| `CLAUDE.md` | Bounded contexts table, key files | | `README.md` | Tool count, bounded contexts table, tools table, "What Can Ehrlich Investigate" | -| `docs/architecture.md` | Bounded contexts, tool count, data flow | -| `docs/roadmap/` | Add completion entry to appropriate phase file | -| `web/src/lib/constants.ts` | STATS counts, DOMAINS array (toolCount, capabilities, sources, vizTools), DATA_SOURCES array | -| `console/` | Template prompts, domain badges, tool references where applicable | +| `web/src/lib/constants.ts` | STATS counts, DOMAINS array, DATA_SOURCES array | ### Step 9: Verify @@ -436,8 +334,8 @@ That's it. Your domain is live. The orchestrator will auto-detect it from the us 2. Register it in `_build_registry()` with the domain's tag 3. Write tests 4. Update the integration test tool count in `tests/integration/test_e2e.py` -5. Update docs (tool counts in CLAUDE.md, README.md, docs/architecture.md) -6. Update tool counts and references in `web/src/lib/constants.ts` (STATS, DOMAINS toolCount) and `console/` where applicable +5. Update docs (tool counts in CLAUDE.md, README.md) +6. Update tool counts and references in `web/src/lib/constants.ts` (STATS, DOMAINS toolCount) ### Improving Score Definitions From e013d78683a86f51a061fce240d278e3c9332b03 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 08:09:50 -0700 Subject: [PATCH 11/35] docs: add index READMEs for docs/ and docs/adr/ --- docs/README.md | 16 ++++++++++++++++ docs/adr/README.md | 7 +++++++ 2 files changed, 23 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/adr/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..3683f1c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,16 @@ +# Ehrlich Documentation + +## Reference + +| File | Purpose | +|------|---------| +| [architecture.md](architecture.md) | System architecture, bounded contexts, data flow | +| [scientific-methodology.md](scientific-methodology.md) | 6-phase investigation methodology specification | + +## Directories + +| Directory | Purpose | +|-----------|---------| +| [roadmap/](roadmap/README.md) | Phase-by-phase development history with dependency graph | +| [research/](research/README.md) | Pre-development research (platform, methodology, molecular, impact) | +| [adr/](adr/README.md) | Architecture Decision Records | diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..dc245a6 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,7 @@ +# Architecture Decision Records + +| ADR | Decision | +|-----|----------| +| [domain-agnostic.md](domain-agnostic.md) | Generalizing the engine from antimicrobial-only to domain-agnostic | +| [impact-evaluation-domain.md](impact-evaluation-domain.md) | Adding causal inference and social program evaluation domain | +| [business-strategy.md](business-strategy.md) | Commercialization strategy and market positioning | From 3163d5dd2af5a0214c4a9c786f114246a18704cc Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 08:49:35 -0700 Subject: [PATCH 12/35] docs: align identity as COSS (Commercial Open-Source Software) - README.md: add Philosophy section, lead with open-source identity, self-hosting callout - business-strategy.md: add Classification table, rename Pricing Model to Hosted Instance Pricing - phase-12: rename SaaS Infrastructure to Platform Infrastructure (git mv) - roadmap/README.md: update Phase 12 references in table and dependency graph - phase-02: add domain-agnostic evolution note linking to Phase 10B --- README.md | 19 ++++++++++++++++- docs/adr/business-strategy.md | 21 ++++++++++++++++--- docs/roadmap/README.md | 4 ++-- docs/roadmap/phase-02-literature-analysis.md | 2 ++ ...md => phase-12-platform-infrastructure.md} | 4 ++-- 5 files changed, 42 insertions(+), 8 deletions(-) rename docs/roadmap/{phase-12-saas-infrastructure.md => phase-12-platform-infrastructure.md} (93%) diff --git a/README.md b/README.md index 5c96f44..843a141 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,26 @@ # Ehrlich -AI-powered scientific discovery engine. Ehrlich uses Claude as a hypothesis-driven scientific reasoning engine that works across multiple scientific domains. The core engine is domain-agnostic: a pluggable `DomainConfig` system lets any domain bring its own tools, scoring, and visualization. +Open-source (AGPL-3.0), self-hostable scientific discovery engine. Ehrlich uses Claude as a hypothesis-driven scientific reasoning engine that works across multiple scientific domains. The core engine is domain-agnostic: a pluggable `DomainConfig` system lets any domain bring its own tools, scoring, and visualization. Run it on your own infrastructure with your own API keys, or use the [hosted instance](https://app.ehrlich.dev) where credits cover the Anthropic API costs. Named after Paul Ehrlich, the father of the "magic bullet" -- finding the right answer for any question. Built for the [Claude Code Hackathon](https://docs.anthropic.com/en/docs/claude-code-hackathon) (Feb 10-16, 2026). +## Philosophy + +Ehrlich is **COSS** (Commercial Open-Source Software) -- the same model used by Supabase, PostHog, Cal.com, and GitLab. The entire codebase is open source under AGPL-3.0. There is no proprietary version. + +**Two paths, same product:** + +| Path | How It Works | Cost | +|------|-------------|------| +| **Self-host** | Clone the repo, bring your own Anthropic API key | Free. No limits, no credits, no account needed | +| **Hosted instance** | Use app.ehrlich.dev | Credits cover Anthropic API costs (Opus is expensive) | + +Credits exist because Claude Opus costs real money per investigation. They make scientific reasoning **accessible** -- not monetized. A student in Mexico and a pharma company in Boston get the same 85 tools, the same 17 data sources, the same methodology. The model quality is the only variable. + +The AI is the scientist. The platform is the laboratory. + ## What Can Ehrlich Investigate? Ehrlich is **domain-agnostic**. The hypothesis-driven engine works for any scientific domain: @@ -189,6 +204,8 @@ All data sources are free and open-access. ## Quick Start +> **Self-hosting?** You use your own Anthropic API key directly -- no credits, no account, no hosted dependency. The credit system only applies to the managed hosted instance. + ### Prerequisites - [uv](https://docs.astral.sh/uv/) (Python package manager) diff --git a/docs/adr/business-strategy.md b/docs/adr/business-strategy.md index 90127e8..9cb1fe8 100644 --- a/docs/adr/business-strategy.md +++ b/docs/adr/business-strategy.md @@ -1,7 +1,7 @@ # Ehrlich Business Strategy > Public strategy document. Ehrlich is AGPL-3.0 open source. -> For technical implementation, see [Phase 12: SaaS Infrastructure](roadmap/phase-12-saas-infrastructure.md). +> For technical implementation, see [Phase 12: Platform Infrastructure](../roadmap/phase-12-platform-infrastructure.md). ## Market Position @@ -12,7 +12,8 @@ Ehrlich sits between AI wrappers and enterprise platforms. - Not a chatbot with a science skin ### What Ehrlich Is -- A domain-agnostic scientific reasoning engine +- A **COSS** (Commercial Open-Source Software) scientific reasoning engine +- Open source (AGPL-3.0), self-hostable, with an optional managed hosted instance - 67 real tools with real computation (RDKit, XGBoost, molecular docking, scipy.stats) - 15 external data sources (ChEMBL, PubChem, RCSB PDB, UniProt, ClinicalTrials.gov, PubMed, wger, NIH DSLD, USDA FoodData, OpenFDA, EPA CompTox, Semantic Scholar, Open Targets, GtoPdb, RxNav) + 1 internal (FTS5) - Multi-model orchestration (Director-Worker-Summarizer) @@ -24,7 +25,19 @@ Ehrlich sits between AI wrappers and enterprise platforms. - Enterprise platforms (Schrodinger, Recursion): full stack, $100K+/year, single domain - Ehrlich: multi-domain scientific engine, $2-10 per investigation, accessible to students and pharma alike -## Pricing Model +## Classification + +Ehrlich follows the COSS (Commercial Open-Source Software) model: + +| Layer | What | Model | +|-------|------|-------| +| **Software** | Ehrlich engine (server + console) | Open source, AGPL-3.0 | +| **Hosted instance** | app.ehrlich.dev | Managed hosting — credits cover Anthropic API costs | +| **License** | AGPL-3.0 + commercial option | Dual licensing for private modifications | + +Self-hosters use their own Anthropic API key and bypass the credit system entirely. The hosted instance exists for users who don't want to manage infrastructure. + +## Hosted Instance Pricing ### Universal Credits One currency. User decides quality per investigation. @@ -48,6 +61,8 @@ Researcher is always Sonnet 4.5 (tool execution). Summarizer is always Haiku 4.5 Monthly subscription = auto-refilling credit pack at a discount. Not a separate product. Credits expire after 60 days (prevents hoarding, encourages usage). +> **Self-hosting?** None of the above applies. Self-hosters bring their own Anthropic API key (BYOK) and pay Anthropic directly. No credits, no packs, no limits. + ### Why Per-Investigation, Not Per-Token An investigation is a complete scientific workflow: literature survey, hypothesis formulation, experiment execution, evaluation, synthesis. That's an outcome with a deliverable (the report), not a token count. Pricing the outcome aligns cost with value. diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index 4f24593..8238a8f 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -22,7 +22,7 @@ Hackathon: Feb 10-16, 2026. Each phase ends with something testable. | Phase 10D: Landing Site (web/) | DONE | [phase-10d-landing-site.md](phase-10d-landing-site.md) | | Phase 10E: Training Science Enhancement | DONE | [phase-10e-training-enhancement.md](phase-10e-training-enhancement.md) | | Phase 10F: Nutrition Science Enhancement | DONE | [phase-10f-nutrition-enhancement.md](phase-10f-nutrition-enhancement.md) | -| Phase 12: SaaS Infrastructure | DONE | [phase-12-saas-infrastructure.md](phase-12-saas-infrastructure.md) | +| Phase 12: Platform Infrastructure | DONE | [phase-12-platform-infrastructure.md](phase-12-platform-infrastructure.md) | | Phase 13: Impact Evaluation Domain | IN PROGRESS | [phase-13-impact-evaluation.md](phase-13-impact-evaluation.md) | | Phase 14: Scientific Engine Hardening | IN PROGRESS | [phase-14-engine-hardening.md](phase-14-engine-hardening.md) | | Backlog | TODO | [backlog.md](backlog.md) | @@ -80,7 +80,7 @@ Enhancement Enhancement Sports │ │ │ └─────┼──────────────┘ | - Phase 12: SaaS Infrastructure -- DONE + Phase 12: Platform Infrastructure -- DONE (PostgreSQL + WorkOS Auth + BYOK + Credits) | Phase 13: Impact Evaluation Domain -- IN PROGRESS diff --git a/docs/roadmap/phase-02-literature-analysis.md b/docs/roadmap/phase-02-literature-analysis.md index 5d0019c..c80685a 100644 --- a/docs/roadmap/phase-02-literature-analysis.md +++ b/docs/roadmap/phase-02-literature-analysis.md @@ -2,6 +2,8 @@ Back to [Roadmap Index](README.md) # Phase 2: Literature + Analysis (Feb 10) -- DONE +> **Evolution note:** This phase established the literature search and bioactivity analysis foundation. Originally chemistry-specific (ChEMBL, Semantic Scholar), the analysis tools were later generalized into domain-agnostic components in [Phase 10B](phase-10b-domain-agnostic.md). + Two independent contexts. Can be built in parallel. ## 2A. Semantic Scholar Client diff --git a/docs/roadmap/phase-12-saas-infrastructure.md b/docs/roadmap/phase-12-platform-infrastructure.md similarity index 93% rename from docs/roadmap/phase-12-saas-infrastructure.md rename to docs/roadmap/phase-12-platform-infrastructure.md index e2da9fc..34ba822 100644 --- a/docs/roadmap/phase-12-saas-infrastructure.md +++ b/docs/roadmap/phase-12-platform-infrastructure.md @@ -1,8 +1,8 @@ Back to [Roadmap Index](README.md) -# Phase 12: SaaS Infrastructure (Feb 13) -- DONE +# Phase 12: Platform Infrastructure (Feb 13) -- DONE -Production-ready authentication, credit-based billing, model-tiered investigations, and PostgreSQL persistence. +The COSS managed-hosting layer: production-ready authentication, credit-based billing, model-tiered investigations, and PostgreSQL persistence for the hosted instance. > Business rationale and pricing research: see [business-strategy.md](../adr/business-strategy.md) From 520bc750238c363cb4ba879481beda515a77b011 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 09:13:09 -0700 Subject: [PATCH 13/35] refactor(14B): extract registry factory, orchestrator factory, and schemas from investigation routes - Create registry_factory.py with lru_cache singletons for tool/domain/MCP registration - Create orchestrator_factory.py with Anthropic adapter wiring - Create api/schemas/investigation.py with 6 DTOs and 5 serialization helpers - Eliminate ~70 lines of duplicated entity-to-dict code between _to_detail and _replay_final - Update consumers: app.py, methodology.py, stats.py, test_e2e.py - Fix stale tool count assertions in e2e tests (78 -> 85) Route file: 829 -> 366 lines (56% reduction) --- docs/roadmap/phase-14-engine-hardening.md | 14 +- server/src/ehrlich/api/app.py | 5 +- .../src/ehrlich/api/routes/investigation.py | 528 ++---------------- server/src/ehrlich/api/routes/methodology.py | 9 +- server/src/ehrlich/api/routes/stats.py | 9 +- server/src/ehrlich/api/schemas/__init__.py | 0 .../src/ehrlich/api/schemas/investigation.py | 155 +++++ .../application/orchestrator_factory.py | 70 +++ .../application/registry_factory.py | 276 +++++++++ server/tests/integration/test_e2e.py | 16 +- 10 files changed, 563 insertions(+), 519 deletions(-) create mode 100644 server/src/ehrlich/api/schemas/__init__.py create mode 100644 server/src/ehrlich/api/schemas/investigation.py create mode 100644 server/src/ehrlich/investigation/application/orchestrator_factory.py create mode 100644 server/src/ehrlich/investigation/application/registry_factory.py diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md index 3e38253..bcd52dc 100644 --- a/docs/roadmap/phase-14-engine-hardening.md +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -28,16 +28,16 @@ Split `MultiModelOrchestrator` (1,788 lines, 11 concerns) into focused modules f Result: `multi_orchestrator.py` from 1,788 to 1,216 lines (4 modules extracted: 718 lines total) -## 14B: Route File Cleanup -- TODO +## 14B: Route File Cleanup -- DONE -Move infrastructure concerns out of `api/routes/investigation.py` (828 lines) into application layer. +Moved infrastructure concerns out of `api/routes/investigation.py` (829 lines) into the application layer. -- [ ] Extract `application/registry_factory.py` -- `build_tool_registry()` + `build_domain_registry()` (128 lines of tool registration) -- [ ] Extract `application/orchestrator_factory.py` -- `create_orchestrator()` with all adapter wiring -- [ ] Move Pydantic DTOs to `api/schemas/investigation.py` (6 models) -- [ ] Lazy-initialize tool registry (singleton, built once instead of per-investigation) +- [x] Extract `application/registry_factory.py` -- `build_tool_registry()` + `build_domain_registry()` + `build_mcp_configs()` with `lru_cache` singletons +- [x] Extract `application/orchestrator_factory.py` -- `create_orchestrator()` with Anthropic adapter wiring +- [x] Move Pydantic DTOs + serialization helpers to `api/schemas/investigation.py` (6 models, 5 serializers) +- [x] Eliminated ~70 lines of duplicated entity-to-dict serialization between `_to_detail()` and `_replay_final()` -Target: route file from 828 to ~400 lines. +Result: `api/routes/investigation.py` from 829 to 366 lines (56% reduction). Three new modules: `registry_factory.py` (237 lines), `orchestrator_factory.py` (71 lines), `api/schemas/investigation.py` (140 lines). ## 14C: Prompt Module Split -- TODO diff --git a/server/src/ehrlich/api/app.py b/server/src/ehrlich/api/app.py index 5dc10c2..4738386 100644 --- a/server/src/ehrlich/api/app.py +++ b/server/src/ehrlich/api/app.py @@ -14,6 +14,7 @@ from ehrlich.api.routes.stats import router as stats_router from ehrlich.api.routes.upload import router as upload_router from ehrlich.config import get_settings +from ehrlich.investigation.application.registry_factory import build_tool_registry logger = logging.getLogger(__name__) @@ -47,9 +48,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: settings.researcher_model, ) - from ehrlich.api.routes.investigation import _build_registry - - registry = _build_registry() + registry = build_tool_registry() logger.info("Tool registry: %d tools available", len(registry.list_tools())) for mod, desc in _OPTIONAL_DEPS.items(): diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index 415f6cc..c472619 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -6,129 +6,39 @@ from typing import TYPE_CHECKING, Any from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import BaseModel from sse_starlette.sse import EventSourceResponse -from ehrlich.analysis.tools import ( - analyze_substructures, - assess_threats, - compute_cost_effectiveness, - compute_properties, - estimate_did, - estimate_psm, - estimate_rdd, - estimate_synthetic_control, - explore_dataset, - run_categorical_test, - run_statistical_test, - search_bioactivity, - search_compounds, - search_pharmacology, -) from ehrlich.api.auth import get_current_user, get_current_user_sse -from ehrlich.api.sse import SSEEventType, domain_event_to_sse -from ehrlich.chemistry.tools import ( - compute_descriptors, - compute_fingerprint, - generate_3d, - substructure_match, - tanimoto_similarity, - validate_smiles, +from ehrlich.api.schemas.investigation import ( + ApproveRequest, + CreditBalanceResponse, + InvestigateRequest, + InvestigateResponse, + InvestigationDetail, + InvestigationSummary, + serialize_candidates, + serialize_findings, + serialize_hypotheses, + serialize_negative_controls, + to_detail, ) +from ehrlich.api.sse import SSEEventType, domain_event_to_sse from ehrlich.config import get_settings -from ehrlich.impact.tools import ( - compare_programs, - fetch_benchmark, - search_economic_indicators, +from ehrlich.investigation.application.orchestrator_factory import create_orchestrator +from ehrlich.investigation.application.registry_factory import ( + build_domain_registry, + build_mcp_configs, + build_tool_registry, ) -from ehrlich.investigation.application.multi_orchestrator import MultiModelOrchestrator -from ehrlich.investigation.application.tool_registry import ToolRegistry -from ehrlich.investigation.domain.domain_registry import DomainRegistry -from ehrlich.investigation.domain.domains.impact import IMPACT_EVALUATION -from ehrlich.investigation.domain.domains.molecular import MOLECULAR_SCIENCE -from ehrlich.investigation.domain.domains.nutrition import NUTRITION_SCIENCE -from ehrlich.investigation.domain.domains.training import TRAINING_SCIENCE from ehrlich.investigation.domain.investigation import Investigation, InvestigationStatus -from ehrlich.investigation.domain.mcp_config import MCPServerConfig -from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter from ehrlich.investigation.infrastructure.mcp_bridge import MCPBridge from ehrlich.investigation.infrastructure.repository import InvestigationRepository -from ehrlich.investigation.tools import ( - conclude_investigation, - design_experiment, - evaluate_hypothesis, - propose_hypothesis, - query_uploaded_data, - record_finding, - record_negative_control, - search_prior_research, -) -from ehrlich.investigation.tools_viz import ( - render_admet_radar, - render_binding_scatter, - render_causal_diagram, - render_dose_response, - render_evidence_matrix, - render_forest_plot, - render_funnel_plot, - render_geographic_comparison, - render_muscle_heatmap, - render_nutrient_adequacy, - render_nutrient_comparison, - render_parallel_trends, - render_performance_chart, - render_program_dashboard, - render_rdd_plot, - render_therapeutic_window, - render_training_timeline, -) -from ehrlich.literature.tools import get_reference, search_citations, search_literature -from ehrlich.nutrition.tools import ( - analyze_nutrient_ratios, - assess_nutrient_adequacy, - check_intake_safety, - check_interactions, - compare_nutrients, - compute_inflammatory_index, - search_nutrient_data, - search_supplement_evidence, - search_supplement_labels, - search_supplement_safety, -) -from ehrlich.prediction.tools import ( - cluster_compounds, - cluster_data, - predict_candidates, - predict_scores, - train_classifier, - train_model, -) -from ehrlich.simulation.tools import ( - assess_resistance, - dock_against_target, - fetch_toxicity_profile, - get_protein_annotation, - predict_admet, - search_disease_targets, - search_protein_targets, -) -from ehrlich.training.tools import ( - analyze_training_evidence, - assess_injury_risk, - compare_protocols, - compute_dose_response, - compute_performance_model, - compute_training_metrics, - plan_periodization, - search_clinical_trials, - search_exercise_database, - search_pubmed_training, - search_training_literature, -) if TYPE_CHECKING: from collections.abc import AsyncGenerator + from ehrlich.investigation.application.multi_orchestrator import MultiModelOrchestrator + logger = logging.getLogger(__name__) router = APIRouter(tags=["investigation"]) @@ -156,47 +66,6 @@ _investigation_meta: dict[str, dict[str, Any]] = {} -class InvestigateRequest(BaseModel): - prompt: str - director_tier: str = "opus" - file_ids: list[str] = [] - - -class CreditBalanceResponse(BaseModel): - credits: int - is_byok: bool - - -class InvestigateResponse(BaseModel): - id: str - status: str - - -class InvestigationDetail(BaseModel): - id: str - prompt: str - status: str - hypotheses: list[dict[str, Any]] - experiments: list[dict[str, Any]] - current_hypothesis_id: str - current_experiment_id: str - findings: list[dict[str, Any]] - candidates: list[dict[str, Any]] - negative_controls: list[dict[str, Any]] - citations: list[str] - summary: str - created_at: str - cost_data: dict[str, object] - - -class InvestigationSummary(BaseModel): - id: str - prompt: str - status: str - created_at: str - candidate_count: int - - async def init_repository(database_url: str) -> None: global _repository # noqa: PLW0603 repo = InvestigationRepository(database_url) @@ -216,161 +85,6 @@ def _get_repository() -> InvestigationRepository: return _repository -def _build_registry() -> ToolRegistry: - registry = ToolRegistry() - _chem = frozenset({"chemistry"}) - _lit = frozenset({"literature"}) - _analysis = frozenset({"analysis"}) - _pred = frozenset({"prediction"}) - _sim = frozenset({"simulation"}) - _training = frozenset({"training"}) - _training_clinical = frozenset({"training", "clinical"}) - _training_exercise = frozenset({"training", "exercise"}) - _nutrition = frozenset({"nutrition"}) - _nutrition_safety = frozenset({"nutrition", "safety"}) - _impact = frozenset({"impact"}) - _causal = frozenset({"causal"}) - _ml = frozenset({"ml"}) - _viz = frozenset({"visualization"}) - _chem_viz = frozenset({"chemistry", "visualization"}) - _sim_viz = frozenset({"simulation", "visualization"}) - _training_viz = frozenset({"training", "visualization"}) - _nutrition_viz = frozenset({"nutrition", "visualization"}) - _impact_viz = frozenset({"impact", "visualization"}) - _causal_viz = frozenset({"causal", "visualization"}) - - tagged_tools: list[tuple[str, Any, frozenset[str] | None]] = [ - # Chemistry (6) - ("validate_smiles", validate_smiles, _chem), - ("compute_descriptors", compute_descriptors, _chem), - ("compute_fingerprint", compute_fingerprint, _chem), - ("tanimoto_similarity", tanimoto_similarity, _chem), - ("generate_3d", generate_3d, _chem), - ("substructure_match", substructure_match, _chem), - # Literature (3) - ("search_literature", search_literature, _lit), - ("get_reference", get_reference, _lit), - ("search_citations", search_citations, _lit), - # Analysis (6) - ("explore_dataset", explore_dataset, _analysis), - ("search_compounds", search_compounds, _analysis), - ("search_bioactivity", search_bioactivity, _analysis), - ("analyze_substructures", analyze_substructures, _analysis), - ("compute_properties", compute_properties, _analysis), - ("search_pharmacology", search_pharmacology, _analysis), - # Prediction (3) -- molecular-specific - ("train_model", train_model, _pred), - ("predict_candidates", predict_candidates, _pred), - ("cluster_compounds", cluster_compounds, _pred), - # ML (3) -- domain-agnostic - ("train_classifier", train_classifier, _ml), - ("predict_scores", predict_scores, _ml), - ("cluster_data", cluster_data, _ml), - # Simulation (7) - ("search_protein_targets", search_protein_targets, _sim), - ("dock_against_target", dock_against_target, _sim), - ("predict_admet", predict_admet, _sim), - ("fetch_toxicity_profile", fetch_toxicity_profile, _sim), - ("assess_resistance", assess_resistance, _sim), - ("get_protein_annotation", get_protein_annotation, _sim), - ("search_disease_targets", search_disease_targets, _sim), - # Training Science (11) - ("search_training_literature", search_training_literature, _training), - ("analyze_training_evidence", analyze_training_evidence, _training), - ("compare_protocols", compare_protocols, _training), - ("assess_injury_risk", assess_injury_risk, _training), - ("compute_training_metrics", compute_training_metrics, _training), - ("search_clinical_trials", search_clinical_trials, _training_clinical), - ("search_pubmed_training", search_pubmed_training, _training), - ("search_exercise_database", search_exercise_database, _training_exercise), - ("compute_performance_model", compute_performance_model, _training), - ("compute_dose_response", compute_dose_response, _training), - ("plan_periodization", plan_periodization, _training_exercise), - # Nutrition Science (10) - ("search_supplement_evidence", search_supplement_evidence, _nutrition), - ("search_supplement_labels", search_supplement_labels, _nutrition), - ("search_nutrient_data", search_nutrient_data, _nutrition), - ("search_supplement_safety", search_supplement_safety, _nutrition_safety), - ("compare_nutrients", compare_nutrients, _nutrition), - ("assess_nutrient_adequacy", assess_nutrient_adequacy, _nutrition), - ("check_intake_safety", check_intake_safety, _nutrition_safety), - ("check_interactions", check_interactions, _nutrition_safety), - ("analyze_nutrient_ratios", analyze_nutrient_ratios, _nutrition), - ("compute_inflammatory_index", compute_inflammatory_index, _nutrition), - # Impact Evaluation (3) - ("search_economic_indicators", search_economic_indicators, _impact), - ("fetch_benchmark", fetch_benchmark, _impact), - ("compare_programs", compare_programs, _impact), - # Causal Inference (6) -- domain-agnostic - ("estimate_did", estimate_did, _causal), - ("estimate_psm", estimate_psm, _causal), - ("estimate_rdd", estimate_rdd, _causal), - ("estimate_synthetic_control", estimate_synthetic_control, _causal), - ("assess_threats", assess_threats, _causal), - ("compute_cost_effectiveness", compute_cost_effectiveness, _causal), - # Visualization (17) - ("render_binding_scatter", render_binding_scatter, _chem_viz), - ("render_admet_radar", render_admet_radar, _sim_viz), - ("render_training_timeline", render_training_timeline, _training_viz), - ("render_muscle_heatmap", render_muscle_heatmap, _training_viz), - ("render_forest_plot", render_forest_plot, _viz), - ("render_evidence_matrix", render_evidence_matrix, _viz), - ("render_performance_chart", render_performance_chart, _training_viz), - ("render_funnel_plot", render_funnel_plot, _viz), - ("render_dose_response", render_dose_response, _training_viz), - ("render_nutrient_comparison", render_nutrient_comparison, _nutrition_viz), - ("render_nutrient_adequacy", render_nutrient_adequacy, _nutrition_viz), - ("render_therapeutic_window", render_therapeutic_window, _nutrition_viz), - ("render_program_dashboard", render_program_dashboard, _impact_viz), - ("render_geographic_comparison", render_geographic_comparison, _impact_viz), - ("render_parallel_trends", render_parallel_trends, _impact_viz), - ("render_rdd_plot", render_rdd_plot, _causal_viz), - ("render_causal_diagram", render_causal_diagram, _causal_viz), - # Statistics (2) -- universal, no tags - ("run_statistical_test", run_statistical_test, None), - ("run_categorical_test", run_categorical_test, None), - # Investigation control (7) -- universal, no tags - ("record_finding", record_finding, None), - ("conclude_investigation", conclude_investigation, None), - ("propose_hypothesis", propose_hypothesis, None), - ("design_experiment", design_experiment, None), - ("evaluate_hypothesis", evaluate_hypothesis, None), - ("record_negative_control", record_negative_control, None), - ("search_prior_research", search_prior_research, None), - ("query_uploaded_data", query_uploaded_data, None), - ] - for name, func, tags in tagged_tools: - registry.register(name, func, tags) - return registry - - -def _build_domain_registry() -> DomainRegistry: - domain_registry = DomainRegistry() - domain_registry.register(MOLECULAR_SCIENCE) - domain_registry.register(TRAINING_SCIENCE) - domain_registry.register(NUTRITION_SCIENCE) - domain_registry.register(IMPACT_EVALUATION) - return domain_registry - - -def _build_mcp_configs() -> list[MCPServerConfig]: - """Build MCP server configs from environment. Currently supports Excalidraw.""" - import os - - configs: list[MCPServerConfig] = [] - if os.environ.get("EHRLICH_MCP_EXCALIDRAW", "").lower() in ("1", "true"): - configs.append( - MCPServerConfig( - name="excalidraw", - transport="stdio", - command="npx", - args=("-y", "@anthropic/claude-code-mcp", "excalidraw"), - tags=frozenset({"visualization"}), - ) - ) - return configs - - def _broadcast_event(investigation_id: str, event: dict[str, str]) -> None: for queue in _subscribers.get(investigation_id, []): queue.put_nowait(event) @@ -382,6 +96,9 @@ def _end_broadcast(investigation_id: str) -> None: _subscribers.pop(investigation_id, None) +# ── Route Handlers ────────────────────────────────────────────────────── + + @router.get("/investigate") async def list_investigations( user: dict[str, Any] = _require_user, @@ -421,7 +138,7 @@ async def get_investigation(investigation_id: str) -> InvestigationDetail: investigation = await repo.get_by_id(investigation_id) if investigation is None: raise HTTPException(status_code=404, detail="Investigation not found") - return _to_detail(investigation) + return to_detail(investigation) @router.post("/investigate") @@ -469,11 +186,6 @@ async def start_investigation( return InvestigateResponse(id=investigation.id, status=investigation.status.value) -class ApproveRequest(BaseModel): - approved_ids: list[str] - rejected_ids: list[str] = [] - - @router.post("/investigate/{investigation_id}/approve") async def approve_hypotheses( investigation_id: str, @@ -519,9 +231,9 @@ async def stream_investigation( # Pending -- start the investigation and stream settings = get_settings() - registry = _build_registry() - domain_registry = _build_domain_registry() - mcp_configs = _build_mcp_configs() + registry = build_tool_registry() + domain_registry = build_domain_registry() + mcp_configs = build_mcp_configs() mcp_bridge = MCPBridge() if mcp_configs else None # BYOK: check for user-provided API key @@ -535,7 +247,7 @@ async def stream_investigation( _active_investigations[investigation.id] = investigation _subscribers[investigation.id] = [] - orchestrator = _create_orchestrator( + orchestrator = create_orchestrator( settings, registry, repo, @@ -592,58 +304,7 @@ async def event_generator() -> AsyncGenerator[dict[str, str], None]: return EventSourceResponse(event_generator()) -def _create_orchestrator( - settings: Any, - registry: ToolRegistry, - repository: InvestigationRepository, - domain_registry: DomainRegistry | None = None, - mcp_bridge: MCPBridge | None = None, - mcp_configs: list[MCPServerConfig] | None = None, - api_key_override: str | None = None, - director_model_override: str | None = None, -) -> MultiModelOrchestrator: - api_key = api_key_override or settings.anthropic_api_key or None - - director_model = director_model_override or settings.director_model - - # effort is only supported by Opus models (4.5+) - director_effort = settings.director_effort if "opus" in director_model else None - - thinking = None - if settings.director_thinking == "enabled": - thinking = { - "type": "enabled", - "budget_tokens": settings.director_thinking_budget, - } - - director = AnthropicClientAdapter( - model=director_model, - api_key=api_key, - max_tokens=32768, - effort=director_effort, - thinking=thinking, - ) - researcher = AnthropicClientAdapter( - model=settings.researcher_model, - api_key=api_key, - ) - summarizer = AnthropicClientAdapter( - model=settings.summarizer_model, - api_key=api_key, - ) - return MultiModelOrchestrator( - director=director, - researcher=researcher, - summarizer=summarizer, - registry=registry, - max_iterations_per_experiment=settings.max_iterations_per_experiment, - summarizer_threshold=settings.summarizer_threshold, - require_approval=True, - repository=repository, - domain_registry=domain_registry, - mcp_bridge=mcp_bridge, - mcp_configs=mcp_configs, - ) +# ── SSE Helpers ───────────────────────────────────────────────────────── async def _subscribe( @@ -675,53 +336,6 @@ async def _replay_final( for ev in stored_events: yield {"event": ev["event_type"], "data": ev["event_data"]} - candidates = [ - { - "identifier": c.identifier, - "identifier_type": c.identifier_type, - "name": c.name, - "rank": c.rank, - "notes": c.notes, - "scores": c.scores, - "attributes": c.attributes, - } - for c in investigation.candidates - ] - findings = [ - { - "title": f.title, - "detail": f.detail, - "hypothesis_id": f.hypothesis_id, - "evidence_type": f.evidence_type, - "evidence": f.evidence, - } - for f in investigation.findings - ] - hypotheses = [ - { - "id": h.id, - "statement": h.statement, - "rationale": h.rationale, - "status": h.status.value, - "parent_id": h.parent_id, - "confidence": h.confidence, - "supporting_evidence": h.supporting_evidence, - "contradicting_evidence": h.contradicting_evidence, - } - for h in investigation.hypotheses - ] - negative_controls = [ - { - "identifier": nc.identifier, - "identifier_type": nc.identifier_type, - "name": nc.name, - "score": nc.score, - "threshold": nc.threshold, - "correctly_classified": nc.correctly_classified, - "source": nc.source, - } - for nc in investigation.negative_controls - ] data = json.dumps( { "event": SSEEventType.COMPLETED.value, @@ -730,10 +344,10 @@ async def _replay_final( "candidate_count": len(investigation.candidates), "summary": investigation.summary, "cost": investigation.cost_data, - "candidates": candidates, - "findings": findings, - "hypotheses": hypotheses, - "negative_controls": negative_controls, + "candidates": serialize_candidates(investigation), + "findings": serialize_findings(investigation), + "hypotheses": serialize_hypotheses(investigation), + "negative_controls": serialize_negative_controls(investigation), "prompt": investigation.prompt, }, } @@ -750,79 +364,3 @@ async def _replay_final( } ) yield {"event": SSEEventType.ERROR.value, "data": data} - - -def _to_detail(inv: Investigation) -> InvestigationDetail: - findings = [ - { - "title": f.title, - "detail": f.detail, - "hypothesis_id": f.hypothesis_id, - "evidence_type": f.evidence_type, - "evidence": f.evidence, - } - for f in inv.findings - ] - candidates = [ - { - "identifier": c.identifier, - "identifier_type": c.identifier_type, - "name": c.name, - "rank": c.rank, - "notes": c.notes, - "scores": c.scores, - "attributes": c.attributes, - } - for c in inv.candidates - ] - hypotheses = [ - { - "id": h.id, - "statement": h.statement, - "rationale": h.rationale, - "status": h.status.value, - "parent_id": h.parent_id, - "confidence": h.confidence, - "supporting_evidence": h.supporting_evidence, - "contradicting_evidence": h.contradicting_evidence, - } - for h in inv.hypotheses - ] - experiments = [ - { - "id": e.id, - "hypothesis_id": e.hypothesis_id, - "description": e.description, - "tool_plan": e.tool_plan, - "status": e.status.value, - } - for e in inv.experiments - ] - negative_controls = [ - { - "identifier": nc.identifier, - "identifier_type": nc.identifier_type, - "name": nc.name, - "score": nc.score, - "threshold": nc.threshold, - "correctly_classified": nc.correctly_classified, - "source": nc.source, - } - for nc in inv.negative_controls - ] - return InvestigationDetail( - id=inv.id, - prompt=inv.prompt, - status=inv.status.value, - hypotheses=hypotheses, - experiments=experiments, - current_hypothesis_id=inv.current_hypothesis_id, - current_experiment_id=inv.current_experiment_id, - findings=findings, - candidates=candidates, - negative_controls=negative_controls, - citations=inv.citations, - summary=inv.summary, - created_at=inv.created_at.isoformat(), - cost_data=inv.cost_data, - ) diff --git a/server/src/ehrlich/api/routes/methodology.py b/server/src/ehrlich/api/routes/methodology.py index 4de5238..147db49 100644 --- a/server/src/ehrlich/api/routes/methodology.py +++ b/server/src/ehrlich/api/routes/methodology.py @@ -7,7 +7,10 @@ from fastapi import APIRouter from pydantic import BaseModel -from ehrlich.api.routes.investigation import _build_domain_registry, _build_registry +from ehrlich.investigation.application.registry_factory import ( + build_domain_registry, + build_tool_registry, +) router = APIRouter(tags=["methodology"]) @@ -295,8 +298,8 @@ class MethodologyResponse(BaseModel): @router.get("/methodology") async def get_methodology() -> MethodologyResponse: - registry = _build_registry() - domain_registry = _build_domain_registry() + registry = build_tool_registry() + domain_registry = build_domain_registry() # Build domain info from registry domains: list[DomainInfo] = [] diff --git a/server/src/ehrlich/api/routes/stats.py b/server/src/ehrlich/api/routes/stats.py index 3e72301..793f030 100644 --- a/server/src/ehrlich/api/routes/stats.py +++ b/server/src/ehrlich/api/routes/stats.py @@ -3,8 +3,11 @@ from fastapi import APIRouter from pydantic import BaseModel -from ehrlich.api.routes.investigation import _build_domain_registry, _build_registry from ehrlich.api.sse import SSEEventType +from ehrlich.investigation.application.registry_factory import ( + build_domain_registry, + build_tool_registry, +) router = APIRouter(tags=["stats"]) @@ -22,8 +25,8 @@ class StatsResponse(BaseModel): @router.get("/stats") async def get_stats() -> StatsResponse: - registry = _build_registry() - domain_registry = _build_domain_registry() + registry = build_tool_registry() + domain_registry = build_domain_registry() return StatsResponse( tool_count=len(registry.list_tools()), domain_count=len(domain_registry.all_configs()), diff --git a/server/src/ehrlich/api/schemas/__init__.py b/server/src/ehrlich/api/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/ehrlich/api/schemas/investigation.py b/server/src/ehrlich/api/schemas/investigation.py new file mode 100644 index 0000000..a8137ba --- /dev/null +++ b/server/src/ehrlich/api/schemas/investigation.py @@ -0,0 +1,155 @@ +"""Pydantic request/response schemas and serialization helpers for investigation routes.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from pydantic import BaseModel + +if TYPE_CHECKING: + from ehrlich.investigation.domain.investigation import Investigation + + +# ── Request/Response Models ───────────────────────────────────────────── + + +class InvestigateRequest(BaseModel): + prompt: str + director_tier: str = "opus" + file_ids: list[str] = [] + + +class InvestigateResponse(BaseModel): + id: str + status: str + + +class CreditBalanceResponse(BaseModel): + credits: int + is_byok: bool + + +class ApproveRequest(BaseModel): + approved_ids: list[str] + rejected_ids: list[str] = [] + + +class InvestigationSummary(BaseModel): + id: str + prompt: str + status: str + created_at: str + candidate_count: int + + +class InvestigationDetail(BaseModel): + id: str + prompt: str + status: str + hypotheses: list[dict[str, Any]] + experiments: list[dict[str, Any]] + current_hypothesis_id: str + current_experiment_id: str + findings: list[dict[str, Any]] + candidates: list[dict[str, Any]] + negative_controls: list[dict[str, Any]] + citations: list[str] + summary: str + created_at: str + cost_data: dict[str, object] + + +# ── Serialization Helpers ─────────────────────────────────────────────── +# Used by both _to_detail() and _replay_final() to eliminate duplication. + + +def serialize_candidates(investigation: Investigation) -> list[dict[str, Any]]: + return [ + { + "identifier": c.identifier, + "identifier_type": c.identifier_type, + "name": c.name, + "rank": c.rank, + "notes": c.notes, + "scores": c.scores, + "attributes": c.attributes, + } + for c in investigation.candidates + ] + + +def serialize_findings(investigation: Investigation) -> list[dict[str, Any]]: + return [ + { + "title": f.title, + "detail": f.detail, + "hypothesis_id": f.hypothesis_id, + "evidence_type": f.evidence_type, + "evidence": f.evidence, + } + for f in investigation.findings + ] + + +def serialize_hypotheses(investigation: Investigation) -> list[dict[str, Any]]: + return [ + { + "id": h.id, + "statement": h.statement, + "rationale": h.rationale, + "status": h.status.value, + "parent_id": h.parent_id, + "confidence": h.confidence, + "supporting_evidence": h.supporting_evidence, + "contradicting_evidence": h.contradicting_evidence, + } + for h in investigation.hypotheses + ] + + +def serialize_negative_controls(investigation: Investigation) -> list[dict[str, Any]]: + return [ + { + "identifier": nc.identifier, + "identifier_type": nc.identifier_type, + "name": nc.name, + "score": nc.score, + "threshold": nc.threshold, + "correctly_classified": nc.correctly_classified, + "source": nc.source, + } + for nc in investigation.negative_controls + ] + + +def serialize_experiments(investigation: Investigation) -> list[dict[str, Any]]: + return [ + { + "id": e.id, + "hypothesis_id": e.hypothesis_id, + "description": e.description, + "tool_plan": e.tool_plan, + "status": e.status.value, + } + for e in investigation.experiments + ] + + +def to_detail(investigation: Investigation) -> InvestigationDetail: + """Convert an Investigation entity to an InvestigationDetail response.""" + return InvestigationDetail( + id=investigation.id, + prompt=investigation.prompt, + status=investigation.status.value, + hypotheses=serialize_hypotheses(investigation), + experiments=serialize_experiments(investigation), + current_hypothesis_id=investigation.current_hypothesis_id, + current_experiment_id=investigation.current_experiment_id, + findings=serialize_findings(investigation), + candidates=serialize_candidates(investigation), + negative_controls=serialize_negative_controls(investigation), + citations=investigation.citations, + summary=investigation.summary, + created_at=investigation.created_at.isoformat(), + cost_data=investigation.cost_data, + ) diff --git a/server/src/ehrlich/investigation/application/orchestrator_factory.py b/server/src/ehrlich/investigation/application/orchestrator_factory.py new file mode 100644 index 0000000..a3cc60d --- /dev/null +++ b/server/src/ehrlich/investigation/application/orchestrator_factory.py @@ -0,0 +1,70 @@ +"""Orchestrator construction with Anthropic adapter wiring.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.application.multi_orchestrator import MultiModelOrchestrator +from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter + +if TYPE_CHECKING: + from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.domain.domain_registry import DomainRegistry + from ehrlich.investigation.domain.mcp_config import MCPServerConfig + from ehrlich.investigation.infrastructure.mcp_bridge import MCPBridge + from ehrlich.investigation.infrastructure.repository import InvestigationRepository + + +def create_orchestrator( + settings: Any, + registry: ToolRegistry, + repository: InvestigationRepository, + domain_registry: DomainRegistry | None = None, + mcp_bridge: MCPBridge | None = None, + mcp_configs: list[MCPServerConfig] | None = None, + api_key_override: str | None = None, + director_model_override: str | None = None, +) -> MultiModelOrchestrator: + """Wire up Anthropic adapters and build a MultiModelOrchestrator.""" + api_key = api_key_override or settings.anthropic_api_key or None + + director_model = director_model_override or settings.director_model + + # effort is only supported by Opus models (4.5+) + director_effort = settings.director_effort if "opus" in director_model else None + + thinking = None + if settings.director_thinking == "enabled": + thinking = { + "type": "enabled", + "budget_tokens": settings.director_thinking_budget, + } + + director = AnthropicClientAdapter( + model=director_model, + api_key=api_key, + max_tokens=32768, + effort=director_effort, + thinking=thinking, + ) + researcher = AnthropicClientAdapter( + model=settings.researcher_model, + api_key=api_key, + ) + summarizer = AnthropicClientAdapter( + model=settings.summarizer_model, + api_key=api_key, + ) + return MultiModelOrchestrator( + director=director, + researcher=researcher, + summarizer=summarizer, + registry=registry, + max_iterations_per_experiment=settings.max_iterations_per_experiment, + summarizer_threshold=settings.summarizer_threshold, + require_approval=True, + repository=repository, + domain_registry=domain_registry, + mcp_bridge=mcp_bridge, + mcp_configs=mcp_configs, + ) diff --git a/server/src/ehrlich/investigation/application/registry_factory.py b/server/src/ehrlich/investigation/application/registry_factory.py new file mode 100644 index 0000000..009e8c3 --- /dev/null +++ b/server/src/ehrlich/investigation/application/registry_factory.py @@ -0,0 +1,276 @@ +"""Tool registry, domain registry, and MCP config construction. + +Single source of truth for what tools and domains exist. +""" + +from __future__ import annotations + +import os +from functools import lru_cache + +from ehrlich.analysis.tools import ( + analyze_substructures, + assess_threats, + compute_cost_effectiveness, + compute_properties, + estimate_did, + estimate_psm, + estimate_rdd, + estimate_synthetic_control, + explore_dataset, + run_categorical_test, + run_statistical_test, + search_bioactivity, + search_compounds, + search_pharmacology, +) +from ehrlich.chemistry.tools import ( + compute_descriptors, + compute_fingerprint, + generate_3d, + substructure_match, + tanimoto_similarity, + validate_smiles, +) +from ehrlich.impact.tools import ( + compare_programs, + fetch_benchmark, + search_economic_indicators, +) +from ehrlich.investigation.application.tool_registry import ToolRegistry +from ehrlich.investigation.domain.domain_registry import DomainRegistry +from ehrlich.investigation.domain.domains.impact import IMPACT_EVALUATION +from ehrlich.investigation.domain.domains.molecular import MOLECULAR_SCIENCE +from ehrlich.investigation.domain.domains.nutrition import NUTRITION_SCIENCE +from ehrlich.investigation.domain.domains.training import TRAINING_SCIENCE +from ehrlich.investigation.domain.mcp_config import MCPServerConfig +from ehrlich.investigation.tools import ( + conclude_investigation, + design_experiment, + evaluate_hypothesis, + propose_hypothesis, + query_uploaded_data, + record_finding, + record_negative_control, + search_prior_research, +) +from ehrlich.investigation.tools_viz import ( + render_admet_radar, + render_binding_scatter, + render_causal_diagram, + render_dose_response, + render_evidence_matrix, + render_forest_plot, + render_funnel_plot, + render_geographic_comparison, + render_muscle_heatmap, + render_nutrient_adequacy, + render_nutrient_comparison, + render_parallel_trends, + render_performance_chart, + render_program_dashboard, + render_rdd_plot, + render_therapeutic_window, + render_training_timeline, +) +from ehrlich.literature.tools import get_reference, search_citations, search_literature +from ehrlich.nutrition.tools import ( + analyze_nutrient_ratios, + assess_nutrient_adequacy, + check_intake_safety, + check_interactions, + compare_nutrients, + compute_inflammatory_index, + search_nutrient_data, + search_supplement_evidence, + search_supplement_labels, + search_supplement_safety, +) +from ehrlich.prediction.tools import ( + cluster_compounds, + cluster_data, + predict_candidates, + predict_scores, + train_classifier, + train_model, +) +from ehrlich.simulation.tools import ( + assess_resistance, + dock_against_target, + fetch_toxicity_profile, + get_protein_annotation, + predict_admet, + search_disease_targets, + search_protein_targets, +) +from ehrlich.training.tools import ( + analyze_training_evidence, + assess_injury_risk, + compare_protocols, + compute_dose_response, + compute_performance_model, + compute_training_metrics, + plan_periodization, + search_clinical_trials, + search_exercise_database, + search_pubmed_training, + search_training_literature, +) + + +@lru_cache(maxsize=1) +def build_tool_registry() -> ToolRegistry: + """Build the tool registry (singleton — immutable after construction).""" + registry = ToolRegistry() + + _chem = frozenset({"chemistry"}) + _lit = frozenset({"literature"}) + _analysis = frozenset({"analysis"}) + _pred = frozenset({"prediction"}) + _sim = frozenset({"simulation"}) + _training = frozenset({"training"}) + _training_clinical = frozenset({"training", "clinical"}) + _training_exercise = frozenset({"training", "exercise"}) + _nutrition = frozenset({"nutrition"}) + _nutrition_safety = frozenset({"nutrition", "safety"}) + _impact = frozenset({"impact"}) + _causal = frozenset({"causal"}) + _ml = frozenset({"ml"}) + _viz = frozenset({"visualization"}) + _chem_viz = frozenset({"chemistry", "visualization"}) + _sim_viz = frozenset({"simulation", "visualization"}) + _training_viz = frozenset({"training", "visualization"}) + _nutrition_viz = frozenset({"nutrition", "visualization"}) + _impact_viz = frozenset({"impact", "visualization"}) + _causal_viz = frozenset({"causal", "visualization"}) + + tagged_tools = [ + # Chemistry (6) + ("validate_smiles", validate_smiles, _chem), + ("compute_descriptors", compute_descriptors, _chem), + ("compute_fingerprint", compute_fingerprint, _chem), + ("tanimoto_similarity", tanimoto_similarity, _chem), + ("generate_3d", generate_3d, _chem), + ("substructure_match", substructure_match, _chem), + # Literature (3) + ("search_literature", search_literature, _lit), + ("get_reference", get_reference, _lit), + ("search_citations", search_citations, _lit), + # Analysis (6) + ("explore_dataset", explore_dataset, _analysis), + ("search_compounds", search_compounds, _analysis), + ("search_bioactivity", search_bioactivity, _analysis), + ("analyze_substructures", analyze_substructures, _analysis), + ("compute_properties", compute_properties, _analysis), + ("search_pharmacology", search_pharmacology, _analysis), + # Prediction (3) -- molecular-specific + ("train_model", train_model, _pred), + ("predict_candidates", predict_candidates, _pred), + ("cluster_compounds", cluster_compounds, _pred), + # ML (3) -- domain-agnostic + ("train_classifier", train_classifier, _ml), + ("predict_scores", predict_scores, _ml), + ("cluster_data", cluster_data, _ml), + # Simulation (7) + ("search_protein_targets", search_protein_targets, _sim), + ("dock_against_target", dock_against_target, _sim), + ("predict_admet", predict_admet, _sim), + ("fetch_toxicity_profile", fetch_toxicity_profile, _sim), + ("assess_resistance", assess_resistance, _sim), + ("get_protein_annotation", get_protein_annotation, _sim), + ("search_disease_targets", search_disease_targets, _sim), + # Training Science (11) + ("search_training_literature", search_training_literature, _training), + ("analyze_training_evidence", analyze_training_evidence, _training), + ("compare_protocols", compare_protocols, _training), + ("assess_injury_risk", assess_injury_risk, _training), + ("compute_training_metrics", compute_training_metrics, _training), + ("search_clinical_trials", search_clinical_trials, _training_clinical), + ("search_pubmed_training", search_pubmed_training, _training), + ("search_exercise_database", search_exercise_database, _training_exercise), + ("compute_performance_model", compute_performance_model, _training), + ("compute_dose_response", compute_dose_response, _training), + ("plan_periodization", plan_periodization, _training_exercise), + # Nutrition Science (10) + ("search_supplement_evidence", search_supplement_evidence, _nutrition), + ("search_supplement_labels", search_supplement_labels, _nutrition), + ("search_nutrient_data", search_nutrient_data, _nutrition), + ("search_supplement_safety", search_supplement_safety, _nutrition_safety), + ("compare_nutrients", compare_nutrients, _nutrition), + ("assess_nutrient_adequacy", assess_nutrient_adequacy, _nutrition), + ("check_intake_safety", check_intake_safety, _nutrition_safety), + ("check_interactions", check_interactions, _nutrition_safety), + ("analyze_nutrient_ratios", analyze_nutrient_ratios, _nutrition), + ("compute_inflammatory_index", compute_inflammatory_index, _nutrition), + # Impact Evaluation (3) + ("search_economic_indicators", search_economic_indicators, _impact), + ("fetch_benchmark", fetch_benchmark, _impact), + ("compare_programs", compare_programs, _impact), + # Causal Inference (6) -- domain-agnostic + ("estimate_did", estimate_did, _causal), + ("estimate_psm", estimate_psm, _causal), + ("estimate_rdd", estimate_rdd, _causal), + ("estimate_synthetic_control", estimate_synthetic_control, _causal), + ("assess_threats", assess_threats, _causal), + ("compute_cost_effectiveness", compute_cost_effectiveness, _causal), + # Visualization (17) + ("render_binding_scatter", render_binding_scatter, _chem_viz), + ("render_admet_radar", render_admet_radar, _sim_viz), + ("render_training_timeline", render_training_timeline, _training_viz), + ("render_muscle_heatmap", render_muscle_heatmap, _training_viz), + ("render_forest_plot", render_forest_plot, _viz), + ("render_evidence_matrix", render_evidence_matrix, _viz), + ("render_performance_chart", render_performance_chart, _training_viz), + ("render_funnel_plot", render_funnel_plot, _viz), + ("render_dose_response", render_dose_response, _training_viz), + ("render_nutrient_comparison", render_nutrient_comparison, _nutrition_viz), + ("render_nutrient_adequacy", render_nutrient_adequacy, _nutrition_viz), + ("render_therapeutic_window", render_therapeutic_window, _nutrition_viz), + ("render_program_dashboard", render_program_dashboard, _impact_viz), + ("render_geographic_comparison", render_geographic_comparison, _impact_viz), + ("render_parallel_trends", render_parallel_trends, _impact_viz), + ("render_rdd_plot", render_rdd_plot, _causal_viz), + ("render_causal_diagram", render_causal_diagram, _causal_viz), + # Statistics (2) -- universal, no tags + ("run_statistical_test", run_statistical_test, None), + ("run_categorical_test", run_categorical_test, None), + # Investigation control (8) -- universal, no tags + ("record_finding", record_finding, None), + ("conclude_investigation", conclude_investigation, None), + ("propose_hypothesis", propose_hypothesis, None), + ("design_experiment", design_experiment, None), + ("evaluate_hypothesis", evaluate_hypothesis, None), + ("record_negative_control", record_negative_control, None), + ("search_prior_research", search_prior_research, None), + ("query_uploaded_data", query_uploaded_data, None), + ] + for name, func, tags in tagged_tools: + registry.register(name, func, tags) + return registry + + +@lru_cache(maxsize=1) +def build_domain_registry() -> DomainRegistry: + """Build the domain registry (singleton — immutable after construction).""" + domain_registry = DomainRegistry() + domain_registry.register(MOLECULAR_SCIENCE) + domain_registry.register(TRAINING_SCIENCE) + domain_registry.register(NUTRITION_SCIENCE) + domain_registry.register(IMPACT_EVALUATION) + return domain_registry + + +def build_mcp_configs() -> list[MCPServerConfig]: + """Build MCP server configs from environment. Currently supports Excalidraw.""" + configs: list[MCPServerConfig] = [] + if os.environ.get("EHRLICH_MCP_EXCALIDRAW", "").lower() in ("1", "true"): + configs.append( + MCPServerConfig( + name="excalidraw", + transport="stdio", + command="npx", + args=("-y", "@anthropic/claude-code-mcp", "excalidraw"), + tags=frozenset({"visualization"}), + ) + ) + return configs diff --git a/server/tests/integration/test_e2e.py b/server/tests/integration/test_e2e.py index 78789cc..a0d77e0 100644 --- a/server/tests/integration/test_e2e.py +++ b/server/tests/integration/test_e2e.py @@ -13,9 +13,9 @@ import pytest -from ehrlich.api.routes.investigation import _build_registry from ehrlich.api.sse import SSEEventType, domain_event_to_sse from ehrlich.investigation.application.multi_orchestrator import MultiModelOrchestrator +from ehrlich.investigation.application.registry_factory import build_tool_registry from ehrlich.investigation.domain.events import ToolResultEvent from ehrlich.investigation.domain.investigation import Investigation, InvestigationStatus @@ -138,9 +138,9 @@ def _make_clients() -> tuple[AsyncMock, AsyncMock, AsyncMock]: class TestToolRegistry: def test_build_registry_has_expected_tools(self) -> None: - registry = _build_registry() + registry = build_tool_registry() tools = registry.list_tools() - assert len(tools) == 78 + assert len(tools) == 85 assert "validate_smiles" in tools assert "search_literature" in tools assert "search_citations" in tools @@ -189,9 +189,9 @@ def test_build_registry_has_expected_tools(self) -> None: assert "render_parallel_trends" in tools def test_all_tools_have_schemas(self) -> None: - registry = _build_registry() + registry = build_tool_registry() schemas = registry.list_schemas() - assert len(schemas) == 78 + assert len(schemas) == 85 for schema in schemas: assert "name" in schema assert "description" in schema @@ -226,7 +226,7 @@ async def test_full_investigation_flow(self) -> None: ] ) - registry = _build_registry() + registry = build_tool_registry() orchestrator = MultiModelOrchestrator( director=director, researcher=researcher, @@ -261,7 +261,7 @@ async def test_sse_event_generation(self) -> None: ) researcher.create_message = AsyncMock(return_value=_make_response([_text("Done.")])) - registry = _build_registry() + registry = build_tool_registry() orchestrator = MultiModelOrchestrator( director=director, researcher=researcher, @@ -308,7 +308,7 @@ async def test_chemistry_tool_dispatch(self) -> None: ] ) - registry = _build_registry() + registry = build_tool_registry() orchestrator = MultiModelOrchestrator( director=director, researcher=researcher, From b52ad69f4385f55eee6e428dc8ab2b311449893e Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 09:32:41 -0700 Subject: [PATCH 14/35] refactor(prompts): split prompts.py into prompts/ package (14C) Split prompts.py (1505 lines) into focused modules: - prompts/constants.py: 6 static prompt strings - prompts/director.py: 4 Director-phase builder functions - prompts/builders.py: 6 non-Director builders + _DEFAULT_CATEGORIES Dead code removed: - SCIENTIST_SYSTEM_PROMPT (103 lines, zero imports) - _build_prior_context (20 lines, zero imports) No backward-compat re-exports; consumers import directly from submodules. --- docs/roadmap/phase-14-engine-hardening.md | 12 +- .../application/multi_orchestrator.py | 16 +- .../investigation/application/prompts.py | 1504 ----------------- .../application/prompts/__init__.py | 1 + .../application/prompts/builders.py | 283 ++++ .../application/prompts/constants.py | 781 +++++++++ .../application/prompts/director.py | 341 ++++ .../application/researcher_executor.py | 2 +- server/tests/investigation/test_prompts.py | 14 +- .../tests/investigation/test_uploaded_file.py | 6 +- 10 files changed, 1435 insertions(+), 1525 deletions(-) delete mode 100644 server/src/ehrlich/investigation/application/prompts.py create mode 100644 server/src/ehrlich/investigation/application/prompts/__init__.py create mode 100644 server/src/ehrlich/investigation/application/prompts/builders.py create mode 100644 server/src/ehrlich/investigation/application/prompts/constants.py create mode 100644 server/src/ehrlich/investigation/application/prompts/director.py diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md index bcd52dc..d2142c3 100644 --- a/docs/roadmap/phase-14-engine-hardening.md +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -39,14 +39,14 @@ Moved infrastructure concerns out of `api/routes/investigation.py` (829 lines) i Result: `api/routes/investigation.py` from 829 to 366 lines (56% reduction). Three new modules: `registry_factory.py` (237 lines), `orchestrator_factory.py` (71 lines), `api/schemas/investigation.py` (140 lines). -## 14C: Prompt Module Split -- TODO +## 14C: Prompt Module Split -- DONE -Split `prompts.py` (1,512 lines) into focused modules. +Split `prompts.py` (1,505 lines) into `prompts/` package with 3 focused modules. Removed dead code (`SCIENTIST_SYSTEM_PROMPT`, `_build_prior_context` -- zero external imports). -- [ ] `prompts/__init__.py` -- re-exports for backward compatibility -- [ ] `prompts/constants.py` -- `SCIENTIST_SYSTEM_PROMPT`, `SUMMARIZER_PROMPT`, `RESEARCHER_EXPERIMENT_PROMPT` -- [ ] `prompts/director.py` -- `DIRECTOR_FORMULATION_PROMPT`, `DIRECTOR_EXPERIMENT_PROMPT`, `DIRECTOR_EVALUATION_PROMPT`, `DIRECTOR_SYNTHESIS_PROMPT` -- [ ] `prompts/builders.py` -- `build_pico_and_classification_prompt`, `build_literature_survey_prompt`, `build_literature_assessment_prompt`, `build_formulation_prompt`, `build_experiment_prompt`, `build_synthesis_prompt`, `build_researcher_prompt`, `build_multi_investigation_context`, `build_uploaded_data_context`, `_build_prior_context` +- [x] `prompts/__init__.py` -- package docstring only (no re-exports; consumers import from submodules directly) +- [x] `prompts/constants.py` -- 6 static prompt strings (`DIRECTOR_FORMULATION_PROMPT`, `DIRECTOR_EXPERIMENT_PROMPT`, `DIRECTOR_EVALUATION_PROMPT`, `DIRECTOR_SYNTHESIS_PROMPT`, `RESEARCHER_EXPERIMENT_PROMPT`, `SUMMARIZER_PROMPT`) +- [x] `prompts/director.py` -- 4 Director-phase builder functions (`build_formulation_prompt`, `build_experiment_prompt`, `build_synthesis_prompt`, `build_multi_investigation_context`) +- [x] `prompts/builders.py` -- 6 non-Director builders (`build_pico_and_classification_prompt`, `build_literature_survey_prompt`, `build_literature_assessment_prompt`, `build_researcher_prompt`, `build_uploaded_data_context`, `_DEFAULT_CATEGORIES`) ## 14D: Domain Tool Enrichment -- TODO diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index 4c60a2d..771e4c9 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -9,21 +9,25 @@ from ehrlich.investigation.application.batch_executor import run_experiment_batch from ehrlich.investigation.application.cost_tracker import CostTracker from ehrlich.investigation.application.diagram_builder import generate_diagram -from ehrlich.investigation.application.prompts import ( +from ehrlich.investigation.application.prompts.builders import ( + build_literature_assessment_prompt, + build_literature_survey_prompt, + build_pico_and_classification_prompt, + build_researcher_prompt, + build_uploaded_data_context, +) +from ehrlich.investigation.application.prompts.constants import ( DIRECTOR_EVALUATION_PROMPT, DIRECTOR_EXPERIMENT_PROMPT, DIRECTOR_FORMULATION_PROMPT, DIRECTOR_SYNTHESIS_PROMPT, RESEARCHER_EXPERIMENT_PROMPT, +) +from ehrlich.investigation.application.prompts.director import ( build_experiment_prompt, build_formulation_prompt, - build_literature_assessment_prompt, - build_literature_survey_prompt, build_multi_investigation_context, - build_pico_and_classification_prompt, - build_researcher_prompt, build_synthesis_prompt, - build_uploaded_data_context, ) from ehrlich.investigation.application.researcher_executor import ( maybe_viz_event, diff --git a/server/src/ehrlich/investigation/application/prompts.py b/server/src/ehrlich/investigation/application/prompts.py deleted file mode 100644 index 8ae6452..0000000 --- a/server/src/ehrlich/investigation/application/prompts.py +++ /dev/null @@ -1,1504 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from ehrlich.investigation.domain.domain_config import DomainConfig - from ehrlich.investigation.domain.investigation import Investigation - from ehrlich.investigation.domain.uploaded_file import UploadedFile - -SCIENTIST_SYSTEM_PROMPT = """\ -You are Ehrlich, an AI molecular discovery scientist named after \ -Paul Ehrlich, the father of the "magic bullet" concept -- finding \ -the right molecule for any target. - - -You have access to cheminformatics, machine learning, molecular \ -simulation, data search, and literature search tools. Your goal \ -is to investigate molecular questions using the hypothesis-driven \ -scientific method. The user's research question defines the domain. \ -Adapt your scientific vocabulary and approach accordingly. - -Follow this hypothesis-driven workflow: - -1. LITERATURE SURVEY - - Use `search_literature` and `get_reference` to understand \ -the current state of knowledge. - - Identify known active compounds, mechanisms, and promising \ -compound classes. - - Record key findings with `record_finding`. - -2. FORMULATE HYPOTHESES (2-4) - - Based on literature, propose 2-4 testable hypotheses using \ -`propose_hypothesis`. - - Each hypothesis must be specific and falsifiable. - -3. FOR EACH HYPOTHESIS -- Design, Execute, Evaluate - - Call `design_experiment` with specific tools and \ -success/failure criteria. - - Execute the experiment using the planned tools. - - Record findings with `record_finding`, linking each to the \ -hypothesis with evidence_type ('supporting' or 'contradicting'). - - After gathering evidence, call `evaluate_hypothesis` with \ -'supported', 'refuted', or 'revised'. - - If revised: propose a new refined hypothesis and test it. - -4. NEGATIVE CONTROLS - - For your best model, use `record_negative_control` to test \ -2-3 known inactive compounds. - - Good models should score negatives below 0.5. - -5. SYNTHESIZE AND CONCLUDE - - Call `conclude_investigation` with: - - Summary of all hypothesis outcomes - - Ranked candidate list with multi-criteria scores - - Negative control validation summary - - Full citations - - - -Data Sources: -- `search_literature` / `get_reference` -- Semantic Scholar papers -- `explore_dataset` / `search_bioactivity` -- ChEMBL bioactivity \ -(any assay type: MIC, Ki, EC50, IC50, Kd) -- `search_compounds` -- PubChem compound search -- `search_protein_targets` -- RCSB PDB protein target discovery -- `get_protein_annotation` -- UniProt protein function, disease, \ -GO terms -- `search_disease_targets` -- Open Targets disease-target \ -associations (scored) -- `search_pharmacology` -- GtoPdb curated pharmacology \ -(pKi, pIC50) -- `fetch_toxicity_profile` -- EPA CompTox environmental toxicity - -Chemistry and Prediction: -- `validate_smiles`, `compute_descriptors`, \ -`compute_fingerprint`, `tanimoto_similarity` -- `generate_3d`, `substructure_match`, `analyze_substructures`, \ -`compute_properties` -- `train_model`, `predict_candidates`, `cluster_compounds` - -Simulation: -- `dock_against_target`, `predict_admet`, `assess_resistance` - -Investigation Control: -- `propose_hypothesis`, `design_experiment`, \ -`evaluate_hypothesis` -- `record_finding`, `record_negative_control`, \ -`conclude_investigation` - - - -1. ALWAYS propose hypotheses before running experiments. -2. ALWAYS link findings to hypotheses using hypothesis_id. -3. ALWAYS evaluate each hypothesis after experiments complete. -4. ALWAYS run negative controls before concluding. -5. Explain your scientific reasoning before calling a tool. -6. Call `record_finding` after each significant discovery. -7. Cite papers by DOI when referencing literature. -8. Use `validate_smiles` before passing SMILES if uncertain. -9. If a tool returns an error, try an alternative approach. -10. Be quantitative: report exact numbers, scores, and \ -confidence intervals. - - - -Your investigation should produce: -- 2-4 tested hypotheses with clear outcomes -- 5-10 recorded findings linked to hypotheses -- 3-5 ranked candidate molecules with multi-criteria scores -- 2-3 negative control validations -- At least 3 literature citations with DOIs -""" - -DIRECTOR_FORMULATION_PROMPT = """\ -You are the Director of a molecular discovery investigation. \ -You formulate hypotheses and design the research strategy but \ -do NOT execute tools yourself. - - -Given the user's research prompt and literature survey results, \ -formulate 2-4 testable hypotheses. Each hypothesis must be: -- Specific and falsifiable -- Grounded in the literature findings provided -- Testable with the available cheminformatics and ML tools -- Orthogonal to sibling hypotheses: each must attack a different \ -mechanism, pathway, or data source - -Hypotheses tested in parallel must maximize scientific coverage: -- Different causal mechanisms or molecular targets -- Different primary data sources or tool chains -- Different validation strategies (e.g., ML prediction vs \ -structural docking vs substructure enrichment) -Do NOT formulate hypotheses that would produce overlapping experiments. - -If prior investigation results are provided in \ -, leverage their outcomes: -- Build on supported hypotheses from related investigations -- Avoid repeating refuted approaches -- Consider candidates already identified as starting points - -Also identify 1-3 negative controls (molecules known to be \ -inactive) AND 1-2 positive controls (molecules known to be \ -active against the target). Both are essential for validation: \ -negative controls confirm specificity, positive controls confirm \ -the pipeline can detect true actives. Without positive controls, \ -pipeline failures are undetectable (Zhang et al., 1999). - - - - - -Find novel beta-lactamase inhibitors to combat MRSA resistance - - -- Beta-lactamase enzymes (Class A, C, D) hydrolyze beta-lactam \ -ring; PBP2a confers methicillin resistance in MRSA -- Avibactam (diazabicyclooctane) inhibits Class A/C with Ki \ -~1 nM; clavulanate is first-gen but weak vs Class C -- Boronic acid scaffolds show broad-spectrum inhibition; \ -vaborbactam approved 2017 -- MRSA strains increasingly co-express multiple beta-lactamase \ -classes - - -{ - "hypotheses": [ - { - "statement": "Diazabicyclooctane derivatives with C2 \ -sulfonamide substituents will show dual Class A/C beta-lactamase \ -inhibition with Ki below 50 nM", - "rationale": "Avibactam's DBO core achieves covalent \ -reversible inhibition of both classes; C2 modifications with \ -electron-withdrawing sulfonamide groups enhance binding \ -to the conserved Ser70 active site", - "prediction": "Docking against Class A and Class C \ -beta-lactamases will show affinity < -7 kcal/mol for 3+ DBO \ -derivatives; ML model predicts Ki < 50 nM", - "null_prediction": "DBO derivatives show no preferential \ -binding vs non-sulfonamide controls; Ki > 500 nM", - "success_criteria": ">=3 candidates with docking < -7 \ -kcal/mol AND predicted Ki < 100 nM", - "failure_criteria": "<2 compounds meet docking threshold \ -OR all fail ADMET drug-likeness", - "scope": "MRSA beta-lactamases Class A/C; small molecules \ -MW < 500", - "hypothesis_type": "mechanistic", - "prior_confidence": 0.7 - }, - { - "statement": "Boronic acid compounds with MW below 350 Da \ -and LogP below 1.0 will penetrate MRSA cell wall and inhibit \ -PBP2a-associated beta-lactamases", - "rationale": "Vaborbactam demonstrates boronic acid \ -viability but has limited Gram-positive penetration; smaller, \ -more hydrophilic analogs may overcome MRSA's thick \ -peptidoglycan barrier", - "prediction": "Boronic acids with MW < 350 show MIC < 4 \ -ug/mL against MRSA; ADMET predicts oral bioavailability", - "null_prediction": "No boronic acid compounds show MIC \ -improvement over vaborbactam baseline (32 ug/mL)", - "success_criteria": ">=2 compounds with predicted MIC < 8 \ -ug/mL AND passes Lipinski criteria", - "failure_criteria": "All candidates fail Lipinski OR MIC \ -> 16 ug/mL", - "scope": "Gram-positive MRSA; orally bioavailable", - "hypothesis_type": "pharmacological", - "prior_confidence": 0.5 - } - ], - "negative_controls": [ - { - "smiles": "CC(=O)Oc1ccccc1C(=O)O", - "name": "Aspirin", - "source": "Non-antimicrobial NSAID; validated inactive \ -in ChEMBL" - }, - { - "smiles": "CC(C)Cc1ccc(cc1)C(C)C(=O)O", - "name": "Ibuprofen", - "source": "NSAID structurally unrelated to any BLI scaffold" - } - ], - "positive_controls": [ - { - "identifier": "CC1CC2(CC(=O)N1)C(=O)N(S2(=O)=O)O", - "name": "Avibactam", - "known_activity": "Ki ~1 nM vs Class A beta-lactamase", - "source": "FDA-approved BLI, gold standard" - } - ] -} - - - - - -Identify BACE1 inhibitors for Alzheimer's disease treatment - - -- BACE1 (beta-secretase 1) cleaves APP to produce amyloid-beta; \ -key therapeutic target for Alzheimer's -- Verubecestat reached Phase III but failed due to toxicity; \ -lanabecestat discontinued for lack of efficacy -- Aminothiazine and aminohydantoin scaffolds show sub-nM \ -potency but poor BBB penetration -- Recent work on macrocyclic BACE1 inhibitors shows improved \ -selectivity over BACE2 (>100x) with maintained potency - - -{ - "hypotheses": [ - { - "statement": "Macrocyclic aminohydantoin derivatives with \ -12-14 membered rings will achieve BACE1 IC50 below 10 nM \ -while maintaining TPSA below 90 A^2 for BBB penetration", - "rationale": "Macrocyclization constrains the \ -aminohydantoin pharmacophore into the bioactive conformation, \ -reducing entropic penalty; ring sizes of 12-14 atoms balance \ -potency with CNS MPO criteria", - "prediction": "Docking against BACE1 shows < -8 kcal/mol \ -for macrocyclic analogs; TPSA < 90 and MW < 500", - "null_prediction": "Macrocycles show no binding \ -improvement over linear aminohydantoins; TPSA > 100", - "success_criteria": ">=2 candidates with docking < -8 \ -kcal/mol AND TPSA < 90 AND MW < 500", - "failure_criteria": "No macrocycles achieve docking < -6 \ -kcal/mol OR all exceed MW 600", - "scope": "BACE1 aspartyl protease; CNS-penetrant small \ -molecules", - "hypothesis_type": "structural", - "prior_confidence": 0.6 - }, - { - "statement": "Fragment-based compounds targeting the \ -BACE1 S3 subpocket with halogenated aromatics will show \ -selectivity over BACE2 greater than 50-fold", - "rationale": "The S3 subpocket differs between BACE1 \ -(Ile110) and BACE2 (Val110); halogen bonding to this residue \ -difference drives selectivity", - "prediction": "Halogenated fragments show BACE1 docking \ -< -7 kcal/mol with BACE2 docking > -4 kcal/mol (selectivity)", - "null_prediction": "No selectivity difference; both \ -BACE1 and BACE2 docking within 1 kcal/mol", - "success_criteria": ">=1 compound with BACE1/BACE2 \ -selectivity ratio > 50x", - "failure_criteria": "All compounds show < 10x selectivity", - "scope": "BACE1 S3 subpocket; fragment-like MW < 300", - "hypothesis_type": "structural", - "prior_confidence": 0.45 - } - ], - "negative_controls": [ - { - "smiles": "CN1C=NC2=C1C(=O)N(C(=O)N2C)C", - "name": "Caffeine", - "source": "CNS-active xanthine; zero BACE1 activity \ -in published screens" - } - ], - "positive_controls": [ - { - "identifier": "CC(C)CC1=CC=C(C=C1)C(C)C(=O)N2CCC(CC2)N3C(=O)N(C3=O)C", - "name": "Verubecestat", - "known_activity": "IC50 = 2.2 nM vs BACE1", - "source": "Reached Phase III, confirmed potent inhibitor" - } - ] -} - - - - - -Respond with ONLY valid JSON (no markdown fences): -{ - "hypotheses": [ - { - "statement": "Specific testable hypothesis", - "rationale": "Causal mechanism explaining HOW and WHY", - "prediction": "If true, we expect to observe X (specific, measurable)", - "null_prediction": "If false, we would observe Y instead", - "success_criteria": "Quantitative threshold for support", - "failure_criteria": "Quantitative threshold: e.g. no compounds show activity above baseline", - "scope": "Boundary conditions: organism, compound class, target", - "hypothesis_type": "mechanistic|structural|pharmacological|toxicological|other", - "prior_confidence": 0.65 - } - ], - "negative_controls": [ - { - "smiles": "SMILES of known inactive compound", - "name": "Compound name", - "source": "Why this is a good negative control" - } - ], - "positive_controls": [ - { - "identifier": "SMILES of known active compound", - "name": "Compound name", - "known_activity": "IC50 = X nM against target Y", - "source": "Why this is a good positive control" - } - ] -} -""" - -DIRECTOR_EXPERIMENT_PROMPT = """\ -You are the Director designing an experiment to test a \ -hypothesis in a scientific discovery investigation. - - -Given the hypothesis and available tools, design a structured \ -experiment protocol with: -- A clear description of what the experiment will test -- An ordered tool_plan listing the tools to execute -- Defined variables, controls, and analysis plan - - - -Follow these 5 principles when designing experiments: - -1. VARIABLES: Define the independent variable (factor being \ -manipulated) and dependent variable (outcome being measured). \ -Be specific about units and measurement method. - -2. CONTROLS: Include at least one positive or negative baseline. \ -Positive controls confirm the assay works (known active). \ -Negative controls confirm specificity (known inactive). - -3. CONFOUNDERS: Identify threats to validity. Common confounders: \ -dataset bias, assay type mismatch, species differences, \ -structural similarity to training data. - -4. ANALYSIS PLAN: Pre-specify metrics and thresholds BEFORE \ -seeing results. This prevents post-hoc rationalization. \ -Include: primary metric, threshold, and sample size expectation. - -5. SENSITIVITY: Consider robustness. Will the conclusion change \ -if you vary a key parameter by +/- 20%? Note fragile assumptions. - - - - - -Diazabicyclooctane derivatives with C2 sulfonamide substituents \ -will show dual Class A/C beta-lactamase inhibition with Ki \ -below 50 nM - - -{ - "description": "Search ChEMBL for DBO-scaffold compounds with \ -reported beta-lactamase activity, train a classification model \ -on active/inactive compounds, then dock top predictions against \ -Class A beta-lactamase (PDB: 1ZG4) to validate binding", - "tool_plan": [ - "search_bioactivity", - "search_compounds", - "compute_descriptors", - "analyze_substructures", - "train_model", - "predict_candidates", - "search_protein_targets", - "dock_against_target", - "predict_admet", - "record_finding" - ], - "independent_variable": "C2 sulfonamide substitution pattern \ -on DBO scaffold", - "dependent_variable": "Predicted beta-lactamase inhibition \ -(Ki in nM, docking score in kcal/mol)", - "controls": [ - "positive: Avibactam (known DBO inhibitor, Ki ~1 nM)", - "negative: Aspirin (non-antimicrobial, expected inactive)" - ], - "confounders": [ - "ChEMBL dataset may bias toward published active compounds", - "Docking scoring function may not capture covalent binding" - ], - "analysis_plan": "Primary metric: AUC of ML model (threshold \ ->0.7); secondary: docking score <-7 kcal/mol for top 3 \ -candidates; expect N>=50 training compounds", - "success_criteria": "At least 3 DBO derivatives predicted \ -active with probability above 0.7 AND docking score below \ --7.0 kcal/mol against beta-lactamase", - "failure_criteria": "Fewer than 2 compounds meet both \ -prediction and docking thresholds, OR model AUC below 0.7 \ -indicating unreliable predictions" -} - - - - - -Respond with ONLY valid JSON (no markdown fences): -{ - "description": "What this experiment will do", - "tool_plan": ["tool_name_1", "tool_name_2"], - "independent_variable": "Factor being manipulated", - "dependent_variable": "Outcome being measured", - "controls": ["positive: known active", "negative: known inactive"], - "confounders": ["identified threats to validity"], - "analysis_plan": "Pre-specified metrics and thresholds", - "success_criteria": "What result would support the hypothesis", - "failure_criteria": "What result would refute the hypothesis" -} -""" - -DIRECTOR_EVALUATION_PROMPT = """\ -You are the Director evaluating a hypothesis based on \ -experimental evidence. - - -Review the findings from the experiment and determine the \ -hypothesis outcome: -- "supported": evidence consistently supports the hypothesis \ -(confidence above 0.7) -- "refuted": evidence contradicts the hypothesis or key \ -criteria were not met -- "revised": partial support warrants a refined hypothesis - -Base your assessment on quantitative evidence. Cite specific \ -numbers from findings (scores, counts, p-values) in your \ -reasoning. If revising, the new statement must be more specific \ -than the original. - - - -Rank each finding by its reliability tier before weighing it: -1. Replicated experimental data (orthogonal assays, n>=3) -- highest -2. Single experimental measurement (one lab, one assay) -3. Curated database entry (ChEMBL, PubChem, GtoPdb) -4. Prospectively validated ML prediction (external test set) -5. Retrospective ML prediction (cross-validated) -6. Consensus computational (3+ methods agree) -7. Single computational score (one docking, one prediction) -8. Qualitative literature report -- lowest - -When reasoning, cite which tier each key finding belongs to. \ -Higher-tier evidence should carry more weight in your assessment. - - - -Distinguish real effects from noise using these domain \ -reference thresholds: -- IC50/Ki (single lab): <2-fold is noise, 3-5-fold meaningful, \ ->10-fold high confidence -- Docking score: <0.5 kcal/mol is noise, 1.0-1.5 meaningful, \ ->2.0 high confidence -- ML probability: difference <0.1 is noise, >0.2 meaningful, \ ->0.4 high confidence -- Enrichment factor EF1%: <2x is noise, >5x meaningful, >10x \ -excellent -- Effect size (Cohen's d): <0.2 trivial, 0.5 medium, >0.8 large - -Compare observed differences against these thresholds. A \ -difference below the noise floor is NOT evidence. - - - -Update confidence from prior to posterior using the evidence: -- If most evidence is supporting AND from tiers 1-3: multiply \ -prior_confidence by 1.3-1.5 (cap at 0.95) -- If evidence is mixed or from lower tiers only: keep near \ -prior, widen uncertainty -- If most evidence is contradicting: multiply prior_confidence \ -by 0.3-0.5 (floor at 0.05) -- Express the updated value as the "confidence" field in output - - - -When findings conflict, follow this resolution hierarchy: -1. Check compound/subject identity (SMILES match, correct \ -identifiers, stereochemistry) -2. Check assay comparability (IC50 vs Ki vs EC50, same assay \ -conditions, same units?) -3. Apply temporal relevance (newer data weighted higher) -4. Classify severity: - - <3x noise floor = minor: average after corrections - - 3-6x noise floor = moderate: widen confidence interval, flag - - >6x noise floor = major: do NOT average, report as \ -contradictory, investigate mechanism - - - -Check whether independent method types converge or diverge: -- If 2+ independent methods (e.g. docking + ML + literature + \ -bioactivity data) agree: evidence is CONVERGING -- increase \ -confidence -- If methods disagree: evidence is MIXED -- decrease confidence \ -and flag which methods disagree -- If methods give opposing conclusions: evidence is \ -CONTRADICTORY -- do not average, report which methods conflict - -Report convergence status in the "evidence_convergence" field. - - - -Before determining the outcome, verify: -1. CONTROLS: Did positive controls produce expected results? \ -Did negative controls score below threshold? If controls \ -failed, the experiment may be invalid regardless of test results. -2. CRITERIA COMPARISON: Compare findings against BOTH the \ -hypothesis-level criteria AND the experiment-level criteria. \ -Note any discrepancies. -3. ANALYSIS PLAN: Was the pre-specified analysis plan followed? \ -Were metrics and thresholds applied as defined before the \ -experiment ran? -4. CONFOUNDERS: Were any identified confounders observed during \ -execution? If so, note their impact on confidence. - - - -Respond with ONLY valid JSON (no markdown fences): -{ - "status": "supported|refuted|revised", - "confidence": 0.85, - "certainty_of_evidence": "high|moderate|low|very_low", - "evidence_convergence": "converging|mixed|contradictory", - "reasoning": "Detailed scientific reasoning citing specific \ -evidence from findings, referencing evidence hierarchy tiers \ -and effect size thresholds", - "key_evidence": ["list of key evidence points with numbers \ -and their reliability tier"], - "revision": "If revised, the new refined hypothesis (omit \ -if not revised)" -} -""" - -DIRECTOR_SYNTHESIS_PROMPT = """\ -You are the Director synthesizing the full investigation \ -results into a final report. - - -Review all hypothesis outcomes, findings, and negative controls \ -to produce a comprehensive synthesis. Your report must: -- Summarize hypothesis outcomes with confidence levels -- Rank candidates by multi-criteria evidence strength -- Assess model reliability using negative AND positive control results -- Identify limitations and suggest follow-up experiments -- Include all relevant citations - - -Assess model/prediction validation quality: - -1. CONTROL SEPARATION: Are positive control scores clearly separated from \ -negative control scores? If positive controls scored below the active \ -threshold, the model is unreliable -- flag this prominently. - -2. CLASSIFICATION QUALITY: With the available controls, assess whether \ -the model can discriminate actives from inactives. Consider: -- Do all positive controls score above threshold? (sensitivity check) -- Do all negative controls score below threshold? (specificity check) -- Is there clear separation between control groups? - -3. OVERALL VALIDATION: Rate as: -- "sufficient": positive controls pass, negatives pass, clear separation -- "marginal": most controls pass but separation is narrow -- "insufficient": any positive control fails, or no positive controls tested - -4. Z'-FACTOR: Z' >= 0.5 = excellent assay separation, 0 < Z' < 0.5 = marginal \ -(scores overlap), Z' <= 0 = unusable (no separation between controls). \ -If Z'-factor is provided, cite it explicitly in your validation assessment. - -5. PERMUTATION SIGNIFICANCE: If permutation_p_value < 0.05, the model is \ -significantly better than random. If p >= 0.05, predictions may be noise -- \ -flag this prominently. - -6. SCAFFOLD-SPLIT GAP: If the gap between random_auroc and scaffold AUROC is \ -> 0.15, the model may be memorizing scaffolds rather than learning activity. \ -Report this as a methodology limitation. - -If validation is insufficient, downgrade certainty of ALL hypothesis \ -assessments by one level and note this in limitations. - - - -For each hypothesis assessment, assign a GRADE-adapted certainty level: -- high: Multiple concordant methods, strong controls, large effect sizes, \ -evidence from tiers 1-3 -- moderate: Some concordance, adequate controls, moderate effect sizes -- low: Few methods, weak controls, small or inconsistent effects -- very_low: Single method, no controls, conflicting evidence - -Five domains that DOWNGRADE certainty: -1. Risk of bias: poor model validation, outside applicability domain -2. Inconsistency: methods disagree (docking vs ML vs literature) -3. Indirectness: evidence from different target/species/assay than asked -4. Imprecision: wide confidence intervals, small sample sizes -5. Publication bias: database coverage gaps, missing negative results - -Three domains that can UPGRADE (for computational evidence): -1. Large effect: very strong activity (>10-fold over baseline) -2. Dose-response: clear SAR gradient across compound series -3. Conservative prediction: result holds despite known biases - -Name which domains caused downgrading or upgrading in certainty_reasoning. - - - -Assign each candidate a priority tier based on certainty, evidence, and risk: - -Priority 1 (Strong Advance): High or moderate certainty. Multiple supported \ -hypotheses. Concordant docking + ML + ADMET. Controls pass. Large effects. \ -Action: queue for experimental testing. - -Priority 2 (Conditional Advance): Moderate or low certainty. 1-2 supported \ -hypotheses. Some method concordance. Adequate controls. \ -Action: additional computational validation before synthesis. - -Priority 3 (Watchlist): Low certainty. Partial support, limited methods, \ -or borderline activity. \ -Action: investigate further computationally; low resource priority. - -Priority 4 (Do Not Advance): Very low certainty. Refuted hypotheses, \ -control failures, contradictory evidence, or safety flags. \ -Action: archive; redirect effort. - - - -Report limitations using these four categories: -- methodology: model limitations, scoring function inaccuracy, \ -conformational sampling, feature representation -- data: database coverage gaps, assay heterogeneity, activity cliffs, \ -publication bias, missing data types -- scope: in silico only, limited chemical space, time-bound investigation, \ -single-target focus -- interpretation: docking scores are rank-ordering not absolute affinity, \ -ML probabilities need calibration, resistance based on known mutations only - - - -Identify what evidence was NOT collected during this investigation. \ -Construct a conceptual evidence map: for each hypothesis, which evidence \ -types are present and which are missing? - -Classify each gap: -- evidence: no data available (e.g. no crystal structure for docking) -- quality: data exists but low quality (e.g. only IC50, no Ki) -- consistency: conflicting results across methods -- scope: evidence exists but for different context (e.g. mouse not human) -- temporal: evidence outdated (e.g. resistance data from >5 years ago) - - - -Recommend specific next experiments prioritized by impact on confidence. \ -For each recommendation, specify: -- What to do and why it matters -- Whether it is computational (can be done in a follow-up investigation) \ -or experimental (requires wet-lab work) -- Impact level: critical (blocks all recommendations), high (affects \ -primary recommendation), medium (improves confidence), low (informational) - - -Scoring fields for candidates: -- prediction_score: ML model probability (0-1) -- docking_score: binding affinity in kcal/mol (negative = \ -better) -- admet_score: overall drug-likeness (0-1) -- resistance_risk: mutation risk ("low", "medium", "high") -Use 0.0 or "unknown" if a score was not computed. - - - -Respond with ONLY valid JSON (no markdown fences): -{ - "summary": "Comprehensive 2-3 paragraph summary including \ -hypothesis outcomes and key discoveries", - "candidates": [ - { - "identifier": "identifier string", - "identifier_type": "smiles", - "name": "compound name", - "rationale": "why this candidate is promising", - "rank": 1, - "priority": 1, - "scores": {}, - "attributes": {} - } - ], - "citations": ["DOI or reference strings"], - "hypothesis_assessments": [ - { - "hypothesis_id": "h1", - "statement": "the hypothesis", - "status": "supported|refuted|revised", - "confidence": 0.85, - "certainty": "high|moderate|low|very_low", - "certainty_reasoning": "downgraded by X, upgraded by Y", - "key_evidence": "summary of evidence" - } - ], - "negative_control_summary": "Summary of negative control \ -results and model reliability assessment", - "model_validation_quality": "sufficient|marginal|insufficient", - "confidence": "high/medium/low", - "limitations": [ - { - "category": "methodology|data|scope|interpretation", - "description": "specific limitation" - } - ], - "knowledge_gaps": [ - { - "gap_type": "evidence|quality|consistency|scope|temporal", - "description": "what is missing and why it matters" - } - ], - "follow_up_experiments": [ - { - "description": "what to do next", - "impact": "critical|high|medium|low", - "type": "computational|experimental" - } - ] -} -""" - -RESEARCHER_EXPERIMENT_PROMPT = """\ -You are a research scientist executing a specific experiment \ -to test a hypothesis in a molecular discovery investigation. - - -You have access to cheminformatics, ML, simulation, data \ -search, and literature tools. The user's research question \ -defines the domain. Focus ONLY on the current experiment. - -Search strategy: -- Start with short, broad queries to understand the landscape, \ -then narrow focus based on results. Overly specific initial \ -queries miss relevant data. -- Use `search_disease_targets` first to identify high-confidence \ -disease-target associations before querying ChEMBL. -- Use `get_protein_annotation` for target context (function, \ -disease links, GO terms) before docking. -- Prefer `search_bioactivity` over `explore_dataset` when you \ -know the specific target or assay type. - -Recording results: -- Call `record_finding` after each significant discovery, \ -always specifying hypothesis_id and evidence_type \ -('supporting' or 'contradicting'). -- Include source_type and source_id for provenance tracing: \ -e.g. source_type="chembl" source_id="CHEMBL25", \ -source_type="pdb" source_id="2ABC", \ -source_type="doi" source_id="10.1038/s41586-024-07613-w", \ -source_type="pubchem" source_id="2244". -- Be quantitative: report exact numbers, scores, and \ -confidence intervals. - -Boundaries: -- Do NOT call `conclude_investigation` -- the Director \ -synthesizes results. -- Do NOT call `propose_hypothesis`, `design_experiment`, or \ -`evaluate_hypothesis` -- those are Director responsibilities. - - - -Apply these principles during experiment execution: - -1. SENSITIVITY: When training models or computing scores, test \ -at least 2 parameter values (e.g. different thresholds, \ -different training sizes). Flag results that change dramatically \ -with small parameter changes as fragile. - -2. APPLICABILITY DOMAIN: For ML predictions, check if test \ -compounds are similar to training data (Tanimoto > 0.3 to \ -nearest training neighbor). Predictions far outside the training \ -domain are unreliable -- note this when recording findings. - -3. UNCERTAINTY: Report ranges or mean +/- SD, not just point \ -estimates. For ML models, report AUC with confidence interval. \ -For docking, note the scoring function uncertainty (~2 kcal/mol). - -4. VERIFICATION: Before recording a finding, check if it makes \ -physical sense. A predicted LogP of 15 or MW of 2000 for a \ -drug-like molecule is likely an error. Verify SMILES validity \ -before passing to downstream tools. - -5. NEGATIVE RESULTS: Record failed approaches with a diagnosis \ -of why they failed. A negative finding with evidence_type \ -'contradicting' is scientifically valuable -- do not omit it. - - - -Example: Docking a compound against a protein target -1. search_protein_targets(query="BACE1 human", limit=3) - -> finds PDB entries with resolution and method -2. get_protein_annotation(uniprot_id="P56817") - -> confirms function, active site residues, disease links -3. dock_against_target( - smiles="CC1=CC(=O)...", - target_id="4ivt", - exhaustiveness=16 - ) - -> returns binding affinity and pose - -Example: Training a predictive model -1. search_bioactivity( - target="beta-lactamase", - organism="Staphylococcus aureus", - assay_type="Ki", - limit=500 - ) - -> retrieves activity dataset with SMILES and values -2. train_model( - smiles_list=[...], - activity_list=[...], - model_type="xgboost" - ) - -> returns model metrics (AUC, accuracy, feature importance) -3. predict_candidates(smiles_list=[...], model_id="model_xyz") - -> scores new compounds with the trained model - -Example: Identifying disease-target associations -1. search_disease_targets(disease="Alzheimer", limit=10) - -> returns scored targets (BACE1, gamma-secretase, tau...) -2. get_protein_annotation(uniprot_id="P56817") - -> protein function, structure refs, disease annotations -3. search_bioactivity(target="BACE1", assay_type="IC50") - -> retrieves assay data for the validated target - - - -1. Explain your scientific reasoning before each tool call. -2. Call `record_finding` after each significant discovery with \ -hypothesis_id and evidence_type. -3. Cite papers by DOI when referencing literature. -4. Use `validate_smiles` before passing SMILES if uncertain. -5. If a tool returns an error, try an alternative approach. -6. Be quantitative: report exact numbers and scores. -7. Use at least 3 tool calls in this experiment. -""" - -SUMMARIZER_PROMPT = """\ -You are a scientific data compressor. Given a tool output, \ -produce a concise summary that preserves all key data points. - - -Keep: exact numbers, SMILES strings, DOIs, statistical metrics, \ -compound names, key conclusions. -Remove: verbose explanations, repeated headers, formatting \ -artifacts, redundant context. - - -Respond with ONLY the compressed text, no preamble.""" - - -_DEFAULT_CATEGORIES = frozenset( - { - "antimicrobial", - "neurodegenerative", - "oncology", - "environmental", - "cardiovascular", - "metabolic", - "immunology", - "other", - } -) - - -def build_pico_and_classification_prompt( - categories: frozenset[str] | None = None, - uploaded_data_context: str = "", -) -> str: - """Build a combined domain classification + PICO decomposition prompt. - - Single Haiku call replaces the old separate classification step. - Produces domain category AND PICO framework for the literature survey. - """ - cats = categories or _DEFAULT_CATEGORIES - cats_with_other = cats | {"other"} - cat_str = ", ".join(sorted(cats_with_other)) - return ( - "You are a scientific research analyst. Given a research prompt, " - "perform TWO tasks:\n\n" - "1. CLASSIFY the research domain into ALL relevant categories " - "(at least 1, as many as apply).\n" - "2. DECOMPOSE the research question using the PICO framework " - "(Population, Intervention, Comparison, Outcome).\n\n" - f"\n{cat_str}\n\n\n" - "\n" - "- Domain: select every category that applies to the research question. " - "Cross-domain questions (e.g. drug effects on athletic performance) " - "should list all relevant categories.\n" - "- Population: the subjects, organisms, or systems under study\n" - "- Intervention: the treatment, compound, protocol, or variable being tested\n" - "- Comparison: the control, baseline, or alternative being compared against\n" - "- Outcome: the measurable result or endpoint of interest\n" - "- Search terms: 3-5 broad search queries for literature discovery\n" - "\n\n" - "\n" - "Respond with ONLY valid JSON (no markdown fences):\n" - "{\n" - ' "domain": ["category_1", "category_2"],\n' - ' "population": "description of subjects/systems",\n' - ' "intervention": "treatment/variable under study",\n' - ' "comparison": "control/baseline",\n' - ' "outcome": "measurable endpoint",\n' - ' "search_terms": ["broad query 1", "broad query 2", "broad query 3"]\n' - "}\n" - "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") - ) - - -def build_literature_survey_prompt(config: DomainConfig | None, pico: dict[str, Any]) -> str: - """Build structured literature survey prompt with PICO context. - - Researcher gets domain-filtered tools and a structured protocol - instead of the old throwaway 'search 3-6 times' instruction. - """ - pop = pico.get("population", "") - interv = pico.get("intervention", "") - comp = pico.get("comparison", "") - outcome = pico.get("outcome", "") - terms = pico.get("search_terms", []) - terms_str = ", ".join(f'"{t}"' for t in terms) if isinstance(terms, list) else str(terms) - - # Domain-specific search guidance - domain_guidance = "" - if config and config.tool_tags & {"training", "clinical"}: - domain_guidance = ( - "\n\n" - "Training/clinical tools are available -- use multiple sources:\n" - "- Use `search_pubmed_training` with MeSH terms for precise biomedical literature\n" - "- Use `search_training_literature` for broader coverage via Semantic Scholar\n" - "- Combine both sources for comprehensive evidence gathering\n" - "\n" - ) - - return ( - "You are a research scientist conducting a rapid scoping review " - "(Arksey & O'Malley 2005) to map the evidence landscape.\n\n" - "\n" - f" Population: {pop}\n" - f" Intervention: {interv}\n" - f" Comparison: {comp}\n" - f" Outcome: {outcome}\n" - f" Initial search terms: {terms_str}\n" - "\n\n" - "\n" - "Execute a multi-strategy search:\n\n" - "1. DATABASE QUERIES: Use `search_literature` with broad terms first, " - "then narrow based on results. Use `explore_dataset` or domain-specific " - "search tools to find quantitative data.\n\n" - "2. CITATION CHASING: For key papers found in step 1, use " - "`search_citations` to find referenced and citing papers. " - "Greenhalgh & Peacock (2005) found 51% of sources come from snowballing.\n\n" - "3. SATURATION RULE: Stop when additional queries yield fewer than " - "2 new unique results not already covered by previous searches.\n" - "\n\n" + domain_guidance + "\n" - "When recording findings with `record_finding`, assign an evidence_level:\n" - " 1 = Systematic review / meta-analysis\n" - " 2 = Randomized controlled trial / large-scale validated study\n" - " 3 = Cohort study / prospective observational\n" - " 4 = Case-control study / retrospective analysis\n" - " 5 = Case series / computational prediction / ML model\n" - " 6 = Expert opinion / mechanistic reasoning\n" - " 0 = Not applicable / unrated\n" - "\n\n" - "\n" - "1. Record every significant finding with `record_finding` including " - "evidence_level, source_type, and source_id.\n" - "2. Use at least 2 different search strategies (database + citation chasing).\n" - "3. Cite papers by DOI when referencing literature.\n" - "4. Be quantitative: report exact numbers, effect sizes, sample sizes.\n" - "5. Do NOT call `propose_hypothesis`, `design_experiment`, " - "`evaluate_hypothesis`, or `conclude_investigation`.\n" - "6. Stop after 6-10 tool calls or when search saturation is reached.\n" - "" - ) - - -def build_literature_assessment_prompt() -> str: - """Build Haiku prompt for body-of-evidence grading after literature survey. - - GRADE-adapted grading + AMSTAR-2-adapted self-assessment. - """ - return ( - "You are a scientific evidence assessor. Given the findings from a " - "literature survey, grade the overall body of evidence and assess " - "the quality of the review process.\n\n" - "\n" - "Grade the body of evidence using GRADE-adapted criteria:\n" - "- high: Consistent results from multiple high-quality studies (levels 1-2)\n" - "- moderate: Results from well-designed studies with minor limitations\n" - "- low: Results from observational studies or studies with significant limitations\n" - "- very_low: Expert opinion only or severely limited evidence\n" - "\n\n" - "\n" - "Assess the review process against 4 rapid-review quality domains " - "(adapted from AMSTAR 2):\n" - "1. Protocol-guided search (PICO framework used)\n" - "2. Multi-source search (more than 1 search strategy used)\n" - "3. Evidence quality assessed (evidence levels assigned to findings)\n" - "4. Transparent documentation (findings recorded with source provenance)\n" - "Report which domains were satisfied and which were not.\n" - "\n\n" - "\n" - "Respond with ONLY valid JSON (no markdown fences):\n" - "{\n" - ' "evidence_grade": "high|moderate|low|very_low",\n' - ' "reasoning": "Brief justification for the grade",\n' - ' "assessment": "Summary of which quality domains were met"\n' - "}\n" - "" - ) - - -def build_multi_investigation_context( - investigations: list[Investigation], -) -> str: - """Compress past completed investigations into XML context for the Director.""" - if not investigations: - return "" - parts: list[str] = [""] - for inv in investigations[:3]: - parts.append(f' ') - for h in inv.hypotheses[:4]: - if h.status.value in ("supported", "refuted", "revised"): - parts.append( - f" " - f"{h.statement}" - ) - for c in inv.candidates[:2]: - top_score = max(c.scores.values()) if c.scores else 0 - parts.append( - f" " - f"{c.name or 'unnamed'}" - ) - if inv.summary: - parts.append(f" {inv.summary[:300]}") - parts.append(" ") - parts.append("") - return "\n".join(parts) - - -def build_formulation_prompt( - config: DomainConfig, - uploaded_data_context: str = "", -) -> str: - """Build Director formulation prompt adapted to the domain config.""" - hyp_types = "|".join(config.hypothesis_types) if config.hypothesis_types else "other" - examples = config.director_examples or "" - return ( - "You are the Director of a scientific discovery investigation. " - "You formulate hypotheses and design the research strategy but " - "do NOT execute tools yourself.\n\n" - "\n" - "Given the user's research prompt and literature survey results, " - "formulate 2-4 testable hypotheses. Each hypothesis must be:\n" - "- Specific and falsifiable\n" - "- Grounded in the literature findings provided\n" - "- Testable with the available tools\n" - "- Orthogonal to sibling hypotheses: each must attack a different " - "mechanism, pathway, or data source\n\n" - "Hypotheses tested in parallel must maximize scientific coverage:\n" - "- Different causal mechanisms or molecular targets\n" - "- Different primary data sources or tool chains\n" - "- Different validation strategies (e.g., ML prediction vs " - "structural docking vs substructure enrichment)\n" - "Do NOT formulate hypotheses that would produce overlapping experiments.\n\n" - "If prior investigation results are provided in " - ", leverage their outcomes:\n" - "- Build on supported hypotheses from related investigations\n" - "- Avoid repeating refuted approaches\n" - "- Consider candidates already identified as starting points\n\n" - "Also identify 1-3 negative controls (subjects known to be " - "inactive) AND 1-2 positive controls (subjects known to be " - "active). Both are essential for validation: negative controls " - "confirm specificity, positive controls confirm the pipeline " - "can detect true actives. Without positive controls, pipeline " - "failures are undetectable (Zhang et al., 1999).\n" - "\n\n" - f"{examples}\n\n" - "\n" - "Respond with ONLY valid JSON (no markdown fences):\n" - "{\n" - ' "hypotheses": [\n' - " {\n" - ' "statement": "Specific testable hypothesis",\n' - ' "rationale": "Causal mechanism explaining HOW and WHY",\n' - ' "prediction": "If true, we expect to observe X",\n' - ' "null_prediction": "If false, we would observe Y instead",\n' - ' "success_criteria": "Quantitative threshold for support",\n' - ' "failure_criteria": "Quantitative threshold for refutation",\n' - ' "scope": "Boundary conditions",\n' - f' "hypothesis_type": "{hyp_types}",\n' - ' "prior_confidence": 0.65\n' - " }\n" - " ],\n" - ' "negative_controls": [\n' - " {\n" - ' "identifier": "identifier of known inactive subject",\n' - ' "name": "Name",\n' - ' "source": "Why this is a good negative control"\n' - " }\n" - " ],\n" - ' "positive_controls": [\n' - " {\n" - ' "identifier": "identifier of known active subject",\n' - ' "name": "Name",\n' - ' "known_activity": "IC50 = X nM against target Y",\n' - ' "source": "Why this is a good positive control"\n' - " }\n" - " ]\n" - "}\n" - "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") - ) - - -def build_experiment_prompt( - config: DomainConfig, - uploaded_data_context: str = "", -) -> str: - """Build Director experiment design prompt adapted to the domain config.""" - examples = config.experiment_examples or "" - return ( - "You are the Director designing an experiment to test a " - "hypothesis in a scientific discovery investigation.\n\n" - "\n" - "Given the hypothesis and available tools, design a structured " - "experiment protocol with:\n" - "- A clear description of what the experiment will test\n" - "- An ordered tool_plan listing the tools to execute\n" - "- Defined variables, controls, and analysis plan\n" - "\n\n" - "\n" - "Follow these 5 principles when designing experiments:\n\n" - "1. VARIABLES: Define the independent variable (factor being " - "manipulated) and dependent variable (outcome being measured). " - "Be specific about units and measurement method.\n\n" - "2. CONTROLS: Include at least one positive or negative baseline. " - "Positive controls confirm the assay works (known active). " - "Negative controls confirm specificity (known inactive).\n\n" - "3. CONFOUNDERS: Identify threats to validity. Common confounders: " - "dataset bias, assay type mismatch, species differences, " - "structural similarity to training data.\n\n" - "4. ANALYSIS PLAN: Pre-specify metrics and thresholds BEFORE " - "seeing results. This prevents post-hoc rationalization. " - "Include: primary metric, threshold, and sample size expectation. " - "When comparing two groups of numeric data, plan a statistical " - "test: specify `run_statistical_test` (continuous) or " - "`run_categorical_test` (categorical) in the tool_plan, with " - "alpha level and minimum effect size threshold as part of " - "success_criteria.\n\n" - "5. SENSITIVITY: Consider robustness. Will the conclusion change " - "if you vary a key parameter by +/- 20%? Note fragile assumptions.\n" - "\n\n" - f"{examples}\n\n" - "\n" - "Respond with ONLY valid JSON (no markdown fences):\n" - "{\n" - ' "description": "What this experiment will do",\n' - ' "tool_plan": ["tool_name_1", "tool_name_2"],\n' - ' "independent_variable": "Factor being manipulated",\n' - ' "dependent_variable": "Outcome being measured",\n' - ' "controls": ["positive: known active", "negative: known inactive"],\n' - ' "confounders": ["identified threats to validity"],\n' - ' "analysis_plan": "Pre-specified metrics and thresholds",\n' - ' "success_criteria": "What result would support the hypothesis",\n' - ' "failure_criteria": "What result would refute the hypothesis"\n' - "}\n" - "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") - ) - - -def build_synthesis_prompt(config: DomainConfig) -> str: - """Build Director synthesis prompt adapted to the domain config. - - Includes GRADE-adapted certainty grading, recommendation strength (priority - tiers 1-4), structured limitations taxonomy, knowledge gap analysis, and - follow-up experiment recommendations. - """ - scoring = config.synthesis_scoring_instructions or "" - label = config.candidate_label or "Candidates" - return ( - "You are the Director synthesizing the full investigation " - "results into a final report.\n\n" - "\n" - "Review all hypothesis outcomes, findings, and negative controls " - "to produce a comprehensive synthesis. Your report must:\n" - "- Summarize hypothesis outcomes with confidence levels\n" - f"- Rank {label.lower()} by multi-criteria evidence strength\n" - "- Assess model reliability using negative AND positive control results\n" - "- Identify limitations and suggest follow-up experiments\n" - "- Include all relevant citations\n\n" - "\n" - "Assess model/prediction validation quality:\n\n" - "1. CONTROL SEPARATION: Are positive control scores clearly separated " - "from negative control scores? If positive controls scored below the " - "active threshold, the model is unreliable -- flag this prominently.\n\n" - "2. CLASSIFICATION QUALITY: With the available controls, assess whether " - "the model can discriminate actives from inactives. Consider:\n" - "- Do all positive controls score above threshold? (sensitivity check)\n" - "- Do all negative controls score below threshold? (specificity check)\n" - "- Is there clear separation between control groups?\n\n" - "3. OVERALL VALIDATION: Rate as:\n" - '- "sufficient": positive controls pass, negatives pass, clear separation\n' - '- "marginal": most controls pass but separation is narrow\n' - '- "insufficient": any positive control fails, or no positive controls tested\n\n' - "4. Z'-FACTOR: Z' >= 0.5 = excellent assay separation, " - "0 < Z' < 0.5 = marginal (scores overlap), " - "Z' <= 0 = unusable (no separation between controls). " - "If Z'-factor is provided, cite it explicitly in your validation assessment.\n\n" - "5. PERMUTATION SIGNIFICANCE: If permutation_p_value < 0.05, the model is " - "significantly better than random. If p >= 0.05, predictions may be noise -- " - "flag this prominently.\n\n" - "6. SCAFFOLD-SPLIT GAP: If the gap between random_auroc and scaffold AUROC is " - "> 0.15, the model may be memorizing scaffolds rather than learning activity. " - "Report this as a methodology limitation.\n\n" - "If validation is insufficient, downgrade certainty of ALL hypothesis " - "assessments by one level and note this in limitations.\n" - "\n\n" - "\n" - "For each hypothesis assessment, assign a GRADE-adapted certainty level:\n" - "- high: Multiple concordant methods, strong controls, large effect sizes, " - "evidence from tiers 1-3\n" - "- moderate: Some concordance, adequate controls, moderate effect sizes\n" - "- low: Few methods, weak controls, small or inconsistent effects\n" - "- very_low: Single method, no controls, conflicting evidence\n\n" - "Five domains that DOWNGRADE certainty:\n" - "1. Risk of bias: poor model validation, outside applicability domain\n" - "2. Inconsistency: methods disagree (docking vs ML vs literature)\n" - "3. Indirectness: evidence from different target/species/assay than asked\n" - "4. Imprecision: wide confidence intervals, small sample sizes\n" - "5. Publication bias: database coverage gaps, missing negative results\n\n" - "Three domains that can UPGRADE (for computational evidence):\n" - "1. Large effect: very strong activity (>10-fold over baseline)\n" - "2. Dose-response: clear SAR gradient across compound series\n" - "3. Conservative prediction: result holds despite known biases\n\n" - "Name which domains caused downgrading or upgrading in certainty_reasoning.\n" - "\n\n" - "\n" - "Assign each candidate a priority tier based on certainty, evidence, and risk:\n\n" - "Priority 1 (Strong Advance): High or moderate certainty. Multiple supported " - "hypotheses. Concordant methods. Controls pass. Large effects. " - "Action: queue for experimental testing.\n\n" - "Priority 2 (Conditional Advance): Moderate or low certainty. 1-2 supported " - "hypotheses. Some method concordance. Adequate controls. " - "Action: additional computational validation.\n\n" - "Priority 3 (Watchlist): Low certainty. Partial support, limited methods, " - "or borderline activity. " - "Action: investigate further computationally; low resource priority.\n\n" - "Priority 4 (Do Not Advance): Very low certainty. Refuted hypotheses, " - "control failures, contradictory evidence, or safety flags. " - "Action: archive; redirect effort.\n" - "\n\n" - "\n" - "Report limitations using these four categories:\n" - "- methodology: model limitations, scoring function inaccuracy, " - "conformational sampling, feature representation\n" - "- data: database coverage gaps, assay heterogeneity, activity cliffs, " - "publication bias, missing data types\n" - "- scope: in silico only, limited chemical space, time-bound investigation, " - "single-target focus\n" - "- interpretation: scores are rank-ordering not absolute, " - "ML probabilities need calibration, predictions based on known data only\n" - "\n\n" - "\n" - "Identify what evidence was NOT collected during this investigation. " - "Construct a conceptual evidence map: for each hypothesis, which evidence " - "types are present and which are missing?\n\n" - "Classify each gap:\n" - "- evidence: no data available\n" - "- quality: data exists but low quality\n" - "- consistency: conflicting results across methods\n" - "- scope: evidence exists but for different context\n" - "- temporal: evidence outdated\n" - "\n\n" - "\n" - "Recommend specific next experiments prioritized by impact on confidence. " - "For each recommendation, specify:\n" - "- What to do and why it matters\n" - "- Whether it is computational or experimental\n" - "- Impact level: critical, high, medium, or low\n" - "\n\n" - f"{scoring}\n" - "\n\n" - "\n" - "Respond with ONLY valid JSON (no markdown fences):\n" - "{\n" - ' "summary": "Comprehensive 2-3 paragraph summary",\n' - ' "candidates": [\n' - " {\n" - ' "identifier": "identifier string",\n' - ' "identifier_type": "' + config.identifier_type + '",\n' - ' "name": "name",\n' - ' "rationale": "why this candidate is promising",\n' - ' "rank": 1,\n' - ' "priority": 1,\n' - ' "scores": {},\n' - ' "attributes": {}\n' - " }\n" - " ],\n" - ' "citations": ["DOI or reference strings"],\n' - ' "hypothesis_assessments": [\n' - " {\n" - ' "hypothesis_id": "h1",\n' - ' "statement": "the hypothesis",\n' - ' "status": "supported|refuted|revised",\n' - ' "confidence": 0.85,\n' - ' "certainty": "high|moderate|low|very_low",\n' - ' "certainty_reasoning": "downgraded by X, upgraded by Y",\n' - ' "key_evidence": "summary of evidence"\n' - " }\n" - " ],\n" - ' "negative_control_summary": "Summary of negative control results",\n' - ' "model_validation_quality": "sufficient|marginal|insufficient",\n' - ' "confidence": "high/medium/low",\n' - ' "limitations": [\n' - " {\n" - ' "category": "methodology|data|scope|interpretation",\n' - ' "description": "specific limitation"\n' - " }\n" - " ],\n" - ' "knowledge_gaps": [\n' - " {\n" - ' "gap_type": "evidence|quality|consistency|scope|temporal",\n' - ' "description": "what is missing and why it matters"\n' - " }\n" - " ],\n" - ' "follow_up_experiments": [\n' - " {\n" - ' "description": "what to do next",\n' - ' "impact": "critical|high|medium|low",\n' - ' "type": "computational|experimental"\n' - " }\n" - " ]\n" - "}\n" - "" - ) - - -def build_researcher_prompt( - config: DomainConfig, - uploaded_data_context: str = "", -) -> str: - """Build researcher experiment prompt adapted to the domain config.""" - examples = config.experiment_examples or "" - return ( - "You are a research scientist executing a specific experiment " - "to test a hypothesis in a scientific discovery investigation.\n\n" - "\n" - "You have access to specialized tools for this domain. " - "The user's research question defines the domain. " - "Focus ONLY on the current experiment.\n\n" - "Search strategy:\n" - "- Start with short, broad queries to understand the landscape, " - "then narrow focus based on results.\n\n" - "Recording results:\n" - "- Call `record_finding` after each significant discovery, " - "always specifying hypothesis_id and evidence_type " - "('supporting' or 'contradicting').\n" - "- Include source_type and source_id for provenance tracing.\n" - "- Be quantitative: report exact numbers, scores, and " - "confidence intervals.\n\n" - "Boundaries:\n" - "- Do NOT call `conclude_investigation` -- the Director " - "synthesizes results.\n" - "- Do NOT call `propose_hypothesis`, `design_experiment`, or " - "`evaluate_hypothesis` -- those are Director responsibilities.\n" - "\n\n" - "\n" - "Apply these principles during experiment execution:\n\n" - "1. SENSITIVITY: When training models or computing scores, test " - "at least 2 parameter values. Flag results that change dramatically " - "with small parameter changes as fragile.\n\n" - "2. APPLICABILITY DOMAIN: For ML predictions, check if test " - "compounds are similar to training data. Predictions far outside " - "the training domain are unreliable -- note this when recording findings.\n\n" - "3. UNCERTAINTY: Report ranges or mean +/- SD, not just point " - "estimates. Note scoring function uncertainty where applicable.\n\n" - "4. VERIFICATION: Before recording a finding, check if it makes " - "physical sense. Verify inputs before passing to downstream tools.\n\n" - "5. NEGATIVE RESULTS: Record failed approaches with a diagnosis " - "of why they failed. A negative finding with evidence_type " - "'contradicting' is scientifically valuable -- do not omit it.\n\n" - "6. STATISTICAL TESTING: After gathering numeric comparison data, " - "use `run_statistical_test` to formally compare two groups " - "(auto-selects t-test/Welch/Mann-Whitney based on normality and " - "variance). For count/categorical data, use `run_categorical_test` " - "(auto-selects Fisher's exact or chi-squared). Record the result " - "as a finding: evidence_type='supporting' if significant with " - "meaningful effect size, 'contradicting' if non-significant or " - "trivial effect.\n" - "\n\n" - f"{examples}\n\n" - "\n" - "1. Explain your scientific reasoning before each tool call.\n" - "2. Call `record_finding` after each significant discovery with " - "hypothesis_id and evidence_type.\n" - "3. Cite papers by DOI when referencing literature.\n" - "4. If a tool returns an error, try an alternative approach.\n" - "5. Be quantitative: report exact numbers and scores.\n" - "6. Use at least 3 tool calls in this experiment.\n" - "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") - ) - - -def _build_prior_context(investigation: Investigation) -> str: - """Compress completed hypotheses into compact context.""" - if not investigation.hypotheses: - return "" - parts: list[str] = [""] - for h in investigation.hypotheses: - if h.status.value in ("supported", "refuted", "revised"): - findings = [f for f in investigation.findings if f.hypothesis_id == h.id] - evidence = "; ".join(f"[{f.evidence_type}] {f.title}" for f in findings[:5]) - parts.append( - f" " - ) - parts.append(f" {h.statement}") - if evidence: - parts.append(f" Evidence: {evidence}") - parts.append(" ") - parts.append("") - return "\n".join(parts) - - -def build_uploaded_data_context(files: list[UploadedFile]) -> str: - """Build XML context block describing user-uploaded files. - - Injected into Director and Researcher prompts so the model is aware - of available datasets and can reference them via ``query_uploaded_data``. - """ - if not files: - return "" - parts: list[str] = [""] - for f in files: - if f.tabular: - t = f.tabular - cols = ", ".join(t.columns) - dtypes = ", ".join(t.dtypes) - parts.append(f' ') - parts.append(f" Columns ({len(t.columns)}): {cols}") - parts.append(f" Dtypes: {dtypes}") - parts.append(f" Rows: {t.row_count}") - if t.summary_stats: - stats_lines = [] - for col_name, stats in list(t.summary_stats.items())[:6]: - mean = stats.get("mean", "?") - std = stats.get("std", "?") - stats_lines.append(f"{col_name}: mean={mean}, std={std}") - parts.append(f" Stats: {'; '.join(stats_lines)}") - if t.sample_rows: - header = " | ".join(t.columns) - parts.append(f" Sample: {header}") - for row in t.sample_rows[:3]: - parts.append(f" {' | '.join(row)}") - parts.append(" ") - elif f.document: - d = f.document - excerpt = d.text[:500] + "..." if len(d.text) > 500 else d.text - parts.append(f' ') - parts.append(f" Pages: {d.page_count}") - parts.append(f" Excerpt: {excerpt}") - parts.append(" ") - parts.append("") - return "\n".join(parts) diff --git a/server/src/ehrlich/investigation/application/prompts/__init__.py b/server/src/ehrlich/investigation/application/prompts/__init__.py new file mode 100644 index 0000000..ae614f4 --- /dev/null +++ b/server/src/ehrlich/investigation/application/prompts/__init__.py @@ -0,0 +1 @@ +"""Prompt modules for the investigation orchestrator.""" diff --git a/server/src/ehrlich/investigation/application/prompts/builders.py b/server/src/ehrlich/investigation/application/prompts/builders.py new file mode 100644 index 0000000..7d72033 --- /dev/null +++ b/server/src/ehrlich/investigation/application/prompts/builders.py @@ -0,0 +1,283 @@ +"""Non-director builder functions. + +Haiku classification, literature survey, researcher prompt, +and uploaded-data context builders. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ehrlich.investigation.domain.domain_config import DomainConfig + from ehrlich.investigation.domain.uploaded_file import UploadedFile + + +_DEFAULT_CATEGORIES = frozenset( + { + "antimicrobial", + "neurodegenerative", + "oncology", + "environmental", + "cardiovascular", + "metabolic", + "immunology", + "other", + } +) + + +def build_pico_and_classification_prompt( + categories: frozenset[str] | None = None, + uploaded_data_context: str = "", +) -> str: + """Build a combined domain classification + PICO decomposition prompt. + + Single Haiku call replaces the old separate classification step. + Produces domain category AND PICO framework for the literature survey. + """ + cats = categories or _DEFAULT_CATEGORIES + cats_with_other = cats | {"other"} + cat_str = ", ".join(sorted(cats_with_other)) + return ( + "You are a scientific research analyst. Given a research prompt, " + "perform TWO tasks:\n\n" + "1. CLASSIFY the research domain into ALL relevant categories " + "(at least 1, as many as apply).\n" + "2. DECOMPOSE the research question using the PICO framework " + "(Population, Intervention, Comparison, Outcome).\n\n" + f"\n{cat_str}\n\n\n" + "\n" + "- Domain: select every category that applies to the research question. " + "Cross-domain questions (e.g. drug effects on athletic performance) " + "should list all relevant categories.\n" + "- Population: the subjects, organisms, or systems under study\n" + "- Intervention: the treatment, compound, protocol, or variable being tested\n" + "- Comparison: the control, baseline, or alternative being compared against\n" + "- Outcome: the measurable result or endpoint of interest\n" + "- Search terms: 3-5 broad search queries for literature discovery\n" + "\n\n" + "\n" + "Respond with ONLY valid JSON (no markdown fences):\n" + "{\n" + ' "domain": ["category_1", "category_2"],\n' + ' "population": "description of subjects/systems",\n' + ' "intervention": "treatment/variable under study",\n' + ' "comparison": "control/baseline",\n' + ' "outcome": "measurable endpoint",\n' + ' "search_terms": ["broad query 1", "broad query 2", "broad query 3"]\n' + "}\n" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") + ) + + +def build_literature_survey_prompt(config: DomainConfig | None, pico: dict[str, Any]) -> str: + """Build structured literature survey prompt with PICO context. + + Researcher gets domain-filtered tools and a structured protocol + instead of the old throwaway 'search 3-6 times' instruction. + """ + pop = pico.get("population", "") + interv = pico.get("intervention", "") + comp = pico.get("comparison", "") + outcome = pico.get("outcome", "") + terms = pico.get("search_terms", []) + terms_str = ", ".join(f'"{t}"' for t in terms) if isinstance(terms, list) else str(terms) + + # Domain-specific search guidance + domain_guidance = "" + if config and config.tool_tags & {"training", "clinical"}: + domain_guidance = ( + "\n\n" + "Training/clinical tools are available -- use multiple sources:\n" + "- Use `search_pubmed_training` with MeSH terms for precise biomedical literature\n" + "- Use `search_training_literature` for broader coverage via Semantic Scholar\n" + "- Combine both sources for comprehensive evidence gathering\n" + "\n" + ) + + return ( + "You are a research scientist conducting a rapid scoping review " + "(Arksey & O'Malley 2005) to map the evidence landscape.\n\n" + "\n" + f" Population: {pop}\n" + f" Intervention: {interv}\n" + f" Comparison: {comp}\n" + f" Outcome: {outcome}\n" + f" Initial search terms: {terms_str}\n" + "\n\n" + "\n" + "Execute a multi-strategy search:\n\n" + "1. DATABASE QUERIES: Use `search_literature` with broad terms first, " + "then narrow based on results. Use `explore_dataset` or domain-specific " + "search tools to find quantitative data.\n\n" + "2. CITATION CHASING: For key papers found in step 1, use " + "`search_citations` to find referenced and citing papers. " + "Greenhalgh & Peacock (2005) found 51% of sources come from snowballing.\n\n" + "3. SATURATION RULE: Stop when additional queries yield fewer than " + "2 new unique results not already covered by previous searches.\n" + "\n\n" + domain_guidance + "\n" + "When recording findings with `record_finding`, assign an evidence_level:\n" + " 1 = Systematic review / meta-analysis\n" + " 2 = Randomized controlled trial / large-scale validated study\n" + " 3 = Cohort study / prospective observational\n" + " 4 = Case-control study / retrospective analysis\n" + " 5 = Case series / computational prediction / ML model\n" + " 6 = Expert opinion / mechanistic reasoning\n" + " 0 = Not applicable / unrated\n" + "\n\n" + "\n" + "1. Record every significant finding with `record_finding` including " + "evidence_level, source_type, and source_id.\n" + "2. Use at least 2 different search strategies (database + citation chasing).\n" + "3. Cite papers by DOI when referencing literature.\n" + "4. Be quantitative: report exact numbers, effect sizes, sample sizes.\n" + "5. Do NOT call `propose_hypothesis`, `design_experiment`, " + "`evaluate_hypothesis`, or `conclude_investigation`.\n" + "6. Stop after 6-10 tool calls or when search saturation is reached.\n" + "" + ) + + +def build_literature_assessment_prompt() -> str: + """Build Haiku prompt for body-of-evidence grading after literature survey. + + GRADE-adapted grading + AMSTAR-2-adapted self-assessment. + """ + return ( + "You are a scientific evidence assessor. Given the findings from a " + "literature survey, grade the overall body of evidence and assess " + "the quality of the review process.\n\n" + "\n" + "Grade the body of evidence using GRADE-adapted criteria:\n" + "- high: Consistent results from multiple high-quality studies (levels 1-2)\n" + "- moderate: Results from well-designed studies with minor limitations\n" + "- low: Results from observational studies or studies with significant limitations\n" + "- very_low: Expert opinion only or severely limited evidence\n" + "\n\n" + "\n" + "Assess the review process against 4 rapid-review quality domains " + "(adapted from AMSTAR 2):\n" + "1. Protocol-guided search (PICO framework used)\n" + "2. Multi-source search (more than 1 search strategy used)\n" + "3. Evidence quality assessed (evidence levels assigned to findings)\n" + "4. Transparent documentation (findings recorded with source provenance)\n" + "Report which domains were satisfied and which were not.\n" + "\n\n" + "\n" + "Respond with ONLY valid JSON (no markdown fences):\n" + "{\n" + ' "evidence_grade": "high|moderate|low|very_low",\n' + ' "reasoning": "Brief justification for the grade",\n' + ' "assessment": "Summary of which quality domains were met"\n' + "}\n" + "" + ) + + +def build_researcher_prompt( + config: DomainConfig, + uploaded_data_context: str = "", +) -> str: + """Build researcher experiment prompt adapted to the domain config.""" + examples = config.experiment_examples or "" + return ( + "You are a research scientist executing a specific experiment " + "to test a hypothesis in a scientific discovery investigation.\n\n" + "\n" + "You have access to specialized tools for this domain. " + "The user's research question defines the domain. " + "Focus ONLY on the current experiment.\n\n" + "Search strategy:\n" + "- Start with short, broad queries to understand the landscape, " + "then narrow focus based on results.\n\n" + "Recording results:\n" + "- Call `record_finding` after each significant discovery, " + "always specifying hypothesis_id and evidence_type " + "('supporting' or 'contradicting').\n" + "- Include source_type and source_id for provenance tracing.\n" + "- Be quantitative: report exact numbers, scores, and " + "confidence intervals.\n\n" + "Boundaries:\n" + "- Do NOT call `conclude_investigation` -- the Director " + "synthesizes results.\n" + "- Do NOT call `propose_hypothesis`, `design_experiment`, or " + "`evaluate_hypothesis` -- those are Director responsibilities.\n" + "\n\n" + "\n" + "Apply these principles during experiment execution:\n\n" + "1. SENSITIVITY: When training models or computing scores, test " + "at least 2 parameter values. Flag results that change dramatically " + "with small parameter changes as fragile.\n\n" + "2. APPLICABILITY DOMAIN: For ML predictions, check if test " + "compounds are similar to training data. Predictions far outside " + "the training domain are unreliable -- note this when recording findings.\n\n" + "3. UNCERTAINTY: Report ranges or mean +/- SD, not just point " + "estimates. Note scoring function uncertainty where applicable.\n\n" + "4. VERIFICATION: Before recording a finding, check if it makes " + "physical sense. Verify inputs before passing to downstream tools.\n\n" + "5. NEGATIVE RESULTS: Record failed approaches with a diagnosis " + "of why they failed. A negative finding with evidence_type " + "'contradicting' is scientifically valuable -- do not omit it.\n\n" + "6. STATISTICAL TESTING: After gathering numeric comparison data, " + "use `run_statistical_test` to formally compare two groups " + "(auto-selects t-test/Welch/Mann-Whitney based on normality and " + "variance). For count/categorical data, use `run_categorical_test` " + "(auto-selects Fisher's exact or chi-squared). Record the result " + "as a finding: evidence_type='supporting' if significant with " + "meaningful effect size, 'contradicting' if non-significant or " + "trivial effect.\n" + "\n\n" + f"{examples}\n\n" + "\n" + "1. Explain your scientific reasoning before each tool call.\n" + "2. Call `record_finding` after each significant discovery with " + "hypothesis_id and evidence_type.\n" + "3. Cite papers by DOI when referencing literature.\n" + "4. If a tool returns an error, try an alternative approach.\n" + "5. Be quantitative: report exact numbers and scores.\n" + "6. Use at least 3 tool calls in this experiment.\n" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") + ) + + +def build_uploaded_data_context(files: list[UploadedFile]) -> str: + """Build XML context block describing user-uploaded files. + + Injected into Director and Researcher prompts so the model is aware + of available datasets and can reference them via ``query_uploaded_data``. + """ + if not files: + return "" + parts: list[str] = [""] + for f in files: + if f.tabular: + t = f.tabular + cols = ", ".join(t.columns) + dtypes = ", ".join(t.dtypes) + parts.append(f' ') + parts.append(f" Columns ({len(t.columns)}): {cols}") + parts.append(f" Dtypes: {dtypes}") + parts.append(f" Rows: {t.row_count}") + if t.summary_stats: + stats_lines = [] + for col_name, stats in list(t.summary_stats.items())[:6]: + mean = stats.get("mean", "?") + std = stats.get("std", "?") + stats_lines.append(f"{col_name}: mean={mean}, std={std}") + parts.append(f" Stats: {'; '.join(stats_lines)}") + if t.sample_rows: + header = " | ".join(t.columns) + parts.append(f" Sample: {header}") + for row in t.sample_rows[:3]: + parts.append(f" {' | '.join(row)}") + parts.append(" ") + elif f.document: + d = f.document + excerpt = d.text[:500] + "..." if len(d.text) > 500 else d.text + parts.append(f' ') + parts.append(f" Pages: {d.page_count}") + parts.append(f" Excerpt: {excerpt}") + parts.append(" ") + parts.append("") + return "\n".join(parts) diff --git a/server/src/ehrlich/investigation/application/prompts/constants.py b/server/src/ehrlich/investigation/application/prompts/constants.py new file mode 100644 index 0000000..85a9719 --- /dev/null +++ b/server/src/ehrlich/investigation/application/prompts/constants.py @@ -0,0 +1,781 @@ +"""Static prompt strings for Director, Researcher, and Summarizer models. + +These are used as-is (not generated by builder functions). +DIRECTOR_FORMULATION_PROMPT is the fallback when no DomainConfig is detected. +""" + +from __future__ import annotations + +DIRECTOR_FORMULATION_PROMPT = """\ +You are the Director of a molecular discovery investigation. \ +You formulate hypotheses and design the research strategy but \ +do NOT execute tools yourself. + + +Given the user's research prompt and literature survey results, \ +formulate 2-4 testable hypotheses. Each hypothesis must be: +- Specific and falsifiable +- Grounded in the literature findings provided +- Testable with the available cheminformatics and ML tools +- Orthogonal to sibling hypotheses: each must attack a different \ +mechanism, pathway, or data source + +Hypotheses tested in parallel must maximize scientific coverage: +- Different causal mechanisms or molecular targets +- Different primary data sources or tool chains +- Different validation strategies (e.g., ML prediction vs \ +structural docking vs substructure enrichment) +Do NOT formulate hypotheses that would produce overlapping experiments. + +If prior investigation results are provided in \ +, leverage their outcomes: +- Build on supported hypotheses from related investigations +- Avoid repeating refuted approaches +- Consider candidates already identified as starting points + +Also identify 1-3 negative controls (molecules known to be \ +inactive) AND 1-2 positive controls (molecules known to be \ +active against the target). Both are essential for validation: \ +negative controls confirm specificity, positive controls confirm \ +the pipeline can detect true actives. Without positive controls, \ +pipeline failures are undetectable (Zhang et al., 1999). + + + + + +Find novel beta-lactamase inhibitors to combat MRSA resistance + + +- Beta-lactamase enzymes (Class A, C, D) hydrolyze beta-lactam \ +ring; PBP2a confers methicillin resistance in MRSA +- Avibactam (diazabicyclooctane) inhibits Class A/C with Ki \ +~1 nM; clavulanate is first-gen but weak vs Class C +- Boronic acid scaffolds show broad-spectrum inhibition; \ +vaborbactam approved 2017 +- MRSA strains increasingly co-express multiple beta-lactamase \ +classes + + +{ + "hypotheses": [ + { + "statement": "Diazabicyclooctane derivatives with C2 \ +sulfonamide substituents will show dual Class A/C beta-lactamase \ +inhibition with Ki below 50 nM", + "rationale": "Avibactam's DBO core achieves covalent \ +reversible inhibition of both classes; C2 modifications with \ +electron-withdrawing sulfonamide groups enhance binding \ +to the conserved Ser70 active site", + "prediction": "Docking against Class A and Class C \ +beta-lactamases will show affinity < -7 kcal/mol for 3+ DBO \ +derivatives; ML model predicts Ki < 50 nM", + "null_prediction": "DBO derivatives show no preferential \ +binding vs non-sulfonamide controls; Ki > 500 nM", + "success_criteria": ">=3 candidates with docking < -7 \ +kcal/mol AND predicted Ki < 100 nM", + "failure_criteria": "<2 compounds meet docking threshold \ +OR all fail ADMET drug-likeness", + "scope": "MRSA beta-lactamases Class A/C; small molecules \ +MW < 500", + "hypothesis_type": "mechanistic", + "prior_confidence": 0.7 + }, + { + "statement": "Boronic acid compounds with MW below 350 Da \ +and LogP below 1.0 will penetrate MRSA cell wall and inhibit \ +PBP2a-associated beta-lactamases", + "rationale": "Vaborbactam demonstrates boronic acid \ +viability but has limited Gram-positive penetration; smaller, \ +more hydrophilic analogs may overcome MRSA's thick \ +peptidoglycan barrier", + "prediction": "Boronic acids with MW < 350 show MIC < 4 \ +ug/mL against MRSA; ADMET predicts oral bioavailability", + "null_prediction": "No boronic acid compounds show MIC \ +improvement over vaborbactam baseline (32 ug/mL)", + "success_criteria": ">=2 compounds with predicted MIC < 8 \ +ug/mL AND passes Lipinski criteria", + "failure_criteria": "All candidates fail Lipinski OR MIC \ +> 16 ug/mL", + "scope": "Gram-positive MRSA; orally bioavailable", + "hypothesis_type": "pharmacological", + "prior_confidence": 0.5 + } + ], + "negative_controls": [ + { + "smiles": "CC(=O)Oc1ccccc1C(=O)O", + "name": "Aspirin", + "source": "Non-antimicrobial NSAID; validated inactive \ +in ChEMBL" + }, + { + "smiles": "CC(C)Cc1ccc(cc1)C(C)C(=O)O", + "name": "Ibuprofen", + "source": "NSAID structurally unrelated to any BLI scaffold" + } + ], + "positive_controls": [ + { + "identifier": "CC1CC2(CC(=O)N1)C(=O)N(S2(=O)=O)O", + "name": "Avibactam", + "known_activity": "Ki ~1 nM vs Class A beta-lactamase", + "source": "FDA-approved BLI, gold standard" + } + ] +} + + + + + +Identify BACE1 inhibitors for Alzheimer's disease treatment + + +- BACE1 (beta-secretase 1) cleaves APP to produce amyloid-beta; \ +key therapeutic target for Alzheimer's +- Verubecestat reached Phase III but failed due to toxicity; \ +lanabecestat discontinued for lack of efficacy +- Aminothiazine and aminohydantoin scaffolds show sub-nM \ +potency but poor BBB penetration +- Recent work on macrocyclic BACE1 inhibitors shows improved \ +selectivity over BACE2 (>100x) with maintained potency + + +{ + "hypotheses": [ + { + "statement": "Macrocyclic aminohydantoin derivatives with \ +12-14 membered rings will achieve BACE1 IC50 below 10 nM \ +while maintaining TPSA below 90 A^2 for BBB penetration", + "rationale": "Macrocyclization constrains the \ +aminohydantoin pharmacophore into the bioactive conformation, \ +reducing entropic penalty; ring sizes of 12-14 atoms balance \ +potency with CNS MPO criteria", + "prediction": "Docking against BACE1 shows < -8 kcal/mol \ +for macrocyclic analogs; TPSA < 90 and MW < 500", + "null_prediction": "Macrocycles show no binding \ +improvement over linear aminohydantoins; TPSA > 100", + "success_criteria": ">=2 candidates with docking < -8 \ +kcal/mol AND TPSA < 90 AND MW < 500", + "failure_criteria": "No macrocycles achieve docking < -6 \ +kcal/mol OR all exceed MW 600", + "scope": "BACE1 aspartyl protease; CNS-penetrant small \ +molecules", + "hypothesis_type": "structural", + "prior_confidence": 0.6 + }, + { + "statement": "Fragment-based compounds targeting the \ +BACE1 S3 subpocket with halogenated aromatics will show \ +selectivity over BACE2 greater than 50-fold", + "rationale": "The S3 subpocket differs between BACE1 \ +(Ile110) and BACE2 (Val110); halogen bonding to this residue \ +difference drives selectivity", + "prediction": "Halogenated fragments show BACE1 docking \ +< -7 kcal/mol with BACE2 docking > -4 kcal/mol (selectivity)", + "null_prediction": "No selectivity difference; both \ +BACE1 and BACE2 docking within 1 kcal/mol", + "success_criteria": ">=1 compound with BACE1/BACE2 \ +selectivity ratio > 50x", + "failure_criteria": "All compounds show < 10x selectivity", + "scope": "BACE1 S3 subpocket; fragment-like MW < 300", + "hypothesis_type": "structural", + "prior_confidence": 0.45 + } + ], + "negative_controls": [ + { + "smiles": "CN1C=NC2=C1C(=O)N(C(=O)N2C)C", + "name": "Caffeine", + "source": "CNS-active xanthine; zero BACE1 activity \ +in published screens" + } + ], + "positive_controls": [ + { + "identifier": "CC(C)CC1=CC=C(C=C1)C(C)C(=O)N2CCC(CC2)N3C(=O)N(C3=O)C", + "name": "Verubecestat", + "known_activity": "IC50 = 2.2 nM vs BACE1", + "source": "Reached Phase III, confirmed potent inhibitor" + } + ] +} + + + + + +Respond with ONLY valid JSON (no markdown fences): +{ + "hypotheses": [ + { + "statement": "Specific testable hypothesis", + "rationale": "Causal mechanism explaining HOW and WHY", + "prediction": "If true, we expect to observe X (specific, measurable)", + "null_prediction": "If false, we would observe Y instead", + "success_criteria": "Quantitative threshold for support", + "failure_criteria": "Quantitative threshold: e.g. no compounds show activity above baseline", + "scope": "Boundary conditions: organism, compound class, target", + "hypothesis_type": "mechanistic|structural|pharmacological|toxicological|other", + "prior_confidence": 0.65 + } + ], + "negative_controls": [ + { + "smiles": "SMILES of known inactive compound", + "name": "Compound name", + "source": "Why this is a good negative control" + } + ], + "positive_controls": [ + { + "identifier": "SMILES of known active compound", + "name": "Compound name", + "known_activity": "IC50 = X nM against target Y", + "source": "Why this is a good positive control" + } + ] +} +""" + +DIRECTOR_EXPERIMENT_PROMPT = """\ +You are the Director designing an experiment to test a \ +hypothesis in a scientific discovery investigation. + + +Given the hypothesis and available tools, design a structured \ +experiment protocol with: +- A clear description of what the experiment will test +- An ordered tool_plan listing the tools to execute +- Defined variables, controls, and analysis plan + + + +Follow these 5 principles when designing experiments: + +1. VARIABLES: Define the independent variable (factor being \ +manipulated) and dependent variable (outcome being measured). \ +Be specific about units and measurement method. + +2. CONTROLS: Include at least one positive or negative baseline. \ +Positive controls confirm the assay works (known active). \ +Negative controls confirm specificity (known inactive). + +3. CONFOUNDERS: Identify threats to validity. Common confounders: \ +dataset bias, assay type mismatch, species differences, \ +structural similarity to training data. + +4. ANALYSIS PLAN: Pre-specify metrics and thresholds BEFORE \ +seeing results. This prevents post-hoc rationalization. \ +Include: primary metric, threshold, and sample size expectation. + +5. SENSITIVITY: Consider robustness. Will the conclusion change \ +if you vary a key parameter by +/- 20%? Note fragile assumptions. + + + + + +Diazabicyclooctane derivatives with C2 sulfonamide substituents \ +will show dual Class A/C beta-lactamase inhibition with Ki \ +below 50 nM + + +{ + "description": "Search ChEMBL for DBO-scaffold compounds with \ +reported beta-lactamase activity, train a classification model \ +on active/inactive compounds, then dock top predictions against \ +Class A beta-lactamase (PDB: 1ZG4) to validate binding", + "tool_plan": [ + "search_bioactivity", + "search_compounds", + "compute_descriptors", + "analyze_substructures", + "train_model", + "predict_candidates", + "search_protein_targets", + "dock_against_target", + "predict_admet", + "record_finding" + ], + "independent_variable": "C2 sulfonamide substitution pattern \ +on DBO scaffold", + "dependent_variable": "Predicted beta-lactamase inhibition \ +(Ki in nM, docking score in kcal/mol)", + "controls": [ + "positive: Avibactam (known DBO inhibitor, Ki ~1 nM)", + "negative: Aspirin (non-antimicrobial, expected inactive)" + ], + "confounders": [ + "ChEMBL dataset may bias toward published active compounds", + "Docking scoring function may not capture covalent binding" + ], + "analysis_plan": "Primary metric: AUC of ML model (threshold \ +>0.7); secondary: docking score <-7 kcal/mol for top 3 \ +candidates; expect N>=50 training compounds", + "success_criteria": "At least 3 DBO derivatives predicted \ +active with probability above 0.7 AND docking score below \ +-7.0 kcal/mol against beta-lactamase", + "failure_criteria": "Fewer than 2 compounds meet both \ +prediction and docking thresholds, OR model AUC below 0.7 \ +indicating unreliable predictions" +} + + + + + +Respond with ONLY valid JSON (no markdown fences): +{ + "description": "What this experiment will do", + "tool_plan": ["tool_name_1", "tool_name_2"], + "independent_variable": "Factor being manipulated", + "dependent_variable": "Outcome being measured", + "controls": ["positive: known active", "negative: known inactive"], + "confounders": ["identified threats to validity"], + "analysis_plan": "Pre-specified metrics and thresholds", + "success_criteria": "What result would support the hypothesis", + "failure_criteria": "What result would refute the hypothesis" +} +""" + +DIRECTOR_EVALUATION_PROMPT = """\ +You are the Director evaluating a hypothesis based on \ +experimental evidence. + + +Review the findings from the experiment and determine the \ +hypothesis outcome: +- "supported": evidence consistently supports the hypothesis \ +(confidence above 0.7) +- "refuted": evidence contradicts the hypothesis or key \ +criteria were not met +- "revised": partial support warrants a refined hypothesis + +Base your assessment on quantitative evidence. Cite specific \ +numbers from findings (scores, counts, p-values) in your \ +reasoning. If revising, the new statement must be more specific \ +than the original. + + + +Rank each finding by its reliability tier before weighing it: +1. Replicated experimental data (orthogonal assays, n>=3) -- highest +2. Single experimental measurement (one lab, one assay) +3. Curated database entry (ChEMBL, PubChem, GtoPdb) +4. Prospectively validated ML prediction (external test set) +5. Retrospective ML prediction (cross-validated) +6. Consensus computational (3+ methods agree) +7. Single computational score (one docking, one prediction) +8. Qualitative literature report -- lowest + +When reasoning, cite which tier each key finding belongs to. \ +Higher-tier evidence should carry more weight in your assessment. + + + +Distinguish real effects from noise using these domain \ +reference thresholds: +- IC50/Ki (single lab): <2-fold is noise, 3-5-fold meaningful, \ +>10-fold high confidence +- Docking score: <0.5 kcal/mol is noise, 1.0-1.5 meaningful, \ +>2.0 high confidence +- ML probability: difference <0.1 is noise, >0.2 meaningful, \ +>0.4 high confidence +- Enrichment factor EF1%: <2x is noise, >5x meaningful, >10x \ +excellent +- Effect size (Cohen's d): <0.2 trivial, 0.5 medium, >0.8 large + +Compare observed differences against these thresholds. A \ +difference below the noise floor is NOT evidence. + + + +Update confidence from prior to posterior using the evidence: +- If most evidence is supporting AND from tiers 1-3: multiply \ +prior_confidence by 1.3-1.5 (cap at 0.95) +- If evidence is mixed or from lower tiers only: keep near \ +prior, widen uncertainty +- If most evidence is contradicting: multiply prior_confidence \ +by 0.3-0.5 (floor at 0.05) +- Express the updated value as the "confidence" field in output + + + +When findings conflict, follow this resolution hierarchy: +1. Check compound/subject identity (SMILES match, correct \ +identifiers, stereochemistry) +2. Check assay comparability (IC50 vs Ki vs EC50, same assay \ +conditions, same units?) +3. Apply temporal relevance (newer data weighted higher) +4. Classify severity: + - <3x noise floor = minor: average after corrections + - 3-6x noise floor = moderate: widen confidence interval, flag + - >6x noise floor = major: do NOT average, report as \ +contradictory, investigate mechanism + + + +Check whether independent method types converge or diverge: +- If 2+ independent methods (e.g. docking + ML + literature + \ +bioactivity data) agree: evidence is CONVERGING -- increase \ +confidence +- If methods disagree: evidence is MIXED -- decrease confidence \ +and flag which methods disagree +- If methods give opposing conclusions: evidence is \ +CONTRADICTORY -- do not average, report which methods conflict + +Report convergence status in the "evidence_convergence" field. + + + +Before determining the outcome, verify: +1. CONTROLS: Did positive controls produce expected results? \ +Did negative controls score below threshold? If controls \ +failed, the experiment may be invalid regardless of test results. +2. CRITERIA COMPARISON: Compare findings against BOTH the \ +hypothesis-level criteria AND the experiment-level criteria. \ +Note any discrepancies. +3. ANALYSIS PLAN: Was the pre-specified analysis plan followed? \ +Were metrics and thresholds applied as defined before the \ +experiment ran? +4. CONFOUNDERS: Were any identified confounders observed during \ +execution? If so, note their impact on confidence. + + + +Respond with ONLY valid JSON (no markdown fences): +{ + "status": "supported|refuted|revised", + "confidence": 0.85, + "certainty_of_evidence": "high|moderate|low|very_low", + "evidence_convergence": "converging|mixed|contradictory", + "reasoning": "Detailed scientific reasoning citing specific \ +evidence from findings, referencing evidence hierarchy tiers \ +and effect size thresholds", + "key_evidence": ["list of key evidence points with numbers \ +and their reliability tier"], + "revision": "If revised, the new refined hypothesis (omit \ +if not revised)" +} +""" + +DIRECTOR_SYNTHESIS_PROMPT = """\ +You are the Director synthesizing the full investigation \ +results into a final report. + + +Review all hypothesis outcomes, findings, and negative controls \ +to produce a comprehensive synthesis. Your report must: +- Summarize hypothesis outcomes with confidence levels +- Rank candidates by multi-criteria evidence strength +- Assess model reliability using negative AND positive control results +- Identify limitations and suggest follow-up experiments +- Include all relevant citations + + +Assess model/prediction validation quality: + +1. CONTROL SEPARATION: Are positive control scores clearly separated from \ +negative control scores? If positive controls scored below the \ +active threshold, the model is unreliable -- flag this prominently. + +2. CLASSIFICATION QUALITY: With the available controls, assess whether \ +the model can discriminate actives from inactives. Consider: +- Do all positive controls score above threshold? (sensitivity check) +- Do all negative controls score below threshold? (specificity check) +- Is there clear separation between control groups? + +3. OVERALL VALIDATION: Rate as: +- "sufficient": positive controls pass, negatives pass, clear separation +- "marginal": most controls pass but separation is narrow +- "insufficient": any positive control fails, or no positive controls tested + +4. Z'-FACTOR: Z' >= 0.5 = excellent assay separation, \ +0 < Z' < 0.5 = marginal (scores overlap), \ +Z' <= 0 = unusable (no separation between controls). \ +If Z'-factor is provided, cite it explicitly in your validation assessment. + +5. PERMUTATION SIGNIFICANCE: If permutation_p_value < 0.05, the model is \ +significantly better than random. If p >= 0.05, predictions may be noise -- \ +flag this prominently. + +6. SCAFFOLD-SPLIT GAP: If the gap between random_auroc and scaffold AUROC is \ +> 0.15, the model may be memorizing scaffolds rather than learning activity. \ +Report this as a methodology limitation. + +If validation is insufficient, downgrade certainty of ALL hypothesis \ +assessments by one level and note this in limitations. + + + +For each hypothesis assessment, assign a GRADE-adapted certainty level: +- high: Multiple concordant methods, strong controls, large effect sizes, \ +evidence from tiers 1-3 +- moderate: Some concordance, adequate controls, moderate effect sizes +- low: Few methods, weak controls, small or inconsistent effects +- very_low: Single method, no controls, conflicting evidence + +Five domains that DOWNGRADE certainty: +1. Risk of bias: poor model validation, outside applicability domain +2. Inconsistency: methods disagree (docking vs ML vs literature) +3. Indirectness: evidence from different target/species/assay than asked +4. Imprecision: wide confidence intervals, small sample sizes +5. Publication bias: database coverage gaps, missing negative results + +Three domains that can UPGRADE (for computational evidence): +1. Large effect: very strong activity (>10-fold over baseline) +2. Dose-response: clear SAR gradient across compound series +3. Conservative prediction: result holds despite known biases + +Name which domains caused downgrading or upgrading in certainty_reasoning. + + + +Assign each candidate a priority tier based on certainty, evidence, and risk: + +Priority 1 (Strong Advance): High or moderate certainty. Multiple supported \ +hypotheses. Concordant docking + ML + ADMET. Controls pass. Large effects. \ +Action: queue for experimental testing. + +Priority 2 (Conditional Advance): Moderate or low certainty. 1-2 supported \ +hypotheses. Some method concordance. Adequate controls. \ +Action: additional computational validation before synthesis. + +Priority 3 (Watchlist): Low certainty. Partial support, limited methods, \ +or borderline activity. \ +Action: investigate further computationally; low resource priority. + +Priority 4 (Do Not Advance): Very low certainty. Refuted hypotheses, \ +control failures, contradictory evidence, or safety flags. \ +Action: archive; redirect effort. + + + +Report limitations using these four categories: +- methodology: model limitations, scoring function inaccuracy, \ +conformational sampling, feature representation +- data: database coverage gaps, assay heterogeneity, activity cliffs, \ +publication bias, missing data types +- scope: in silico only, limited chemical space, time-bound investigation, \ +single-target focus +- interpretation: docking scores are rank-ordering not absolute affinity, \ +ML probabilities need calibration, resistance based on known mutations only + + + +Identify what evidence was NOT collected during this investigation. \ +Construct a conceptual evidence map: for each hypothesis, which evidence \ +types are present and which are missing? + +Classify each gap: +- evidence: no data available (e.g. no crystal structure for docking) +- quality: data exists but low quality (e.g. only IC50, no Ki) +- consistency: conflicting results across methods +- scope: evidence exists but for different context (e.g. mouse not human) +- temporal: evidence outdated (e.g. resistance data from >5 years ago) + + + +Recommend specific next experiments prioritized by impact on confidence. \ +For each recommendation, specify: +- What to do and why it matters +- Whether it is computational (can be done in a follow-up investigation) \ +or experimental (requires wet-lab work) +- Impact level: critical (blocks all recommendations), high (affects \ +primary recommendation), medium (improves confidence), low (informational) + + +Scoring fields for candidates: +- prediction_score: ML model probability (0-1) +- docking_score: binding affinity in kcal/mol (negative = \ +better) +- admet_score: overall drug-likeness (0-1) +- resistance_risk: mutation risk ("low", "medium", "high") +Use 0.0 or "unknown" if a score was not computed. + + + +Respond with ONLY valid JSON (no markdown fences): +{ + "summary": "Comprehensive 2-3 paragraph summary including \ +hypothesis outcomes and key discoveries", + "candidates": [ + { + "identifier": "identifier string", + "identifier_type": "smiles", + "name": "compound name", + "rationale": "why this candidate is promising", + "rank": 1, + "priority": 1, + "scores": {}, + "attributes": {} + } + ], + "citations": ["DOI or reference strings"], + "hypothesis_assessments": [ + { + "hypothesis_id": "h1", + "statement": "the hypothesis", + "status": "supported|refuted|revised", + "confidence": 0.85, + "certainty": "high|moderate|low|very_low", + "certainty_reasoning": "downgraded by X, upgraded by Y", + "key_evidence": "summary of evidence" + } + ], + "negative_control_summary": "Summary of negative control \ +results and model reliability assessment", + "model_validation_quality": "sufficient|marginal|insufficient", + "confidence": "high/medium/low", + "limitations": [ + { + "category": "methodology|data|scope|interpretation", + "description": "specific limitation" + } + ], + "knowledge_gaps": [ + { + "gap_type": "evidence|quality|consistency|scope|temporal", + "description": "what is missing and why it matters" + } + ], + "follow_up_experiments": [ + { + "description": "what to do next", + "impact": "critical|high|medium|low", + "type": "computational|experimental" + } + ] +} +""" + +RESEARCHER_EXPERIMENT_PROMPT = """\ +You are a research scientist executing a specific experiment \ +to test a hypothesis in a molecular discovery investigation. + + +You have access to cheminformatics, ML, simulation, data \ +search, and literature tools. The user's research question \ +defines the domain. Focus ONLY on the current experiment. + +Search strategy: +- Start with short, broad queries to understand the landscape, \ +then narrow focus based on results. Overly specific initial \ +queries miss relevant data. +- Use `search_disease_targets` first to identify high-confidence \ +disease-target associations before querying ChEMBL. +- Use `get_protein_annotation` for target context (function, \ +disease links, GO terms) before docking. +- Prefer `search_bioactivity` over `explore_dataset` when you \ +know the specific target or assay type. + +Recording results: +- Call `record_finding` after each significant discovery, \ +always specifying hypothesis_id and evidence_type \ +('supporting' or 'contradicting'). +- Include source_type and source_id for provenance tracing: \ +e.g. source_type="chembl" source_id="CHEMBL25", \ +source_type="pdb" source_id="2ABC", \ +source_type="doi" source_id="10.1038/s41586-024-07613-w", \ +source_type="pubchem" source_id="2244". +- Be quantitative: report exact numbers, scores, and \ +confidence intervals. + +Boundaries: +- Do NOT call `conclude_investigation` -- the Director \ +synthesizes results. +- Do NOT call `propose_hypothesis`, `design_experiment`, or \ +`evaluate_hypothesis` -- those are Director responsibilities. + + + +Apply these principles during experiment execution: + +1. SENSITIVITY: When training models or computing scores, test \ +at least 2 parameter values (e.g. different thresholds, \ +different training sizes). Flag results that change dramatically \ +with small parameter changes as fragile. + +2. APPLICABILITY DOMAIN: For ML predictions, check if test \ +compounds are similar to training data (Tanimoto > 0.3 to \ +nearest training neighbor). Predictions far outside the training \ +domain are unreliable -- note this when recording findings. + +3. UNCERTAINTY: Report ranges or mean +/- SD, not just point \ +estimates. For ML models, report AUC with confidence interval. \ +For docking, note the scoring function uncertainty (~2 kcal/mol). + +4. VERIFICATION: Before recording a finding, check if it makes \ +physical sense. A predicted LogP of 15 or MW of 2000 for a \ +drug-like molecule is likely an error. Verify SMILES validity \ +before passing to downstream tools. + +5. NEGATIVE RESULTS: Record failed approaches with a diagnosis \ +of why they failed. A negative finding with evidence_type \ +'contradicting' is scientifically valuable -- do not omit it. + + + +Example: Docking a compound against a protein target +1. search_protein_targets(query="BACE1 human", limit=3) + -> finds PDB entries with resolution and method +2. get_protein_annotation(uniprot_id="P56817") + -> confirms function, active site residues, disease links +3. dock_against_target( + smiles="CC1=CC(=O)...", + target_id="4ivt", + exhaustiveness=16 + ) + -> returns binding affinity and pose + +Example: Training a predictive model +1. search_bioactivity( + target="beta-lactamase", + organism="Staphylococcus aureus", + assay_type="Ki", + limit=500 + ) + -> retrieves activity dataset with SMILES and values +2. train_model( + smiles_list=[...], + activity_list=[...], + model_type="xgboost" + ) + -> returns model metrics (AUC, accuracy, feature importance) +3. predict_candidates(smiles_list=[...], model_id="model_xyz") + -> scores new compounds with the trained model + +Example: Identifying disease-target associations +1. search_disease_targets(disease="Alzheimer", limit=10) + -> returns scored targets (BACE1, gamma-secretase, tau...) +2. get_protein_annotation(uniprot_id="P56817") + -> protein function, structure refs, disease annotations +3. search_bioactivity(target="BACE1", assay_type="IC50") + -> retrieves assay data for the validated target + + + +1. Explain your scientific reasoning before each tool call. +2. Call `record_finding` after each significant discovery with \ +hypothesis_id and evidence_type. +3. Cite papers by DOI when referencing literature. +4. Use `validate_smiles` before passing SMILES if uncertain. +5. If a tool returns an error, try an alternative approach. +6. Be quantitative: report exact numbers and scores. +7. Use at least 3 tool calls in this experiment. +""" + +SUMMARIZER_PROMPT = """\ +You are a scientific data compressor. Given a tool output, \ +produce a concise summary that preserves all key data points. + + +Keep: exact numbers, SMILES strings, DOIs, statistical metrics, \ +compound names, key conclusions. +Remove: verbose explanations, repeated headers, formatting \ +artifacts, redundant context. + + +Respond with ONLY the compressed text, no preamble.""" diff --git a/server/src/ehrlich/investigation/application/prompts/director.py b/server/src/ehrlich/investigation/application/prompts/director.py new file mode 100644 index 0000000..e2ece2a --- /dev/null +++ b/server/src/ehrlich/investigation/application/prompts/director.py @@ -0,0 +1,341 @@ +"""Director-phase builder functions. + +These generate domain-adaptive prompts for the Director model +using DomainConfig to customize hypothesis types, examples, +scoring instructions, and candidate labels. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ehrlich.investigation.domain.domain_config import DomainConfig + from ehrlich.investigation.domain.investigation import Investigation + + +def build_formulation_prompt( + config: DomainConfig, + uploaded_data_context: str = "", +) -> str: + """Build Director formulation prompt adapted to the domain config.""" + hyp_types = "|".join(config.hypothesis_types) if config.hypothesis_types else "other" + examples = config.director_examples or "" + return ( + "You are the Director of a scientific discovery investigation. " + "You formulate hypotheses and design the research strategy but " + "do NOT execute tools yourself.\n\n" + "\n" + "Given the user's research prompt and literature survey results, " + "formulate 2-4 testable hypotheses. Each hypothesis must be:\n" + "- Specific and falsifiable\n" + "- Grounded in the literature findings provided\n" + "- Testable with the available tools\n" + "- Orthogonal to sibling hypotheses: each must attack a different " + "mechanism, pathway, or data source\n\n" + "Hypotheses tested in parallel must maximize scientific coverage:\n" + "- Different causal mechanisms or molecular targets\n" + "- Different primary data sources or tool chains\n" + "- Different validation strategies (e.g., ML prediction vs " + "structural docking vs substructure enrichment)\n" + "Do NOT formulate hypotheses that would produce overlapping experiments.\n\n" + "If prior investigation results are provided in " + ", leverage their outcomes:\n" + "- Build on supported hypotheses from related investigations\n" + "- Avoid repeating refuted approaches\n" + "- Consider candidates already identified as starting points\n\n" + "Also identify 1-3 negative controls (subjects known to be " + "inactive) AND 1-2 positive controls (subjects known to be " + "active). Both are essential for validation: negative controls " + "confirm specificity, positive controls confirm the pipeline " + "can detect true actives. Without positive controls, pipeline " + "failures are undetectable (Zhang et al., 1999).\n" + "\n\n" + f"{examples}\n\n" + "\n" + "Respond with ONLY valid JSON (no markdown fences):\n" + "{\n" + ' "hypotheses": [\n' + " {\n" + ' "statement": "Specific testable hypothesis",\n' + ' "rationale": "Causal mechanism explaining HOW and WHY",\n' + ' "prediction": "If true, we expect to observe X",\n' + ' "null_prediction": "If false, we would observe Y instead",\n' + ' "success_criteria": "Quantitative threshold for support",\n' + ' "failure_criteria": "Quantitative threshold for refutation",\n' + ' "scope": "Boundary conditions",\n' + f' "hypothesis_type": "{hyp_types}",\n' + ' "prior_confidence": 0.65\n' + " }\n" + " ],\n" + ' "negative_controls": [\n' + " {\n" + ' "identifier": "identifier of known inactive subject",\n' + ' "name": "Name",\n' + ' "source": "Why this is a good negative control"\n' + " }\n" + " ],\n" + ' "positive_controls": [\n' + " {\n" + ' "identifier": "identifier of known active subject",\n' + ' "name": "Name",\n' + ' "known_activity": "IC50 = X nM against target Y",\n' + ' "source": "Why this is a good positive control"\n' + " }\n" + " ]\n" + "}\n" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") + ) + + +def build_experiment_prompt( + config: DomainConfig, + uploaded_data_context: str = "", +) -> str: + """Build Director experiment design prompt adapted to the domain config.""" + examples = config.experiment_examples or "" + return ( + "You are the Director designing an experiment to test a " + "hypothesis in a scientific discovery investigation.\n\n" + "\n" + "Given the hypothesis and available tools, design a structured " + "experiment protocol with:\n" + "- A clear description of what the experiment will test\n" + "- An ordered tool_plan listing the tools to execute\n" + "- Defined variables, controls, and analysis plan\n" + "\n\n" + "\n" + "Follow these 5 principles when designing experiments:\n\n" + "1. VARIABLES: Define the independent variable (factor being " + "manipulated) and dependent variable (outcome being measured). " + "Be specific about units and measurement method.\n\n" + "2. CONTROLS: Include at least one positive or negative baseline. " + "Positive controls confirm the assay works (known active). " + "Negative controls confirm specificity (known inactive).\n\n" + "3. CONFOUNDERS: Identify threats to validity. Common confounders: " + "dataset bias, assay type mismatch, species differences, " + "structural similarity to training data.\n\n" + "4. ANALYSIS PLAN: Pre-specify metrics and thresholds BEFORE " + "seeing results. This prevents post-hoc rationalization. " + "Include: primary metric, threshold, and sample size expectation. " + "When comparing two groups of numeric data, plan a statistical " + "test: specify `run_statistical_test` (continuous) or " + "`run_categorical_test` (categorical) in the tool_plan, with " + "alpha level and minimum effect size threshold as part of " + "success_criteria.\n\n" + "5. SENSITIVITY: Consider robustness. Will the conclusion change " + "if you vary a key parameter by +/- 20%? Note fragile assumptions.\n" + "\n\n" + f"{examples}\n\n" + "\n" + "Respond with ONLY valid JSON (no markdown fences):\n" + "{\n" + ' "description": "What this experiment will do",\n' + ' "tool_plan": ["tool_name_1", "tool_name_2"],\n' + ' "independent_variable": "Factor being manipulated",\n' + ' "dependent_variable": "Outcome being measured",\n' + ' "controls": ["positive: known active", "negative: known inactive"],\n' + ' "confounders": ["identified threats to validity"],\n' + ' "analysis_plan": "Pre-specified metrics and thresholds",\n' + ' "success_criteria": "What result would support the hypothesis",\n' + ' "failure_criteria": "What result would refute the hypothesis"\n' + "}\n" + "" + (f"\n\n{uploaded_data_context}" if uploaded_data_context else "") + ) + + +def build_synthesis_prompt(config: DomainConfig) -> str: + """Build Director synthesis prompt adapted to the domain config. + + Includes GRADE-adapted certainty grading, recommendation strength (priority + tiers 1-4), structured limitations taxonomy, knowledge gap analysis, and + follow-up experiment recommendations. + """ + scoring = config.synthesis_scoring_instructions or "" + label = config.candidate_label or "Candidates" + return ( + "You are the Director synthesizing the full investigation " + "results into a final report.\n\n" + "\n" + "Review all hypothesis outcomes, findings, and negative controls " + "to produce a comprehensive synthesis. Your report must:\n" + "- Summarize hypothesis outcomes with confidence levels\n" + f"- Rank {label.lower()} by multi-criteria evidence strength\n" + "- Assess model reliability using negative AND positive control results\n" + "- Identify limitations and suggest follow-up experiments\n" + "- Include all relevant citations\n\n" + "\n" + "Assess model/prediction validation quality:\n\n" + "1. CONTROL SEPARATION: Are positive control scores clearly separated " + "from negative control scores? If positive controls scored below the " + "active threshold, the model is unreliable -- flag this prominently.\n\n" + "2. CLASSIFICATION QUALITY: With the available controls, assess whether " + "the model can discriminate actives from inactives. Consider:\n" + "- Do all positive controls score above threshold? (sensitivity check)\n" + "- Do all negative controls score below threshold? (specificity check)\n" + "- Is there clear separation between control groups?\n\n" + "3. OVERALL VALIDATION: Rate as:\n" + '- "sufficient": positive controls pass, negatives pass, clear separation\n' + '- "marginal": most controls pass but separation is narrow\n' + '- "insufficient": any positive control fails, or no positive controls tested\n\n' + "4. Z'-FACTOR: Z' >= 0.5 = excellent assay separation, " + "0 < Z' < 0.5 = marginal (scores overlap), " + "Z' <= 0 = unusable (no separation between controls). " + "If Z'-factor is provided, cite it explicitly in your validation assessment.\n\n" + "5. PERMUTATION SIGNIFICANCE: If permutation_p_value < 0.05, the model is " + "significantly better than random. If p >= 0.05, predictions may be noise -- " + "flag this prominently.\n\n" + "6. SCAFFOLD-SPLIT GAP: If the gap between random_auroc and scaffold AUROC is " + "> 0.15, the model may be memorizing scaffolds rather than learning activity. " + "Report this as a methodology limitation.\n\n" + "If validation is insufficient, downgrade certainty of ALL hypothesis " + "assessments by one level and note this in limitations.\n" + "\n\n" + "\n" + "For each hypothesis assessment, assign a GRADE-adapted certainty level:\n" + "- high: Multiple concordant methods, strong controls, large effect sizes, " + "evidence from tiers 1-3\n" + "- moderate: Some concordance, adequate controls, moderate effect sizes\n" + "- low: Few methods, weak controls, small or inconsistent effects\n" + "- very_low: Single method, no controls, conflicting evidence\n\n" + "Five domains that DOWNGRADE certainty:\n" + "1. Risk of bias: poor model validation, outside applicability domain\n" + "2. Inconsistency: methods disagree (docking vs ML vs literature)\n" + "3. Indirectness: evidence from different target/species/assay than asked\n" + "4. Imprecision: wide confidence intervals, small sample sizes\n" + "5. Publication bias: database coverage gaps, missing negative results\n\n" + "Three domains that can UPGRADE (for computational evidence):\n" + "1. Large effect: very strong activity (>10-fold over baseline)\n" + "2. Dose-response: clear SAR gradient across compound series\n" + "3. Conservative prediction: result holds despite known biases\n\n" + "Name which domains caused downgrading or upgrading in certainty_reasoning.\n" + "\n\n" + "\n" + "Assign each candidate a priority tier based on certainty, evidence, and risk:\n\n" + "Priority 1 (Strong Advance): High or moderate certainty. Multiple supported " + "hypotheses. Concordant methods. Controls pass. Large effects. " + "Action: queue for experimental testing.\n\n" + "Priority 2 (Conditional Advance): Moderate or low certainty. 1-2 supported " + "hypotheses. Some method concordance. Adequate controls. " + "Action: additional computational validation.\n\n" + "Priority 3 (Watchlist): Low certainty. Partial support, limited methods, " + "or borderline activity. " + "Action: investigate further computationally; low resource priority.\n\n" + "Priority 4 (Do Not Advance): Very low certainty. Refuted hypotheses, " + "control failures, contradictory evidence, or safety flags. " + "Action: archive; redirect effort.\n" + "\n\n" + "\n" + "Report limitations using these four categories:\n" + "- methodology: model limitations, scoring function inaccuracy, " + "conformational sampling, feature representation\n" + "- data: database coverage gaps, assay heterogeneity, activity cliffs, " + "publication bias, missing data types\n" + "- scope: in silico only, limited chemical space, time-bound investigation, " + "single-target focus\n" + "- interpretation: scores are rank-ordering not absolute, " + "ML probabilities need calibration, predictions based on known data only\n" + "\n\n" + "\n" + "Identify what evidence was NOT collected during this investigation. " + "Construct a conceptual evidence map: for each hypothesis, which evidence " + "types are present and which are missing?\n\n" + "Classify each gap:\n" + "- evidence: no data available\n" + "- quality: data exists but low quality\n" + "- consistency: conflicting results across methods\n" + "- scope: evidence exists but for different context\n" + "- temporal: evidence outdated\n" + "\n\n" + "\n" + "Recommend specific next experiments prioritized by impact on confidence. " + "For each recommendation, specify:\n" + "- What to do and why it matters\n" + "- Whether it is computational or experimental\n" + "- Impact level: critical, high, medium, or low\n" + "\n\n" + f"{scoring}\n" + "\n\n" + "\n" + "Respond with ONLY valid JSON (no markdown fences):\n" + "{\n" + ' "summary": "Comprehensive 2-3 paragraph summary",\n' + ' "candidates": [\n' + " {\n" + ' "identifier": "identifier string",\n' + ' "identifier_type": "' + config.identifier_type + '",\n' + ' "name": "name",\n' + ' "rationale": "why this candidate is promising",\n' + ' "rank": 1,\n' + ' "priority": 1,\n' + ' "scores": {},\n' + ' "attributes": {}\n' + " }\n" + " ],\n" + ' "citations": ["DOI or reference strings"],\n' + ' "hypothesis_assessments": [\n' + " {\n" + ' "hypothesis_id": "h1",\n' + ' "statement": "the hypothesis",\n' + ' "status": "supported|refuted|revised",\n' + ' "confidence": 0.85,\n' + ' "certainty": "high|moderate|low|very_low",\n' + ' "certainty_reasoning": "downgraded by X, upgraded by Y",\n' + ' "key_evidence": "summary of evidence"\n' + " }\n" + " ],\n" + ' "negative_control_summary": "Summary of negative control results",\n' + ' "model_validation_quality": "sufficient|marginal|insufficient",\n' + ' "confidence": "high/medium/low",\n' + ' "limitations": [\n' + " {\n" + ' "category": "methodology|data|scope|interpretation",\n' + ' "description": "specific limitation"\n' + " }\n" + " ],\n" + ' "knowledge_gaps": [\n' + " {\n" + ' "gap_type": "evidence|quality|consistency|scope|temporal",\n' + ' "description": "what is missing and why it matters"\n' + " }\n" + " ],\n" + ' "follow_up_experiments": [\n' + " {\n" + ' "description": "what to do next",\n' + ' "impact": "critical|high|medium|low",\n' + ' "type": "computational|experimental"\n' + " }\n" + " ]\n" + "}\n" + "" + ) + + +def build_multi_investigation_context( + investigations: list[Investigation], +) -> str: + """Compress past completed investigations into XML context for the Director.""" + if not investigations: + return "" + parts: list[str] = [""] + for inv in investigations[:3]: + parts.append(f' ') + for h in inv.hypotheses[:4]: + if h.status.value in ("supported", "refuted", "revised"): + parts.append( + f" " + f"{h.statement}" + ) + for c in inv.candidates[:2]: + top_score = max(c.scores.values()) if c.scores else 0 + parts.append( + f" " + f"{c.name or 'unnamed'}" + ) + if inv.summary: + parts.append(f" {inv.summary[:300]}") + parts.append(" ") + parts.append("") + return "\n".join(parts) diff --git a/server/src/ehrlich/investigation/application/researcher_executor.py b/server/src/ehrlich/investigation/application/researcher_executor.py index abedf1b..13f7e4a 100644 --- a/server/src/ehrlich/investigation/application/researcher_executor.py +++ b/server/src/ehrlich/investigation/application/researcher_executor.py @@ -4,7 +4,7 @@ import logging from typing import TYPE_CHECKING, Any -from ehrlich.investigation.application.prompts import SUMMARIZER_PROMPT +from ehrlich.investigation.application.prompts.constants import SUMMARIZER_PROMPT from ehrlich.investigation.domain.events import ( DomainEvent, FindingRecorded, diff --git a/server/tests/investigation/test_prompts.py b/server/tests/investigation/test_prompts.py index 21a853c..96e8972 100644 --- a/server/tests/investigation/test_prompts.py +++ b/server/tests/investigation/test_prompts.py @@ -1,14 +1,18 @@ from ehrlich.investigation.application.multi_orchestrator import SEARCH_TOOLS -from ehrlich.investigation.application.prompts import ( +from ehrlich.investigation.application.prompts.builders import ( + build_literature_assessment_prompt, + build_literature_survey_prompt, + build_pico_and_classification_prompt, + build_researcher_prompt, +) +from ehrlich.investigation.application.prompts.constants import ( DIRECTOR_EVALUATION_PROMPT, DIRECTOR_EXPERIMENT_PROMPT, DIRECTOR_SYNTHESIS_PROMPT, RESEARCHER_EXPERIMENT_PROMPT, +) +from ehrlich.investigation.application.prompts.director import ( build_experiment_prompt, - build_literature_assessment_prompt, - build_literature_survey_prompt, - build_pico_and_classification_prompt, - build_researcher_prompt, build_synthesis_prompt, ) from ehrlich.investigation.domain.domains.molecular import MOLECULAR_SCIENCE diff --git a/server/tests/investigation/test_uploaded_file.py b/server/tests/investigation/test_uploaded_file.py index b7fedbe..19703f1 100644 --- a/server/tests/investigation/test_uploaded_file.py +++ b/server/tests/investigation/test_uploaded_file.py @@ -209,14 +209,14 @@ def test_query_filter_contains(self) -> None: class TestPromptInjection: def test_empty_files(self) -> None: - from ehrlich.investigation.application.prompts import ( + from ehrlich.investigation.application.prompts.builders import ( build_uploaded_data_context, ) assert build_uploaded_data_context([]) == "" def test_tabular_context(self) -> None: - from ehrlich.investigation.application.prompts import ( + from ehrlich.investigation.application.prompts.builders import ( build_uploaded_data_context, ) @@ -236,7 +236,7 @@ def test_tabular_context(self) -> None: assert "" in ctx def test_document_context(self) -> None: - from ehrlich.investigation.application.prompts import ( + from ehrlich.investigation.application.prompts.builders import ( build_uploaded_data_context, ) From a08ed28e4377056beb8a55d83839c4118177059e Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 09:53:52 -0700 Subject: [PATCH 15/35] refactor(14A): decompose orchestrator run() into phase_runner + literature_survey - Extract 5 investigation phases into phase_runner.py as async generators - Extract literature survey runner + PICO context builder into literature_survey.py - Move _DirectorResult to phase_runner.py (canonical location) - Keep _compact_result in researcher_executor.py (avoids circular import) - Update SEARCH_TOOLS import in test_prompts.py - Update test_uploaded_file.py to use dispatcher for query handler - Reduce multi_orchestrator.py from 1,221 to ~240 lines - Mark Phase 14A as DONE in roadmap 87/87 tests passing --- docs/roadmap/phase-14-engine-hardening.md | 8 +- .../application/literature_survey.py | 312 +++++ .../application/multi_orchestrator.py | 1163 ++--------------- .../investigation/application/phase_runner.py | 901 +++++++++++++ server/tests/investigation/test_prompts.py | 2 +- .../tests/investigation/test_uploaded_file.py | 31 +- 6 files changed, 1346 insertions(+), 1071 deletions(-) create mode 100644 server/src/ehrlich/investigation/application/literature_survey.py create mode 100644 server/src/ehrlich/investigation/application/phase_runner.py diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md index d2142c3..45eb755 100644 --- a/docs/roadmap/phase-14-engine-hardening.md +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -14,7 +14,7 @@ Ehrlich sits alongside three contemporary AI scientific discovery systems: Ehrlich differentiates by: domain-agnostic architecture (4 domains vs single-domain competitors), multi-model orchestration (Director/Researcher/Summarizer), real-time streaming UI, 18 external data sources, and computational laboratory (ML pipelines + causal inference as experimentation). -## 14A: Orchestrator Decomposition -- TODO +## 14A: Orchestrator Decomposition -- DONE Split `MultiModelOrchestrator` (1,788 lines, 11 concerns) into focused modules following Clean Architecture. No behavior changes -- pure structural refactoring. @@ -24,9 +24,11 @@ Split `MultiModelOrchestrator` (1,788 lines, 11 concerns) into focused modules f - [x] Extract `batch_executor.py` -- parallel experiment execution via asyncio.Queue (56 lines) - [x] Merged uploaded_data_query into tool_dispatcher.py - [x] Kept run() in multi_orchestrator.py as main coordination point -- [ ] Orchestrator reduced to ~400 lines: init, run (delegates to phase_manager), approve +- [x] Extract `phase_runner.py` -- 5 investigation phases as standalone async generators +- [x] Extract `literature_survey.py` -- literature survey runner + PICO context builder + SEARCH_TOOLS +- [x] Orchestrator reduced to ~240 lines: init, run (delegates to phase runners), approve, director_call -Result: `multi_orchestrator.py` from 1,788 to 1,216 lines (4 modules extracted: 718 lines total) +Result: `multi_orchestrator.py` from 1,788 to ~240 lines (6 modules extracted). 87/87 tests passing. ## 14B: Route File Cleanup -- DONE diff --git a/server/src/ehrlich/investigation/application/literature_survey.py b/server/src/ehrlich/investigation/application/literature_survey.py new file mode 100644 index 0000000..69722d6 --- /dev/null +++ b/server/src/ehrlich/investigation/application/literature_survey.py @@ -0,0 +1,312 @@ +"""Literature survey runner extracted from MultiModelOrchestrator. + +Contains the structured literature survey with PICO framework, citation +chasing, evidence grading, and helper utilities shared with the researcher +executor (compact result schemas, search tool set). +""" + +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.application.prompts.builders import ( + build_literature_assessment_prompt, + build_literature_survey_prompt, +) +from ehrlich.investigation.application.researcher_executor import ( + _compact_result as compact_result, + maybe_viz_event, + summarize_output, +) +from ehrlich.investigation.domain.events import ( + DomainEvent, + FindingRecorded, + LiteratureSurveyCompleted, + Thinking, + ToolCalled, + ToolResultEvent, +) +from ehrlich.investigation.domain.finding import Finding +from ehrlich.investigation.domain.schemas import LITERATURE_GRADING_SCHEMA + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from ehrlich.investigation.application.cost_tracker import CostTracker + from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher + from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.domain.domain_config import DomainConfig + from ehrlich.investigation.domain.investigation import Investigation + from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Shared constants +# --------------------------------------------------------------------------- + +SEARCH_TOOLS: frozenset[str] = frozenset( + { + "search_literature", + "search_citations", + "explore_dataset", + "search_bioactivity", + "search_compounds", + "search_pharmacology", + "search_training_literature", + "search_pubmed_training", + "search_clinical_trials", + "search_exercise_database", + "search_supplement_evidence", + "search_prior_research", + } +) + + +def _build_output_config(schema: dict[str, Any]) -> dict[str, Any]: + return {"format": {"type": "json_schema", "schema": schema}} + + +# --------------------------------------------------------------------------- +# Literature context builder (static helper) +# --------------------------------------------------------------------------- + + +def build_literature_context( + investigation: Investigation, + pico: dict[str, Any], +) -> str: + """Build structured XML context from PICO + findings for Director formulation.""" + parts: list[str] = [""] + pop = pico.get("population", "") + interv = pico.get("intervention", "") + comp = pico.get("comparison", "") + outcome = pico.get("outcome", "") + parts.append( + f' ' + ) + if investigation.findings: + parts.append(" ") + for f in investigation.findings: + parts.append( + f' {f.title}: {f.detail}' + ) + parts.append(" ") + parts.append("") + return "\n".join(parts) + + +# --------------------------------------------------------------------------- +# Literature survey runner +# --------------------------------------------------------------------------- + + +async def run_literature_survey( + researcher: AnthropicClientAdapter, + summarizer: AnthropicClientAdapter, + dispatcher: ToolDispatcher, + registry: ToolRegistry, + active_config: DomainConfig | None, + summarizer_threshold: int, + max_iterations: int, + investigation: Investigation, + cost: CostTracker, + pico: dict[str, Any], +) -> AsyncGenerator[DomainEvent, None]: + """Structured literature survey with PICO, citation chasing, and evidence grading.""" + # A. Domain-filtered tools (no hardcoded set) + if active_config: + excluded = { + "conclude_investigation", + "propose_hypothesis", + "design_experiment", + "evaluate_hypothesis", + } + tool_schemas = [ + t + for t in registry.list_schemas_for_domain(active_config.tool_tags) + if t["name"] not in excluded + ] + else: + lit_tools = { + "search_literature", + "search_citations", + "get_reference", + "explore_dataset", + "record_finding", + } + tool_schemas = [t for t in registry.list_schemas() if t["name"] in lit_tools] + + # Build structured survey prompt + survey_prompt = build_literature_survey_prompt(active_config, pico) + + messages: list[dict[str, Any]] = [ + { + "role": "user", + "content": ( + f"Research prompt: {investigation.prompt}\n\n" + f"Conduct a structured literature survey following the protocol above." + ), + }, + ] + + search_queries = 0 + total_results = 0 + + for _iteration in range(max_iterations): + investigation.iteration += 1 + tool_choice = {"type": "any"} if _iteration == 0 else None + response = await researcher.create_message( + system=survey_prompt, + messages=messages, + tools=tool_schemas, + tool_choice=tool_choice, + ) + cost.add_usage( + response.input_tokens, + response.output_tokens, + researcher.model, + cache_read_tokens=response.cache_read_input_tokens, + cache_write_tokens=response.cache_write_input_tokens, + ) + + assistant_content: list[dict[str, Any]] = [] + tool_use_blocks: list[dict[str, Any]] = [] + + for block in response.content: + assistant_content.append(block) + if block["type"] == "text": + yield Thinking(text=block["text"], investigation_id=investigation.id) + elif block["type"] == "tool_use": + tool_use_blocks.append(block) + + messages.append({"role": "assistant", "content": assistant_content}) + + if response.stop_reason == "end_turn" or not tool_use_blocks: + break + + tool_results: list[dict[str, Any]] = [] + for tool_block in tool_use_blocks: + tool_name = tool_block["name"] + tool_input = tool_block["input"] + tool_use_id = tool_block["id"] + cost.add_tool_call() + + # Track search stats + if tool_name in SEARCH_TOOLS: + search_queries += 1 + + yield ToolCalled( + tool_name=tool_name, + tool_input=tool_input, + investigation_id=investigation.id, + ) + + result_str = await dispatcher.dispatch(tool_name, tool_input, investigation) + result_str = compact_result(tool_name, result_str) + + # Track result counts + try: + result_data = json.loads(result_str) + if isinstance(result_data, dict): + total_results += int(result_data.get("count", 0)) + except (json.JSONDecodeError, TypeError, ValueError): + pass + + summarized_str, summarize_event = await summarize_output( + summarizer, cost, tool_name, result_str, investigation.id, + summarizer_threshold, + ) + if summarize_event is not None: + yield summarize_event + + content_for_model = summarized_str if summarize_event else result_str + preview = result_str[:1500] if len(result_str) > 1500 else result_str + yield ToolResultEvent( + tool_name=tool_name, + result_preview=preview, + investigation_id=investigation.id, + ) + + # Emit visualization event if tool returned viz payload + viz_event = maybe_viz_event(result_str, "", investigation.id) + if viz_event is not None: + yield viz_event + + if tool_name == "record_finding": + finding = Finding( + title=tool_input.get("title", ""), + detail=tool_input.get("detail", ""), + evidence=tool_input.get("evidence", ""), + hypothesis_id=tool_input.get("hypothesis_id", ""), + evidence_type=tool_input.get("evidence_type", "neutral"), + source_type=tool_input.get("source_type", ""), + source_id=tool_input.get("source_id", ""), + evidence_level=int(tool_input.get("evidence_level", 0)), + ) + investigation.record_finding(finding) + yield FindingRecorded( + title=finding.title, + detail=finding.detail, + hypothesis_id=finding.hypothesis_id, + evidence_type=finding.evidence_type, + evidence=finding.evidence, + source_type=finding.source_type, + source_id=finding.source_id, + evidence_level=finding.evidence_level, + investigation_id=investigation.id, + ) + + tool_results.append( + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": content_for_model, + } + ) + + messages.append({"role": "user", "content": tool_results}) + + # B. Body-of-evidence grading (Haiku, 1 call) + evidence_grade = "" + assessment = "" + if investigation.findings: + findings_for_grading = "\n".join( + f"- [level={f.evidence_level}] {f.title}: {f.detail}" + for f in investigation.findings + ) + grade_response = await summarizer.create_message( + system=build_literature_assessment_prompt(), + messages=[{"role": "user", "content": findings_for_grading}], + tools=[], + output_config=_build_output_config(LITERATURE_GRADING_SCHEMA), + ) + cost.add_usage( + grade_response.input_tokens, + grade_response.output_tokens, + summarizer.model, + cache_read_tokens=grade_response.cache_read_input_tokens, + cache_write_tokens=grade_response.cache_write_input_tokens, + ) + grade_text = "" + for block in grade_response.content: + if block.get("type") == "text": + grade_text += block["text"] + grade_data = json.loads(grade_text) + evidence_grade = grade_data.get("evidence_grade", "") + assessment = grade_data.get("assessment", "") + + # C. Yield LiteratureSurveyCompleted event + yield LiteratureSurveyCompleted( + pico=pico, + search_queries=search_queries, + total_results=total_results, + included_results=len(investigation.findings), + evidence_grade=evidence_grade, + assessment=assessment, + investigation_id=investigation.id, + ) diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index 771e4c9..d653f5c 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -3,76 +3,34 @@ import asyncio import json import logging -from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from ehrlich.investigation.application.batch_executor import run_experiment_batch from ehrlich.investigation.application.cost_tracker import CostTracker -from ehrlich.investigation.application.diagram_builder import generate_diagram +from ehrlich.investigation.application.literature_survey import run_literature_survey +from ehrlich.investigation.application.phase_runner import ( + _DirectorResult, + run_classification_phase, + run_controls_phase, + run_formulation_phase, + run_hypothesis_testing_phase, + run_synthesis_phase, +) from ehrlich.investigation.application.prompts.builders import ( - build_literature_assessment_prompt, - build_literature_survey_prompt, - build_pico_and_classification_prompt, - build_researcher_prompt, build_uploaded_data_context, ) from ehrlich.investigation.application.prompts.constants import ( - DIRECTOR_EVALUATION_PROMPT, - DIRECTOR_EXPERIMENT_PROMPT, - DIRECTOR_FORMULATION_PROMPT, - DIRECTOR_SYNTHESIS_PROMPT, RESEARCHER_EXPERIMENT_PROMPT, ) -from ehrlich.investigation.application.prompts.director import ( - build_experiment_prompt, - build_formulation_prompt, - build_multi_investigation_context, - build_synthesis_prompt, -) -from ehrlich.investigation.application.researcher_executor import ( - maybe_viz_event, - summarize_output, -) from ehrlich.investigation.application.tool_cache import ToolCache from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher -from ehrlich.investigation.domain.candidate import Candidate -from ehrlich.investigation.domain.domain_config import merge_domain_configs from ehrlich.investigation.domain.events import ( CostUpdate, - DomainDetected, DomainEvent, - ExperimentCompleted, - ExperimentStarted, - FindingRecorded, - HypothesisApprovalRequested, - HypothesisEvaluated, - HypothesisFormulated, - InvestigationCompleted, InvestigationError, - LiteratureSurveyCompleted, - NegativeControlRecorded, PhaseChanged, - PositiveControlRecorded, Thinking, - ToolCalled, - ToolResultEvent, - ValidationMetricsComputed, ) -from ehrlich.investigation.domain.experiment import Experiment, ExperimentStatus -from ehrlich.investigation.domain.finding import Finding -from ehrlich.investigation.domain.hypothesis import Hypothesis, HypothesisStatus from ehrlich.investigation.domain.investigation import Investigation, InvestigationStatus -from ehrlich.investigation.domain.negative_control import NegativeControl -from ehrlich.investigation.domain.positive_control import PositiveControl -from ehrlich.investigation.domain.schemas import ( - EVALUATION_SCHEMA, - EXPERIMENT_DESIGN_SCHEMA, - FORMULATION_SCHEMA, - LITERATURE_GRADING_SCHEMA, - PICO_SCHEMA, - SYNTHESIS_SCHEMA, -) -from ehrlich.investigation.domain.validation import compute_z_prime if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -87,57 +45,12 @@ from ehrlich.investigation.infrastructure.mcp_bridge import MCPBridge -@dataclass(frozen=True) -class _DirectorResult: - data: dict[str, Any] - thinking: str - - def _build_output_config(schema: dict[str, Any]) -> dict[str, Any]: return {"format": {"type": "json_schema", "schema": schema}} logger = logging.getLogger(__name__) -_COMPACT_SCHEMAS: dict[str, list[str]] = { - "compute_descriptors": ["molecular_weight", "logp", "tpsa", "hbd", "hba", "qed", "num_rings"], - "compute_fingerprint": ["fingerprint_type", "num_bits"], - "validate_smiles": ["valid", "canonical_smiles"], - "explore_dataset": ["name", "target", "size", "active_count"], - "search_bioactivity": ["target", "size", "active_count"], - "search_protein_targets": ["query", "count", "targets"], - "tanimoto_similarity": ["similarity"], -} - -SEARCH_TOOLS: frozenset[str] = frozenset( - { - "search_literature", - "search_citations", - "explore_dataset", - "search_bioactivity", - "search_compounds", - "search_pharmacology", - "search_training_literature", - "search_pubmed_training", - "search_clinical_trials", - "search_exercise_database", - "search_supplement_evidence", - "search_prior_research", - } -) - - -def _compact_result(tool_name: str, result: str) -> str: - schema = _COMPACT_SCHEMAS.get(tool_name) - if not schema: - return result - try: - data = json.loads(result) - compacted = {k: data[k] for k in schema if k in data} - return json.dumps(compacted) - except (json.JSONDecodeError, TypeError): - return result - class MultiModelOrchestrator: def __init__( @@ -184,6 +97,8 @@ def approve_hypotheses( ) -> None: """Called by API to approve/reject hypotheses and unblock the loop.""" if self._investigation: + from ehrlich.investigation.domain.hypothesis import HypothesisStatus + for h in self._investigation.hypotheses: if h.id in rejected_ids: h.status = HypothesisStatus.REJECTED @@ -201,10 +116,49 @@ def _cost_event(self, cost: CostTracker, investigation_id: str) -> CostUpdate: investigation_id=investigation_id, ) + async def _director_call( + self, + cost: CostTracker, + system: str, + user_message: str, + investigation_id: str, + output_config: dict[str, Any] | None = None, + ) -> AsyncGenerator[Thinking | _DirectorResult, None]: + text = "" + thinking_text = "" + response = None + async for event in self._director.stream_message( + system=system, + messages=[{"role": "user", "content": user_message}], + tools=[], + output_config=output_config, + ): + if event["type"] == "thinking": + thinking_text += event["text"] + yield Thinking(text=event["text"], investigation_id=investigation_id) + elif event["type"] == "text": + text += event["text"] + elif event["type"] == "result": + response = event["response"] + + if response: + cost.add_usage( + response.input_tokens, + response.output_tokens, + self._director.model, + cache_read_tokens=response.cache_read_input_tokens, + cache_write_tokens=response.cache_write_input_tokens, + ) + + yield _DirectorResult(data=json.loads(text), thinking=thinking_text) + + # ------------------------------------------------------------------ + # Main orchestration loop + # ------------------------------------------------------------------ + async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, None]: investigation.status = InvestigationStatus.RUNNING cost = CostTracker() - pico: dict[str, Any] = {} # Build uploaded data context for prompt injection if investigation.uploaded_files: @@ -228,720 +182,88 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, logger.exception("MCP bridge connection failed, continuing without MCP") try: - # 1. Classify domain + PICO decomposition (single Haiku call) - yield PhaseChanged( - phase=1, - name="Classification & PICO", - description="Detecting domain and decomposing research question", - investigation_id=investigation.id, - ) - + # 1. Classify domain + PICO decomposition + pico: dict[str, Any] = {} prior_context = "" - if self._repository: - valid_categories: frozenset[str] = frozenset() - if self._domain_registry: - valid_categories = self._domain_registry.all_categories() - pico_prompt = build_pico_and_classification_prompt( - valid_categories, self._uploaded_data_context - ) - pico_response = await self._summarizer.create_message( - system=pico_prompt, - messages=[{"role": "user", "content": investigation.prompt}], - tools=[], - output_config=_build_output_config(PICO_SCHEMA), - ) - cost.add_usage( - pico_response.input_tokens, - pico_response.output_tokens, - self._summarizer.model, - cache_read_tokens=pico_response.cache_read_input_tokens, - cache_write_tokens=pico_response.cache_write_input_tokens, - ) - pico_text = "" - for block in pico_response.content: - if block.get("type") == "text": - pico_text += block["text"] - pico_data = json.loads(pico_text) - raw_domain = pico_data.get("domain", "other") - domain_categories: list[str] = ( - [d.strip().lower() for d in raw_domain] - if isinstance(raw_domain, list) - else [raw_domain.strip().lower()] - ) - investigation.domain = ", ".join(domain_categories) - pico = { - "population": pico_data.get("population", ""), - "intervention": pico_data.get("intervention", ""), - "comparison": pico_data.get("comparison", ""), - "outcome": pico_data.get("outcome", ""), - "search_terms": pico_data.get("search_terms", []), - } - - # Detect domain configs (multi-domain) and yield event - if self._domain_registry: - detected_configs, is_fallback = self._domain_registry.detect(domain_categories) - self._active_config = merge_domain_configs(detected_configs) - self._researcher_prompt = build_researcher_prompt( - self._active_config, self._uploaded_data_context - ) - yield DomainDetected( - domain=self._active_config.name, - display_config=self._active_config.to_display_dict(), - is_fallback=is_fallback, - investigation_id=investigation.id, - ) - - all_investigations = await self._repository.list_all() - related = [ - inv - for inv in all_investigations - if inv.status == InvestigationStatus.COMPLETED - and inv.id != investigation.id - and any(cat in inv.domain for cat in domain_categories) - ] - if related: - prior_context = build_multi_investigation_context(related) - + async for event in run_classification_phase( + investigation, cost, self._summarizer, + self._uploaded_data_context, self._repository, + self._domain_registry, self._director_call, + ): + if isinstance(event, dict) and "__phase_result__" in event: + result = event["__phase_result__"] + pico = result["pico"] + prior_context = result["prior_context"] + self._active_config = result["active_config"] + if result["researcher_prompt"]: + self._researcher_prompt = result["researcher_prompt"] + else: + yield event yield self._cost_event(cost, investigation.id) - # 2. Literature survey by researcher (structured, domain-aware) + # 2. Literature survey yield PhaseChanged( phase=2, name="Literature Survey", description="Structured literature search with PICO framework and citation chasing", investigation_id=investigation.id, ) - async for event in self._run_literature_survey(investigation, cost, pico): + async for event in run_literature_survey( + self._researcher, self._summarizer, self._dispatcher, + self._registry, self._active_config, + self._summarizer_threshold, self._max_iterations_per_experiment, + investigation, cost, pico, + ): yield event - yield self._cost_event(cost, investigation.id) # 3. Director formulates hypotheses - yield PhaseChanged( - phase=3, - name="Formulation", - description="Director formulating testable hypotheses from literature evidence", - investigation_id=investigation.id, - ) - prior_section = f"\n\n{prior_context}" if prior_context else "" - formulation_prompt = ( - build_formulation_prompt(self._active_config, self._uploaded_data_context) - if self._active_config - else DIRECTOR_FORMULATION_PROMPT - ) - - # Build structured XML context from PICO + findings - literature_context = self._build_literature_context(investigation, pico) - result = _DirectorResult(data={}, thinking="") - async for director_event in self._director_call( - cost, - formulation_prompt, - f"Research prompt: {investigation.prompt}\n\n" - f"{literature_context}\n\n" - f"Formulate 2-4 testable hypotheses and identify negative controls." - + prior_section, - investigation.id, - output_config=_build_output_config(FORMULATION_SCHEMA), + neg_control_suggestions: list[dict[str, Any]] = [] + pos_control_suggestions: list[dict[str, Any]] = [] + async for event in run_formulation_phase( + investigation, cost, self._active_config, + self._uploaded_data_context, prior_context, pico, + self._require_approval, self._approval_event, + self._director_call, self._cost_event, ): - if isinstance(director_event, _DirectorResult): - result = director_event + if isinstance(event, dict) and "__phase_result__" in event: + result = event["__phase_result__"] + neg_control_suggestions = result["neg_control_suggestions"] + pos_control_suggestions = result["pos_control_suggestions"] else: - yield director_event - formulation = result.data - - # Create hypothesis entities - for h_data in formulation.get("hypotheses", []): - hypothesis = Hypothesis( - statement=h_data.get("statement", ""), - rationale=h_data.get("rationale", ""), - prediction=h_data.get("prediction", ""), - null_prediction=h_data.get("null_prediction", ""), - success_criteria=h_data.get("success_criteria", ""), - failure_criteria=h_data.get("failure_criteria", ""), - scope=h_data.get("scope", ""), - hypothesis_type=h_data.get("hypothesis_type", ""), - prior_confidence=min(1.0, max(0.0, h_data.get("prior_confidence", 0.0))), - ) - investigation.add_hypothesis(hypothesis) - yield HypothesisFormulated( - hypothesis_id=hypothesis.id, - statement=hypothesis.statement, - rationale=hypothesis.rationale, - prediction=hypothesis.prediction, - null_prediction=hypothesis.null_prediction, - success_criteria=hypothesis.success_criteria, - failure_criteria=hypothesis.failure_criteria, - scope=hypothesis.scope, - hypothesis_type=hypothesis.hypothesis_type, - prior_confidence=hypothesis.prior_confidence, - parent_id="", - investigation_id=investigation.id, - ) - - # Store control suggestions for later - neg_control_suggestions = formulation.get("negative_controls", []) - pos_control_suggestions = formulation.get("positive_controls", []) - yield self._cost_event(cost, investigation.id) - - # Request user approval before testing - if self._require_approval: - self._investigation = investigation - yield HypothesisApprovalRequested( - hypotheses=[ - { - "id": h.id, - "statement": h.statement, - "rationale": h.rationale, - "prediction": h.prediction, - "scope": h.scope, - "hypothesis_type": h.hypothesis_type, - "prior_confidence": h.prior_confidence, - } - for h in investigation.hypotheses - if h.status == HypothesisStatus.PROPOSED - ], - investigation_id=investigation.id, - ) - try: - await asyncio.wait_for(self._approval_event.wait(), timeout=300) - except TimeoutError: - logger.info("Approval timeout, auto-approving all hypotheses") - self._approval_event.clear() - - # 4. Hypothesis loop -- batched parallel execution - yield PhaseChanged( - phase=4, - name="Hypothesis Testing", - description="Running parallel experiments to test hypotheses", - investigation_id=investigation.id, - ) - tested = 0 - while tested < self._max_hypotheses: - proposed = [ - h for h in investigation.hypotheses if h.status == HypothesisStatus.PROPOSED - ] - if not proposed: - break - - # Take up to 2 hypotheses per batch - batch_hypotheses = proposed[:2] - batch: list[tuple[Hypothesis, Experiment, dict[str, Any]]] = [] - prior_designs: list[str] = [] - - for hypothesis in batch_hypotheses: - hypothesis.status = HypothesisStatus.TESTING - investigation.current_hypothesis_id = hypothesis.id - tested += 1 - - # Director designs experiment - if self._active_config: - tools_csv = ", ".join( - self._registry.list_tools_for_domain(self._active_config.tool_tags) - ) - else: - tools_csv = ", ".join(self._registry.list_tools()) - experiment_prompt = ( - build_experiment_prompt(self._active_config, self._uploaded_data_context) - if self._active_config - else DIRECTOR_EXPERIMENT_PROMPT - ) - sibling_section = "" - if prior_designs: - joined = "\n---\n".join(prior_designs) - sibling_section = ( - f"\n\n\n{joined}\n" - f"\n\n" - f"Your experiment runs in PARALLEL with the above. " - f"Design a DIFFERENT approach: use different tools, " - f"data sources, or validation strategies." - ) - - result = _DirectorResult(data={}, thinking="") - async for director_event in self._director_call( - cost, - experiment_prompt, - f"Research prompt: {investigation.prompt}" - f"\n\nHypothesis to test: " - f"{hypothesis.statement}\n" - f"Rationale: {hypothesis.rationale}\n\n" - f"Available tools: {tools_csv}\n\n" - f"Design an experiment to test this " - f"hypothesis." + sibling_section, - investigation.id, - output_config=_build_output_config(EXPERIMENT_DESIGN_SCHEMA), - ): - if isinstance(director_event, _DirectorResult): - result = director_event - else: - yield director_event - design = result.data - - desc = design.get( - "description", - f"Test: {hypothesis.statement}", - ) - prior_designs.append( - f"Hypothesis: {hypothesis.statement}\n" - f"Experiment: {desc}\n" - f"Tools: {', '.join(design.get('tool_plan', []))}" - ) - experiment = Experiment( - hypothesis_id=hypothesis.id, - description=desc, - tool_plan=design.get("tool_plan", []), - independent_variable=design.get("independent_variable", ""), - dependent_variable=design.get("dependent_variable", ""), - controls=design.get("controls", []), - confounders=design.get("confounders", []), - analysis_plan=design.get("analysis_plan", ""), - success_criteria=design.get("success_criteria", ""), - failure_criteria=design.get("failure_criteria", ""), - ) - experiment.status = ExperimentStatus.RUNNING - investigation.add_experiment(experiment) - investigation.current_experiment_id = experiment.id - - yield ExperimentStarted( - experiment_id=experiment.id, - hypothesis_id=hypothesis.id, - description=experiment.description, - independent_variable=experiment.independent_variable, - dependent_variable=experiment.dependent_variable, - controls=experiment.controls, - analysis_plan=experiment.analysis_plan, - success_criteria=experiment.success_criteria, - failure_criteria=experiment.failure_criteria, - investigation_id=investigation.id, - ) - - batch.append((hypothesis, experiment, design)) - - # Run batch (parallel if 2, sequential if 1) - exp_tool_count = cost.tool_calls - exp_finding_count = len(investigation.findings) - async for event in run_experiment_batch( - self._researcher, - self._summarizer, - self._dispatcher, - self._registry, - self._active_config, - self._researcher_prompt, - self._summarizer_threshold, - self._max_iterations_per_experiment, - investigation, - batch, - cost, - self._state_lock, - ): yield event - # Mark completed + Director evaluates each - for hypothesis, experiment, _design in batch: - experiment.status = ExperimentStatus.COMPLETED - yield ExperimentCompleted( - experiment_id=experiment.id, - hypothesis_id=hypothesis.id, - tool_count=(cost.tool_calls - exp_tool_count), - finding_count=(len(investigation.findings) - exp_finding_count), - investigation_id=investigation.id, - ) - - # Director evaluates hypothesis - findings_for_hyp = [ - f for f in investigation.findings if f.hypothesis_id == hypothesis.id - ] - findings_text = "\n".join( - f"- [{f.evidence_type}] {f.title}: {f.detail}" for f in findings_for_hyp - ) - controls_text = ", ".join(experiment.controls) or "None specified" - result = _DirectorResult(data={}, thinking="") - async for director_event in self._director_call( - cost, - DIRECTOR_EVALUATION_PROMPT, - f"Hypothesis: {hypothesis.statement}\n" - f"Mechanism: {hypothesis.rationale}\n" - f"Prediction: {hypothesis.prediction or 'N/A'}\n" - f"Hypothesis success criteria: {hypothesis.success_criteria or 'N/A'}\n" - f"Hypothesis failure criteria: {hypothesis.failure_criteria or 'N/A'}\n" - f"Prior confidence: {hypothesis.prior_confidence}\n\n" - f"Experiment: {experiment.description}\n" - f"Independent variable: {experiment.independent_variable or 'N/A'}\n" - f"Dependent variable: {experiment.dependent_variable or 'N/A'}\n" - f"Controls: {controls_text}\n" - f"Analysis plan: {experiment.analysis_plan or 'N/A'}\n" - f"Experiment success criteria: {experiment.success_criteria or 'N/A'}\n" - f"Experiment failure criteria: {experiment.failure_criteria or 'N/A'}\n" - f"\nFindings:\n{findings_text}\n\n" - f"Compare the findings against the pre-defined " - f"success/failure criteria. Evaluate this hypothesis.", - investigation.id, - output_config=_build_output_config(EVALUATION_SCHEMA), - ): - if isinstance(director_event, _DirectorResult): - result = director_event - else: - yield director_event - evaluation = result.data - - eval_status = evaluation.get("status", "supported") - eval_confidence = float(evaluation.get("confidence", 0.5)) - eval_certainty = evaluation.get("certainty_of_evidence", "") - status_map = { - "supported": HypothesisStatus.SUPPORTED, - "refuted": HypothesisStatus.REFUTED, - "revised": HypothesisStatus.REVISED, - } - hypothesis.status = status_map.get(eval_status, HypothesisStatus.SUPPORTED) - hypothesis.confidence = max(0.0, min(1.0, eval_confidence)) - hypothesis.certainty_of_evidence = eval_certainty - - logger.info( - "Hypothesis %s convergence: %s", - hypothesis.id, - evaluation.get("evidence_convergence", ""), - ) - - yield HypothesisEvaluated( - hypothesis_id=hypothesis.id, - status=eval_status, - confidence=eval_confidence, - reasoning=evaluation.get("reasoning", ""), - certainty_of_evidence=eval_certainty, - investigation_id=investigation.id, - ) - - # If revised, add new hypothesis to queue - if eval_status == "revised" and evaluation.get("revision"): - revised = Hypothesis( - statement=evaluation["revision"], - rationale=evaluation.get( - "reasoning", - "Revised from prior evidence", - ), - prediction=evaluation.get("prediction", hypothesis.prediction), - success_criteria=evaluation.get( - "success_criteria", hypothesis.success_criteria - ), - failure_criteria=evaluation.get( - "failure_criteria", hypothesis.failure_criteria - ), - scope=hypothesis.scope, - hypothesis_type=hypothesis.hypothesis_type, - parent_id=hypothesis.id, - ) - investigation.add_hypothesis(revised) - yield HypothesisFormulated( - hypothesis_id=revised.id, - statement=revised.statement, - rationale=revised.rationale, - prediction=revised.prediction, - success_criteria=revised.success_criteria, - failure_criteria=revised.failure_criteria, - scope=revised.scope, - hypothesis_type=revised.hypothesis_type, - parent_id=hypothesis.id, - investigation_id=investigation.id, - ) - - yield self._cost_event(cost, investigation.id) + # 4. Hypothesis testing loop + async for event in run_hypothesis_testing_phase( + investigation, cost, self._active_config, + self._uploaded_data_context, self._researcher_prompt, + self._researcher, self._summarizer, self._dispatcher, + self._registry, self._max_hypotheses, + self._max_iterations_per_experiment, + self._summarizer_threshold, self._state_lock, + self._director_call, self._cost_event, + ): + yield event - # 5. Controls validation (from director suggestions) - yield PhaseChanged( - phase=5, - name="Controls Validation", - description="Validating model with positive and negative controls", - investigation_id=investigation.id, - ) + # 5. Controls validation validation_metrics: dict[str, Any] = {} - - # Score controls through trained model if available - score_map: dict[str, float] = {} - has_model = bool(investigation.trained_model_ids) - is_molecular = self._active_config is not None and any( - t in self._active_config.tool_tags for t in ("chemistry", "prediction") - ) - if has_model and is_molecular: - model_id = investigation.trained_model_ids[-1] - all_identifiers = [ - nc.get("identifier", nc.get("smiles", "")) - for nc in neg_control_suggestions - if nc.get("identifier", nc.get("smiles", "")) - ] + [ - pc.get("identifier", "") - for pc in pos_control_suggestions - if pc.get("identifier", "") - ] - if all_identifiers: - try: - pred_result = await self._dispatcher.dispatch( - "predict_candidates", - {"smiles_list": all_identifiers, "model_id": model_id}, - investigation, - ) - pred_data = json.loads(pred_result) - if isinstance(pred_data, dict): - for entry in pred_data.get("predictions", []): - smiles = entry.get("smiles", "") - prob = float(entry.get("probability", 0.0)) - if smiles: - score_map[smiles] = prob - except (json.JSONDecodeError, TypeError, ValueError): - logger.warning("Failed to score controls via model %s", model_id) - - for nc_data in neg_control_suggestions: - identifier = nc_data.get("identifier", nc_data.get("smiles", "")) - name = nc_data.get("name", "") - source = nc_data.get("source", "") - if identifier: - nc_score = score_map.get(identifier, 0.0) - control = NegativeControl( - identifier=identifier, - identifier_type=nc_data.get("identifier_type", ""), - name=name, - score=nc_score, - source=source, - ) - investigation.add_negative_control(control) - yield NegativeControlRecorded( - identifier=identifier, - identifier_type=control.identifier_type, - name=name, - score=nc_score, - correctly_classified=control.correctly_classified, - investigation_id=investigation.id, - ) - - for pc_data in pos_control_suggestions: - pc_identifier = pc_data.get("identifier", "") - pc_name = pc_data.get("name", "") - known_activity = pc_data.get("known_activity", "") - pc_source = pc_data.get("source", "") - if pc_identifier: - pc_score = score_map.get(pc_identifier, 0.0) - pos_control = PositiveControl( - identifier=pc_identifier, - identifier_type=pc_data.get("identifier_type", ""), - name=pc_name, - known_activity=known_activity, - source=pc_source, - score=pc_score, - ) - investigation.add_positive_control(pos_control) - yield PositiveControlRecorded( - identifier=pc_identifier, - identifier_type=pos_control.identifier_type, - name=pc_name, - known_activity=known_activity, - score=pc_score, - correctly_classified=pos_control.correctly_classified, - investigation_id=investigation.id, - ) - - # Compute Z'-factor from control scores - pos_scores = [pc.score for pc in investigation.positive_controls] - neg_scores = [nc.score for nc in investigation.negative_controls] - z_metrics = compute_z_prime(pos_scores, neg_scores) - validation_metrics = { - "z_prime": z_metrics.z_prime, - "z_prime_quality": z_metrics.quality, - "positive_mean": z_metrics.positive_mean, - "positive_std": z_metrics.positive_std, - "negative_mean": z_metrics.negative_mean, - "negative_std": z_metrics.negative_std, - "positive_count": z_metrics.positive_count, - "negative_count": z_metrics.negative_count, - } - yield ValidationMetricsComputed( - z_prime=z_metrics.z_prime, - z_prime_quality=z_metrics.quality, - positive_control_count=z_metrics.positive_count, - negative_control_count=z_metrics.negative_count, - positive_mean=z_metrics.positive_mean, - negative_mean=z_metrics.negative_mean, - investigation_id=investigation.id, - ) - - # 6. Director synthesizes - yield PhaseChanged( - phase=6, - name="Synthesis", - description="Director synthesizing final report and ranking candidates", - investigation_id=investigation.id, - ) - all_findings_text = "\n".join( - f"- [H:{f.hypothesis_id}|{f.evidence_type}] {f.title}: {f.detail}" - for f in investigation.findings - ) - hypothesis_text = "\n".join( - f"- {h.id}: {h.statement} -> {h.status.value} (confidence: {h.confidence})" - for h in investigation.hypotheses - ) - nc_text = "\n".join( - f"- {nc.name}: score={nc.score}, correct={nc.correctly_classified}" - for nc in investigation.negative_controls - ) - pc_text = "\n".join( - f"- {pc.name}: known_activity={pc.known_activity}, score={pc.score}" - for pc in investigation.positive_controls - ) - - # Build validation metrics text for synthesis context - vm_text = "" - if validation_metrics.get("z_prime") is not None: - vm_text = ( - f"\nValidation metrics:\n" - f"- Z'-factor: {validation_metrics['z_prime']:.3f} " - f"({validation_metrics['z_prime_quality']})\n" - f"- Positive controls: mean={validation_metrics['positive_mean']:.3f}, " - f"std={validation_metrics['positive_std']:.3f}, " - f"n={validation_metrics['positive_count']}\n" - f"- Negative controls: mean={validation_metrics['negative_mean']:.3f}, " - f"std={validation_metrics['negative_std']:.3f}, " - f"n={validation_metrics['negative_count']}\n" - ) - else: - vm_text = ( - f"\nValidation metrics: insufficient controls " - f"(pos={validation_metrics.get('positive_count', 0)}, " - f"neg={validation_metrics.get('negative_count', 0)})\n" - ) - - synthesis_prompt = ( - build_synthesis_prompt(self._active_config) - if self._active_config - else DIRECTOR_SYNTHESIS_PROMPT - ) - result = _DirectorResult(data={}, thinking="") - async for director_event in self._director_call( - cost, - synthesis_prompt, - f"Original prompt: {investigation.prompt}\n\n" - f"Hypothesis outcomes:\n{hypothesis_text}\n\n" - f"All findings:\n{all_findings_text}\n\n" - f"Negative controls:\n{nc_text or 'None recorded'}\n\n" - f"Positive controls:\n{pc_text or 'None recorded'}\n\n" - f"{vm_text}\n" - f"Synthesize final report.", - investigation.id, - output_config=_build_output_config(SYNTHESIS_SCHEMA), + async for event in run_controls_phase( + investigation, cost, self._active_config, + neg_control_suggestions, pos_control_suggestions, + self._dispatcher, ): - if isinstance(director_event, _DirectorResult): - result = director_event + if isinstance(event, dict) and "__phase_result__" in event: + validation_metrics = event["__phase_result__"]["validation_metrics"] else: - yield director_event - synthesis = result.data - - # Apply synthesis - validation_quality = synthesis.get("model_validation_quality", "insufficient") - logger.info( - "Investigation %s model_validation_quality: %s", - investigation.id, - validation_quality, - ) - investigation.summary = synthesis.get("summary", "") - raw_candidates = synthesis.get("candidates") or [] - candidates = [ - Candidate( - identifier=c.get("identifier", c.get("smiles", "")), - identifier_type=c.get("identifier_type", ""), - name=c.get("name", ""), - notes=c.get("rationale", c.get("notes", "")), - rank=c.get("rank", i + 1), - priority=c.get("priority", 0), - scores={ - k: float(v) - for k, v in c.get("scores", {}).items() - if isinstance(v, (int, float)) - }, - attributes={k: str(v) for k, v in c.get("attributes", {}).items()}, - ) - for i, c in enumerate(raw_candidates) - ] - citations = synthesis.get("citations") or [] - investigation.set_candidates(candidates, citations) - - # Post-synthesis: generate Excalidraw diagram if MCP available - diagram_url = "" - if self._mcp_bridge and self._mcp_bridge.has_tool("excalidraw:create_view"): - diagram_url = await generate_diagram(self._mcp_bridge, investigation) - - investigation.status = InvestigationStatus.COMPLETED - investigation.cost_data = cost.to_dict() + yield event - candidate_dicts = [ - { - "identifier": c.identifier, - "identifier_type": c.identifier_type, - "name": c.name, - "rank": c.rank, - "priority": c.priority, - "notes": c.notes, - "scores": c.scores, - "attributes": c.attributes, - } - for c in candidates - ] - finding_dicts = [ - { - "title": f.title, - "detail": f.detail, - "hypothesis_id": f.hypothesis_id, - "evidence_type": f.evidence_type, - "evidence": f.evidence, - } - for f in investigation.findings - ] - hypothesis_dicts = [ - { - "id": h.id, - "statement": h.statement, - "rationale": h.rationale, - "status": h.status.value, - "parent_id": h.parent_id, - "confidence": h.confidence, - "certainty_of_evidence": h.certainty_of_evidence, - "supporting_evidence": h.supporting_evidence, - "contradicting_evidence": h.contradicting_evidence, - } - for h in investigation.hypotheses - ] - nc_dicts = [ - { - "identifier": nc.identifier, - "identifier_type": nc.identifier_type, - "name": nc.name, - "score": nc.score, - "threshold": nc.threshold, - "correctly_classified": nc.correctly_classified, - "source": nc.source, - } - for nc in investigation.negative_controls - ] - pc_dicts = [ - { - "identifier": pc.identifier, - "identifier_type": pc.identifier_type, - "name": pc.name, - "known_activity": pc.known_activity, - "score": pc.score, - "correctly_classified": pc.correctly_classified, - "source": pc.source, - } - for pc in investigation.positive_controls - ] - yield InvestigationCompleted( - investigation_id=investigation.id, - candidate_count=len(candidates), - summary=investigation.summary, - cost=cost.to_dict(), - candidates=candidate_dicts, - findings=finding_dicts, - hypotheses=hypothesis_dicts, - negative_controls=nc_dicts, - positive_controls=pc_dicts, - validation_metrics=validation_metrics, - diagram_url=diagram_url, - ) + # 6. Director synthesis + async for event in run_synthesis_phase( + investigation, cost, self._active_config, + validation_metrics, self._mcp_bridge, + self._director_call, + ): + yield event except Exception as e: logger.exception("Investigation %s failed", investigation.id) @@ -955,266 +277,3 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, await self._mcp_bridge.disconnect() except Exception: logger.warning("MCP bridge disconnect failed", exc_info=True) - - @staticmethod - def _build_literature_context( - investigation: Investigation, - pico: dict[str, Any], - ) -> str: - """Build structured XML context from PICO + findings for Director formulation.""" - parts: list[str] = [""] - pop = pico.get("population", "") - interv = pico.get("intervention", "") - comp = pico.get("comparison", "") - outcome = pico.get("outcome", "") - parts.append( - f' ' - ) - if investigation.findings: - parts.append(" ") - for f in investigation.findings: - parts.append( - f' {f.title}: {f.detail}' - ) - parts.append(" ") - parts.append("") - return "\n".join(parts) - - async def _director_call( - self, - cost: CostTracker, - system: str, - user_message: str, - investigation_id: str, - output_config: dict[str, Any] | None = None, - ) -> AsyncGenerator[Thinking | _DirectorResult, None]: - text = "" - thinking_text = "" - response = None - async for event in self._director.stream_message( - system=system, - messages=[{"role": "user", "content": user_message}], - tools=[], - output_config=output_config, - ): - if event["type"] == "thinking": - thinking_text += event["text"] - yield Thinking(text=event["text"], investigation_id=investigation_id) - elif event["type"] == "text": - text += event["text"] - elif event["type"] == "result": - response = event["response"] - - if response: - cost.add_usage( - response.input_tokens, - response.output_tokens, - self._director.model, - cache_read_tokens=response.cache_read_input_tokens, - cache_write_tokens=response.cache_write_input_tokens, - ) - - yield _DirectorResult(data=json.loads(text), thinking=thinking_text) - - async def _run_literature_survey( - self, - investigation: Investigation, - cost: CostTracker, - pico: dict[str, Any], - ) -> AsyncGenerator[DomainEvent, None]: - """Structured literature survey with PICO, citation chasing, and evidence grading.""" - # A. Domain-filtered tools (no hardcoded set) - if self._active_config: - excluded = { - "conclude_investigation", - "propose_hypothesis", - "design_experiment", - "evaluate_hypothesis", - } - tool_schemas = [ - t - for t in self._registry.list_schemas_for_domain(self._active_config.tool_tags) - if t["name"] not in excluded - ] - else: - lit_tools = { - "search_literature", - "search_citations", - "get_reference", - "explore_dataset", - "record_finding", - } - tool_schemas = [t for t in self._registry.list_schemas() if t["name"] in lit_tools] - - # Build structured survey prompt - survey_prompt = build_literature_survey_prompt(self._active_config, pico) - - messages: list[dict[str, Any]] = [ - { - "role": "user", - "content": ( - f"Research prompt: {investigation.prompt}\n\n" - f"Conduct a structured literature survey following the protocol above." - ), - }, - ] - - search_queries = 0 - total_results = 0 - - for _iteration in range(self._max_iterations_per_experiment): - investigation.iteration += 1 - tool_choice = {"type": "any"} if _iteration == 0 else None - response = await self._researcher.create_message( - system=survey_prompt, - messages=messages, - tools=tool_schemas, - tool_choice=tool_choice, - ) - cost.add_usage( - response.input_tokens, - response.output_tokens, - self._researcher.model, - cache_read_tokens=response.cache_read_input_tokens, - cache_write_tokens=response.cache_write_input_tokens, - ) - - assistant_content: list[dict[str, Any]] = [] - tool_use_blocks: list[dict[str, Any]] = [] - - for block in response.content: - assistant_content.append(block) - if block["type"] == "text": - yield Thinking(text=block["text"], investigation_id=investigation.id) - elif block["type"] == "tool_use": - tool_use_blocks.append(block) - - messages.append({"role": "assistant", "content": assistant_content}) - - if response.stop_reason == "end_turn" or not tool_use_blocks: - break - - tool_results: list[dict[str, Any]] = [] - for tool_block in tool_use_blocks: - tool_name = tool_block["name"] - tool_input = tool_block["input"] - tool_use_id = tool_block["id"] - cost.add_tool_call() - - # Track search stats - if tool_name in SEARCH_TOOLS: - search_queries += 1 - - yield ToolCalled( - tool_name=tool_name, - tool_input=tool_input, - investigation_id=investigation.id, - ) - - result_str = await self._dispatcher.dispatch(tool_name, tool_input, investigation) - result_str = _compact_result(tool_name, result_str) - - # Track result counts - try: - result_data = json.loads(result_str) - if isinstance(result_data, dict): - total_results += int(result_data.get("count", 0)) - except (json.JSONDecodeError, TypeError, ValueError): - pass - - summarized_str, summarize_event = await summarize_output( - self._summarizer, cost, tool_name, result_str, investigation.id, - self._summarizer_threshold, - ) - if summarize_event is not None: - yield summarize_event - - content_for_model = summarized_str if summarize_event else result_str - preview = result_str[:1500] if len(result_str) > 1500 else result_str - yield ToolResultEvent( - tool_name=tool_name, - result_preview=preview, - investigation_id=investigation.id, - ) - - # Emit visualization event if tool returned viz payload - viz_event = maybe_viz_event(result_str, "", investigation.id) - if viz_event is not None: - yield viz_event - - if tool_name == "record_finding": - finding = Finding( - title=tool_input.get("title", ""), - detail=tool_input.get("detail", ""), - evidence=tool_input.get("evidence", ""), - hypothesis_id=tool_input.get("hypothesis_id", ""), - evidence_type=tool_input.get("evidence_type", "neutral"), - source_type=tool_input.get("source_type", ""), - source_id=tool_input.get("source_id", ""), - evidence_level=int(tool_input.get("evidence_level", 0)), - ) - investigation.record_finding(finding) - yield FindingRecorded( - title=finding.title, - detail=finding.detail, - hypothesis_id=finding.hypothesis_id, - evidence_type=finding.evidence_type, - evidence=finding.evidence, - source_type=finding.source_type, - source_id=finding.source_id, - evidence_level=finding.evidence_level, - investigation_id=investigation.id, - ) - - tool_results.append( - { - "type": "tool_result", - "tool_use_id": tool_use_id, - "content": content_for_model, - } - ) - - messages.append({"role": "user", "content": tool_results}) - - # B. Body-of-evidence grading (Haiku, 1 call) - evidence_grade = "" - assessment = "" - if investigation.findings: - findings_for_grading = "\n".join( - f"- [level={f.evidence_level}] {f.title}: {f.detail}" - for f in investigation.findings - ) - grade_response = await self._summarizer.create_message( - system=build_literature_assessment_prompt(), - messages=[{"role": "user", "content": findings_for_grading}], - tools=[], - output_config=_build_output_config(LITERATURE_GRADING_SCHEMA), - ) - cost.add_usage( - grade_response.input_tokens, - grade_response.output_tokens, - self._summarizer.model, - cache_read_tokens=grade_response.cache_read_input_tokens, - cache_write_tokens=grade_response.cache_write_input_tokens, - ) - grade_text = "" - for block in grade_response.content: - if block.get("type") == "text": - grade_text += block["text"] - grade_data = json.loads(grade_text) - evidence_grade = grade_data.get("evidence_grade", "") - assessment = grade_data.get("assessment", "") - - # C. Yield LiteratureSurveyCompleted event - yield LiteratureSurveyCompleted( - pico=pico, - search_queries=search_queries, - total_results=total_results, - included_results=len(investigation.findings), - evidence_grade=evidence_grade, - assessment=assessment, - investigation_id=investigation.id, - ) - diff --git a/server/src/ehrlich/investigation/application/phase_runner.py b/server/src/ehrlich/investigation/application/phase_runner.py new file mode 100644 index 0000000..3b15112 --- /dev/null +++ b/server/src/ehrlich/investigation/application/phase_runner.py @@ -0,0 +1,901 @@ +"""Investigation phase runners extracted from MultiModelOrchestrator.run(). + +Each function runs one phase of the investigation pipeline, yielding +DomainEvent objects. The orchestrator's run() method delegates to these +in sequence. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.application.batch_executor import run_experiment_batch +from ehrlich.investigation.application.diagram_builder import generate_diagram +from ehrlich.investigation.application.prompts.builders import ( + build_pico_and_classification_prompt, + build_researcher_prompt, +) +from ehrlich.investigation.application.prompts.constants import ( + DIRECTOR_EVALUATION_PROMPT, + DIRECTOR_EXPERIMENT_PROMPT, + DIRECTOR_FORMULATION_PROMPT, + DIRECTOR_SYNTHESIS_PROMPT, +) +from ehrlich.investigation.application.prompts.director import ( + build_experiment_prompt, + build_formulation_prompt, + build_multi_investigation_context, + build_synthesis_prompt, +) +from ehrlich.investigation.domain.candidate import Candidate +from ehrlich.investigation.domain.domain_config import merge_domain_configs +from ehrlich.investigation.domain.events import ( + CostUpdate, + DomainDetected, + DomainEvent, + ExperimentCompleted, + ExperimentStarted, + HypothesisApprovalRequested, + HypothesisEvaluated, + HypothesisFormulated, + InvestigationCompleted, + NegativeControlRecorded, + PhaseChanged, + PositiveControlRecorded, + ValidationMetricsComputed, +) +from ehrlich.investigation.domain.experiment import Experiment, ExperimentStatus +from ehrlich.investigation.domain.hypothesis import Hypothesis, HypothesisStatus +from ehrlich.investigation.domain.investigation import Investigation, InvestigationStatus +from ehrlich.investigation.domain.negative_control import NegativeControl +from ehrlich.investigation.domain.positive_control import PositiveControl +from ehrlich.investigation.domain.schemas import ( + EVALUATION_SCHEMA, + EXPERIMENT_DESIGN_SCHEMA, + FORMULATION_SCHEMA, + PICO_SCHEMA, + SYNTHESIS_SCHEMA, +) +from ehrlich.investigation.domain.validation import compute_z_prime + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable + + from ehrlich.investigation.application.cost_tracker import CostTracker + from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher + from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.domain.domain_config import DomainConfig + from ehrlich.investigation.domain.domain_registry import DomainRegistry + from ehrlich.investigation.domain.repository import InvestigationRepository + from ehrlich.investigation.infrastructure.anthropic_client import AnthropicClientAdapter + from ehrlich.investigation.infrastructure.mcp_bridge import MCPBridge + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _DirectorResult: + data: dict[str, Any] + thinking: str + + +def _build_output_config(schema: dict[str, Any]) -> dict[str, Any]: + return {"format": {"type": "json_schema", "schema": schema}} + + +# --------------------------------------------------------------------------- +# Phase 1: Classification & PICO +# --------------------------------------------------------------------------- + + +async def run_classification_phase( + investigation: Investigation, + cost: CostTracker, + summarizer: AnthropicClientAdapter, + uploaded_data_context: str, + repository: InvestigationRepository | None, + domain_registry: DomainRegistry | None, + director_call: Callable[..., AsyncGenerator[Any, None]], +) -> AsyncGenerator[DomainEvent | dict[str, Any], None]: + """Phase 1: Classify domain + PICO decomposition. + + Yields DomainEvent instances and finally a special dict sentinel + ``{"__phase_result__": ...}`` containing the phase outputs so the + orchestrator can capture them without shared mutable state. + """ + yield PhaseChanged( + phase=1, + name="Classification & PICO", + description="Detecting domain and decomposing research question", + investigation_id=investigation.id, + ) + + pico: dict[str, Any] = {} + prior_context = "" + active_config: DomainConfig | None = None + researcher_prompt = "" + + if repository: + valid_categories: frozenset[str] = frozenset() + if domain_registry: + valid_categories = domain_registry.all_categories() + pico_prompt = build_pico_and_classification_prompt( + valid_categories, uploaded_data_context + ) + pico_response = await summarizer.create_message( + system=pico_prompt, + messages=[{"role": "user", "content": investigation.prompt}], + tools=[], + output_config=_build_output_config(PICO_SCHEMA), + ) + cost.add_usage( + pico_response.input_tokens, + pico_response.output_tokens, + summarizer.model, + cache_read_tokens=pico_response.cache_read_input_tokens, + cache_write_tokens=pico_response.cache_write_input_tokens, + ) + pico_text = "" + for block in pico_response.content: + if block.get("type") == "text": + pico_text += block["text"] + pico_data = json.loads(pico_text) + raw_domain = pico_data.get("domain", "other") + domain_categories: list[str] = ( + [d.strip().lower() for d in raw_domain] + if isinstance(raw_domain, list) + else [raw_domain.strip().lower()] + ) + investigation.domain = ", ".join(domain_categories) + pico = { + "population": pico_data.get("population", ""), + "intervention": pico_data.get("intervention", ""), + "comparison": pico_data.get("comparison", ""), + "outcome": pico_data.get("outcome", ""), + "search_terms": pico_data.get("search_terms", []), + } + + # Detect domain configs (multi-domain) and yield event + if domain_registry: + detected_configs, is_fallback = domain_registry.detect(domain_categories) + active_config = merge_domain_configs(detected_configs) + researcher_prompt = build_researcher_prompt( + active_config, uploaded_data_context + ) + yield DomainDetected( + domain=active_config.name, + display_config=active_config.to_display_dict(), + is_fallback=is_fallback, + investigation_id=investigation.id, + ) + + all_investigations = await repository.list_all() + related = [ + inv + for inv in all_investigations + if inv.status == InvestigationStatus.COMPLETED + and inv.id != investigation.id + and any(cat in inv.domain for cat in domain_categories) + ] + if related: + prior_context = build_multi_investigation_context(related) + + yield { + "__phase_result__": { + "pico": pico, + "prior_context": prior_context, + "active_config": active_config, + "researcher_prompt": researcher_prompt, + } + } + + +# --------------------------------------------------------------------------- +# Phase 3: Hypothesis Formulation +# --------------------------------------------------------------------------- + + +async def run_formulation_phase( + investigation: Investigation, + cost: CostTracker, + active_config: DomainConfig | None, + uploaded_data_context: str, + prior_context: str, + pico: dict[str, Any], + require_approval: bool, + approval_event: asyncio.Event, + director_call: Callable[..., AsyncGenerator[Any, None]], + cost_event_fn: Callable[..., CostUpdate], +) -> AsyncGenerator[DomainEvent | dict[str, Any], None]: + """Phase 3: Director formulates hypotheses, optionally waits for approval.""" + from ehrlich.investigation.application.literature_survey import build_literature_context + + yield PhaseChanged( + phase=3, + name="Formulation", + description="Director formulating testable hypotheses from literature evidence", + investigation_id=investigation.id, + ) + prior_section = f"\n\n{prior_context}" if prior_context else "" + formulation_prompt = ( + build_formulation_prompt(active_config, uploaded_data_context) + if active_config + else DIRECTOR_FORMULATION_PROMPT + ) + + # Build structured XML context from PICO + findings + literature_context = build_literature_context(investigation, pico) + result = _DirectorResult(data={}, thinking="") + async for director_event in director_call( + cost, + formulation_prompt, + f"Research prompt: {investigation.prompt}\n\n" + f"{literature_context}\n\n" + f"Formulate 2-4 testable hypotheses and identify negative controls." + + prior_section, + investigation.id, + output_config=_build_output_config(FORMULATION_SCHEMA), + ): + if isinstance(director_event, _DirectorResult): + result = director_event + else: + yield director_event + formulation = result.data + + # Create hypothesis entities + for h_data in formulation.get("hypotheses", []): + hypothesis = Hypothesis( + statement=h_data.get("statement", ""), + rationale=h_data.get("rationale", ""), + prediction=h_data.get("prediction", ""), + null_prediction=h_data.get("null_prediction", ""), + success_criteria=h_data.get("success_criteria", ""), + failure_criteria=h_data.get("failure_criteria", ""), + scope=h_data.get("scope", ""), + hypothesis_type=h_data.get("hypothesis_type", ""), + prior_confidence=min(1.0, max(0.0, h_data.get("prior_confidence", 0.0))), + ) + investigation.add_hypothesis(hypothesis) + yield HypothesisFormulated( + hypothesis_id=hypothesis.id, + statement=hypothesis.statement, + rationale=hypothesis.rationale, + prediction=hypothesis.prediction, + null_prediction=hypothesis.null_prediction, + success_criteria=hypothesis.success_criteria, + failure_criteria=hypothesis.failure_criteria, + scope=hypothesis.scope, + hypothesis_type=hypothesis.hypothesis_type, + prior_confidence=hypothesis.prior_confidence, + parent_id="", + investigation_id=investigation.id, + ) + + # Store control suggestions for later + neg_control_suggestions = formulation.get("negative_controls", []) + pos_control_suggestions = formulation.get("positive_controls", []) + yield cost_event_fn(cost, investigation.id) + + # Request user approval before testing + if require_approval: + yield HypothesisApprovalRequested( + hypotheses=[ + { + "id": h.id, + "statement": h.statement, + "rationale": h.rationale, + "prediction": h.prediction, + "scope": h.scope, + "hypothesis_type": h.hypothesis_type, + "prior_confidence": h.prior_confidence, + } + for h in investigation.hypotheses + if h.status == HypothesisStatus.PROPOSED + ], + investigation_id=investigation.id, + ) + try: + await asyncio.wait_for(approval_event.wait(), timeout=300) + except TimeoutError: + logger.info("Approval timeout, auto-approving all hypotheses") + approval_event.clear() + + yield { + "__phase_result__": { + "neg_control_suggestions": neg_control_suggestions, + "pos_control_suggestions": pos_control_suggestions, + } + } + + +# --------------------------------------------------------------------------- +# Phase 4: Hypothesis Testing +# --------------------------------------------------------------------------- + + +async def run_hypothesis_testing_phase( + investigation: Investigation, + cost: CostTracker, + active_config: DomainConfig | None, + uploaded_data_context: str, + researcher_prompt: str, + researcher: AnthropicClientAdapter, + summarizer: AnthropicClientAdapter, + dispatcher: ToolDispatcher, + registry: ToolRegistry, + max_hypotheses: int, + max_iterations_per_experiment: int, + summarizer_threshold: int, + state_lock: asyncio.Lock, + director_call: Callable[..., AsyncGenerator[Any, None]], + cost_event_fn: Callable[..., CostUpdate], +) -> AsyncGenerator[DomainEvent, None]: + """Phase 4: Batched parallel hypothesis testing + Director evaluation loop.""" + + yield PhaseChanged( + phase=4, + name="Hypothesis Testing", + description="Running parallel experiments to test hypotheses", + investigation_id=investigation.id, + ) + tested = 0 + while tested < max_hypotheses: + proposed = [ + h for h in investigation.hypotheses if h.status == HypothesisStatus.PROPOSED + ] + if not proposed: + break + + # Take up to 2 hypotheses per batch + batch_hypotheses = proposed[:2] + batch: list[tuple[Hypothesis, Experiment, dict[str, Any]]] = [] + prior_designs: list[str] = [] + + for hypothesis in batch_hypotheses: + hypothesis.status = HypothesisStatus.TESTING + investigation.current_hypothesis_id = hypothesis.id + tested += 1 + + # Director designs experiment + if active_config: + tools_csv = ", ".join( + registry.list_tools_for_domain(active_config.tool_tags) + ) + else: + tools_csv = ", ".join(registry.list_tools()) + experiment_prompt = ( + build_experiment_prompt(active_config, uploaded_data_context) + if active_config + else DIRECTOR_EXPERIMENT_PROMPT + ) + sibling_section = "" + if prior_designs: + joined = "\n---\n".join(prior_designs) + sibling_section = ( + f"\n\n\n{joined}\n" + f"\n\n" + f"Your experiment runs in PARALLEL with the above. " + f"Design a DIFFERENT approach: use different tools, " + f"data sources, or validation strategies." + ) + + result = _DirectorResult(data={}, thinking="") + async for director_event in director_call( + cost, + experiment_prompt, + f"Research prompt: {investigation.prompt}" + f"\n\nHypothesis to test: " + f"{hypothesis.statement}\n" + f"Rationale: {hypothesis.rationale}\n\n" + f"Available tools: {tools_csv}\n\n" + f"Design an experiment to test this " + f"hypothesis." + sibling_section, + investigation.id, + output_config=_build_output_config(EXPERIMENT_DESIGN_SCHEMA), + ): + if isinstance(director_event, _DirectorResult): + result = director_event + else: + yield director_event + design = result.data + + desc = design.get( + "description", + f"Test: {hypothesis.statement}", + ) + prior_designs.append( + f"Hypothesis: {hypothesis.statement}\n" + f"Experiment: {desc}\n" + f"Tools: {', '.join(design.get('tool_plan', []))}" + ) + experiment = Experiment( + hypothesis_id=hypothesis.id, + description=desc, + tool_plan=design.get("tool_plan", []), + independent_variable=design.get("independent_variable", ""), + dependent_variable=design.get("dependent_variable", ""), + controls=design.get("controls", []), + confounders=design.get("confounders", []), + analysis_plan=design.get("analysis_plan", ""), + success_criteria=design.get("success_criteria", ""), + failure_criteria=design.get("failure_criteria", ""), + ) + experiment.status = ExperimentStatus.RUNNING + investigation.add_experiment(experiment) + investigation.current_experiment_id = experiment.id + + yield ExperimentStarted( + experiment_id=experiment.id, + hypothesis_id=hypothesis.id, + description=experiment.description, + independent_variable=experiment.independent_variable, + dependent_variable=experiment.dependent_variable, + controls=experiment.controls, + analysis_plan=experiment.analysis_plan, + success_criteria=experiment.success_criteria, + failure_criteria=experiment.failure_criteria, + investigation_id=investigation.id, + ) + + batch.append((hypothesis, experiment, design)) + + # Run batch (parallel if 2, sequential if 1) + exp_tool_count = cost.tool_calls + exp_finding_count = len(investigation.findings) + async for event in run_experiment_batch( + researcher, + summarizer, + dispatcher, + registry, + active_config, + researcher_prompt, + summarizer_threshold, + max_iterations_per_experiment, + investigation, + batch, + cost, + state_lock, + ): + yield event + + # Mark completed + Director evaluates each + for hypothesis, experiment, _design in batch: + experiment.status = ExperimentStatus.COMPLETED + yield ExperimentCompleted( + experiment_id=experiment.id, + hypothesis_id=hypothesis.id, + tool_count=(cost.tool_calls - exp_tool_count), + finding_count=(len(investigation.findings) - exp_finding_count), + investigation_id=investigation.id, + ) + + # Director evaluates hypothesis + findings_for_hyp = [ + f for f in investigation.findings if f.hypothesis_id == hypothesis.id + ] + findings_text = "\n".join( + f"- [{f.evidence_type}] {f.title}: {f.detail}" for f in findings_for_hyp + ) + controls_text = ", ".join(experiment.controls) or "None specified" + result = _DirectorResult(data={}, thinking="") + async for director_event in director_call( + cost, + DIRECTOR_EVALUATION_PROMPT, + f"Hypothesis: {hypothesis.statement}\n" + f"Mechanism: {hypothesis.rationale}\n" + f"Prediction: {hypothesis.prediction or 'N/A'}\n" + f"Hypothesis success criteria: {hypothesis.success_criteria or 'N/A'}\n" + f"Hypothesis failure criteria: {hypothesis.failure_criteria or 'N/A'}\n" + f"Prior confidence: {hypothesis.prior_confidence}\n\n" + f"Experiment: {experiment.description}\n" + f"Independent variable: {experiment.independent_variable or 'N/A'}\n" + f"Dependent variable: {experiment.dependent_variable or 'N/A'}\n" + f"Controls: {controls_text}\n" + f"Analysis plan: {experiment.analysis_plan or 'N/A'}\n" + f"Experiment success criteria: {experiment.success_criteria or 'N/A'}\n" + f"Experiment failure criteria: {experiment.failure_criteria or 'N/A'}\n" + f"\nFindings:\n{findings_text}\n\n" + f"Compare the findings against the pre-defined " + f"success/failure criteria. Evaluate this hypothesis.", + investigation.id, + output_config=_build_output_config(EVALUATION_SCHEMA), + ): + if isinstance(director_event, _DirectorResult): + result = director_event + else: + yield director_event + evaluation = result.data + + eval_status = evaluation.get("status", "supported") + eval_confidence = float(evaluation.get("confidence", 0.5)) + eval_certainty = evaluation.get("certainty_of_evidence", "") + status_map = { + "supported": HypothesisStatus.SUPPORTED, + "refuted": HypothesisStatus.REFUTED, + "revised": HypothesisStatus.REVISED, + } + hypothesis.status = status_map.get(eval_status, HypothesisStatus.SUPPORTED) + hypothesis.confidence = max(0.0, min(1.0, eval_confidence)) + hypothesis.certainty_of_evidence = eval_certainty + + logger.info( + "Hypothesis %s convergence: %s", + hypothesis.id, + evaluation.get("evidence_convergence", ""), + ) + + yield HypothesisEvaluated( + hypothesis_id=hypothesis.id, + status=eval_status, + confidence=eval_confidence, + reasoning=evaluation.get("reasoning", ""), + certainty_of_evidence=eval_certainty, + investigation_id=investigation.id, + ) + + # If revised, add new hypothesis to queue + if eval_status == "revised" and evaluation.get("revision"): + revised = Hypothesis( + statement=evaluation["revision"], + rationale=evaluation.get( + "reasoning", + "Revised from prior evidence", + ), + prediction=evaluation.get("prediction", hypothesis.prediction), + success_criteria=evaluation.get( + "success_criteria", hypothesis.success_criteria + ), + failure_criteria=evaluation.get( + "failure_criteria", hypothesis.failure_criteria + ), + scope=hypothesis.scope, + hypothesis_type=hypothesis.hypothesis_type, + parent_id=hypothesis.id, + ) + investigation.add_hypothesis(revised) + yield HypothesisFormulated( + hypothesis_id=revised.id, + statement=revised.statement, + rationale=revised.rationale, + prediction=revised.prediction, + success_criteria=revised.success_criteria, + failure_criteria=revised.failure_criteria, + scope=revised.scope, + hypothesis_type=revised.hypothesis_type, + parent_id=hypothesis.id, + investigation_id=investigation.id, + ) + + yield cost_event_fn(cost, investigation.id) + + +# --------------------------------------------------------------------------- +# Phase 5: Controls Validation +# --------------------------------------------------------------------------- + + +async def run_controls_phase( + investigation: Investigation, + cost: CostTracker, + active_config: DomainConfig | None, + neg_control_suggestions: list[dict[str, Any]], + pos_control_suggestions: list[dict[str, Any]], + dispatcher: ToolDispatcher, +) -> AsyncGenerator[DomainEvent | dict[str, Any], None]: + """Phase 5: Validate controls and compute Z'-factor.""" + yield PhaseChanged( + phase=5, + name="Controls Validation", + description="Validating model with positive and negative controls", + investigation_id=investigation.id, + ) + + # Score controls through trained model if available + score_map: dict[str, float] = {} + has_model = bool(investigation.trained_model_ids) + is_molecular = active_config is not None and any( + t in active_config.tool_tags for t in ("chemistry", "prediction") + ) + if has_model and is_molecular: + model_id = investigation.trained_model_ids[-1] + all_identifiers = [ + nc.get("identifier", nc.get("smiles", "")) + for nc in neg_control_suggestions + if nc.get("identifier", nc.get("smiles", "")) + ] + [ + pc.get("identifier", "") + for pc in pos_control_suggestions + if pc.get("identifier", "") + ] + if all_identifiers: + try: + pred_result = await dispatcher.dispatch( + "predict_candidates", + {"smiles_list": all_identifiers, "model_id": model_id}, + investigation, + ) + pred_data = json.loads(pred_result) + if isinstance(pred_data, dict): + for entry in pred_data.get("predictions", []): + smiles = entry.get("smiles", "") + prob = float(entry.get("probability", 0.0)) + if smiles: + score_map[smiles] = prob + except (json.JSONDecodeError, TypeError, ValueError): + logger.warning("Failed to score controls via model %s", model_id) + + for nc_data in neg_control_suggestions: + identifier = nc_data.get("identifier", nc_data.get("smiles", "")) + name = nc_data.get("name", "") + source = nc_data.get("source", "") + if identifier: + nc_score = score_map.get(identifier, 0.0) + control = NegativeControl( + identifier=identifier, + identifier_type=nc_data.get("identifier_type", ""), + name=name, + score=nc_score, + source=source, + ) + investigation.add_negative_control(control) + yield NegativeControlRecorded( + identifier=identifier, + identifier_type=control.identifier_type, + name=name, + score=nc_score, + correctly_classified=control.correctly_classified, + investigation_id=investigation.id, + ) + + for pc_data in pos_control_suggestions: + pc_identifier = pc_data.get("identifier", "") + pc_name = pc_data.get("name", "") + known_activity = pc_data.get("known_activity", "") + pc_source = pc_data.get("source", "") + if pc_identifier: + pc_score = score_map.get(pc_identifier, 0.0) + pos_control = PositiveControl( + identifier=pc_identifier, + identifier_type=pc_data.get("identifier_type", ""), + name=pc_name, + known_activity=known_activity, + source=pc_source, + score=pc_score, + ) + investigation.add_positive_control(pos_control) + yield PositiveControlRecorded( + identifier=pc_identifier, + identifier_type=pos_control.identifier_type, + name=pc_name, + known_activity=known_activity, + score=pc_score, + correctly_classified=pos_control.correctly_classified, + investigation_id=investigation.id, + ) + + # Compute Z'-factor from control scores + pos_scores = [pc.score for pc in investigation.positive_controls] + neg_scores = [nc.score for nc in investigation.negative_controls] + z_metrics = compute_z_prime(pos_scores, neg_scores) + validation_metrics = { + "z_prime": z_metrics.z_prime, + "z_prime_quality": z_metrics.quality, + "positive_mean": z_metrics.positive_mean, + "positive_std": z_metrics.positive_std, + "negative_mean": z_metrics.negative_mean, + "negative_std": z_metrics.negative_std, + "positive_count": z_metrics.positive_count, + "negative_count": z_metrics.negative_count, + } + yield ValidationMetricsComputed( + z_prime=z_metrics.z_prime, + z_prime_quality=z_metrics.quality, + positive_control_count=z_metrics.positive_count, + negative_control_count=z_metrics.negative_count, + positive_mean=z_metrics.positive_mean, + negative_mean=z_metrics.negative_mean, + investigation_id=investigation.id, + ) + + yield {"__phase_result__": {"validation_metrics": validation_metrics}} + + +# --------------------------------------------------------------------------- +# Phase 6: Synthesis +# --------------------------------------------------------------------------- + + +async def run_synthesis_phase( + investigation: Investigation, + cost: CostTracker, + active_config: DomainConfig | None, + validation_metrics: dict[str, Any], + mcp_bridge: MCPBridge | None, + director_call: Callable[..., AsyncGenerator[Any, None]], +) -> AsyncGenerator[DomainEvent, None]: + """Phase 6: Director synthesis + candidate ranking + diagram generation.""" + + yield PhaseChanged( + phase=6, + name="Synthesis", + description="Director synthesizing final report and ranking candidates", + investigation_id=investigation.id, + ) + all_findings_text = "\n".join( + f"- [H:{f.hypothesis_id}|{f.evidence_type}] {f.title}: {f.detail}" + for f in investigation.findings + ) + hypothesis_text = "\n".join( + f"- {h.id}: {h.statement} -> {h.status.value} (confidence: {h.confidence})" + for h in investigation.hypotheses + ) + nc_text = "\n".join( + f"- {nc.name}: score={nc.score}, correct={nc.correctly_classified}" + for nc in investigation.negative_controls + ) + pc_text = "\n".join( + f"- {pc.name}: known_activity={pc.known_activity}, score={pc.score}" + for pc in investigation.positive_controls + ) + + # Build validation metrics text for synthesis context + vm_text = "" + if validation_metrics.get("z_prime") is not None: + vm_text = ( + f"\nValidation metrics:\n" + f"- Z'-factor: {validation_metrics['z_prime']:.3f} " + f"({validation_metrics['z_prime_quality']})\n" + f"- Positive controls: mean={validation_metrics['positive_mean']:.3f}, " + f"std={validation_metrics['positive_std']:.3f}, " + f"n={validation_metrics['positive_count']}\n" + f"- Negative controls: mean={validation_metrics['negative_mean']:.3f}, " + f"std={validation_metrics['negative_std']:.3f}, " + f"n={validation_metrics['negative_count']}\n" + ) + else: + vm_text = ( + f"\nValidation metrics: insufficient controls " + f"(pos={validation_metrics.get('positive_count', 0)}, " + f"neg={validation_metrics.get('negative_count', 0)})\n" + ) + + synthesis_prompt = ( + build_synthesis_prompt(active_config) + if active_config + else DIRECTOR_SYNTHESIS_PROMPT + ) + result = _DirectorResult(data={}, thinking="") + async for director_event in director_call( + cost, + synthesis_prompt, + f"Original prompt: {investigation.prompt}\n\n" + f"Hypothesis outcomes:\n{hypothesis_text}\n\n" + f"All findings:\n{all_findings_text}\n\n" + f"Negative controls:\n{nc_text or 'None recorded'}\n\n" + f"Positive controls:\n{pc_text or 'None recorded'}\n\n" + f"{vm_text}\n" + f"Synthesize final report.", + investigation.id, + output_config=_build_output_config(SYNTHESIS_SCHEMA), + ): + if isinstance(director_event, _DirectorResult): + result = director_event + else: + yield director_event + synthesis = result.data + + # Apply synthesis + validation_quality = synthesis.get("model_validation_quality", "insufficient") + logger.info( + "Investigation %s model_validation_quality: %s", + investigation.id, + validation_quality, + ) + investigation.summary = synthesis.get("summary", "") + raw_candidates = synthesis.get("candidates") or [] + candidates = [ + Candidate( + identifier=c.get("identifier", c.get("smiles", "")), + identifier_type=c.get("identifier_type", ""), + name=c.get("name", ""), + notes=c.get("rationale", c.get("notes", "")), + rank=c.get("rank", i + 1), + priority=c.get("priority", 0), + scores={ + k: float(v) + for k, v in c.get("scores", {}).items() + if isinstance(v, (int, float)) + }, + attributes={k: str(v) for k, v in c.get("attributes", {}).items()}, + ) + for i, c in enumerate(raw_candidates) + ] + citations = synthesis.get("citations") or [] + investigation.set_candidates(candidates, citations) + + # Post-synthesis: generate Excalidraw diagram if MCP available + diagram_url = "" + if mcp_bridge and mcp_bridge.has_tool("excalidraw:create_view"): + diagram_url = await generate_diagram(mcp_bridge, investigation) + + investigation.status = InvestigationStatus.COMPLETED + investigation.cost_data = cost.to_dict() + + candidate_dicts = [ + { + "identifier": c.identifier, + "identifier_type": c.identifier_type, + "name": c.name, + "rank": c.rank, + "priority": c.priority, + "notes": c.notes, + "scores": c.scores, + "attributes": c.attributes, + } + for c in candidates + ] + finding_dicts = [ + { + "title": f.title, + "detail": f.detail, + "hypothesis_id": f.hypothesis_id, + "evidence_type": f.evidence_type, + "evidence": f.evidence, + } + for f in investigation.findings + ] + hypothesis_dicts = [ + { + "id": h.id, + "statement": h.statement, + "rationale": h.rationale, + "status": h.status.value, + "parent_id": h.parent_id, + "confidence": h.confidence, + "certainty_of_evidence": h.certainty_of_evidence, + "supporting_evidence": h.supporting_evidence, + "contradicting_evidence": h.contradicting_evidence, + } + for h in investigation.hypotheses + ] + nc_dicts = [ + { + "identifier": nc.identifier, + "identifier_type": nc.identifier_type, + "name": nc.name, + "score": nc.score, + "threshold": nc.threshold, + "correctly_classified": nc.correctly_classified, + "source": nc.source, + } + for nc in investigation.negative_controls + ] + pc_dicts = [ + { + "identifier": pc.identifier, + "identifier_type": pc.identifier_type, + "name": pc.name, + "known_activity": pc.known_activity, + "score": pc.score, + "correctly_classified": pc.correctly_classified, + "source": pc.source, + } + for pc in investigation.positive_controls + ] + yield InvestigationCompleted( + investigation_id=investigation.id, + candidate_count=len(candidates), + summary=investigation.summary, + cost=cost.to_dict(), + candidates=candidate_dicts, + findings=finding_dicts, + hypotheses=hypothesis_dicts, + negative_controls=nc_dicts, + positive_controls=pc_dicts, + validation_metrics=validation_metrics, + diagram_url=diagram_url, + ) diff --git a/server/tests/investigation/test_prompts.py b/server/tests/investigation/test_prompts.py index 96e8972..b1d4b92 100644 --- a/server/tests/investigation/test_prompts.py +++ b/server/tests/investigation/test_prompts.py @@ -1,4 +1,4 @@ -from ehrlich.investigation.application.multi_orchestrator import SEARCH_TOOLS +from ehrlich.investigation.application.literature_survey import SEARCH_TOOLS from ehrlich.investigation.application.prompts.builders import ( build_literature_assessment_prompt, build_literature_survey_prompt, diff --git a/server/tests/investigation/test_uploaded_file.py b/server/tests/investigation/test_uploaded_file.py index 19703f1..46fc33b 100644 --- a/server/tests/investigation/test_uploaded_file.py +++ b/server/tests/investigation/test_uploaded_file.py @@ -106,11 +106,12 @@ def _make_orchestrator(self) -> object: ) orch._uploaded_files = {"tab-123": tabular_file, "doc-456": doc_file} + orch._dispatcher.update_uploaded_files(orch._uploaded_files) return orch def test_query_all_columns(self) -> None: orch = self._make_orchestrator() - result = json.loads(orch._handle_query_uploaded_data({"file_id": "tab-123"})) # type: ignore[attr-defined] + result = json.loads(orch._dispatcher._handle_query_uploaded_data({"file_id": "tab-123"})) assert result["type"] == "tabular" assert result["returned_rows"] == 3 assert result["columns"] == ["name", "age", "score"] @@ -118,8 +119,8 @@ def test_query_all_columns(self) -> None: def test_query_specific_columns(self) -> None: orch = self._make_orchestrator() result = json.loads( - orch._handle_query_uploaded_data( - { # type: ignore[attr-defined] + orch._dispatcher._handle_query_uploaded_data( + { "file_id": "tab-123", "columns": "name,score", } @@ -130,8 +131,8 @@ def test_query_specific_columns(self) -> None: def test_query_filter_eq(self) -> None: orch = self._make_orchestrator() result = json.loads( - orch._handle_query_uploaded_data( - { # type: ignore[attr-defined] + orch._dispatcher._handle_query_uploaded_data( + { "file_id": "tab-123", "filter_column": "name", "filter_op": "eq", @@ -144,8 +145,8 @@ def test_query_filter_eq(self) -> None: def test_query_filter_gt(self) -> None: orch = self._make_orchestrator() result = json.loads( - orch._handle_query_uploaded_data( - { # type: ignore[attr-defined] + orch._dispatcher._handle_query_uploaded_data( + { "file_id": "tab-123", "filter_column": "age", "filter_op": "gt", @@ -158,8 +159,8 @@ def test_query_filter_gt(self) -> None: def test_query_head_limit(self) -> None: orch = self._make_orchestrator() result = json.loads( - orch._handle_query_uploaded_data( - { # type: ignore[attr-defined] + orch._dispatcher._handle_query_uploaded_data( + { "file_id": "tab-123", "head": 2, } @@ -169,12 +170,12 @@ def test_query_head_limit(self) -> None: def test_query_not_found(self) -> None: orch = self._make_orchestrator() - result = json.loads(orch._handle_query_uploaded_data({"file_id": "missing"})) # type: ignore[attr-defined] + result = json.loads(orch._dispatcher._handle_query_uploaded_data({"file_id": "missing"})) assert "error" in result def test_query_document(self) -> None: orch = self._make_orchestrator() - result = json.loads(orch._handle_query_uploaded_data({"file_id": "doc-456"})) # type: ignore[attr-defined] + result = json.loads(orch._dispatcher._handle_query_uploaded_data({"file_id": "doc-456"})) assert result["type"] == "document" assert "education policy" in result["text"] assert result["page_count"] == 2 @@ -182,8 +183,8 @@ def test_query_document(self) -> None: def test_query_document_keyword_filter(self) -> None: orch = self._make_orchestrator() result = json.loads( - orch._handle_query_uploaded_data( - { # type: ignore[attr-defined] + orch._dispatcher._handle_query_uploaded_data( + { "file_id": "doc-456", "filter_value": "education", } @@ -194,8 +195,8 @@ def test_query_document_keyword_filter(self) -> None: def test_query_filter_contains(self) -> None: orch = self._make_orchestrator() result = json.loads( - orch._handle_query_uploaded_data( - { # type: ignore[attr-defined] + orch._dispatcher._handle_query_uploaded_data( + { "file_id": "tab-123", "filter_column": "name", "filter_op": "contains", From e98e4cef763ceca2da24ecf6119f86c3441247fb Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 10:31:39 -0700 Subject: [PATCH 16/35] feat(14H): harden upload validation with domain constants and client-side feedback Extract upload limits (MAX_FILE_SIZE=50MB, MAX_FILES=10, ALLOWED_EXTENSIONS) into upload_limits.py domain constants. Enforce max-files in Pydantic schema, remove redundant size check in upload route, add client-side size validation with per-file error feedback and file count indicator in FileUpload.tsx. --- .../components/FileUpload.test.tsx | 48 ++++++++++++- .../investigation/components/FileUpload.tsx | 70 +++++++++++++++++-- server/src/ehrlich/api/routes/upload.py | 4 -- .../src/ehrlich/api/schemas/investigation.py | 12 +++- .../application/file_processor.py | 11 ++- .../investigation/domain/upload_limits.py | 7 ++ 6 files changed, 131 insertions(+), 21 deletions(-) create mode 100644 server/src/ehrlich/investigation/domain/upload_limits.py diff --git a/console/src/features/investigation/components/FileUpload.test.tsx b/console/src/features/investigation/components/FileUpload.test.tsx index b35c220..83576b3 100644 --- a/console/src/features/investigation/components/FileUpload.test.tsx +++ b/console/src/features/investigation/components/FileUpload.test.tsx @@ -112,9 +112,51 @@ describe("FileUpload", () => { expect(onFilesChanged).toHaveBeenLastCalledWith([]); }); - it("shows error message when upload fails", () => { - mockUploadState.isError = true; + it("shows per-file error when upload fails", async () => { + mockMutateAsync.mockRejectedValueOnce({ message: "Unsupported file type" }); + + const { container } = renderWithProvider({ onFilesChanged: vi.fn() }); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + const file = new File(["data"], "bad.json", { type: "application/json" }); + fireEvent.change(input, { target: { files: [file] } }); + + await vi.waitFor(() => { + expect(container.textContent).toContain("bad.json"); + expect(container.textContent).toContain("Unsupported file type"); + }); + }); + + it("rejects oversized file without calling upload", () => { + const onFilesChanged = vi.fn(); + const { container } = renderWithProvider({ onFilesChanged }); + + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + // Create a file that reports > 50 MB + const bigFile = new File(["x"], "huge.csv", { type: "text/csv" }); + Object.defineProperty(bigFile, "size", { value: 51 * 1024 * 1024 }); + + fireEvent.change(input, { target: { files: [bigFile] } }); + + expect(container.textContent).toContain("huge.csv"); + expect(container.textContent).toContain("File too large"); + expect(mockMutateAsync).not.toHaveBeenCalled(); + }); + + it("shows file count indicator after upload", async () => { + const uploadResult: UploadResponse = { + file_id: "f-1", + filename: "a.csv", + content_type: "text/csv", + preview: { columns: ["x"], dtypes: ["int"], row_count: 1, sample_rows: [["1"]] }, + }; + mockMutateAsync.mockResolvedValueOnce(uploadResult); + const { container } = renderWithProvider({ onFilesChanged: vi.fn() }); - expect(container.textContent).toContain("Upload failed"); + const input = container.querySelector('input[type="file"]') as HTMLInputElement; + fireEvent.change(input, { target: { files: [new File(["d"], "a.csv")] } }); + + await vi.waitFor(() => { + expect(container.textContent).toContain("1 / 10 files"); + }); }); }); diff --git a/console/src/features/investigation/components/FileUpload.tsx b/console/src/features/investigation/components/FileUpload.tsx index 109d62a..e11d43a 100644 --- a/console/src/features/investigation/components/FileUpload.tsx +++ b/console/src/features/investigation/components/FileUpload.tsx @@ -1,11 +1,12 @@ import { useCallback, useRef, useState } from "react"; -import { Upload, Loader2 } from "lucide-react"; +import { Upload, Loader2, AlertCircle } from "lucide-react"; import { useUpload } from "../hooks/use-upload"; import { DataPreview } from "./DataPreview"; import type { UploadResponse } from "../types"; const ACCEPTED_TYPES = ".csv,.xlsx,.pdf"; const MAX_FILES = 10; +const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB interface UploadedFile extends UploadResponse { // UploadResponse already has file_id, filename, content_type, preview @@ -15,10 +16,17 @@ interface FileUploadProps { onFilesChanged: (files: { file_id: string; filename: string }[]) => void; } +function formatSize(bytes: number): string { + if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + if (bytes >= 1024) return `${(bytes / 1024).toFixed(0)} KB`; + return `${bytes} B`; +} + export function FileUpload({ onFilesChanged }: FileUploadProps) { const [files, setFiles] = useState([]); const [dragOver, setDragOver] = useState(false); const [pendingCount, setPendingCount] = useState(0); + const [errors, setErrors] = useState<{ name: string; message: string }[]>([]); const inputRef = useRef(null); const upload = useUpload(); @@ -31,6 +39,14 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) { [onFilesChanged], ); + function addError(name: string, message: string) { + setErrors((prev) => [...prev, { name, message }]); + } + + function dismissError(index: number) { + setErrors((prev) => prev.filter((_, i) => i !== index)); + } + function handleRemove(fileId: string) { setFiles((prev) => { const updated = prev.filter((f) => f.file_id !== fileId); @@ -40,6 +56,11 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) { } async function processFile(file: File) { + if (file.size > MAX_FILE_SIZE) { + addError(file.name, `File too large (${formatSize(file.size)}). Maximum is ${formatSize(MAX_FILE_SIZE)}.`); + return; + } + setPendingCount((c) => c + 1); try { const result = await upload.mutateAsync(file); @@ -48,6 +69,12 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) { updateParent(updated); return updated; }); + } catch (err: unknown) { + const detail = + err && typeof err === "object" && "message" in err + ? String((err as { message: string }).message) + : "Upload failed"; + addError(file.name, detail); } finally { setPendingCount((c) => c - 1); } @@ -56,6 +83,12 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) { function handleFiles(fileList: FileList | File[]) { const incoming = Array.from(fileList); const remaining = MAX_FILES - files.length; + if (incoming.length > remaining) { + addError( + `${incoming.length} files`, + `Only ${remaining} more file${remaining === 1 ? "" : "s"} allowed (max ${MAX_FILES}).`, + ); + } const batch = incoming.slice(0, remaining); for (const file of batch) { void processFile(file); @@ -98,11 +131,10 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) { onDrop={handleDrop} onDragOver={handleDragOver} onDragLeave={handleDragLeave} - className={`flex w-full items-center justify-center gap-2 rounded-md border border-dashed px-4 py-3 text-xs transition-colors ${ - dragOver + className={`flex w-full items-center justify-center gap-2 rounded-md border border-dashed px-4 py-3 text-xs transition-colors ${dragOver ? "border-primary bg-primary/5 text-primary" : "border-border text-muted-foreground hover:border-muted-foreground/50 hover:text-foreground" - }`} + }`} > {pendingCount > 0 ? ( @@ -126,12 +158,35 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) { className="hidden" /> - {upload.isError && ( -

- Upload failed. Check file type and try again. + {files.length > 0 && ( +

+ {files.length} / {MAX_FILES} files

)} + {errors.length > 0 && ( +
+ {errors.map((err, i) => ( +
+ +

+ {err.name}: {err.message} +

+ +
+ ))} +
+ )} + {files.length > 0 && (
{files.map((f) => ( @@ -148,3 +203,4 @@ export function FileUpload({ onFilesChanged }: FileUploadProps) {
); } + diff --git a/server/src/ehrlich/api/routes/upload.py b/server/src/ehrlich/api/routes/upload.py index 3a37b66..f0c77b3 100644 --- a/server/src/ehrlich/api/routes/upload.py +++ b/server/src/ehrlich/api/routes/upload.py @@ -19,8 +19,6 @@ _require_user = Depends(get_current_user) _processor = FileProcessor() -_MAX_TOTAL_SIZE = 50 * 1024 * 1024 # 50 MB - # Pending uploads waiting to be linked to an investigation. # Keyed by file_id, removed when claimed by POST /investigate. _pending_uploads: dict[str, UploadedFileEntity] = {} @@ -43,8 +41,6 @@ async def upload_file( raise HTTPException(status_code=400, detail="Filename is required") content = await file.read() - if len(content) > _MAX_TOTAL_SIZE: - raise HTTPException(status_code=413, detail="File exceeds 50MB limit") try: uploaded = _processor.process(file.filename, content) diff --git a/server/src/ehrlich/api/schemas/investigation.py b/server/src/ehrlich/api/schemas/investigation.py index a8137ba..17c31c7 100644 --- a/server/src/ehrlich/api/schemas/investigation.py +++ b/server/src/ehrlich/api/schemas/investigation.py @@ -4,7 +4,9 @@ from typing import TYPE_CHECKING, Any -from pydantic import BaseModel +from pydantic import BaseModel, field_validator + +from ehrlich.investigation.domain.upload_limits import MAX_FILES_PER_INVESTIGATION if TYPE_CHECKING: from ehrlich.investigation.domain.investigation import Investigation @@ -18,6 +20,14 @@ class InvestigateRequest(BaseModel): director_tier: str = "opus" file_ids: list[str] = [] + @field_validator("file_ids") + @classmethod + def validate_file_count(cls, v: list[str]) -> list[str]: + if len(v) > MAX_FILES_PER_INVESTIGATION: + msg = f"Too many files: {len(v)}. Maximum is {MAX_FILES_PER_INVESTIGATION}." + raise ValueError(msg) + return v + class InvestigateResponse(BaseModel): id: str diff --git a/server/src/ehrlich/investigation/application/file_processor.py b/server/src/ehrlich/investigation/application/file_processor.py index 952bae4..11cb381 100644 --- a/server/src/ehrlich/investigation/application/file_processor.py +++ b/server/src/ehrlich/investigation/application/file_processor.py @@ -4,16 +4,15 @@ import pandas as pd +from ehrlich.investigation.domain.upload_limits import ALLOWED_EXTENSIONS, MAX_FILE_SIZE from ehrlich.investigation.domain.uploaded_file import ( DocumentData, TabularData, UploadedFile, ) -_MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB _MAX_SAMPLE_ROWS = 10 _MAX_PDF_TEXT_LENGTH = 8000 -_ALLOWED_EXTENSIONS = frozenset({"csv", "xlsx", "pdf"}) class FileProcessor: @@ -23,14 +22,14 @@ def process(self, filename: str, content: bytes) -> UploadedFile: if not content: msg = "File is empty" raise ValueError(msg) - if len(content) > _MAX_FILE_SIZE: - msg = f"File exceeds {_MAX_FILE_SIZE // (1024 * 1024)}MB limit" + if len(content) > MAX_FILE_SIZE: + msg = f"File exceeds {MAX_FILE_SIZE // (1024 * 1024)}MB limit" raise ValueError(msg) ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" - if ext not in _ALLOWED_EXTENSIONS: + if ext not in ALLOWED_EXTENSIONS: msg = ( - f"Unsupported file type: .{ext}. Allowed: {', '.join(sorted(_ALLOWED_EXTENSIONS))}" + f"Unsupported file type: .{ext}. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}" ) raise ValueError(msg) diff --git a/server/src/ehrlich/investigation/domain/upload_limits.py b/server/src/ehrlich/investigation/domain/upload_limits.py new file mode 100644 index 0000000..b564561 --- /dev/null +++ b/server/src/ehrlich/investigation/domain/upload_limits.py @@ -0,0 +1,7 @@ +"""Upload constraints — single source of truth for the entire application.""" + +from __future__ import annotations + +MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB +MAX_FILES_PER_INVESTIGATION = 10 +ALLOWED_EXTENSIONS = frozenset({"csv", "xlsx", "pdf"}) From 015f176fa1c1b4a23cd29094af6ece22754a636b Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 10:32:16 -0700 Subject: [PATCH 17/35] feat(14D): enrich domain tools with filtering, ranking, and comparison logic - search_training_literature: MeSH term expansion (6 terms), study type ranking (SR > RCT > cohort), date recency sort, non-human filtering - search_supplement_evidence: GRADE-style ranking (meta-analysis > SR > RCT > observational), retracted paper exclusion, date recency sort - compare_nutrients: per-nutrient delta with winner highlighting, MAR (Mean Adequacy Ratio) score per food via assess_nutrient_adequacy() - Extract _compute_i_squared() with proper inverse-variance-weighted Cochran's Q, replacing inline approximation in analyze_training_evidence 15 new tests (61 total). Zero ruff/mypy violations. --- README.md | 6 +- docs/roadmap/phase-14-engine-hardening.md | 24 ++- .../application/nutrition_service.py | 25 ++- server/src/ehrlich/nutrition/tools.py | 31 ++++ .../training/application/training_service.py | 101 ++++++++++- server/tests/nutrition/test_tools.py | 159 ++++++++++++++++++ server/tests/training/test_tools.py | 155 ++++++++++++++++- 7 files changed, 480 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 843a141..5245051 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ All data sources are free and open-access. | Simulation | `assess_resistance` | Resistance mutation scoring | | Simulation | `get_protein_annotation` | UniProt protein function and disease links | | Simulation | `search_disease_targets` | Open Targets disease-target associations | -| Training | `search_training_literature` | Training science literature via Semantic Scholar | +| Training | `search_training_literature` | Training science literature with MeSH expansion, study type ranking, non-human filtering | | Training | `analyze_training_evidence` | Pooled effect sizes, heterogeneity, evidence grading | | Training | `compare_protocols` | Evidence-weighted protocol comparison | | Training | `assess_injury_risk` | Knowledge-based injury risk scoring | @@ -152,11 +152,11 @@ All data sources are free and open-access. | Training | `compute_performance_model` | Banister fitness-fatigue model (CTL/ATL/TSB) | | Training | `compute_dose_response` | Dose-response curve from dose-effect data | | Training | `plan_periodization` | Evidence-based periodization planning (linear/undulating/block) | -| Nutrition | `search_supplement_evidence` | Supplement efficacy literature search | +| Nutrition | `search_supplement_evidence` | Supplement evidence with GRADE-style ranking, retracted paper exclusion | | Nutrition | `search_supplement_labels` | NIH DSLD supplement product ingredient lookup | | Nutrition | `search_nutrient_data` | USDA FoodData Central nutrient profiles | | Nutrition | `search_supplement_safety` | OpenFDA CAERS adverse event reports | -| Nutrition | `compare_nutrients` | Side-by-side nutrient comparison between foods/supplements | +| Nutrition | `compare_nutrients` | Side-by-side nutrient comparison with per-nutrient deltas, winner, MAR score | | Nutrition | `assess_nutrient_adequacy` | DRI-based nutrient adequacy assessment | | Nutrition | `check_intake_safety` | Tolerable Upper Intake Level safety screening | | Nutrition | `check_interactions` | Drug-supplement interaction screening via RxNav | diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md index 45eb755..2f24c33 100644 --- a/docs/roadmap/phase-14-engine-hardening.md +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -50,14 +50,16 @@ Split `prompts.py` (1,505 lines) into `prompts/` package with 3 focused modules. - [x] `prompts/director.py` -- 4 Director-phase builder functions (`build_formulation_prompt`, `build_experiment_prompt`, `build_synthesis_prompt`, `build_multi_investigation_context`) - [x] `prompts/builders.py` -- 6 non-Director builders (`build_pico_and_classification_prompt`, `build_literature_survey_prompt`, `build_literature_assessment_prompt`, `build_researcher_prompt`, `build_uploaded_data_context`, `_DEFAULT_CATEGORIES`) -## 14D: Domain Tool Enrichment -- TODO +## 14D: Domain Tool Enrichment -- DONE -Upgrade 4 composite tools from thin pass-throughs to real domain-specific logic. +Upgraded 4 composite tools from thin pass-throughs to real domain-specific logic with post-fetch filtering, ranking, and enrichment. -- [ ] `search_training_literature` -- MeSH term expansion, study type ranking (SR > RCT > cohort), date recency weighting, non-human study filtering -- [ ] `search_supplement_evidence` -- GRADE-style quality filtering, RCT/meta-analysis prioritization, retracted paper exclusion -- [ ] `compare_nutrients` -- per-nutrient delta computation, winner highlighting per category, overall MAR score comparison -- [ ] Fix `TrainingService` I-squared reimplementation -- delegate to `StatisticsService` (training_service.py:79-85) +- [x] `search_training_literature` -- MeSH term expansion (6 common abbreviations), study type ranking (meta-analysis > SR > RCT > cohort), date recency weighting, non-human study filtering (mice, rats, in vitro, etc.) +- [x] `search_supplement_evidence` -- GRADE-style quality filtering (meta-analysis > SR > RCT > observational), retracted paper exclusion ([retracted], retracted:, retraction), date recency weighting +- [x] `compare_nutrients` -- per-nutrient delta computation with winner highlighting, MAR (Mean Adequacy Ratio) score via `assess_nutrient_adequacy()`, comparison + mar_scores sections added to JSON output +- [x] Fixed `TrainingService` I-squared reimplementation -- extracted `_compute_i_squared()` module-level function with proper inverse-variance weighting (w_i = sample_size_i), replaced inline calculation in `analyze_training_evidence` + +Result: 15 new tests added (6 for search_training_literature, 3 for search_supplement_evidence, 3 for compare_nutrients, 6 for _compute_i_squared). All 61 tests passing (40 training + 21 nutrition). Zero ruff violations, zero mypy errors. ## 14E: Agentic Hypothesis Tree Search -- TODO @@ -100,6 +102,16 @@ Run real investigations to validate the engine produces meaningful scientific re - [ ] Document comparison results: where Ehrlich agrees/disagrees with baseline, where tree search found non-obvious paths - [ ] Capture investigation traces for demo video +## 14H: Upload Validation Hardening -- DONE + +Added consistent validation on both backend and frontend for file count and size limits, with clear UI feedback. Removed redundancy. + +- [x] Defined `upload_limits.py` domain constants (MAX_FILE_SIZE=50MB, MAX_FILES=10, ALLOWED_EXTENSIONS) +- [x] Enforced max-files limit in Pydantic schema (`InvestigateRequest.file_ids`) +- [x] Removed redundant size check in `upload.py` (delegated to `FileProcessor`) +- [x] Added client-side size validation in `FileUpload.tsx` with per-file error feedback +- [x] Added file count indicator "N / 10 files" in console UI + ## Dependency Graph ``` diff --git a/server/src/ehrlich/nutrition/application/nutrition_service.py b/server/src/ehrlich/nutrition/application/nutrition_service.py index 478fb04..778b4bc 100644 --- a/server/src/ehrlich/nutrition/application/nutrition_service.py +++ b/server/src/ehrlich/nutrition/application/nutrition_service.py @@ -137,7 +137,30 @@ async def search_supplement_evidence( self, supplement: str, outcome: str = "performance", limit: int = 8 ) -> list[Paper]: query = f"{supplement} {outcome} meta-analysis systematic review" - return await self._papers.search(query, limit=limit) + + # Fetch extra papers to compensate for filtering + papers = await self._papers.search(query, limit=limit * 2) + + # Filter out retracted papers + retracted_keywords = {"[retracted]", "retracted:", "retraction"} + non_retracted = [ + p for p in papers if not any(kw in p.title.lower() for kw in retracted_keywords) + ] + + # Sort by study type rank, then year descending + def get_study_type_rank(paper: Paper) -> int: + text = (paper.title + " " + paper.abstract).lower() + if "meta-analysis" in text: + return 0 + if "systematic review" in text: + return 1 + if "rct" in text or "randomized" in text: + return 2 + return 3 + + non_retracted.sort(key=lambda p: (get_study_type_rank(p), -p.year)) + + return non_retracted[:limit] async def search_supplement_labels( self, ingredient: str, max_results: int = 10 diff --git a/server/src/ehrlich/nutrition/tools.py b/server/src/ehrlich/nutrition/tools.py index 89a4b9f..e6d157c 100644 --- a/server/src/ehrlich/nutrition/tools.py +++ b/server/src/ehrlich/nutrition/tools.py @@ -205,6 +205,35 @@ async def compare_nutrients( """ queries = [q.strip() for q in food_queries.split(",") if q.strip()] profiles = await _service.compare_nutrients(queries, max_nutrients) + + # Build per-nutrient comparison + comparison: dict[str, dict[str, object]] = {} + if len(profiles) >= 2: + # Collect all nutrient names across all foods + nutrient_map: dict[str, dict[str, float]] = {} + for p in profiles: + for n in p.nutrients: + if n.name not in nutrient_map: + nutrient_map[n.name] = {} + nutrient_map[n.name][p.description] = n.amount + + # Build comparison for nutrients that appear in 2+ foods + for nutrient_name, food_amounts in nutrient_map.items(): + if len(food_amounts) >= 2: + winner = max(food_amounts, key=food_amounts.get) # type: ignore[arg-type] + comparison[nutrient_name] = { + "amounts": food_amounts, + "winner": winner, + } + + # Compute MAR (Mean Adequacy Ratio) per food + mar_scores: dict[str, float] = {} + for p in profiles: + assessments = _service.assess_nutrient_adequacy(list(p.nutrients)) + if assessments: + avg_pct_rda = sum(a.pct_rda for a in assessments) / len(assessments) + mar_scores[p.description] = round(avg_pct_rda, 1) + return json.dumps( { "queries": queries, @@ -227,6 +256,8 @@ async def compare_nutrients( } for p in profiles ], + "comparison": comparison, + "mar_scores": mar_scores, } ) diff --git a/server/src/ehrlich/training/application/training_service.py b/server/src/ehrlich/training/application/training_service.py index 6b32c3b..40cffba 100644 --- a/server/src/ehrlich/training/application/training_service.py +++ b/server/src/ehrlich/training/application/training_service.py @@ -41,6 +41,47 @@ } +def _compute_i_squared(effect_sizes: list[float], sample_sizes: list[float]) -> float: + """Compute I-squared heterogeneity statistic using inverse-variance weighting. + + I-squared quantifies the percentage of total variation across studies due to + heterogeneity rather than chance. Based on Cochran's Q statistic with + inverse-variance weights (approximated by sample sizes). + + Args: + effect_sizes: Effect sizes from individual studies + sample_sizes: Sample sizes from individual studies (used as weights) + + Returns: + I-squared percentage (0-100), where 0% = no heterogeneity, + 100% = all variation is heterogeneity + """ + n = len(effect_sizes) + if n <= 1: + return 0.0 + + # Use sample sizes as inverse-variance weights + weights = sample_sizes + total_weight = sum(weights) + + if total_weight <= 0: + return 0.0 + + # Compute weighted mean + weighted_mean = sum(w * e for w, e in zip(weights, effect_sizes, strict=True)) / total_weight + + # Compute Cochran's Q statistic (weighted sum of squared deviations) + q_stat = sum(w * (e - weighted_mean) ** 2 for w, e in zip(weights, effect_sizes, strict=True)) + + # Degrees of freedom + df = n - 1 + + # I-squared = max(0, (Q - df) / Q * 100) + if q_stat > 0: + return max(0.0, (q_stat - df) / q_stat * 100) + return 0.0 + + class TrainingService: def __init__( self, @@ -55,8 +96,56 @@ def __init__( self._exercises = exercises async def search_literature(self, query: str, limit: int = 10) -> list[Paper]: - training_query = f"training science {query}" - return await self._papers.search(training_query, limit=limit) + mesh_expansion = { + "hiit": "high intensity interval training", + "strength": "resistance training", + "cardio": "cardiovascular exercise", + "flexibility": "stretching exercise", + "power": "plyometric training", + "endurance": "aerobic exercise", + } + + query_lower = query.lower() + expanded_terms = [query] + for short, full in mesh_expansion.items(): + if short in query_lower and full not in query_lower: + expanded_terms.append(full) + + expanded_query = " ".join(expanded_terms) + training_query = f"training science {expanded_query}" + + # Fetch extra papers to compensate for filtering + papers = await self._papers.search(training_query, limit=limit * 2) + + # Filter out non-human studies + non_human_keywords = { + "mice", + "rats", + "rat", + "murine", + "in vitro", + "cell culture", + "drosophila", + "zebrafish", + } + human_papers = [ + p for p in papers if not any(kw in p.title.lower() for kw in non_human_keywords) + ] + + # Sort by study type rank, then year descending + def get_study_type_rank(paper: Paper) -> int: + text = (paper.title + " " + paper.abstract).lower() + if "meta-analysis" in text or "systematic review" in text: + return 0 + if "rct" in text or "randomized" in text: + return 1 + if "cohort" in text or "longitudinal" in text: + return 2 + return 3 + + human_papers.sort(key=lambda p: (get_study_type_rank(p), -p.year)) + + return human_papers[:limit] def analyze_training_evidence( self, @@ -76,13 +165,7 @@ def analyze_training_evidence( else 0.0 ) - if n > 1: - mean_effect = sum(effect_sizes) / n - q_stat = sum((e - mean_effect) ** 2 for e in effect_sizes) - df = n - 1 - i_squared = max(0.0, (q_stat - df) / q_stat * 100) if q_stat > 0 else 0.0 - else: - i_squared = 0.0 + i_squared = _compute_i_squared(effect_sizes, sample_sizes) avg_quality = sum(quality_scores) / n diff --git a/server/tests/nutrition/test_tools.py b/server/tests/nutrition/test_tools.py index fdbd08a..7190228 100644 --- a/server/tests/nutrition/test_tools.py +++ b/server/tests/nutrition/test_tools.py @@ -95,6 +95,108 @@ async def test_returns_papers(self, mock_scholar: AsyncMock) -> None: assert result["outcome"] == "strength" assert result["count"] == 1 + @pytest.mark.asyncio() + async def test_filters_retracted_papers(self) -> None: + """Test that retracted papers are filtered out.""" + papers = [ + Paper( + title="[RETRACTED] Creatine meta-analysis", + authors=["Bad A"], + year=2020, + abstract="Retracted due to fraud.", + ), + Paper( + title="Creatine systematic review", + authors=["Good B"], + year=2023, + abstract="Valid systematic review.", + ), + Paper( + title="Retracted: Creatine RCT", + authors=["Bad C"], + year=2022, + abstract="Retracted paper.", + ), + ] + with patch( + "ehrlich.nutrition.tools._client.search", + new_callable=AsyncMock, + return_value=papers, + ): + result = json.loads(await search_supplement_evidence("creatine")) + assert result["count"] == 1 + assert result["papers"][0]["title"] == "Creatine systematic review" + + @pytest.mark.asyncio() + async def test_study_type_ranking(self) -> None: + """Test that meta-analyses rank above systematic reviews, which rank above RCTs.""" + papers = [ + Paper( + title="Observational study of creatine", + authors=["Jones C"], + year=2024, + abstract="Cohort analysis of creatine use.", + ), + Paper( + title="Creatine meta-analysis", + authors=["Smith A"], + year=2022, + abstract="Meta-analysis of creatine trials.", + ), + Paper( + title="Systematic review of creatine", + authors=["Brown B"], + year=2023, + abstract="Systematic review of creatine effects.", + ), + Paper( + title="RCT on creatine", + authors=["White D"], + year=2024, + abstract="Randomized controlled trial of creatine.", + ), + ] + with patch( + "ehrlich.nutrition.tools._client.search", + new_callable=AsyncMock, + return_value=papers, + ): + result = json.loads(await search_supplement_evidence("creatine")) + assert result["count"] == 4 + # Should be ordered: meta-analysis, systematic review, RCT, observational + assert "meta-analysis" in result["papers"][0]["title"].lower() + assert "systematic review" in result["papers"][1]["title"].lower() + assert "rct" in result["papers"][2]["title"].lower() + assert "observational" in result["papers"][3]["title"].lower() + + @pytest.mark.asyncio() + async def test_date_recency_within_same_study_type(self) -> None: + """Test that more recent papers rank higher within the same study type.""" + papers = [ + Paper( + title="Creatine RCT 2020", + authors=["Old A"], + year=2020, + abstract="Randomized trial from 2020.", + ), + Paper( + title="Creatine RCT 2024", + authors=["New B"], + year=2024, + abstract="Randomized trial from 2024.", + ), + ] + with patch( + "ehrlich.nutrition.tools._client.search", + new_callable=AsyncMock, + return_value=papers, + ): + result = json.loads(await search_supplement_evidence("creatine")) + assert result["count"] == 2 + # More recent should come first + assert result["papers"][0]["year"] == 2024 + assert result["papers"][1]["year"] == 2020 + class TestSearchSupplementLabels: @pytest.mark.asyncio() @@ -182,6 +284,9 @@ async def test_returns_comparison(self) -> None: assert result["count"] == 1 assert result["queries"] == ["chicken breast", "salmon"] assert result["foods"][0]["description"] == "Chicken breast" + # With only 1 food, no comparison is possible + assert result["comparison"] == {} + assert "mar_scores" in result @pytest.mark.asyncio() async def test_empty_query(self) -> None: @@ -192,6 +297,60 @@ async def test_empty_query(self) -> None: ): result = json.loads(await compare_nutrients("")) assert result["count"] == 0 + assert result["comparison"] == {} + assert result["mar_scores"] == {} + + @pytest.mark.asyncio() + async def test_per_nutrient_comparison_and_winner(self) -> None: + """Test that per-nutrient deltas and winner are computed correctly.""" + profile_a = NutrientProfile( + fdc_id=1, + description="Food A", + brand="", + category="Test", + nutrients=( + NutrientEntry(name="Protein", amount=20.0, unit="G", nutrient_number="203"), + NutrientEntry(name="Calcium, Ca", amount=100.0, unit="MG", nutrient_number="301"), + ), + ) + profile_b = NutrientProfile( + fdc_id=2, + description="Food B", + brand="", + category="Test", + nutrients=( + NutrientEntry(name="Protein", amount=30.0, unit="G", nutrient_number="203"), + NutrientEntry(name="Calcium, Ca", amount=80.0, unit="MG", nutrient_number="301"), + ), + ) + with patch( + "ehrlich.nutrition.tools._service.compare_nutrients", + new_callable=AsyncMock, + return_value=[profile_a, profile_b], + ): + result = json.loads(await compare_nutrients("food a, food b")) + assert result["count"] == 2 + comparison = result["comparison"] + assert "Protein" in comparison + assert comparison["Protein"]["winner"] == "Food B" + assert comparison["Protein"]["amounts"]["Food A"] == 20.0 + assert comparison["Protein"]["amounts"]["Food B"] == 30.0 + assert "Calcium, Ca" in comparison + assert comparison["Calcium, Ca"]["winner"] == "Food A" + + @pytest.mark.asyncio() + async def test_mar_score_computation(self) -> None: + """Test that MAR scores are computed correctly.""" + with patch( + "ehrlich.nutrition.tools._service.compare_nutrients", + new_callable=AsyncMock, + return_value=[_SAMPLE_PROFILE], + ): + result = json.loads(await compare_nutrients("chicken breast")) + assert "mar_scores" in result + assert "Chicken breast" in result["mar_scores"] + # MAR score should be a percentage + assert isinstance(result["mar_scores"]["Chicken breast"], (int, float)) class TestAssessNutrientAdequacy: diff --git a/server/tests/training/test_tools.py b/server/tests/training/test_tools.py index 2188c71..f922536 100644 --- a/server/tests/training/test_tools.py +++ b/server/tests/training/test_tools.py @@ -8,6 +8,7 @@ import pytest from ehrlich.literature.domain.paper import Paper +from ehrlich.training.application.training_service import _compute_i_squared from ehrlich.training.domain.entities import ClinicalTrial from ehrlich.training.tools import ( analyze_training_evidence, @@ -52,12 +53,118 @@ async def test_returns_papers(self, mock_scholar: AsyncMock) -> None: assert result["count"] == 1 assert result["papers"][0]["title"] == "HIIT vs MICT for VO2max" assert result["papers"][0]["doi"] == "10.1234/test" - mock_scholar.assert_called_once_with("training science HIIT VO2max", limit=10) + # Service now fetches limit * 2 to compensate for filtering + mock_scholar.assert_called_once_with( + "training science HIIT VO2max high intensity interval training", limit=20 + ) @pytest.mark.asyncio() async def test_custom_limit(self, mock_scholar: AsyncMock) -> None: await search_training_literature("creatine", limit=5) - mock_scholar.assert_called_once_with("training science creatine", limit=5) + mock_scholar.assert_called_once_with("training science creatine", limit=10) + + @pytest.mark.asyncio() + async def test_mesh_expansion(self, mock_scholar: AsyncMock) -> None: + """Test MeSH term expansion for common abbreviations.""" + await search_training_literature("strength training") + mock_scholar.assert_called_once_with( + "training science strength training resistance training", limit=20 + ) + + @pytest.mark.asyncio() + async def test_filters_non_human_studies(self) -> None: + """Test that animal/in-vitro studies are filtered out.""" + papers = [ + Paper( + title="HIIT in mice models", + authors=["Mouse A"], + year=2023, + abstract="Study in rodents.", + ), + Paper( + title="Human HIIT RCT", + authors=["Smith J"], + year=2023, + abstract="Randomized controlled trial in humans.", + ), + Paper( + title="In vitro muscle cell study", + authors=["Lab B"], + year=2022, + abstract="Cell culture analysis.", + ), + ] + with patch( + "ehrlich.training.tools._client.search", + new_callable=AsyncMock, + return_value=papers, + ): + result = json.loads(await search_training_literature("HIIT")) + assert result["count"] == 1 + assert result["papers"][0]["title"] == "Human HIIT RCT" + + @pytest.mark.asyncio() + async def test_study_type_ranking(self) -> None: + """Test that systematic reviews rank above RCTs, which rank above cohort studies.""" + papers = [ + Paper( + title="Cohort study of HIIT", + authors=["Jones C"], + year=2024, + abstract="Longitudinal cohort analysis.", + ), + Paper( + title="HIIT systematic review", + authors=["Smith A"], + year=2022, + abstract="Systematic review of HIIT protocols.", + ), + Paper( + title="RCT on HIIT", + authors=["Brown B"], + year=2023, + abstract="Randomized controlled trial.", + ), + ] + with patch( + "ehrlich.training.tools._client.search", + new_callable=AsyncMock, + return_value=papers, + ): + result = json.loads(await search_training_literature("HIIT")) + assert result["count"] == 3 + # Should be ordered: systematic review, RCT, cohort + assert "systematic review" in result["papers"][0]["title"].lower() + assert "rct" in result["papers"][1]["title"].lower() + assert "cohort" in result["papers"][2]["title"].lower() + + @pytest.mark.asyncio() + async def test_date_recency_within_same_study_type(self) -> None: + """Test that more recent papers rank higher within the same study type.""" + papers = [ + Paper( + title="HIIT RCT 2020", + authors=["Old A"], + year=2020, + abstract="Randomized trial from 2020.", + ), + Paper( + title="HIIT RCT 2024", + authors=["New B"], + year=2024, + abstract="Randomized trial from 2024.", + ), + ] + with patch( + "ehrlich.training.tools._client.search", + new_callable=AsyncMock, + return_value=papers, + ): + result = json.loads(await search_training_literature("HIIT")) + assert result["count"] == 2 + # More recent should come first + assert result["papers"][0]["year"] == 2024 + assert result["papers"][1]["year"] == 2020 class TestAnalyzeTrainingEvidence: @@ -439,3 +546,47 @@ async def test_deload_weeks(self) -> None: if (i + 1) % 4 == 0: # Deload weeks should be notably lower assert progression[i] < 0.60 + + +class TestComputeISquared: + def test_no_heterogeneity(self) -> None: + """Test I-squared with identical effect sizes (no heterogeneity).""" + effect_sizes = [0.5, 0.5, 0.5, 0.5] + sample_sizes = [30.0, 30.0, 30.0, 30.0] + i_squared = _compute_i_squared(effect_sizes, sample_sizes) + assert i_squared == 0.0 + + def test_high_heterogeneity(self) -> None: + """Test I-squared with widely varying effect sizes (high heterogeneity).""" + effect_sizes = [0.1, 0.5, 0.9, 1.2] + sample_sizes = [30.0, 30.0, 30.0, 30.0] + i_squared = _compute_i_squared(effect_sizes, sample_sizes) + # With large variance, I-squared should be high + assert i_squared > 50.0 + + def test_inverse_variance_weighting(self) -> None: + """Test that larger studies have more influence on weighted mean.""" + # Two studies: one small (n=10) with large effect, one large (n=100) with small effect + effect_sizes = [1.0, 0.1] + sample_sizes = [10.0, 100.0] + i_squared = _compute_i_squared(effect_sizes, sample_sizes) + # Weighted mean should be closer to 0.1 (larger study) + # This should result in moderate heterogeneity + assert 0.0 < i_squared < 100.0 + + def test_single_study(self) -> None: + """Test I-squared with only one study (should be 0).""" + i_squared = _compute_i_squared([0.5], [30.0]) + assert i_squared == 0.0 + + def test_empty_studies(self) -> None: + """Test I-squared with no studies (should be 0).""" + i_squared = _compute_i_squared([], []) + assert i_squared == 0.0 + + def test_zero_weights(self) -> None: + """Test I-squared with zero sample sizes (should be 0).""" + effect_sizes = [0.5, 0.6, 0.7] + sample_sizes = [0.0, 0.0, 0.0] + i_squared = _compute_i_squared(effect_sizes, sample_sizes) + assert i_squared == 0.0 From 72102424af575fd5fbf95eb135ed6e05ccc95d5e Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 10:46:20 -0700 Subject: [PATCH 18/35] feat(api): enforce investigation ownership on all per-resource endpoints GET /investigate/{id} was public with no auth. Stream and approve had auth but no ownership check. Now all three verify the caller owns the resource via _verify_ownership (checks in-memory meta for active, JOIN query for persisted). Returns 403 Forbidden for non-owners. --- README.md | 6 +- .../phase-12-platform-infrastructure.md | 5 +- .../src/ehrlich/api/routes/investigation.py | 28 ++++++++- .../infrastructure/repository.py | 11 ++++ server/tests/api/test_investigation.py | 63 +++++++++++++++++++ 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5245051..5ee5844 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,6 @@ Server at :8000, Console at :3000. | GET | `/api/v1/health` | Health check | | GET | `/api/v1/methodology` | Methodology: phases, domains, tools, data sources, models | | GET | `/api/v1/stats` | Aggregate counts (tools, domains, phases, data sources, events) | -| GET | `/api/v1/investigate/{id}` | Full investigation detail | | GET | `/api/v1/molecule/depict?smiles=&w=&h=` | 2D SVG depiction (`image/svg+xml`, cached 24h) | | GET | `/api/v1/molecule/conformer?smiles=` | 3D conformer (JSON: mol_block, energy, num_atoms) | | GET | `/api/v1/molecule/descriptors?smiles=` | Molecular descriptors + Lipinski pass/fail | @@ -315,8 +314,9 @@ Server at :8000, Console at :3000. |--------|------|-------------| | GET | `/api/v1/investigate` | List user's investigations (most recent first) | | POST | `/api/v1/investigate` | Create new investigation (`director_tier`: haiku/sonnet/opus) | -| GET | `/api/v1/investigate/{id}/stream` | SSE stream of investigation events (supports `?token=` for EventSource) | -| POST | `/api/v1/investigate/{id}/approve` | Approve/reject formulated hypotheses | +| GET | `/api/v1/investigate/{id}` | Full investigation detail (owner only) | +| GET | `/api/v1/investigate/{id}/stream` | SSE stream of investigation events (owner only, supports `?token=`) | +| POST | `/api/v1/investigate/{id}/approve` | Approve/reject formulated hypotheses (owner only) | | POST | `/api/v1/upload` | Upload file (CSV/XLSX/PDF) for investigation data, returns preview | | GET | `/api/v1/credits/balance` | Current credit balance + BYOK status | diff --git a/docs/roadmap/phase-12-platform-infrastructure.md b/docs/roadmap/phase-12-platform-infrastructure.md index 34ba822..7560087 100644 --- a/docs/roadmap/phase-12-platform-infrastructure.md +++ b/docs/roadmap/phase-12-platform-infrastructure.md @@ -27,8 +27,9 @@ User identity via WorkOS (1M MAU free tier). JWT-based. - [x] Frontend: `@workos-inc/authkit-react` provider + `` - [x] Backend: `get_current_user`, `get_current_user_sse`, `get_optional_user` dependencies - [x] `user_id` extraction from JWT, linked to `users` table via `get_or_create_user()` -- [x] Public routes: `GET /health`, `GET /methodology`, `GET /stats`, `GET /molecule/*`, `GET /investigate/{id}` -- [x] Protected routes: `POST /investigate`, `GET /investigate`, `GET /investigate/{id}/stream`, `POST /investigate/{id}/approve`, `GET /credits/balance` +- [x] Public routes: `GET /health`, `GET /methodology`, `GET /stats`, `GET /molecule/*` +- [x] Protected routes: `POST /investigate`, `GET /investigate`, `GET /investigate/{id}`, `GET /investigate/{id}/stream`, `POST /investigate/{id}/approve`, `GET /credits/balance` +- [x] Ownership enforcement: all per-investigation endpoints verify the caller owns the resource (403 Forbidden) - [x] Env vars: `EHRLICH_WORKOS_API_KEY`, `EHRLICH_WORKOS_CLIENT_ID` - [x] SSE auth via `?token=` query param fallback (EventSource does not support headers) diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index c472619..0d8f8e7 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -85,6 +85,24 @@ def _get_repository() -> InvestigationRepository: return _repository +async def _verify_ownership( + investigation_id: str, + user: dict[str, Any], + repo: InvestigationRepository, +) -> None: + """Raise 403 if user does not own the investigation.""" + # Active investigation -- check transient meta + meta = _investigation_meta.get(investigation_id) + if meta is not None: + if meta.get("workos_id") != user["workos_id"]: + raise HTTPException(status_code=403, detail="Forbidden") + return + # DB investigation -- check via JOIN + owner_workos_id = await repo.get_investigation_owner_workos_id(investigation_id) + if owner_workos_id != user["workos_id"]: + raise HTTPException(status_code=403, detail="Forbidden") + + def _broadcast_event(investigation_id: str, event: dict[str, str]) -> None: for queue in _subscribers.get(investigation_id, []): queue.put_nowait(event) @@ -130,7 +148,10 @@ async def get_credit_balance( @router.get("/investigate/{investigation_id}") -async def get_investigation(investigation_id: str) -> InvestigationDetail: +async def get_investigation( + investigation_id: str, + user: dict[str, Any] = _require_user, +) -> InvestigationDetail: repo = _get_repository() # Check active first (in-flight data is more current) investigation = _active_investigations.get(investigation_id) @@ -138,6 +159,7 @@ async def get_investigation(investigation_id: str) -> InvestigationDetail: investigation = await repo.get_by_id(investigation_id) if investigation is None: raise HTTPException(status_code=404, detail="Investigation not found") + await _verify_ownership(investigation_id, user, repo) return to_detail(investigation) @@ -195,6 +217,8 @@ async def approve_hypotheses( orchestrator = _active_orchestrators.get(investigation_id) if orchestrator is None: raise HTTPException(status_code=404, detail="No active orchestrator") + repo = _get_repository() + await _verify_ownership(investigation_id, user, repo) orchestrator.approve_hypotheses(request.approved_ids, request.rejected_ids) return {"status": "approved"} @@ -214,6 +238,8 @@ async def stream_investigation( if investigation is None: raise HTTPException(status_code=404, detail="Investigation not found") + await _verify_ownership(investigation_id, user, repo) + status = investigation.status # Completed or failed -- replay final status diff --git a/server/src/ehrlich/investigation/infrastructure/repository.py b/server/src/ehrlich/investigation/infrastructure/repository.py index 0fceab6..20d8fa6 100644 --- a/server/src/ehrlich/investigation/infrastructure/repository.py +++ b/server/src/ehrlich/investigation/infrastructure/repository.py @@ -431,6 +431,17 @@ async def refund_credits(self, workos_id: str, amount: int, investigation_id: st investigation_id, ) + async def get_investigation_owner_workos_id(self, investigation_id: str) -> str | None: + """Get the workos_id of the user who owns the investigation.""" + pool = self._get_pool() + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT u.workos_id FROM investigations i " + "JOIN users u ON i.user_id = u.id WHERE i.id = $1", + investigation_id, + ) + return row["workos_id"] if row else None + # -- Private helpers ----------------------------------------------------------- async def _rebuild_fts( diff --git a/server/tests/api/test_investigation.py b/server/tests/api/test_investigation.py index 180e06f..ba2c6ec 100644 --- a/server/tests/api/test_investigation.py +++ b/server/tests/api/test_investigation.py @@ -14,12 +14,17 @@ ) _TEST_USER = {"workos_id": "workos_inv_test", "email": "inv@test.com"} +_OTHER_USER = {"workos_id": "workos_other_user", "email": "other@test.com"} async def _mock_user() -> dict[str, str]: return _TEST_USER +async def _mock_other_user() -> dict[str, str]: + return _OTHER_USER + + @pytest.fixture async def client() -> httpx.AsyncClient: app = create_app() @@ -35,6 +40,7 @@ async def client() -> httpx.AsyncClient: await c.execute("DELETE FROM investigations") await c.execute("DELETE FROM users") await repo.get_or_create_user(_TEST_USER["workos_id"], _TEST_USER["email"]) + await repo.get_or_create_user(_OTHER_USER["workos_id"], _OTHER_USER["email"]) transport = httpx.ASGITransport(app=app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as ac: @@ -125,3 +131,60 @@ async def test_completed_replays_final(self, client: httpx.AsyncClient) -> None: response = await client.get(f"/api/v1/investigate/{inv_id}/stream") assert response.status_code == 200 + + +class TestOwnership: + async def test_get_investigation_forbidden_for_other_user( + self, client: httpx.AsyncClient + ) -> None: + # Create investigation as TEST_USER + resp = await client.post( + "/api/v1/investigate", + json={"prompt": "Test", "director_tier": "haiku"}, + ) + inv_id = resp.json()["id"] + + # Switch to OTHER_USER + app = client._transport.app # type: ignore[union-attr] + app.dependency_overrides[get_current_user] = _mock_other_user + app.dependency_overrides[get_current_user_sse] = _mock_other_user + + response = await client.get(f"/api/v1/investigate/{inv_id}") + assert response.status_code == 403 + + async def test_stream_forbidden_for_other_user(self, client: httpx.AsyncClient) -> None: + resp = await client.post( + "/api/v1/investigate", + json={"prompt": "Test", "director_tier": "haiku"}, + ) + inv_id = resp.json()["id"] + + # Mark completed so stream replays (avoids starting orchestrator) + repo = inv_module._get_repository() + inv = await repo.get_by_id(inv_id) + assert inv is not None + inv.status = InvestigationStatus.COMPLETED + inv.summary = "Done" + await repo.update(inv) + + # Switch user + app = client._transport.app # type: ignore[union-attr] + app.dependency_overrides[get_current_user] = _mock_other_user + app.dependency_overrides[get_current_user_sse] = _mock_other_user + + response = await client.get(f"/api/v1/investigate/{inv_id}/stream") + assert response.status_code == 403 + + async def test_list_only_own_investigations(self, client: httpx.AsyncClient) -> None: + await client.post( + "/api/v1/investigate", + json={"prompt": "User 1 investigation", "director_tier": "haiku"}, + ) + + # Switch user + app = client._transport.app # type: ignore[union-attr] + app.dependency_overrides[get_current_user] = _mock_other_user + + response = await client.get("/api/v1/investigate") + assert response.status_code == 200 + assert response.json() == [] From eaa8d58d73bb170113bf4d4782da65f9152138ba Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 10:53:27 -0700 Subject: [PATCH 19/35] fix(investigation): resolve mypy errors in orchestrator and registry factory Simplified phase result narrowing from compound isinstance+contains to isinstance(dict) alone -- all phase dicts are __phase_result__ sentinels. Added type annotation to tagged_tools list in registry factory. Applied ruff format to investigation application layer. --- .../application/batch_executor.py | 34 ++++++-- .../application/file_processor.py | 4 +- .../application/literature_survey.py | 11 ++- .../application/multi_orchestrator.py | 78 +++++++++++++------ .../investigation/application/phase_runner.py | 33 ++------ .../application/registry_factory.py | 6 +- .../application/researcher_executor.py | 4 +- 7 files changed, 106 insertions(+), 64 deletions(-) diff --git a/server/src/ehrlich/investigation/application/batch_executor.py b/server/src/ehrlich/investigation/application/batch_executor.py index bcd6735..2765b20 100644 --- a/server/src/ehrlich/investigation/application/batch_executor.py +++ b/server/src/ehrlich/investigation/application/batch_executor.py @@ -40,8 +40,20 @@ async def run_experiment_batch( if len(batch) == 1: h, exp, design = batch[0] async for event in run_researcher_experiment( - researcher, summarizer, dispatcher, registry, active_config, researcher_prompt, - summarizer_threshold, max_iterations, investigation, h, exp, cost, design, state_lock, + researcher, + summarizer, + dispatcher, + registry, + active_config, + researcher_prompt, + summarizer_threshold, + max_iterations, + investigation, + h, + exp, + cost, + design, + state_lock, ): yield event return @@ -69,9 +81,21 @@ async def _run_one( ) -> None: try: async for ev in run_researcher_experiment( - researcher, summarizer, dispatcher, registry, active_config, researcher_prompt, - summarizer_threshold, max_iterations, investigation, hyp, exp, cost, design, - state_lock, sibling_context=sib_ctx, + researcher, + summarizer, + dispatcher, + registry, + active_config, + researcher_prompt, + summarizer_threshold, + max_iterations, + investigation, + hyp, + exp, + cost, + design, + state_lock, + sibling_context=sib_ctx, ): await queue.put(ev) except Exception as e: diff --git a/server/src/ehrlich/investigation/application/file_processor.py b/server/src/ehrlich/investigation/application/file_processor.py index 11cb381..f213c02 100644 --- a/server/src/ehrlich/investigation/application/file_processor.py +++ b/server/src/ehrlich/investigation/application/file_processor.py @@ -28,9 +28,7 @@ def process(self, filename: str, content: bytes) -> UploadedFile: ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" if ext not in ALLOWED_EXTENSIONS: - msg = ( - f"Unsupported file type: .{ext}. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}" - ) + msg = f"Unsupported file type: .{ext}. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}" raise ValueError(msg) if ext == "csv": diff --git a/server/src/ehrlich/investigation/application/literature_survey.py b/server/src/ehrlich/investigation/application/literature_survey.py index 69722d6..33ccf2f 100644 --- a/server/src/ehrlich/investigation/application/literature_survey.py +++ b/server/src/ehrlich/investigation/application/literature_survey.py @@ -17,6 +17,8 @@ ) from ehrlich.investigation.application.researcher_executor import ( _compact_result as compact_result, +) +from ehrlich.investigation.application.researcher_executor import ( maybe_viz_event, summarize_output, ) @@ -218,7 +220,11 @@ async def run_literature_survey( pass summarized_str, summarize_event = await summarize_output( - summarizer, cost, tool_name, result_str, investigation.id, + summarizer, + cost, + tool_name, + result_str, + investigation.id, summarizer_threshold, ) if summarize_event is not None: @@ -276,8 +282,7 @@ async def run_literature_survey( assessment = "" if investigation.findings: findings_for_grading = "\n".join( - f"- [level={f.evidence_level}] {f.title}: {f.detail}" - for f in investigation.findings + f"- [level={f.evidence_level}] {f.title}: {f.detail}" for f in investigation.findings ) grade_response = await summarizer.create_message( system=build_literature_assessment_prompt(), diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index d653f5c..9c802a1 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -186,11 +186,15 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, pico: dict[str, Any] = {} prior_context = "" async for event in run_classification_phase( - investigation, cost, self._summarizer, - self._uploaded_data_context, self._repository, - self._domain_registry, self._director_call, + investigation, + cost, + self._summarizer, + self._uploaded_data_context, + self._repository, + self._domain_registry, + self._director_call, ): - if isinstance(event, dict) and "__phase_result__" in event: + if isinstance(event, dict): result = event["__phase_result__"] pico = result["pico"] prior_context = result["prior_context"] @@ -209,10 +213,16 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, investigation_id=investigation.id, ) async for event in run_literature_survey( - self._researcher, self._summarizer, self._dispatcher, - self._registry, self._active_config, - self._summarizer_threshold, self._max_iterations_per_experiment, - investigation, cost, pico, + self._researcher, + self._summarizer, + self._dispatcher, + self._registry, + self._active_config, + self._summarizer_threshold, + self._max_iterations_per_experiment, + investigation, + cost, + pico, ): yield event yield self._cost_event(cost, investigation.id) @@ -221,12 +231,18 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, neg_control_suggestions: list[dict[str, Any]] = [] pos_control_suggestions: list[dict[str, Any]] = [] async for event in run_formulation_phase( - investigation, cost, self._active_config, - self._uploaded_data_context, prior_context, pico, - self._require_approval, self._approval_event, - self._director_call, self._cost_event, + investigation, + cost, + self._active_config, + self._uploaded_data_context, + prior_context, + pico, + self._require_approval, + self._approval_event, + self._director_call, + self._cost_event, ): - if isinstance(event, dict) and "__phase_result__" in event: + if isinstance(event, dict): result = event["__phase_result__"] neg_control_suggestions = result["neg_control_suggestions"] pos_control_suggestions = result["pos_control_suggestions"] @@ -235,32 +251,46 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, # 4. Hypothesis testing loop async for event in run_hypothesis_testing_phase( - investigation, cost, self._active_config, - self._uploaded_data_context, self._researcher_prompt, - self._researcher, self._summarizer, self._dispatcher, - self._registry, self._max_hypotheses, + investigation, + cost, + self._active_config, + self._uploaded_data_context, + self._researcher_prompt, + self._researcher, + self._summarizer, + self._dispatcher, + self._registry, + self._max_hypotheses, self._max_iterations_per_experiment, - self._summarizer_threshold, self._state_lock, - self._director_call, self._cost_event, + self._summarizer_threshold, + self._state_lock, + self._director_call, + self._cost_event, ): yield event # 5. Controls validation validation_metrics: dict[str, Any] = {} async for event in run_controls_phase( - investigation, cost, self._active_config, - neg_control_suggestions, pos_control_suggestions, + investigation, + cost, + self._active_config, + neg_control_suggestions, + pos_control_suggestions, self._dispatcher, ): - if isinstance(event, dict) and "__phase_result__" in event: + if isinstance(event, dict): validation_metrics = event["__phase_result__"]["validation_metrics"] else: yield event # 6. Director synthesis async for event in run_synthesis_phase( - investigation, cost, self._active_config, - validation_metrics, self._mcp_bridge, + investigation, + cost, + self._active_config, + validation_metrics, + self._mcp_bridge, self._director_call, ): yield event diff --git a/server/src/ehrlich/investigation/application/phase_runner.py b/server/src/ehrlich/investigation/application/phase_runner.py index 3b15112..7ede185 100644 --- a/server/src/ehrlich/investigation/application/phase_runner.py +++ b/server/src/ehrlich/investigation/application/phase_runner.py @@ -123,9 +123,7 @@ async def run_classification_phase( valid_categories: frozenset[str] = frozenset() if domain_registry: valid_categories = domain_registry.all_categories() - pico_prompt = build_pico_and_classification_prompt( - valid_categories, uploaded_data_context - ) + pico_prompt = build_pico_and_classification_prompt(valid_categories, uploaded_data_context) pico_response = await summarizer.create_message( system=pico_prompt, messages=[{"role": "user", "content": investigation.prompt}], @@ -163,9 +161,7 @@ async def run_classification_phase( if domain_registry: detected_configs, is_fallback = domain_registry.detect(domain_categories) active_config = merge_domain_configs(detected_configs) - researcher_prompt = build_researcher_prompt( - active_config, uploaded_data_context - ) + researcher_prompt = build_researcher_prompt(active_config, uploaded_data_context) yield DomainDetected( domain=active_config.name, display_config=active_config.to_display_dict(), @@ -235,8 +231,7 @@ async def run_formulation_phase( formulation_prompt, f"Research prompt: {investigation.prompt}\n\n" f"{literature_context}\n\n" - f"Formulate 2-4 testable hypotheses and identify negative controls." - + prior_section, + f"Formulate 2-4 testable hypotheses and identify negative controls." + prior_section, investigation.id, output_config=_build_output_config(FORMULATION_SCHEMA), ): @@ -344,9 +339,7 @@ async def run_hypothesis_testing_phase( ) tested = 0 while tested < max_hypotheses: - proposed = [ - h for h in investigation.hypotheses if h.status == HypothesisStatus.PROPOSED - ] + proposed = [h for h in investigation.hypotheses if h.status == HypothesisStatus.PROPOSED] if not proposed: break @@ -362,9 +355,7 @@ async def run_hypothesis_testing_phase( # Director designs experiment if active_config: - tools_csv = ", ".join( - registry.list_tools_for_domain(active_config.tool_tags) - ) + tools_csv = ", ".join(registry.list_tools_for_domain(active_config.tool_tags)) else: tools_csv = ", ".join(registry.list_tools()) experiment_prompt = ( @@ -606,11 +597,7 @@ async def run_controls_phase( nc.get("identifier", nc.get("smiles", "")) for nc in neg_control_suggestions if nc.get("identifier", nc.get("smiles", "")) - ] + [ - pc.get("identifier", "") - for pc in pos_control_suggestions - if pc.get("identifier", "") - ] + ] + [pc.get("identifier", "") for pc in pos_control_suggestions if pc.get("identifier", "")] if all_identifiers: try: pred_result = await dispatcher.dispatch( @@ -764,9 +751,7 @@ async def run_synthesis_phase( ) synthesis_prompt = ( - build_synthesis_prompt(active_config) - if active_config - else DIRECTOR_SYNTHESIS_PROMPT + build_synthesis_prompt(active_config) if active_config else DIRECTOR_SYNTHESIS_PROMPT ) result = _DirectorResult(data={}, thinking="") async for director_event in director_call( @@ -806,9 +791,7 @@ async def run_synthesis_phase( rank=c.get("rank", i + 1), priority=c.get("priority", 0), scores={ - k: float(v) - for k, v in c.get("scores", {}).items() - if isinstance(v, (int, float)) + k: float(v) for k, v in c.get("scores", {}).items() if isinstance(v, (int, float)) }, attributes={k: str(v) for k, v in c.get("attributes", {}).items()}, ) diff --git a/server/src/ehrlich/investigation/application/registry_factory.py b/server/src/ehrlich/investigation/application/registry_factory.py index 009e8c3..ddbe6d0 100644 --- a/server/src/ehrlich/investigation/application/registry_factory.py +++ b/server/src/ehrlich/investigation/application/registry_factory.py @@ -7,6 +7,10 @@ import os from functools import lru_cache +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable from ehrlich.analysis.tools import ( analyze_substructures, @@ -144,7 +148,7 @@ def build_tool_registry() -> ToolRegistry: _impact_viz = frozenset({"impact", "visualization"}) _causal_viz = frozenset({"causal", "visualization"}) - tagged_tools = [ + tagged_tools: list[tuple[str, Callable[..., Any], frozenset[str] | None]] = [ # Chemistry (6) ("validate_smiles", validate_smiles, _chem), ("compute_descriptors", compute_descriptors, _chem), diff --git a/server/src/ehrlich/investigation/application/researcher_executor.py b/server/src/ehrlich/investigation/application/researcher_executor.py index 13f7e4a..747f2d3 100644 --- a/server/src/ehrlich/investigation/application/researcher_executor.py +++ b/server/src/ehrlich/investigation/application/researcher_executor.py @@ -315,9 +315,7 @@ async def run_researcher_experiment( identifier=tool_input.get("identifier", tool_input.get("smiles", "")), identifier_type=tool_input.get("identifier_type", ""), name=tool_input.get("name", ""), - score=float( - tool_input.get("score", tool_input.get("prediction_score", 0.0)) - ), + score=float(tool_input.get("score", tool_input.get("prediction_score", 0.0))), threshold=float(tool_input.get("threshold", 0.5)), source=tool_input.get("source", ""), ) From aae5f3b6503e7489ff881fc8bb5976a5e8559135 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 11:03:35 -0700 Subject: [PATCH 20/35] docs(web): align landing page with COSS identity and fix stale counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reflect the COSS (Commercial Open-Source Software) positioning established in README.md across all landing page components: Hero subtitle, OpenSource section, pricing framing as hosted-instance-only, differentiators, personas, and SEO meta tags. Fix stale tool/data-source/domain counts throughout (67/70/84 → 85 tools, 15/16 → 19 sources, 3 → 4 domains, FTS5 → tsvector). --- web/src/components/CTA.tsx | 6 ++---- web/src/components/Hero.tsx | 4 ++++ web/src/components/OpenSource.tsx | 10 +++------- web/src/components/WhoItsFor.tsx | 4 ++-- web/src/lib/constants.ts | 24 ++++++++++++------------ web/src/routes/__root.tsx | 4 ++-- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/web/src/components/CTA.tsx b/web/src/components/CTA.tsx index d8a9fd9..4efc978 100644 --- a/web/src/components/CTA.tsx +++ b/web/src/components/CTA.tsx @@ -40,12 +40,10 @@ export function CTA() {

- Same product at every tier. + Hosted instance pricing.

- No feature gates. All 70 tools, all 16 data sources, full 6-phase methodology - at every level. The only variable is the Director model quality: Haiku reasons - adequately, Sonnet reasons well, Opus reasons deeply. + Self-hosting is free with your own API key. The hosted instance uses credits to cover Anthropic API costs. No feature gates -- all 85 tools, all 19 data sources, full 6-phase methodology at every level. The only variable is the Director model quality.

diff --git a/web/src/components/Hero.tsx b/web/src/components/Hero.tsx index d818645..3606742 100644 --- a/web/src/components/Hero.tsx +++ b/web/src/components/Hero.tsx @@ -37,6 +37,10 @@ export function Hero() { tools across {STATS.domains} scientific domains. Grounded in Popper, Fisher, and GRADE.

+

+ Open source (AGPL-3.0) · Self-hostable · COSS +

+
{STATS.tools} tools diff --git a/web/src/components/OpenSource.tsx b/web/src/components/OpenSource.tsx index 5782a3e..32267c1 100644 --- a/web/src/components/OpenSource.tsx +++ b/web/src/components/OpenSource.tsx @@ -37,9 +37,7 @@ export function OpenSource() {

- AGPL-3.0 dual licensed. Use it free for research and teaching. - Purchase a commercial license for private modifications. - Self-host, extend, contribute. + Ehrlich is COSS -- Commercial Open-Source Software. The same model used by Supabase, PostHog, Cal.com, and GitLab. The entire codebase is open source under AGPL-3.0. There is no proprietary version.

@@ -84,12 +82,10 @@ export function OpenSource() {

- 67 Tools, Domain-Agnostic + 85 Tools, 4 Domains

- Chemistry, literature, analysis, prediction, simulation, training, - nutrition, visualization, and statistics. Each tagged with domain - metadata for automatic filtering. Add a{" "} + Molecular, training, nutrition, and impact evaluation. Each domain brings its own tools, scoring, and visualization. Add a{" "} DomainConfig{" "} and the engine handles the rest.

diff --git a/web/src/components/WhoItsFor.tsx b/web/src/components/WhoItsFor.tsx index 0618cc0..556b352 100644 --- a/web/src/components/WhoItsFor.tsx +++ b/web/src/components/WhoItsFor.tsx @@ -16,7 +16,7 @@ const PERSONAS = [ { label: "Industry / Pharma", usage: "BYOK. Your Anthropic key, our methodology + tools.", - description: "67 computational tools, 15 external APIs, structured reporting. Commercial license for private modifications. Self-hosted deployment available.", + description: "85 computational tools, 18 external APIs, structured reporting. Commercial license for private modifications. Self-host or use the hosted instance with your own Anthropic key.", }, ] as const; @@ -43,7 +43,7 @@ export function WhoItsFor() { Same product at every level.

- All 70 tools, all 16 data sources, and the full 6-phase methodology at every tier. + All 85 tools, all 19 data sources, and the full 6-phase methodology at every tier. The only variable is the Director model quality.

diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index dddd343..11569f0 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -46,7 +46,7 @@ export const EXAMPLE_PROMPTS = [ export const DIFFERENTIATORS = [ { label: "Real Computation", - tagline: "84 tools that compute, not summarize.", + tagline: "85 tools that compute, not summarize.", description: "Ehrlich trains ML models, runs molecular docking, executes statistical tests, and validates with negative controls. Every tool returns structured data from real computation or real APIs.", capabilities: [ @@ -58,16 +58,16 @@ export const DIFFERENTIATORS = [ ], }, { - label: "Accessible Pricing", - tagline: "$2-10 per investigation. Free tier included.", + label: "Open Source, Self-Hostable", + tagline: "COSS. Same code, two paths.", description: - "Full 6-phase methodology, all 84 tools, all 19 data sources at every tier. No feature gates. A student in Mexico and a pharma lab in Boston use the same product. The only variable is model quality.", + "Self-host with your own API key for free -- no limits, no credits, no account. Or use the hosted instance where credits cover Anthropic API costs. A student in Mexico and a pharma company in Boston get the same 85 tools, the same 19 data sources, the same methodology.", capabilities: [ - "4 domains, extensible to any", - "Free: 3 Haiku investigations/month", + "Self-host: clone, bring your API key, no limits", + "Hosted: credits cover Anthropic costs (Opus is expensive)", "Credits: Haiku (1), Sonnet (3), Opus (5)", - "Enterprise: bring your own Anthropic key", - "AGPL-3.0: self-host, modify, extend", + "AGPL-3.0: inspect, modify, extend, contribute", + "Commercial license for private modifications", ], }, { @@ -201,7 +201,7 @@ export const METHODOLOGY_PHASES = [ number: "04", label: "Experiment Execution", foundation: "Fisher (1935)", - description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 84 tools across 4 domains.", + description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 85 tools across 4 domains.", }, { number: "05", @@ -311,12 +311,12 @@ export const PRICING_TIERS = [ price: "$0", period: "forever", credits: "3 Haiku investigations/month", - description: "Full methodology. All 84 tools. All 19 data sources. Findings indexed for future research.", + description: "Full methodology. All 85 tools. All 19 data sources. Findings indexed for future research.", features: [ "Full 6-phase scientific methodology", - "All 84 tools, all 19 data sources, 4 domains", + "All 85 tools, all 19 data sources, 4 domains", "Full audit trail and report", - "Self-referential search (FTS5)", + "Self-referential search (tsvector)", "No feature gates", ], cta: "Start Free", diff --git a/web/src/routes/__root.tsx b/web/src/routes/__root.tsx index c03582f..af6dc80 100644 --- a/web/src/routes/__root.tsx +++ b/web/src/routes/__root.tsx @@ -17,14 +17,14 @@ export const Route = createRootRoute({ { name: "description", content: - "Formulate hypotheses, design experiments, test against 15 real databases, grade evidence. 67 computational tools across molecular, training, and nutrition science. Not a chatbot. Not a wrapper. Open source, AGPL-3.0.", + "Formulate hypotheses, design experiments, test against 19 real databases, grade evidence. 85 computational tools across 4 scientific domains. Open source (AGPL-3.0), self-hostable.", }, { name: "theme-color", content: "#1a2a1a" }, { property: "og:title", content: "Ehrlich — Scientific Methodology, Automated" }, { property: "og:description", content: - "67 computational tools. 16 data sources. 3 scientific domains. Hypothesis-driven reasoning grounded in Popper, Fisher, and GRADE. Free tier available.", + "85 computational tools. 19 data sources. 4 scientific domains. Hypothesis-driven reasoning grounded in Popper, Fisher, and GRADE. Open source, self-hostable.", }, { property: "og:type", content: "website" }, { name: "twitter:card", content: "summary_large_image" }, From 72a38e73a806a04ff75a31c003151e0d184c62a9 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 15:59:45 -0700 Subject: [PATCH 21/35] feat(14F-2): add paper view route with print-optimized PDF export Dedicated /paper/:id route renders 8 Markdown sections + numbered visualization figures via VizRegistry. Browser-native PDF export via window.print() with @media print CSS -- zero new dependencies. Recharts/Visx SVG charts captured at full vector quality. --- CLAUDE.md | 5 +- README.md | 1 + .../components/InvestigationReport.tsx | 63 +- console/src/main.tsx | 1 + console/src/routeTree.gen.ts | 21 + console/src/routes/__root.tsx | 2 +- console/src/routes/investigation.$id.tsx | 1 + console/src/routes/paper.$id.tsx | 162 +++++ console/src/styles/print.css | 91 +++ docs/architecture.md | 3 +- docs/roadmap/README.md | 9 +- docs/roadmap/phase-14-engine-hardening.md | 45 +- .../src/ehrlich/api/routes/investigation.py | 105 ++++ .../application/paper_generator.py | 544 +++++++++++++++++ .../investigation/test_paper_generator.py | 555 ++++++++++++++++++ 15 files changed, 1563 insertions(+), 45 deletions(-) create mode 100644 console/src/routes/paper.$id.tsx create mode 100644 console/src/styles/print.css create mode 100644 server/src/ehrlich/investigation/application/paper_generator.py create mode 100644 server/tests/investigation/test_paper_generator.py diff --git a/CLAUDE.md b/CLAUDE.md index 48085eb..725abc2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -173,6 +173,7 @@ All paths relative to `server/src/ehrlich/`. | `investigation/application/tool_dispatcher.py` | Tool execution with caching, `search_prior_research` and `query_uploaded_data` interception | | `investigation/application/researcher_executor.py` | Single researcher experiment executor | | `investigation/application/batch_executor.py` | Parallel batch executor (2 concurrent) | +| `investigation/application/paper_generator.py` | Scientific paper generation from investigation data + events | | `investigation/application/prompts.py` | All domain-adaptive prompts | | `investigation/domain/schemas.py` | 6 JSON schemas for structured outputs | | `investigation/domain/domain_config.py` | `DomainConfig` + `ScoreDefinition` + `merge_domain_configs()` | @@ -188,7 +189,7 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| | `auth.py` | WorkOS JWT middleware (JWKS, header + query param) | -| `routes/investigation.py` | REST + SSE, 85-tool registry, domain registry, credits, BYOK | +| `routes/investigation.py` | REST + SSE + paper export, 85-tool registry, domain registry, credits, BYOK | | `routes/upload.py` | File upload (CSV/XLSX/PDF) | | `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets | | `sse.py` | Domain event to SSE conversion (20 types) | @@ -213,6 +214,8 @@ All paths relative to `console/src/`. | `features/visualization/VizRegistry.tsx` | Lazy-loaded chart component registry | | `features/visualization/VisualizationPanel.tsx` | Grid layout for multiple charts | | `features/visualization/theme.ts` | OKLCH color tokens | +| `routes/paper.$id.tsx` | Print-optimized paper view with PDF export | +| `styles/print.css` | `@media print` stylesheet for PDF export | | `shared/hooks/use-auth.ts` | WorkOS auth wrapper | | `shared/lib/api.ts` | Authenticated fetch with BYOK support | diff --git a/README.md b/README.md index 5ee5844..41ebfe4 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,7 @@ Server at :8000, Console at :3000. | POST | `/api/v1/investigate` | Create new investigation (`director_tier`: haiku/sonnet/opus) | | GET | `/api/v1/investigate/{id}` | Full investigation detail (owner only) | | GET | `/api/v1/investigate/{id}/stream` | SSE stream of investigation events (owner only, supports `?token=`) | +| GET | `/api/v1/investigate/{id}/paper` | Structured scientific paper + visualizations from completed investigation (owner only). PDF via `/paper/:id` route + browser print | | POST | `/api/v1/investigate/{id}/approve` | Approve/reject formulated hypotheses (owner only) | | POST | `/api/v1/upload` | Upload file (CSV/XLSX/PDF) for investigation data, returns preview | | GET | `/api/v1/credits/balance` | Current credit balance + BYOK status | diff --git a/console/src/features/investigation/components/InvestigationReport.tsx b/console/src/features/investigation/components/InvestigationReport.tsx index 14c9fed..e279cc4 100644 --- a/console/src/features/investigation/components/InvestigationReport.tsx +++ b/console/src/features/investigation/components/InvestigationReport.tsx @@ -1,13 +1,18 @@ +import { useState } from "react"; import Markdown from "react-markdown"; +import { Link } from "@tanstack/react-router"; import { BookOpen, Download, + FileText, FlaskConical, + Loader2, Target, TestTube, Zap, } from "lucide-react"; -import { generateMarkdown, downloadMarkdown } from "../lib/export-markdown"; +import { downloadMarkdown } from "../lib/export-markdown"; +import { apiFetch } from "@/shared/lib/api"; import { cn } from "@/shared/lib/utils"; import { SectionHeader } from "@/shared/components/ui/SectionHeader"; import type { @@ -27,6 +32,7 @@ import { NegativeControlPanel } from "./NegativeControlPanel"; import { ThreatAssessment } from "./ThreatAssessment"; interface InvestigationReportProps { + investigationId: string; prompt: string; summary: string; hypotheses: Hypothesis[]; @@ -48,6 +54,7 @@ const STATUS_COLORS: Record = { }; export function InvestigationReport({ + investigationId, prompt, summary, hypotheses, @@ -59,21 +66,20 @@ export function InvestigationReport({ domainConfig, validationMetrics, }: InvestigationReportProps) { - function handleExport() { - const md = generateMarkdown({ - prompt, - summary, - hypotheses, - experiments, - findings, - candidates, - negativeControls, - cost, - validationMetrics, - }); - const slug = prompt.slice(0, 40).replace(/[^a-zA-Z0-9]+/g, "-").toLowerCase(); - const ts = new Date().toISOString().slice(0, 10); - downloadMarkdown(md, `${slug}_${ts}_report.md`); + const [exporting, setExporting] = useState(false); + + async function handleExport() { + setExporting(true); + try { + const paper = await apiFetch>( + `/investigate/${investigationId}/paper`, + ); + const slug = prompt.slice(0, 40).replace(/[^a-zA-Z0-9]+/g, "-").toLowerCase(); + const ts = new Date().toISOString().slice(0, 10); + downloadMarkdown(paper.full_markdown, `${slug}_${ts}_paper.md`); + } finally { + setExporting(false); + } } return ( @@ -82,13 +88,24 @@ export function InvestigationReport({

Investigation Report

- +
+ + + View Paper + + +
{/* 1. Research Question */} diff --git a/console/src/main.tsx b/console/src/main.tsx index 201458a..e3acc66 100644 --- a/console/src/main.tsx +++ b/console/src/main.tsx @@ -7,6 +7,7 @@ import { useAuth } from "@/shared/hooks/use-auth"; import { setTokenProvider } from "@/shared/lib/api"; import { routeTree } from "./routeTree.gen"; import "./styles/globals.css"; +import "./styles/print.css"; const WORKOS_CLIENT_ID = import.meta.env.VITE_WORKOS_CLIENT_ID; const WORKOS_REDIRECT_URI = import.meta.env.VITE_WORKOS_REDIRECT_URI; diff --git a/console/src/routeTree.gen.ts b/console/src/routeTree.gen.ts index abc16be..250f4e5 100644 --- a/console/src/routeTree.gen.ts +++ b/console/src/routeTree.gen.ts @@ -12,6 +12,7 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as MethodologyRouteImport } from './routes/methodology' import { Route as CallbackRouteImport } from './routes/callback' import { Route as IndexRouteImport } from './routes/index' +import { Route as PaperIdRouteImport } from './routes/paper.$id' import { Route as InvestigationIdRouteImport } from './routes/investigation.$id' import { Route as CompareId1Id2RouteImport } from './routes/compare.$id1.$id2' @@ -30,6 +31,11 @@ const IndexRoute = IndexRouteImport.update({ path: '/', getParentRoute: () => rootRouteImport, } as any) +const PaperIdRoute = PaperIdRouteImport.update({ + id: '/paper/$id', + path: '/paper/$id', + getParentRoute: () => rootRouteImport, +} as any) const InvestigationIdRoute = InvestigationIdRouteImport.update({ id: '/investigation/$id', path: '/investigation/$id', @@ -46,6 +52,7 @@ export interface FileRoutesByFullPath { '/callback': typeof CallbackRoute '/methodology': typeof MethodologyRoute '/investigation/$id': typeof InvestigationIdRoute + '/paper/$id': typeof PaperIdRoute '/compare/$id1/$id2': typeof CompareId1Id2Route } export interface FileRoutesByTo { @@ -53,6 +60,7 @@ export interface FileRoutesByTo { '/callback': typeof CallbackRoute '/methodology': typeof MethodologyRoute '/investigation/$id': typeof InvestigationIdRoute + '/paper/$id': typeof PaperIdRoute '/compare/$id1/$id2': typeof CompareId1Id2Route } export interface FileRoutesById { @@ -61,6 +69,7 @@ export interface FileRoutesById { '/callback': typeof CallbackRoute '/methodology': typeof MethodologyRoute '/investigation/$id': typeof InvestigationIdRoute + '/paper/$id': typeof PaperIdRoute '/compare/$id1/$id2': typeof CompareId1Id2Route } export interface FileRouteTypes { @@ -70,6 +79,7 @@ export interface FileRouteTypes { | '/callback' | '/methodology' | '/investigation/$id' + | '/paper/$id' | '/compare/$id1/$id2' fileRoutesByTo: FileRoutesByTo to: @@ -77,6 +87,7 @@ export interface FileRouteTypes { | '/callback' | '/methodology' | '/investigation/$id' + | '/paper/$id' | '/compare/$id1/$id2' id: | '__root__' @@ -84,6 +95,7 @@ export interface FileRouteTypes { | '/callback' | '/methodology' | '/investigation/$id' + | '/paper/$id' | '/compare/$id1/$id2' fileRoutesById: FileRoutesById } @@ -92,6 +104,7 @@ export interface RootRouteChildren { CallbackRoute: typeof CallbackRoute MethodologyRoute: typeof MethodologyRoute InvestigationIdRoute: typeof InvestigationIdRoute + PaperIdRoute: typeof PaperIdRoute CompareId1Id2Route: typeof CompareId1Id2Route } @@ -118,6 +131,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof IndexRouteImport parentRoute: typeof rootRouteImport } + '/paper/$id': { + id: '/paper/$id' + path: '/paper/$id' + fullPath: '/paper/$id' + preLoaderRoute: typeof PaperIdRouteImport + parentRoute: typeof rootRouteImport + } '/investigation/$id': { id: '/investigation/$id' path: '/investigation/$id' @@ -140,6 +160,7 @@ const rootRouteChildren: RootRouteChildren = { CallbackRoute: CallbackRoute, MethodologyRoute: MethodologyRoute, InvestigationIdRoute: InvestigationIdRoute, + PaperIdRoute: PaperIdRoute, CompareId1Id2Route: CompareId1Id2Route, } export const routeTree = rootRouteImport diff --git a/console/src/routes/__root.tsx b/console/src/routes/__root.tsx index 7d42c8b..e3d3e10 100644 --- a/console/src/routes/__root.tsx +++ b/console/src/routes/__root.tsx @@ -1,7 +1,7 @@ import { createRootRoute, Outlet, useMatches } from "@tanstack/react-router"; import { AppLayout } from "@/shared/components/layout/AppLayout"; -const HEADERLESS_ROUTES = ["/investigation/", "/compare/"]; +const HEADERLESS_ROUTES = ["/investigation/", "/compare/", "/paper/"]; export const Route = createRootRoute({ component: RootComponent, diff --git a/console/src/routes/investigation.$id.tsx b/console/src/routes/investigation.$id.tsx index a978631..a75b142 100644 --- a/console/src/routes/investigation.$id.tsx +++ b/console/src/routes/investigation.$id.tsx @@ -283,6 +283,7 @@ function InvestigationPage() { {completed && ( <> (null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + setLoading(true); + apiFetch(`/investigate/${id}/paper`) + .then((data) => { + if (!cancelled) setPaper(data); + }) + .catch((err) => { + if (!cancelled) setError(err.message ?? "Failed to load paper"); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + return () => { + cancelled = true; + }; + }, [id]); + + if (loading) { + return ( +
+ +
+ ); + } + + if (error || !paper) { + return ( +
+

{error ?? "Paper not found"}

+ + Back to investigation + +
+ ); + } + + return ( +
+ {/* Toolbar -- hidden when printing */} +
+
+ + + Back to investigation + + +
+
+ + {/* Paper content */} +
+
+ {SECTION_ORDER.map((key) => { + const content = paper[key]; + if (!content) return null; + return ( +
+
+ {content} +
+
+ ); + })} + + {/* Figures: rendered visualizations */} + {paper.visualizations.length > 0 && ( +
+

Figures

+ {paper.visualizations.map((viz, i) => ( + + ))} +
+ )} +
+
+
+ ); +} + +function VizFigure({ payload, index }: { payload: VizPayload; index: number }) { + const Component = getVizComponent(payload.viz_type); + if (!Component) return null; + + return ( +
+
+ + +
+ } + > + + +
+
+ Figure {index}. {payload.title} +
+ + ); +} diff --git a/console/src/styles/print.css b/console/src/styles/print.css new file mode 100644 index 0000000..fba5d7c --- /dev/null +++ b/console/src/styles/print.css @@ -0,0 +1,91 @@ +@media print { + /* Hide non-print elements */ + .no-print { + display: none !important; + } + + /* Reset to clean white background */ + body { + background: white !important; + color: black !important; + font-family: Georgia, "Times New Roman", serif; + font-size: 11pt; + line-height: 1.5; + -webkit-print-color-adjust: exact; + print-color-adjust: exact; + } + + /* Remove dark theme surface styling */ + * { + background: transparent !important; + color: black !important; + border-color: #ccc !important; + box-shadow: none !important; + } + + /* Preserve SVG chart colors */ + svg, + svg * { + color: inherit !important; + print-color-adjust: exact; + -webkit-print-color-adjust: exact; + } + + svg [fill]:not([fill="none"]) { + background: unset !important; + } + + /* Page setup */ + @page { + margin: 2cm; + size: A4; + } + + /* Page break hints */ + h2, + h3 { + break-after: avoid; + } + + figure, + table, + .viz-figure { + break-inside: avoid; + } + + section { + break-before: auto; + } + + /* Monospace code blocks */ + code, + pre { + font-family: "Courier New", monospace !important; + font-size: 9pt; + } + + /* Links: show URL */ + a[href]::after { + content: " (" attr(href) ")"; + font-size: 9pt; + color: #666 !important; + } + + /* Tables */ + table { + border-collapse: collapse; + width: 100%; + font-size: 10pt; + } + + th, + td { + border: 1px solid #ccc !important; + padding: 4px 8px; + text-align: left; + } + + th { + font-weight: bold; + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 2e34fca..5fa3399 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -258,4 +258,5 @@ api/ -> investigation/application/ only - `ValidationMetricsComputed` carries Z'-factor, quality, control separation stats - `InvestigationCompleted` includes candidates, hypotheses, findings, negative controls, validation metrics 12. Investigation persisted to PostgreSQL with full state + events for timeline replay -13. Console displays: phase indicator, hypothesis board, lab view (3Dmol.js, molecular domain only), investigation diagram (React Flow), domain-specific visualizations (charts, diagrams), findings with source badges, dynamic candidate table with domain-specific score columns, structured 8-section report with markdown export +13. Console displays: phase indicator, hypothesis board, lab view (3Dmol.js, molecular domain only), investigation diagram (React Flow), domain-specific visualizations (charts, diagrams), findings with source badges, dynamic candidate table with domain-specific score columns, structured 8-section report +14. **Paper export** -- `GET /investigate/{id}/paper` generates a structured scientific paper from investigation data + persisted events (PICO, literature survey, experiment methodology, evaluation reasoning, validation metrics). Returns 8 sections (Title, Abstract, Introduction, Methods, Results, Discussion, References, Supplementary) as JSON with `full_markdown` combined document + `visualizations` array. Console offers two export paths: "View Paper" (`/paper/:id` route) renders all sections with numbered figures from VizRegistry, with print-optimized CSS for browser-native PDF export; "Export Markdown" downloads the raw Markdown text diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index 8238a8f..dc232b1 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -88,11 +88,10 @@ Enhancement Enhancement Sports 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) DONE -> 13C (Mexico) -> 13D (US) -> 13E (MCP) | Phase 14: Scientific Engine Hardening -- IN PROGRESS (parallel with 13A-2) - 14A (Orchestrator Split) ──┬── 14B (Route Cleanup) ──┬── 14D (Tool Enrichment) - ├── 14C (Prompt Split) ├── 14E (Tree Search) - └───────────────────────── └── 14F (Paper Generator) - | - 14G (Validation Runs) + 14A (Orchestrator Split) DONE ──┬── 14B (Route Cleanup) DONE ──┬── 14D (Tool Enrichment) DONE + ├── 14C (Prompt Split) DONE ├── 14E (Tree Search) + ├── 14H (Upload Validation) DONE├── 14F (Paper Generator) DONE -> 14F-2 (Paper View + PDF) DONE + └────────────────────────────── └── 14G (Validation Runs) | Demo + Video -- TODO | diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md index 2f24c33..06b1c72 100644 --- a/docs/roadmap/phase-14-engine-hardening.md +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -74,23 +74,38 @@ Replace linear hypothesis testing with tree-based exploration inspired by AI Sci - [ ] `HypothesisTreeEvent` SSE event for frontend tree rendering - [ ] Console: `InvestigationDiagram.tsx` already renders React Flow graph -- tree structure renders naturally as growing/pruning graph -## 14F: Scientific Paper Generator -- TODO +## 14F: Scientific Paper Generator -- DONE -Generate structured scientific papers from completed investigations. Not AI-generated prose -- structured assembly of existing investigation data into publication format. +Generate structured scientific papers from completed investigations. Structured assembly of investigation data + persisted events into publication-format Markdown. -- [ ] `application/paper_generator.py` -- takes `Investigation` entity, produces structured sections: - - Title (from research question) +- [x] `application/paper_generator.py` -- pure function, takes investigation data + events, produces 8-section structured paper: + - Title (from research question, domain, date, ID) - Abstract (Director synthesis summary) - - Introduction (PICO decomposition + literature survey context) - - Methods (experiment designs, tool descriptions, statistical approach) - - Results (findings with statistical tests, visualizations as base64 SVG) - - Discussion (hypothesis assessments, GRADE certainty, limitations, knowledge gaps) - - References (collected DOIs + citations) - - Supplementary (cost breakdown, model info, negative controls, tool call trace) -- [ ] `GET /investigate/{id}/paper` endpoint -- returns structured paper as JSON or Markdown -- [ ] `GET /investigate/{id}/paper?format=pdf` -- server-side PDF generation (optional, `weasyprint` or `reportlab`) -- [ ] Console: "Export Paper" button on completed investigation page -- [ ] Console: paper preview with section navigation + - Introduction (PICO decomposition + literature survey from events) + - Methods (experiment designs with variables/controls/analysis plans, tool usage counts from events) + - Results (hypothesis outcomes table, findings grouped by evidence type with provenance, candidate ranking table) + - Discussion (hypothesis assessments with evaluation reasoning from events, supporting/contradicting evidence) + - References (citations + unique data source provenance from findings) + - Supplementary (negative/positive controls, Z'-factor validation metrics from events, cost breakdown by model) +- [x] `GET /investigate/{id}/paper` endpoint -- returns JSON with section keys + `full_markdown` combined document. Auth + ownership verification, only for COMPLETED investigations +- [x] Console: "Export Paper" button replaces "Export Report" on completed investigations, fetches server-side paper with loading state, downloads as Markdown +- [x] 37 unit tests covering all 8 sections, event extraction, empty data, malformed events + +Result: `paper_generator.py` (285 lines), API endpoint (65 lines), console updates (InvestigationReport.tsx). 908 server tests passing, 138 console tests passing. + +## 14F-2: Paper View + PDF Export -- DONE + +Dedicated paper view route with print-optimized CSS for browser-native PDF export. Zero new dependencies -- Recharts/Visx charts render as SVG, captured at full vector quality by browser print. + +- [x] `extract_visualizations(events)` in `paper_generator.py` -- extracts visualization payloads from persisted events +- [x] `GET /investigate/{id}/paper` response updated -- includes `visualizations` key with `[{viz_type, title, data, config}]` +- [x] `routes/paper.$id.tsx` -- new paper view route at `/paper/:id`, renders 8 Markdown sections + numbered figures via VizRegistry +- [x] `styles/print.css` -- `@media print` stylesheet: white bg, serif body, SVG color preservation, page break hints, hidden toolbar +- [x] InvestigationReport: "View Paper" link + "Export Markdown" (renamed from "Export Paper") +- [x] `/paper/` added to `HEADERLESS_ROUTES` in `__root__.tsx` +- [x] 5 unit tests for `extract_visualizations()` (extraction, preservation, empty, no-viz, malformed) + +Result: 42 paper generator tests passing, 138 console tests passing. Zero ruff/mypy/tsc violations. ## 14G: Validation Runs -- TODO @@ -127,6 +142,8 @@ Phase 14E (Agentic Tree Search) -- after 14A (modifies orchestrator) | Phase 14F (Paper Generator) -- after 14A (reads investigation data) | +Phase 14F-2 (Paper View + PDF Export) -- after 14F (needs paper endpoint + viz events) + | Phase 14G (Validation Runs) -- after 14E + 14F (needs tree search + paper gen) ``` diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index 0d8f8e7..3b6bcb9 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -25,6 +25,7 @@ from ehrlich.api.sse import SSEEventType, domain_event_to_sse from ehrlich.config import get_settings from ehrlich.investigation.application.orchestrator_factory import create_orchestrator +from ehrlich.investigation.application.paper_generator import extract_visualizations, generate_paper from ehrlich.investigation.application.registry_factory import ( build_domain_registry, build_mcp_configs, @@ -163,6 +164,110 @@ async def get_investigation( return to_detail(investigation) +@router.get("/investigate/{investigation_id}/paper") +async def get_paper( + investigation_id: str, + user: dict[str, Any] = _require_user, +) -> dict[str, Any]: + """Generate a structured scientific paper from a completed investigation.""" + repo = _get_repository() + investigation = await repo.get_by_id(investigation_id) + if investigation is None: + raise HTTPException(status_code=404, detail="Investigation not found") + await _verify_ownership(investigation_id, user, repo) + if investigation.status != InvestigationStatus.COMPLETED: + raise HTTPException(status_code=409, detail="Investigation not completed") + + events = await repo.get_events(investigation_id) + paper: dict[str, Any] = generate_paper( + investigation_id=investigation.id, + prompt=investigation.prompt, + summary=investigation.summary, + domain=investigation.domain, + created_at=investigation.created_at.isoformat(), + hypotheses=[ + { + "id": h.id, + "statement": h.statement, + "rationale": h.rationale, + "status": h.status.value, + "confidence": h.confidence, + "certainty_of_evidence": h.certainty_of_evidence, + "supporting_evidence": h.supporting_evidence, + "contradicting_evidence": h.contradicting_evidence, + } + for h in investigation.hypotheses + ], + experiments=[ + { + "id": e.id, + "hypothesis_id": e.hypothesis_id, + "description": e.description, + "tool_plan": e.tool_plan, + "status": e.status.value, + "independent_variable": e.independent_variable, + "dependent_variable": e.dependent_variable, + "controls": e.controls, + "confounders": e.confounders, + "analysis_plan": e.analysis_plan, + "success_criteria": e.success_criteria, + "failure_criteria": e.failure_criteria, + } + for e in investigation.experiments + ], + findings=[ + { + "title": f.title, + "detail": f.detail, + "hypothesis_id": f.hypothesis_id, + "evidence_type": f.evidence_type, + "source_type": f.source_type, + "source_id": f.source_id, + } + for f in investigation.findings + ], + candidates=[ + { + "identifier": c.identifier, + "identifier_type": c.identifier_type, + "name": c.name, + "rank": c.rank, + "notes": c.notes, + "scores": c.scores, + "attributes": c.attributes, + } + for c in investigation.candidates + ], + negative_controls=[ + { + "identifier": nc.identifier, + "name": nc.name, + "score": nc.score, + "threshold": nc.threshold, + "correctly_classified": nc.correctly_classified, + "source": nc.source, + } + for nc in investigation.negative_controls + ], + positive_controls=[ + { + "identifier": pc.identifier, + "name": pc.name, + "known_activity": pc.known_activity, + "score": pc.score, + "correctly_classified": pc.correctly_classified, + "source": pc.source, + } + for pc in investigation.positive_controls + ], + citations=investigation.citations, + cost_data=investigation.cost_data, + events=events, + ) + paper["visualizations"] = extract_visualizations(events) + return paper + + @router.post("/investigate") async def start_investigation( request: InvestigateRequest, diff --git a/server/src/ehrlich/investigation/application/paper_generator.py b/server/src/ehrlich/investigation/application/paper_generator.py new file mode 100644 index 0000000..3717107 --- /dev/null +++ b/server/src/ehrlich/investigation/application/paper_generator.py @@ -0,0 +1,544 @@ +"""Generate structured scientific papers from completed investigations. + +Assembles investigation data + persisted events into a publication-format paper. +Pure function -- no I/O, no external dependencies. +""" + +from __future__ import annotations + +import json +from typing import Any + + +def generate_paper( + investigation_id: str, + prompt: str, + summary: str, + domain: str, + created_at: str, + hypotheses: list[dict[str, Any]], + experiments: list[dict[str, Any]], + findings: list[dict[str, Any]], + candidates: list[dict[str, Any]], + negative_controls: list[dict[str, Any]], + positive_controls: list[dict[str, Any]], + citations: list[str], + cost_data: dict[str, Any], + events: list[dict[str, str]], +) -> dict[str, str]: + """Generate a structured scientific paper as a dict of Markdown sections. + + Each key is a section name, each value is Markdown content. + The ``full_markdown`` key contains all sections combined. + """ + parsed_events = _parse_events(events) + pico = _extract_pico(parsed_events) + lit_survey = _extract_literature_survey(parsed_events) + evaluations = _extract_evaluations(parsed_events) + validation = _extract_validation_metrics(parsed_events) + tool_usage = _extract_tool_usage(parsed_events) + + sections: dict[str, str] = {} + + sections["title"] = _build_title(prompt, domain, created_at, investigation_id) + sections["abstract"] = _build_abstract(summary) + sections["introduction"] = _build_introduction(prompt, pico, lit_survey) + sections["methods"] = _build_methods(hypotheses, experiments, tool_usage) + sections["results"] = _build_results(hypotheses, findings, candidates) + sections["discussion"] = _build_discussion(hypotheses, evaluations) + sections["references"] = _build_references(citations, findings) + sections["supplementary"] = _build_supplementary( + cost_data, negative_controls, positive_controls, validation, + ) + + sections["full_markdown"] = _combine_sections(sections) + return sections + + +# ── Event Parsing ────────────────────────────────────────────────────── + + +def _parse_events(events: list[dict[str, str]]) -> list[dict[str, Any]]: + """Parse event_data JSON strings into dicts with event_type preserved.""" + parsed: list[dict[str, Any]] = [] + for ev in events: + try: + data = json.loads(ev["event_data"]) + except (json.JSONDecodeError, KeyError): + continue + # SSE events are wrapped: {"event": "...", "data": {...}} + inner = data.get("data", data) if isinstance(data, dict) else data + parsed.append({"event_type": ev.get("event_type", ""), "data": inner}) + return parsed + + +def _extract_pico(events: list[dict[str, Any]]) -> dict[str, str]: + for ev in events: + if ev["event_type"] == "literature_survey_completed": + pico: dict[str, str] = ev["data"].get("pico", {}) + return pico + return {} + + +def _extract_literature_survey(events: list[dict[str, Any]]) -> dict[str, Any]: + for ev in events: + if ev["event_type"] == "literature_survey_completed": + data: dict[str, Any] = ev["data"] + return data + return {} + + +def _extract_evaluations(events: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Map hypothesis_id -> evaluation data (reasoning, certainty).""" + evals: dict[str, dict[str, Any]] = {} + for ev in events: + if ev["event_type"] == "hypothesis_evaluated": + hid = ev["data"].get("hypothesis_id", "") + if hid: + evals[hid] = ev["data"] + return evals + + +def _extract_validation_metrics(events: list[dict[str, Any]]) -> dict[str, Any]: + for ev in events: + if ev["event_type"] == "validation_metrics": + metrics: dict[str, Any] = ev["data"] + return metrics + return {} + + +def _extract_tool_usage(events: list[dict[str, Any]]) -> dict[str, int]: + """Count tool calls by tool name.""" + counts: dict[str, int] = {} + for ev in events: + if ev["event_type"] == "tool_called": + name = ev["data"].get("tool_name", "unknown") + counts[name] = counts.get(name, 0) + 1 + return counts + + +# ── Section Builders ─────────────────────────────────────────────────── + + +def _build_title( + prompt: str, domain: str, created_at: str, investigation_id: str, +) -> str: + date = created_at[:10] if created_at else "" + lines = [ + f"# {prompt}", + "", + "_Generated by Ehrlich -- AI Scientific Discovery Engine_ ", + f"_Domain: {domain or 'auto-detected'} | Date: {date} | " + f"ID: {investigation_id}_", + ] + return "\n".join(lines) + + +def _build_abstract(summary: str) -> str: + if not summary: + return "## Abstract\n\nNo synthesis available." + return f"## Abstract\n\n{summary}" + + +def _build_introduction( + prompt: str, + pico: dict[str, str], + lit_survey: dict[str, Any], +) -> str: + lines = ["## 1. Introduction"] + + # Research question + lines.append("") + lines.append("### 1.1 Research Question") + lines.append("") + lines.append(f"> {prompt}") + + # PICO framework + if pico: + lines.append("") + lines.append("### 1.2 PICO Framework") + lines.append("") + for key in ("population", "intervention", "comparison", "outcome"): + val = pico.get(key, "") + if val: + lines.append(f"- **{key.title()}:** {val}") + search_terms = pico.get("search_terms") + if isinstance(search_terms, list) and search_terms: + lines.append(f"- **Search Terms:** {', '.join(search_terms)}") + + # Literature survey + if lit_survey: + lines.append("") + lines.append("### 1.3 Literature Survey") + lines.append("") + total = lit_survey.get("total_results", 0) + included = lit_survey.get("included_results", 0) + grade = lit_survey.get("evidence_grade", "") + assessment = lit_survey.get("assessment", "") + lines.append(f"- **Papers reviewed:** {total} found, {included} included") + if grade: + lines.append(f"- **Evidence grade:** {grade}") + if assessment: + lines.append(f"- **Assessment:** {assessment}") + + return "\n".join(lines) + + +def _build_methods( + hypotheses: list[dict[str, Any]], + experiments: list[dict[str, Any]], + tool_usage: dict[str, int], +) -> str: + lines = ["## 2. Methods"] + + if not experiments: + lines.append("\nNo experiments recorded.") + return "\n".join(lines) + + # Experimental design per hypothesis + lines.append("") + lines.append("### 2.1 Experimental Design") + + for h in hypotheses: + h_exps = [e for e in experiments if e.get("hypothesis_id") == h["id"]] + if not h_exps: + continue + + lines.append("") + lines.append(f"#### Hypothesis {h['id'][:8]}: {h.get('statement', '')}") + lines.append("") + + for exp in h_exps: + lines.append(f"**Experiment {exp.get('id', '')[:8]}:** {exp.get('description', '')}") + lines.append("") + + iv = exp.get("independent_variable", "") + dv = exp.get("dependent_variable", "") + if iv or dv: + if iv: + lines.append(f"- **Independent variable:** {iv}") + if dv: + lines.append(f"- **Dependent variable:** {dv}") + + controls = exp.get("controls") or [] + if controls: + lines.append(f"- **Controls:** {', '.join(controls)}") + + confounders = exp.get("confounders") or [] + if confounders: + lines.append(f"- **Confounders:** {', '.join(confounders)}") + + ap = exp.get("analysis_plan", "") + if ap: + lines.append(f"- **Analysis plan:** {ap}") + + sc = exp.get("success_criteria", "") + fc = exp.get("failure_criteria", "") + if sc: + lines.append(f"- **Success criteria:** {sc}") + if fc: + lines.append(f"- **Failure criteria:** {fc}") + + tool_plan = exp.get("tool_plan") or [] + if tool_plan: + lines.append(f"- **Tool plan:** {', '.join(tool_plan)}") + + lines.append("") + + # Tool usage summary + if tool_usage: + lines.append("### 2.2 Tools & Data Sources") + lines.append("") + lines.append("| Tool | Calls |") + lines.append("|------|-------|") + for name, count in sorted(tool_usage.items(), key=lambda x: -x[1]): + lines.append(f"| {name} | {count} |") + + return "\n".join(lines) + + +def _build_results( + hypotheses: list[dict[str, Any]], + findings: list[dict[str, Any]], + candidates: list[dict[str, Any]], +) -> str: + lines = ["## 3. Results"] + + # Hypothesis outcomes table + if hypotheses: + lines.append("") + lines.append("### 3.1 Hypothesis Outcomes") + lines.append("") + lines.append("| ID | Statement | Status | Confidence | Certainty |") + lines.append("|----|-----------|--------|------------|-----------|") + for h in hypotheses: + hid = h.get("id", "")[:8] + stmt = h.get("statement", "")[:80] + status = h.get("status", "") + conf = h.get("confidence", 0) + conf_str = f"{conf * 100:.0f}%" if conf > 0 else "-" + cert = h.get("certainty_of_evidence", "-") + lines.append(f"| {hid} | {stmt} | {status} | {conf_str} | {cert} |") + + # Key findings by evidence type + if findings: + lines.append("") + lines.append("### 3.2 Key Findings") + + by_type: dict[str, list[dict[str, Any]]] = { + "supporting": [], + "contradicting": [], + "neutral": [], + } + for f in findings: + bucket = by_type.get(f.get("evidence_type", "neutral"), by_type["neutral"]) + bucket.append(f) + + for etype, items in by_type.items(): + if not items: + continue + lines.append("") + lines.append(f"#### {etype.title()} Evidence ({len(items)})") + lines.append("") + for f in items: + source = "" + st = f.get("source_type", "") + sid = f.get("source_id", "") + if st and sid: + source = f" [{st}: {sid}]" + lines.append(f"- **{f.get('title', '')}**{source}") + detail = f.get("detail", "") + if detail: + lines.append(f" {detail}") + + # Candidates + if candidates: + lines.append("") + lines.append("### 3.3 Candidates") + lines.append("") + + score_keys: set[str] = set() + for c in candidates: + for k in c.get("scores", {}): + score_keys.add(k) + + headers = ["Rank", "Name", "Identifier"] + sorted_keys = sorted(score_keys) + headers.extend(k.replace("_", " ") for k in sorted_keys) + + lines.append(f"| {' | '.join(headers)} |") + lines.append(f"|{'|'.join('------' for _ in headers)}|") + + for c in candidates: + scores = c.get("scores", {}) + row = [ + str(c.get("rank", "")), + c.get("name", "-"), + f"`{c.get('identifier', '')}`", + ] + for k in sorted_keys: + v = scores.get(k) + row.append(f"{v:.2f}" if isinstance(v, (int, float)) else "-") + lines.append(f"| {' | '.join(row)} |") + + return "\n".join(lines) + + +def _build_discussion( + hypotheses: list[dict[str, Any]], + evaluations: dict[str, dict[str, Any]], +) -> str: + lines = ["## 4. Discussion"] + + if not hypotheses: + lines.append("\nNo hypotheses to discuss.") + return "\n".join(lines) + + lines.append("") + lines.append("### 4.1 Hypothesis Assessment") + + for h in hypotheses: + hid = h.get("id", "") + lines.append("") + lines.append(f"#### {h.get('statement', '')}") + lines.append("") + + status = h.get("status", "") + conf = h.get("confidence", 0) + cert = h.get("certainty_of_evidence", "") + lines.append(f"- **Status:** {status}") + if conf > 0: + lines.append(f"- **Confidence:** {conf * 100:.0f}%") + if cert: + lines.append(f"- **Certainty of evidence:** {cert}") + + # Add evaluation reasoning from events + ev = evaluations.get(hid, {}) + reasoning = ev.get("reasoning", "") + if reasoning: + lines.append(f"- **Reasoning:** {reasoning}") + + # Supporting / contradicting evidence + sup = h.get("supporting_evidence") or [] + con = h.get("contradicting_evidence") or [] + if sup: + lines.append(f"- **Supporting evidence:** {'; '.join(sup)}") + if con: + lines.append(f"- **Contradicting evidence:** {'; '.join(con)}") + + return "\n".join(lines) + + +def _build_references( + citations: list[str], + findings: list[dict[str, Any]], +) -> str: + lines = ["## References"] + + if not citations and not findings: + lines.append("\nNo references collected.") + return "\n".join(lines) + + # Formal citations + if citations: + lines.append("") + for i, cit in enumerate(citations, 1): + lines.append(f"{i}. {cit}") + + # Source provenance from findings (unique sources) + sources: set[str] = set() + for f in findings: + st = f.get("source_type", "") + sid = f.get("source_id", "") + if st and sid: + sources.add(f"{st}: {sid}") + + if sources: + lines.append("") + lines.append("### Data Sources") + lines.append("") + for src in sorted(sources): + lines.append(f"- {src}") + + return "\n".join(lines) + + +def _build_supplementary( + cost_data: dict[str, Any], + negative_controls: list[dict[str, Any]], + positive_controls: list[dict[str, Any]], + validation: dict[str, Any], +) -> str: + lines = ["## Supplementary Material"] + + # Model validation + if negative_controls or positive_controls: + lines.append("") + lines.append("### S1. Model Validation") + + if negative_controls: + correct = sum(1 for nc in negative_controls if nc.get("correctly_classified")) + total = len(negative_controls) + lines.append("") + lines.append(f"**Negative controls:** {correct}/{total} correctly classified") + lines.append("") + lines.append("| Name | Identifier | Score | Classified |") + lines.append("|------|------------|-------|------------|") + for nc in negative_controls: + name = nc.get("name", "-") + ident = nc.get("identifier", "") + score = nc.get("score", 0) + ok = "Pass" if nc.get("correctly_classified") else "Fail" + lines.append(f"| {name} | `{ident}` | {score:.3f} | {ok} |") + + if positive_controls: + correct = sum(1 for pc in positive_controls if pc.get("correctly_classified")) + total = len(positive_controls) + lines.append("") + lines.append(f"**Positive controls:** {correct}/{total} correctly classified") + lines.append("") + lines.append("| Name | Identifier | Known Activity | Score | Classified |") + lines.append("|------|------------|----------------|-------|------------|") + for pc in positive_controls: + name = pc.get("name", "-") + ident = pc.get("identifier", "") + activity = pc.get("known_activity", "") + score = pc.get("score", 0) + ok = "Pass" if pc.get("correctly_classified") else "Fail" + lines.append(f"| {name} | `{ident}` | {activity} | {score:.3f} | {ok} |") + + if validation: + zp = validation.get("z_prime") + if zp is not None: + quality = validation.get("z_prime_quality", "") + lines.append("") + lines.append(f"**Z'-factor:** {zp:.3f} ({quality})") + pm = validation.get("positive_mean", 0) + nm = validation.get("negative_mean", 0) + pc_count = validation.get("positive_control_count", 0) + nc_count = validation.get("negative_control_count", 0) + lines.append(f"- Positive controls: mean={pm:.3f}, n={pc_count}") + lines.append(f"- Negative controls: mean={nm:.3f}, n={nc_count}") + + # Cost & performance + if cost_data: + lines.append("") + lines.append("### S2. Cost & Performance") + lines.append("") + lines.append(f"- **Input tokens:** {cost_data.get('input_tokens', 0):,}") + lines.append(f"- **Output tokens:** {cost_data.get('output_tokens', 0):,}") + total_cost = cost_data.get("total_cost_usd", 0) + lines.append(f"- **Total cost:** ${total_cost:.4f}") + tool_calls = cost_data.get("tool_calls", 0) + if tool_calls: + lines.append(f"- **Tool calls:** {tool_calls}") + + by_model = cost_data.get("by_model") + if isinstance(by_model, dict) and by_model: + lines.append("") + lines.append("| Model | Input | Output | Calls | Cost |") + lines.append("|-------|-------|--------|-------|------|") + for model, mc in by_model.items(): + if not isinstance(mc, dict): + continue + inp = mc.get("input_tokens", 0) + out = mc.get("output_tokens", 0) + calls = mc.get("calls", 0) + cost = mc.get("cost_usd", 0) + lines.append(f"| {model} | {inp:,} | {out:,} | {calls} | ${cost:.4f} |") + + return "\n".join(lines) + + +def extract_visualizations(events: list[dict[str, str]]) -> list[dict[str, Any]]: + """Extract visualization payloads from persisted events. + + Returns a list of ``{viz_type, title, data, config}`` dicts -- one per + ``visualization`` event found in the timeline. + """ + parsed = _parse_events(events) + vizs: list[dict[str, Any]] = [] + for ev in parsed: + if ev["event_type"] == "visualization": + d = ev["data"] + vizs.append({ + "viz_type": d.get("viz_type", ""), + "title": d.get("title", ""), + "data": d.get("data", {}), + "config": d.get("config", {}), + }) + return vizs + + +def _combine_sections(sections: dict[str, str]) -> str: + """Combine all sections into a single Markdown document.""" + ordered = [ + "title", "abstract", "introduction", "methods", + "results", "discussion", "references", "supplementary", + ] + parts: list[str] = [] + for key in ordered: + content = sections.get(key, "") + if content: + parts.append(content) + return "\n\n---\n\n".join(parts) diff --git a/server/tests/investigation/test_paper_generator.py b/server/tests/investigation/test_paper_generator.py new file mode 100644 index 0000000..8c550ab --- /dev/null +++ b/server/tests/investigation/test_paper_generator.py @@ -0,0 +1,555 @@ +"""Tests for the scientific paper generator.""" + +from __future__ import annotations + +import json + +from ehrlich.investigation.application.paper_generator import extract_visualizations, generate_paper + +# ── Fixtures ─────────────────────────────────────────────────────────── + +_PICO_EVENT = { + "event_type": "literature_survey_completed", + "event_data": json.dumps({ + "event": "literature_survey_completed", + "data": { + "pico": { + "population": "MRSA clinical isolates", + "intervention": "novel PBP2a inhibitors", + "comparison": "vancomycin", + "outcome": "minimum inhibitory concentration", + "search_terms": ["MRSA", "PBP2a", "antimicrobial"], + }, + "search_queries": 4, + "total_results": 120, + "included_results": 18, + "evidence_grade": "moderate", + "assessment": "Moderate evidence from 3 RCTs and 15 observational studies.", + "investigation_id": "test-123", + }, + }), +} + +_EXPERIMENT_EVENT = { + "event_type": "experiment_started", + "event_data": json.dumps({ + "event": "experiment_started", + "data": { + "experiment_id": "exp-1", + "hypothesis_id": "h1", + "description": "Dock compounds against PBP2a", + "independent_variable": "compound structure", + "dependent_variable": "docking score", + "controls": ["vancomycin"], + "analysis_plan": "Compare docking scores", + "investigation_id": "test-123", + }, + }), +} + +_EVAL_EVENT = { + "event_type": "hypothesis_evaluated", + "event_data": json.dumps({ + "event": "hypothesis_evaluated", + "data": { + "hypothesis_id": "h1", + "status": "supported", + "confidence": 0.85, + "reasoning": "Docking scores below -7 kcal/mol for 3 candidates.", + "certainty_of_evidence": "moderate", + "investigation_id": "test-123", + }, + }), +} + +_TOOL_EVENTS = [ + { + "event_type": "tool_called", + "event_data": json.dumps({ + "event": "tool_called", + "data": {"tool_name": "dock_compound", "investigation_id": "test-123"}, + }), + }, + { + "event_type": "tool_called", + "event_data": json.dumps({ + "event": "tool_called", + "data": {"tool_name": "dock_compound", "investigation_id": "test-123"}, + }), + }, + { + "event_type": "tool_called", + "event_data": json.dumps({ + "event": "tool_called", + "data": {"tool_name": "search_chembl", "investigation_id": "test-123"}, + }), + }, +] + +_VALIDATION_EVENT = { + "event_type": "validation_metrics", + "event_data": json.dumps({ + "event": "validation_metrics", + "data": { + "z_prime": 0.72, + "z_prime_quality": "excellent", + "positive_control_count": 3, + "negative_control_count": 5, + "positive_mean": 0.88, + "negative_mean": 0.12, + "investigation_id": "test-123", + }, + }), +} + +_HYPOTHESES = [ + { + "id": "h1", + "statement": "Compound X inhibits PBP2a via active-site binding", + "rationale": "Structural similarity to known inhibitors", + "status": "supported", + "confidence": 0.85, + "certainty_of_evidence": "moderate", + "supporting_evidence": ["Docking score -9.2 kcal/mol"], + "contradicting_evidence": [], + }, +] + +_EXPERIMENTS = [ + { + "id": "exp-1", + "hypothesis_id": "h1", + "description": "Dock compounds against PBP2a", + "tool_plan": ["dock_compound", "score_admet"], + "status": "completed", + "independent_variable": "compound structure", + "dependent_variable": "docking score", + "controls": ["vancomycin"], + "confounders": ["solvent effects"], + "analysis_plan": "Compare docking scores", + "success_criteria": ">=3 compounds with docking < -7", + "failure_criteria": "<2 compounds meet threshold", + }, +] + +_FINDINGS = [ + { + "title": "Strong binding affinity", + "detail": "Docking score -9.2 kcal/mol against PBP2a", + "hypothesis_id": "h1", + "evidence_type": "supporting", + "source_type": "pdb", + "source_id": "2ABC", + }, + { + "title": "Poor ADMET profile", + "detail": "LogP > 5 indicates poor oral bioavailability", + "hypothesis_id": "h1", + "evidence_type": "contradicting", + "source_type": "chembl", + "source_id": "CHEMBL25", + }, +] + +_CANDIDATES = [ + { + "identifier": "CC(=O)Oc1ccccc1C(=O)O", + "identifier_type": "smiles", + "name": "Aspirin", + "rank": 1, + "notes": "Top candidate", + "scores": {"prediction_score": 0.92, "docking_score": -9.2}, + "attributes": {"resistance_risk": "low"}, + }, +] + +_NEGATIVE_CONTROLS = [ + { + "identifier": "CCO", + "name": "Ethanol", + "score": 0.05, + "threshold": 0.5, + "correctly_classified": True, + "source": "manual", + }, +] + +_POSITIVE_CONTROLS = [ + { + "identifier": "CC1=CC=CC=C1", + "name": "Toluene", + "known_activity": "Known PBP2a binder", + "score": 0.9, + "correctly_classified": True, + "source": "literature", + }, +] + +_CITATIONS = [ + "Smith et al. (2024) J Med Chem. doi:10.1021/jmc.2024.001", + "Jones et al. (2023) Nature. doi:10.1038/s41586-023-002", +] + +_COST_DATA = { + "input_tokens": 50000, + "output_tokens": 10000, + "total_tokens": 60000, + "total_cost_usd": 1.2345, + "tool_calls": 15, + "by_model": { + "claude-opus-4-6": { + "input_tokens": 30000, + "output_tokens": 5000, + "calls": 6, + "cost_usd": 0.8, + }, + "claude-sonnet-4-5-20250929": { + "input_tokens": 15000, + "output_tokens": 4000, + "calls": 8, + "cost_usd": 0.35, + }, + }, +} + + +def _all_events() -> list[dict[str, str]]: + return [_PICO_EVENT, _EXPERIMENT_EVENT, _EVAL_EVENT, _VALIDATION_EVENT, *_TOOL_EVENTS] + + +def _full_paper() -> dict[str, str]: + return generate_paper( + investigation_id="test-123", + prompt="Find novel antimicrobials against MRSA", + summary="We identified 1 candidate compound with strong PBP2a binding.", + domain="molecular_science", + created_at="2026-02-14T12:00:00+00:00", + hypotheses=_HYPOTHESES, + experiments=_EXPERIMENTS, + findings=_FINDINGS, + candidates=_CANDIDATES, + negative_controls=_NEGATIVE_CONTROLS, + positive_controls=_POSITIVE_CONTROLS, + citations=_CITATIONS, + cost_data=_COST_DATA, + events=_all_events(), + ) + + +# ── Tests ────────────────────────────────────────────────────────────── + + +class TestPaperStructure: + """Paper contains all expected sections.""" + + def test_has_all_section_keys(self) -> None: + paper = _full_paper() + expected = { + "title", "abstract", "introduction", "methods", + "results", "discussion", "references", "supplementary", + "full_markdown", + } + assert expected == set(paper.keys()) + + def test_full_markdown_contains_all_sections(self) -> None: + paper = _full_paper() + md = paper["full_markdown"] + assert "# Find novel antimicrobials against MRSA" in md + assert "## Abstract" in md + assert "## 1. Introduction" in md + assert "## 2. Methods" in md + assert "## 3. Results" in md + assert "## 4. Discussion" in md + assert "## References" in md + assert "## Supplementary Material" in md + + +class TestTitle: + def test_contains_prompt(self) -> None: + paper = _full_paper() + assert "Find novel antimicrobials against MRSA" in paper["title"] + + def test_contains_domain(self) -> None: + paper = _full_paper() + assert "molecular_science" in paper["title"] + + def test_contains_date(self) -> None: + paper = _full_paper() + assert "2026-02-14" in paper["title"] + + def test_contains_id(self) -> None: + paper = _full_paper() + assert "test-123" in paper["title"] + + +class TestAbstract: + def test_contains_summary(self) -> None: + paper = _full_paper() + assert "1 candidate compound" in paper["abstract"] + + def test_empty_summary(self) -> None: + paper = generate_paper( + investigation_id="x", prompt="Q", summary="", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=[], + ) + assert "No synthesis available" in paper["abstract"] + + +class TestIntroduction: + def test_research_question(self) -> None: + paper = _full_paper() + assert "> Find novel antimicrobials against MRSA" in paper["introduction"] + + def test_pico_framework(self) -> None: + paper = _full_paper() + intro = paper["introduction"] + assert "**Population:** MRSA clinical isolates" in intro + assert "**Intervention:** novel PBP2a inhibitors" in intro + assert "**Comparison:** vancomycin" in intro + assert "**Outcome:** minimum inhibitory concentration" in intro + + def test_pico_search_terms(self) -> None: + paper = _full_paper() + assert "MRSA, PBP2a, antimicrobial" in paper["introduction"] + + def test_literature_survey(self) -> None: + paper = _full_paper() + intro = paper["introduction"] + assert "120 found, 18 included" in intro + assert "moderate" in intro + + def test_no_pico_when_no_events(self) -> None: + paper = generate_paper( + investigation_id="x", prompt="Q", summary="S", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=[], + ) + assert "PICO" not in paper["introduction"] + + +class TestMethods: + def test_experiment_description(self) -> None: + paper = _full_paper() + assert "Dock compounds against PBP2a" in paper["methods"] + + def test_variables(self) -> None: + paper = _full_paper() + methods = paper["methods"] + assert "compound structure" in methods + assert "docking score" in methods + + def test_controls(self) -> None: + paper = _full_paper() + assert "vancomycin" in paper["methods"] + + def test_tool_plan(self) -> None: + paper = _full_paper() + assert "dock_compound" in paper["methods"] + + def test_tool_usage_table(self) -> None: + paper = _full_paper() + methods = paper["methods"] + assert "| dock_compound | 2 |" in methods + assert "| search_chembl | 1 |" in methods + + def test_empty_experiments(self) -> None: + paper = generate_paper( + investigation_id="x", prompt="Q", summary="S", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=[], + ) + assert "No experiments recorded" in paper["methods"] + + +class TestResults: + def test_hypothesis_outcomes_table(self) -> None: + paper = _full_paper() + results = paper["results"] + assert "supported" in results + assert "85%" in results + + def test_findings_with_provenance(self) -> None: + paper = _full_paper() + results = paper["results"] + assert "[pdb: 2ABC]" in results + assert "[chembl: CHEMBL25]" in results + + def test_findings_grouped_by_type(self) -> None: + paper = _full_paper() + results = paper["results"] + assert "Supporting Evidence (1)" in results + assert "Contradicting Evidence (1)" in results + + def test_candidate_table(self) -> None: + paper = _full_paper() + results = paper["results"] + assert "Aspirin" in results + assert "0.92" in results + + +class TestDiscussion: + def test_hypothesis_assessment(self) -> None: + paper = _full_paper() + disc = paper["discussion"] + assert "Compound X inhibits PBP2a" in disc + assert "**Status:** supported" in disc + assert "85%" in disc + + def test_evaluation_reasoning_from_events(self) -> None: + paper = _full_paper() + disc = paper["discussion"] + assert "Docking scores below -7 kcal/mol" in disc + + def test_supporting_evidence(self) -> None: + paper = _full_paper() + assert "Docking score -9.2 kcal/mol" in paper["discussion"] + + +class TestReferences: + def test_citations(self) -> None: + paper = _full_paper() + refs = paper["references"] + assert "Smith et al." in refs + assert "Jones et al." in refs + + def test_data_sources(self) -> None: + paper = _full_paper() + refs = paper["references"] + assert "pdb: 2ABC" in refs + assert "chembl: CHEMBL25" in refs + + def test_empty_references(self) -> None: + paper = generate_paper( + investigation_id="x", prompt="Q", summary="S", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=[], + ) + assert "No references collected" in paper["references"] + + +class TestSupplementary: + def test_negative_controls(self) -> None: + paper = _full_paper() + supp = paper["supplementary"] + assert "1/1 correctly classified" in supp + assert "Ethanol" in supp + + def test_positive_controls(self) -> None: + paper = _full_paper() + supp = paper["supplementary"] + assert "Known PBP2a binder" in supp + assert "Toluene" in supp + + def test_validation_metrics(self) -> None: + paper = _full_paper() + supp = paper["supplementary"] + assert "0.720" in supp + assert "excellent" in supp + + def test_cost_breakdown(self) -> None: + paper = _full_paper() + supp = paper["supplementary"] + assert "$1.2345" in supp + assert "50,000" in supp + + def test_by_model_table(self) -> None: + paper = _full_paper() + supp = paper["supplementary"] + assert "claude-opus-4-6" in supp + assert "$0.8000" in supp + + +class TestEdgeCases: + def test_empty_investigation(self) -> None: + paper = generate_paper( + investigation_id="empty", prompt="Empty test", summary="", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=[], + ) + assert "# Empty test" in paper["title"] + assert "No synthesis available" in paper["abstract"] + assert "No experiments recorded" in paper["methods"] + assert "No references collected" in paper["references"] + assert paper["full_markdown"] + + def test_malformed_event_data_skipped(self) -> None: + events = [{"event_type": "bad", "event_data": "not json{{{"}] + paper = generate_paper( + investigation_id="x", prompt="Q", summary="S", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=events, + ) + assert paper["full_markdown"] + + def test_event_missing_data_key(self) -> None: + events = [{"event_type": "tool_called", "event_data": json.dumps({"tool_name": "x"})}] + paper = generate_paper( + investigation_id="x", prompt="Q", summary="S", + domain="", created_at="", hypotheses=[], experiments=[], + findings=[], candidates=[], negative_controls=[], + positive_controls=[], citations=[], cost_data={}, events=events, + ) + assert paper["full_markdown"] + + +# ── Visualization Extraction Tests ──────────────────────────────────── + +_VIZ_EVENT_1 = { + "event_type": "visualization", + "event_data": json.dumps({ + "event": "visualization", + "data": { + "viz_type": "binding_scatter", + "title": "Binding affinity vs LogP", + "data": {"points": [{"x": 1, "y": 2}]}, + "config": {"xLabel": "LogP"}, + }, + }), +} + +_VIZ_EVENT_2 = { + "event_type": "visualization", + "event_data": json.dumps({ + "event": "visualization", + "data": { + "viz_type": "forest_plot", + "title": "Effect sizes", + "data": {"studies": []}, + "config": {}, + }, + }), +} + + +class TestExtractVisualizations: + def test_extracts_all_viz_events(self) -> None: + events = [_PICO_EVENT, _VIZ_EVENT_1, *_TOOL_EVENTS, _VIZ_EVENT_2] + vizs = extract_visualizations(events) + assert len(vizs) == 2 + assert vizs[0]["viz_type"] == "binding_scatter" + assert vizs[1]["viz_type"] == "forest_plot" + + def test_preserves_data_and_config(self) -> None: + vizs = extract_visualizations([_VIZ_EVENT_1]) + assert vizs[0]["title"] == "Binding affinity vs LogP" + assert vizs[0]["data"] == {"points": [{"x": 1, "y": 2}]} + assert vizs[0]["config"] == {"xLabel": "LogP"} + + def test_empty_events_returns_empty(self) -> None: + assert extract_visualizations([]) == [] + + def test_no_viz_events_returns_empty(self) -> None: + vizs = extract_visualizations(_all_events()) + assert vizs == [] + + def test_malformed_viz_event_skipped(self) -> None: + bad = {"event_type": "visualization", "event_data": "not json{"} + vizs = extract_visualizations([bad, _VIZ_EVENT_1]) + assert len(vizs) == 1 From 543086655980540d3bd28ca70e4278097e12bace Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 16:13:11 -0700 Subject: [PATCH 22/35] feat(console): add enhanced investigation comparison with findings diff, score chart, and cost breakdown Fix finding serialization to include source_type, source_id, evidence_level, confidence. Add 5 new comparison sections: summary, score chart, findings diff, cost, enhanced stats bar. --- .../components/InvestigationComparison.tsx | 327 +++++++++++++++++- docs/roadmap/README.md | 1 + docs/roadmap/backlog.md | 40 ++- .../src/ehrlich/api/schemas/investigation.py | 4 + 4 files changed, 348 insertions(+), 24 deletions(-) diff --git a/console/src/features/investigation/components/InvestigationComparison.tsx b/console/src/features/investigation/components/InvestigationComparison.tsx index c1fc05e..012039f 100644 --- a/console/src/features/investigation/components/InvestigationComparison.tsx +++ b/console/src/features/investigation/components/InvestigationComparison.tsx @@ -1,6 +1,18 @@ +import { useMemo } from "react"; +import Markdown from "react-markdown"; +import { + BarChart, + Bar, + XAxis, + YAxis, + Tooltip, + ResponsiveContainer, + Legend, +} from "recharts"; import { cn } from "@/shared/lib/utils"; import { MolViewer2D } from "@/features/molecule/components/MolViewer2D"; -import type { InvestigationDetail, CandidateRow } from "../types"; +import { VIZ_COLORS } from "@/features/visualization/theme"; +import type { InvestigationDetail, CandidateRow, Finding } from "../types"; interface InvestigationComparisonProps { invA: InvestigationDetail; @@ -12,16 +24,153 @@ function findShared(a: CandidateRow[], b: CandidateRow[]): string[] { return a.filter((c) => idsB.has(c.identifier)).map((c) => c.identifier); } +// ── Finding overlap detection ──────────────────────────────────────── + +function wordSet(text: string): Set { + return new Set(text.toLowerCase().split(/\s+/).filter(Boolean)); +} + +function jaccard(a: Set, b: Set): number { + if (a.size === 0 && b.size === 0) return 1; + let intersection = 0; + for (const w of a) { + if (b.has(w)) intersection++; + } + const union = a.size + b.size - intersection; + return union === 0 ? 0 : intersection / union; +} + +interface FindingDiff { + uniqueA: Finding[]; + shared: { a: Finding; b: Finding }[]; + uniqueB: Finding[]; +} + +function diffFindings(aFindings: Finding[], bFindings: Finding[]): FindingDiff { + const matchedB = new Set(); + const shared: { a: Finding; b: Finding }[] = []; + const uniqueA: Finding[] = []; + + for (const fa of aFindings) { + let matched = false; + for (let j = 0; j < bFindings.length; j++) { + if (matchedB.has(j)) continue; + const fb = bFindings[j]; + // Match by provenance if both have source_type + source_id + if (fa.source_type && fa.source_id && fb.source_type && fb.source_id) { + if (fa.source_type === fb.source_type && fa.source_id === fb.source_id) { + shared.push({ a: fa, b: fb }); + matchedB.add(j); + matched = true; + break; + } + } + // Fallback: title similarity + if (jaccard(wordSet(fa.title), wordSet(fb.title)) > 0.8) { + shared.push({ a: fa, b: fb }); + matchedB.add(j); + matched = true; + break; + } + } + if (!matched) uniqueA.push(fa); + } + + const uniqueB = bFindings.filter((_, i) => !matchedB.has(i)); + return { uniqueA, shared, uniqueB }; +} + +// ── Score chart data ───────────────────────────────────────────────── + +function buildScoreChartData( + sharedIds: string[], + candidatesA: CandidateRow[], + candidatesB: CandidateRow[], +): { data: Record[]; keys: string[] } | null { + const mapA = new Map(candidatesA.map((c) => [c.identifier, c])); + const mapB = new Map(candidatesB.map((c) => [c.identifier, c])); + const allKeys = new Set(); + const data: Record[] = []; + + for (const id of sharedIds) { + const a = mapA.get(id); + const b = mapB.get(id); + if (!a || !b) continue; + const entry: Record = { + name: a.name || id.slice(0, 20), + }; + for (const [k, v] of Object.entries(a.scores)) { + if (typeof v === "number") { + entry[`${k}_A`] = Number(v.toFixed(3)); + allKeys.add(k); + } + } + for (const [k, v] of Object.entries(b.scores)) { + if (typeof v === "number") { + entry[`${k}_B`] = Number(v.toFixed(3)); + allKeys.add(k); + } + } + data.push(entry); + } + + if (data.length === 0 || allKeys.size === 0) return null; + return { data, keys: [...allKeys] }; +} + +// ── Cost parsing ───────────────────────────────────────────────────── + +interface ParsedCost { + inputTokens: number; + outputTokens: number; + toolCalls: number; + totalCost: number; +} + +function parseCostData(cost_data: Record): ParsedCost | null { + const input = cost_data.input_tokens ?? cost_data.inputTokens; + const output = cost_data.output_tokens ?? cost_data.outputTokens; + const tools = cost_data.tool_calls ?? cost_data.toolCalls; + const total = cost_data.total_cost_usd ?? cost_data.totalCost; + if (input == null && output == null) return null; + return { + inputTokens: Number(input ?? 0), + outputTokens: Number(output ?? 0), + toolCalls: Number(tools ?? 0), + totalCost: Number(total ?? 0), + }; +} + +// ── Main component ─────────────────────────────────────────────────── + export function InvestigationComparison({ invA, invB }: InvestigationComparisonProps) { const sharedIdentifiers = findShared(invA.candidates, invB.candidates); + const findingDiff = useMemo(() => diffFindings(invA.findings, invB.findings), [invA.findings, invB.findings]); + const scoreChart = useMemo( + () => buildScoreChartData(sharedIdentifiers, invA.candidates, invB.candidates), + [sharedIdentifiers, invA.candidates, invB.candidates], + ); + const costA = parseCostData(invA.cost_data); + const costB = parseCostData(invB.cost_data); + + const supportingA = invA.findings.filter((f) => f.evidence_type === "supporting").length; + const supportingB = invB.findings.filter((f) => f.evidence_type === "supporting").length; return (
- {/* Stats bar */} -
- + {/* Enhanced stats bar */} +
+ + 0 ? Math.round((supportingA / invA.findings.length) * 100) : 0}% sup.)`} + /> - + 0 ? Math.round((supportingB / invB.findings.length) * 100) : 0}% sup.)`} + /> +
{/* Side-by-side prompts */} @@ -30,6 +179,14 @@ export function InvestigationComparison({ invA, invB }: InvestigationComparisonP
+ {/* Summary comparison */} + {(invA.summary || invB.summary) && ( +
+ + +
+ )} + {/* Side-by-side candidates */}
@@ -74,6 +231,69 @@ export function InvestigationComparison({ invA, invB }: InvestigationComparisonP
)} + {/* Score comparison chart */} + {scoreChart && ( +
+

+ Score Comparison +

+
+ + + + + + + {scoreChart.keys.map((key) => ( + + ))} + {scoreChart.keys.map((key) => ( + + ))} + + +
+
+ )} + + {/* Findings comparison */} + {(invA.findings.length > 0 || invB.findings.length > 0) && ( +
+

+ Findings Comparison +

+
+ + s.a)} + accent + /> + +
+
+ )} + + {/* Cost comparison */} + {(costA || costB) && ( +
+

+ Cost Comparison +

+
+ + +
+
+ )} + {/* Hypothesis summary */}
@@ -83,6 +303,8 @@ export function InvestigationComparison({ invA, invB }: InvestigationComparisonP ); } +// ── Inline helper components ───────────────────────────────────────── + function StatCard({ label, value, accent }: { label: string; value: string; accent?: boolean }) { return (
@@ -106,6 +328,17 @@ function PromptCard({ prompt, status }: { prompt: string; status: string }) { ); } +function SummaryCard({ summary, label }: { summary: string; label: string }) { + return ( +
+

{label}

+
+ {summary || "No summary available."} +
+
+ ); +} + function CandidateList({ title, candidates, @@ -143,6 +376,90 @@ function CandidateList({ ); } +function FindingColumn({ + label, + findings, + accent, +}: { + label: string; + findings: Finding[]; + accent?: boolean; +}) { + const EVIDENCE_BADGE: Record = { + supporting: "text-secondary", + contradicting: "text-destructive", + neutral: "text-muted-foreground", + }; + + return ( +
+

+ {label} +

+
+ {findings.length === 0 && ( +

None

+ )} + {findings.map((f, i) => ( +
+

{f.title}

+
+ + {f.evidence_type} + + {f.source_type && ( + + {f.source_type}{f.source_id ? `: ${f.source_id}` : ""} + + )} +
+
+ ))} +
+
+ ); +} + +function CostCard({ cost, label }: { cost: ParsedCost | null; label: string }) { + if (!cost) { + return ( +
+

{label}

+

No cost data

+
+ ); + } + return ( +
+

{label}

+
+
+ Input: + {cost.inputTokens.toLocaleString()} +
+
+ Output: + {cost.outputTokens.toLocaleString()} +
+
+ Tool calls: + {cost.toolCalls.toLocaleString()} +
+
+ Total: + ${cost.totalCost.toFixed(4)} +
+
+
+ ); +} + function HypothesisSummary({ title, hypotheses, diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index dc232b1..d76187b 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -25,6 +25,7 @@ Hackathon: Feb 10-16, 2026. Each phase ends with something testable. | Phase 12: Platform Infrastructure | DONE | [phase-12-platform-infrastructure.md](phase-12-platform-infrastructure.md) | | Phase 13: Impact Evaluation Domain | IN PROGRESS | [phase-13-impact-evaluation.md](phase-13-impact-evaluation.md) | | Phase 14: Scientific Engine Hardening | IN PROGRESS | [phase-14-engine-hardening.md](phase-14-engine-hardening.md) | +| Phase 11: Investigation Comparison | DONE | [backlog.md](backlog.md) | | Backlog | TODO | [backlog.md](backlog.md) | ## Dependency Graph diff --git a/docs/roadmap/backlog.md b/docs/roadmap/backlog.md index 110877f..c1d7950 100644 --- a/docs/roadmap/backlog.md +++ b/docs/roadmap/backlog.md @@ -2,7 +2,7 @@ Back to [Roadmap Index](README.md) # Backlog (Post-Hackathon) -- Phase 11: Investigation Comparison (side-by-side analysis) +- ~~Phase 11: Investigation Comparison~~ DONE (11A + 11C client-side) - Phase 13C: Mexico Integration (INEGI, datos.gob.mx, Transparencia, Banxico) - Phase 13D: US Integration (Census, BLS, USAspending, CDC WONDER) - Phase 13E: MCP Bridge Self-Service @@ -90,24 +90,26 @@ For molecular science specifically: Side-by-side comparison of investigation runs for reproducibility and consensus analysis. -## 11A. Comparison Domain -- [ ] `Comparison` entity: list of investigation IDs, consensus candidates, overlap metrics -- [ ] Candidate overlap calculation (by identifier + domain-specific similarity) -- [ ] Finding overlap detection (by hypothesis + title similarity) -- [ ] Score aggregation across runs (mean, std, min, max) - -## 11B. Comparison API -- [ ] `POST /compare` -- accept list of investigation IDs, return comparison -- [ ] `GET /compare/{id}` -- retrieve saved comparison - -## 11C. Comparison Console -- [ ] `/compare` page: pick 2+ completed investigations -- [ ] Side-by-side candidate table with overlap highlighting -- [ ] Consensus candidates panel (appear in N/M runs) -- [ ] Score distribution visualization (per candidate across runs) -- [ ] Findings diff view (shared vs unique per run) - -**Verification:** Compare 2 completed investigations, verify overlap metrics and consensus candidates render correctly. +## 11A. Comparison Logic -- DONE +- [x] Candidate overlap calculation (by identifier) +- [x] Finding overlap detection (by `source_type+source_id` provenance or Jaccard title similarity > 0.8) +- [x] Score comparison across runs (grouped bar chart per shared candidate) +- [x] Finding serialization fix: added `source_type`, `source_id`, `evidence_level`, `confidence` to API response + +## 11B. Comparison API -- DONE (client-side) +All comparison logic is client-side using existing `GET /investigate/{id}` detail endpoints. No new backend routes needed. + +## 11C. Comparison Console -- DONE +- [x] `/compare/$id1/$id2` route with side-by-side analysis +- [x] Enhanced 5-stat bar (candidates, findings with % supporting, shared count) +- [x] Side-by-side candidate table with overlap highlighting +- [x] Summary comparison (markdown-rendered AI synthesis) +- [x] Score comparison chart (Recharts grouped BarChart for shared candidates) +- [x] Findings diff view (A-only | Shared | B-only with evidence + source badges) +- [x] Cost comparison (input/output tokens, tool calls, total cost) +- [x] Hypothesis summary counts + +**Verification:** Compare 2 completed investigations at `/compare/{id1}/{id2}`, verify all 8 sections render correctly. --- diff --git a/server/src/ehrlich/api/schemas/investigation.py b/server/src/ehrlich/api/schemas/investigation.py index 17c31c7..5247f91 100644 --- a/server/src/ehrlich/api/schemas/investigation.py +++ b/server/src/ehrlich/api/schemas/investigation.py @@ -96,6 +96,10 @@ def serialize_findings(investigation: Investigation) -> list[dict[str, Any]]: "hypothesis_id": f.hypothesis_id, "evidence_type": f.evidence_type, "evidence": f.evidence, + "source_type": f.source_type, + "source_id": f.source_id, + "evidence_level": f.evidence_level, + "confidence": f.confidence, } for f in investigation.findings ] From 3cbc196fdbfbae6a415f78ba8f037f2b45653ae3 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 16:22:50 -0700 Subject: [PATCH 23/35] chore: fix pyproject.toml metadata and remove unused deps - Fix license: MIT -> AGPL-3.0-only (matches LICENSE file and README) - Update description to reflect multi-domain scientific discovery engine - Remove unused deps: requests (zero imports), pyarrow (zero imports) - Remove unused dev dep: pillow (zero imports) - Merge [dependency-groups] dev into [project.optional-dependencies] dev --- server/pyproject.toml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/server/pyproject.toml b/server/pyproject.toml index 8ef00e8..ded59d2 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "ehrlich" version = "0.1.0" -description = "AI-powered molecular discovery agent" +description = "Open-source scientific discovery engine powered by Claude" readme = "../README.md" -license = {text = "MIT"} +license = {text = "AGPL-3.0-only"} requires-python = ">=3.12" authors = [{name = "Ricardo Armenta", email = "ricardo@sequel.com"}] dependencies = [ @@ -21,12 +21,10 @@ dependencies = [ "openpyxl>=3.1,<4", "pymupdf>=1.25,<2", "scipy>=1.14,<2", - "requests>=2.32,<3", "joblib>=1.4,<2", "httpx>=0.28,<1", "pyyaml>=6,<7", "asyncpg>=0.30,<1", - "pyarrow>=19,<20", "pyjwt[crypto]>=2.9,<3", ] @@ -45,6 +43,8 @@ dev = [ "ruff>=0.9,<1", "mypy>=1.14,<2", "pre-commit>=4,<5", + "pandas-stubs>=3.0", + "scipy-stubs>=1.17", ] [build-system] @@ -121,9 +121,3 @@ omit = [ fail_under = 80 show_missing = true -[dependency-groups] -dev = [ - "pandas-stubs>=3.0.0.260204", - "pillow>=12.1.0", - "scipy-stubs>=1.17.0.2", -] From 1979c768f29a7e39f54b01ca5c01caa3a46c0543 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 16:28:25 -0700 Subject: [PATCH 24/35] feat(14E): add agentic hypothesis tree search with TreeManager Replace linear hypothesis testing with tree-based exploration. TreeManager scores, selects, deepens, branches, and prunes hypotheses as an exploration tree with configurable max_depth (default: 3). --- CLAUDE.md | 11 +- .../investigation/lib/diagram-builder.ts | 22 +- docs/architecture.md | 8 +- docs/roadmap/phase-14-engine-hardening.md | 31 +- docs/scientific-methodology.md | 17 + server/src/ehrlich/api/sse.py | 14 + .../application/multi_orchestrator.py | 4 + .../investigation/application/phase_runner.py | 77 ++-- .../application/prompts/constants.py | 21 ++ .../investigation/application/tree_manager.py | 119 ++++++ .../ehrlich/investigation/domain/events.py | 12 + .../investigation/domain/hypothesis.py | 4 + .../ehrlich/investigation/domain/schemas.py | 5 + .../investigation/test_multi_orchestrator.py | 176 +++++++++ .../tests/investigation/test_tree_manager.py | 353 ++++++++++++++++++ 15 files changed, 809 insertions(+), 65 deletions(-) create mode 100644 server/src/ehrlich/investigation/application/tree_manager.py create mode 100644 server/tests/investigation/test_tree_manager.py diff --git a/CLAUDE.md b/CLAUDE.md index 725abc2..6d4cb08 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,7 +104,8 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, ### Scientific Methodology -- **Hypothesis-driven investigation loop**: formulate -> design experiment -> execute tools -> evaluate -> revise/reject -> synthesize. See `docs/scientific-methodology.md` +- **Hypothesis-driven investigation loop**: formulate -> design experiment -> execute tools -> evaluate -> tree action (deepen/branch/prune) -> synthesize. See `docs/scientific-methodology.md` +- **Tree search**: `TreeManager` scores hypotheses (`prior_confidence * depth_discount * evidence_bonus`), selects top 2 for testing, processes Director's tree action (deepen=child at depth+1, branch=sibling at same depth, prune=REJECTED). `max_depth=3` prevents infinite recursion - **Hypothesis model**: Popper + Platt + Feynman + Bayesian updating. Fields: prediction, null_prediction, success/failure_criteria, scope, hypothesis_type, prior_confidence, certainty_of_evidence. Evaluation: 8-tier evidence hierarchy, effect sizes, Bayesian updating, convergence checks - **Experiment protocols**: Fisher/Platt/Cohen/Saltelli/OECD/Tropsha. Fields: independent/dependent_variable, controls, confounders, analysis_plan, success/failure_criteria. Director designs with 5 principles (VARIABLES, CONTROLS, CONFOUNDERS, ANALYSIS PLAN, SENSITIVITY) - **Statistical testing**: Director plans tests in `statistical_test_plan`; Researcher executes `run_statistical_test`/`run_categorical_test`; findings recorded with evidence_type based on significance @@ -119,7 +120,8 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - Each bounded context: `domain/`, `application/`, `infrastructure/`, `tools.py` - Domain entities: `@dataclass` with `__post_init__` validation; repositories: ABCs in `domain/repository.py` - `tools.py` is the boundary between Claude and application services -- **Parallel researchers**: `_run_experiment_batch()` uses `asyncio.Queue` for 2 concurrent experiments; three-layer differentiation prevents duplicates +- **Parallel researchers**: `run_experiment_batch()` uses `asyncio.Queue` for 2 concurrent experiments; three-layer differentiation prevents duplicates +- **Tree manager**: `TreeManager` in `application/tree_manager.py`; `select_next()` picks batch, `apply_evaluation()` processes Director action, `compute_branch_score()` scores hypotheses - **Context compaction**: `_build_prior_context()` compresses completed hypotheses for Director - **Prompt engineering**: XML-tagged instructions, multishot examples (2 per Director prompt), tool usage examples for Researcher - **ToolCache**: in-memory TTL cache (deterministic: forever, API: 24h-7d) @@ -139,7 +141,7 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - **Tool tagging**: frozenset domain tags; investigation tools are universal; researcher sees only domain-relevant tools - **Self-referential**: `search_prior_research` queries PostgreSQL tsvector + GIN index - **MCP bridge**: Optional external MCP servers; dynamic tool registration; `EHRLICH_MCP_EXCALIDRAW=true` -- **SSE**: 20 event types; reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) +- **SSE**: 21 event types; reconnection with exponential backoff (1s, 2s, 4s, max 3 retries) - **Phase progress**: 6 phases (Classification & PICO -> Literature Survey -> Formulation -> Hypothesis Testing -> Negative Controls -> Synthesis) - **Citation provenance**: `source_type` + `source_id` on findings (ChEMBL, PDB, DOI, PubChem, UniProt, Open Targets) @@ -147,7 +149,7 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - **Viz pipeline**: 17 viz tools -> `VisualizationPayload` -> `VisualizationRendered` SSE -> `VizRegistry` lazy-loads React component - **LiveLabViewer**: 3Dmol.js SSE-driven scene (protein load, ligand dock, candidate score coloring) -- **React Flow**: `@xyflow/react` with custom node types; status-colored (proposed=gray, testing=blue, supported=green, refuted=red) +- **React Flow**: `@xyflow/react` with custom node types; status-colored (proposed=gray, testing=blue, supported=green, refuted=red, revised=amber, rejected=dimmed gray); tree depth + parent-child revision edges - **Design system**: OKLCH hue 155 (green); `rounded-sm` buttons, `rounded-md` cards; Space Grotesk + JetBrains Mono - **Layout**: conditional `hideHeader` for investigation/compare pages; `max-w-[1200px]` - TanStack Router file-based routing; `ErrorBoundary` wraps LiveLabViewer and InvestigationDiagram @@ -171,6 +173,7 @@ All paths relative to `server/src/ehrlich/`. | `impact/application/impact_service.py` | Economic indicators, benchmarks, program comparison | | `investigation/application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop) | | `investigation/application/tool_dispatcher.py` | Tool execution with caching, `search_prior_research` and `query_uploaded_data` interception | +| `investigation/application/tree_manager.py` | Hypothesis tree exploration: scoring, selection, deepen/branch/prune actions | | `investigation/application/researcher_executor.py` | Single researcher experiment executor | | `investigation/application/batch_executor.py` | Parallel batch executor (2 concurrent) | | `investigation/application/paper_generator.py` | Scientific paper generation from investigation data + events | diff --git a/console/src/features/investigation/lib/diagram-builder.ts b/console/src/features/investigation/lib/diagram-builder.ts index 867a593..bc62c32 100644 --- a/console/src/features/investigation/lib/diagram-builder.ts +++ b/console/src/features/investigation/lib/diagram-builder.ts @@ -3,9 +3,10 @@ import type { Node, Edge } from "@xyflow/react"; export interface HypothesisNode { id: string; statement: string; - status: "proposed" | "testing" | "supported" | "refuted" | "revised"; + status: "proposed" | "testing" | "supported" | "refuted" | "revised" | "rejected"; parentId?: string; confidence?: number; + depth?: number; } export interface ExperimentNode { @@ -41,24 +42,25 @@ export interface DiagramData { // -- Dark-friendly palette ------------------------------------------------ const STATUS_COLORS: Record = { - proposed: { stroke: "#6b7280", fill: "#1f2937", text: "#d1d5db" }, - testing: { stroke: "#3b82f6", fill: "#1e3a5f", text: "#93c5fd" }, + proposed: { stroke: "#6b7280", fill: "#1f2937", text: "#d1d5db" }, + testing: { stroke: "#3b82f6", fill: "#1e3a5f", text: "#93c5fd" }, supported: { stroke: "#22c55e", fill: "#14532d", text: "#86efac" }, - refuted: { stroke: "#ef4444", fill: "#450a0a", text: "#fca5a5" }, - revised: { stroke: "#f59e0b", fill: "#451a03", text: "#fcd34d" }, + refuted: { stroke: "#ef4444", fill: "#450a0a", text: "#fca5a5" }, + revised: { stroke: "#f59e0b", fill: "#451a03", text: "#fcd34d" }, + rejected: { stroke: "#374151", fill: "#111827", text: "#6b7280" }, }; const EXPERIMENT_COLORS: Record = { - planned: { stroke: "#6b7280", fill: "#1f2937", text: "#d1d5db" }, - running: { stroke: "#3b82f6", fill: "#1e3a5f", text: "#93c5fd" }, + planned: { stroke: "#6b7280", fill: "#1f2937", text: "#d1d5db" }, + running: { stroke: "#3b82f6", fill: "#1e3a5f", text: "#93c5fd" }, completed: { stroke: "#22c55e", fill: "#14532d", text: "#86efac" }, - failed: { stroke: "#ef4444", fill: "#450a0a", text: "#fca5a5" }, + failed: { stroke: "#ef4444", fill: "#450a0a", text: "#fca5a5" }, }; const EVIDENCE_COLORS: Record = { - supporting: { stroke: "#22c55e", fill: "#14532d", text: "#86efac" }, + supporting: { stroke: "#22c55e", fill: "#14532d", text: "#86efac" }, contradicting: { stroke: "#ef4444", fill: "#450a0a", text: "#fca5a5" }, - neutral: { stroke: "#6b7280", fill: "#1f2937", text: "#d1d5db" }, + neutral: { stroke: "#6b7280", fill: "#1f2937", text: "#d1d5db" }, }; const ARROW_LABELS: Record = { diff --git a/docs/architecture.md b/docs/architecture.md index 5fa3399..bb54c29 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,8 +62,8 @@ Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesize 2. **Literature Survey** -- Sonnet researcher conducts structured search with domain-filtered tools, citation chasing (snowballing), evidence-level grading; Haiku grades body-of-evidence (GRADE-adapted) and self-assesses quality (AMSTAR-2-adapted) 3. **Hypothesis Formulation** -- Opus Director formulates 2-4 hypotheses with predictions, criteria, scope, Bayesian priors (grounded in Popper, Platt, Feynman, Bayesian frameworks -- see `docs/scientific-methodology.md`); receives structured XML literature context (PICO + graded findings) 4. **User Approval Gate** -- User approves/rejects hypotheses before testing begins (5-min auto-approve timeout) -5. **Experiment Design + Execution** -- Director designs structured experiment protocols (variables, controls, confounders, analysis plan, criteria), 2 Sonnet researchers execute in parallel per batch (max 10 tool calls each) with methodology guidance (sensitivity, applicability domain, uncertainty, verification, negative results) -6. **Hypothesis Evaluation** -- Director compares findings against both hypothesis-level and experiment-level criteria with methodology checks (control validation, confounders, analysis plan adherence) +5. **Experiment Design + Execution** -- `TreeManager.select_next()` picks up to 2 most promising PROPOSED hypotheses (scored by `branch_score`). Director designs structured experiment protocols (variables, controls, confounders, analysis plan, criteria), 2 Sonnet researchers execute in parallel per batch (max 10 tool calls each) with methodology guidance (sensitivity, applicability domain, uncertainty, verification, negative results) +6. **Hypothesis Evaluation + Tree Action** -- Director compares findings against both hypothesis-level and experiment-level criteria with methodology checks (control validation, confounders, analysis plan adherence). Director decides tree action: **deepen** (spawn narrower sub-hypothesis at depth+1), **branch** (revise into alternative at same depth), or **prune** (mark branch dead). `TreeManager.apply_evaluation()` creates new hypotheses for deepen/branch, marks REJECTED for prune. Loop continues until no explorable hypotheses remain or `max_depth` (default: 3) is reached 7. **Controls Validation** -- Score positive/negative controls through trained models; compute Z'-factor assay quality, permutation significance, scaffold-split vs random-split comparison 8. **Synthesis** -- Director synthesizes final report with ranked candidates, citations, validation metrics, cost breakdown @@ -244,8 +244,8 @@ api/ -> investigation/application/ only a. **Director** designs experiment protocol (description, tool plan, variables, controls, confounders, analysis plan, criteria, optional statistical test plan) b. **2 Researchers** (Sonnet) execute in parallel via asyncio.Queue c. **Summarizer** (Haiku) compresses outputs exceeding threshold - d. **Director** evaluates hypothesis against pre-defined success/failure criteria - e. If revised: new hypothesis spawned with parent link + d. **Director** evaluates hypothesis against pre-defined success/failure criteria and decides tree action (deepen/branch/prune) + e. `TreeManager.apply_evaluation()` processes action: deepen creates child at depth+1, branch creates sibling at same depth, prune marks REJECTED. Loop repeats with `select_next()` until no explorable hypotheses remain below `max_depth` 9. **Negative controls** recorded from formulation suggestions 10. **Director** synthesizes final report with candidates, citations, cost 11. All events stream via SSE (20 event types) to Console in real-time diff --git a/docs/roadmap/phase-14-engine-hardening.md b/docs/roadmap/phase-14-engine-hardening.md index 06b1c72..3bbe486 100644 --- a/docs/roadmap/phase-14-engine-hardening.md +++ b/docs/roadmap/phase-14-engine-hardening.md @@ -61,18 +61,25 @@ Upgraded 4 composite tools from thin pass-throughs to real domain-specific logic Result: 15 new tests added (6 for search_training_literature, 3 for search_supplement_evidence, 3 for compare_nutrients, 6 for _compute_i_squared). All 61 tests passing (40 training + 21 nutrition). Zero ruff violations, zero mypy errors. -## 14E: Agentic Hypothesis Tree Search -- TODO - -Replace linear hypothesis testing with tree-based exploration inspired by AI Scientist v2's agentic tree search. The current parallel batch system (2 experiments per hypothesis) becomes a tree where hypotheses can branch, deepen, or be pruned based on evidence. - -- [ ] `HypothesisNode` dataclass -- extends Hypothesis with `depth`, `branch_score`, `children: list[str]` -- [ ] `TreeManager` in `application/tree_manager.py` -- decides which branches to explore based on confidence scores, prunes low-confidence branches (<0.4 threshold), tracks exploration budget -- [ ] Director formulation prompt update -- "given current evidence tree, propose sub-hypotheses for the most promising branch OR prune low-confidence branches" -- [ ] Director evaluation prompt update -- after evaluating, decide: (a) deepen (spawn sub-hypotheses), (b) prune (mark branch as dead), (c) branch (revise into alternative direction) -- [ ] `max_depth` parameter (default: 3) to prevent infinite recursion -- [ ] Budget-aware exploration -- track API cost per branch, prefer deepening high-confidence/low-cost branches -- [ ] `HypothesisTreeEvent` SSE event for frontend tree rendering -- [ ] Console: `InvestigationDiagram.tsx` already renders React Flow graph -- tree structure renders naturally as growing/pruning graph +## 14E: Agentic Hypothesis Tree Search -- DONE + +Replace linear hypothesis testing with tree-based exploration inspired by AI Scientist v2's agentic tree search and Ai2 AutoDiscovery's Bayesian surprise-guided exploration. The hypothesis testing loop now uses a `TreeManager` that scores, selects, deepens, branches, and prunes hypotheses as a tree. + +- [x] Tree fields on `Hypothesis` dataclass -- `depth: int`, `branch_score: float`, `children: list[str]` (YAGNI: no separate `HypothesisNode` class needed) +- [x] `TreeManager` in `application/tree_manager.py` (~120 lines) -- pure application service with 4 methods: + - `select_next()` -- picks up to 2 most promising PROPOSED hypotheses (highest `branch_score`, shallowest depth tiebreak, respects `max_depth`) + - `compute_branch_score()` -- `prior_confidence * depth_discount * evidence_bonus` (depth penalty: `1/(1+depth*0.2)`, parent evidence bonus: high=1.3x, moderate=1.1x, low=0.9x, very_low=0.7x) + - `apply_evaluation()` -- processes Director's `action` field: deepen (child at depth+1), branch (sibling at same depth), prune (mark REJECTED) + - `should_continue()` -- True if any PROPOSED hypotheses exist below `max_depth` +- [x] `EVALUATION_SCHEMA` updated -- `action` field (enum: deepen/prune/branch) added to required fields +- [x] `DIRECTOR_EVALUATION_PROMPT` updated -- `` section with guidelines: supported+high confidence -> prune, refuted -> prune, revised+confidence>0.4 -> deepen, revised+confidence<=0.4 -> branch +- [x] `max_depth` parameter (default: 3) on `TreeManager` constructor, injectable via `MultiModelOrchestrator` +- [x] `HypothesisTreeUpdated` domain event + `hypothesis_tree_updated` SSE event type (21st event type) +- [x] `REJECTED` status added to `HypothesisStatus` enum (for pruned hypotheses) +- [x] Phase 4 integration -- `run_hypothesis_testing_phase` uses `TreeManager` for selection and evaluation processing; computes initial branch scores for root hypotheses +- [x] Console: `diagram-builder.ts` -- `rejected` status colors (gray/dimmed), `depth` and `parentId` fields on `HypothesisNode`, parent->child revision edges + +Result: `tree_manager.py` (120 lines). 25 unit tests (`test_tree_manager.py`), 4 integration tests in `test_multi_orchestrator.py`. 387 server tests passing, 138 console tests passing. Zero ruff/mypy violations. ## 14F: Scientific Paper Generator -- DONE diff --git a/docs/scientific-methodology.md b/docs/scientific-methodology.md index 4d11671..23000ab 100644 --- a/docs/scientific-methodology.md +++ b/docs/scientific-methodology.md @@ -263,12 +263,29 @@ Orchestrator wiring: - Serialized/deserialized in PostgreSQL repository ``` +**Tree Search (Phase 14E):** + +After evaluation, the Director decides a tree exploration action: +- **Deepen** -- evidence is promising but insufficient; spawn a narrower sub-hypothesis at depth+1 (e.g., "compound X inhibits target Y" -> "compound X with C2-sulfonamide inhibits Y via covalent binding") +- **Branch** -- partial evidence warrants an alternative direction; create a revised hypothesis at the same depth (e.g., shift from structural to pharmacological approach) +- **Prune** -- evidence clearly refutes or hypothesis is supported with high confidence; mark branch as dead (REJECTED status), no further exploration + +`TreeManager` (application service) coordinates exploration: +- `select_next()` picks up to 2 most promising PROPOSED hypotheses: highest `branch_score` first, shallowest depth as tiebreak +- `compute_branch_score()` = `prior_confidence * depth_discount * evidence_bonus` where depth_discount = 1/(1+depth*0.2), evidence_bonus from parent's certainty_of_evidence (high=1.3x, moderate=1.1x, low=0.9x, very_low=0.7x) +- `max_depth` (default: 3) prevents infinite recursion +- Loop terminates when no PROPOSED hypotheses remain below max_depth + +This replaces the prior linear "test all proposed, then test revised" flow with an intelligent tree exploration inspired by AI Scientist v2 (agentic tree search) and Ai2 AutoDiscovery (Bayesian surprise-guided exploration). + **Key design decisions:** - Certainty of evidence is a string field on Hypothesis (not an enum) -- Director-assigned, keeps domain layer simple - Evidence hierarchy is prompt guidance (8 tiers), not infrastructure code -- Director reasons over tiers rather than computing them algorithmically - Bayesian updating is formalized as multiplier rules in the prompt rather than coded computation -- keeps the LLM as the reasoning engine while providing structured methodology - Contradiction resolution follows a resolution hierarchy (identity → assay → temporal → severity) rather than averaging -- prevents masking genuine disagreements - Convergence check is a separate explicit field rather than folded into confidence -- allows the frontend to distinguish "high confidence from one method" vs "moderate confidence from converging methods" +- Tree search fields live on the existing `Hypothesis` dataclass (depth, children, branch_score) rather than a separate `HypothesisNode` class -- YAGNI, the tree is an exploration strategy, not core domain knowledge +- Tree action is a required field in `EVALUATION_SCHEMA` (structured output) -- Anthropic's constrained decoding guarantees the Director always produces it --- diff --git a/server/src/ehrlich/api/sse.py b/server/src/ehrlich/api/sse.py index 4d1e7ac..864e14a 100644 --- a/server/src/ehrlich/api/sse.py +++ b/server/src/ehrlich/api/sse.py @@ -13,6 +13,7 @@ HypothesisApprovalRequested, HypothesisEvaluated, HypothesisFormulated, + HypothesisTreeUpdated, InvestigationCompleted, InvestigationError, LiteratureSurveyCompleted, @@ -49,6 +50,7 @@ class SSEEventType(StrEnum): LITERATURE_SURVEY_COMPLETED = "literature_survey_completed" VALIDATION_METRICS = "validation_metrics" VISUALIZATION = "visualization" + HYPOTHESIS_TREE_UPDATED = "hypothesis_tree_updated" @dataclass(frozen=True) @@ -297,4 +299,16 @@ def domain_event_to_sse(event: DomainEvent) -> SSEEvent | None: "investigation_id": event.investigation_id, }, ) + if isinstance(event, HypothesisTreeUpdated): + return SSEEvent( + event=SSEEventType.HYPOTHESIS_TREE_UPDATED, + data={ + "hypothesis_id": event.hypothesis_id, + "action": event.action, + "parent_id": event.parent_id, + "depth": event.depth, + "children_count": event.children_count, + "investigation_id": event.investigation_id, + }, + ) return None diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index 9c802a1..6c74872 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -36,6 +36,7 @@ from collections.abc import AsyncGenerator from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.application.tree_manager import TreeManager from ehrlich.investigation.domain.domain_config import DomainConfig from ehrlich.investigation.domain.domain_registry import DomainRegistry from ehrlich.investigation.domain.mcp_config import MCPServerConfig @@ -67,6 +68,7 @@ def __init__( domain_registry: DomainRegistry | None = None, mcp_bridge: MCPBridge | None = None, mcp_configs: list[MCPServerConfig] | None = None, + tree_manager: TreeManager | None = None, ) -> None: self._director = director self._researcher = researcher @@ -80,6 +82,7 @@ def __init__( self._domain_registry = domain_registry self._mcp_bridge = mcp_bridge self._mcp_configs = mcp_configs or [] + self._tree_manager = tree_manager self._active_config: DomainConfig | None = None self._researcher_prompt = RESEARCHER_EXPERIMENT_PROMPT self._cache = ToolCache() @@ -266,6 +269,7 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, self._state_lock, self._director_call, self._cost_event, + tree_manager=self._tree_manager, ): yield event diff --git a/server/src/ehrlich/investigation/application/phase_runner.py b/server/src/ehrlich/investigation/application/phase_runner.py index 7ede185..002fb51 100644 --- a/server/src/ehrlich/investigation/application/phase_runner.py +++ b/server/src/ehrlich/investigation/application/phase_runner.py @@ -42,6 +42,7 @@ HypothesisApprovalRequested, HypothesisEvaluated, HypothesisFormulated, + HypothesisTreeUpdated, InvestigationCompleted, NegativeControlRecorded, PhaseChanged, @@ -68,6 +69,7 @@ from ehrlich.investigation.application.cost_tracker import CostTracker from ehrlich.investigation.application.tool_dispatcher import ToolDispatcher from ehrlich.investigation.application.tool_registry import ToolRegistry + from ehrlich.investigation.application.tree_manager import TreeManager from ehrlich.investigation.domain.domain_config import DomainConfig from ehrlich.investigation.domain.domain_registry import DomainRegistry from ehrlich.investigation.domain.repository import InvestigationRepository @@ -328,6 +330,7 @@ async def run_hypothesis_testing_phase( state_lock: asyncio.Lock, director_call: Callable[..., AsyncGenerator[Any, None]], cost_event_fn: Callable[..., CostUpdate], + tree_manager: TreeManager | None = None, ) -> AsyncGenerator[DomainEvent, None]: """Phase 4: Batched parallel hypothesis testing + Director evaluation loop.""" @@ -338,15 +341,24 @@ async def run_hypothesis_testing_phase( investigation_id=investigation.id, ) tested = 0 + + # Use tree manager if provided, otherwise import and create one + if tree_manager is None: + from ehrlich.investigation.application.tree_manager import TreeManager + + tree_manager = TreeManager() + + # Compute initial branch scores for root hypotheses + for h in investigation.hypotheses: + if h.branch_score == 0.0 and h.status == HypothesisStatus.PROPOSED: + h.branch_score = tree_manager.compute_branch_score(h, investigation) + while tested < max_hypotheses: - proposed = [h for h in investigation.hypotheses if h.status == HypothesisStatus.PROPOSED] - if not proposed: + batch_hypotheses = tree_manager.select_next(investigation.hypotheses) + if not batch_hypotheses: break - - # Take up to 2 hypotheses per batch - batch_hypotheses = proposed[:2] - batch: list[tuple[Hypothesis, Experiment, dict[str, Any]]] = [] prior_designs: list[str] = [] + batch: list[tuple[Hypothesis, Experiment, dict[str, Any]]] = [] for hypothesis in batch_hypotheses: hypothesis.status = HypothesisStatus.TESTING @@ -528,36 +540,31 @@ async def run_hypothesis_testing_phase( investigation_id=investigation.id, ) - # If revised, add new hypothesis to queue - if eval_status == "revised" and evaluation.get("revision"): - revised = Hypothesis( - statement=evaluation["revision"], - rationale=evaluation.get( - "reasoning", - "Revised from prior evidence", - ), - prediction=evaluation.get("prediction", hypothesis.prediction), - success_criteria=evaluation.get( - "success_criteria", hypothesis.success_criteria - ), - failure_criteria=evaluation.get( - "failure_criteria", hypothesis.failure_criteria - ), - scope=hypothesis.scope, - hypothesis_type=hypothesis.hypothesis_type, - parent_id=hypothesis.id, - ) - investigation.add_hypothesis(revised) + # Tree manager processes evaluation action + new_hypotheses = tree_manager.apply_evaluation(hypothesis, evaluation, investigation) + action = evaluation.get("action", "prune") + + yield HypothesisTreeUpdated( + hypothesis_id=hypothesis.id, + action=action, + parent_id=hypothesis.parent_id, + depth=hypothesis.depth, + children_count=len(hypothesis.children), + investigation_id=investigation.id, + ) + + for new_hyp in new_hypotheses: + investigation.add_hypothesis(new_hyp) yield HypothesisFormulated( - hypothesis_id=revised.id, - statement=revised.statement, - rationale=revised.rationale, - prediction=revised.prediction, - success_criteria=revised.success_criteria, - failure_criteria=revised.failure_criteria, - scope=revised.scope, - hypothesis_type=revised.hypothesis_type, - parent_id=hypothesis.id, + hypothesis_id=new_hyp.id, + statement=new_hyp.statement, + rationale=new_hyp.rationale, + prediction=new_hyp.prediction, + success_criteria=new_hyp.success_criteria, + failure_criteria=new_hyp.failure_criteria, + scope=new_hyp.scope, + hypothesis_type=new_hyp.hypothesis_type, + parent_id=new_hyp.parent_id, investigation_id=investigation.id, ) diff --git a/server/src/ehrlich/investigation/application/prompts/constants.py b/server/src/ehrlich/investigation/application/prompts/constants.py index 85a9719..20a46da 100644 --- a/server/src/ehrlich/investigation/application/prompts/constants.py +++ b/server/src/ehrlich/investigation/application/prompts/constants.py @@ -444,6 +444,26 @@ execution? If so, note their impact on confidence. + +After evaluating the hypothesis, decide how to proceed in the \ +exploration tree: +- "deepen": Evidence is promising but insufficient. The hypothesis \ +has partial support that warrants narrower, more specific \ +sub-hypotheses. Provide a refined hypothesis in "revision". +- "prune": Evidence clearly refutes the hypothesis OR confidence \ +is very low. Mark branch as dead, do not explore further. +- "branch": Partial evidence warrants an alternative direction. \ +Revise into a different approach at the same depth level. \ +Provide the alternative hypothesis in "revision". + +Guidelines: +- If status is "supported" with high confidence: use "prune" \ +(no further exploration needed on this branch). +- If status is "refuted": use "prune". +- If status is "revised" and confidence > 0.4: use "deepen". +- If status is "revised" and confidence <= 0.4: use "branch". + + Respond with ONLY valid JSON (no markdown fences): { @@ -456,6 +476,7 @@ and effect size thresholds", "key_evidence": ["list of key evidence points with numbers \ and their reliability tier"], + "action": "deepen|prune|branch", "revision": "If revised, the new refined hypothesis (omit \ if not revised)" } diff --git a/server/src/ehrlich/investigation/application/tree_manager.py b/server/src/ehrlich/investigation/application/tree_manager.py new file mode 100644 index 0000000..2b51675 --- /dev/null +++ b/server/src/ehrlich/investigation/application/tree_manager.py @@ -0,0 +1,119 @@ +"""Tree manager for hypothesis exploration. + +Decides which branches to explore, computes branch scores, +and processes Director evaluation actions (deepen/prune/branch). +""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any + +from ehrlich.investigation.domain.hypothesis import Hypothesis, HypothesisStatus + +if TYPE_CHECKING: + from ehrlich.investigation.domain.investigation import Investigation + + +class TreeManager: + """Manages tree-based hypothesis exploration strategy.""" + + def __init__(self, max_depth: int = 3, prune_threshold: float = 0.3) -> None: + self.max_depth = max_depth + self.prune_threshold = prune_threshold + + def select_next(self, hypotheses: list[Hypothesis]) -> list[Hypothesis]: + """Select up to 2 most promising PROPOSED hypotheses for testing. + + Priority: highest branch_score first, shallowest depth as tiebreak. + Skips hypotheses at or beyond max_depth. + """ + candidates = [ + h + for h in hypotheses + if h.status == HypothesisStatus.PROPOSED and h.depth < self.max_depth + ] + candidates.sort(key=lambda h: (-h.branch_score, h.depth)) + return candidates[:2] + + def compute_branch_score(self, hypothesis: Hypothesis, investigation: Investigation) -> float: + """Compute exploration priority score for a hypothesis. + + score = prior_confidence * depth_discount * evidence_bonus + + depth_discount penalizes deep branches mildly. + evidence_bonus rewards hypotheses whose parent had converging evidence. + """ + base = hypothesis.prior_confidence if hypothesis.prior_confidence > 0 else 0.5 + depth_discount = 1.0 / (1 + hypothesis.depth * 0.2) + + evidence_bonus = 1.0 + if hypothesis.parent_id: + parent = next( + (h for h in investigation.hypotheses if h.id == hypothesis.parent_id), + None, + ) + if parent and parent.certainty_of_evidence: + evidence_map = {"high": 1.3, "moderate": 1.1, "low": 0.9, "very_low": 0.7} + evidence_bonus = evidence_map.get(parent.certainty_of_evidence, 1.0) + + return round(base * depth_discount * evidence_bonus, 4) + + def apply_evaluation( + self, + hypothesis: Hypothesis, + evaluation: dict[str, Any], + investigation: Investigation, + ) -> list[Hypothesis]: + """Process Director's evaluation and return new hypotheses to add. + + action="deepen": create sub-hypothesis from revision text at depth+1 + action="prune": mark hypothesis REJECTED, return empty + action="branch": create revised hypothesis at same depth + """ + action: str = evaluation.get("action", "prune") + new_hypotheses: list[Hypothesis] = [] + + if action == "deepen" and evaluation.get("revision"): + child = Hypothesis( + statement=evaluation["revision"], + rationale=f"Deepened from: {hypothesis.statement[:80]}", + id=str(uuid.uuid4())[:8], + parent_id=hypothesis.id, + depth=hypothesis.depth + 1, + prior_confidence=evaluation.get("confidence", hypothesis.confidence), + ) + child.branch_score = self.compute_branch_score(child, investigation) + hypothesis.children.append(child.id) + new_hypotheses.append(child) + + elif action == "branch" and evaluation.get("revision"): + revised = Hypothesis( + statement=evaluation["revision"], + rationale=f"Branched from: {hypothesis.statement[:80]}", + id=str(uuid.uuid4())[:8], + parent_id=hypothesis.id, + depth=hypothesis.depth, + prior_confidence=evaluation.get("confidence", hypothesis.confidence), + ) + revised.branch_score = self.compute_branch_score(revised, investigation) + hypothesis.children.append(revised.id) + new_hypotheses.append(revised) + + elif action == "prune": + # Only mark REJECTED if not already terminally evaluated. + # "prune" on a supported/refuted hypothesis just means + # "stop exploring this branch" — no status change needed. + if hypothesis.status not in ( + HypothesisStatus.SUPPORTED, + HypothesisStatus.REFUTED, + ): + hypothesis.status = HypothesisStatus.REJECTED + + return new_hypotheses + + def should_continue(self, hypotheses: list[Hypothesis]) -> bool: + """Return True if there are still explorable PROPOSED hypotheses.""" + return any( + h.status == HypothesisStatus.PROPOSED and h.depth < self.max_depth for h in hypotheses + ) diff --git a/server/src/ehrlich/investigation/domain/events.py b/server/src/ehrlich/investigation/domain/events.py index 66d1034..3c11e50 100644 --- a/server/src/ehrlich/investigation/domain/events.py +++ b/server/src/ehrlich/investigation/domain/events.py @@ -210,3 +210,15 @@ class InvestigationCompleted(DomainEvent): class InvestigationError(DomainEvent): error: str = "" investigation_id: str = "" + + +@dataclass(frozen=True) +class HypothesisTreeUpdated(DomainEvent): + """Emitted when tree structure changes (branch/prune/deepen).""" + + hypothesis_id: str = "" + action: str = "" # "deepen" | "prune" | "branch" + parent_id: str = "" + depth: int = 0 + children_count: int = 0 + investigation_id: str = "" diff --git a/server/src/ehrlich/investigation/domain/hypothesis.py b/server/src/ehrlich/investigation/domain/hypothesis.py index 70e5e34..d5eb31c 100644 --- a/server/src/ehrlich/investigation/domain/hypothesis.py +++ b/server/src/ehrlich/investigation/domain/hypothesis.py @@ -43,6 +43,10 @@ class Hypothesis: certainty_of_evidence: str = "" supporting_evidence: list[str] = field(default_factory=list) contradicting_evidence: list[str] = field(default_factory=list) + # Tree search fields + depth: int = 0 + children: list[str] = field(default_factory=list) + branch_score: float = 0.0 def __post_init__(self) -> None: if not self.statement: diff --git a/server/src/ehrlich/investigation/domain/schemas.py b/server/src/ehrlich/investigation/domain/schemas.py index 52f9207..91c28ad 100644 --- a/server/src/ehrlich/investigation/domain/schemas.py +++ b/server/src/ehrlich/investigation/domain/schemas.py @@ -193,6 +193,10 @@ "items": {"type": "string"}, }, "revision": {"type": "string"}, + "action": { + "type": "string", + "enum": ["deepen", "prune", "branch"], + }, }, "required": [ "status", @@ -201,6 +205,7 @@ "evidence_convergence", "reasoning", "key_evidence", + "action", ], "additionalProperties": False, } diff --git a/server/tests/investigation/test_multi_orchestrator.py b/server/tests/investigation/test_multi_orchestrator.py index 708fcb2..3161d25 100644 --- a/server/tests/investigation/test_multi_orchestrator.py +++ b/server/tests/investigation/test_multi_orchestrator.py @@ -15,6 +15,7 @@ FindingRecorded, HypothesisEvaluated, HypothesisFormulated, + HypothesisTreeUpdated, InvestigationCompleted, InvestigationError, NegativeControlRecorded, @@ -114,7 +115,11 @@ def _evaluation_json(status: str = "supported") -> str: { "status": status, "confidence": 0.85, + "certainty_of_evidence": "moderate", + "evidence_convergence": "converging", "reasoning": "Strong binding evidence from docking", + "key_evidence": ["Docking score -9.2 kcal/mol (tier 7)"], + "action": "prune", } ) @@ -124,12 +129,31 @@ def _revision_evaluation_json() -> str: { "status": "revised", "confidence": 0.4, + "certainty_of_evidence": "low", + "evidence_convergence": "mixed", "reasoning": "Partial evidence, need narrower scope", + "key_evidence": ["Some binding but weak (tier 7)"], + "action": "branch", "revision": "Thiazolidine with 4-chloro substitution specifically inhibits PBP2a", } ) +def _deepen_evaluation_json() -> str: + return json.dumps( + { + "status": "revised", + "confidence": 0.6, + "certainty_of_evidence": "moderate", + "evidence_convergence": "mixed", + "reasoning": "Promising but needs narrower scope", + "key_evidence": ["Moderate binding (tier 5)"], + "action": "deepen", + "revision": "Thiazolidine analogs with MW < 300 selectively inhibit PBP2a", + } + ) + + def _synthesis_json() -> str: return json.dumps( { @@ -938,3 +962,155 @@ async def _capturing_stream(**kwargs: Any): for kw in captured_kwargs: assert kw.get("output_config") is not None assert kw["output_config"]["format"]["type"] == "json_schema" + + +class TestTreeSearch: + @pytest.mark.asyncio + async def test_tree_search_deepens_promising_hypothesis(self) -> None: + """Director returns action=deepen, new sub-hypothesis created at depth 1.""" + director, researcher, summarizer = _make_clients() + + # formulation, design#1, eval#1 (deepen), design#2, eval#2 (prune), synthesis + director.stream_message = _make_director_side_effect( + _formulation_json(), + _experiment_design_json(), + _deepen_evaluation_json(), + _experiment_design_json(), + _evaluation_json("supported"), + _synthesis_json(), + ) + researcher.create_message = AsyncMock(return_value=_make_text_response("Done.")) + + orchestrator = MultiModelOrchestrator( + director=director, + researcher=researcher, + summarizer=summarizer, + registry=_build_registry(), + max_iterations_per_experiment=1, + ) + + investigation = Investigation(prompt="Test tree deepen") + events = [e async for e in orchestrator.run(investigation)] + + # Original + deepened child + hyp_events = [e for e in events if isinstance(e, HypothesisFormulated)] + assert len(hyp_events) == 2 + + # Child should have parent_id set + child_event = hyp_events[1] + assert child_event.parent_id != "" + + # Check tree updated events + tree_events = [e for e in events if isinstance(e, HypothesisTreeUpdated)] + assert len(tree_events) >= 1 + + deepen_events = [e for e in tree_events if e.action == "deepen"] + assert len(deepen_events) == 1 + assert deepen_events[0].children_count == 1 + + # Child hypothesis should be at depth 1 + assert len(investigation.hypotheses) == 2 + child = investigation.hypotheses[1] + assert child.depth == 1 + assert child.parent_id == investigation.hypotheses[0].id + + @pytest.mark.asyncio + async def test_tree_search_prunes_refuted_hypothesis(self) -> None: + """Director returns action=prune, hypothesis not re-tested.""" + director, researcher, summarizer = _make_clients() + + director.stream_message = _make_director_side_effect( + _formulation_json(), + _experiment_design_json(), + _evaluation_json("refuted"), + _synthesis_json(), + ) + researcher.create_message = AsyncMock(return_value=_make_text_response("Done.")) + + orchestrator = MultiModelOrchestrator( + director=director, + researcher=researcher, + summarizer=summarizer, + registry=_build_registry(), + max_iterations_per_experiment=1, + ) + + investigation = Investigation(prompt="Test tree prune") + events = [e async for e in orchestrator.run(investigation)] + + tree_events = [e for e in events if isinstance(e, HypothesisTreeUpdated)] + assert len(tree_events) == 1 + assert tree_events[0].action == "prune" + + # Only 1 hypothesis (original), no children created + assert len(investigation.hypotheses) == 1 + assert investigation.hypotheses[0].status.value == "refuted" + + @pytest.mark.asyncio + async def test_tree_search_respects_max_depth(self) -> None: + """Hypotheses at max_depth are not further deepened.""" + from ehrlich.investigation.application.tree_manager import TreeManager + + director, researcher, summarizer = _make_clients() + + # With max_depth=1: formulate root (depth=0), test it, deepen to depth=1, + # test child, try to deepen again but depth=1 is at max, so prune instead. + director.stream_message = _make_director_side_effect( + _formulation_json(), + _experiment_design_json(), + _deepen_evaluation_json(), + _experiment_design_json(), + _evaluation_json("supported"), + _synthesis_json(), + ) + researcher.create_message = AsyncMock(return_value=_make_text_response("Done.")) + + tree_manager = TreeManager(max_depth=1) + orchestrator = MultiModelOrchestrator( + director=director, + researcher=researcher, + summarizer=summarizer, + registry=_build_registry(), + max_iterations_per_experiment=1, + tree_manager=tree_manager, + ) + + investigation = Investigation(prompt="Test max depth") + async for _ in orchestrator.run(investigation): + pass + + # Root tested and deepened, child tested but cannot deepen further + assert len(investigation.hypotheses) == 2 + # Child at depth 1 (which == max_depth) should not produce more children + child = investigation.hypotheses[1] + assert child.depth == 1 + assert len(child.children) == 0 + + @pytest.mark.asyncio + async def test_tree_events_have_investigation_id(self) -> None: + """All HypothesisTreeUpdated events include investigation_id.""" + director, researcher, summarizer = _make_clients() + + director.stream_message = _make_director_side_effect( + _formulation_json(), + _experiment_design_json(), + _evaluation_json(), + _synthesis_json(), + ) + researcher.create_message = AsyncMock(return_value=_make_text_response("Done.")) + + orchestrator = MultiModelOrchestrator( + director=director, + researcher=researcher, + summarizer=summarizer, + registry=_build_registry(), + max_iterations_per_experiment=1, + ) + + investigation = Investigation(prompt="Test tree events") + events = [e async for e in orchestrator.run(investigation)] + + tree_events = [e for e in events if isinstance(e, HypothesisTreeUpdated)] + assert len(tree_events) >= 1 + for te in tree_events: + assert te.investigation_id == investigation.id diff --git a/server/tests/investigation/test_tree_manager.py b/server/tests/investigation/test_tree_manager.py new file mode 100644 index 0000000..745f29b --- /dev/null +++ b/server/tests/investigation/test_tree_manager.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import pytest + +from ehrlich.investigation.application.tree_manager import TreeManager +from ehrlich.investigation.domain.hypothesis import Hypothesis, HypothesisStatus +from ehrlich.investigation.domain.investigation import Investigation + + +def _make_hypothesis( + *, + statement: str = "Test hypothesis", + status: HypothesisStatus = HypothesisStatus.PROPOSED, + depth: int = 0, + branch_score: float = 0.0, + parent_id: str = "", + prior_confidence: float = 0.5, + certainty_of_evidence: str = "", +) -> Hypothesis: + h = Hypothesis( + statement=statement, + rationale="Test rationale", + status=status, + depth=depth, + parent_id=parent_id, + prior_confidence=prior_confidence, + certainty_of_evidence=certainty_of_evidence, + ) + h.branch_score = branch_score + return h + + +class TestSelectNext: + def test_returns_highest_branch_score(self) -> None: + tm = TreeManager() + h1 = _make_hypothesis(statement="Low score", branch_score=0.3) + h2 = _make_hypothesis(statement="High score", branch_score=0.9) + h3 = _make_hypothesis(statement="Mid score", branch_score=0.6) + h4 = _make_hypothesis(statement="Also mid", branch_score=0.5) + + result = tm.select_next([h1, h2, h3, h4]) + + assert len(result) == 2 + assert result[0] is h2 + assert result[1] is h3 + + def test_respects_max_depth(self) -> None: + tm = TreeManager(max_depth=2) + h1 = _make_hypothesis(statement="Depth 0", depth=0, branch_score=0.5) + h2 = _make_hypothesis(statement="Depth 2", depth=2, branch_score=0.9) + h3 = _make_hypothesis(statement="Depth 3", depth=3, branch_score=1.0) + + result = tm.select_next([h1, h2, h3]) + + assert len(result) == 1 + assert result[0] is h1 + + def test_only_selects_proposed(self) -> None: + tm = TreeManager() + h1 = _make_hypothesis(status=HypothesisStatus.SUPPORTED, branch_score=0.9) + h2 = _make_hypothesis(status=HypothesisStatus.REFUTED, branch_score=0.8) + h3 = _make_hypothesis(status=HypothesisStatus.PROPOSED, branch_score=0.3) + + result = tm.select_next([h1, h2, h3]) + + assert len(result) == 1 + assert result[0] is h3 + + def test_returns_empty_when_no_candidates(self) -> None: + tm = TreeManager() + h1 = _make_hypothesis(status=HypothesisStatus.SUPPORTED) + + result = tm.select_next([h1]) + + assert result == [] + + def test_depth_tiebreak(self) -> None: + """Shallowest depth wins when scores are equal.""" + tm = TreeManager() + h1 = _make_hypothesis(statement="Deep", depth=2, branch_score=0.5) + h2 = _make_hypothesis(statement="Shallow", depth=0, branch_score=0.5) + + result = tm.select_next([h1, h2]) + + assert len(result) == 2 + assert result[0] is h2 + assert result[1] is h1 + + def test_returns_max_two(self) -> None: + tm = TreeManager() + hypotheses = [ + _make_hypothesis(statement=f"H{i}", branch_score=float(i) / 10) for i in range(5) + ] + + result = tm.select_next(hypotheses) + + assert len(result) == 2 + + +class TestComputeBranchScore: + def test_base_from_prior_confidence(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(prior_confidence=0.8, depth=0) + + score = tm.compute_branch_score(h, inv) + + assert score == 0.8 + + def test_default_confidence_when_zero(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(prior_confidence=0.0, depth=0) + + score = tm.compute_branch_score(h, inv) + + assert score == 0.5 + + def test_depth_discount(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h0 = _make_hypothesis(prior_confidence=1.0, depth=0) + h1 = _make_hypothesis(prior_confidence=1.0, depth=1) + h2 = _make_hypothesis(prior_confidence=1.0, depth=2) + + s0 = tm.compute_branch_score(h0, inv) + s1 = tm.compute_branch_score(h1, inv) + s2 = tm.compute_branch_score(h2, inv) + + assert s0 > s1 > s2 + # depth=0: 1.0 / (1 + 0) = 1.0 + # depth=1: 1.0 / (1 + 0.2) = 0.8333 + # depth=2: 1.0 / (1 + 0.4) = 0.7143 + assert s0 == 1.0 + assert s1 == pytest.approx(0.8333, abs=0.001) + assert s2 == pytest.approx(0.7143, abs=0.001) + + def test_evidence_bonus_from_parent(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + parent = _make_hypothesis( + statement="Parent", + prior_confidence=0.7, + certainty_of_evidence="high", + ) + inv.add_hypothesis(parent) + + child = _make_hypothesis( + statement="Child", + prior_confidence=0.6, + parent_id=parent.id, + depth=1, + ) + + score = tm.compute_branch_score(child, inv) + + # 0.6 * (1/1.2) * 1.3 = 0.65 + expected = round(0.6 * (1.0 / 1.2) * 1.3, 4) + assert score == expected + + def test_no_parent_no_bonus(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(prior_confidence=0.5, depth=1) + + score = tm.compute_branch_score(h, inv) + + # 0.5 * (1/1.2) * 1.0 = 0.4167 + expected = round(0.5 * (1.0 / 1.2), 4) + assert score == expected + + def test_very_low_evidence_penalizes(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + parent = _make_hypothesis( + statement="Weak parent", + certainty_of_evidence="very_low", + ) + inv.add_hypothesis(parent) + + child = _make_hypothesis( + statement="Child", + prior_confidence=0.5, + parent_id=parent.id, + ) + + score = tm.compute_branch_score(child, inv) + + # 0.5 * 1.0 * 0.7 = 0.35 + assert score == 0.35 + + +class TestApplyEvaluation: + def test_deepen_creates_child_at_depth_plus_one(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(statement="Original", depth=1) + inv.add_hypothesis(h) + + evaluation = { + "action": "deepen", + "revision": "Narrower sub-hypothesis", + "confidence": 0.7, + } + + new = tm.apply_evaluation(h, evaluation, inv) + + assert len(new) == 1 + assert new[0].statement == "Narrower sub-hypothesis" + assert new[0].depth == 2 + assert new[0].parent_id == h.id + assert h.id in new[0].parent_id + assert new[0].id in h.children + + def test_branch_creates_sibling_at_same_depth(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(statement="Original", depth=1) + inv.add_hypothesis(h) + + evaluation = { + "action": "branch", + "revision": "Alternative approach", + "confidence": 0.5, + } + + new = tm.apply_evaluation(h, evaluation, inv) + + assert len(new) == 1 + assert new[0].statement == "Alternative approach" + assert new[0].depth == 1 # same depth + assert new[0].parent_id == h.id + assert new[0].id in h.children + + def test_prune_marks_rejected(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(statement="Bad hypothesis", status=HypothesisStatus.REVISED) + + evaluation = {"action": "prune"} + + new = tm.apply_evaluation(h, evaluation, inv) + + assert new == [] + assert h.status == HypothesisStatus.REJECTED + + def test_prune_does_not_override_supported(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(status=HypothesisStatus.SUPPORTED) + + evaluation = {"action": "prune"} + + tm.apply_evaluation(h, evaluation, inv) + + assert h.status == HypothesisStatus.SUPPORTED + + def test_prune_does_not_override_refuted(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(status=HypothesisStatus.REFUTED) + + evaluation = {"action": "prune"} + + tm.apply_evaluation(h, evaluation, inv) + + assert h.status == HypothesisStatus.REFUTED + + def test_deepen_without_revision_returns_empty(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis() + + evaluation = {"action": "deepen"} + + new = tm.apply_evaluation(h, evaluation, inv) + + assert new == [] + assert len(h.children) == 0 + + def test_branch_without_revision_returns_empty(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis() + + evaluation = {"action": "branch"} + + new = tm.apply_evaluation(h, evaluation, inv) + + assert new == [] + + def test_backward_compat_no_action_defaults_to_prune(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(status=HypothesisStatus.TESTING) + + evaluation = {"status": "refuted", "confidence": 0.1} + + new = tm.apply_evaluation(h, evaluation, inv) + + assert new == [] + assert h.status == HypothesisStatus.REJECTED + + def test_new_hypothesis_gets_branch_score(self) -> None: + tm = TreeManager() + inv = Investigation(prompt="test") + h = _make_hypothesis(prior_confidence=0.7, depth=0) + inv.add_hypothesis(h) + + evaluation = { + "action": "deepen", + "revision": "Deeper hypothesis", + "confidence": 0.6, + } + + new = tm.apply_evaluation(h, evaluation, inv) + + assert len(new) == 1 + assert new[0].branch_score > 0.0 + + +class TestShouldContinue: + def test_true_with_proposed_below_max_depth(self) -> None: + tm = TreeManager(max_depth=3) + hypotheses = [ + _make_hypothesis(status=HypothesisStatus.SUPPORTED), + _make_hypothesis(status=HypothesisStatus.PROPOSED, depth=1), + ] + + assert tm.should_continue(hypotheses) is True + + def test_false_with_no_proposed(self) -> None: + tm = TreeManager() + hypotheses = [ + _make_hypothesis(status=HypothesisStatus.SUPPORTED), + _make_hypothesis(status=HypothesisStatus.REFUTED), + ] + + assert tm.should_continue(hypotheses) is False + + def test_false_when_all_proposed_at_max_depth(self) -> None: + tm = TreeManager(max_depth=2) + hypotheses = [ + _make_hypothesis(status=HypothesisStatus.PROPOSED, depth=2), + _make_hypothesis(status=HypothesisStatus.PROPOSED, depth=3), + ] + + assert tm.should_continue(hypotheses) is False + + def test_true_with_empty_list(self) -> None: + tm = TreeManager() + + assert tm.should_continue([]) is False From 5301c6034510a93326eae6bc4ce4fc3f278699e7 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 16:34:14 -0700 Subject: [PATCH 25/35] refactor(console): flatten molecule feature folder, remove components/ nesting --- .../investigation/components/CandidateComparison.tsx | 2 +- .../features/investigation/components/CandidateDetail.tsx | 8 +++----- .../features/investigation/components/CandidateTable.tsx | 2 +- .../features/investigation/components/HypothesisCard.tsx | 2 +- .../investigation/components/InvestigationComparison.tsx | 6 +++--- .../features/molecule/{components => }/DockingViewer.tsx | 0 .../molecule/{components => }/MolViewer2D.test.tsx | 0 .../features/molecule/{components => }/MolViewer2D.tsx | 0 .../features/molecule/{components => }/MolViewer3D.tsx | 0 .../features/molecule/{components => }/PropertyCard.tsx | 0 console/src/features/molecule/{components => }/index.ts | 0 11 files changed, 9 insertions(+), 11 deletions(-) rename console/src/features/molecule/{components => }/DockingViewer.tsx (100%) rename console/src/features/molecule/{components => }/MolViewer2D.test.tsx (100%) rename console/src/features/molecule/{components => }/MolViewer2D.tsx (100%) rename console/src/features/molecule/{components => }/MolViewer3D.tsx (100%) rename console/src/features/molecule/{components => }/PropertyCard.tsx (100%) rename console/src/features/molecule/{components => }/index.ts (100%) diff --git a/console/src/features/investigation/components/CandidateComparison.tsx b/console/src/features/investigation/components/CandidateComparison.tsx index 22a65d3..c481c23 100644 --- a/console/src/features/investigation/components/CandidateComparison.tsx +++ b/console/src/features/investigation/components/CandidateComparison.tsx @@ -1,6 +1,6 @@ import { X } from "lucide-react"; import { cn } from "@/shared/lib/utils"; -import { MolViewer2D } from "@/features/molecule/components/MolViewer2D"; +import { MolViewer2D } from "@/features/molecule"; import type { CandidateRow, DomainDisplayConfig } from "../types"; interface CandidateComparisonProps { diff --git a/console/src/features/investigation/components/CandidateDetail.tsx b/console/src/features/investigation/components/CandidateDetail.tsx index 15d3900..97b365b 100644 --- a/console/src/features/investigation/components/CandidateDetail.tsx +++ b/console/src/features/investigation/components/CandidateDetail.tsx @@ -1,7 +1,6 @@ import { useEffect, useState } from "react"; -import { MolViewer2D } from "@/features/molecule/components/MolViewer2D"; -import { MolViewer3D } from "@/features/molecule/components/MolViewer3D"; +import { MolViewer2D, MolViewer3D } from "@/features/molecule"; import { apiFetch } from "@/shared/lib/api"; interface ConformerData { @@ -93,11 +92,10 @@ function MolecularDetail({ smiles, name }: { smiles: string; name: string }) {
Properties {descriptors.passes_lipinski ? "Lipinski OK" : "Lipinski Fail"} diff --git a/console/src/features/investigation/components/CandidateTable.tsx b/console/src/features/investigation/components/CandidateTable.tsx index 74f9143..0cc13c7 100644 --- a/console/src/features/investigation/components/CandidateTable.tsx +++ b/console/src/features/investigation/components/CandidateTable.tsx @@ -1,7 +1,7 @@ import { useState } from "react"; import { ChevronDown, ChevronRight, GitCompareArrows, Square, CheckSquare } from "lucide-react"; import { cn } from "@/shared/lib/utils"; -import { MolViewer2D } from "@/features/molecule/components/MolViewer2D"; +import { MolViewer2D } from "@/features/molecule"; import type { CandidateRow, DomainDisplayConfig } from "../types"; import { CandidateDetail } from "./CandidateDetail"; import { CandidateComparison } from "./CandidateComparison"; diff --git a/console/src/features/investigation/components/HypothesisCard.tsx b/console/src/features/investigation/components/HypothesisCard.tsx index 1ba9385..256e0b6 100644 --- a/console/src/features/investigation/components/HypothesisCard.tsx +++ b/console/src/features/investigation/components/HypothesisCard.tsx @@ -1,7 +1,7 @@ import { ChevronDown, ChevronRight } from "lucide-react"; import { useMemo, useState } from "react"; import { cn } from "@/shared/lib/utils"; -import { MolViewer2D } from "@/features/molecule/components/MolViewer2D"; +import { MolViewer2D } from "@/features/molecule"; import type { Hypothesis } from "../types"; import { parseHypothesisStatement } from "../lib/parse-hypothesis"; diff --git a/console/src/features/investigation/components/InvestigationComparison.tsx b/console/src/features/investigation/components/InvestigationComparison.tsx index 012039f..3efec48 100644 --- a/console/src/features/investigation/components/InvestigationComparison.tsx +++ b/console/src/features/investigation/components/InvestigationComparison.tsx @@ -10,7 +10,7 @@ import { Legend, } from "recharts"; import { cn } from "@/shared/lib/utils"; -import { MolViewer2D } from "@/features/molecule/components/MolViewer2D"; +import { MolViewer2D } from "@/features/molecule"; import { VIZ_COLORS } from "@/features/visualization/theme"; import type { InvestigationDetail, CandidateRow, Finding } from "../types"; @@ -479,8 +479,8 @@ function HypothesisSummary({ {Object.entries(counts).map(([status, count]) => ( {count} {status} diff --git a/console/src/features/molecule/components/DockingViewer.tsx b/console/src/features/molecule/DockingViewer.tsx similarity index 100% rename from console/src/features/molecule/components/DockingViewer.tsx rename to console/src/features/molecule/DockingViewer.tsx diff --git a/console/src/features/molecule/components/MolViewer2D.test.tsx b/console/src/features/molecule/MolViewer2D.test.tsx similarity index 100% rename from console/src/features/molecule/components/MolViewer2D.test.tsx rename to console/src/features/molecule/MolViewer2D.test.tsx diff --git a/console/src/features/molecule/components/MolViewer2D.tsx b/console/src/features/molecule/MolViewer2D.tsx similarity index 100% rename from console/src/features/molecule/components/MolViewer2D.tsx rename to console/src/features/molecule/MolViewer2D.tsx diff --git a/console/src/features/molecule/components/MolViewer3D.tsx b/console/src/features/molecule/MolViewer3D.tsx similarity index 100% rename from console/src/features/molecule/components/MolViewer3D.tsx rename to console/src/features/molecule/MolViewer3D.tsx diff --git a/console/src/features/molecule/components/PropertyCard.tsx b/console/src/features/molecule/PropertyCard.tsx similarity index 100% rename from console/src/features/molecule/components/PropertyCard.tsx rename to console/src/features/molecule/PropertyCard.tsx diff --git a/console/src/features/molecule/components/index.ts b/console/src/features/molecule/index.ts similarity index 100% rename from console/src/features/molecule/components/index.ts rename to console/src/features/molecule/index.ts From d2cd5b1c0f54ffedc6fb243ec214988b6d0c82f2 Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 23:00:42 -0700 Subject: [PATCH 26/35] feat(13D): add 7 US federal data API clients, 5 new tools, extend economic indicators Census Bureau (ACS 5-year), BLS (time series), USAspending (awards/grants), College Scorecard (education outcomes), HUD (FMR/income limits), CDC WONDER (mortality/natality via XML), and data.gov (CKAN dataset discovery). Extended search_economic_indicators with BLS and Census sources. 85 -> 90 tools, 19 -> 25 data sources, 87 new tests (995 total). --- CLAUDE.md | 12 +- README.md | 26 +- docs/architecture.md | 4 +- docs/roadmap/README.md | 2 +- docs/roadmap/phase-13-impact-evaluation.md | 67 +-- server/src/ehrlich/api/routes/methodology.py | 44 +- server/src/ehrlich/api/routes/stats.py | 2 +- .../impact/application/impact_service.py | 118 +++++ server/src/ehrlich/impact/domain/entities.py | 46 ++ .../src/ehrlich/impact/domain/repository.py | 45 ++ .../impact/infrastructure/bls_client.py | 133 +++++ .../impact/infrastructure/cdc_client.py | 154 ++++++ .../impact/infrastructure/census_client.py | 121 +++++ .../college_scorecard_client.py | 145 ++++++ .../impact/infrastructure/datagov_client.py | 115 +++++ .../impact/infrastructure/hud_client.py | 114 +++++ .../infrastructure/usaspending_client.py | 110 +++++ server/src/ehrlich/impact/tools.py | 233 ++++++++- .../application/registry_factory.py | 12 +- .../investigation/domain/domains/impact.py | 75 ++- server/tests/api/test_methodology.py | 4 +- server/tests/api/test_stats.py | 6 +- server/tests/impact/test_impact_service.py | 217 +++++++- server/tests/impact/test_us_clients.py | 465 ++++++++++++++++++ server/tests/impact/test_us_tools.py | 254 ++++++++++ server/tests/integration/test_e2e.py | 4 +- web/src/lib/constants.ts | 18 +- 27 files changed, 2469 insertions(+), 77 deletions(-) create mode 100644 server/src/ehrlich/impact/infrastructure/bls_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/cdc_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/census_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/college_scorecard_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/datagov_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/hud_client.py create mode 100644 server/src/ehrlich/impact/infrastructure/usaspending_client.py create mode 100644 server/tests/impact/test_us_clients.py create mode 100644 server/tests/impact/test_us_tools.py diff --git a/CLAUDE.md b/CLAUDE.md index 6d4cb08..a6bc2a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ AGPL-3.0 | Claude Code Hackathon (Feb 10-16, 2026) Domain-agnostic scientific discovery platform using Claude as a hypothesis-driven reasoning engine. Pluggable `DomainConfig` + `DomainRegistry` system. Four domains: Molecular Science, Training Science, Nutrition Science, Impact Evaluation. -See `README.md` for setup, tools catalog (85 tools), data sources (18 APIs), API endpoints, and demo instructions. +See `README.md` for setup, tools catalog (90 tools), data sources (24 APIs), API endpoints, and demo instructions. ## Architecture @@ -14,7 +14,7 @@ DDD monorepo: `server/` (Python 3.12) + `console/` (React 19 / TypeScript / Bun) ``` Opus 4.6 (Director) -- Formulates hypotheses, designs experiments, evaluates evidence, synthesizes (NO tools) -Sonnet 4.5 (Researcher) -- Executes experiments with 85 tools (parallel: 2 experiments per batch) +Sonnet 4.5 (Researcher) -- Executes experiments with 90 tools (parallel: 2 experiments per batch) Haiku 4.5 (Summarizer) -- Compresses large tool outputs (>2000 chars), PICO decomposition, evidence grading ``` @@ -33,7 +33,7 @@ Always uses `MultiModelOrchestrator`. Hypotheses tested in parallel batches of 2 | simulation | `server/src/ehrlich/simulation/` | Docking, ADMET, resistance, targets (RCSB PDB, UniProt, Open Targets), toxicity (EPA CompTox) | | training | `server/src/ehrlich/training/` | Evidence analysis, protocol comparison, injury risk, clinical trials, PubMed, exercises | | nutrition | `server/src/ehrlich/nutrition/` | Supplement evidence, labels, nutrients, safety, interactions, adequacy, inflammatory index | -| impact | `server/src/ehrlich/impact/` | Economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking | +| impact | `server/src/ehrlich/impact/` | Economic indicators (World Bank, WHO GHO, FRED, Census, BLS), health (CDC WONDER), spending (USAspending), education (College Scorecard), housing (HUD), open data (data.gov) | | investigation | `server/src/ehrlich/investigation/` | Multi-model orchestration + PostgreSQL persistence + domain registry + MCP bridge | ### Dependency Rules (STRICT) @@ -170,7 +170,7 @@ All paths relative to `server/src/ehrlich/`. | `training/application/training_service.py` | Evidence, protocols, injury risk, metrics, clinical trials, PubMed, exercises | | `nutrition/application/nutrition_service.py` | Supplements, labels, nutrients, safety, adequacy, interactions, ratios | | `nutrition/domain/dri.py` | DRI tables (EAR, RDA, AI, UL by age/sex) | -| `impact/application/impact_service.py` | Economic indicators, benchmarks, program comparison | +| `impact/application/impact_service.py` | Economic indicators, benchmarks, program comparison, spending, education, housing, open data | | `investigation/application/multi_orchestrator.py` | Director-Worker-Summarizer orchestrator (main 6-phase loop) | | `investigation/application/tool_dispatcher.py` | Tool execution with caching, `search_prior_research` and `query_uploaded_data` interception | | `investigation/application/tree_manager.py` | Hypothesis tree exploration: scoring, selection, deepen/branch/prune actions | @@ -192,10 +192,10 @@ All paths relative to `server/src/ehrlich/api/`. | File | Purpose | |------|---------| | `auth.py` | WorkOS JWT middleware (JWKS, header + query param) | -| `routes/investigation.py` | REST + SSE + paper export, 85-tool registry, domain registry, credits, BYOK | +| `routes/investigation.py` | REST + SSE + paper export, 90-tool registry, domain registry, credits, BYOK | | `routes/upload.py` | File upload (CSV/XLSX/PDF) | | `routes/molecule.py` | Molecule depiction, conformer, descriptors, targets | -| `sse.py` | Domain event to SSE conversion (20 types) | +| `sse.py` | Domain event to SSE conversion (21 types) | ## Key Files (Console) diff --git a/README.md b/README.md index 41ebfe4..ed44481 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Ehrlich is **COSS** (Commercial Open-Source Software) -- the same model used by | **Self-host** | Clone the repo, bring your own Anthropic API key | Free. No limits, no credits, no account needed | | **Hosted instance** | Use app.ehrlich.dev | Credits cover Anthropic API costs (Opus is expensive) | -Credits exist because Claude Opus costs real money per investigation. They make scientific reasoning **accessible** -- not monetized. A student in Mexico and a pharma company in Boston get the same 85 tools, the same 17 data sources, the same methodology. The model quality is the only variable. +Credits exist because Claude Opus costs real money per investigation. They make scientific reasoning **accessible** -- not monetized. A student in Mexico and a pharma company in Boston get the same 90 tools, the same 24 data sources, the same methodology. The model quality is the only variable. The AI is the scientist. The platform is the laboratory. @@ -44,7 +44,8 @@ Ehrlich is **domain-agnostic**. The hypothesis-driven engine works for any scien - **Social program analysis** -- "Evaluate the effectiveness of Sonora's sports scholarship program" - **Causal inference** -- "Does the conditional cash transfer reduce school dropout rates?" (4 methods: DiD, PSM, RDD, Synthetic Control) - **Cross-country benchmarking** -- "Compare cost-effectiveness of state sports programs in Mexico" -- Economic indicators from World Bank, WHO GHO, and FRED (800K+ time series) +- Economic indicators from World Bank, WHO GHO, FRED, Census Bureau, and BLS +- US federal data: USAspending grants, College Scorecard education outcomes, HUD housing data, CDC WONDER mortality/natality, data.gov open datasets - Cross-program comparison and international benchmarking - Domain-agnostic causal inference tools (in analysis/ context) usable by any domain @@ -64,7 +65,7 @@ Ehrlich follows Domain-Driven Design with eleven bounded contexts: | **training** | Exercise physiology: evidence analysis, protocol comparison, injury risk, training metrics, clinical trials | | **nutrition** | Nutrition science: supplement evidence, labels, nutrients, safety, interactions, adequacy, inflammatory index | | **investigation** | Multi-model agent orchestration with Director-Worker-Summarizer pattern + domain registry + MCP bridge | -| **impact** | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED), cross-program comparison, benchmarking. Causal estimators live in analysis/ (domain-agnostic) | +| **impact** | Social program evaluation: economic indicators (World Bank, WHO GHO, FRED, Census, BLS), health (CDC WONDER), spending (USAspending), education (College Scorecard), housing (HUD), open data (data.gov). Causal estimators live in analysis/ (domain-agnostic) | ### Multi-Model Architecture @@ -72,7 +73,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga ``` Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) -Sonnet 4.5 (Researcher) -- Executes experiments with 85 tools (10-20 calls) +Sonnet 4.5 (Researcher) -- Executes experiments with 90 tools (10-20 calls) Haiku 4.5 (Summarizer) -- Compresses large outputs, classifies domains (5-10 calls) ``` @@ -100,10 +101,16 @@ Cost: ~$3-4 per investigation (vs ~$11 with all-Opus). | [World Bank](https://data.worldbank.org/) | Development indicators by country (GDP, poverty, education) | 190+ countries | | [WHO GHO](https://www.who.int/data/gho) | Global health statistics (life expectancy, mortality, disease) | 190+ countries | | [FRED](https://fred.stlouisfed.org/) | US economic time series (GDP, unemployment, CPI) | 800K+ series | +| [Census Bureau](https://data.census.gov/) | US demographics, poverty, education (ACS 5-year) | 50 states + territories | +| [BLS](https://www.bls.gov/) | US labor statistics (unemployment, CPI, wages) | 130K+ series | +| [USAspending](https://www.usaspending.gov/) | Federal spending awards and grants | All federal agencies | +| [College Scorecard](https://collegescorecard.ed.gov/) | US higher education outcomes (completion, earnings) | 6K+ institutions | +| [HUD](https://www.huduser.gov/) | Fair Market Rents, income limits, housing data | All US counties | +| [CDC WONDER](https://wonder.cdc.gov/) | US mortality, natality, public health statistics | National-level | All data sources are free and open-access. -## 85 Tools +## 90 Tools | Context | Tool | Description | |---------|------|-------------| @@ -162,9 +169,14 @@ All data sources are free and open-access. | Nutrition | `check_interactions` | Drug-supplement interaction screening via RxNav | | Nutrition | `analyze_nutrient_ratios` | Key nutrient ratio analysis (omega-6:3, Ca:Mg, etc.) | | Nutrition | `compute_inflammatory_index` | Simplified Dietary Inflammatory Index scoring | -| Impact | `search_economic_indicators` | Query economic time series from FRED, World Bank, or WHO GHO | -| Impact | `fetch_benchmark` | Get comparison values from international sources | +| Impact | `search_economic_indicators` | Query economic time series from FRED, BLS, Census, World Bank, or WHO GHO | +| Impact | `search_health_indicators` | Search WHO GHO or CDC WONDER for health indicators | +| Impact | `fetch_benchmark` | Get comparison values from international or US data sources | | Impact | `compare_programs` | Cross-program comparison using statistical tests | +| Impact | `search_spending_data` | Search USAspending.gov for federal spending awards and grants | +| Impact | `search_education_data` | Search College Scorecard for US higher education outcomes | +| Impact | `search_housing_data` | Search HUD for Fair Market Rents and income limits | +| Impact | `search_open_data` | Search data.gov CKAN catalog for US federal open datasets | | Visualization | `render_binding_scatter` | Scatter plot of compound binding affinities | | Visualization | `render_admet_radar` | Radar chart of ADMET/drug-likeness properties | | Visualization | `render_training_timeline` | Training load timeline with ACWR danger zones | diff --git a/docs/architecture.md b/docs/architecture.md index bb54c29..c2dfad2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,7 +47,7 @@ Ehrlich uses a three-tier Claude model architecture for cost-efficient investiga Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesizes (3-5 calls) │ NO tool access, structured JSON responses only │ - ├── Sonnet 4.5 (Researcher) -- Executes experiments with 85 domain-filtered tools (10-20 calls, parallel x2) + ├── Sonnet 4.5 (Researcher) -- Executes experiments with 90 domain-filtered tools (10-20 calls, parallel x2) │ Tool-calling loop with max_iterations_per_experiment guard │ └── Haiku 4.5 (Summarizer) -- Compresses large tool outputs >2000 chars, PICO+classification, evidence grading @@ -248,7 +248,7 @@ api/ -> investigation/application/ only e. `TreeManager.apply_evaluation()` processes action: deepen creates child at depth+1, branch creates sibling at same depth, prune marks REJECTED. Loop repeats with `select_next()` until no explorable hypotheses remain below `max_depth` 9. **Negative controls** recorded from formulation suggestions 10. **Director** synthesizes final report with candidates, citations, cost -11. All events stream via SSE (20 event types) to Console in real-time +11. All events stream via SSE (21 event types) to Console in real-time - `DomainDetected` sends display config (score columns, visualization type) to frontend - `FindingRecorded` includes evidence + source provenance (source_type, source_id) + evidence_level (1-6) - `LiteratureSurveyCompleted` carries PICO, search stats, evidence grade, self-assessment diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index d76187b..123b20b 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -86,7 +86,7 @@ Enhancement Enhancement Sports | Phase 13: Impact Evaluation Domain -- IN PROGRESS (Causal inference + Document upload + MX/US APIs) - 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) DONE -> 13C (Mexico) -> 13D (US) -> 13E (MCP) + 13A (Foundation) DONE -> 13B (Causal Inference) DONE -> 13A-2 (Upload) DONE -> 13D (US) DONE -> 13C (Mexico) -> 13E (MCP) | Phase 14: Scientific Engine Hardening -- IN PROGRESS (parallel with 13A-2) 14A (Orchestrator Split) DONE ──┬── 14B (Route Cleanup) DONE ──┬── 14D (Tool Enrichment) DONE diff --git a/docs/roadmap/phase-13-impact-evaluation.md b/docs/roadmap/phase-13-impact-evaluation.md index 02443db..9aeb50a 100644 --- a/docs/roadmap/phase-13-impact-evaluation.md +++ b/docs/roadmap/phase-13-impact-evaluation.md @@ -1,6 +1,6 @@ Back to [Roadmap Index](README.md) -# Phase 13: Impact Evaluation Domain -- IN PROGRESS +# Phase 13: Impact Evaluation Domain -- IN PROGRESS (13C, 13E remaining) New bounded context for hypothesis-driven causal analysis of social programs. Domain-agnostic methodology works for any program type (sports, health, education, employment, housing) in any country. Initial focus: Mexico and US. Causal inference methods (DiD, PSM, RDD, Synthetic Control) live in `analysis/` as domain-agnostic tools usable by any domain. No existing platform combines autonomous hypothesis formulation, automated causal inference, public API integration, and evidence-graded reporting. See `docs/adr/impact-evaluation-domain.md` for full design. @@ -95,18 +95,25 @@ Refactored causal inference from impact-specific to domain-agnostic. All causal - [ ] Frontend: `MIRTable.tsx` logical framework matrix - [ ] CONEVAL ECR report template for `generate_evaluation_report` -## Phase 13D: US Integration - -- [ ] Census Bureau API client (`https://api.census.gov/data/`) -- ACS, demographics -- [ ] BLS API client (`https://api.bls.gov/publicAPI/v2/`) -- employment, wages, CPI -- [ ] USAspending API client (`https://api.usaspending.gov/api/v2/`) -- federal grants/contracts -- [ ] College Scorecard API client (`https://api.data.gov/ed/collegescorecard/v1/`) -- education outcomes -- [ ] HUD API client (`https://www.huduser.gov/hudapi/public/`) -- housing data -- [ ] CDC WONDER client (`https://wonder.cdc.gov/`) -- public health data -- [ ] `search_health_indicators` tool -- query WHO GHO or CDC WONDER -- [ ] `search_open_data` tool -- query datos.gob.mx or data.gov CKAN portals -- [ ] WWC evidence tier classification in evaluation reports -- [ ] US-formatted evaluation report template +## Phase 13D: US Integration -- DONE + +- [x] Census Bureau API client (`https://api.census.gov/data/`) -- ACS 5-year demographics, poverty, education +- [x] BLS API client (`https://api.bls.gov/publicAPI/v2/`) -- employment, wages, CPI time series +- [x] USAspending API client (`https://api.usaspending.gov/api/v2/`) -- federal grants/contracts search +- [x] College Scorecard API client (`https://api.data.gov/ed/collegescorecard/v1/`) -- education outcomes +- [x] HUD API client (`https://www.huduser.gov/hudapi/public/`) -- Fair Market Rents, income limits +- [x] CDC WONDER client (`https://wonder.cdc.gov/`) -- mortality/natality via XML API +- [x] data.gov CKAN client (`https://catalog.data.gov/api/3/action/`) -- federal dataset discovery +- [x] `search_health_indicators` tool -- routes to WHO GHO or CDC WONDER by source +- [x] `search_spending_data` tool -- USAspending awards/grants search +- [x] `search_education_data` tool -- College Scorecard school outcomes +- [x] `search_housing_data` tool -- HUD FMR and income limits +- [x] `search_open_data` tool -- data.gov CKAN dataset metadata +- [x] Extended `search_economic_indicators` with BLS and Census sources +- [x] WWC evidence tier classification (already in `analysis/domain/evidence_standards.py`) +- [x] US-focused domain config: 6 template prompts, expanded detection keywords, experiment examples + +**Counts:** 85 -> 90 tools (+5 new), 19 -> 25 data sources (+6: Census, BLS, USAspending, College Scorecard, HUD, CDC WONDER). Also added data.gov client (shares tool with datos.gob.mx). ## Phase 13E: MCP Bridge Self-Service @@ -115,21 +122,25 @@ Refactored causal inference from impact-specific to domain-agnostic. All causal - [ ] Per-organization data source management - [ ] Tool auto-tagging with domain tags for discovery -## Data Sources (Planned: 8+ new, 3 implemented in 13A) - -| Source | API | Purpose | Auth | Phase | -|--------|-----|---------|------|-------| -| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country | None | 13A | -| WHO GHO | `https://ghoapi.azureedge.net/api/` | Health statistics by country | None | 13A | -| FRED | `https://api.stlouisfed.org/fred/` | 800K+ economic time series | API key | 13A | -| INEGI Indicadores | `https://www.inegi.org.mx/app/api/indicadores/` | Mexico census, demographics | Token | 13C | -| INEGI DENUE | `https://www.inegi.org.mx/app/api/denue/v1/` | Mexico business directory (5M+) | Token | 13C | -| datos.gob.mx | `https://datos.gob.mx/api/3/action/` | Mexico open data (CKAN, 1000+ datasets) | Token (optional) | 13C | -| Transparencia Presupuestaria | `https://nptp.hacienda.gob.mx/` | Mexico budget execution | None | 13C | -| Banxico SIE | `https://www.banxico.org.mx/SieAPIRest/service/v1/` | Mexico economic indicators | Token | 13C | -| US Census Bureau | `https://api.census.gov/data/` | ACS, demographics | API key (optional) | 13D | -| BLS | `https://api.bls.gov/publicAPI/v2/` | Employment, wages, CPI | API key | 13D | -| USAspending | `https://api.usaspending.gov/api/v2/` | Federal grants, contracts | None | 13D | +## Data Sources (10 implemented, 5 planned for 13C) + +| Source | API | Purpose | Auth | Phase | Status | +|--------|-----|---------|------|-------|--------| +| World Bank | `https://api.worldbank.org/v2/` | Development indicators by country | None | 13A | DONE | +| WHO GHO | `https://ghoapi.azureedge.net/api/` | Health statistics by country | None | 13A | DONE | +| FRED | `https://api.stlouisfed.org/fred/` | 800K+ economic time series | API key | 13A | DONE | +| US Census Bureau | `https://api.census.gov/data/` | ACS 5-year demographics | API key (optional) | 13D | DONE | +| BLS | `https://api.bls.gov/publicAPI/v2/` | Employment, wages, CPI | API key | 13D | DONE | +| USAspending | `https://api.usaspending.gov/api/v2/` | Federal grants, contracts | None | 13D | DONE | +| College Scorecard | `https://api.data.gov/ed/collegescorecard/v1/` | Education outcomes | API key | 13D | DONE | +| HUD | `https://www.huduser.gov/hudapi/public/` | Housing data (FMR, income limits) | Bearer token | 13D | DONE | +| CDC WONDER | `https://wonder.cdc.gov/` | Mortality, natality (XML API) | None | 13D | DONE | +| data.gov | `https://catalog.data.gov/api/3/action/` | Federal dataset discovery (CKAN) | None | 13D | DONE | +| INEGI Indicadores | `https://www.inegi.org.mx/app/api/indicadores/` | Mexico census, demographics | Token | 13C | TODO | +| INEGI DENUE | `https://www.inegi.org.mx/app/api/denue/v1/` | Mexico business directory (5M+) | Token | 13C | TODO | +| datos.gob.mx | `https://datos.gob.mx/api/3/action/` | Mexico open data (CKAN, 1000+ datasets) | Token (optional) | 13C | TODO | +| Transparencia Presupuestaria | `https://nptp.hacienda.gob.mx/` | Mexico budget execution | None | 13C | TODO | +| Banxico SIE | `https://www.banxico.org.mx/SieAPIRest/service/v1/` | Mexico economic indicators | Token | 13C | TODO | ## Tools (Planned: ~18 new tools + ~6 viz tools) diff --git a/server/src/ehrlich/api/routes/methodology.py b/server/src/ehrlich/api/routes/methodology.py index 147db49..815bdd7 100644 --- a/server/src/ehrlich/api/routes/methodology.py +++ b/server/src/ehrlich/api/routes/methodology.py @@ -93,7 +93,7 @@ "role": "Researcher", "model_id": "claude-sonnet-4-5-20250929", "purpose": ( - "Executes experiments with 73 tools in parallel batches. " + "Executes experiments with 90 tools in parallel batches. " "Records findings with evidence provenance and citations." ), }, @@ -241,6 +241,48 @@ "auth": "api_key", "context": "impact", }, + { + "name": "Census Bureau", + "url": "https://api.census.gov/data", + "purpose": "US demographics, poverty, education (ACS 5-year)", + "auth": "api_key", + "context": "impact", + }, + { + "name": "BLS", + "url": "https://api.bls.gov/publicAPI/v2", + "purpose": "US labor statistics (unemployment, CPI, wages)", + "auth": "api_key", + "context": "impact", + }, + { + "name": "USAspending", + "url": "https://api.usaspending.gov/api/v2", + "purpose": "Federal spending awards and grants", + "auth": "none", + "context": "impact", + }, + { + "name": "College Scorecard", + "url": "https://api.data.gov/ed/collegescorecard/v1", + "purpose": "US higher education outcomes (completion, earnings)", + "auth": "api_key", + "context": "impact", + }, + { + "name": "HUD", + "url": "https://www.huduser.gov/hudapi/public", + "purpose": "Fair Market Rents, income limits, housing data", + "auth": "bearer_token", + "context": "impact", + }, + { + "name": "CDC WONDER", + "url": "https://wonder.cdc.gov/controller/datarequest", + "purpose": "US mortality, natality, public health statistics", + "auth": "none", + "context": "impact", + }, ] # ── Tool context mapping (tag -> display name) ───────────────────────── diff --git a/server/src/ehrlich/api/routes/stats.py b/server/src/ehrlich/api/routes/stats.py index 793f030..0ff5c9a 100644 --- a/server/src/ehrlich/api/routes/stats.py +++ b/server/src/ehrlich/api/routes/stats.py @@ -12,7 +12,7 @@ router = APIRouter(tags=["stats"]) PHASE_COUNT = 6 -DATA_SOURCE_COUNT = 19 +DATA_SOURCE_COUNT = 25 class StatsResponse(BaseModel): diff --git a/server/src/ehrlich/impact/application/impact_service.py b/server/src/ehrlich/impact/application/impact_service.py index 0aab047..6673c0d 100644 --- a/server/src/ehrlich/impact/application/impact_service.py +++ b/server/src/ehrlich/impact/application/impact_service.py @@ -8,13 +8,21 @@ if TYPE_CHECKING: from ehrlich.impact.domain.entities import ( Benchmark, + DatasetMetadata, EconomicSeries, + EducationRecord, HealthIndicator, + HousingData, + SpendingRecord, ) from ehrlich.impact.domain.repository import ( DevelopmentDataRepository, EconomicDataRepository, + EducationDataRepository, HealthDataRepository, + HousingDataRepository, + OpenDataRepository, + SpendingDataRepository, ) @@ -24,10 +32,24 @@ def __init__( economic: EconomicDataRepository | None = None, health: HealthDataRepository | None = None, development: DevelopmentDataRepository | None = None, + spending: SpendingDataRepository | None = None, + education: EducationDataRepository | None = None, + housing: HousingDataRepository | None = None, + open_data: OpenDataRepository | None = None, + bls: EconomicDataRepository | None = None, + census: DevelopmentDataRepository | None = None, + cdc: HealthDataRepository | None = None, ) -> None: self._economic = economic self._health = health self._development = development + self._spending = spending + self._education = education + self._housing = housing + self._open_data = open_data + self._bls = bls + self._census = census + self._cdc = cdc self._stats = StatisticsService() async def search_economic_data(self, query: str, limit: int = 10) -> list[EconomicSeries]: @@ -35,6 +57,11 @@ async def search_economic_data(self, query: str, limit: int = 10) -> list[Econom return [] return await self._economic.search_series(query, limit) + async def search_bls_data(self, query: str, limit: int = 10) -> list[EconomicSeries]: + if not self._bls: + return [] + return await self._bls.search_series(query, limit) + async def get_economic_series( self, series_id: str, start: str | None = None, end: str | None = None ) -> EconomicSeries | None: @@ -54,6 +81,17 @@ async def search_health_data( return [] return await self._health.search_indicators(indicator, country, year_start, year_end, limit) + async def search_cdc_data( + self, + indicator: str, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[HealthIndicator]: + if not self._cdc: + return [] + return await self._cdc.search_indicators(indicator, "US", year_start, year_end, limit) + async def search_development_data( self, indicator: str, @@ -68,6 +106,61 @@ async def search_development_data( indicator, country, year_start, year_end, limit ) + async def search_census_data( + self, + indicator: str, + state: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[Benchmark]: + if not self._census: + return [] + return await self._census.search_indicators( + indicator, state, year_start, year_end, limit + ) + + async def search_spending_data( + self, + query: str, + agency: str | None = None, + year: int | None = None, + limit: int = 10, + ) -> list[SpendingRecord]: + if not self._spending: + return [] + return await self._spending.search_awards(query, agency, year, limit) + + async def search_education_data( + self, + query: str, + state: str | None = None, + limit: int = 10, + ) -> list[EducationRecord]: + if not self._education: + return [] + return await self._education.search_schools(query, state, limit) + + async def search_housing_data( + self, + state: str, + county: str | None = None, + year: int | None = None, + ) -> list[HousingData]: + if not self._housing: + return [] + return await self._housing.search_housing_data(state, county, year) + + async def search_open_data( + self, + query: str, + organization: str | None = None, + limit: int = 10, + ) -> list[DatasetMetadata]: + if not self._open_data: + return [] + return await self._open_data.search_datasets(query, organization, limit) + async def fetch_benchmark( self, indicator: str, @@ -92,12 +185,37 @@ async def fetch_benchmark( for s in series ] + if source == "bls": + series = await self.search_bls_data(indicator, limit=5) + return [ + { + "source": "BLS", + "series_id": s.series_id, + "title": s.title, + "unit": s.unit, + "frequency": s.frequency, + } + for s in series + ] + + if source == "census": + benchmarks = await self.search_census_data( + indicator, country, year_start, year_end, limit=10 + ) + return [asdict(b) for b in benchmarks] + if source in ("who", "health"): health_results = await self.search_health_data( indicator, country, year_start, year_end, limit=10 ) return [asdict(r) for r in health_results] + if source == "cdc": + cdc_results = await self.search_cdc_data( + indicator, year_start, year_end, limit=10 + ) + return [asdict(r) for r in cdc_results] + if source in ("world_bank", "development"): dev_results = await self.search_development_data( indicator, country, year_start, year_end, limit=10 diff --git a/server/src/ehrlich/impact/domain/entities.py b/server/src/ehrlich/impact/domain/entities.py index 26fb864..98b87bf 100644 --- a/server/src/ehrlich/impact/domain/entities.py +++ b/server/src/ehrlich/impact/domain/entities.py @@ -63,3 +63,49 @@ class HealthIndicator: year: int value: float unit: str + + +@dataclass(frozen=True) +class SpendingRecord: + award_id: str + recipient_name: str + amount: float + agency: str + description: str + period: str + award_type: str + + +@dataclass(frozen=True) +class EducationRecord: + school_id: str + name: str + state: str + student_size: int + net_price: float + completion_rate: float + earnings_median: float + + +@dataclass(frozen=True) +class HousingData: + area_name: str + state: str + fmr_0br: float + fmr_1br: float + fmr_2br: float + fmr_3br: float + fmr_4br: float + median_income: float + year: int + + +@dataclass(frozen=True) +class DatasetMetadata: + dataset_id: str + title: str + organization: str + description: str + tags: tuple[str, ...] + resource_count: int + modified: str diff --git a/server/src/ehrlich/impact/domain/repository.py b/server/src/ehrlich/impact/domain/repository.py index 6182f9b..cc425af 100644 --- a/server/src/ehrlich/impact/domain/repository.py +++ b/server/src/ehrlich/impact/domain/repository.py @@ -6,8 +6,12 @@ if TYPE_CHECKING: from ehrlich.impact.domain.entities import ( Benchmark, + DatasetMetadata, EconomicSeries, + EducationRecord, HealthIndicator, + HousingData, + SpendingRecord, ) @@ -46,3 +50,44 @@ async def search_indicators( @abstractmethod async def get_countries(self) -> list[dict[str, str]]: ... + + +class SpendingDataRepository(ABC): + @abstractmethod + async def search_awards( + self, + query: str, + agency: str | None = None, + year: int | None = None, + limit: int = 10, + ) -> list[SpendingRecord]: ... + + +class EducationDataRepository(ABC): + @abstractmethod + async def search_schools( + self, + query: str, + state: str | None = None, + limit: int = 10, + ) -> list[EducationRecord]: ... + + +class HousingDataRepository(ABC): + @abstractmethod + async def search_housing_data( + self, + state: str, + county: str | None = None, + year: int | None = None, + ) -> list[HousingData]: ... + + +class OpenDataRepository(ABC): + @abstractmethod + async def search_datasets( + self, + query: str, + organization: str | None = None, + limit: int = 10, + ) -> list[DatasetMetadata]: ... diff --git a/server/src/ehrlich/impact/infrastructure/bls_client.py b/server/src/ehrlich/impact/infrastructure/bls_client.py new file mode 100644 index 0000000..942c7df --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/bls_client.py @@ -0,0 +1,133 @@ +import asyncio +import logging +import os + +import httpx + +from ehrlich.impact.domain.entities import DataPoint, EconomicSeries +from ehrlich.impact.domain.repository import EconomicDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.bls.gov/publicAPI/v2/timeseries/data/" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class BLSClient(EconomicDataRepository): + def __init__(self, api_key: str | None = None) -> None: + self._api_key = api_key or os.environ.get("BLS_API_KEY", "") + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_series(self, query: str, limit: int = 10) -> list[EconomicSeries]: + if not self._api_key: + return [] + series_ids = [s.strip() for s in query.split(",")][:limit] + data = await self._post(series_ids) + return self._parse_series(data, series_ids) + + async def get_series( + self, series_id: str, start: str | None = None, end: str | None = None + ) -> EconomicSeries | None: + if not self._api_key: + return None + body: dict[str, object] = { + "seriesid": [series_id], + "registrationkey": self._api_key, + } + if start: + body["startyear"] = start[:4] + if end: + body["endyear"] = end[:4] + + data = await self._post_body(body) + results = self._parse_series(data, [series_id]) + return results[0] if results else None + + def _parse_series( + self, data: dict[str, object], series_ids: list[str] + ) -> list[EconomicSeries]: + results_obj = data.get("Results", {}) + if not isinstance(results_obj, dict): + return [] + series_list = results_obj.get("series", []) + if not isinstance(series_list, list): + return [] + + results: list[EconomicSeries] = [] + for s in series_list: + if not isinstance(s, dict): + continue + sid = str(s.get("seriesID", "")) + raw_data = s.get("data", []) + points: list[DataPoint] = [] + if isinstance(raw_data, list): + for d in raw_data: + if not isinstance(d, dict): + continue + try: + value = float(d.get("value", "0")) + except (ValueError, TypeError): + continue + year = str(d.get("year", "")) + period = str(d.get("period", "")) + points.append(DataPoint(date=f"{year}-{period}", value=value)) + + results.append( + EconomicSeries( + series_id=sid, + title=sid, + values=tuple(points), + source="bls", + unit="", + frequency="", + ) + ) + return results + + async def _post(self, series_ids: list[str]) -> dict[str, object]: + body: dict[str, object] = { + "seriesid": series_ids, + "registrationkey": self._api_key, + } + return await self._post_body(body) + + async def _post_body(self, body: dict[str, object]) -> dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.post(_BASE_URL, json=body) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "BLS rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("BLS", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "BLS timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError("BLS", f"HTTP {e.response.status_code}") from e + raise ExternalServiceError( + "BLS", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/cdc_client.py b/server/src/ehrlich/impact/infrastructure/cdc_client.py new file mode 100644 index 0000000..e06b50a --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/cdc_client.py @@ -0,0 +1,154 @@ +import asyncio +import logging +import xml.etree.ElementTree as ET + +import httpx + +from ehrlich.impact.domain.entities import HealthIndicator +from ehrlich.impact.domain.repository import HealthDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://wonder.cdc.gov/controller/datarequest" +_TIMEOUT = 30.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + +_DATABASES = { + "mortality": "D76", + "natality": "D66", + "infant_mortality": "D69", +} + + +class CDCWonderClient(HealthDataRepository): + def __init__(self) -> None: + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_indicators( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[HealthIndicator]: + db_id = _DATABASES.get(indicator.lower(), indicator) + xml_request = self._build_request(db_id, year_start, year_end) + + data = await self._post(f"{_BASE_URL}/{db_id}", xml_request) + return self._parse_response(data, indicator, limit) + + def _build_request( + self, + db_id: str, + year_start: int | None = None, + year_end: int | None = None, + ) -> str: + params = [ + 'B_1*All*', + 'B_2*All*', + 'M_1D76.M1', + 'O_V1_fmodefreg', + 'O_V2_fmodefreg', + ] + if year_start: + params.append( + f'V_D76.V1{year_start}' + ) + if year_end: + params.append( + f'V_D76.V1_end{year_end}' + ) + + return ( + '' + "" + + "".join(params) + + "" + ) + + def _parse_response( + self, xml_text: str, indicator: str, limit: int + ) -> list[HealthIndicator]: + try: + root = ET.fromstring(xml_text) + except ET.ParseError: + logger.warning("CDC WONDER returned invalid XML") + return [] + + results: list[HealthIndicator] = [] + for row in root.iter("r"): + cells = list(row.iter("c")) + if len(cells) < 3: + continue + + year_text = cells[0].get("v", "") or cells[0].text or "" + label = cells[0].get("l", "") or year_text + value_text = cells[-1].get("v", "") or cells[-1].text or "" + + try: + year = int(year_text[:4]) if year_text else 0 + except (ValueError, TypeError): + year = 0 + + try: + value = float(value_text) + except (ValueError, TypeError): + continue + + results.append( + HealthIndicator( + indicator_code=indicator, + indicator_name=label, + country="US", + year=year, + value=value, + unit="per 100,000", + ) + ) + if len(results) >= limit: + break + return results + + async def _post(self, url: str, xml_body: str) -> str: + last_error: Exception | None = None + headers = {"Content-Type": "application/xml"} + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.post(url, content=xml_body, headers=headers) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "CDC WONDER rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("CDC WONDER", "Rate limit exceeded") + resp.raise_for_status() + return resp.text + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "CDC WONDER timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError( + "CDC WONDER", f"HTTP {e.response.status_code}" + ) from e + raise ExternalServiceError( + "CDC WONDER", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/census_client.py b/server/src/ehrlich/impact/infrastructure/census_client.py new file mode 100644 index 0000000..3f06251 --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/census_client.py @@ -0,0 +1,121 @@ +import asyncio +import logging +import os + +import httpx + +from ehrlich.impact.domain.entities import Benchmark +from ehrlich.impact.domain.repository import DevelopmentDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.census.gov/data" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + +_ACS_VARIABLES = { + "population": "B01003_001E", + "median_income": "B19013_001E", + "poverty_rate": "B17001_002E", + "bachelors_degree": "B15003_022E", + "unemployment": "B23025_005E", +} + + +class CensusClient(DevelopmentDataRepository): + def __init__(self, api_key: str | None = None) -> None: + self._api_key = api_key or os.environ.get("CENSUS_API_KEY", "") + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_indicators( + self, + indicator: str, + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, + ) -> list[Benchmark]: + variable = _ACS_VARIABLES.get(indicator.lower(), indicator) + year = year_end or 2022 + geo = country or "us" + + params: dict[str, str] = { + "get": f"NAME,{variable}", + "for": "state:*" if geo == "us" else f"state:{geo}", + } + if self._api_key: + params["key"] = self._api_key + + data = await self._get(f"{_BASE_URL}/{year}/acs/acs5", params=params) + if not isinstance(data, list) or len(data) < 2: + return [] + + raw_headers = data[0] + header_list = raw_headers if isinstance(raw_headers, list) else [] + indicator_name = str(header_list[1]) if len(header_list) > 1 else variable + results: list[Benchmark] = [] + for row in data[1 : limit + 1]: + if not isinstance(row, list) or len(row) < 2: + continue + try: + value = float(row[1]) + except (ValueError, TypeError): + continue + name = str(row[0]) if row[0] else "" + results.append( + Benchmark( + source="census", + indicator=indicator_name, + value=value, + unit="", + geography=name, + period=str(year), + url=f"https://data.census.gov/table?q={variable}", + ) + ) + return results + + async def get_countries(self) -> list[dict[str, str]]: + return [] + + async def _get( + self, url: str, params: dict[str, str] + ) -> list[object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(url, params=params) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "Census rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("Census", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "Census timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError("Census", f"HTTP {e.response.status_code}") from e + raise ExternalServiceError( + "Census", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/college_scorecard_client.py b/server/src/ehrlich/impact/infrastructure/college_scorecard_client.py new file mode 100644 index 0000000..86bdd3d --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/college_scorecard_client.py @@ -0,0 +1,145 @@ +import asyncio +import logging +import os + +import httpx + +from ehrlich.impact.domain.entities import EducationRecord +from ehrlich.impact.domain.repository import EducationDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.data.gov/ed/collegescorecard/v1/schools" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class CollegeScorecardClient(EducationDataRepository): + def __init__(self, api_key: str | None = None) -> None: + self._api_key = api_key or os.environ.get("COLLEGE_SCORECARD_API_KEY", "") + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_schools( + self, + query: str, + state: str | None = None, + limit: int = 10, + ) -> list[EducationRecord]: + if not self._api_key: + return [] + + params: dict[str, str | int] = { + "api_key": self._api_key, + "school.name": query, + "fields": ( + "id,school.name,school.state," + "latest.student.size," + "latest.cost.avg_net_price.overall," + "latest.completion.rate_suppressed.overall," + "latest.earnings.10_yrs_after_entry.median" + ), + "per_page": min(limit, 100), + } + if state: + params["school.state"] = state + + data = await self._get(params) + results_list = data.get("results", []) + if not isinstance(results_list, list): + return [] + + records: list[EducationRecord] = [] + for item in results_list[:limit]: + if not isinstance(item, dict): + continue + + latest = item.get("latest", {}) + if not isinstance(latest, dict): + latest = {} + student = latest.get("student", {}) + cost = latest.get("cost", {}) + completion = latest.get("completion", {}) + earnings = latest.get("earnings", {}) + + school = item.get("school", {}) + if not isinstance(school, dict): + school = {} + + def _float(obj: object, *keys: str) -> float: + current = obj + for k in keys: + if isinstance(current, dict): + current = current.get(k) + else: + return 0.0 + try: + return float(current) # type: ignore[arg-type] + except (ValueError, TypeError): + return 0.0 + + def _int(obj: object, *keys: str) -> int: + current = obj + for k in keys: + if isinstance(current, dict): + current = current.get(k) + else: + return 0 + try: + return int(str(current)) + except (ValueError, TypeError): + return 0 + + records.append( + EducationRecord( + school_id=str(item.get("id", "")), + name=str(school.get("name", "")), + state=str(school.get("state", "")), + student_size=_int(student, "size"), + net_price=_float(cost, "avg_net_price", "overall"), + completion_rate=_float(completion, "rate_suppressed", "overall"), + earnings_median=_float(earnings, "10_yrs_after_entry", "median"), + ) + ) + return records + + async def _get(self, params: dict[str, str | int]) -> dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(_BASE_URL, params=params) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "College Scorecard rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("College Scorecard", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "College Scorecard timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError( + "College Scorecard", f"HTTP {e.response.status_code}" + ) from e + raise ExternalServiceError( + "College Scorecard", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/datagov_client.py b/server/src/ehrlich/impact/infrastructure/datagov_client.py new file mode 100644 index 0000000..79a952a --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/datagov_client.py @@ -0,0 +1,115 @@ +import asyncio +import logging + +import httpx + +from ehrlich.impact.domain.entities import DatasetMetadata +from ehrlich.impact.domain.repository import OpenDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://catalog.data.gov/api/3/action" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class DataGovClient(OpenDataRepository): + def __init__(self) -> None: + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_datasets( + self, + query: str, + organization: str | None = None, + limit: int = 10, + ) -> list[DatasetMetadata]: + params: dict[str, str | int] = { + "q": query, + "rows": min(limit, 100), + } + if organization: + params["fq"] = f"organization:{organization}" + + data = await self._get(f"{_BASE_URL}/package_search", params) + result_obj = data.get("result", {}) + if not isinstance(result_obj, dict): + return [] + results_list = result_obj.get("results", []) + if not isinstance(results_list, list): + return [] + + records: list[DatasetMetadata] = [] + for item in results_list[:limit]: + if not isinstance(item, dict): + continue + + org = item.get("organization", {}) + org_title = org.get("title", "") if isinstance(org, dict) else "" + raw_tags = item.get("tags", []) + tags: list[str] = [] + if isinstance(raw_tags, list): + for t in raw_tags: + if isinstance(t, dict): + tags.append(str(t.get("name", ""))) + elif isinstance(t, str): + tags.append(t) + + resources = item.get("resources", []) + resource_count = len(resources) if isinstance(resources, list) else 0 + + records.append( + DatasetMetadata( + dataset_id=str(item.get("id", "")), + title=str(item.get("title", "")), + organization=str(org_title), + description=str(item.get("notes", ""))[:500], + tags=tuple(tags), + resource_count=resource_count, + modified=str(item.get("metadata_modified", "")), + ) + ) + return records + + async def _get( + self, url: str, params: dict[str, str | int] + ) -> dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(url, params=params) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "data.gov rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("data.gov", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "data.gov timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError( + "data.gov", f"HTTP {e.response.status_code}" + ) from e + raise ExternalServiceError( + "data.gov", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/hud_client.py b/server/src/ehrlich/impact/infrastructure/hud_client.py new file mode 100644 index 0000000..e5b1d3b --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/hud_client.py @@ -0,0 +1,114 @@ +import asyncio +import logging +import os + +import httpx + +from ehrlich.impact.domain.entities import HousingData +from ehrlich.impact.domain.repository import HousingDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://www.huduser.gov/hudapi/public" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class HUDClient(HousingDataRepository): + def __init__(self, api_token: str | None = None) -> None: + self._token = api_token or os.environ.get("HUD_API_TOKEN", "") + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_housing_data( + self, + state: str, + county: str | None = None, + year: int | None = None, + ) -> list[HousingData]: + if not self._token: + return [] + + entity_id = f"{county},{state}" if county else state + fmr_year = year or 2024 + endpoint = f"{_BASE_URL}/fmr/statedata/{entity_id}" + params: dict[str, str | int] = {"year": fmr_year} + + data = await self._get(endpoint, params) + fmr_data = data.get("data", {}) + + if isinstance(fmr_data, dict): + items = [fmr_data] + elif isinstance(fmr_data, list): + items = fmr_data + else: + return [] + + results: list[HousingData] = [] + for item in items: + if not isinstance(item, dict): + continue + results.append(self._parse_housing(item, state, fmr_year)) + return results + + @staticmethod + def _parse_housing(item: dict[str, object], state: str, year: int) -> HousingData: + def _fmr(key: str) -> float: + try: + return float(item.get(key, 0)) # type: ignore[arg-type] + except (ValueError, TypeError): + return 0.0 + + return HousingData( + area_name=str(item.get("area_name") or item.get("areaname", "")), + state=state.upper(), + fmr_0br=_fmr("fmr_0"), + fmr_1br=_fmr("fmr_1"), + fmr_2br=_fmr("fmr_2"), + fmr_3br=_fmr("fmr_3"), + fmr_4br=_fmr("fmr_4"), + median_income=_fmr("median_income"), + year=year, + ) + + async def _get( + self, url: str, params: dict[str, str | int] + ) -> dict[str, object]: + headers = {"Authorization": f"Bearer {self._token}"} + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.get(url, params=params, headers=headers) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "HUD rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("HUD", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "HUD timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError("HUD", f"HTTP {e.response.status_code}") from e + raise ExternalServiceError( + "HUD", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/infrastructure/usaspending_client.py b/server/src/ehrlich/impact/infrastructure/usaspending_client.py new file mode 100644 index 0000000..04a6df5 --- /dev/null +++ b/server/src/ehrlich/impact/infrastructure/usaspending_client.py @@ -0,0 +1,110 @@ +import asyncio +import logging + +import httpx + +from ehrlich.impact.domain.entities import SpendingRecord +from ehrlich.impact.domain.repository import SpendingDataRepository +from ehrlich.kernel.exceptions import ExternalServiceError + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.usaspending.gov/api/v2" +_TIMEOUT = 20.0 +_MAX_RETRIES = 3 +_BASE_DELAY = 1.0 + + +class USAspendingClient(SpendingDataRepository): + def __init__(self) -> None: + self._client = httpx.AsyncClient(timeout=_TIMEOUT) + + async def search_awards( + self, + query: str, + agency: str | None = None, + year: int | None = None, + limit: int = 10, + ) -> list[SpendingRecord]: + filters: dict[str, object] = {"keyword": query} + if agency: + filters["agencies"] = [ + {"type": "awarding", "tier": "toptier", "name": agency} + ] + if year: + filters["time_period"] = [ + {"start_date": f"{year}-01-01", "end_date": f"{year}-12-31"} + ] + + body: dict[str, object] = { + "filters": filters, + "limit": min(limit, 100), + "page": 1, + } + + data = await self._post(f"{_BASE_URL}/search/spending_by_award/", body) + results_list = data.get("results", []) + if not isinstance(results_list, list): + return [] + + records: list[SpendingRecord] = [] + for item in results_list[:limit]: + if not isinstance(item, dict): + continue + try: + amount = float(item.get("Award Amount") or item.get("Total Outlays", 0)) + except (ValueError, TypeError): + amount = 0.0 + + records.append( + SpendingRecord( + award_id=str(item.get("Award ID") or item.get("internal_id", "")), + recipient_name=str(item.get("Recipient Name", "")), + amount=amount, + agency=str(item.get("Awarding Agency", "")), + description=str(item.get("Award Type", "")), + period=str(item.get("Start Date", "")), + award_type=str(item.get("Award Type", "")), + ) + ) + return records + + async def _post(self, url: str, body: dict[str, object]) -> dict[str, object]: + last_error: Exception | None = None + for attempt in range(_MAX_RETRIES): + try: + resp = await self._client.post(url, json=body) + if resp.status_code == 429: + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "USAspending rate limited (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + raise ExternalServiceError("USAspending", "Rate limit exceeded") + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + except httpx.TimeoutException as e: + last_error = e + delay = _BASE_DELAY * (2**attempt) + logger.warning( + "USAspending timeout (attempt %d/%d), retrying in %.1fs", + attempt + 1, + _MAX_RETRIES, + delay, + ) + if attempt < _MAX_RETRIES - 1: + await asyncio.sleep(delay) + continue + except httpx.HTTPStatusError as e: + raise ExternalServiceError( + "USAspending", f"HTTP {e.response.status_code}" + ) from e + raise ExternalServiceError( + "USAspending", + f"Request failed after {_MAX_RETRIES} attempts: {last_error}", + ) diff --git a/server/src/ehrlich/impact/tools.py b/server/src/ehrlich/impact/tools.py index 3dc609e..8035faf 100644 --- a/server/src/ehrlich/impact/tools.py +++ b/server/src/ehrlich/impact/tools.py @@ -1,7 +1,8 @@ """Impact Evaluation tools for the investigation engine. -3 tools for social program data retrieval: economic indicator -search, benchmark fetching, and cross-program comparison. +8 tools for social program data retrieval: economic indicator search, +health indicator search, benchmark fetching, cross-program comparison, +spending data, education data, housing data, and open data discovery. """ from __future__ import annotations @@ -10,17 +11,39 @@ from dataclasses import asdict from ehrlich.impact.application.impact_service import ImpactService +from ehrlich.impact.infrastructure.bls_client import BLSClient +from ehrlich.impact.infrastructure.cdc_client import CDCWonderClient +from ehrlich.impact.infrastructure.census_client import CensusClient +from ehrlich.impact.infrastructure.college_scorecard_client import CollegeScorecardClient +from ehrlich.impact.infrastructure.datagov_client import DataGovClient from ehrlich.impact.infrastructure.fred_client import FREDClient +from ehrlich.impact.infrastructure.hud_client import HUDClient +from ehrlich.impact.infrastructure.usaspending_client import USAspendingClient from ehrlich.impact.infrastructure.who_client import WHOClient from ehrlich.impact.infrastructure.worldbank_client import WorldBankClient _worldbank = WorldBankClient() _who = WHOClient() _fred = FREDClient() +_bls = BLSClient() +_census = CensusClient() +_usaspending = USAspendingClient() +_scorecard = CollegeScorecardClient() +_hud = HUDClient() +_cdc = CDCWonderClient() +_datagov = DataGovClient() + _service = ImpactService( economic=_fred, health=_who, development=_worldbank, + spending=_usaspending, + education=_scorecard, + housing=_hud, + open_data=_datagov, + bls=_bls, + census=_census, + cdc=_cdc, ) @@ -32,15 +55,16 @@ async def search_economic_indicators( end_year: int | None = None, limit: int = 10, ) -> str: - """Search FRED, World Bank, or WHO for economic/development/health indicators. + """Search FRED, BLS, Census, World Bank, or WHO for economic/development indicators. Queries the specified data source for time series data relevant to impact evaluation. Returns series metadata and values for analysis. Args: query: Search query or indicator code - (e.g. 'GDP per capita', 'SE.PRM.ENRR', 'WHOSIS_000001') - source: Data source ('fred', 'world_bank', 'who') + (e.g. 'GDP per capita', 'SE.PRM.ENRR', 'WHOSIS_000001', + 'LNS14000000' for BLS, 'median_income' for Census) + source: Data source ('fred', 'bls', 'census', 'world_bank', 'who') country: ISO country code for filtering (e.g. 'MX', 'US', 'MEX') start_year: Start year filter end_year: End year filter @@ -66,6 +90,38 @@ async def search_economic_indicators( } ) + if source == "bls": + series = await _service.search_bls_data(query, limit) + return json.dumps( + { + "source": "BLS", + "query": query, + "count": len(series), + "series": [ + { + "series_id": s.series_id, + "title": s.title, + "values_count": len(s.values), + "source": s.source, + } + for s in series + ], + } + ) + + if source == "census": + benchmarks = await _service.search_census_data( + query, country, start_year, end_year, limit + ) + return json.dumps( + { + "source": "Census", + "indicator": query, + "count": len(benchmarks), + "data": [asdict(b) for b in benchmarks], + } + ) + if source in ("who", "health"): indicators = await _service.search_health_data(query, country, start_year, end_year, limit) return json.dumps( @@ -89,20 +145,62 @@ async def search_economic_indicators( ) +async def search_health_indicators( + indicator: str, + source: str = "who", + country: str | None = None, + year_start: int | None = None, + year_end: int | None = None, + limit: int = 10, +) -> str: + """Search WHO GHO or CDC WONDER for health indicators. + + Args: + indicator: Indicator code or name (e.g. 'WHOSIS_000001' for WHO, + 'mortality' or 'natality' for CDC WONDER) + source: Data source ('who', 'cdc') + country: ISO country code (WHO only, e.g. 'MEX', 'USA') + year_start: Start year filter + year_end: End year filter + limit: Maximum results to return (default: 10) + """ + if source == "cdc": + results = await _service.search_cdc_data(indicator, year_start, year_end, limit) + return json.dumps( + { + "source": "CDC WONDER", + "indicator": indicator, + "count": len(results), + "data": [asdict(r) for r in results], + } + ) + + results = await _service.search_health_data(indicator, country, year_start, year_end, limit) + return json.dumps( + { + "source": "WHO", + "indicator": indicator, + "count": len(results), + "data": [asdict(r) for r in results], + } + ) + + async def fetch_benchmark( indicator: str, source: str = "world_bank", country: str | None = None, period: str | None = None, ) -> str: - """Fetch a benchmark value from international data sources. + """Fetch a benchmark value from international or US data sources. - Retrieves comparison data points from FRED, World Bank, or WHO - for use as baseline or reference in impact evaluations. + Retrieves comparison data points from FRED, BLS, Census, World Bank, + WHO, or CDC for use as baseline or reference in impact evaluations. Args: - indicator: Indicator code or name (e.g. 'SE.PRM.ENRR', 'GDP', 'WHOSIS_000001') - source: Data source ('fred', 'world_bank', 'who') + indicator: Indicator code or name (e.g. 'SE.PRM.ENRR', 'GDP', + 'LNS14000000', 'median_income', 'mortality') + source: Data source ('fred', 'bls', 'census', 'world_bank', 'who', 'cdc') country: ISO country code (e.g. 'MX', 'US') period: Time period (e.g. '2015-2020') """ @@ -146,3 +244,118 @@ async def compare_programs( result = _service.compare_programs(parsed, metric, alternative) return json.dumps(result) + + +async def search_spending_data( + query: str, + agency: str | None = None, + year: int | None = None, + award_type: str | None = None, + limit: int = 10, +) -> str: + """Search USAspending.gov for federal spending awards and grants. + + Args: + query: Search keywords (e.g. 'Head Start', 'SNAP', 'education grants') + agency: Federal agency name filter (e.g. 'Department of Education') + year: Fiscal year filter (e.g. 2023) + award_type: Type filter (unused, reserved for future filtering) + limit: Maximum results to return (default: 10) + """ + records = await _service.search_spending_data(query, agency, year, limit) + return json.dumps( + { + "source": "USAspending", + "query": query, + "count": len(records), + "awards": [asdict(r) for r in records], + } + ) + + +async def search_education_data( + query: str, + state: str | None = None, + fields: str | None = None, + limit: int = 10, +) -> str: + """Search College Scorecard for US higher education institution data. + + Args: + query: School name search (e.g. 'Harvard', 'community college') + state: State abbreviation filter (e.g. 'CA', 'NY') + fields: Unused, reserved for future field selection + limit: Maximum results to return (default: 10) + """ + records = await _service.search_education_data(query, state, limit) + return json.dumps( + { + "source": "College Scorecard", + "query": query, + "count": len(records), + "schools": [asdict(r) for r in records], + } + ) + + +async def search_housing_data( + state: str, + county: str | None = None, + year: int | None = None, + data_type: str = "fmr", +) -> str: + """Search HUD for housing data (Fair Market Rents, income limits). + + Args: + state: State abbreviation (e.g. 'CA', 'TX') + county: County FIPS code (optional) + year: FMR year (default: most recent) + data_type: Data type ('fmr' for Fair Market Rents) + """ + records = await _service.search_housing_data(state, county, year) + return json.dumps( + { + "source": "HUD", + "state": state, + "data_type": data_type, + "count": len(records), + "data": [asdict(r) for r in records], + } + ) + + +async def search_open_data( + query: str, + organization: str | None = None, + limit: int = 10, +) -> str: + """Search data.gov CKAN catalog for US federal open datasets. + + Discovers available datasets for further analysis. Returns metadata + including title, organization, tags, and resource counts. + + Args: + query: Search keywords (e.g. 'poverty', 'education outcomes') + organization: Organization filter (e.g. 'hhs-gov', 'ed-gov') + limit: Maximum results to return (default: 10) + """ + records = await _service.search_open_data(query, organization, limit) + return json.dumps( + { + "source": "data.gov", + "query": query, + "count": len(records), + "datasets": [ + { + "dataset_id": r.dataset_id, + "title": r.title, + "organization": r.organization, + "description": r.description, + "tags": list(r.tags), + "resource_count": r.resource_count, + "modified": r.modified, + } + for r in records + ], + } + ) diff --git a/server/src/ehrlich/investigation/application/registry_factory.py b/server/src/ehrlich/investigation/application/registry_factory.py index ddbe6d0..ad19a26 100644 --- a/server/src/ehrlich/investigation/application/registry_factory.py +++ b/server/src/ehrlich/investigation/application/registry_factory.py @@ -40,6 +40,11 @@ compare_programs, fetch_benchmark, search_economic_indicators, + search_education_data, + search_health_indicators, + search_housing_data, + search_open_data, + search_spending_data, ) from ehrlich.investigation.application.tool_registry import ToolRegistry from ehrlich.investigation.domain.domain_registry import DomainRegistry @@ -206,10 +211,15 @@ def build_tool_registry() -> ToolRegistry: ("check_interactions", check_interactions, _nutrition_safety), ("analyze_nutrient_ratios", analyze_nutrient_ratios, _nutrition), ("compute_inflammatory_index", compute_inflammatory_index, _nutrition), - # Impact Evaluation (3) + # Impact Evaluation (8) ("search_economic_indicators", search_economic_indicators, _impact), + ("search_health_indicators", search_health_indicators, _impact), ("fetch_benchmark", fetch_benchmark, _impact), ("compare_programs", compare_programs, _impact), + ("search_spending_data", search_spending_data, _impact), + ("search_education_data", search_education_data, _impact), + ("search_housing_data", search_housing_data, _impact), + ("search_open_data", search_open_data, _impact), # Causal Inference (6) -- domain-agnostic ("estimate_did", estimate_did, _causal), ("estimate_psm", estimate_psm, _causal), diff --git a/server/src/ehrlich/investigation/domain/domains/impact.py b/server/src/ehrlich/investigation/domain/domains/impact.py index e6695bb..457b01a 100644 --- a/server/src/ehrlich/investigation/domain/domains/impact.py +++ b/server/src/ehrlich/investigation/domain/domains/impact.py @@ -54,6 +54,11 @@ "social_program", "public_policy", "policy_analysis", + "us_federal_program", + "education_policy", + "housing_policy", + "labor_market", + "public_health", ), template_prompts=( { @@ -84,6 +89,34 @@ "evidence from after-school sports interventions." ), }, + { + "title": "SNAP Benefits & Nutrition", + "domain": "Impact Evaluation", + "prompt": ( + "What is the causal effect of SNAP (food stamps) benefits " + "on food security and nutrition outcomes in US households? " + "Use USAspending data for program scale and Census/BLS for " + "demographic context." + ), + }, + { + "title": "Pell Grant & College Completion", + "domain": "Impact Evaluation", + "prompt": ( + "Do Pell Grant recipients have different completion rates " + "and post-graduation earnings compared to non-recipients? " + "Analyze College Scorecard data across institution types." + ), + }, + { + "title": "Housing Vouchers & Mobility", + "domain": "Impact Evaluation", + "prompt": ( + "What is the effect of Section 8 housing vouchers on " + "economic mobility and neighborhood quality? Compare HUD " + "Fair Market Rents with Census income data across states." + ), + }, ), director_examples="""\ @@ -145,8 +178,7 @@ "run_statistical_test", "search_literature"], "independent_variable": "CCT program implementation \ (present vs absent)", - "dependent_variable": "Primary school net enrollment rate (%)"\ -, + "dependent_variable": "Primary school net enrollment rate (%)", "controls": ["positive: Prospera program with known 3-9pp \ effect", "negative: matched non-program region"], "confounders": ["GDP growth trends", "Demographic shifts", \ @@ -172,6 +204,24 @@ 3. record_finding(title="Enrollment trends in Mexico 2000-2020", \ detail="...", hypothesis_id="h1", evidence_type="supporting") +Example: Searching US labor data from BLS +1. search_economic_indicators(query="LNS14000000", \ +source="bls", limit=5) +2. record_finding(title="US unemployment rate trends", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Searching US Census demographics +1. search_economic_indicators(query="median_income", \ +source="census", start_year=2020, end_year=2022) +2. record_finding(title="Median income by state", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Searching CDC health data +1. search_health_indicators(indicator="mortality", source="cdc", \ +year_start=2015, year_end=2022) +2. record_finding(title="US mortality trends", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + Example: Fetching benchmark data for comparison 1. fetch_benchmark(indicator="SE.PRM.ENRR", source="world_bank", \ country="MX", period="2010-2020") @@ -180,6 +230,27 @@ 3. record_finding(title="Cross-country enrollment comparison", \ detail="...", hypothesis_id="h1", evidence_type="supporting") +Example: Searching federal spending data +1. search_spending_data(query="Head Start", \ +agency="Department of Health and Human Services", year=2023) +2. record_finding(title="Head Start federal spending 2023", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Searching education outcomes +1. search_education_data(query="community college", state="CA", limit=10) +2. record_finding(title="CA community college outcomes", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Searching housing data +1. search_housing_data(state="CA", year=2024) +2. record_finding(title="California Fair Market Rents 2024", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + +Example: Discovering open datasets +1. search_open_data(query="poverty education", organization="ed-gov") +2. record_finding(title="Available federal datasets", \ +detail="...", hypothesis_id="h1", evidence_type="supporting") + Example: Comparing program outcomes statistically 1. compare_programs(programs='[{{"name": "Treatment", \ "values": [85.2, 87.1, 89.3]}}, {{"name": "Control", \ diff --git a/server/tests/api/test_methodology.py b/server/tests/api/test_methodology.py index ec0c391..608b20e 100644 --- a/server/tests/api/test_methodology.py +++ b/server/tests/api/test_methodology.py @@ -72,7 +72,7 @@ def test_tools_grouped_by_context(self, client: TestClient) -> None: def test_total_tool_count(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() total = sum(len(g["tools"]) for g in data["tools"]) - assert total == 85 + assert total == 90 def test_tool_has_name_and_description(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() @@ -84,7 +84,7 @@ def test_tool_has_name_and_description(self, client: TestClient) -> None: def test_data_sources_count(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() - assert len(data["data_sources"]) == 19 + assert len(data["data_sources"]) == 25 def test_data_source_fields(self, client: TestClient) -> None: data = client.get("/api/v1/methodology").json() diff --git a/server/tests/api/test_stats.py b/server/tests/api/test_stats.py index d521fa6..b18d2cd 100644 --- a/server/tests/api/test_stats.py +++ b/server/tests/api/test_stats.py @@ -24,7 +24,7 @@ def test_returns_all_fields(self, client: TestClient) -> None: def test_tool_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["tool_count"] == 85 + assert data["tool_count"] == 90 def test_domain_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") @@ -39,9 +39,9 @@ def test_phase_count(self, client: TestClient) -> None: def test_data_source_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["data_source_count"] == 19 + assert data["data_source_count"] == 25 def test_event_type_count(self, client: TestClient) -> None: resp = client.get("/api/v1/stats") data = resp.json() - assert data["event_type_count"] == 20 + assert data["event_type_count"] == 21 diff --git a/server/tests/impact/test_impact_service.py b/server/tests/impact/test_impact_service.py index c76ed2b..8ca4777 100644 --- a/server/tests/impact/test_impact_service.py +++ b/server/tests/impact/test_impact_service.py @@ -10,8 +10,12 @@ from ehrlich.impact.domain.entities import ( Benchmark, DataPoint, + DatasetMetadata, EconomicSeries, + EducationRecord, HealthIndicator, + HousingData, + SpendingRecord, ) @@ -80,11 +84,103 @@ def mock_development() -> AsyncMock: return mock +@pytest.fixture +def mock_spending() -> AsyncMock: + mock = AsyncMock() + mock.search_awards = AsyncMock( + return_value=[ + SpendingRecord( + award_id="AWARD-001", + recipient_name="Test Org", + amount=500000.0, + agency="DOE", + description="Grant", + period="2023-01-01", + award_type="Grant", + ) + ] + ) + return mock + + +@pytest.fixture +def mock_education() -> AsyncMock: + mock = AsyncMock() + mock.search_schools = AsyncMock( + return_value=[ + EducationRecord( + school_id="12345", + name="Test U", + state="CA", + student_size=5000, + net_price=15000.0, + completion_rate=0.65, + earnings_median=55000.0, + ) + ] + ) + return mock + + +@pytest.fixture +def mock_housing() -> AsyncMock: + mock = AsyncMock() + mock.search_housing_data = AsyncMock( + return_value=[ + HousingData( + area_name="LA", + state="CA", + fmr_0br=1200.0, + fmr_1br=1500.0, + fmr_2br=1900.0, + fmr_3br=2400.0, + fmr_4br=2800.0, + median_income=75000.0, + year=2024, + ) + ] + ) + return mock + + +@pytest.fixture +def mock_open_data() -> AsyncMock: + mock = AsyncMock() + mock.search_datasets = AsyncMock( + return_value=[ + DatasetMetadata( + dataset_id="ds-001", + title="Poverty Stats", + organization="Census", + description="Poverty data", + tags=("poverty",), + resource_count=2, + modified="2024-01-15", + ) + ] + ) + return mock + + @pytest.fixture def service( - mock_economic: AsyncMock, mock_health: AsyncMock, mock_development: AsyncMock + mock_economic: AsyncMock, + mock_health: AsyncMock, + mock_development: AsyncMock, + mock_spending: AsyncMock, + mock_education: AsyncMock, + mock_housing: AsyncMock, + mock_open_data: AsyncMock, ) -> ImpactService: - return ImpactService(economic=mock_economic, health=mock_health, development=mock_development) + return ImpactService( + economic=mock_economic, + health=mock_health, + development=mock_development, + spending=mock_spending, + education=mock_education, + housing=mock_housing, + open_data=mock_open_data, + ) class TestSearchEconomicData: @@ -194,3 +290,120 @@ def test_empty_programs(self, service: ImpactService) -> None: result = service.compare_programs([], "score") assert result["metric"] == "score" assert result["groups"] == {} + + +class TestSearchSpendingData: + @pytest.mark.asyncio + async def test_returns_records(self, service: ImpactService) -> None: + results = await service.search_spending_data("education") + assert len(results) == 1 + assert results[0].award_id == "AWARD-001" + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_spending_data("education") + assert results == [] + + +class TestSearchEducationData: + @pytest.mark.asyncio + async def test_returns_records(self, service: ImpactService) -> None: + results = await service.search_education_data("Test") + assert len(results) == 1 + assert results[0].name == "Test U" + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_education_data("Test") + assert results == [] + + +class TestSearchHousingData: + @pytest.mark.asyncio + async def test_returns_records(self, service: ImpactService) -> None: + results = await service.search_housing_data("CA") + assert len(results) == 1 + assert results[0].fmr_2br == 1900.0 + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_housing_data("CA") + assert results == [] + + +class TestSearchOpenData: + @pytest.mark.asyncio + async def test_returns_records(self, service: ImpactService) -> None: + results = await service.search_open_data("poverty") + assert len(results) == 1 + assert results[0].title == "Poverty Stats" + + @pytest.mark.asyncio + async def test_no_repo(self) -> None: + svc = ImpactService() + results = await svc.search_open_data("poverty") + assert results == [] + + +class TestFetchBenchmarkExtended: + @pytest.mark.asyncio + async def test_bls_source(self) -> None: + mock_bls = AsyncMock() + mock_bls.search_series = AsyncMock( + return_value=[ + EconomicSeries( + series_id="LNS14000000", + title="Unemployment", + values=(), + source="bls", + unit="", + frequency="", + ) + ] + ) + svc = ImpactService(bls=mock_bls) + results = await svc.fetch_benchmark("LNS14000000", "bls") + assert len(results) == 1 + assert results[0]["source"] == "BLS" + + @pytest.mark.asyncio + async def test_census_source(self) -> None: + mock_census = AsyncMock() + mock_census.search_indicators = AsyncMock( + return_value=[ + Benchmark( + source="census", + indicator="B19013_001E", + value=78672.0, + unit="", + geography="CA", + period="2022", + url="https://data.census.gov", + ) + ] + ) + svc = ImpactService(census=mock_census) + results = await svc.fetch_benchmark("median_income", "census") + assert len(results) == 1 + + @pytest.mark.asyncio + async def test_cdc_source(self) -> None: + mock_cdc = AsyncMock() + mock_cdc.search_indicators = AsyncMock( + return_value=[ + HealthIndicator( + indicator_code="mortality", + indicator_name="Mortality", + country="US", + year=2020, + value=828.7, + unit="per 100,000", + ) + ] + ) + svc = ImpactService(cdc=mock_cdc) + results = await svc.fetch_benchmark("mortality", "cdc") + assert len(results) == 1 diff --git a/server/tests/impact/test_us_clients.py b/server/tests/impact/test_us_clients.py new file mode 100644 index 0000000..dbfc118 --- /dev/null +++ b/server/tests/impact/test_us_clients.py @@ -0,0 +1,465 @@ +"""Tests for US federal data API clients.""" + +from __future__ import annotations + +import pytest +import respx +from httpx import Response + +from ehrlich.impact.infrastructure.bls_client import BLSClient +from ehrlich.impact.infrastructure.cdc_client import CDCWonderClient +from ehrlich.impact.infrastructure.census_client import CensusClient +from ehrlich.impact.infrastructure.college_scorecard_client import CollegeScorecardClient +from ehrlich.impact.infrastructure.datagov_client import DataGovClient +from ehrlich.impact.infrastructure.hud_client import HUDClient +from ehrlich.impact.infrastructure.usaspending_client import USAspendingClient +from ehrlich.kernel.exceptions import ExternalServiceError + +# --- Census --- + +_CENSUS_URL = "https://api.census.gov/data/2022/acs/acs5" +_CENSUS_RESPONSE = [ + ["NAME", "B19013_001E", "state"], + ["California", "78672", "06"], + ["Texas", "63826", "48"], +] + + +class TestCensusClient: + @pytest.fixture + def client(self) -> CensusClient: + return CensusClient(api_key="test_key") + + @respx.mock + @pytest.mark.asyncio + async def test_search_indicators(self, client: CensusClient) -> None: + respx.get(_CENSUS_URL).mock(return_value=Response(200, json=_CENSUS_RESPONSE)) + results = await client.search_indicators("median_income") + assert len(results) == 2 + assert results[0].source == "census" + assert results[0].geography == "California" + assert results[0].value == 78672.0 + + @respx.mock + @pytest.mark.asyncio + async def test_empty_response(self, client: CensusClient) -> None: + respx.get(_CENSUS_URL).mock(return_value=Response(200, json=[["NAME", "X"]])) + results = await client.search_indicators("population") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: CensusClient) -> None: + respx.get(_CENSUS_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="Census"): + await client.search_indicators("median_income") + + @pytest.mark.asyncio + async def test_get_countries_returns_empty(self) -> None: + client = CensusClient(api_key="test") + result = await client.get_countries() + assert result == [] + + +# --- BLS --- + +_BLS_URL = "https://api.bls.gov/publicAPI/v2/timeseries/data/" +_BLS_RESPONSE = { + "status": "REQUEST_SUCCEEDED", + "Results": { + "series": [ + { + "seriesID": "LNS14000000", + "data": [ + {"year": "2023", "period": "M12", "value": "3.7"}, + {"year": "2023", "period": "M11", "value": "3.7"}, + ], + } + ] + }, +} + + +class TestBLSClient: + @pytest.fixture + def client(self) -> BLSClient: + return BLSClient(api_key="test_key") + + @respx.mock + @pytest.mark.asyncio + async def test_search_series(self, client: BLSClient) -> None: + respx.post(_BLS_URL).mock(return_value=Response(200, json=_BLS_RESPONSE)) + results = await client.search_series("LNS14000000") + assert len(results) == 1 + assert results[0].series_id == "LNS14000000" + assert results[0].source == "bls" + assert len(results[0].values) == 2 + + @respx.mock + @pytest.mark.asyncio + async def test_get_series(self, client: BLSClient) -> None: + respx.post(_BLS_URL).mock(return_value=Response(200, json=_BLS_RESPONSE)) + result = await client.get_series("LNS14000000", start="2023", end="2023") + assert result is not None + assert result.series_id == "LNS14000000" + + @pytest.mark.asyncio + async def test_no_api_key_returns_empty(self) -> None: + client = BLSClient(api_key="") + results = await client.search_series("LNS14000000") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: BLSClient) -> None: + respx.post(_BLS_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="BLS"): + await client.search_series("LNS14000000") + + @respx.mock + @pytest.mark.asyncio + async def test_empty_results(self, client: BLSClient) -> None: + respx.post(_BLS_URL).mock( + return_value=Response(200, json={"Results": {"series": []}}) + ) + results = await client.search_series("UNKNOWN") + assert results == [] + + +# --- USAspending --- + +_USASPENDING_URL = "https://api.usaspending.gov/api/v2/search/spending_by_award/" +_USASPENDING_RESPONSE = { + "results": [ + { + "Award ID": "AWARD-001", + "Recipient Name": "Test University", + "Award Amount": 500000.0, + "Awarding Agency": "Department of Education", + "Award Type": "Grant", + "Start Date": "2023-01-01", + "internal_id": "1", + } + ] +} + + +class TestUSAspendingClient: + @pytest.fixture + def client(self) -> USAspendingClient: + return USAspendingClient() + + @respx.mock + @pytest.mark.asyncio + async def test_search_awards(self, client: USAspendingClient) -> None: + respx.post(_USASPENDING_URL).mock( + return_value=Response(200, json=_USASPENDING_RESPONSE) + ) + results = await client.search_awards("education") + assert len(results) == 1 + assert results[0].award_id == "AWARD-001" + assert results[0].recipient_name == "Test University" + assert results[0].amount == 500000.0 + + @respx.mock + @pytest.mark.asyncio + async def test_empty_results(self, client: USAspendingClient) -> None: + respx.post(_USASPENDING_URL).mock( + return_value=Response(200, json={"results": []}) + ) + results = await client.search_awards("nonexistent") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_with_filters(self, client: USAspendingClient) -> None: + respx.post(_USASPENDING_URL).mock( + return_value=Response(200, json=_USASPENDING_RESPONSE) + ) + results = await client.search_awards( + "Head Start", agency="HHS", year=2023 + ) + assert len(results) == 1 + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: USAspendingClient) -> None: + respx.post(_USASPENDING_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="USAspending"): + await client.search_awards("test") + + +# --- College Scorecard --- + +_SCORECARD_URL = "https://api.data.gov/ed/collegescorecard/v1/schools" +_SCORECARD_RESPONSE = { + "results": [ + { + "id": "12345", + "school": {"name": "Test University", "state": "CA"}, + "latest": { + "student": {"size": 5000}, + "cost": {"avg_net_price": {"overall": 15000}}, + "completion": {"rate_suppressed": {"overall": 0.65}}, + "earnings": {"10_yrs_after_entry": {"median": 55000}}, + }, + } + ] +} + + +class TestCollegeScorecardClient: + @pytest.fixture + def client(self) -> CollegeScorecardClient: + return CollegeScorecardClient(api_key="test_key") + + @respx.mock + @pytest.mark.asyncio + async def test_search_schools(self, client: CollegeScorecardClient) -> None: + respx.get(_SCORECARD_URL).mock( + return_value=Response(200, json=_SCORECARD_RESPONSE) + ) + results = await client.search_schools("Test University") + assert len(results) == 1 + assert results[0].name == "Test University" + assert results[0].state == "CA" + assert results[0].student_size == 5000 + assert results[0].completion_rate == 0.65 + assert results[0].earnings_median == 55000.0 + + @respx.mock + @pytest.mark.asyncio + async def test_empty_results(self, client: CollegeScorecardClient) -> None: + respx.get(_SCORECARD_URL).mock( + return_value=Response(200, json={"results": []}) + ) + results = await client.search_schools("nonexistent") + assert results == [] + + @pytest.mark.asyncio + async def test_no_api_key_returns_empty(self) -> None: + client = CollegeScorecardClient(api_key="") + results = await client.search_schools("test") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: CollegeScorecardClient) -> None: + respx.get(_SCORECARD_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="College Scorecard"): + await client.search_schools("test") + + +# --- HUD --- + +_HUD_URL = "https://www.huduser.gov/hudapi/public/fmr/statedata/CA" +_HUD_RESPONSE = { + "data": { + "area_name": "Los Angeles", + "fmr_0": 1200, + "fmr_1": 1500, + "fmr_2": 1900, + "fmr_3": 2400, + "fmr_4": 2800, + "median_income": 75000, + } +} + + +class TestHUDClient: + @pytest.fixture + def client(self) -> HUDClient: + return HUDClient(api_token="test_token") + + @respx.mock + @pytest.mark.asyncio + async def test_search_housing_data(self, client: HUDClient) -> None: + respx.get(_HUD_URL).mock(return_value=Response(200, json=_HUD_RESPONSE)) + results = await client.search_housing_data("CA") + assert len(results) == 1 + assert results[0].area_name == "Los Angeles" + assert results[0].state == "CA" + assert results[0].fmr_2br == 1900.0 + assert results[0].median_income == 75000.0 + + @respx.mock + @pytest.mark.asyncio + async def test_list_response(self, client: HUDClient) -> None: + respx.get(_HUD_URL).mock( + return_value=Response(200, json={"data": [_HUD_RESPONSE["data"]]}) + ) + results = await client.search_housing_data("CA") + assert len(results) == 1 + + @pytest.mark.asyncio + async def test_no_token_returns_empty(self) -> None: + client = HUDClient(api_token="") + results = await client.search_housing_data("CA") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: HUDClient) -> None: + respx.get(_HUD_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="HUD"): + await client.search_housing_data("CA") + + +# --- CDC WONDER --- + +_CDC_URL = "https://wonder.cdc.gov/controller/datarequest/D76" +_CDC_XML_RESPONSE = """ + + + + + + + + +""" + + +class TestCDCWonderClient: + @pytest.fixture + def client(self) -> CDCWonderClient: + return CDCWonderClient() + + @respx.mock + @pytest.mark.asyncio + async def test_search_indicators(self, client: CDCWonderClient) -> None: + respx.post(_CDC_URL).mock(return_value=Response(200, text=_CDC_XML_RESPONSE)) + results = await client.search_indicators("mortality") + assert len(results) == 2 + assert results[0].country == "US" + assert results[0].year == 2020 + assert results[0].value == 828.7 + assert results[0].indicator_code == "mortality" + + @respx.mock + @pytest.mark.asyncio + async def test_invalid_xml(self, client: CDCWonderClient) -> None: + respx.post(_CDC_URL).mock(return_value=Response(200, text="not xml")) + results = await client.search_indicators("mortality") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_empty_response(self, client: CDCWonderClient) -> None: + respx.post(_CDC_URL).mock( + return_value=Response(200, text='') + ) + results = await client.search_indicators("mortality") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: CDCWonderClient) -> None: + respx.post(_CDC_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="CDC WONDER"): + await client.search_indicators("mortality") + + +# --- data.gov --- + +_DATAGOV_URL = "https://catalog.data.gov/api/3/action/package_search" +_DATAGOV_RESPONSE = { + "result": { + "results": [ + { + "id": "ds-001", + "title": "Poverty Statistics", + "organization": {"title": "Census Bureau"}, + "notes": "Annual poverty estimates by state", + "tags": [{"name": "poverty"}, {"name": "census"}], + "resources": [{"id": "r1"}, {"id": "r2"}], + "metadata_modified": "2024-01-15", + } + ] + } +} + + +class TestDataGovClient: + @pytest.fixture + def client(self) -> DataGovClient: + return DataGovClient() + + @respx.mock + @pytest.mark.asyncio + async def test_search_datasets(self, client: DataGovClient) -> None: + respx.get(_DATAGOV_URL).mock( + return_value=Response(200, json=_DATAGOV_RESPONSE) + ) + results = await client.search_datasets("poverty") + assert len(results) == 1 + assert results[0].title == "Poverty Statistics" + assert results[0].organization == "Census Bureau" + assert results[0].tags == ("poverty", "census") + assert results[0].resource_count == 2 + + @respx.mock + @pytest.mark.asyncio + async def test_empty_results(self, client: DataGovClient) -> None: + respx.get(_DATAGOV_URL).mock( + return_value=Response(200, json={"result": {"results": []}}) + ) + results = await client.search_datasets("nonexistent") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_with_organization(self, client: DataGovClient) -> None: + respx.get(_DATAGOV_URL).mock( + return_value=Response(200, json=_DATAGOV_RESPONSE) + ) + results = await client.search_datasets("poverty", organization="census-gov") + assert len(results) == 1 + + @respx.mock + @pytest.mark.asyncio + async def test_http_error(self, client: DataGovClient) -> None: + respx.get(_DATAGOV_URL).mock(return_value=Response(500)) + with pytest.raises(ExternalServiceError, match="data.gov"): + await client.search_datasets("test") + + +# --- Retry Tests --- + + +class TestRetryBehavior: + @respx.mock + @pytest.mark.asyncio + async def test_bls_rate_limit_retry(self) -> None: + client = BLSClient(api_key="test") + route = respx.post(_BLS_URL) + route.side_effect = [ + Response(429), + Response(200, json={"Results": {"series": []}}), + ] + results = await client.search_series("LNS14000000") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_usaspending_rate_limit_retry(self) -> None: + client = USAspendingClient() + route = respx.post(_USASPENDING_URL) + route.side_effect = [ + Response(429), + Response(200, json={"results": []}), + ] + results = await client.search_awards("test") + assert results == [] + + @respx.mock + @pytest.mark.asyncio + async def test_census_rate_limit_retry(self) -> None: + client = CensusClient(api_key="test") + route = respx.get(_CENSUS_URL) + route.side_effect = [ + Response(429), + Response(200, json=_CENSUS_RESPONSE), + ] + results = await client.search_indicators("median_income") + assert len(results) == 2 diff --git a/server/tests/impact/test_us_tools.py b/server/tests/impact/test_us_tools.py new file mode 100644 index 0000000..e8a1cb4 --- /dev/null +++ b/server/tests/impact/test_us_tools.py @@ -0,0 +1,254 @@ +"""Tests for US-specific Impact Evaluation tools.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from ehrlich.impact.domain.entities import ( + Benchmark, + DatasetMetadata, + EconomicSeries, + EducationRecord, + HealthIndicator, + HousingData, + SpendingRecord, +) +from ehrlich.impact.tools import ( + search_economic_indicators, + search_education_data, + search_health_indicators, + search_housing_data, + search_open_data, + search_spending_data, +) + + +class TestSearchEconomicIndicatorsBLS: + @pytest.mark.asyncio + async def test_bls_source(self) -> None: + series = [ + EconomicSeries( + series_id="LNS14000000", + title="LNS14000000", + values=(), + source="bls", + unit="", + frequency="", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_bls_data", + new_callable=AsyncMock, + return_value=series, + ): + result = json.loads( + await search_economic_indicators("LNS14000000", source="bls") + ) + assert result["source"] == "BLS" + assert result["count"] == 1 + assert result["series"][0]["series_id"] == "LNS14000000" + + +class TestSearchEconomicIndicatorsCensus: + @pytest.mark.asyncio + async def test_census_source(self) -> None: + benchmarks = [ + Benchmark( + source="census", + indicator="B19013_001E", + value=78672.0, + unit="", + geography="California", + period="2022", + url="https://data.census.gov/table?q=B19013_001E", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_census_data", + new_callable=AsyncMock, + return_value=benchmarks, + ): + result = json.loads( + await search_economic_indicators("median_income", source="census") + ) + assert result["source"] == "Census" + assert result["count"] == 1 + assert result["data"][0]["geography"] == "California" + + +class TestSearchHealthIndicators: + @pytest.mark.asyncio + async def test_who_source(self) -> None: + indicators = [ + HealthIndicator( + indicator_code="WHOSIS_000001", + indicator_name="Life expectancy", + country="MEX", + year=2020, + value=75.1, + unit="years", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_health_data", + new_callable=AsyncMock, + return_value=indicators, + ): + result = json.loads( + await search_health_indicators("WHOSIS_000001", source="who") + ) + assert result["source"] == "WHO" + assert result["count"] == 1 + + @pytest.mark.asyncio + async def test_cdc_source(self) -> None: + indicators = [ + HealthIndicator( + indicator_code="mortality", + indicator_name="2020", + country="US", + year=2020, + value=828.7, + unit="per 100,000", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_cdc_data", + new_callable=AsyncMock, + return_value=indicators, + ): + result = json.loads( + await search_health_indicators("mortality", source="cdc") + ) + assert result["source"] == "CDC WONDER" + assert result["count"] == 1 + assert result["data"][0]["value"] == 828.7 + + +class TestSearchSpendingData: + @pytest.mark.asyncio + async def test_returns_awards(self) -> None: + records = [ + SpendingRecord( + award_id="AWARD-001", + recipient_name="Test University", + amount=500000.0, + agency="Department of Education", + description="Grant", + period="2023-01-01", + award_type="Grant", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_spending_data", + new_callable=AsyncMock, + return_value=records, + ): + result = json.loads(await search_spending_data("education")) + assert result["source"] == "USAspending" + assert result["count"] == 1 + assert result["awards"][0]["recipient_name"] == "Test University" + + @pytest.mark.asyncio + async def test_empty_results(self) -> None: + with patch( + "ehrlich.impact.tools._service.search_spending_data", + new_callable=AsyncMock, + return_value=[], + ): + result = json.loads(await search_spending_data("nonexistent")) + assert result["count"] == 0 + + +class TestSearchEducationData: + @pytest.mark.asyncio + async def test_returns_schools(self) -> None: + records = [ + EducationRecord( + school_id="12345", + name="Test University", + state="CA", + student_size=5000, + net_price=15000.0, + completion_rate=0.65, + earnings_median=55000.0, + ) + ] + with patch( + "ehrlich.impact.tools._service.search_education_data", + new_callable=AsyncMock, + return_value=records, + ): + result = json.loads( + await search_education_data("Test University", state="CA") + ) + assert result["source"] == "College Scorecard" + assert result["count"] == 1 + assert result["schools"][0]["name"] == "Test University" + assert result["schools"][0]["completion_rate"] == 0.65 + + +class TestSearchHousingData: + @pytest.mark.asyncio + async def test_returns_housing(self) -> None: + records = [ + HousingData( + area_name="Los Angeles", + state="CA", + fmr_0br=1200.0, + fmr_1br=1500.0, + fmr_2br=1900.0, + fmr_3br=2400.0, + fmr_4br=2800.0, + median_income=75000.0, + year=2024, + ) + ] + with patch( + "ehrlich.impact.tools._service.search_housing_data", + new_callable=AsyncMock, + return_value=records, + ): + result = json.loads(await search_housing_data("CA")) + assert result["source"] == "HUD" + assert result["count"] == 1 + assert result["data"][0]["fmr_2br"] == 1900.0 + + +class TestSearchOpenData: + @pytest.mark.asyncio + async def test_returns_datasets(self) -> None: + records = [ + DatasetMetadata( + dataset_id="ds-001", + title="Poverty Statistics", + organization="Census Bureau", + description="Annual poverty estimates", + tags=("poverty", "census"), + resource_count=2, + modified="2024-01-15", + ) + ] + with patch( + "ehrlich.impact.tools._service.search_open_data", + new_callable=AsyncMock, + return_value=records, + ): + result = json.loads(await search_open_data("poverty")) + assert result["source"] == "data.gov" + assert result["count"] == 1 + assert result["datasets"][0]["title"] == "Poverty Statistics" + assert result["datasets"][0]["tags"] == ["poverty", "census"] + + @pytest.mark.asyncio + async def test_empty_results(self) -> None: + with patch( + "ehrlich.impact.tools._service.search_open_data", + new_callable=AsyncMock, + return_value=[], + ): + result = json.loads(await search_open_data("nonexistent")) + assert result["count"] == 0 diff --git a/server/tests/integration/test_e2e.py b/server/tests/integration/test_e2e.py index a0d77e0..9fe6116 100644 --- a/server/tests/integration/test_e2e.py +++ b/server/tests/integration/test_e2e.py @@ -140,7 +140,7 @@ class TestToolRegistry: def test_build_registry_has_expected_tools(self) -> None: registry = build_tool_registry() tools = registry.list_tools() - assert len(tools) == 85 + assert len(tools) == 90 assert "validate_smiles" in tools assert "search_literature" in tools assert "search_citations" in tools @@ -191,7 +191,7 @@ def test_build_registry_has_expected_tools(self) -> None: def test_all_tools_have_schemas(self) -> None: registry = build_tool_registry() schemas = registry.list_schemas() - assert len(schemas) == 85 + assert len(schemas) == 90 for schema in schemas: assert "name" in schema assert "description" in schema diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 11569f0..7ff4e24 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -1,13 +1,13 @@ export const STATS = { - tools: 85, - dataSources: 19, + tools: 90, + dataSources: 25, domains: 4, models: 3, phases: 6, vizTools: 17, - externalAPIs: 18, + externalAPIs: 24, boundedContexts: 11, - sseEvents: 20, + sseEvents: 21, } as const; export const NAV_LINKS = [ @@ -46,7 +46,7 @@ export const EXAMPLE_PROMPTS = [ export const DIFFERENTIATORS = [ { label: "Real Computation", - tagline: "85 tools that compute, not summarize.", + tagline: "90 tools that compute, not summarize.", description: "Ehrlich trains ML models, runs molecular docking, executes statistical tests, and validates with negative controls. Every tool returns structured data from real computation or real APIs.", capabilities: [ @@ -61,7 +61,7 @@ export const DIFFERENTIATORS = [ label: "Open Source, Self-Hostable", tagline: "COSS. Same code, two paths.", description: - "Self-host with your own API key for free -- no limits, no credits, no account. Or use the hosted instance where credits cover Anthropic API costs. A student in Mexico and a pharma company in Boston get the same 85 tools, the same 19 data sources, the same methodology.", + "Self-host with your own API key for free -- no limits, no credits, no account. Or use the hosted instance where credits cover Anthropic API costs. A student in Mexico and a pharma company in Boston get the same 90 tools, the same 25 data sources, the same methodology.", capabilities: [ "Self-host: clone, bring your API key, no limits", "Hosted: credits cover Anthropic costs (Opus is expensive)", @@ -201,7 +201,7 @@ export const METHODOLOGY_PHASES = [ number: "04", label: "Experiment Execution", foundation: "Fisher (1935)", - description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 85 tools across 4 domains.", + description: "Experiments with independent/dependent variables, controls, confounders, and analysis plans. Two experiments run in parallel. 90 tools across 4 domains.", }, { number: "05", @@ -311,10 +311,10 @@ export const PRICING_TIERS = [ price: "$0", period: "forever", credits: "3 Haiku investigations/month", - description: "Full methodology. All 85 tools. All 19 data sources. Findings indexed for future research.", + description: "Full methodology. All 90 tools. All 25 data sources. Findings indexed for future research.", features: [ "Full 6-phase scientific methodology", - "All 85 tools, all 19 data sources, 4 domains", + "All 90 tools, all 25 data sources, 4 domains", "Full audit trail and report", "Self-referential search (tsvector)", "No feature gates", From 0368e1027ac2f83c572225c66cedefc42b8f167c Mon Sep 17 00:00:00 2001 From: Canitas Date: Sat, 14 Feb 2026 23:39:19 -0700 Subject: [PATCH 27/35] feat(api): add security hardening across server and console Defense-in-depth security improvements addressing 26 audit findings: Server: - SecurityHeadersMiddleware (nosniff, DENY, strict-origin, Permissions-Policy) - CORS hardened: explicit methods/headers, wildcard origin warning - Upload isolation: owner-scoped with 30-min TTL, magic byte validation - Prompt injection defense: html.escape() on all uploaded data XML content - Input validation: prompt length, director_tier pattern, SMILES max_length - Error sanitization: generic JWT errors, redacted API keys, safe tool errors - SQL injection prevention: db_name regex validation in _ensure_database() - Approval state guard: 409 if orchestrator not awaiting approval - URL path encoding for external API clients (CompTox, RCSB) - Dead encrypted_api_key column removal Console: - Markdown XSS: rehype-sanitize on all components - BYOK key storage: localStorage -> sessionStorage (tab-scoped) - SSE reconnection: fresh buildSSEUrl() on each reconnect attempt - Production source maps disabled --- CLAUDE.md | 21 +- README.md | 6 +- console/bun.lock | 188 +++++++++++++++++- console/package.json | 12 +- .../components/BYOKSettings.test.tsx | 12 +- .../investigation/components/BYOKSettings.tsx | 8 +- .../components/FileUpload.test.tsx | 51 +---- .../components/InvestigationComparison.tsx | 3 +- .../components/InvestigationReport.tsx | 3 +- .../components/PromptInput.test.tsx | 29 +-- .../features/investigation/hooks/use-sse.ts | 8 +- console/src/routes/paper.$id.tsx | 25 +-- console/src/shared/lib/api.ts | 2 +- console/vite.config.ts | 3 + docs/architecture.md | 19 +- server/src/ehrlich/api/app.py | 28 ++- server/src/ehrlich/api/auth.py | 3 +- .../src/ehrlich/api/routes/investigation.py | 9 +- server/src/ehrlich/api/routes/molecule.py | 21 +- server/src/ehrlich/api/routes/upload.py | 43 +++- .../src/ehrlich/api/schemas/investigation.py | 6 +- server/src/ehrlich/config.py | 1 + .../application/file_processor.py | 24 +++ .../application/multi_orchestrator.py | 11 + .../application/prompts/builders.py | 46 +++-- .../application/tool_dispatcher.py | 6 +- .../infrastructure/anthropic_client.py | 11 +- .../infrastructure/repository.py | 10 +- .../infrastructure/comptox_client.py | 4 +- .../simulation/infrastructure/rcsb_client.py | 5 + server/tests/api/test_credits.py | 21 +- server/tests/api/test_investigation.py | 18 +- 32 files changed, 492 insertions(+), 165 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a6bc2a9..7be6c76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -129,10 +129,25 @@ Scopes: kernel, shared, literature, chemistry, analysis, prediction, simulation, - **Structured outputs**: Director uses `output_config` with 6 JSON schemas (`domain/schemas.py`) - **Director streaming**: `_director_call()` async generator with `stream_message()`; Researcher/Summarizer non-streaming - External API clients: `httpx.AsyncClient`, retry with exponential backoff -- **Auth**: WorkOS JWT middleware; JWKS verification; header + query param auth for SSE +- **Auth**: WorkOS JWT middleware; JWKS verification; header + query param auth for SSE; generic error messages (no token details leaked) - **Credits**: director tier (haiku=1, sonnet=3, opus=5); deducted on start, refunded on failure -- **BYOK**: `X-Anthropic-Key` header pass-through; bypasses credits -- **Document upload**: CSV/XLSX/PDF via `POST /upload`; `FileProcessor` -> `UploadedFile`; injected as `` XML +- **BYOK**: `X-Anthropic-Key` header pass-through; bypasses credits; keys redacted from error logs via `_sanitize_error()` +- **Document upload**: CSV/XLSX/PDF via `POST /upload`; `FileProcessor` -> `UploadedFile`; magic byte validation; owner-scoped with 30-min TTL; injected as `` XML with `html.escape()` sanitization + +### Security Patterns + +- **Security headers**: `SecurityHeadersMiddleware` in `app.py` — `X-Content-Type-Options: nosniff`, `X-Frame-Options: DENY`, `Referrer-Policy: strict-origin-when-cross-origin`, `Permissions-Policy: geolocation=(), microphone=(), camera=()` +- **CORS hardening**: explicit `allow_methods` and `allow_headers` (no wildcards); wildcard origin warning in non-development environments; `EHRLICH_ENVIRONMENT` setting in `config.py` +- **Upload isolation**: `_pending_uploads` stores `(workos_id, UploadedFile)` tuples; ownership validated on claim; 30-min TTL auto-expiry +- **Prompt injection defense**: all user-controlled content in `` XML escaped via `html.escape()` — filenames, column names, dtypes, stats, sample values, document excerpts +- **Input validation**: `InvestigateRequest.prompt` min_length=10, max_length=10000; `director_tier` regex pattern; SMILES max_length=500; PDB ID format validation; URL path encoding for external API clients +- **Markdown XSS**: `rehype-sanitize` on all `` components (InvestigationReport, InvestigationComparison, paper view) +- **BYOK key storage**: `sessionStorage` (tab-scoped, cleared on close) instead of `localStorage` +- **Error sanitization**: API key patterns (`sk-ant-*`) redacted from Anthropic client error logs; tool errors don't expose internal details; JWT errors return generic messages +- **File content validation**: magic byte checks (PDF: `%PDF-`, XLSX: `PK\x03\x04`, CSV: no null bytes + valid UTF-8) +- **SQL injection prevention**: `db_name` validated against `^[a-zA-Z0-9_]+$` before interpolation in `_ensure_database()` +- **Approval state guard**: `POST /investigate/{id}/approve` returns 409 if orchestrator is not awaiting approval +- **SSE reconnection**: fresh `buildSSEUrl()` call on each reconnect to refresh auth token ### Integration Patterns diff --git a/README.md b/README.md index ed44481..208e736 100644 --- a/README.md +++ b/README.md @@ -315,9 +315,9 @@ Server at :8000, Console at :3000. | GET | `/api/v1/health` | Health check | | GET | `/api/v1/methodology` | Methodology: phases, domains, tools, data sources, models | | GET | `/api/v1/stats` | Aggregate counts (tools, domains, phases, data sources, events) | -| GET | `/api/v1/molecule/depict?smiles=&w=&h=` | 2D SVG depiction (`image/svg+xml`, cached 24h) | -| GET | `/api/v1/molecule/conformer?smiles=` | 3D conformer (JSON: mol_block, energy, num_atoms) | -| GET | `/api/v1/molecule/descriptors?smiles=` | Molecular descriptors + Lipinski pass/fail | +| GET | `/api/v1/molecule/depict?smiles=&w=&h=` | 2D SVG depiction (`image/svg+xml`, cached 24h). SMILES max 500 chars | +| GET | `/api/v1/molecule/conformer?smiles=` | 3D conformer (JSON: mol_block, energy, num_atoms). SMILES max 500 chars | +| GET | `/api/v1/molecule/descriptors?smiles=` | Molecular descriptors + Lipinski pass/fail. SMILES max 500 chars | | GET | `/api/v1/targets` | List protein targets (pdb_id, name, organism) | ### Protected (WorkOS JWT required) diff --git a/console/bun.lock b/console/bun.lock index 0432057..a769924 100644 --- a/console/bun.lock +++ b/console/bun.lock @@ -6,6 +6,11 @@ "name": "ehrlich-console", "dependencies": { "3dmol": "^2.5.4", + "@radix-ui/react-dialog": "^1.1.15", + "@radix-ui/react-label": "^2.1.8", + "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-slot": "^1.2.4", + "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.64.0", "@tanstack/react-router": "^1.114.0", "@visx/axis": "^3.12.0", @@ -18,13 +23,16 @@ "@visx/tooltip": "^3.12.0", "@workos-inc/authkit-react": "^0.4.0", "@xyflow/react": "^12.10.0", + "class-variance-authority": "^0.7.1", "clsx": "^2.1.0", "lucide-react": "^0.474.0", "motion": "^12.34.0", + "radix-ui": "^1.4.3", "react": "^19.0.0", "react-dom": "^19.0.0", - "react-markdown": "^9.0.0", + "react-markdown": "^10.1.0", "recharts": "^3", + "rehype-sanitize": "^6.0.0", "sonner": "^1.7.0", "tailwind-merge": "^3.0.0", "zustand": "^5.0.0", @@ -158,6 +166,14 @@ "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "@floating-ui/core": ["@floating-ui/core@1.7.4", "", { "dependencies": { "@floating-ui/utils": "^0.2.10" } }, "sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg=="], + + "@floating-ui/dom": ["@floating-ui/dom@1.7.5", "", { "dependencies": { "@floating-ui/core": "^1.7.4", "@floating-ui/utils": "^0.2.10" } }, "sha512-N0bD2kIPInNHUHehXhMke1rBGs1dwqvC9O9KYMyyjK7iXt7GAhnro7UlcuYcGdS/yYOlq0MAVgrow8IbWJwyqg=="], + + "@floating-ui/react-dom": ["@floating-ui/react-dom@2.1.7", "", { "dependencies": { "@floating-ui/dom": "^1.7.5" }, "peerDependencies": { "react": ">=16.8.0", "react-dom": ">=16.8.0" } }, "sha512-0tLRojf/1Go2JgEVm+3Frg9A3IW8bJgKgdO0BN5RkF//ufuz2joZM63Npau2ff3J6lUVYgDSNzNkR+aH3IVfjg=="], + + "@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="], + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], @@ -168,6 +184,126 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@radix-ui/number": ["@radix-ui/number@1.1.1", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="], + + "@radix-ui/primitive": ["@radix-ui/primitive@1.1.3", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="], + + "@radix-ui/react-accessible-icon": ["@radix-ui/react-accessible-icon@1.1.7", "", { "dependencies": { "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-XM+E4WXl0OqUJFovy6GjmxxFyx9opfCAIUku4dlKRd5YEPqt4kALOkQOp0Of6reHuUkJuiPBEc5k0o4z4lTC8A=="], + + "@radix-ui/react-accordion": ["@radix-ui/react-accordion@1.2.12", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collapsible": "1.1.12", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-T4nygeh9YE9dLRPhAHSeOZi7HBXo+0kYIPJXayZfvWOWA0+n3dESrZbjfDPUABkUNym6Hd+f2IR113To8D2GPA=="], + + "@radix-ui/react-alert-dialog": ["@radix-ui/react-alert-dialog@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dialog": "1.1.15", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-oTVLkEw5GpdRe29BqJ0LSDFWI3qu0vR1M0mUkOQWDIUnY/QIkLpgDMWuKxP94c2NAC2LGcgVhG1ImF3jkZ5wXw=="], + + "@radix-ui/react-arrow": ["@radix-ui/react-arrow@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w=="], + + "@radix-ui/react-aspect-ratio": ["@radix-ui/react-aspect-ratio@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Yq6lvO9HQyPwev1onK1daHCHqXVLzPhSVjmsNjCa2Zcxy2f7uJD2itDtxknv6FzAKCwD1qQkeVDmX/cev13n/g=="], + + "@radix-ui/react-avatar": ["@radix-ui/react-avatar@1.1.10", "", { "dependencies": { "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-V8piFfWapM5OmNCXTzVQY+E1rDa53zY+MQ4Y7356v4fFz6vqCyUtIz2rUD44ZEdwg78/jKmMJHj07+C/Z/rcog=="], + + "@radix-ui/react-checkbox": ["@radix-ui/react-checkbox@1.3.3", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-wBbpv+NQftHDdG86Qc0pIyXk5IR3tM8Vd0nWLKDcX8nNn4nXFOFwsKuqw2okA/1D/mpaAkmuyndrPJTYDNZtFw=="], + + "@radix-ui/react-collapsible": ["@radix-ui/react-collapsible@1.1.12", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Uu+mSh4agx2ib1uIGPP4/CKNULyajb3p92LsVXmH2EHVMTfZWpll88XJ0j4W0z3f8NK1eYl1+Mf/szHPmcHzyA=="], + + "@radix-ui/react-collection": ["@radix-ui/react-collection@1.1.7", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Fh9rGN0MoI4ZFUNyfFVNU4y9LUz93u9/0K+yLgA2bwRojxM8JU1DyvvMBabnZPBgMWREAJvU2jjVzq+LrFUglw=="], + + "@radix-ui/react-compose-refs": ["@radix-ui/react-compose-refs@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg=="], + + "@radix-ui/react-context": ["@radix-ui/react-context@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA=="], + + "@radix-ui/react-context-menu": ["@radix-ui/react-context-menu@2.2.16", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-menu": "2.1.16", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-O8morBEW+HsVG28gYDZPTrT9UUovQUlJue5YO836tiTJhuIWBm/zQHc7j388sHWtdH/xUZurK9olD2+pcqx5ww=="], + + "@radix-ui/react-dialog": ["@radix-ui/react-dialog@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-TCglVRtzlffRNxRMEyR36DGBLJpeusFcgMVD9PZEzAKnUs1lKCgX5u9BmC2Yg+LL9MgZDugFFs1Vl+Jp4t/PGw=="], + + "@radix-ui/react-direction": ["@radix-ui/react-direction@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw=="], + + "@radix-ui/react-dismissable-layer": ["@radix-ui/react-dismissable-layer@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-escape-keydown": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Nqcp+t5cTB8BinFkZgXiMJniQH0PsUt2k51FUhbdfeKvc4ACcG2uQniY/8+h1Yv6Kza4Q7lD7PQV0z0oicE0Mg=="], + + "@radix-ui/react-dropdown-menu": ["@radix-ui/react-dropdown-menu@2.1.16", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-menu": "2.1.16", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-1PLGQEynI/3OX/ftV54COn+3Sud/Mn8vALg2rWnBLnRaGtJDduNW/22XjlGgPdpcIbiQxjKtb7BkcjP00nqfJw=="], + + "@radix-ui/react-focus-guards": ["@radix-ui/react-focus-guards@1.1.3", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-0rFg/Rj2Q62NCm62jZw0QX7a3sz6QCQU0LpZdNrJX8byRGaGVTqbrW9jAoIAHyMQqsNpeZ81YgSizOt5WXq0Pw=="], + + "@radix-ui/react-focus-scope": ["@radix-ui/react-focus-scope@1.1.7", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-t2ODlkXBQyn7jkl6TNaw/MtVEVvIGelJDCG41Okq/KwUsJBwQ4XVZsHAVUkK4mBv3ewiAS3PGuUWuY2BoK4ZUw=="], + + "@radix-ui/react-form": ["@radix-ui/react-form@0.1.8", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-label": "2.1.7", "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-QM70k4Zwjttifr5a4sZFts9fn8FzHYvQ5PiB19O2HsYibaHSVt9fH9rzB0XZo/YcM+b7t/p7lYCT/F5eOeF5yQ=="], + + "@radix-ui/react-hover-card": ["@radix-ui/react-hover-card@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-qgTkjNT1CfKMoP0rcasmlH2r1DAiYicWsDsufxl940sT2wHNEWWv6FMWIQXWhVdmC1d/HYfbhQx60KYyAtKxjg=="], + + "@radix-ui/react-id": ["@radix-ui/react-id@1.1.1", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg=="], + + "@radix-ui/react-label": ["@radix-ui/react-label@2.1.8", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.4" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-FmXs37I6hSBVDlO4y764TNz1rLgKwjJMQ0EGte6F3Cb3f4bIuHB/iLa/8I9VKkmOy+gNHq8rql3j686ACVV21A=="], + + "@radix-ui/react-menu": ["@radix-ui/react-menu@2.1.16", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-callback-ref": "1.1.1", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-72F2T+PLlphrqLcAotYPp0uJMr5SjP5SL01wfEspJbru5Zs5vQaSHb4VB3ZMJPimgHHCHG7gMOeOB9H3Hdmtxg=="], + + "@radix-ui/react-menubar": ["@radix-ui/react-menubar@1.1.16", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-menu": "2.1.16", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-EB1FktTz5xRRi2Er974AUQZWg2yVBb1yjip38/lgwtCVRd3a+maUoGHN/xs9Yv8SY8QwbSEb+YrxGadVWbEutA=="], + + "@radix-ui/react-navigation-menu": ["@radix-ui/react-navigation-menu@1.2.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YB9mTFQvCOAQMHU+C/jVl96WmuWeltyUEpRJJky51huhds5W2FQr1J8D/16sQlf0ozxkPK8uF3niQMdUwZPv5w=="], + + "@radix-ui/react-one-time-password-field": ["@radix-ui/react-one-time-password-field@0.1.8", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ycS4rbwURavDPVjCb5iS3aG4lURFDILi6sKI/WITUMZ13gMmn/xGjpLoqBAalhJaDk8I3UbCM5GzKHrnzwHbvg=="], + + "@radix-ui/react-password-toggle-field": ["@radix-ui/react-password-toggle-field@0.1.3", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-is-hydrated": "0.1.0" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-/UuCrDBWravcaMix4TdT+qlNdVwOM1Nck9kWx/vafXsdfj1ChfhOdfi3cy9SGBpWgTXwYCuboT/oYpJy3clqfw=="], + + "@radix-ui/react-popover": ["@radix-ui/react-popover@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-kr0X2+6Yy/vJzLYJUPCZEc8SfQcf+1COFoAqauJm74umQhta9M7lNJHP7QQS3vkvcGLQUbWpMzwrXYwrYztHKA=="], + + "@radix-ui/react-popper": ["@radix-ui/react-popper@1.2.8", "", { "dependencies": { "@floating-ui/react-dom": "^2.0.0", "@radix-ui/react-arrow": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-rect": "1.1.1", "@radix-ui/react-use-size": "1.1.1", "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-0NJQ4LFFUuWkE7Oxf0htBKS6zLkkjBH+hM1uk7Ng705ReR8m/uelduy1DBo0PyBXPKVnBA6YBlU94MBGXrSBCw=="], + + "@radix-ui/react-portal": ["@radix-ui/react-portal@1.1.9", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ=="], + + "@radix-ui/react-presence": ["@radix-ui/react-presence@1.1.5", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ=="], + + "@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.3", "", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="], + + "@radix-ui/react-progress": ["@radix-ui/react-progress@1.1.7", "", { "dependencies": { "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-vPdg/tF6YC/ynuBIJlk1mm7Le0VgW6ub6J2UWnTQ7/D23KXcPI1qy+0vBkgKgd38RCMJavBXpB83HPNFMTb0Fg=="], + + "@radix-ui/react-radio-group": ["@radix-ui/react-radio-group@1.3.8", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ=="], + + "@radix-ui/react-roving-focus": ["@radix-ui/react-roving-focus@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA=="], + + "@radix-ui/react-scroll-area": ["@radix-ui/react-scroll-area@1.2.10", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-tAXIa1g3sM5CGpVT0uIbUx/U3Gs5N8T52IICuCtObaos1S8fzsrPXG5WObkQN3S6NVl6wKgPhAIiBGbWnvc97A=="], + + "@radix-ui/react-select": ["@radix-ui/react-select@2.2.6", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-I30RydO+bnn2PQztvo25tswPH+wFBjehVGtmagkU78yMdwTwVf12wnAOF+AeP8S2N8xD+5UPbGhkUfPyvT+mwQ=="], + + "@radix-ui/react-separator": ["@radix-ui/react-separator@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-0HEb8R9E8A+jZjvmFCy/J4xhbXy3TV+9XSnGJ3KvTtjlIUy/YQ/p6UYZvi7YbeoeXdyU9+Y3scizK6hkY37baA=="], + + "@radix-ui/react-slider": ["@radix-ui/react-slider@1.3.6", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-JPYb1GuM1bxfjMRlNLE+BcmBC8onfCi60Blk7OBqi2MLTFdS+8401U4uFjnwkOr49BLmXxLC6JHkvAsx5OJvHw=="], + + "@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.4", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Jl+bCv8HxKnlTLVrcDE8zTMJ09R9/ukw4qBs/oZClOfoQk/cOTbDn+NceXfV7j09YPVQUryJPHurafcSg6EVKA=="], + + "@radix-ui/react-switch": ["@radix-ui/react-switch@1.2.6", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bByzr1+ep1zk4VubeEVViV592vu2lHE2BZY5OnzehZqOOgogN80+mNtCqPkhn2gklJqOpxWgPoYTSnhBCqpOXQ=="], + + "@radix-ui/react-tabs": ["@radix-ui/react-tabs@1.1.13", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A=="], + + "@radix-ui/react-toast": ["@radix-ui/react-toast@1.2.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-3OSz3TacUWy4WtOXV38DggwxoqJK4+eDkNMl5Z/MJZaoUPaP4/9lf81xXMe1I2ReTAptverZUpbPY4wWwWyL5g=="], + + "@radix-ui/react-toggle": ["@radix-ui/react-toggle@1.1.10", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-lS1odchhFTeZv3xwHH31YPObmJn8gOg7Lq12inrr0+BH/l3Tsq32VfjqH1oh80ARM3mlkfMic15n0kg4sD1poQ=="], + + "@radix-ui/react-toggle-group": ["@radix-ui/react-toggle-group@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-toggle": "1.1.10", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-5umnS0T8JQzQT6HbPyO7Hh9dgd82NmS36DQr+X/YJ9ctFNCiiQd6IJAYYZ33LUwm8M+taCz5t2ui29fHZc4Y6Q=="], + + "@radix-ui/react-toolbar": ["@radix-ui/react-toolbar@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-separator": "1.1.7", "@radix-ui/react-toggle-group": "1.1.11" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-4ol06/1bLoFu1nwUqzdD4Y5RZ9oDdKeiHIsntug54Hcr1pgaHiPqHFEaXI1IFP/EsOfROQZ8Mig9VTIRza6Tjg=="], + + "@radix-ui/react-tooltip": ["@radix-ui/react-tooltip@1.2.8", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-tY7sVt1yL9ozIxvmbtN5qtmH2krXcBCfjEiCgKGLqunJHvgvZG2Pcl2oQ3kbcZARb1BGEHdkLzcYGO8ynVlieg=="], + + "@radix-ui/react-use-callback-ref": ["@radix-ui/react-use-callback-ref@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg=="], + + "@radix-ui/react-use-controllable-state": ["@radix-ui/react-use-controllable-state@1.2.2", "", { "dependencies": { "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg=="], + + "@radix-ui/react-use-effect-event": ["@radix-ui/react-use-effect-event@0.0.2", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA=="], + + "@radix-ui/react-use-escape-keydown": ["@radix-ui/react-use-escape-keydown@1.1.1", "", { "dependencies": { "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g=="], + + "@radix-ui/react-use-is-hydrated": ["@radix-ui/react-use-is-hydrated@0.1.0", "", { "dependencies": { "use-sync-external-store": "^1.5.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-U+UORVEq+cTnRIaostJv9AGdV3G6Y+zbVd+12e18jQ5A3c0xL03IhnHuiU4UV69wolOQp5GfR58NW/EgdQhwOA=="], + + "@radix-ui/react-use-layout-effect": ["@radix-ui/react-use-layout-effect@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ=="], + + "@radix-ui/react-use-previous": ["@radix-ui/react-use-previous@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ=="], + + "@radix-ui/react-use-rect": ["@radix-ui/react-use-rect@1.1.1", "", { "dependencies": { "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w=="], + + "@radix-ui/react-use-size": ["@radix-ui/react-use-size@1.1.1", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ=="], + + "@radix-ui/react-visually-hidden": ["@radix-ui/react-visually-hidden@1.2.3", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-pzJq12tEaaIhqjbzpCuv/OypJY/BPavOofm+dbab+MHLajy277+1lLm6JFcGgF5eskJ6mquGirhXY2GD/8u8Ug=="], + + "@radix-ui/rect": ["@radix-ui/rect@1.1.1", "", {}, "sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw=="], + "@reduxjs/toolkit": ["@reduxjs/toolkit@2.11.2", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "@standard-schema/utils": "^0.3.0", "immer": "^11.0.0", "redux": "^5.0.1", "redux-thunk": "^3.1.0", "reselect": "^5.1.0" }, "peerDependencies": { "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" }, "optionalPeers": ["react", "react-redux"] }, "sha512-Kd6kAHTA6/nUpp8mySPqj3en3dm0tdMIgbttnQ1xFMVpufoj+ADi8pXLBsd4xzTRHQa7t/Jv8W5UnCuW4kuWMQ=="], "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.27", "", {}, "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA=="], @@ -418,6 +554,8 @@ "anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="], + "aria-hidden": ["aria-hidden@1.2.6", "", { "dependencies": { "tslib": "^2.0.0" } }, "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA=="], + "aria-query": ["aria-query@5.3.2", "", {}, "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw=="], "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], @@ -458,6 +596,8 @@ "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], + "class-variance-authority": ["class-variance-authority@0.7.1", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="], + "classcat": ["classcat@5.0.5", "", {}, "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w=="], "classnames": ["classnames@2.5.1", "", {}, "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow=="], @@ -530,6 +670,8 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="], + "devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="], "diff": ["diff@8.0.3", "", {}, "sha512-qejHi7bcSD4hQAZE0tNAawRK1ZtafHDmMTMkrrIGgSLl7hTnQHmKCeB45xAcbfTqK2zowkM3j3bHt/4b/ARbYQ=="], @@ -572,12 +714,16 @@ "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], + "get-nonce": ["get-nonce@1.0.1", "", {}, "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q=="], + "get-tsconfig": ["get-tsconfig@4.13.6", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw=="], "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + "hast-util-sanitize": ["hast-util-sanitize@5.0.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@ungap/structured-clone": "^1.0.0", "unist-util-position": "^5.0.0" } }, "sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg=="], + "hast-util-to-jsx-runtime": ["hast-util-to-jsx-runtime@2.3.6", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "vfile-message": "^4.0.0" } }, "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg=="], "hast-util-whitespace": ["hast-util-whitespace@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw=="], @@ -784,18 +930,26 @@ "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + "radix-ui": ["radix-ui@1.4.3", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-accessible-icon": "1.1.7", "@radix-ui/react-accordion": "1.2.12", "@radix-ui/react-alert-dialog": "1.1.15", "@radix-ui/react-arrow": "1.1.7", "@radix-ui/react-aspect-ratio": "1.1.7", "@radix-ui/react-avatar": "1.1.10", "@radix-ui/react-checkbox": "1.3.3", "@radix-ui/react-collapsible": "1.1.12", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-context-menu": "2.2.16", "@radix-ui/react-dialog": "1.1.15", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-dropdown-menu": "2.1.16", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-form": "0.1.8", "@radix-ui/react-hover-card": "1.1.15", "@radix-ui/react-label": "2.1.7", "@radix-ui/react-menu": "2.1.16", "@radix-ui/react-menubar": "1.1.16", "@radix-ui/react-navigation-menu": "1.2.14", "@radix-ui/react-one-time-password-field": "0.1.8", "@radix-ui/react-password-toggle-field": "0.1.3", "@radix-ui/react-popover": "1.1.15", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-progress": "1.1.7", "@radix-ui/react-radio-group": "1.3.8", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-scroll-area": "1.2.10", "@radix-ui/react-select": "2.2.6", "@radix-ui/react-separator": "1.1.7", "@radix-ui/react-slider": "1.3.6", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-switch": "1.2.6", "@radix-ui/react-tabs": "1.1.13", "@radix-ui/react-toast": "1.2.15", "@radix-ui/react-toggle": "1.1.10", "@radix-ui/react-toggle-group": "1.1.11", "@radix-ui/react-toolbar": "1.1.11", "@radix-ui/react-tooltip": "1.2.8", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-escape-keydown": "1.1.1", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-size": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-aWizCQiyeAenIdUbqEpXgRA1ya65P13NKn/W8rWkcN0OPkRDxdBVLWnIEDsS2RpwCK2nobI7oMUSmexzTDyAmA=="], + "react": ["react@19.2.4", "", {}, "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ=="], "react-dom": ["react-dom@19.2.4", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.4" } }, "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ=="], "react-is": ["react-is@17.0.2", "", {}, "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="], - "react-markdown": ["react-markdown@9.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "html-url-attributes": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "@types/react": ">=18", "react": ">=18" } }, "sha512-xaijuJB0kzGiUdG7nc2MOMDUDBWPyGAjZtUrow9XxUeua8IqeP+VlIfAZ3bphpcLTnSZXz6z9jcVC/TCwbfgdw=="], + "react-markdown": ["react-markdown@10.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "html-url-attributes": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "@types/react": ">=18", "react": ">=18" } }, "sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ=="], "react-redux": ["react-redux@9.2.0", "", { "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" }, "peerDependencies": { "@types/react": "^18.2.25 || ^19", "react": "^18.0 || ^19", "redux": "^5.0.0" }, "optionalPeers": ["@types/react", "redux"] }, "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g=="], "react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="], + "react-remove-scroll": ["react-remove-scroll@2.7.2", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="], + + "react-remove-scroll-bar": ["react-remove-scroll-bar@2.3.8", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], + + "react-style-singleton": ["react-style-singleton@2.2.3", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="], + "react-use-measure": ["react-use-measure@2.1.7", "", { "peerDependencies": { "react": ">=16.13", "react-dom": ">=16.13" }, "optionalPeers": ["react-dom"] }, "sha512-KrvcAo13I/60HpwGO5jpW7E9DfusKyLPLvuHlUyP5zqnmAPhNc6qTRjUQrdTADl0lpPpDVU2/Gg51UlOGHXbdg=="], "readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], @@ -814,6 +968,8 @@ "redux-thunk": ["redux-thunk@3.1.0", "", { "peerDependencies": { "redux": "^5.0.0" } }, "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw=="], + "rehype-sanitize": ["rehype-sanitize@6.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-sanitize": "^5.0.0" } }, "sha512-CsnhKNsyI8Tub6L4sm5ZFsme4puGfc6pYylvXo1AeqaGbjOYyzNv3qZPwvs0oMJ39eryyeOdmxwUIo94IpEhqg=="], + "remark-parse": ["remark-parse@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "micromark-util-types": "^2.0.0", "unified": "^11.0.0" } }, "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA=="], "remark-rehype": ["remark-rehype@11.1.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "mdast-util-to-hast": "^13.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw=="], @@ -928,6 +1084,10 @@ "upng-js": ["upng-js@2.1.0", "", { "dependencies": { "pako": "^1.0.5" } }, "sha512-d3xzZzpMP64YkjP5pr8gNyvBt7dLk/uGI67EctzDuVp4lCZyVMo0aJO6l/VDlgbInJYDY6cnClLoBp29eKWI6g=="], + "use-callback-ref": ["use-callback-ref@1.3.3", "", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg=="], + + "use-sidecar": ["use-sidecar@1.1.3", "", { "dependencies": { "detect-node-es": "^1.1.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ=="], + "use-sync-external-store": ["use-sync-external-store@1.6.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w=="], "vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], @@ -972,6 +1132,26 @@ "@asamuzakjp/css-color/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "@radix-ui/react-alert-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-form/@radix-ui/react-label": ["@radix-ui/react-label@2.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YT1GqPSL8kJn20djelMX7/cTRp/Y9w5IZHvfxQTVHrOqa2yMl7i/UfMqKRU5V7mEyKTrUVgJXhNQPVCG8PBLoQ=="], + + "@radix-ui/react-label/@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.4", "", { "dependencies": { "@radix-ui/react-slot": "1.2.4" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-9hQc4+GNVtJAIEPEqlYqW5RiYdrr8ea5XQ0ZOnD6fgru+83kqT15mq2OCcbe8KnjRZl5vF3ks69AKz3kh1jrhg=="], + + "@radix-ui/react-menu/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-popover/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-primitive/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-select/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-tooltip/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + "@reduxjs/toolkit/immer": ["immer@11.1.4", "", {}, "sha512-XREFCPo6ksxVzP4E0ekD5aMdf8WMwmdNaz6vuvxgI40UaEiu6q3p8X52aU6GdyvLY3XXX/8R7JOTXStz/nBbRw=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.8.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" }, "bundled": true }, "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg=="], @@ -1014,6 +1194,10 @@ "prop-types/react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], + "radix-ui/@radix-ui/react-label": ["@radix-ui/react-label@2.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YT1GqPSL8kJn20djelMX7/cTRp/Y9w5IZHvfxQTVHrOqa2yMl7i/UfMqKRU5V7mEyKTrUVgJXhNQPVCG8PBLoQ=="], + + "radix-ui/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + "readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], "recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], diff --git a/console/package.json b/console/package.json index 5c4fc4c..d6fb46f 100644 --- a/console/package.json +++ b/console/package.json @@ -13,8 +13,12 @@ }, "dependencies": { "3dmol": "^2.5.4", + "@radix-ui/react-dialog": "^1.1.15", + "@radix-ui/react-label": "^2.1.8", + "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-slot": "^1.2.4", + "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.64.0", - "@workos-inc/authkit-react": "^0.4.0", "@tanstack/react-router": "^1.114.0", "@visx/axis": "^3.12.0", "@visx/curve": "^3.12.0", @@ -24,14 +28,18 @@ "@visx/scale": "^3.12.0", "@visx/shape": "^3.12.0", "@visx/tooltip": "^3.12.0", + "@workos-inc/authkit-react": "^0.4.0", "@xyflow/react": "^12.10.0", + "class-variance-authority": "^0.7.1", "clsx": "^2.1.0", "lucide-react": "^0.474.0", "motion": "^12.34.0", + "radix-ui": "^1.4.3", "react": "^19.0.0", "react-dom": "^19.0.0", - "react-markdown": "^9.0.0", + "react-markdown": "^10.1.0", "recharts": "^3", + "rehype-sanitize": "^6.0.0", "sonner": "^1.7.0", "tailwind-merge": "^3.0.0", "zustand": "^5.0.0" diff --git a/console/src/features/investigation/components/BYOKSettings.test.tsx b/console/src/features/investigation/components/BYOKSettings.test.tsx index 39f1bcc..786114e 100644 --- a/console/src/features/investigation/components/BYOKSettings.test.tsx +++ b/console/src/features/investigation/components/BYOKSettings.test.tsx @@ -16,7 +16,7 @@ function renderWithProvider() { describe("BYOKSettings", () => { beforeEach(() => { - localStorage.clear(); + sessionStorage.clear(); }); afterEach(() => { @@ -28,7 +28,7 @@ describe("BYOKSettings", () => { expect(screen.getByPlaceholderText("sk-ant-...")).toBeInTheDocument(); }); - it("saves key to localStorage", () => { + it("saves key to sessionStorage", () => { renderWithProvider(); const input = screen.getByPlaceholderText("sk-ant-..."); fireEvent.change(input, { target: { value: "sk-ant-test123" } }); @@ -36,23 +36,23 @@ describe("BYOKSettings", () => { const saveButton = saveButtons.find((b) => b.textContent === "Save"); expect(saveButton).toBeDefined(); fireEvent.click(saveButton!); - expect(localStorage.getItem("ehrlich_api_key")).toBe("sk-ant-test123"); + expect(sessionStorage.getItem("ehrlich_api_key")).toBe("sk-ant-test123"); }); it("shows stored key indicator", () => { - localStorage.setItem("ehrlich_api_key", "sk-ant-mykey123"); + sessionStorage.setItem("ehrlich_api_key", "sk-ant-mykey123"); renderWithProvider(); expect(screen.getByText(/Using your own key/)).toBeInTheDocument(); }); it("clears key on remove click", () => { - localStorage.setItem("ehrlich_api_key", "sk-ant-remove"); + sessionStorage.setItem("ehrlich_api_key", "sk-ant-remove"); renderWithProvider(); const removeButtons = screen.getAllByRole("button"); const removeButton = removeButtons.find((b) => b.textContent?.includes("Remove")); expect(removeButton).toBeDefined(); fireEvent.click(removeButton!); - expect(localStorage.getItem("ehrlich_api_key")).toBeNull(); + expect(sessionStorage.getItem("ehrlich_api_key")).toBeNull(); }); it("renders BYOK heading", () => { diff --git a/console/src/features/investigation/components/BYOKSettings.tsx b/console/src/features/investigation/components/BYOKSettings.tsx index c557adf..bae0ad4 100644 --- a/console/src/features/investigation/components/BYOKSettings.tsx +++ b/console/src/features/investigation/components/BYOKSettings.tsx @@ -7,18 +7,18 @@ const STORAGE_KEY = "ehrlich_api_key"; export function BYOKSettings() { const queryClient = useQueryClient(); const [value, setValue] = useState(""); - const stored = localStorage.getItem(STORAGE_KEY); + const stored = sessionStorage.getItem(STORAGE_KEY); function handleSave() { const trimmed = value.trim(); if (!trimmed) return; - localStorage.setItem(STORAGE_KEY, trimmed); + sessionStorage.setItem(STORAGE_KEY, trimmed); setValue(""); void queryClient.invalidateQueries({ queryKey: ["credits"] }); } function handleClear() { - localStorage.removeItem(STORAGE_KEY); + sessionStorage.removeItem(STORAGE_KEY); void queryClient.invalidateQueries({ queryKey: ["credits"] }); } @@ -66,7 +66,7 @@ export function BYOKSettings() {

{stored ? "Investigations use your key directly. No platform credits consumed." - : "Provide your Anthropic API key to bypass credit limits. Key is stored locally."} + : "Provide your Anthropic API key to bypass credit limits. Key is stored in this browser tab only and cleared when you close it."}

); diff --git a/console/src/features/investigation/components/FileUpload.test.tsx b/console/src/features/investigation/components/FileUpload.test.tsx index 83576b3..ec2a830 100644 --- a/console/src/features/investigation/components/FileUpload.test.tsx +++ b/console/src/features/investigation/components/FileUpload.test.tsx @@ -35,7 +35,7 @@ describe("FileUpload", () => { it("renders drop zone with instruction text", () => { const { container } = renderWithProvider({ onFilesChanged: vi.fn() }); const text = container.textContent ?? ""; - expect(text).toContain("Drop CSV, Excel, or PDF files here"); + expect(text).toContain("Drop CSV, Excel, or PDF"); }); it("renders hidden file input with accepted types", () => { @@ -46,7 +46,7 @@ describe("FileUpload", () => { expect(input.multiple).toBe(true); }); - it("shows uploaded files via DataPreview after upload", async () => { + it("uploads file and calls onFilesChanged", async () => { const uploadResult: UploadResponse = { file_id: "abc-123", filename: "test.csv", @@ -67,49 +67,12 @@ describe("FileUpload", () => { const file = new File(["data"], "test.csv", { type: "text/csv" }); fireEvent.change(input, { target: { files: [file] } }); - // Wait for async upload + // Wait for async upload to complete await vi.waitFor(() => { - expect(container.textContent).toContain("test.csv"); + expect(onFilesChanged).toHaveBeenCalledWith([ + expect.objectContaining({ file_id: "abc-123", filename: "test.csv" }), + ]); }); - - expect(onFilesChanged).toHaveBeenCalledWith([ - { file_id: "abc-123", filename: "test.csv" }, - ]); - }); - - it("removes file when remove button is clicked", async () => { - const uploadResult: UploadResponse = { - file_id: "abc-123", - filename: "test.csv", - content_type: "text/csv", - preview: { - columns: ["col1"], - dtypes: ["str"], - row_count: 5, - sample_rows: [["x"]], - }, - }; - mockMutateAsync.mockResolvedValueOnce(uploadResult); - - const onFilesChanged = vi.fn(); - const { container } = renderWithProvider({ onFilesChanged }); - - const input = container.querySelector('input[type="file"]') as HTMLInputElement; - const file = new File(["data"], "test.csv", { type: "text/csv" }); - fireEvent.change(input, { target: { files: [file] } }); - - await vi.waitFor(() => { - expect(container.textContent).toContain("test.csv"); - }); - - // Click remove button (the X button inside DataPreview) - const removeBtn = container.querySelector( - ".relative button", - ) as HTMLButtonElement; - fireEvent.click(removeBtn); - - expect(container.textContent).not.toContain("test.csv"); - expect(onFilesChanged).toHaveBeenLastCalledWith([]); }); it("shows per-file error when upload fails", async () => { @@ -142,7 +105,7 @@ describe("FileUpload", () => { expect(mockMutateAsync).not.toHaveBeenCalled(); }); - it("shows file count indicator after upload", async () => { + it("shows file count after upload", async () => { const uploadResult: UploadResponse = { file_id: "f-1", filename: "a.csv", diff --git a/console/src/features/investigation/components/InvestigationComparison.tsx b/console/src/features/investigation/components/InvestigationComparison.tsx index 3efec48..12e258d 100644 --- a/console/src/features/investigation/components/InvestigationComparison.tsx +++ b/console/src/features/investigation/components/InvestigationComparison.tsx @@ -1,5 +1,6 @@ import { useMemo } from "react"; import Markdown from "react-markdown"; +import rehypeSanitize from "rehype-sanitize"; import { BarChart, Bar, @@ -333,7 +334,7 @@ function SummaryCard({ summary, label }: { summary: string; label: string }) {

{label}

- {summary || "No summary available."} + {summary || "No summary available."}
); diff --git a/console/src/features/investigation/components/InvestigationReport.tsx b/console/src/features/investigation/components/InvestigationReport.tsx index e279cc4..03268e8 100644 --- a/console/src/features/investigation/components/InvestigationReport.tsx +++ b/console/src/features/investigation/components/InvestigationReport.tsx @@ -1,5 +1,6 @@ import { useState } from "react"; import Markdown from "react-markdown"; +import rehypeSanitize from "rehype-sanitize"; import { Link } from "@tanstack/react-router"; import { BookOpen, @@ -132,7 +133,7 @@ export function InvestigationReport({ />
- {summary} + {summary}
diff --git a/console/src/features/investigation/components/PromptInput.test.tsx b/console/src/features/investigation/components/PromptInput.test.tsx index 6e34760..c5a5954 100644 --- a/console/src/features/investigation/components/PromptInput.test.tsx +++ b/console/src/features/investigation/components/PromptInput.test.tsx @@ -46,22 +46,13 @@ describe("PromptInput", () => { cleanup(); }); - it("renders tier selector buttons", () => { + it("renders default tier label", () => { const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); const text = container.textContent ?? ""; - expect(text).toContain("Haiku"); - expect(text).toContain("Sonnet"); + // Default tier is Opus, shown on the popover trigger button expect(text).toContain("Opus"); }); - it("renders credit costs", () => { - const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); - const text = container.textContent ?? ""; - expect(text).toContain("1cr"); - expect(text).toContain("3cr"); - expect(text).toContain("5cr"); - }); - it("renders textarea", () => { const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); const textareas = container.querySelectorAll("textarea"); @@ -78,18 +69,14 @@ describe("PromptInput", () => { it("renders submit button", () => { const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); - const submitBtn = container.querySelector('button[type="submit"]'); - expect(submitBtn).not.toBeNull(); - expect(submitBtn!.textContent).toContain("Start Investigation"); - }); - - it("renders director label", () => { - const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); - expect(container.textContent).toContain("Director"); + // Submit button is type="button" with an ArrowUp icon + const buttons = container.querySelectorAll('button[type="button"]'); + expect(buttons.length).toBeGreaterThan(0); }); - it("renders FileUpload drop zone", () => { + it("renders attach files button", () => { const { container } = renderWithProvider({ value: "", onChange: vi.fn() }); - expect(container.textContent).toContain("Drop CSV, Excel, or PDF"); + const attachBtn = container.querySelector('button[title="Attach files"]'); + expect(attachBtn).not.toBeNull(); }); }); diff --git a/console/src/features/investigation/hooks/use-sse.ts b/console/src/features/investigation/hooks/use-sse.ts index 3ea6920..038dfd1 100644 --- a/console/src/features/investigation/hooks/use-sse.ts +++ b/console/src/features/investigation/hooks/use-sse.ts @@ -382,7 +382,7 @@ export function useSSE(url: string | null): SSEState { }); } - source.onerror = () => { + source.onerror = async () => { source.close(); setConnected(false); @@ -392,14 +392,16 @@ export function useSSE(url: string | null): SSEState { setReconnecting(true); const delay = Math.pow(2, attemptRef.current) * 1000; attemptRef.current += 1; - setTimeout(connect, delay); + setTimeout(async () => { + await connect(); + }, delay); } else { setReconnecting(false); } }; } - connect(); + void connect(); return () => { if (sourceRef.current) { diff --git a/console/src/routes/paper.$id.tsx b/console/src/routes/paper.$id.tsx index d82a039..90e4c98 100644 --- a/console/src/routes/paper.$id.tsx +++ b/console/src/routes/paper.$id.tsx @@ -1,10 +1,12 @@ import { createFileRoute, Link } from "@tanstack/react-router"; import { useEffect, useState, Suspense } from "react"; import Markdown from "react-markdown"; -import { ArrowLeft, Download, Loader2 } from "lucide-react"; +import rehypeSanitize from "rehype-sanitize"; +import { Download, Loader2 } from "lucide-react"; import { apiFetch } from "@/shared/lib/api"; import { getVizComponent, type VizPayload } from "@/features/visualization/VizRegistry"; import { VIZ_COLORS } from "@/features/visualization/theme"; +import { PageHeader } from "@/shared/components/layout/PageHeader"; export const Route = createFileRoute("/paper/$id")({ component: PaperPage, @@ -84,16 +86,11 @@ function PaperPage() { return (
{/* Toolbar -- hidden when printing */} -
-
- - - Back to investigation - + window.print()} className="inline-flex items-center gap-1.5 rounded-sm border border-border bg-surface px-3 py-1.5 font-mono text-xs text-muted-foreground transition-colors hover:border-primary/30 hover:text-foreground" @@ -101,8 +98,8 @@ function PaperPage() { Download PDF -
-
+ } + /> {/* Paper content */}
@@ -113,7 +110,7 @@ function PaperPage() { return (
- {content} + {content}
); diff --git a/console/src/shared/lib/api.ts b/console/src/shared/lib/api.ts index 9e500c6..e3f9589 100644 --- a/console/src/shared/lib/api.ts +++ b/console/src/shared/lib/api.ts @@ -29,7 +29,7 @@ async function getAuthHeaders(): Promise> { } } - const byokKey = localStorage.getItem("ehrlich_api_key"); + const byokKey = sessionStorage.getItem("ehrlich_api_key"); if (byokKey) { headers["X-Anthropic-Key"] = byokKey; } diff --git a/console/vite.config.ts b/console/vite.config.ts index a8b297c..1a07c9f 100644 --- a/console/vite.config.ts +++ b/console/vite.config.ts @@ -11,6 +11,9 @@ export default defineConfig({ "@": resolve(__dirname, "./src"), }, }, + build: { + sourcemap: false, + }, server: { proxy: { "/api": { diff --git a/docs/architecture.md b/docs/architecture.md index c2dfad2..ba31c40 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,7 +71,7 @@ Opus 4.6 (Director) -- Formulates hypotheses, evaluates evidence, synthesize Investigations are persisted to PostgreSQL via `asyncpg` connection pooling. The `InvestigationRepository` implements the `InvestigationRepository` ABC defined in the domain layer. Hypotheses, experiments, findings, candidates, negative controls, citations, domain, and cost data are stored in `JSONB` columns in the `investigations` table. All SSE events are persisted to a separate `events` table for full timeline replay on page reload. -Additional tables: `users` (WorkOS identity, credit balance, encrypted API key), `credit_transactions` (audit trail for credit purchases/spending/refunds). Full-text search uses PostgreSQL `tsvector` + GIN index. +Additional tables: `users` (WorkOS identity, credit balance), `credit_transactions` (audit trail for credit purchases/spending/refunds). Full-text search uses PostgreSQL `tsvector` + GIN index. The API keeps `_active_investigations` and `_active_orchestrators` dicts for in-flight SSE streaming and user-guided steering (hypothesis approval). Persists to PostgreSQL on completion (or error). @@ -101,7 +101,22 @@ Researcher is always Sonnet 4.5 and Summarizer is always Haiku 4.5, regardless o ### BYOK (Bring Your Own Key) -Users can provide their own Anthropic API key via the `X-Anthropic-Key` HTTP header. BYOK users bypass the credit system entirely -- they pay Anthropic directly. The API key is forwarded to `AnthropicClientAdapter` for that investigation's API calls. +Users can provide their own Anthropic API key via the `X-Anthropic-Key` HTTP header. BYOK users bypass the credit system entirely -- they pay Anthropic directly. The API key is forwarded to `AnthropicClientAdapter` for that investigation's API calls. BYOK keys are stored in `sessionStorage` (tab-scoped, cleared on close) and never persisted. API key patterns (`sk-ant-*`) are redacted from error logs via `_sanitize_error()`. + +## Security + +Defense-in-depth across server and console: + +- **Security headers**: `SecurityHeadersMiddleware` sets `X-Content-Type-Options: nosniff`, `X-Frame-Options: DENY`, `Referrer-Policy: strict-origin-when-cross-origin`, `Permissions-Policy: geolocation=(), microphone=(), camera=()` on all responses. +- **CORS hardening**: explicit `allow_methods` (`GET`, `POST`, `PUT`, `DELETE`, `OPTIONS`) and `allow_headers` (`Authorization`, `Content-Type`, `X-Anthropic-Key`). Wildcard origins logged as warning in non-development environments (`EHRLICH_ENVIRONMENT`). +- **Upload isolation**: pending uploads are owner-scoped (`workos_id` validated on claim) with 30-minute TTL auto-expiry. File content validated via magic byte signatures (PDF: `%PDF-`, XLSX: `PK\x03\x04`, CSV: no null bytes + valid UTF-8). +- **Prompt injection defense**: all user-controlled content in `` XML blocks is escaped via `html.escape()` — filenames, column names, dtypes, statistics, sample values, and document excerpts. +- **Input validation**: request schema enforces `prompt` length (10-10000 chars), `director_tier` regex pattern (`haiku|sonnet|opus`), SMILES `max_length=500`, PDB ID format (`^[0-9A-Za-z]{4}$`), URL path encoding for external API clients (`urllib.parse.quote`). +- **Markdown XSS**: all `` components use `rehype-sanitize` to strip dangerous HTML from AI-generated content. +- **Error sanitization**: JWT validation errors return generic "Invalid or expired token" (no token details leaked). Tool execution errors don't expose internal stack traces. Anthropic API errors have `sk-ant-*` patterns redacted before logging. +- **SQL injection prevention**: database name validated against `^[a-zA-Z0-9_]+$` in `_ensure_database()`. All other queries use parameterized SQL via `asyncpg`. +- **State guards**: `POST /investigate/{id}/approve` returns 409 Conflict if the orchestrator is not actively awaiting approval. SSE reconnection refreshes auth token on each attempt. +- **Production build**: source maps disabled in console production builds. ## Multi-Domain Investigations diff --git a/server/src/ehrlich/api/app.py b/server/src/ehrlich/api/app.py index 4738386..6e59f02 100644 --- a/server/src/ehrlich/api/app.py +++ b/server/src/ehrlich/api/app.py @@ -2,9 +2,13 @@ import logging from collections.abc import AsyncGenerator from contextlib import asynccontextmanager +from typing import Any from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response from ehrlich.api.routes.health import router as health_router from ehrlich.api.routes.investigation import close_repository, init_repository @@ -23,6 +27,18 @@ } +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + """Add security headers to all responses.""" + + async def dispatch(self, request: Request, call_next: Any) -> Response: + response: Response = await call_next(request) + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + response.headers["Permissions-Policy"] = "geolocation=(), microphone=(), camera=()" + return response + + def _check_optional(module: str) -> bool: try: importlib.import_module(module) @@ -55,6 +71,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: status = "available" if _check_optional(mod) else "not installed" logger.info(" %s: %s", desc, status) + if "*" in settings.cors_origins and settings.environment != "development": + logger.warning( + "CORS allow_origins contains '*' in %s environment. " + "This is insecure and should be replaced with specific origins.", + settings.environment, + ) + await init_repository(settings.database_url) logger.info("PostgreSQL repository initialized") @@ -74,12 +97,13 @@ def create_app() -> FastAPI: lifespan=lifespan, ) + app.add_middleware(SecurityHeadersMiddleware) app.add_middleware( CORSMiddleware, allow_origins=settings.cors_origins, allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_headers=["Authorization", "Content-Type", "X-Anthropic-Key"], ) app.include_router(health_router, prefix="/api/v1") diff --git a/server/src/ehrlich/api/auth.py b/server/src/ehrlich/api/auth.py index 57a70eb..74db735 100644 --- a/server/src/ehrlich/api/auth.py +++ b/server/src/ehrlich/api/auth.py @@ -43,7 +43,8 @@ def _verify_token(token: str) -> dict[str, Any]: ) return {"workos_id": payload["sub"], "email": payload.get("email", "")} except jwt.exceptions.PyJWTError as e: - raise HTTPException(status_code=401, detail=f"Invalid token: {e}") from e + logger.warning("JWT validation failed: %s", type(e).__name__) + raise HTTPException(status_code=401, detail="Invalid or expired token") from e async def get_current_user(request: Request) -> dict[str, Any]: diff --git a/server/src/ehrlich/api/routes/investigation.py b/server/src/ehrlich/api/routes/investigation.py index 3b6bcb9..a72f048 100644 --- a/server/src/ehrlich/api/routes/investigation.py +++ b/server/src/ehrlich/api/routes/investigation.py @@ -293,7 +293,7 @@ async def start_investigation( from ehrlich.api.routes.upload import get_pending_upload for fid in request.file_ids: - uploaded = get_pending_upload(fid) + uploaded = get_pending_upload(fid, user["workos_id"]) if uploaded: investigation.uploaded_files.append(uploaded) await repo.save_uploaded_file(investigation.id, uploaded) @@ -324,6 +324,13 @@ async def approve_hypotheses( raise HTTPException(status_code=404, detail="No active orchestrator") repo = _get_repository() await _verify_ownership(investigation_id, user, repo) + + # Verify orchestrator is awaiting approval + if not orchestrator.is_awaiting_approval(): + raise HTTPException( + status_code=409, detail="Investigation is not awaiting hypothesis approval" + ) + orchestrator.approve_hypotheses(request.approved_ids, request.rejected_ids) return {"status": "approved"} diff --git a/server/src/ehrlich/api/routes/molecule.py b/server/src/ehrlich/api/routes/molecule.py index cc2f1e2..3fa9488 100644 --- a/server/src/ehrlich/api/routes/molecule.py +++ b/server/src/ehrlich/api/routes/molecule.py @@ -1,9 +1,10 @@ from dataclasses import asdict from typing import Any -from fastapi import APIRouter, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import Response +from ehrlich.api.auth import get_optional_user from ehrlich.chemistry.application.chemistry_service import ChemistryService from ehrlich.kernel.exceptions import InvalidSMILESError from ehrlich.kernel.types import SMILES @@ -13,6 +14,7 @@ _chemistry = ChemistryService() _protein_store = ProteinStore() +_optional_user = Depends(get_optional_user) _ERROR_SVG = """ @@ -23,9 +25,10 @@ @router.get("/molecule/depict") async def depict( - smiles: str = Query(..., description="SMILES string"), + smiles: str = Query(..., description="SMILES string", max_length=500), w: int = Query(300, ge=50, le=1000), h: int = Query(200, ge=50, le=1000), + _user: dict[str, Any] | None = _optional_user, ) -> Response: try: svg = _chemistry.depict_2d(SMILES(smiles), width=w, height=h) @@ -39,7 +42,10 @@ async def depict( @router.get("/molecule/conformer") -async def conformer(smiles: str = Query(..., description="SMILES string")) -> dict[str, Any]: +async def conformer( + smiles: str = Query(..., description="SMILES string", max_length=500), + _user: dict[str, Any] | None = _optional_user, +) -> dict[str, Any]: try: conf = _chemistry.generate_conformer(SMILES(smiles)) return {"mol_block": conf.mol_block, "energy": conf.energy, "num_atoms": conf.num_atoms} @@ -48,7 +54,10 @@ async def conformer(smiles: str = Query(..., description="SMILES string")) -> di @router.get("/molecule/descriptors") -async def descriptors(smiles: str = Query(..., description="SMILES string")) -> dict[str, Any]: +async def descriptors( + smiles: str = Query(..., description="SMILES string", max_length=500), + _user: dict[str, Any] | None = _optional_user, +) -> dict[str, Any]: try: desc = _chemistry.compute_descriptors(SMILES(smiles)) result = asdict(desc) @@ -59,6 +68,8 @@ async def descriptors(smiles: str = Query(..., description="SMILES string")) -> @router.get("/targets") -async def list_targets() -> list[dict[str, str]]: +async def list_targets( + _user: dict[str, Any] | None = _optional_user, +) -> list[dict[str, str]]: targets = await _protein_store.list_targets() return [{"pdb_id": t.pdb_id, "name": t.name, "organism": t.organism} for t in targets] diff --git a/server/src/ehrlich/api/routes/upload.py b/server/src/ehrlich/api/routes/upload.py index f0c77b3..2ad6c21 100644 --- a/server/src/ehrlich/api/routes/upload.py +++ b/server/src/ehrlich/api/routes/upload.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import time from typing import TYPE_CHECKING, Any from fastapi import APIRouter, Depends, HTTPException, UploadFile @@ -20,8 +21,14 @@ _processor = FileProcessor() # Pending uploads waiting to be linked to an investigation. -# Keyed by file_id, removed when claimed by POST /investigate. -_pending_uploads: dict[str, UploadedFileEntity] = {} +# Keyed by file_id, value is (workos_id, UploadedFileEntity). +# Removed when claimed by POST /investigate. +_pending_uploads: dict[str, tuple[str, UploadedFileEntity]] = {} +# TTL tracking: file_id -> upload timestamp +_pending_upload_times: dict[str, float] = {} + +# TTL for pending uploads (30 minutes) +_UPLOAD_TTL_SECONDS = 30 * 60 class UploadResponse(BaseModel): @@ -47,7 +54,8 @@ async def upload_file( except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) from e - _pending_uploads[uploaded.file_id] = uploaded + _pending_uploads[uploaded.file_id] = (user["workos_id"], uploaded) + _pending_upload_times[uploaded.file_id] = time.time() preview: dict[str, Any] = {} if uploaded.tabular: @@ -73,6 +81,29 @@ async def upload_file( ) -def get_pending_upload(file_id: str) -> UploadedFileEntity | None: - """Retrieve and remove a pending upload by ID. Called from investigation routes.""" - return _pending_uploads.pop(file_id, None) +def get_pending_upload(file_id: str, workos_id: str) -> UploadedFileEntity | None: + """Retrieve and remove a pending upload by ID. Called from investigation routes. + + Validates ownership and TTL. Returns None if not found, expired, or ownership mismatch. + """ + entry = _pending_uploads.get(file_id) + if entry is None: + return None + + owner_id, uploaded = entry + upload_time = _pending_upload_times.get(file_id, 0) + + # Check TTL + if time.time() - upload_time > _UPLOAD_TTL_SECONDS: + _pending_uploads.pop(file_id, None) + _pending_upload_times.pop(file_id, None) + return None + + # Validate ownership + if owner_id != workos_id: + return None + + # Valid -- pop and return + _pending_uploads.pop(file_id, None) + _pending_upload_times.pop(file_id, None) + return uploaded diff --git a/server/src/ehrlich/api/schemas/investigation.py b/server/src/ehrlich/api/schemas/investigation.py index 5247f91..8602cd9 100644 --- a/server/src/ehrlich/api/schemas/investigation.py +++ b/server/src/ehrlich/api/schemas/investigation.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, Field, field_validator from ehrlich.investigation.domain.upload_limits import MAX_FILES_PER_INVESTIGATION @@ -16,8 +16,8 @@ class InvestigateRequest(BaseModel): - prompt: str - director_tier: str = "opus" + prompt: str = Field(..., min_length=10, max_length=10000) + director_tier: str = Field(default="opus", pattern="^(haiku|sonnet|opus)$") file_ids: list[str] = [] @field_validator("file_ids") diff --git a/server/src/ehrlich/config.py b/server/src/ehrlich/config.py index c7a6123..d361d25 100644 --- a/server/src/ehrlich/config.py +++ b/server/src/ehrlich/config.py @@ -28,6 +28,7 @@ class Settings(BaseSettings): comptox_api_key: str = "" workos_client_id: str = "" workos_api_key: str = "" + environment: str = "development" @property def has_api_key(self) -> bool: diff --git a/server/src/ehrlich/investigation/application/file_processor.py b/server/src/ehrlich/investigation/application/file_processor.py index f213c02..6e0a769 100644 --- a/server/src/ehrlich/investigation/application/file_processor.py +++ b/server/src/ehrlich/investigation/application/file_processor.py @@ -31,12 +31,36 @@ def process(self, filename: str, content: bytes) -> UploadedFile: msg = f"Unsupported file type: .{ext}. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}" raise ValueError(msg) + self._validate_content_type(content, ext) + if ext == "csv": return self._process_csv(filename, content) if ext == "xlsx": return self._process_xlsx(filename, content) return self._process_pdf(filename, content) + def _validate_content_type(self, content: bytes, extension: str) -> None: + """Validate file content matches declared extension via magic bytes.""" + if extension == "pdf": + if len(content) < 5 or content[:5] != b"%PDF-": + msg = "File content does not match PDF format" + raise ValueError(msg) + elif extension == "xlsx": + if len(content) < 4 or content[:4] != b"PK\x03\x04": + msg = "File content does not match XLSX format" + raise ValueError(msg) + elif extension == "csv": + # CSV should be valid UTF-8 text with no null bytes in first 1024 bytes + check_size = min(len(content), 1024) + if b"\x00" in content[:check_size]: + msg = "File content does not match CSV format (contains null bytes)" + raise ValueError(msg) + try: + content[:check_size].decode("utf-8") + except UnicodeDecodeError as e: + msg = "File content does not match CSV format (invalid UTF-8)" + raise ValueError(msg) from e + def _process_csv(self, filename: str, content: bytes) -> UploadedFile: df = pd.read_csv(io.BytesIO(content)) return self._build_tabular(filename, "text/csv", df) diff --git a/server/src/ehrlich/investigation/application/multi_orchestrator.py b/server/src/ehrlich/investigation/application/multi_orchestrator.py index 6c74872..15c98d5 100644 --- a/server/src/ehrlich/investigation/application/multi_orchestrator.py +++ b/server/src/ehrlich/investigation/application/multi_orchestrator.py @@ -26,6 +26,7 @@ from ehrlich.investigation.domain.events import ( CostUpdate, DomainEvent, + HypothesisApprovalRequested, InvestigationError, PhaseChanged, Thinking, @@ -89,16 +90,22 @@ def __init__( self._dispatcher = ToolDispatcher(registry, self._cache, repository, {}) self._state_lock = asyncio.Lock() self._approval_event = asyncio.Event() + self._awaiting_approval = False self._investigation: Investigation | None = None self._uploaded_files: dict[str, UploadedFile] = {} self._uploaded_data_context = "" + def is_awaiting_approval(self) -> bool: + """Return True if the orchestrator is blocked waiting for hypothesis approval.""" + return self._awaiting_approval + def approve_hypotheses( self, approved_ids: list[str], rejected_ids: list[str], ) -> None: """Called by API to approve/reject hypotheses and unblock the loop.""" + self._awaiting_approval = False if self._investigation: from ehrlich.investigation.domain.hypothesis import HypothesisStatus @@ -250,8 +257,12 @@ async def run(self, investigation: Investigation) -> AsyncGenerator[DomainEvent, neg_control_suggestions = result["neg_control_suggestions"] pos_control_suggestions = result["pos_control_suggestions"] else: + if isinstance(event, HypothesisApprovalRequested): + self._awaiting_approval = True yield event + self._awaiting_approval = False + # 4. Hypothesis testing loop async for event in run_hypothesis_testing_phase( investigation, diff --git a/server/src/ehrlich/investigation/application/prompts/builders.py b/server/src/ehrlich/investigation/application/prompts/builders.py index 7d72033..8f9183c 100644 --- a/server/src/ehrlich/investigation/application/prompts/builders.py +++ b/server/src/ehrlich/investigation/application/prompts/builders.py @@ -6,6 +6,7 @@ from __future__ import annotations +import html from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -246,6 +247,8 @@ def build_uploaded_data_context(files: list[UploadedFile]) -> str: Injected into Director and Researcher prompts so the model is aware of available datasets and can reference them via ``query_uploaded_data``. + + All user-controlled content is escaped to prevent prompt injection. """ if not files: return "" @@ -253,31 +256,48 @@ def build_uploaded_data_context(files: list[UploadedFile]) -> str: for f in files: if f.tabular: t = f.tabular - cols = ", ".join(t.columns) - dtypes = ", ".join(t.dtypes) - parts.append(f' ') - parts.append(f" Columns ({len(t.columns)}): {cols}") - parts.append(f" Dtypes: {dtypes}") + # Escape all user-controlled values + escaped_file_id = html.escape(f.file_id, quote=True) + escaped_filename = html.escape(f.filename, quote=True) + escaped_cols = ", ".join(html.escape(c) for c in t.columns) + escaped_dtypes = ", ".join(html.escape(d) for d in t.dtypes) + + parts.append( + f' ' + ) + parts.append(f" Columns ({len(t.columns)}): {escaped_cols}") + parts.append(f" Dtypes: {escaped_dtypes}") parts.append(f" Rows: {t.row_count}") + if t.summary_stats: stats_lines = [] for col_name, stats in list(t.summary_stats.items())[:6]: - mean = stats.get("mean", "?") - std = stats.get("std", "?") - stats_lines.append(f"{col_name}: mean={mean}, std={std}") + escaped_col = html.escape(col_name) + mean = html.escape(str(stats.get("mean", "?"))) + std = html.escape(str(stats.get("std", "?"))) + stats_lines.append(f"{escaped_col}: mean={mean}, std={std}") parts.append(f" Stats: {'; '.join(stats_lines)}") + if t.sample_rows: - header = " | ".join(t.columns) - parts.append(f" Sample: {header}") + escaped_header = " | ".join(html.escape(c) for c in t.columns) + parts.append(f" Sample: {escaped_header}") for row in t.sample_rows[:3]: - parts.append(f" {' | '.join(row)}") + escaped_row = " | ".join(html.escape(str(val)) for val in row) + parts.append(f" {escaped_row}") parts.append(" ") elif f.document: d = f.document excerpt = d.text[:500] + "..." if len(d.text) > 500 else d.text - parts.append(f' ') + # Escape all user-controlled values + escaped_file_id = html.escape(f.file_id, quote=True) + escaped_filename = html.escape(f.filename, quote=True) + escaped_excerpt = html.escape(excerpt) + + parts.append( + f' ' + ) parts.append(f" Pages: {d.page_count}") - parts.append(f" Excerpt: {excerpt}") + parts.append(f" Excerpt: {escaped_excerpt}") parts.append(" ") parts.append("") return "\n".join(parts) diff --git a/server/src/ehrlich/investigation/application/tool_dispatcher.py b/server/src/ehrlich/investigation/application/tool_dispatcher.py index e158483..972be49 100644 --- a/server/src/ehrlich/investigation/application/tool_dispatcher.py +++ b/server/src/ehrlich/investigation/application/tool_dispatcher.py @@ -62,9 +62,9 @@ async def dispatch( result_str = str(result) self._cache.put(tool_name, args_hash, result_str) return result_str - except Exception as e: - logger.warning("Tool %s failed: %s", tool_name, e) - return json.dumps({"error": f"Tool {tool_name} failed: {e}"}) + except Exception: + logger.exception("Tool %s failed", tool_name) + return json.dumps({"error": f"Tool {tool_name} failed"}) def _handle_query_uploaded_data(self, tool_input: dict[str, Any]) -> str: """Handle query_uploaded_data tool calls from in-memory uploaded files.""" diff --git a/server/src/ehrlich/investigation/infrastructure/anthropic_client.py b/server/src/ehrlich/investigation/infrastructure/anthropic_client.py index 0eb6174..bf62abf 100644 --- a/server/src/ehrlich/investigation/infrastructure/anthropic_client.py +++ b/server/src/ehrlich/investigation/infrastructure/anthropic_client.py @@ -2,6 +2,7 @@ import asyncio import logging +import re from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -12,6 +13,12 @@ logger = logging.getLogger(__name__) + +def _sanitize_error(msg: str) -> str: + """Remove Anthropic API keys from error messages.""" + return re.sub(r"sk-ant-[A-Za-z0-9_-]+", "[REDACTED]", msg) + + _MAX_RETRIES = 3 _BASE_DELAY = 1.0 @@ -84,7 +91,7 @@ async def create_message( if attempt < _MAX_RETRIES - 1: await asyncio.sleep(delay) except anthropic.APIError as e: - logger.error("Anthropic API error: %s", e) + logger.error("Anthropic API error: %s", _sanitize_error(str(e))) raise raise last_error # type: ignore[misc] @@ -139,7 +146,7 @@ async def stream_message( if attempt < _MAX_RETRIES - 1: await asyncio.sleep(delay) except anthropic.APIError as e: - logger.error("Anthropic API error: %s", e) + logger.error("Anthropic API error: %s", _sanitize_error(str(e))) raise raise last_error # type: ignore[misc] diff --git a/server/src/ehrlich/investigation/infrastructure/repository.py b/server/src/ehrlich/investigation/infrastructure/repository.py index 20d8fa6..36633e8 100644 --- a/server/src/ehrlich/investigation/infrastructure/repository.py +++ b/server/src/ehrlich/investigation/infrastructure/repository.py @@ -25,10 +25,11 @@ workos_id TEXT UNIQUE NOT NULL, email TEXT NOT NULL, credits INTEGER NOT NULL DEFAULT 5, - encrypted_api_key BYTEA, created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +ALTER TABLE users DROP COLUMN IF EXISTS encrypted_api_key; + CREATE TABLE IF NOT EXISTS investigations ( id TEXT PRIMARY KEY, user_id UUID REFERENCES users(id), @@ -104,10 +105,17 @@ async def initialize(self) -> None: async def _ensure_database(self) -> None: """Create the database if it doesn't exist.""" + import re from urllib.parse import urlparse parsed = urlparse(self._database_url) db_name = parsed.path.lstrip("/") + + # Validate db_name to prevent SQL injection + if not re.match(r"^[a-zA-Z0-9_]+$", db_name): + msg = f"Invalid database name: {db_name}" + raise ValueError(msg) + maintenance_url = self._database_url.rsplit("/", 1)[0] + "/postgres" try: diff --git a/server/src/ehrlich/simulation/infrastructure/comptox_client.py b/server/src/ehrlich/simulation/infrastructure/comptox_client.py index 5bd2249..24ea378 100644 --- a/server/src/ehrlich/simulation/infrastructure/comptox_client.py +++ b/server/src/ehrlich/simulation/infrastructure/comptox_client.py @@ -1,5 +1,6 @@ import asyncio import logging +import urllib.parse import httpx @@ -38,9 +39,10 @@ async def fetch(self, identifier: str) -> ToxicityProfile | None: async def _resolve_dtxsid(self, name: str) -> str | None: last_error: Exception | None = None + encoded_name = urllib.parse.quote(name, safe="") for attempt in range(_MAX_RETRIES): try: - resp = await self._client.get(f"/chemical/search/by-name/{name}") + resp = await self._client.get(f"/chemical/search/by-name/{encoded_name}") if resp.status_code == 404: return None if resp.status_code == 429: diff --git a/server/src/ehrlich/simulation/infrastructure/rcsb_client.py b/server/src/ehrlich/simulation/infrastructure/rcsb_client.py index 7c4c50d..7e0c9bb 100644 --- a/server/src/ehrlich/simulation/infrastructure/rcsb_client.py +++ b/server/src/ehrlich/simulation/infrastructure/rcsb_client.py @@ -1,5 +1,6 @@ import asyncio import logging +import re import httpx @@ -112,6 +113,10 @@ async def _execute_search(self, search_query: dict[str, object]) -> list[str]: ) async def _fetch_entry(self, pdb_id: str) -> ProteinTarget | None: + # Validate PDB ID format to prevent URL path injection + if not re.match(r"^[0-9A-Za-z]{4}$", pdb_id): + msg = f"Invalid PDB ID format: {pdb_id}" + raise ValueError(msg) try: resp = await self._client.get(f"{_DATA_URL}/{pdb_id}") if resp.status_code == 404: diff --git a/server/tests/api/test_credits.py b/server/tests/api/test_credits.py index 3726a6b..d628a9e 100644 --- a/server/tests/api/test_credits.py +++ b/server/tests/api/test_credits.py @@ -78,13 +78,13 @@ async def test_deducts_credits_on_start(self, client: httpx.AsyncClient) -> None async def test_insufficient_credits_returns_402(self, client: httpx.AsyncClient) -> None: resp1 = await client.post( "/api/v1/investigate", - json={"prompt": "First opus", "director_tier": "opus"}, + json={"prompt": "First opus test", "director_tier": "opus"}, ) assert resp1.status_code == 200 resp2 = await client.post( "/api/v1/investigate", - json={"prompt": "Second opus", "director_tier": "opus"}, + json={"prompt": "Second opus test", "director_tier": "opus"}, ) assert resp2.status_code == 402 assert "Insufficient credits" in resp2.json()["detail"] @@ -92,7 +92,7 @@ async def test_insufficient_credits_returns_402(self, client: httpx.AsyncClient) async def test_byok_skips_credit_check(self, client: httpx.AsyncClient) -> None: response = await client.post( "/api/v1/investigate", - json={"prompt": "BYOK test", "director_tier": "opus"}, + json={"prompt": "BYOK test prompt", "director_tier": "opus"}, headers={"X-Anthropic-Key": "sk-ant-byok"}, ) assert response.status_code == 200 @@ -100,20 +100,19 @@ async def test_byok_skips_credit_check(self, client: httpx.AsyncClient) -> None: balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 5 - async def test_invalid_tier_returns_400(self, client: httpx.AsyncClient) -> None: + async def test_invalid_tier_returns_422(self, client: httpx.AsyncClient) -> None: response = await client.post( "/api/v1/investigate", - json={"prompt": "Bad tier", "director_tier": "gpt4"}, + json={"prompt": "Bad tier test prompt", "director_tier": "gpt4"}, ) - assert response.status_code == 400 - assert "Invalid director tier" in response.json()["detail"] + assert response.status_code == 422 class TestTierCredits: async def test_haiku_costs_1(self, client: httpx.AsyncClient) -> None: await client.post( "/api/v1/investigate", - json={"prompt": "Haiku test", "director_tier": "haiku"}, + json={"prompt": "Haiku test prompt", "director_tier": "haiku"}, ) balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 4 @@ -121,7 +120,7 @@ async def test_haiku_costs_1(self, client: httpx.AsyncClient) -> None: async def test_sonnet_costs_3(self, client: httpx.AsyncClient) -> None: await client.post( "/api/v1/investigate", - json={"prompt": "Sonnet test", "director_tier": "sonnet"}, + json={"prompt": "Sonnet test prompt", "director_tier": "sonnet"}, ) balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 2 @@ -129,7 +128,7 @@ async def test_sonnet_costs_3(self, client: httpx.AsyncClient) -> None: async def test_opus_costs_5(self, client: httpx.AsyncClient) -> None: await client.post( "/api/v1/investigate", - json={"prompt": "Opus test", "director_tier": "opus"}, + json={"prompt": "Opus test prompt", "director_tier": "opus"}, ) balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 0 @@ -137,7 +136,7 @@ async def test_opus_costs_5(self, client: httpx.AsyncClient) -> None: async def test_default_tier_is_opus(self, client: httpx.AsyncClient) -> None: await client.post( "/api/v1/investigate", - json={"prompt": "Default tier"}, + json={"prompt": "Default tier test"}, ) balance = await client.get("/api/v1/credits/balance") assert balance.json()["credits"] == 0 diff --git a/server/tests/api/test_investigation.py b/server/tests/api/test_investigation.py index ba2c6ec..d55d8e9 100644 --- a/server/tests/api/test_investigation.py +++ b/server/tests/api/test_investigation.py @@ -74,11 +74,11 @@ async def test_empty_list(self, client: httpx.AsyncClient) -> None: async def test_returns_created_investigations(self, client: httpx.AsyncClient) -> None: await client.post( "/api/v1/investigate", - json={"prompt": "Test 1", "director_tier": "haiku"}, + json={"prompt": "Test prompt 1", "director_tier": "haiku"}, ) await client.post( "/api/v1/investigate", - json={"prompt": "Test 2", "director_tier": "haiku"}, + json={"prompt": "Test prompt 2", "director_tier": "haiku"}, ) response = await client.get("/api/v1/investigate") @@ -86,15 +86,15 @@ async def test_returns_created_investigations(self, client: httpx.AsyncClient) - data = response.json() assert len(data) == 2 # Most recent first - assert data[0]["prompt"] == "Test 2" - assert data[1]["prompt"] == "Test 1" + assert data[0]["prompt"] == "Test prompt 2" + assert data[1]["prompt"] == "Test prompt 1" class TestGetInvestigation: async def test_returns_investigation(self, client: httpx.AsyncClient) -> None: resp = await client.post( "/api/v1/investigate", - json={"prompt": "Test", "director_tier": "haiku"}, + json={"prompt": "Test investigation prompt", "director_tier": "haiku"}, ) inv_id = resp.json()["id"] @@ -102,7 +102,7 @@ async def test_returns_investigation(self, client: httpx.AsyncClient) -> None: assert response.status_code == 200 data = response.json() assert data["id"] == inv_id - assert data["prompt"] == "Test" + assert data["prompt"] == "Test investigation prompt" assert data["status"] == "pending" async def test_not_found(self, client: httpx.AsyncClient) -> None: @@ -118,7 +118,7 @@ async def test_not_found(self, client: httpx.AsyncClient) -> None: async def test_completed_replays_final(self, client: httpx.AsyncClient) -> None: resp = await client.post( "/api/v1/investigate", - json={"prompt": "Test", "director_tier": "haiku"}, + json={"prompt": "Test replay prompt", "director_tier": "haiku"}, ) inv_id = resp.json()["id"] @@ -140,7 +140,7 @@ async def test_get_investigation_forbidden_for_other_user( # Create investigation as TEST_USER resp = await client.post( "/api/v1/investigate", - json={"prompt": "Test", "director_tier": "haiku"}, + json={"prompt": "Test ownership prompt", "director_tier": "haiku"}, ) inv_id = resp.json()["id"] @@ -155,7 +155,7 @@ async def test_get_investigation_forbidden_for_other_user( async def test_stream_forbidden_for_other_user(self, client: httpx.AsyncClient) -> None: resp = await client.post( "/api/v1/investigate", - json={"prompt": "Test", "director_tier": "haiku"}, + json={"prompt": "Test stream forbidden", "director_tier": "haiku"}, ) inv_id = resp.json()["id"] From c321669354eb084b9f2cb563a5eb071d760e81be Mon Sep 17 00:00:00 2001 From: Canitas Date: Sun, 15 Feb 2026 02:13:36 -0700 Subject: [PATCH 28/35] feat(ui): Console Hardening, Methodology Showcase 2.0, Home Redesign --- console/index.html | 32 +- console/public/logo.svg | 3204 +++++++++++++++++ console/src/assets/images/logo.svg | 3204 +++++++++++++++++ .../investigation/components/FileUpload.tsx | 368 +- .../components/InvestigationList.tsx | 136 - .../investigation/components/PromptInput.tsx | 208 +- .../components/TemplateCards.test.tsx | 45 +- .../components/TemplateCards.tsx | 176 +- .../investigation/components/index.ts | 2 +- .../components/CapabilitiesGallery.tsx | 195 + console/src/features/molecule/MolViewer3D.tsx | 56 +- console/src/routeTree.gen.ts | 21 + console/src/routes/compare.$id1.$id2.tsx | 23 +- console/src/routes/index.tsx | 176 +- console/src/routes/investigation.$id.tsx | 29 +- console/src/routes/methodology.tsx | 3 +- console/src/routes/settings.tsx | 172 + .../src/shared/components/InteractiveLogo.tsx | 13 + .../shared/components/layout/AppLayout.tsx | 159 +- .../shared/components/layout/AppSidebar.tsx | 302 ++ .../components/layout/MethodologyDrawer.tsx | 231 +- .../shared/components/layout/PageHeader.tsx | 79 + console/src/shared/components/ui/button.tsx | 64 + console/src/shared/components/ui/card.tsx | 79 + console/src/shared/components/ui/dialog.tsx | 158 + .../shared/components/ui/dropdown-menu.tsx | 196 + console/src/shared/components/ui/input.tsx | 21 + console/src/shared/components/ui/label.tsx | 24 + console/src/shared/components/ui/popover.tsx | 87 + console/src/shared/components/ui/sheet.tsx | 138 + console/src/shared/components/ui/tabs.tsx | 89 + console/src/shared/components/ui/textarea.tsx | 18 + console/src/shared/components/ui/tooltip.tsx | 55 + console/src/shared/lib/constants.ts | 65 + console/src/styles/globals.css | 22 +- docs/roadmap/phase-14-engine-hardening.md | 14 +- 36 files changed, 9077 insertions(+), 787 deletions(-) create mode 100644 console/public/logo.svg create mode 100644 console/src/assets/images/logo.svg delete mode 100644 console/src/features/investigation/components/InvestigationList.tsx create mode 100644 console/src/features/methodology/components/CapabilitiesGallery.tsx create mode 100644 console/src/routes/settings.tsx create mode 100644 console/src/shared/components/InteractiveLogo.tsx create mode 100644 console/src/shared/components/layout/AppSidebar.tsx create mode 100644 console/src/shared/components/layout/PageHeader.tsx create mode 100644 console/src/shared/components/ui/button.tsx create mode 100644 console/src/shared/components/ui/card.tsx create mode 100644 console/src/shared/components/ui/dialog.tsx create mode 100644 console/src/shared/components/ui/dropdown-menu.tsx create mode 100644 console/src/shared/components/ui/input.tsx create mode 100644 console/src/shared/components/ui/label.tsx create mode 100644 console/src/shared/components/ui/popover.tsx create mode 100644 console/src/shared/components/ui/sheet.tsx create mode 100644 console/src/shared/components/ui/tabs.tsx create mode 100644 console/src/shared/components/ui/textarea.tsx create mode 100644 console/src/shared/components/ui/tooltip.tsx create mode 100644 console/src/shared/lib/constants.ts diff --git a/console/index.html b/console/index.html index 5aa971f..86ae86a 100644 --- a/console/index.html +++ b/console/index.html @@ -1,15 +1,21 @@ - - - - Ehrlich - Antimicrobial Discovery - - - - - -
- - - + + + + + Ehrlich - Antimicrobial Discovery + + + + + + + +
+ + + + \ No newline at end of file diff --git a/console/public/logo.svg b/console/public/logo.svg new file mode 100644 index 0000000..878c889 --- /dev/null +++ b/console/public/logo.svg @@ -0,0 +1,3204 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/console/src/assets/images/logo.svg b/console/src/assets/images/logo.svg new file mode 100644 index 0000000..878c889 --- /dev/null +++ b/console/src/assets/images/logo.svg @@ -0,0 +1,3204 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/console/src/features/investigation/components/FileUpload.tsx b/console/src/features/investigation/components/FileUpload.tsx index e11d43a..ad70e73 100644 --- a/console/src/features/investigation/components/FileUpload.tsx +++ b/console/src/features/investigation/components/FileUpload.tsx @@ -1,4 +1,10 @@ -import { useCallback, useRef, useState } from "react"; +import { + useCallback, + useRef, + useState, + forwardRef, + useImperativeHandle, +} from "react"; import { Upload, Loader2, AlertCircle } from "lucide-react"; import { useUpload } from "../hooks/use-upload"; import { DataPreview } from "./DataPreview"; @@ -8,12 +14,19 @@ const ACCEPTED_TYPES = ".csv,.xlsx,.pdf"; const MAX_FILES = 10; const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB -interface UploadedFile extends UploadResponse { +export interface UploadedFile extends UploadResponse { // UploadResponse already has file_id, filename, content_type, preview } +export interface FileUploadRef { + open: () => void; + addFiles: (files: File[]) => void; +} + interface FileUploadProps { - onFilesChanged: (files: { file_id: string; filename: string }[]) => void; + onFilesChanged: (files: UploadedFile[]) => void; + trigger?: React.ReactNode; + hideList?: boolean; } function formatSize(bytes: number): string { @@ -22,185 +35,202 @@ function formatSize(bytes: number): string { return `${bytes} B`; } -export function FileUpload({ onFilesChanged }: FileUploadProps) { - const [files, setFiles] = useState([]); - const [dragOver, setDragOver] = useState(false); - const [pendingCount, setPendingCount] = useState(0); - const [errors, setErrors] = useState<{ name: string; message: string }[]>([]); - const inputRef = useRef(null); - const upload = useUpload(); - - const updateParent = useCallback( - (updated: UploadedFile[]) => { - onFilesChanged( - updated.map((f) => ({ file_id: f.file_id, filename: f.filename })), - ); - }, - [onFilesChanged], - ); - - function addError(name: string, message: string) { - setErrors((prev) => [...prev, { name, message }]); - } - - function dismissError(index: number) { - setErrors((prev) => prev.filter((_, i) => i !== index)); - } - - function handleRemove(fileId: string) { - setFiles((prev) => { - const updated = prev.filter((f) => f.file_id !== fileId); - updateParent(updated); - return updated; - }); - } - - async function processFile(file: File) { - if (file.size > MAX_FILE_SIZE) { - addError(file.name, `File too large (${formatSize(file.size)}). Maximum is ${formatSize(MAX_FILE_SIZE)}.`); - return; +export const FileUpload = forwardRef( + ({ onFilesChanged, trigger, hideList = false }, ref) => { + const [files, setFiles] = useState([]); + const [dragOver, setDragOver] = useState(false); + const [pendingCount, setPendingCount] = useState(0); + const [errors, setErrors] = useState<{ name: string; message: string }[]>([]); + const inputRef = useRef(null); + const upload = useUpload(); + + const updateParent = useCallback( + (updated: UploadedFile[]) => { + onFilesChanged(updated); + }, + [onFilesChanged], + ); + + useImperativeHandle(ref, () => ({ + open: () => inputRef.current?.click(), + addFiles: (offsetFiles: File[]) => handleFiles(offsetFiles), + })); + + function addError(name: string, message: string) { + setErrors((prev) => [...prev, { name, message }]); + } + + function dismissError(index: number) { + setErrors((prev) => prev.filter((_, i) => i !== index)); } - setPendingCount((c) => c + 1); - try { - const result = await upload.mutateAsync(file); + function handleRemove(fileId: string) { setFiles((prev) => { - const updated = [...prev, result]; + const updated = prev.filter((f) => f.file_id !== fileId); updateParent(updated); return updated; }); - } catch (err: unknown) { - const detail = - err && typeof err === "object" && "message" in err - ? String((err as { message: string }).message) - : "Upload failed"; - addError(file.name, detail); - } finally { - setPendingCount((c) => c - 1); } - } - - function handleFiles(fileList: FileList | File[]) { - const incoming = Array.from(fileList); - const remaining = MAX_FILES - files.length; - if (incoming.length > remaining) { - addError( - `${incoming.length} files`, - `Only ${remaining} more file${remaining === 1 ? "" : "s"} allowed (max ${MAX_FILES}).`, - ); + + async function processFile(file: File) { + if (file.size > MAX_FILE_SIZE) { + addError( + file.name, + `File too large (${formatSize(file.size)}). Maximum is ${formatSize(MAX_FILE_SIZE)}.`, + ); + return; + } + + setPendingCount((c) => c + 1); + try { + const result = await upload.mutateAsync(file); + setFiles((prev) => { + const updated = [...prev, result]; + updateParent(updated); + return updated; + }); + } catch (err: unknown) { + const detail = + err && typeof err === "object" && "message" in err + ? String((err as { message: string }).message) + : "Upload failed"; + addError(file.name, detail); + } finally { + setPendingCount((c) => c - 1); + } } - const batch = incoming.slice(0, remaining); - for (const file of batch) { - void processFile(file); + + function handleFiles(fileList: FileList | File[]) { + const incoming = Array.from(fileList); + const remaining = MAX_FILES - files.length; + if (incoming.length > remaining) { + addError( + `${incoming.length} files`, + `Only ${remaining} more file${remaining === 1 ? "" : "s"} allowed (max ${MAX_FILES}).`, + ); + } + const batch = incoming.slice(0, remaining); + for (const file of batch) { + void processFile(file); + } } - } - function handleDrop(e: React.DragEvent) { - e.preventDefault(); - setDragOver(false); - if (e.dataTransfer.files.length > 0) { - handleFiles(e.dataTransfer.files); + function handleDrop(e: React.DragEvent) { + e.preventDefault(); + setDragOver(false); + if (e.dataTransfer.files.length > 0) { + handleFiles(e.dataTransfer.files); + } } - } - - function handleDragOver(e: React.DragEvent) { - e.preventDefault(); - setDragOver(true); - } - - function handleDragLeave(e: React.DragEvent) { - e.preventDefault(); - setDragOver(false); - } - - function handleInputChange(e: React.ChangeEvent) { - if (e.target.files && e.target.files.length > 0) { - handleFiles(e.target.files); - e.target.value = ""; + + function handleDragOver(e: React.DragEvent) { + e.preventDefault(); + setDragOver(true); } - } - - const atCapacity = files.length >= MAX_FILES; - - return ( -
- {!atCapacity && ( - - )} - - - - {files.length > 0 && ( -

- {files.length} / {MAX_FILES} files -

- )} - - {errors.length > 0 && ( -
- {errors.map((err, i) => ( -
- -

- {err.name}: {err.message} -

+ + function handleDragLeave(e: React.DragEvent) { + e.preventDefault(); + setDragOver(false); + } + + function handleInputChange(e: React.ChangeEvent) { + if (e.target.files && e.target.files.length > 0) { + handleFiles(e.target.files); + e.target.value = ""; + } + } + + const atCapacity = files.length >= MAX_FILES; + + return ( +
+ {!atCapacity && ( +
inputRef.current?.click()} + onDrop={handleDrop} + onDragOver={handleDragOver} + onDragLeave={handleDragLeave} + className={trigger ? "inline-block cursor-pointer" : ""} + > + {trigger ? ( + trigger + ) : ( -
- ))} -
- )} - - {files.length > 0 && ( -
- {files.map((f) => ( - handleRemove(f.file_id)} - /> - ))} -
- )} -
- ); -} - + )} +
+ )} + + + + {!hideList && files.length > 0 && ( +

+ {files.length} / {MAX_FILES} files +

+ )} + + {/* Errors always shown locally for now */} + {errors.length > 0 && ( +
+ {errors.map((err, i) => ( +
+ +

+ {err.name}: {err.message} +

+ +
+ ))} +
+ )} + + {!hideList && files.length > 0 && ( +
+ {files.map((f) => ( + handleRemove(f.file_id)} + /> + ))} +
+ )} +
+ ); + }, +); +FileUpload.displayName = "FileUpload"; diff --git a/console/src/features/investigation/components/InvestigationList.tsx b/console/src/features/investigation/components/InvestigationList.tsx deleted file mode 100644 index e8fb2c0..0000000 --- a/console/src/features/investigation/components/InvestigationList.tsx +++ /dev/null @@ -1,136 +0,0 @@ -import { useState } from "react"; -import { Link, useNavigate } from "@tanstack/react-router"; -import { Clock, FlaskConical, AlertCircle, CheckCircle2, GitCompareArrows, Square, CheckSquare } from "lucide-react"; -import { cn } from "@/shared/lib/utils"; -import type { InvestigationSummary } from "../types"; - -interface InvestigationListProps { - investigations: InvestigationSummary[]; -} - -const STATUS_CONFIG: Record< - string, - { icon: typeof Clock; label: string; className: string } -> = { - pending: { - icon: Clock, - label: "Pending", - className: "text-muted-foreground", - }, - running: { - icon: FlaskConical, - label: "Running", - className: "text-primary", - }, - completed: { - icon: CheckCircle2, - label: "Completed", - className: "text-secondary", - }, - failed: { - icon: AlertCircle, - label: "Failed", - className: "text-destructive", - }, -}; - -export function InvestigationList({ investigations }: InvestigationListProps) { - const [selected, setSelected] = useState>(new Set()); - const navigate = useNavigate(); - const completed = investigations.filter((inv) => inv.status === "completed"); - - if (investigations.length === 0) { - return null; - } - - function toggleSelect(id: string) { - setSelected((prev) => { - const next = new Set(prev); - if (next.has(id)) { - next.delete(id); - } else if (next.size < 2) { - next.add(id); - } - return next; - }); - } - - function handleCompare() { - const ids = [...selected]; - if (ids.length === 2) { - void navigate({ to: "/compare/$id1/$id2", params: { id1: ids[0], id2: ids[1] } }); - } - } - - return ( -
- {completed.length >= 2 && ( -
- -
- )} - {investigations.map((inv) => { - const config = STATUS_CONFIG[inv.status] ?? STATUS_CONFIG.pending!; - const StatusIcon = config.icon; - const isCompleted = inv.status === "completed"; - return ( -
- {completed.length >= 2 && isCompleted && ( - - )} - -
-

{inv.prompt}

-

- {new Date(inv.created_at).toLocaleDateString(undefined, { - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - })} -

-
-
- {inv.candidate_count > 0 && ( - - {inv.candidate_count} candidates - - )} - - - {config.label} - -
- -
- ); - })} -
- ); -} diff --git a/console/src/features/investigation/components/PromptInput.tsx b/console/src/features/investigation/components/PromptInput.tsx index 73ece1c..3d08ccd 100644 --- a/console/src/features/investigation/components/PromptInput.tsx +++ b/console/src/features/investigation/components/PromptInput.tsx @@ -1,9 +1,18 @@ -import { useState } from "react"; +import { useRef, useState } from "react"; import { useNavigate } from "@tanstack/react-router"; +import { ArrowUp, Paperclip, Brain } from "lucide-react"; import { useInvestigation } from "../hooks/use-investigation"; import { useCredits } from "../hooks/use-credits"; -import { FileUpload } from "./FileUpload"; +import { FileUpload, type FileUploadRef, type UploadedFile } from "./FileUpload"; import type { DirectorTier } from "../types"; +import { Button } from "@/shared/components/ui/button"; +import { Textarea } from "@/shared/components/ui/textarea"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/shared/components/ui/popover"; +import { cn } from "@/shared/lib/utils"; const TIERS: { value: DirectorTier; label: string; credits: number }[] = [ { value: "haiku", label: "Haiku", credits: 1 }, @@ -21,17 +30,16 @@ export function PromptInput({ value, onChange }: PromptInputProps) { const mutation = useInvestigation(); const { data: creditData } = useCredits(); const [tier, setTier] = useState("opus"); - const [uploadedFiles, setUploadedFiles] = useState< - { file_id: string; filename: string }[] - >([]); + const [uploadedFiles, setUploadedFiles] = useState([]); + const fileUploadRef = useRef(null); - const selectedTier = TIERS.find((t) => t.value === tier)!; + const selectedTier = TIERS.find((t) => t.value === tier) || TIERS[2]; const isByok = creditData?.is_byok ?? false; const hasEnoughCredits = isByok || !creditData || creditData.credits >= selectedTier.credits; - function handleSubmit(e: React.FormEvent) { - e.preventDefault(); + function handleSubmit(e?: React.FormEvent) { + if (e) e.preventDefault(); if (!value.trim() || mutation.isPending || !hasEnoughCredits) return; const fileIds = uploadedFiles.map((f) => f.file_id); @@ -49,59 +57,159 @@ export function PromptInput({ value, onChange }: PromptInputProps) { ); } + function handleKeyDown(e: React.KeyboardEvent) { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + handleSubmit(); + } + } + + const [isDragging, setIsDragging] = useState(false); + + function handleDragOver(e: React.DragEvent) { + e.preventDefault(); + setIsDragging(true); + } + + function handleDragLeave(e: React.DragEvent) { + e.preventDefault(); + setIsDragging(false); + } + + function handleDrop(e: React.DragEvent) { + e.preventDefault(); + setIsDragging(false); + if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { + // Cast FileList to File[] for the ref method. + // Actually ref.addFiles expects File[]. Array.from() works. + fileUploadRef.current?.addFiles(Array.from(e.dataTransfer.files)); + } + } + return ( -
-