diff --git a/.env.example b/.env.example index 894727b..4482a11 100644 --- a/.env.example +++ b/.env.example @@ -1,36 +1,56 @@ # GitHub API (optional, increases rate limits for public repos) GITHUB_TOKEN= -# AI provider: gemini | openai | anthropic | edgeone | mock +# AI provider: gemini | openai | anthropic | mock AI_PROVIDER=mock +# Optional separate provider for semantic search: openai, gemini, or mock. +# EMBEDDING_PROVIDER=openai +# Independent embedding credentials and endpoint (recommended). +# Legacy OPENAI_/GEMINI_ settings are used when these are absent. +# EMBEDDING_API_KEY= +# EMBEDDING_API_BASE=https://api.openai.com/v1 +# EMBEDDING_MODEL=text-embedding-3-small +# OpenAI-compatible example: OPENAI_API_BASE=https://api.featherless.ai/v1 +# OPENAI_EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B +# EMBEDDING_DIMENSIONS=4096 # AI API Key (mandatory) GEMINI_API_KEY= OPENAI_API_KEY= ANTHROPIC_API_KEY= -EDGEONE_API_KEY= # Model overrides (optional) GEMINI_MODEL=gemini-2.5-flash GEMINI_API_VERSION=v1alpha OPENAI_MODEL=gpt-4o-mini ANTHROPIC_MODEL=claude-3-5-haiku-latest -EDGEONE_MODEL=@makers/deepseek-v4-flash # Provider API base URL overrides (optional) OPENAI_API_BASE= ANTHROPIC_API_BASE= -EDGEONE_API_BASE=https://ai-gateway.edgeone.link/v1 # Embedding overrides (optional) OPENAI_EMBEDDING_MODEL=openrouter/openai/text-embedding-3-large -EDGEONE_EMBEDDING_MODEL=openrouter/openai/text-embedding-3-large GEMINI_EMBEDDING_MODEL=gemini-embedding-001 EMBEDDING_DIMENSIONS=3072 # Cache key hashing secret (optional, set a stable random value in production) +# Required in shared mode (including when Redis is configured). Use a random secret. +# Local mode generates a process-only secret when this is blank. TOKEN_CACHE_HASH_SECRET= # Production server binding (optional) HOST=0.0.0.0 PORT=3000 + +# Daily embedding input budget (300 MB; non-refundable reservation per attempt). +# In the default local mode this budget is PER-PROCESS: each instance enforces its own 300 MB. +# To share one budget across instances, use shared-mode rate limiting: +# RATE_LIMIT_MODE=shared plus UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN. +SERVER_EMBEDDING_DAILY_BYTES=300000000 + +API_REQUESTS_PER_MINUTE=600 + +SERVER_AI_DAILY_CALLS=500 + +API_REQUESTS_PER_IP_PER_MINUTE=120 diff --git a/AGENTS.md b/AGENTS.md index f6cc67f..c0ba8e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ ## Project Identity -**gitSdm** (Git Software Dependency Map) — v3.0.0. Graph-first repository analysis tool: visualize file dependencies, AI-powered codebase insights, semantic search, commit timelines, architecture diagrams, and dependency health. Single-page app with an embedded Express backend + Vercel serverless functions. +**gitSdm** (Git Software Dependency Map) — v3.5.11. Interactive repository dependency visualization tool: map file dependencies, AI-powered codebase insights, semantic search, commit timelines, architecture diagrams, and dependency health. Single-page app with an embedded Express backend + Vercel serverless functions. --- @@ -20,7 +20,7 @@ | Graph Viz | react-force-graph-2d, d3-force, d3-hierarchy | | Diagrams | Mermaid 11, html-to-image, jsPDF | | Backend | Express (dev + prod server), Zod validation | -| AI Providers | Gemini, OpenAI, Anthropic, EdgeOne Makers, Mock — unified via `createProvider()` | +| AI Providers | Gemini, OpenAI, Anthropic, Mock — unified via `createProvider()` | | GitHub API | Octokit REST v21 | | Search | In-memory vector store with chunking + embedding + QA engine | | Security | dompurify (sanitization) | @@ -35,15 +35,15 @@ gitSdm/ ├── api/ # Vercel serverless entry points (thin wrappers) │ ├── ai/ # AI endpoint wrappers -│ ├── repo/ # Repo API endpoint wrappers +│ ├── repo/ # Repo API endpoint wrappers (analyze, files, branches, churn, health) │ └── trending.ts # Trending repos endpoint ├── server/ # Backend services & router -│ ├── ai/ # AI provider abstraction + task handlers +│ ├── ai/ # AI provider abstraction, chat config, SSRF protection, prompts & tasks │ │ ├── prompts.ts # Shared AI prompt templates -│ │ ├── provider.ts/test.ts # AI provider (Gemini/OpenAI/Anthropic/EdgeOne/Mock) +│ │ ├── provider.ts/test.ts # AI provider (Gemini/OpenAI/Anthropic/Mock) │ │ ├── service.ts/test.ts # AI service orchestration │ │ └── tasks/ # Individual AI tasks (diagram, explain, onboarding, playground, refactor) -│ ├── cache/ # LRU caching layer (lru.ts/test.ts) +│ ├── cache/ # LRU caching layer & hashed token secrets │ ├── config/ # Env validation & public config (app-config.ts/test.ts) │ ├── env.ts # Environment variable exports │ ├── github/ # GitHub API client (Octokit), fetch-tree, mock-data (+ test files) @@ -53,19 +53,19 @@ gitSdm/ │ │ ├── dependency-analyzer.ts/test.ts │ │ ├── file-classifier.ts/test.ts │ │ ├── import-resolver.ts/test.ts -│ │ └── manifest-parsers/ # docker, go, java, npm, pip, rust + shared registry/types +│ │ └── manifest-parsers/ # docker, go, java, npm, pip (PEP 621/Poetry), rust + shared registry/types │ ├── router/ # Route handlers (ai-routes, repo-routes, search-routes, schemas) -│ ├── search/ # Semantic search: chunker, embeddings, vector store, QA engine, indexing pipeline -│ ├── services/ # Business logic (analyze-repo, churn-service, dependency-health, get-file, npm-registry, trending) -│ ├── utils/ # Errors, context, logger, HTTP helpers +│ ├── search/ # Semantic search: chunker, embeddings, checkpoints, vector store, QA engine, indexing pipeline +│ ├── services/ # Business logic (analyze-repo, churn-service, dependency-health, repo-enrichment, get-file, npm-registry, trending) +│ ├── utils/ # Errors, context, logger, HTTP, limits, SSRF guard, bounded queue │ ├── api-router.ts # Unified API router -│ ├── dev-api.ts # Dev server API middleware +│ ├── dev-api.ts # Vite dev server middleware │ ├── prod-server.ts # Production Express server │ └── vercel-handler.ts # Vercel serverless entry ├── src/ # Frontend SPA │ ├── app/ # Router setup (router.tsx), app providers (providers.tsx) │ ├── components/ # UI components organized by domain -│ │ ├── ui/ # shadcn primitives & Base UI wrappers +│ │ ├── ui/ # shadcn primitives, button-variants & Base UI wrappers │ │ ├── viz/ # Main workspace: ai-sidebar/, architecture/, layout/, learning-path/, top-nav/ │ │ │ ├── ai-sidebar/ # AI center tab, action buttons, intelligence cards, tool cards │ │ │ ├── architecture/ # Mermaid diagram generation, pan/zoom, render-sequence guard @@ -73,31 +73,31 @@ gitSdm/ │ │ │ ├── learning-path/# Guided code walkthroughs │ │ │ ├── top-nav/ # TopNav, BranchSwitcher, HeaderActionMenu, HeaderStats, WorkspaceModeSelector │ │ │ ├── AIErrorCard.tsx, AISidebar.tsx, AnalysisTab.tsx, ArchitectureView.tsx -│ │ │ ├── BottomStatusBar.tsx, DependencyHealthTab.tsx, LearningPathTab.tsx -│ │ │ ├── OverviewTab.tsx, SettingsPopover.tsx, StagedLoader.tsx, VizError.tsx +│ │ │ ├── BottomStatusBar.tsx, ChurnStatus.tsx, DependencyHealthTab.tsx, LearningPathTab.tsx +│ │ │ ├── MotionSettings.tsx, OverviewTab.tsx, SettingsPopover.tsx, StagedLoader.tsx, VizError.tsx │ │ ├── explorer/ # File explorer sidebar + code inspector dock │ │ ├── timeline/ # Commit history view + repo timeline -│ │ ├── contributors/# Contributor analytics -│ │ ├── home/ # Landing page: HeroSection, RepoInput, HomeGraphPreview, Trending, repoPresets -│ │ ├── layout/ # Navbar +│ │ ├── contributors/# Contributor analytics & timeline layout +│ │ ├── home/ # Landing page: HeroSection, RepoInput, HomeGraphPreview, Trending, repoPresets, repo-navigation +│ │ ├── layout/ # Navbar, SiteFooter, InfoPageLayout, PageMetadata │ │ ├── theme/ # ThemeSync │ │ └── ErrorBoundary.tsx │ ├── features/ # Feature modules │ │ ├── graph/ # Graph canvas (ForceGraphCanvas, GraphCanvas, ToolbarDropdowns), widgets, force engine │ │ │ ├── canvas/ # Canvas layout engine, hooks, helpers (filters), widgets (DropdownPanel, LegendPanel) -│ │ │ └── force/ # Force constants, color palettes, blastRadius, buildForceGraphData -│ │ ├── ai/ # AI task frontend hooks (useAiTasks) -│ │ └── search/ # Semantic search UI: SearchBar, SearchResults, QAAnswerView, IndexingStatusPanel -│ ├── hooks/ # Shared hooks (useAnalyzeRepo, useCodeInspectorState, useMobile, useRepoBranches, useRepoTags, useWorkspaceShortcuts) -│ ├── lib/ # Shared utilities: apiClient, clipboard, utils, file-context (+ test files) -│ ├── stores/ # Zustand stores (vizStore — canonical global store) -│ ├── pages/ # Route pages (VizPage, HomePage, SearchPage, NotFoundPage) -│ ├── types/ # TypeScript type definitions (api, domain, github, index) +│ │ │ └── force/ # Force constants, color palettes, blastRadius, buildForceGraphData, reconcileGraph +│ │ ├── ai/ # AI task frontend hooks & task cache (useAiTasks, tool-cache) +│ │ └── search/ # Semantic search UI: SearchBar, SearchResults, SearchIndexControls, SearchEmptyState, QAAnswerView, IndexingStatusPanel +│ ├── hooks/ # Shared hooks (useAnalyzeRepo, useCodeInspectorState, useMobile, useRepoBranches, useRepoTags, useWorkspaceShortcuts, useMotionPreference, useRepoChurn, useRepoHealth) +│ ├── lib/ # Shared utilities: apiClient, clipboard, utils, file-context, motion-preference, page-metadata, churn-progress, workspace-mode (+ test files) +│ ├── stores/ # Zustand stores (vizStore, motionStore, chatConfigStore) +│ ├── pages/ # Route pages (HomePage, VizPage, SearchPage, TermsPage, PrivacyPage, NotFoundPage, search-page-logic) +│ ├── types/ # TypeScript type definitions (api, domain, github, churn, index) │ ├── styles/ # Tailwind CSS global styles (globals.css, interface.css) │ └── main.tsx # App entry point -├── e2e/ # Playwright end-to-end test specs (offline mock-driven) -└── public/ # Static assets -``` +├── public/ # Static assets (og-image.png/svg, robots.txt, sitemap.xml) +├── scripts/ # Prerender and maintenance scripts (prerender.tsx, clean_graphify.py) +└── e2e/ # Playwright end-to-end test specs (offline mock-driven) --- @@ -108,7 +108,7 @@ gitSdm/ All AI interactions go through `server/ai/provider.ts`. Do NOT call SDKs directly in task code. ```text -Task handler → summarizer / task → createProvider(type) → { Gemini | OpenAI | Anthropic | EdgeOne | Mock } +Task handler → summarizer / task → createProvider(type) → { Gemini | OpenAI | Anthropic | Mock } ``` - Uses `createProvider(overrideKey?)` → returns `AIProvider` with `.complete(messages, options?)` @@ -133,7 +133,7 @@ Manual pathname-matching pattern in `server/router/`. Route files export `handle ### API Client (Frontend) -`src/lib/apiClient.ts` — Typed fetch wrapper via `apiFetch(url, options?)`. Authentication tokens stored in `localStorage` (`gitsdm_gemini_api_key`, `gitsdm_github_pat`) and sent as `X-Gemini-API-Key` / `X-GitHub-Token` headers. +`src/lib/apiClient.ts` — Typed fetch wrapper via `apiFetch(url, options?)`. Authentication tokens stored in `localStorage` (`gitsdm_gemini_api_key`, `gitsdm_github_pat`) and sent as `X-AI-API-Key` (with `X-Gemini-API-Key` fallback) / `X-GitHub-Token` headers. ### Toast Notification Pattern @@ -189,7 +189,7 @@ Test runner: **Bun** (`bun test --isolate`). ### Coverage & Test Counts -- **46 test files, 408 tests passing** (991 expect calls), 0 failures. +- **78 test files, 527 tests passing** (1479 expect calls), 0 failures. - Run a single test file: `bun test src/components/home/RepoInput.test.ts` - Run test coverage: `bun test --coverage` @@ -225,22 +225,18 @@ bun run lint # ESLint check | Variable | Default | Description | | ------------------------ | ------------------------------------------ | -------------------------------------- | | `GITHUB_TOKEN` | — | GitHub PAT for API rate limits | -| `AI_PROVIDER` | `mock` | Provider: gemini, openai, anthropic, edgeone, mock | +| `AI_PROVIDER` | `mock` | Provider: gemini, openai, anthropic, mock | | `GEMINI_API_KEY` | — | Gemini API key | | `OPENAI_API_KEY` | — | OpenAI API key | | `ANTHROPIC_API_KEY` | — | Anthropic API key | -| `EDGEONE_API_KEY` | — | EdgeOne Makers Models API key (alias: `MAKERS_MODELS_KEY`) | | `OPENAI_API_BASE` | OpenAI default | Custom API base URL | | `ANTHROPIC_API_BASE` | Anthropic default | Custom API base URL | -| `EDGEONE_API_BASE` | `https://ai-gateway.edgeone.link/v1` | EdgeOne API base URL | | `OPENAI_EMBEDDING_MODEL` | `openrouter/openai/text-embedding-3-large` | Embedding model | -| `EDGEONE_EMBEDDING_MODEL`| `openrouter/openai/text-embedding-3-large` | Embedding model | | `EMBEDDING_DIMENSIONS` | `3072` | Vector dimension count | | `GEMINI_MODEL` | `gemini-2.5-flash` | Gemini model override | | `GEMINI_API_VERSION` | `v1alpha` | Gemini API version | | `OPENAI_MODEL` | `gpt-4o-mini` | OpenAI model override | | `ANTHROPIC_MODEL` | `claude-3-5-haiku-latest` | Anthropic model override | -| `EDGEONE_MODEL` | `@makers/deepseek-v4-flash` | EdgeOne model override | | `GEMINI_EMBEDDING_MODEL` | `gemini-embedding-001` | Gemini embedding model override | | `TOKEN_CACHE_HASH_SECRET`| — | Cache key hashing secret (production) | | `HOST` | `0.0.0.0` | Production server bind | @@ -257,8 +253,32 @@ bun run lint # ESLint check - Zero hardcoded secrets - 250+ LOC ceiling approached only by generated shadcn components +## Versioning Policy + +Version numbers increment progressively based on commit scope and change size: + +- **Major (`+1.x.x`)**: Major milestone or large-scale overhaul (100+ files). +- **Minor (`x.+1.x`)**: Feature additions and substantial enhancements (`feat`). +- **Patch (`x.x.+1`)**: Bug fixes, security hardening, and maintenance (`fix`, `chore`). + +Track progressive increments across commit batches rather than collapsing multiple feature commits into a single minor bump. Releases on the same date are consolidated in the changelog under that date's latest version. + ## CI Requirements - `.github/workflows/ci.yml`: Pinned Bun 1.3.14 (`setup-bun`), `permissions: contents: read`. - All installs use `nick-invision/retry@v3` with `bun install --frozen-lockfile`. - Jobs: `lint` -> `typecheck` -> `test` -> `build` (with 2.5 MB gzipped-JS bundle gate) -> `e2e` (`AI_PROVIDER=mock`). + +## Agent skills + +### Issue tracker + +Issues live as local markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +Default five canonical triage roles. See `docs/agents/triage-labels.md`. + +### Domain docs + +Single-context layout (`CONTEXT.md` + `docs/adr/`). See `docs/agents/domain.md`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 35e70f1..e9ea743 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architecture -`gitSdm` is a graph-first repository analysis application. It visualizes file dependencies, architecture diagrams, module hierarchies, commit history, and AI-driven codebase insights in a single-page app with an embedded Express backend and Vercel serverless functions. +`gitSdm` is an interactive repository dependency visualization tool. It maps file dependencies, architecture diagrams, module hierarchies, commit history, and codebase insights in a single-page app with an embedded Express backend and Vercel serverless functions. --- @@ -22,7 +22,7 @@ │ │ │ │ ┌────────────────────────┐ ┌───────────▼───────────┐ ┌────────────────┐ │ │ │ AI Provider Layer │ │ In-Memory Vector DB │ │ LRU Cache │ │ -│ │ (Gemini/OAI/Anth/Edge) │ │ (Chunking/Embed/QA) │ │ (200 items) │ │ +│ │ (Gemini/OAI/Anth/Mock) │ │ (Chunking/Embed/QA) │ │ (200 items) │ │ │ └────────────────────────┘ └───────────────────────┘ └────────────────┘ │ └──────────────────────────────────────┬──────────────────────────────────────┘ │ (JSON REST API) @@ -67,18 +67,27 @@ - Google Gemini (`gemini-2.5-flash`) - OpenAI (`gpt-4o-mini`) - Anthropic (`claude-3-5-haiku-latest`) - - EdgeOne Makers (`@makers/deepseek-v4-flash`) - Offline Mock Provider (fixtures for testing and development without API keys) +- **Custom Endpoints & SSRF Guard (`server/ai/chat-config.ts`, `server/ai/public-chat-fetch.ts`):** Supports user-defined OpenAI-compatible base URLs and model overrides in Settings. Enforces SSRF security by verifying public HTTPS protocols, blocking internal/loopback/link-local IP addresses, and pinning DNS resolution. +- **Task Lifecycle & Cache Resilience (`src/features/ai/tool-cache.ts`):** In-memory task caching and deduplication ensure in-flight AI requests survive panel remounting and branch switching, while configuration revisions refresh stale cached responses on save. +- **Independent Embedding Pipeline (`server/search/embedding-config.ts`):** Decouples embedding credentials, models, and endpoints from chat settings to keep semantic search isolated. - **AI Tasks (`server/ai/tasks/`):** Dedicated pipelines for architectural summaries, Mermaid flowchart generation, onboarding tours, refactoring risk detection, and dependency health audits. ### 4. Semantic Search & QA Engine -- **Chunker & Embedding Pipeline (`server/search/`):** Breaks repository files into semantic code chunks, computes embeddings, and indexes them in an in-memory vector store with cosine similarity search. -- **Natural Language QA:** Synthesizes context-aware answers to user queries referencing specific source lines. +- **Chunker & Resumable Checkpoints (`server/search/checkpoints.ts`, `server/search/build-snapshot.ts`):** Splits supported repository files into AST/sliding-window code chunks, computes embeddings with rate-limit pacing, and saves intermediate batch checkpoints so interrupted builds resume seamlessly. +- **In-Memory Vector Store & LRU Eviction (`server/search/vector-store.ts`):** Fast cosine-similarity search store with strict byte budgets, 30-minute index lifetimes, and LRU eviction. +- **Natural Language QA (`server/search/qa-engine.ts`):** Synthesizes context-aware answers to user queries referencing specific source lines. -### 5. State Management & Data Flow +### 5. Server Guardrails & Usage Limits -- **Global UI State (`src/stores/vizStore.ts`):** Zustand store persisted to `localStorage` via `zustand/middleware/persist` for filters, layout preferences, and active workspace modes. +- **SSRF Outbound Guard (`server/utils/url-guard.ts`):** Validates outbound HTTP/HTTPS destinations, expands IPv6 addresses to 8 hextets, checks transition prefixes (NAT64 `64:ff9b::/96`, 6to4 `2002::/16`), and rejects private, loopback, multicast, or documentation addresses. +- **Admission & Rate Limits (`server/utils/client-limits.ts`, `server/utils/usage-limits.ts`):** Enforces per-IP and deployment-wide request quotas, trusted proxy header chains, and daily server-funded AI/embedding budgets with Upstash Redis or local sliding windows. +- **Bounded Queues & Request Limits (`server/utils/bounded-queue.ts`, `server/utils/request-limits.ts`):** Caps concurrency and waiting queues with 10s wait and 30s execution timeouts, immediate abort cleanup, and strict 1 MB UTF-8 body limits. + +### 6. State Management & Data Flow + +- **Global UI State (`src/stores/vizStore.ts`, `src/stores/motionStore.ts`, `src/stores/chatConfigStore.ts`):** Zustand stores for workspace modes, filters, responsive drawers, motion preferences, and chat configuration revisions. - **Server State:** TanStack React Query handles server queries with stale-while-revalidate semantics and query keys keyed by `[resource, owner, repo, branch]`. --- @@ -89,21 +98,21 @@ gitSdm/ ├── api/ # Vercel serverless entry points (thin wrappers) │ ├── ai/ # AI task endpoint wrappers -│ ├── repo/ # Repository analysis endpoint wrappers +│ ├── repo/ # Repository analysis, churn, and health endpoints │ └── trending.ts # Trending repositories endpoint ├── server/ # Backend services & router -│ ├── ai/ # AI provider abstraction, prompts, and task handlers +│ ├── ai/ # AI provider abstraction, chat config, SSRF protection, prompts & tasks │ │ └── tasks/ # Individual AI task logic (diagram, explain, onboarding, playground, refactor) -│ ├── cache/ # LRU caching layer +│ ├── cache/ # LRU caching layer & hashed token secrets │ ├── config/ # Runtime config & environment validation │ ├── github/ # GitHub API client (Octokit) & mock fixtures │ ├── graph/ # Graph building, node colors, and layout algorithms │ ├── parser/ # Dependency analysis, file classification & manifest parsers -│ │ └── manifest-parsers/ # npm, pnpm, cargo, pip, go, maven workspace parsers -│ ├── router/ # Modular request route handlers (repo, AI, search) -│ ├── search/ # Semantic search, vector store & QA engine -│ ├── services/ # Application services (analyze-repo, churn, health, trending, npm-registry) -│ ├── utils/ # HTTP, context, and logging helpers +│ │ └── manifest-parsers/ # npm, pnpm, cargo, pip (PEP 621/Poetry), go, maven workspace parsers +│ ├── router/ # Modular request route handlers (repo, AI, search, schemas) +│ ├── search/ # Semantic search: chunker, embeddings, checkpoints, vector store & QA engine +│ ├── services/ # Application services (analyze-repo, churn, health, repo-enrichment, trending, npm-registry) +│ ├── utils/ # HTTP, context, client limits, request limits, SSRF guard, and logging helpers │ ├── api-router.ts # Unified API router │ ├── dev-api.ts # Vite dev server middleware │ ├── env.ts # Environment variable exports @@ -112,21 +121,22 @@ gitSdm/ ├── src/ # Frontend SPA application │ ├── app/ # App entry, router setup, and query client providers │ ├── components/ # UI components organized by domain -│ │ ├── contributors/ # Contributor metrics and activity timeline +│ │ ├── contributors/ # Contributor metrics, activity timeline, and timeline layout │ │ ├── explorer/ # File tree and code inspector dock -│ │ ├── home/ # Landing page (hero, input, preview, trending) -│ │ ├── layout/ # Top navbar and app layout +│ │ ├── home/ # Landing page (hero, input, preview, trending, presets, repo navigation) +│ │ ├── layout/ # Navbar, SiteFooter, InfoPageLayout, and PageMetadata │ │ ├── theme/ # ThemeSync appearance controller │ │ ├── timeline/ # Commit history and repo timeline views -│ │ ├── ui/ # shadcn/ui primitives & Base UI components +│ │ ├── ui/ # shadcn/ui primitives, button-variants & Base UI components │ │ └── viz/ # Main visualization workspace │ │ ├── ai-sidebar/ # Codebase intelligence tabs & action cards │ │ ├── architecture/# Interactive Mermaid diagrams & zoom controls │ │ ├── layout/ # Resizable sidebar & panel drawer │ │ ├── learning-path/# Guided code walkthroughs -│ │ └── top-nav/ # Branch/tag switcher, stats & workspace mode controls +│ │ ├── top-nav/ # Branch/tag switcher, stats & workspace mode controls +│ │ └── AIErrorCard, ChurnStatus, MotionSettings, SettingsPopover, StagedLoader, VizError │ ├── features/ # Feature modules -│ │ ├── ai/ # AI task mutation hooks (useAiTasks) +│ │ ├── ai/ # AI task mutation hooks & resilient cache (useAiTasks, tool-cache) │ │ ├── graph/ # Force graph & D3 canvas engine, widgets & export │ │ │ ├── canvas/ # ForceGraphCanvas, GraphCanvas, and ToolbarDropdowns │ │ │ │ ├── force/ # D3 physics, force painter, minimap, and sync @@ -134,13 +144,15 @@ gitSdm/ │ │ │ │ ├── hooks/ # Canvas state hooks │ │ │ │ └── widgets/# DropdownPanel, LegendPanel, FloatingControls, FilterSummary │ │ │ └── force/ # Force graph data models, constants, and blast radius -│ │ └── search/ # Semantic search interface & indexing status -│ ├── hooks/ # Shared custom React hooks -│ ├── lib/ # API client, clipboard, and string helpers -│ ├── pages/ # Route pages (VizPage, HomePage, SearchPage, NotFoundPage) -│ ├── stores/ # Zustand global stores (vizStore) +│ │ └── search/ # Semantic search interface, controls, empty state, recovery & parsePaths +│ ├── hooks/ # Shared custom React hooks (useAnalyzeRepo, useMotionPreference, useRepoChurn, useRepoHealth) +│ ├── lib/ # API client, clipboard, motion-preference, page-metadata, churn-progress, and workspace-mode +│ ├── pages/ # Route pages (HomePage, VizPage, SearchPage, TermsPage, PrivacyPage, NotFoundPage) +│ ├── stores/ # Zustand global stores (vizStore, motionStore, chatConfigStore) │ ├── styles/ # Tailwind CSS 4 global theme -│ └── types/ # TypeScript domain types and DTOs +│ └── types/ # TypeScript domain types, DTOs, and churn types +├── public/ # Static assets (og-image.png/svg, robots.txt, sitemap.xml) +├── scripts/ # Build and prerender scripts (prerender.tsx, clean_graphify.py) └── e2e/ # Playwright end-to-end test suite ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index ed2f835..a8828dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,111 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +and this project adheres to progressive [Semantic Versioning](https://semver.org/spec/v2.0.0.html) based on commit scope and size: + +- **Major (`+1.x.x`)**: Major architectural rewrite or large-scale overhaul (100+ files). +- **Minor (`x.+1.x`)**: Feature additions and substantial enhancements (`feat`). +- **Patch (`x.x.+1`)**: Bug fixes, security hardening, and maintenance (`fix`, `chore`). + +Releases on the same date are consolidated under that date's latest version. + +## [3.5.11] - 2026-09-14 + +### Follow-up fixes (2026-09-19) + +- Pin embedding connections to validated public addresses, with a bounded body limit that accommodates JSON escaping. +- Avoid replaying shared quota reservations after an ambiguous transport failure. +- Normalize equivalent mapped IPv6 addresses and count each commit author once. +- Keep paused indexing attached to its repository and original scope; show safe error messages. +- Generate and cache AI architecture diagrams against the displayed commit SHA. + +### Security + +- **DNS-Pinned Limiter Requests**: The shared usage limiter now connects only to DNS-validated addresses over `node:https` with the original TLS identity (no re-resolution between validation and connect), so a rebinding hostname cannot divert the Redis bearer token to a private address; redirects fail closed and responses are capped at 64 KiB. +- **Bounded Outbound Request Bodies**: Custom-endpoint chat requests stream through a running-total size check (256 KiB, plus `Content-Length` pre-check) instead of buffering unbounded `arrayBuffer()` calls. +- **HTTPS-Only Embedding Endpoints**: Configured `OPENAI_API_BASE`/`EMBEDDING_API_BASE` values are rejected before constructing the embedding client when they are not HTTPS. +- **Client Quota Canonicalization & Secrecy**: IPv4-mapped forwarded addresses canonicalize from their 32-bit payload (hex forms included) so per-IP buckets cannot be evaded, and client scopes derive from a server-secret HMAC instead of unkeyed SHA-256. +- **Non-Negative Usage Amounts**: `reserveUsage` rejects negative and non-integer amounts before either backend path, closing quota replenishment. +- **Strict IPv6 Compression & Group Validation in SSRF Guard**: Enforces non-zero `::` compression runs (`missing > 0`) and validates each group as 1–4 hexadecimal characters before parsing in `url-guard.ts`, failing closed on malformed zero-run compressions and out-of-range hextets. + +### Fixed + +- **Repository-Scoped Index Operations**: Active indexing operations track owner/repo and reset on repository navigation; query resets bump the mutation revision while preserving the in-flight index build; stale search/ask responses drop on revision, scope, branch, or repo mismatch. +- **Resumable Index State Preservation**: `INDEX_NOT_FOUND` after scope edits and cross-repository status polls no longer hide `indexing`/`paused` operations; retries preserve snapshot and coverage only when resuming a paused build. +- **Indexing Error Sanitization**: Untrusted API error messages are stripped of URLs, tokens, and paths (300-char cap) before display, and pipeline logs use secret-safe error payloads; embedding cooldowns arm only on provider 429s. +- **Credential-Scoped Caches & Churn**: Churn and health query keys carry the credential revision; continuations echo an opaque scope fingerprint and refetch on mismatch; per-file 404s no longer halt whole-batch polling; author counts include both login and commit author names. +- **Tracked Same-Tab Config Writes**: All credential/AI setting writes route through a helper that refreshes chat config (including `VizError` PAT clears); tool cache keys use presence-safe branch encoding and Mermaid requests share the keyed task cache. +- **Dependency-Focused Health Path**: Cold `/api/repo/health` requests skip the full analysis pipeline (no contributors, timeline, commits, or graph), and npm metadata is fetched for every dependency with no 100-item truncation. +- **Responsive Workspace Preservation**: Mobile panel closing and repository switches use a single mode-preserving state update; the Settings popover re-reads stored values on open across desktop/mobile instances. +- **Parser, Queue & HTTP Fixes**: PEP 621 scans stay within `[project]` (optionals only in `[project.optional-dependencies]`), versionless Poetry path/git/URL tables emit with undefined versions, aborted queue entries skip work, stalled bodies return typed 408s, caller abort signals reach providers, and `.git` suffixes strip before query/hash parsing. +- **Test Seams Over Module Mocks**: `analyze-repo` and `get-file` suites drive the real pipeline through stub Octokit clients in `RequestContext` instead of `mock.module()` on shared internal modules; admission tests use an injectable pinned-POST seam. +- **Frontend/Server Type Boundary**: Shared `SearchCoverage`, `IndexingStatus`, `scopeKey`, and `coverageMessage` live in `src/` (types via the barrel, pure helpers via `@/lib`), ending all deep `server/` imports from the frontend; server runtime keeps a documented local mirror where project boundaries forbid sharing. +- **Dev Server 404 & Asset Parity**: Aligned Vite development middleware with production Express server by serving 404 HTML for unmapped client routes and JSON 404 for missing `/assets/*` paths. +- **Mock Repository AI Fallback**: Ensured mock repository owners use deterministic offline fixtures in `executeAiTask` when no user API key is configured. +- **AI Mutation Revision Invalidation**: Tagged AI task mutations with active configuration revisions and validated revision matches in `useAiCenterState`, preventing stale results from blocking fresh requests on settings updates. +- **Canvas Redraw on Metadata Updates**: Linked graph metadata updates directly to `drawNodeCanvasObject` in `ForceGraphCanvas`, repainting updated node metrics immediately without reheating simulation physics. +- **Explicit Timeout Retry Matching**: Replaced ambiguous regex with explicit `/(?:timeout|timed\s*out)/i` in `embedding-retry.ts`. + +## [3.5.7] - 2026-09-13 + +### Security + +- **Full IPv6 Expansion in SSRF Guard**: IPv6 hostnames expand to their complete eight-group form before transition-prefix decoding, so compressed zero runs can no longer shift embedded IPv4 fields (`2002::802:808` no longer decodes as a public host). +- **Strict Well-Known NAT64 Decoding**: Only `64:ff9b::/96` with zero mid-groups decodes its embedded IPv4; the local-use variant `64:ff9b:1::/48` and all other `64:ff9b::*` forms reject outright. +- **Bounded Limiter DNS Resolution**: The shared limiter's DNS lookup is bounded by an injectable deadline (default 3s) and fails closed on timeout, so a hung resolver cannot hold admission slots. +- **Limiter DNS Resolution Validation**: The shared limiter resolves its hostname and re-validates every resolved address through the SSRF guard before connecting, fails closed on lookup failure or non-public addresses, rejects redirects so the bearer token cannot be forwarded, and supports IPv6-literal limiter URLs. +- **SSRF Guard Transition Prefixes**: The URL guard decodes embedded IPv4 destinations in NAT64 (`64:ff9b::/96`) and 6to4 (`2002::/16`) addresses and rejects private/loopback destinations, rejects the `64:ff9b:1::/48` local-use variant and `192.0.0.0/24`. +- **SSRF Guard for Outbound Fetches**: Added `server/utils/url-guard.ts` rejecting non-http(s) schemes and loopback/private/reserved hosts (IPv6 `::/8` and multicast forms, documentation and TEST-NET ranges, bare single-label names) and applied it to the shared rate limiter's Upstash fetch with distinct not-configured/not-permitted errors. +- **Rate-Limit Budget Protection**: Unknown `/api/*` paths now return 404 before any admission or usage bucket is reserved, so unregistered-path floods can no longer exhaust the deployment-wide budget. +- **PAT-Scoped AI Caches**: Included `gitsdm_github_pat` in chat config revisions (refreshed on save/clear) and reset AI panel mutations when the commit/revision changes, preventing cache reuse across credential changes. +- **Quota & Endpoint Security**: Prevented quota bypass from whitespace keys, enabled DNS multi-address sequential fallbacks, clamped `Retry-After` cooldown conversions, and combined caller signals with embedding queues. + +### Fixed + +- **Unified Search Scope Typing**: Scope polling, the search store, and indexing mutations now share the canonical `IndexScope` contract instead of three structural copies. +- **Search Scope Changes During Indexing**: Editing include/exclude filters while an index builds clears stale results and coverage while the operation keeps running, and status polling keeps targeting the build's original scope until it settles. +- **Bounded Queue Listener Leak & Abort Observability**: Detached caller-abort listener when the deadline rejects a never-settling job, and attached active abort listener during execution in `createBoundedQueue` so cancelled jobs immediately vacate waiting slots and abort running work. +- **413 Body Teardown**: Oversized request bodies now flush the response and destroy the socket instead of draining indefinitely. +- **Embedding Timeout Classification**: Matched the OpenAI SDK's "Request timed out." message so client timeouts receive backoff retries. +- **Search Scope Path Validation**: Rejected only whole `..` path components in include/exclude filters, allowing directory names like `a..b.ts`. +- **Search Scope Reset Guard**: Editing include/exclude fields no longer abandons an active or paused indexing operation; new `search-page-logic` helpers cover payloads, branch resolution, and partial-coverage enablement. +- **Graph Metadata Refresh & Simulation Stability**: `reconcileGraph` preserves simulation data identity on metric-only updates to prevent unintended reheating, while updating node visual properties. +- **Status Query Recovery**: Restored bounded retries for the indexing status query so transient failures don't hide existing indexes. +- **Reduced Motion Defaults**: OS-level reduced motion is honored before hydration and whenever no explicit choice is stored, while explicit Full/Reduced selections still win. +- **Churn Error Message & Retry Ordering**: Collapsed duplicate churn failure notices into one message and aligned server failure retry calculation with client (`a.retryAt - b.retryAt`). +- **Custom Endpoint Hardening & Validation Feedback**: Restricted custom endpoint retries strictly to pre-transmission connection failures, prevented retries on size-limit or redirect rejections, and wired real-time validation and error alerts in `SettingsPopover`. +- **Mobile Drawer Exclusivity**: Ensured `setExplorerOpen` and `setAiSidebarOpen` keep drawers mutually exclusive on mobile viewports. +- **Manifest Parser Depth**: Handled standard PEP 621 `dependencies = [...]` arrays, bracketed extras, PEP 508 environment markers, optional dependencies (`[project.optional-dependencies]`), Poetry dev groups (`type: 'dev'`), and ignored brackets inside TOML comments in `pip-parser`. +- **Commit Invalidation & AI Tool Cache**: Included repository commit SHA in tool cache keys, reset active mutations when commit SHA advances in `useAiCenterState`, and added bounded LRU eviction (100 entries). +- **UI & Layout Sync**: Kept `workspaceMode` in sync with individual panel toggles, gated health enrichment on comparison branches, and validated repository naming in navigation. + +## [3.5.0] - 2026-09-12 + +### Added + +- **Custom OpenAI-Compatible Endpoints**: Support for user-defined OpenAI-compatible base URLs and model overrides in Settings. +- **SSRF Protection for Custom Endpoints**: Blocks private, loopback, and link-local IP addresses with pinned DNS resolution. +- **Independent Embedding Configuration**: Decouples `EMBEDDING_PROVIDER`, `EMBEDDING_API_KEY`, `EMBEDDING_API_BASE`, and `EMBEDDING_MODEL` from chat settings. +- **Task Lifecycle Resilience**: Unified in-memory promise caching in `src/features/ai/tool-cache.ts` ensuring in-flight requests survive tab remounts and branch switches. + +### Changed + +- **Unified AI Providers**: Streamlined provider interface across Gemini, OpenAI, Anthropic, and Mock. Consolidated EdgeOne into standard OpenAI-compatible configuration. +- **Client Cache Synchronization**: Settings updates increment a chat configuration revision that invalidates stale cached responses and refreshes active tools. + +### Fixed + +- **Workspace Modes**: Made workspace modes mutually exclusive: Explorer Only opens only the file explorer, Insights Only opens only repository insights, and Focus Mode closes both panels, with desktop and mobile regression tests. + +## [3.0.2] - 2026-09-08 + +### Fixed + +- **Cache Endpoint Security**: Eliminated unauthenticated `/api/cache/clear` endpoint. +- **Mermaid Code Splitting**: Code-split `mermaid` library to dedicated vendor chunk, shrinking `ArchitectureView` from 1.44 MB to 22 KB. +- **Performance & Asset Caching**: Removed font-gating FOIT blocker in `index.html`, enabled `display=swap`, and added immutable Cache-Control headers for hashed static assets across Bun server, Vercel, and EdgeOne. +- **SEO & Metadata**: Added search engine metadata: `robots.txt`, `sitemap.xml`, OpenGraph SVG banner, and social meta tags. +- Replaced marketing buzzword copy across documentation and landing pages. ## [3.0.0] - 2026-09-07 @@ -28,33 +132,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Code Churn overlay (Color by Churn mode) with LRU-cached churn service -- Complexity Score overlay (Color by Complexity / Size by Complexity) -- Unified color architecture with `GRAPH_NODE_PALETTES` and `GRAPH_NODE_LAYER_OFFSETS` -- Legend panel with churn/complexity ramp and blast radius section -- Churn cache bucket in LRU cache (200 max, 1hr TTL) - -### Changed - -- Color architecture refactored for strict visual lane separation (fill/ring/glow/badge) -- Multi-author ring visibility gated at zoom ≥ 0.75 -- Compare-branch modified color: amber (#f59e0b) -- Churn/complexity low end: `#3f3f46` → `#ffffff` -- Legend swatches unified to circles - -## [2.7.5] - 2026-07-03 - -### Added - +- **Code Churn & Complexity Overlays**: Visual overlay for 90-day commit frequency, author ownership risk (3+ distinct authors), and cyclomatic/line complexity with LRU-cached churn service. - **Compare Mode Extension**: Added support for comparing tags and specific commit SHAs in the visual workspace. -- **AI Playground**: New dedicated interface for testing AI prompts and logic directly within the app. +- **AI Playground**: Dedicated interface for testing AI prompts and logic directly within the app. - **Dependency Health Highlights**: Real-time visual alerts and health status mapping for project dependencies. +- Unified color architecture with `GRAPH_NODE_PALETTES` and `GRAPH_NODE_LAYER_OFFSETS`. +- Legend panel with churn/complexity ramp and blast radius section. - Mobile-responsive layout improvements for the home and visualization pages. ### Changed -- **AI Provider Unification**: Refactored the core AI logic to use a unified provider flow, improving reliability across Gemini, OpenAI, and Anthropic. +- **AI Provider Unification**: Refactored the core AI logic to use a unified provider flow across Gemini, OpenAI, and Anthropic. - Purged unused React Flow dependencies in favor of optimized canvas-based rendering for large graphs. +- Color architecture refactored for strict visual lane separation (fill/ring/glow/badge). - Optimized O(N) array lookups to O(1) Map lookups in several critical UI components (LearningPathTab, AnalysisTab). ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b22d8e1..bf4707a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ bun install cp .env.example .env ``` -Make sure to add your `GITHUB_TOKEN` and any AI provider API keys (`GEMINI_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `EDGEONE_API_KEY`) to the `.env` file for full feature support during development. Offline development works out-of-the-box with `AI_PROVIDER=mock`. +Make sure to add your `GITHUB_TOKEN` and any AI provider API keys (`GEMINI_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) to the `.env` file for full feature support during development. Offline development works out-of-the-box with `AI_PROVIDER=mock`. --- @@ -96,6 +96,19 @@ bunx graphify update . --- + +--- + +## Versioning Policy + +Version numbers are incremented progressively based on commit scope and change size: + +- **Major (`+1.x.x`)**: Major milestone or large-scale architectural overhaul (100+ files). +- **Minor (`x.+1.x`)**: Feature additions and substantial enhancements (`feat`). +- **Patch (`x.x.+1`)**: Bug fixes, security hardening, and targeted maintenance (`fix`, `chore`). + +When updating versions, count commits since the previous version and advance progressively per feature/fix commit rather than collapsing multiple feature commits into a single minor bump. Releases on the same date are consolidated in the changelog under that date's latest version. + ## Pull Request Guidelines 1. **Create a Branch**: diff --git a/README.md b/README.md index 801296c..1194f1d 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -

gitSdm (Git Software Dependency Map v3.0.0)

+

gitSdm — Git Software Dependency Map

- Graph-first repository analysis for exploring files, dependencies, modules, architecture diagrams, and AI-powered codebase intelligence. + Interactive dependency maps for exploring files, modules, architecture diagrams, and codebase intelligence.

+ Live Demo License PRs Welcome CI @@ -20,6 +21,34 @@ - 🌡️ **Code Churn & Complexity Overlays** — Heatmap modes visualizing 90-day commit frequency, author ownership risk, and cyclomatic/line complexity. - 🧠 **AI Codebase Intelligence** — Multi-provider AI orchestration for architecture overviews, dependency health audits, refactoring risk analysis, and interactive learning paths. - 🔍 **Semantic Search & QA Engine** — Vector-embedded code chunking with natural language codebase question-answering. + +Search indexes are process-local and scoped to GitHub credentials, commit SHA, and embedding configuration. Use a persistent backend for search; indexes are not shared between serverless instances and are lost on restart. Per process, indexing allows two active jobs with no waiting queue. Each repository is limited to 2,000 supported files, 256 KiB per file, 4,000 chunks, and 32 MiB of estimated index data. Published indexes have a 128 MiB total budget, a 16-index cap, and a 30-minute lifetime, with least-recently-used eviction. Active builds retain at most 32 MiB each in addition to published indexes. Oversized or incomplete builds fail without replacing a published snapshot. + +Embedding operations start at least `EMBEDDING_REQUEST_INTERVAL_MS` apart (default 1,000 ms per process). Provider rate limits return `429 EMBEDDING_RATE_LIMITED` immediately and pause new embedding work for the provider's `Retry-After` duration, or one minute when absent. Timeouts retry at most three attempts with bounded backoff. This pacing does not increase the provider's quota; full-repository indexing may still require a smaller scope or a higher provider quota. + +Provider calls use separate AI and embedding queues, each allowing two active calls and eight waiting calls. Waiting expires after 10 seconds; active calls have a 30-second abort deadline. Operations that ignore cancellation retain their slot until they settle. GitHub requests time out after 15 seconds. AI prompts are capped at 64,000 characters with at most 4,096 output tokens. Mock providers do not spend the hosted-provider budget. + +Repository analysis caches are scoped to the effective GitHub credential, including entries addressed by commit SHA. Repository access is checked before returning cached analysis. Raw credentials are never included in cache keys. + +Set `TOKEN_CACHE_HASH_SECRET` to a securely generated random secret in shared mode, including when Redis configuration enables shared mode automatically. Use the same secret across instances. Local mode uses a random per-process secret when the setting is blank; restarting changes cache identities. Rotating a configured secret also changes those identities. + +Usage controls combine per-IP counters with deployment-wide budgets. Each IP gets 120 expensive API requests per minute by default, including requests using user-provided credentials. Users on the same network may share this allowance. Configure these environment variables on the backend: + +| Variable | Default | Meaning | +| ----------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `API_REQUESTS_PER_MINUTE` | `600` | Shared expensive-route request budget; the per-process eight-active ceiling also applies. | +| `API_REQUESTS_PER_IP_PER_MINUTE` | `120` | Per-client expensive-route budget, checked before global admission. | +| `TRUSTED_PROXY_IPS` | unset | Comma-separated exact proxy IP addresses allowed to supply `X-Forwarded-For`. The chain is checked from right to left, stopping at the first untrusted address. | +| `SERVER_AI_DAILY_CALLS` | `500` | Server-funded completion attempts per UTC day; `0` requires user credentials. User-supplied AI keys still use the bounded queue and prompt limits. | +| `SERVER_EMBEDDING_DAILY_BYTES` | `300000000` | Reserved UTF-8 input bytes per UTC day (300 MB), shared across server-funded embedding requests. Each attempt, including retries, reserves its input size before dispatch. Reservations remain counted if a later cooldown or timeout prevents dispatch; `0` disables paid embeddings. | +| `RATE_LIMIT_MODE` | `local` | Set to `shared` for multiple processes/instances. | +| `UPSTASH_REDIS_REST_URL` / `UPSTASH_REDIS_REST_TOKEN` | unset | Redis REST connection for atomic shared reservations. Setting either also enables shared mode. | + +Direct connections use the socket IP. Without trusted-proxy configuration, proxied users share the proxy's allowance; missing socket identities share an `unknown` bucket. Configure only proxies you control that append or replace the forwarded chain. Local limiter storage caps at 10,000 live counters and rejects new counters at capacity rather than evicting active quotas. IPv6 addresses are normalized but counted individually. + + +Shared mode fails closed when its backend is missing or unavailable. Local counters reset on restart and cannot impose a deployment-wide budget across multiple instances. Configure shared mode before scaling out. These are request/input limits, not a dollar spending cap; provider pricing varies. Reservations are not refunded after failures because upstream work may have been billed. The shared implementation uses the [Redis REST API](https://upstash.com/docs/redis/features/restapi) with atomic Lua reservations. + - 📐 **Architecture Diagrams** — Automated Mermaid diagram generation with interactive zoom/pan and PNG/PDF/SVG export. - 🔀 **Branch, Tag & Commit Comparison** — Visual diffing of dependencies and module trees across Git branches, tags, and specific commit SHAs. - 👥 **Contributor Analytics & Timeline** — Commit history activity charts, timeline weeks, and author metrics. @@ -46,13 +75,38 @@ Open [http://localhost:5173](http://localhost:5173) in your browser. `gitSdm` unifies AI calls across providers with a fallback mock provider for full offline development: -| Provider | `AI_PROVIDER` | Configuration | -| --- | --- | --- | -| **Mock (Default)** | `mock` | No API key required. Uses offline fixtures. | -| **Google Gemini** | `gemini` | Set `GEMINI_API_KEY` (model: `gemini-2.5-flash`). | -| **OpenAI** | `openai` | Set `OPENAI_API_KEY` (model: `gpt-4o-mini`). | -| **Anthropic** | `anthropic` | Set `ANTHROPIC_API_KEY` (model: `claude-3-5-haiku-latest`). | -| **EdgeOne Makers** | `edgeone` | Set `EDGEONE_API_KEY` (model: `@makers/deepseek-v4-flash`). | +| Provider | `AI_PROVIDER` | Configuration | +| ------------------ | ------------- | ----------------------------------------------------------- | +| **Mock (Default)** | `mock` | No API key required. Uses offline fixtures. | +| **Google Gemini** | `gemini` | Set `GEMINI_API_KEY` (model: `gemini-2.5-flash`). | +| **OpenAI** | `openai` | Set `OPENAI_API_KEY` (model: `gpt-4o-mini`). | +| **Anthropic** | `anthropic` | Set `ANTHROPIC_API_KEY` (model: `claude-3-5-haiku-latest`). | + +### Independent embedding configuration + +Set `EMBEDDING_PROVIDER=openai`, `gemini`, or `mock` explicitly to keep semantic +search independent of `AI_PROVIDER`. Set `EMBEDDING_API_KEY`, `EMBEDDING_MODEL`, +and (for OpenAI-compatible services) `EMBEDDING_API_BASE` independently of chat. +Gemini uses its SDK endpoint. Keep `EMBEDDING_DIMENSIONS` matched to the model. + +Legacy OpenAI/Gemini key and embedding-model settings remain fallbacks. Without +`EMBEDDING_PROVIDER`, legacy selection still follows `AI_PROVIDER`. Configure all +embedding settings explicitly before changing chat credentials or endpoints. +Changing embedding model or endpoint invalidates existing search index identities. + +### Browser chat settings + +Settings accepts an AI API key, an explicit chat provider, an optional model, +and an optional OpenAI-compatible base URL. Save applies the fields together; +Clear removes the saved key and chat overrides. These settings apply to AI tasks +and Ask AI, not semantic indexing or embedding generation. Existing saved keys +remain supported. + +Custom URLs require the user's own key and a public HTTPS endpoint. Local and private +network addresses are blocked, DNS addresses are checked and pinned for the connection, +and redirects are not followed. No server allowlist configuration is needed. +Requests use per-request configuration and cache identities include provider, model, +endpoint, and hashed credentials. --- @@ -81,13 +135,13 @@ bun run test:e2e # Run Playwright end-to-end tests (mock mode) ## Documentation -| Document | Contents | -| --- | --- | -| [`ARCHITECTURE.md`](./ARCHITECTURE.md) | Folder structure, service layers, and state flow | -| [`ROADMAP.md`](./ROADMAP.md) | Planned milestones and feature proposals | +| Document | Contents | +| -------------------------------------- | --------------------------------------------------------- | +| [`ARCHITECTURE.md`](./ARCHITECTURE.md) | Folder structure, service layers, and state flow | +| [`ROADMAP.md`](./ROADMAP.md) | Planned milestones and feature proposals | | [`CONTRIBUTING.md`](./CONTRIBUTING.md) | Contribution guidelines, coding conventions, and PR guide | -| [`CHANGELOG.md`](./CHANGELOG.md) | Release history and version updates | -| [`SECURITY.md`](./SECURITY.md) | Vulnerability disclosure policy | +| [`CHANGELOG.md`](./CHANGELOG.md) | Release history and version updates | +| [`SECURITY.md`](./SECURITY.md) | Vulnerability disclosure policy | --- diff --git a/ROADMAP.md b/ROADMAP.md index de466a7..531b703 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -12,9 +12,12 @@ - **Dependency Health Report** — Core health panel, version freshness checker (npm), license compliance audits, and real-time visual highlight alerts (amber borders/warning badges) on the force-directed canvas. - **Code Churn & Hotspot Overlays** — Visual overlay for 90-day commit frequency, author ownership risk (3+ distinct authors), and LRU-cached churn service. - **Complexity Score per Module** — Graph-derived complexity metrics (LOC, import count, export count) with Color/Size by Complexity modes. -- **Unified Multi-Provider AI** — Support for Gemini, OpenAI, Anthropic, EdgeOne Makers, and an offline Mock provider. +- **Unified Multi-Provider AI** — Support for Gemini, OpenAI, Anthropic, and an offline Mock provider. - **Personalized Onboarding Paths** — AI-generated custom reading tours. User pastes a repo URL + describes their goal, and the system returns a guided walkthrough ("start with these 5 files, in this order") with graph node highlights. Implemented as `LearningPathTab`. - **Commit & Tag Snapshot Diffing** — Extended Compare Branch to accept commit SHAs and tags as comparison targets. Includes Branches/Tags/SHA sub-tabs in the picker UI, `/api/repo/tags` endpoint, and full graph overlay for any ref type. +- **Custom OpenAI-Compatible AI Endpoints** — Allows configuring custom models and public HTTPS endpoints in Settings with SSRF DNS pinning and private-address protections. +- **Decoupled Search Embedding Configuration** — Independent configuration of embedding models, credentials, and endpoints isolated from chat provider settings. +- **Resilient AI Task Lifecycle & Cache Invalidation** — Task-level promise sharing across panel remounts and automatic cache invalidation on settings changes. --- diff --git a/SECURITY.md b/SECURITY.md index 4948a4a..bef8988 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -80,6 +80,6 @@ Research conducted in good faith under this policy is authorized: ## Security Best Practices for Users -* **Environment Variables:** Never check in your `GITHUB_TOKEN`, `GEMINI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or `EDGEONE_API_KEY` into git repositories. Use the local `.env` file (which is gitignored). +* **Environment Variables:** Never check in your `GITHUB_TOKEN`, `GEMINI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY` into git repositories. Use the local `.env` file (which is gitignored). * **Private Repositories:** Ensure access tokens passed to `gitSdm` use the minimum necessary scopes (ideally read-only permissions for repository metadata/trees). * **Docker Deployments:** When running via Docker, bind the server to `127.0.0.1` unless external access is explicitly required, and configure a reverse proxy with TLS. diff --git a/api/repo/churn.ts b/api/repo/churn.ts new file mode 100644 index 0000000..33cbea9 --- /dev/null +++ b/api/repo/churn.ts @@ -0,0 +1,2 @@ +import { createVercelHandler } from '../../server/vercel-handler'; +export default createVercelHandler('/api/repo/churn'); diff --git a/api/repo/health.ts b/api/repo/health.ts new file mode 100644 index 0000000..01189fe --- /dev/null +++ b/api/repo/health.ts @@ -0,0 +1,2 @@ +import { createVercelHandler } from '../../server/vercel-handler'; +export default createVercelHandler('/api/repo/health'); diff --git a/docs/agents/domain.md b/docs/agents/domain.md new file mode 100644 index 0000000..3524904 --- /dev/null +++ b/docs/agents/domain.md @@ -0,0 +1,51 @@ +# Domain Docs + +How the engineering skills should consume this repo's domain documentation when exploring the codebase. + +## Before exploring, read these + +- **`CONTEXT.md`** at the repo root, or +- **`CONTEXT-MAP.md`** at the repo root if it exists: it points at one `CONTEXT.md` per context. Read each one relevant to the topic. +- **`docs/adr/`**: read ADRs that touch the area you're about to work in. In multi-context repos, also check `src//docs/adr/` for context-scoped decisions. + +If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved. + +## File structure + +Single-context repo (most repos): + +``` +/ +├── CONTEXT.md +├── docs/adr/ +│ ├── 0001-event-sourced-orders.md +│ └── 0002-postgres-for-write-model.md +└── src/ +``` + +Multi-context repo (presence of `CONTEXT-MAP.md` at the root): + +``` +/ +├── CONTEXT-MAP.md +├── docs/adr/ ← system-wide decisions +└── src/ + ├── ordering/ + │ ├── CONTEXT.md + │ └── docs/adr/ ← context-specific decisions + └── billing/ + ├── CONTEXT.md + └── docs/adr/ +``` + +## Use the glossary's vocabulary + +When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids. + +If the concept you need isn't in the glossary yet, that's a signal: either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`). + +## Flag ADR conflicts + +If your output contradicts an existing ADR, surface it explicitly rather than silently overriding: + +> _Contradicts ADR-0007 (event-sourced orders), but worth reopening because…_ diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 0000000..0209a19 --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,30 @@ +# Issue tracker: Local Markdown + +Issues and specs for this repo live as markdown files in `.scratch/`. + +## Conventions + +- One feature per directory: `.scratch//` +- The spec is `.scratch//spec.md` +- Implementation issues are one file per ticket at `.scratch//issues/-.md`, numbered from `01`, never a single combined tickets file +- Triage state is recorded as a `Status:` line near the top of each issue file (see `triage-labels.md` for the role strings) +- Comments and conversation history append to the bottom of the file under a `## Comments` heading + +## When a skill says "publish to the issue tracker" + +Create a new file under `.scratch//` (creating the directory if needed). + +## When a skill says "fetch the relevant ticket" + +Read the file at the referenced path. The user will normally pass the path or the issue number directly. + +## Wayfinding operations + +Used by `/wayfinder`. The **map** is a file with one **child** file per ticket. + +- **Map**: `.scratch//map.md` (the Notes / Decisions-so-far / Fog body). +- **Child ticket**: `.scratch//issues/NN-.md`, numbered from `01`, with the question in the body. A `Type:` line records the ticket type (`research`/`prototype`/`grilling`/`task`); a `Status:` line records `claimed`/`resolved`. +- **Blocking**: a `Blocked by: NN, NN` line near the top. A ticket is unblocked when every file it lists is `resolved`. +- **Frontier**: scan `.scratch//issues/` for files that are open, unblocked, and unclaimed; first by number wins. +- **Claim**: set `Status: claimed` and save before any work. +- **Resolve**: append the answer under an `## Answer` heading, set `Status: resolved`, then append a context pointer (gist + link) to the map's Decisions-so-far in `map.md`. diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md new file mode 100644 index 0000000..b716855 --- /dev/null +++ b/docs/agents/triage-labels.md @@ -0,0 +1,15 @@ +# Triage Labels + +The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker. + +| Label in mattpocock/skills | Label in our tracker | Meaning | +| -------------------------- | -------------------- | ---------------------------------------- | +| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue | +| `needs-info` | `needs-info` | Waiting on reporter for more information | +| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent | +| `ready-for-human` | `ready-for-human` | Requires human implementation | +| `wontfix` | `wontfix` | Will not be actioned | + +When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table. + +Edit the right-hand column to match whatever vocabulary you actually use. diff --git a/e2e/ai-settings-regressions.e2e.ts b/e2e/ai-settings-regressions.e2e.ts new file mode 100644 index 0000000..870ae16 --- /dev/null +++ b/e2e/ai-settings-regressions.e2e.ts @@ -0,0 +1,87 @@ +import { test, expect } from '@playwright/test'; + +test('health completion is delivered after leaving and returning to the panel', async ({ page }) => { + let release = () => {}; + const held = new Promise((resolve) => { release = resolve; }); + let calls = 0; + await page.route('**/api/ai/health', async (route) => { + calls++; + await held; + await route.fulfill({ json: { + scores: { maintainability: 80, modularity: 80, readability: 80, architecture: 80, complexity: 80 }, + summary: 'Completed after remount', cached: false, + } }); + }); + await page.route('**/api/ai/explain', (route) => route.fulfill({ json: { explanation: 'Example' } })); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await page.getByRole('button', { name: /Health Audit/ }).click(); + await expect.poll(() => calls).toBe(1); + await page.getByRole('tab', { name: 'Overview', exact: true }).click(); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + release(); + await expect(page.getByText('Completed after remount', { exact: true })).toBeVisible(); + expect(calls).toBe(1); +}); + +test('saving chat settings refreshes the active AI architecture diagram', async ({ page }) => { + let calls = 0; + await page.route('**/api/ai/mermaid', (route) => { + calls++; + return route.fulfill({ json: { diagram: `graph TD; A[Model${calls}] --> B[Done]`, cached: false } }); + }); + await page.goto('/mock/todo-app'); + await page.getByRole('button', { name: 'Architecture', exact: true }).click(); + await page.getByRole('button', { name: 'AI Enhanced', exact: true }).click(); + await expect(page.getByText('Model1', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await page.getByRole('textbox', { name: 'AI API key', exact: true }).fill('test-key'); + await page.getByRole('textbox', { name: 'Chat model', exact: true }).fill('new-model'); + await page.getByRole('button', { name: 'Save AI API key', exact: true }).click(); + await expect(page.getByText('Model2', { exact: true })).toBeVisible(); + await expect(page.getByText('Model1', { exact: true })).toHaveCount(0); + expect(calls).toBe(2); +}); + +test('an older explanation cannot hide the current failure', async ({ page }) => { + let release = () => {}; + const held = new Promise((resolve) => { release = resolve; }); + let started = false; + await page.route('**/api/ai/explain', async (route) => { + started = true; + await held; + await route.fulfill({ json: { explanation: 'Old explanation' } }); + }); + await page.route('**/api/ai/explain-lif', (route) => route.fulfill({ status: 500, json: { error: 'Failed' } })); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await expect.poll(() => started).toBe(true); + await page.getByRole('button', { name: 'ELI5 MODE', exact: true }).click(); + await expect(page.getByText('AI explanation failed. Please try again.', { exact: true })).toBeVisible(); + const response = page.waitForResponse('**/api/ai/explain'); + release(); + await response; + await expect(page.getByRole('button', { name: 'Retry Request', exact: true })).toBeVisible(); + await expect(page.getByText('Old explanation', { exact: true })).toHaveCount(0); +}); + +test('saving changed chat settings refreshes cached explanations', async ({ page }) => { + let calls = 0; + await page.route('**/api/ai/explain', (route) => { + calls++; + return route.fulfill({ json: { explanation: `Answer ${calls}` } }); + }); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await expect(page.getByText('Answer 1', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await page.getByRole('textbox', { name: 'AI API key', exact: true }).fill('test-key'); + await page.getByRole('textbox', { name: 'Chat model', exact: true }).fill('test-model'); + await page.getByRole('button', { name: 'Save AI API key', exact: true }).click(); + await expect(page.getByText('Answer 2', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Close settings', exact: true }).click(); + await page.getByRole('tab', { name: 'Overview', exact: true }).click(); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await expect(page.getByText('Answer 2', { exact: true })).toBeVisible(); + expect(calls).toBe(2); +}); diff --git a/e2e/enrichment-regressions.e2e.ts b/e2e/enrichment-regressions.e2e.ts new file mode 100644 index 0000000..aa4ca07 --- /dev/null +++ b/e2e/enrichment-regressions.e2e.ts @@ -0,0 +1,52 @@ +import { test, expect } from '@playwright/test'; + +test('404 and privacy metadata are reset when navigating home', async ({ page }) => { + const response = await page.goto('/missing-page'); + expect(response?.status()).toBe(404); + await expect(page.locator('meta[name="robots"]')).toHaveAttribute('content', 'noindex'); + await page.getByRole('link', { name: 'Go to homepage', exact: true }).click(); + await expect(page).toHaveTitle(/Git Software Dependency Map/); + await expect(page.locator('meta[name="robots"]')).toHaveAttribute('content', 'index, follow'); + await page.goto('/privacy'); + await expect(page).toHaveTitle('Privacy policy — gitSdm'); + await page.getByRole('link', { name: 'Home', exact: true }).click(); + await expect(page.locator('link[rel="canonical"]')).toHaveAttribute('href', 'https://gsdm.site/'); +}); + +test('prerendered homepage stays visible while its route chunk loads', async ({ page }) => { + let release = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + await page.route('**/assets/HomePage-*.js', async (route) => { + await gate; + await route.continue(); + }); + try { + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await expect(page.locator('h1')).toBeVisible(); + await expect(page.getByRole('link', { name: 'Privacy', exact: true })).toBeVisible(); + } finally { + release(); + } +}); + +test('full workspace restores both panels after mobile resize', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto('/mock/todo-app'); + await page.getByRole('button', { name: 'Open menu', exact: true }).click(); + await page.getByRole('button', { name: 'Full Workspace', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Collapse file explorer', exact: true })).toBeVisible(); + await page.setViewportSize({ width: 1256, height: 912 }); + await expect(page.getByRole('button', { name: 'Collapse file explorer', exact: true })).toBeVisible(); + await expect( + page.getByRole('button', { + name: 'Collapse repository insights', + exact: true, + }), + ).toBeVisible(); +}); + +test('missing assets return an actual 404', async ({ request }) => { + expect((await request.get('/assets/missing.js')).status()).toBe(404); +}); diff --git a/e2e/interface.e2e.ts b/e2e/interface.e2e.ts index 9549217..e2b0706 100644 --- a/e2e/interface.e2e.ts +++ b/e2e/interface.e2e.ts @@ -6,7 +6,7 @@ test('settings closes with Escape and restores its accessible trigger', async ({ const trigger = page.getByRole('button', { name: 'Settings and credentials', exact: true }); await trigger.click(); const dialog = page.getByRole('dialog', { name: 'Settings and credentials' }); - const key = dialog.getByLabel('Google AI key', { exact: true }); + const key = dialog.getByLabel('AI API key', { exact: true }); await expect(key).toBeFocused(); await key.press('Escape'); await expect(dialog).toHaveCount(0); @@ -23,18 +23,27 @@ test('learning Explain responds to Enter and synchronizes the inspected file', a await expect(page.getByRole('region', { name: 'File inspector' })).toContainText('src/App.tsx'); }); -test('workspace modes hide both sidebars in focus and open learning on mobile', async ({ page }) => { +test('workspace modes show only the selected sidebar', async ({ page }) => { await page.setViewportSize({ width: 1256, height: 912 }); await page.goto('/mock/todo-app'); await page.getByRole('button', { name: 'Full Workspace', exact: true }).click(); - await page.getByRole('menuitem', { name: 'Focus Mode Minimizes sidebars to focus purely on the canvas', exact: true }).click(); + await page.getByRole('menuitem', { name: /Explorer Only/ }).click(); + await expect(page.getByRole('button', { name: 'Collapse file explorer', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Open repository insights', exact: true })).toBeVisible(); + + await page.getByRole('button', { name: 'Explorer Only', exact: true }).click(); + await page.getByRole('menuitem', { name: /Insights Only/ }).click(); + await expect(page.getByRole('button', { name: 'Open file explorer', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Collapse repository insights', exact: true })).toBeVisible(); + + await page.getByRole('button', { name: 'Insights Only', exact: true }).click(); + await page.getByRole('menuitem', { name: /Focus Mode/ }).click(); await expect(page.getByRole('button', { name: 'Open file explorer', exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Open repository insights', exact: true })).toBeVisible(); await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: 'Open menu', exact: true }).click(); - await page.getByRole('button', { name: 'Learning Mode', exact: true }).click(); - await expect(page.getByRole('tab', { name: 'Learning', exact: true })).toBeVisible(); - await expect(page.getByRole('tab', { name: 'Learning', exact: true })).toHaveAttribute('aria-selected', 'true'); + await page.getByRole('button', { name: 'Insights Only', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Collapse repository insights', exact: true })).toBeVisible(); }); test('layout changes fit the graph after focusing a file', async ({ page }) => { @@ -175,3 +184,19 @@ test('panel resizing preserves the graph and inspector context follows related f await expect(inspector).toContainText('src/context/TodoContext.tsx'); await expect(page.getByRole('tabpanel', { name: 'Details' }).getByRole('heading', { name: 'TodoContext.tsx', exact: true })).toBeVisible(); }); + + test('health audit stops after an API error and retries only on request', async ({ page }) => { + let calls = 0; + await page.route('**/api/ai/**', route => { + if (route.request().url().includes('/health')) calls++; + return route.fulfill({ status: 500, json: { error: 'Internal Server Error' } }); + }); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await page.getByRole('button', { name: /Health Audit/ }).click(); + await expect(page.getByText('AI request failed. Check your settings and try again.', { exact: true })).toBeVisible(); + await page.waitForTimeout(1000); + expect(calls).toBe(1); + await page.getByRole('button', { name: 'Retry Request', exact: true }).click(); + await expect.poll(() => calls).toBe(2); + }); diff --git a/e2e/reduced-motion.e2e.ts b/e2e/reduced-motion.e2e.ts new file mode 100644 index 0000000..23748a6 --- /dev/null +++ b/e2e/reduced-motion.e2e.ts @@ -0,0 +1,69 @@ +import { test, expect } from '@playwright/test'; + +test('mounted buttons respond to motion changes without a reload', async ({ page }) => { + await page.addInitScript(() => { + localStorage.setItem('gitsdm-motion', JSON.stringify({ state: { preference: 'system' }, version: 0 })); + }); + await page.emulateMedia({ reducedMotion: 'no-preference' }); + await page.route('**/api/repo/analyze**', (route) => route.fulfill({ + status: 400, contentType: 'application/json', body: JSON.stringify({ error: 'Analysis failed', code: 'VALIDATION_ERROR' }), + })); + await page.goto('/mock/todo-app'); + const button = page.getByRole('button', { name: 'Back to Home', exact: true }); + await expect(button).toBeVisible(); + await button.hover(); + const scale = () => button.evaluate((element) => new DOMMatrix(getComputedStyle(element).transform).a); + await expect.poll(scale).toBeGreaterThan(1.01); + const original = await button.elementHandle(); + await page.emulateMedia({ reducedMotion: 'reduce' }); + await expect.poll(scale).toBe(1); + expect(await original?.evaluate((element) => element.isConnected)).toBe(true); + await page.emulateMedia({ reducedMotion: 'no-preference' }); + await page.mouse.move(0, 0); + await button.hover(); + await expect.poll(scale).toBeGreaterThan(1.01); +}); + +for (const reducedMotion of ['reduce', 'no-preference'] as const) { + test(`workspace settles and controls work with motion ${reducedMotion}`, async ({ page }) => { + await page.emulateMedia({ reducedMotion }); + await page.addInitScript(() => { + localStorage.setItem('gitsdm-motion', JSON.stringify({ state: { preference: 'system' }, version: 0 })); + }); + const errors: string[] = []; + page.on('pageerror', (error) => errors.push(error.message)); + await page.goto('/'); + await expect(page.getByRole('heading', { name: 'Understand the whole codebase.' })).toBeVisible(); + await page.goto('/mock/todo-app'); + const graph = page.locator('.graph-canvas-host').last(); + await expect(graph.locator('canvas').first()).toBeVisible({ timeout: 15000 }); + await expect(graph).not.toHaveAttribute('data-reduced-settling', '', { timeout: 15000 }); + await page.getByRole('button', { name: 'Zoom in', exact: true }).click(); + for (const layout of ['Horizontal tree', 'Force layout']) { + await page.getByRole('button', { name: 'Layout', exact: true }).click(); + await page.getByRole('button', { name: layout, exact: true }).click(); + await expect(graph).not.toHaveAttribute('data-reduced-settling', '', { timeout: 15000 }); + await expect(graph.locator('canvas').first()).toBeVisible(); + } + await page.emulateMedia({ reducedMotion: reducedMotion === 'reduce' ? 'no-preference' : 'reduce' }); + await expect(graph.locator('canvas').first()).toBeVisible(); + expect(errors).toEqual([]); + }); +} + +test('app motion choice overrides the system setting and persists on reload', async ({ page }) => { + await page.emulateMedia({ reducedMotion: 'no-preference' }); + await page.goto('/mock/todo-app'); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'Reduced', exact: true }).click(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'true'); + await page.reload(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'true'); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await expect(page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'Reduced', exact: true })).toHaveAttribute('aria-pressed', 'true'); + await page.emulateMedia({ reducedMotion: 'reduce' }); + await page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'Full', exact: true }).click(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'false'); + await page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'System', exact: true }).click(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'true'); +}); diff --git a/e2e/search-progress.e2e.ts b/e2e/search-progress.e2e.ts new file mode 100644 index 0000000..042a54a --- /dev/null +++ b/e2e/search-progress.e2e.ts @@ -0,0 +1,107 @@ +import { test, expect } from '@playwright/test'; + +test('Search, Ask and a rejected Resume retain the paused commit', async ({ page }) => { + const coverage = { kind: 'partial', commitSha: 'commit-a', indexedFiles: 32, totalFiles: 40 }; + const paused = { + state: 'paused', + reason: 'EMBEDDING_RATE_LIMITED', + error: 'Paused', + filesProcessed: 32, + totalFiles: 40, + progress: 80, + snapshotSha: 'commit-a', + coverage, + }; + const submitted: string[] = []; + await page.route('**/api/search/status?**', (route) => route.fulfill({ json: paused })); + await page.route('**/api/search', (route) => { + submitted.push(route.request().postDataJSON().branch); + return route.fulfill({ json: { results: [], cached: false, query: 'test query', coverage } }); + }); + await page.route('**/api/search/ask', (route) => { + submitted.push(route.request().postDataJSON().branch); + return route.fulfill({ + json: { answer: 'Only indexed files are covered.', citations: [], cached: false, coverage }, + }); + }); + await page.route('**/api/search/index', (route) => { + submitted.push(route.request().postDataJSON().branch); + return route.fulfill({ + status: 429, + json: { error: 'Admission limit', code: 'USAGE_LIMIT_EXCEEDED', context: { retryAfterSeconds: 60 } }, + }); + }); + await page.goto('/mock/todo-app/search'); + await expect(page.getByText('32 files indexed · 8 remaining')).toBeVisible(); + await page.getByLabel('Search codebase or ask a question').fill('test query'); + await page.getByRole('button', { name: 'Go', exact: true }).click(); + await expect.poll(() => submitted.length).toBe(1); + await page.getByRole('tab', { name: 'Ask', exact: true }).click(); + await expect.poll(() => submitted.length).toBe(2); + await page.getByRole('button', { name: 'Resume', exact: true }).click(); + await expect(page.getByText('Admission limit', { exact: true })).toBeVisible(); + await expect(page.getByText('32 files indexed · 8 remaining')).toBeVisible(); + await expect(page.getByRole('button', { name: 'Resume' })).toBeDisabled(); + await expect(page.getByLabel('Search codebase or ask a question')).toBeEnabled(); + expect(submitted).toEqual(['commit-a', 'commit-a', 'commit-a']); +}); + +test('partial coverage, cooldown, empty results and cancel are visible', async ({ page }) => { + let cancelled = false; + const coverage = { kind: 'partial', commitSha: 'snapshot', indexedFiles: 32, totalFiles: 40 }; + const paused = { + state: 'paused', + reason: 'EMBEDDING_RATE_LIMITED', + error: 'Provider rate limit', + retryAt: Date.now() + 60000, + filesProcessed: 32, + totalFiles: 40, + progress: 80, + coverage, + snapshotSha: 'snapshot', + }; + await page.route('**/api/search/status?**', (route) => route.fulfill({ json: paused })); + await page.route('**/api/search', (route) => + route.fulfill({ json: { results: [], query: 'test query', cached: false, coverage } }), + ); + await page.route('**/api/search/cancel', (route) => { + cancelled = true; + expect(route.request().postDataJSON().branch).toBe('snapshot'); + return route.fulfill({ json: { state: 'idle' } }); + }); + await page.goto('/mock/todo-app/search'); + await expect(page.getByText('32 files indexed · 8 remaining')).toBeVisible(); + await expect(page.getByRole('button', { name: 'Resume' })).toBeDisabled(); + const input = page.getByLabel('Search codebase or ask a question'); + await expect(input).toBeEnabled(); + await input.fill('test query'); + await page.getByRole('button', { name: 'Go', exact: true }).click(); + await expect(page.getByText('No matches in indexed files yet.')).toBeVisible(); + await page.getByRole('button', { name: 'Cancel', exact: true }).click(); + expect(cancelled).toBe(true); +}); + +test('cooldown resumes the same snapshot and completes', async ({ page }) => { + let resumed = false; + const status = { + state: 'paused', + reason: 'USAGE_LIMIT_EXCEEDED', + error: 'Daily budget reached', + retryAt: Date.now() + 2000, + filesProcessed: 0, + totalFiles: 40, + progress: 0, + snapshotSha: 'pinned-sha', + }; + await page.route('**/api/search/status?**', (route) => + route.fulfill({ json: resumed ? { state: 'complete', chunkCount: 40, timestamp: Date.now() } : status }), + ); + await page.route('**/api/search/index', (route) => { + resumed = true; + expect(route.request().postDataJSON().branch).toBe('pinned-sha'); + return route.fulfill({ json: { state: 'complete', chunkCount: 40, timestamp: Date.now() } }); + }); + await page.goto('/mock/todo-app/search'); + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); + expect(resumed).toBe(true); +}); diff --git a/e2e/search-races.e2e.ts b/e2e/search-races.e2e.ts new file mode 100644 index 0000000..0243a2b --- /dev/null +++ b/e2e/search-races.e2e.ts @@ -0,0 +1,132 @@ +import { test, expect } from '@playwright/test'; + +test('an expired paused build cannot auto-resume against another repository', async ({ page }) => { + const now = Date.now(); + await page.clock.install({ time: new Date(now) }); + const starts: Array<{ repo: string; branch?: string; buildId: string }> = []; + await page.route('**/api/search/index', (route) => { + starts.push(route.request().postDataJSON()); + return route.fulfill({ json: { state: 'complete', chunkCount: 1, timestamp: now } }); + }); + await page.route('**/api/search/status?**', (route) => { + const repo = new URL(route.request().url()).searchParams.get('repo'); + return route.fulfill({ json: repo === 'todo-app' ? { + state: 'paused', snapshotSha: 'repo-a-sha', buildId: 'repo-a-build', retryAt: now + 60000, + filesProcessed: 1, totalFiles: 2, progress: 50, reason: 'EMBEDDING_RATE_LIMITED', error: 'Paused', + } : { state: 'idle' } }); + }); + await page.goto('/mock/todo-app/search'); + await expect(page.getByRole('button', { name: 'Resume', exact: true })).toBeVisible(); + await page.evaluate(() => { + window.history.pushState({}, '', '/'); + window.dispatchEvent(new PopStateEvent('popstate')); + }); + await expect(page.getByRole('button', { name: 'Resume', exact: true })).toHaveCount(0); + await page.clock.fastForward(61000); + await page.evaluate(() => { + window.history.pushState({}, '', '/mock/gitsdm/search'); + window.dispatchEvent(new PopStateEvent('popstate')); + }); + await expect(page.getByRole('button', { name: 'Build Index', exact: true })).toBeVisible(); + expect(starts).toHaveLength(0); + await page.getByRole('button', { name: 'Build Index', exact: true }).click(); + await expect.poll(() => starts.length).toBe(1); + expect(starts[0].repo).toBe('gitsdm'); + expect(starts[0].branch).not.toBe('repo-a-sha'); + expect(starts[0].buildId).not.toBe('repo-a-build'); +}); + +test('a paused index belongs only to its repository', async ({ page }) => { + const nextRequests: string[] = []; + await page.route('**/api/search/status?**', (route) => { + const url = new URL(route.request().url()); + if (url.searchParams.get('repo') === 'gitsdm') { + nextRequests.push(url.search); + return route.fulfill({ json: { state: 'idle' } }); + } + return route.fulfill({ json: { state: 'paused', snapshotSha: 'old-repository-sha', buildId: 'old-build', + filesProcessed: 1, totalFiles: 2, progress: 50, reason: 'EMBEDDING_RATE_LIMITED', error: 'Paused' } }); + }); + await page.goto('/mock/todo-app/search'); + await expect(page.getByRole('button', { name: 'Resume', exact: true })).toBeVisible(); + await page.evaluate(() => { + window.history.pushState({}, '', '/mock/gitsdm/search'); + window.dispatchEvent(new PopStateEvent('popstate')); + }); + await expect(page.getByRole('button', { name: 'Build Index', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Resume', exact: true })).toHaveCount(0); + await expect.poll(() => nextRequests.length).toBeGreaterThan(0); + expect(nextRequests.at(-1)).not.toContain('old-repository-sha'); +}); + +test('cancel never restarts from cached paused status while refresh is delayed', async ({ page }) => { + let cancelled = false; + let starts = 0; + const paused = { + state: 'paused', + reason: 'EMBEDDING_RATE_LIMITED', + error: 'Paused', + retryAt: Date.now() + 4000, + snapshotSha: 'commit-a', + filesProcessed: 1, + totalFiles: 2, + progress: 50, + }; + await page.route('**/api/search/status?**', async (route) => { + if (cancelled) { + await new Promise((resolve) => setTimeout(resolve, 6000)); + await route.fulfill({ json: { state: 'idle' } }); + } else await route.fulfill({ json: paused }); + }); + await page.route('**/api/search/cancel', (route) => { + cancelled = true; + return route.fulfill({ json: { state: 'idle' } }); + }); + await page.route('**/api/search/index', (route) => { + starts++; + return route.fulfill({ json: { state: 'complete', chunkCount: 2, timestamp: Date.now() } }); + }); + await page.goto('/mock/todo-app/search'); + await page.getByRole('button', { name: 'Cancel', exact: true }).click(); + await page.waitForTimeout(5000); // Observe past the old retry deadline. + expect(starts).toBe(0); + await expect(page.getByRole('button', { name: 'Build Index', exact: true })).toBeVisible(); +}); + +test('a delayed idle GET cannot hide indexing or stop progress polling', async ({ page }) => { + let releaseStatus!: () => void; + let releaseIndex!: () => void; + const statusGate = new Promise((resolve) => { + releaseStatus = resolve; + }); + const indexGate = new Promise((resolve) => { + releaseIndex = resolve; + }); + let calls = 0; + let started = false; + let complete = false; + const progress = { state: 'indexing', progress: 50, filesProcessed: 1, totalFiles: 2, snapshotSha: 'commit-a' }; + const done = { state: 'complete', chunkCount: 2, timestamp: Date.now() }; + await page.route('**/api/search/status?**', async (route) => { + calls++; + if (!started) { + await statusGate; + await route.fulfill({ json: { state: 'idle' } }); + } else await route.fulfill({ json: complete ? done : progress }); + }); + await page.route('**/api/search/index', async (route) => { + started = true; + await indexGate; + complete = true; + await route.fulfill({ json: done }); + }); + await page.goto('/mock/todo-app/search'); + await expect.poll(() => calls).toBeGreaterThanOrEqual(1); + await page.getByRole('button', { name: 'Build Index', exact: true }).click(); + await expect.poll(() => started).toBe(true); + releaseStatus(); + await expect(page.getByRole('button', { name: 'Cancel', exact: true })).toBeVisible(); + await expect(page.getByText('1 files indexed · 1 remaining')).toBeVisible({ timeout: 8000 }); + releaseIndex(); + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); +}); diff --git a/e2e/search.e2e.ts b/e2e/search.e2e.ts index d8df0d2..0a2d889 100644 --- a/e2e/search.e2e.ts +++ b/e2e/search.e2e.ts @@ -1,6 +1,64 @@ import { test, expect } from '@playwright/test'; +test('index completion, scoped search, and status agree', async ({ request }) => { + const headers = { 'X-GitHub-Token': 'offline-search-test' }; + const data = { owner: 'mock', repo: 'todo-app' }; + const indexed = await request.post('/api/search/index', { data, headers }); + expect(indexed.status()).toBe(200); + expect(await indexed.json()).toMatchObject({ state: 'complete' }); + const status = await request.get('/api/search/status?owner=mock&repo=todo-app', { headers }); + expect(await status.json()).toMatchObject({ state: 'complete' }); + const found = await request.post('/api/search', { data: { ...data, query: 'todo state' }, headers }); + expect(found.status()).toBe(200); + const isolated = await request.post('/api/search', { + data: { ...data, query: 'todo state' }, + headers: { 'X-GitHub-Token': 'different-user' }, + }); + expect(isolated.status()).toBe(404); +}); + test.describe('Search', () => { + test('idle status does not poll and an existing index is restored on entry', async ({ page }) => { + let calls = 0; + await page.route('**/api/search/status?**', (route) => { + calls++; + return route.fulfill({ json: { state: 'idle' } }); + }); + await page.goto('/mock/todo-app/search'); + await expect.poll(() => calls).toBe(1); + // Observe beyond the old three-second polling interval. + await page.waitForTimeout(3500); + expect(calls).toBe(1); + await page.unroute('**/api/search/status?**'); + await page.route('**/api/search/status?**', (route) => + route.fulfill({ + json: { state: 'complete', chunkCount: 12, timestamp: Date.now() }, + }), + ); + await page.reload(); + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); + await expect(page.getByLabel('Search codebase or ask a question')).toBeEnabled(); + }); + test('build completion enables search and a missing snapshot offers rebuilding', async ({ page }) => { + await page.goto('/mock/todo-app/search'); + await expect(page.getByRole('region', { name: 'Indexing status' })).toBeVisible(); + const isReady = await page.getByText('Index ready', { exact: true }).isVisible(); + if (!isReady) { + await page.getByRole('button', { name: 'Build Index', exact: true }).click(); + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); + } + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); + await page.route('**/api/search', (route) => + route.fulfill({ + status: 404, + contentType: 'application/json', + body: JSON.stringify({ error: 'Index expired. Please rebuild.', code: 'INDEX_NOT_FOUND' }), + }), + ); + await page.getByLabel('Search codebase or ask a question').fill('todo state'); + await page.getByRole('button', { name: 'Go', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Build Index', exact: true })).toBeVisible(); + }); test('search page loads and has search input', async ({ page }) => { // Navigate to a repo's search page await page.goto('/mock/gitsdm/search', { timeout: 15000 }); @@ -9,5 +67,4 @@ test.describe('Search', () => { const searchInput = page.locator('input[type="search"], input[placeholder*="Search"i], input[placeholder*="Ask"i]'); await expect(searchInput.first()).toBeVisible({ timeout: 15000 }); }); - }); diff --git a/index.html b/index.html index c50f653..24c98a9 100644 --- a/index.html +++ b/index.html @@ -5,30 +5,41 @@ - + gitSdmᵝ — Git Software Dependency Map + + + + + + + + + + + + + + + + + + + + + - - gitSdmᵝ — Git Software Dependency Map - + + diff --git a/package.json b/package.json index 43a34dc..10fca99 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { "name": "gitsdm", "private": true, - "version": "3.0.0", + "version": "3.5.11", "type": "module", "scripts": { "dev": "bunx --bun vite", - "build": "bunx --bun vite build", + "build": "bunx --bun vite build && bun scripts/prerender.tsx", "build:server": "bunx --bun vite build --ssr server/prod-server.ts --outDir dist-server", "build:docker": "bun run build && bun run build:server", "start": "bun dist-server/prod-server.js", @@ -78,4 +78,4 @@ "esbuild", "protobufjs" ] -} \ No newline at end of file +} diff --git a/playwright.config.ts b/playwright.config.ts index c3d9d08..bece714 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -15,6 +15,11 @@ export default defineConfig({ port: 3000, timeout: 60000, reuseExistingServer: !process.env.CI, - env: { AI_PROVIDER: 'mock' }, + env: { + AI_PROVIDER: 'mock', + EMBEDDING_PROVIDER: 'mock', + API_REQUESTS_PER_MINUTE: '10000', + API_REQUESTS_PER_IP_PER_MINUTE: '10000', + }, }, }); diff --git a/public/og-image.png b/public/og-image.png new file mode 100644 index 0000000..59d3a24 Binary files /dev/null and b/public/og-image.png differ diff --git a/public/og-image.svg b/public/og-image.svg new file mode 100644 index 0000000..9e3ba91 --- /dev/null +++ b/public/og-image.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + gitSdm + + + v3.0 + + + + Interactive Repository Intelligence + + + + Explore files, dependencies, modules, and architecture in your browser. + + + + + + Interactive Graph + + + Mermaid Diagrams + + + Semantic Search + + + AI Insights + + diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..cdcff03 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Allow: / +Disallow: /api/ + +Sitemap: https://gsdm.site/sitemap.xml diff --git a/public/sitemap.xml b/public/sitemap.xml new file mode 100644 index 0000000..b9b639f --- /dev/null +++ b/public/sitemap.xml @@ -0,0 +1,26 @@ + + + + https://gsdm.site/terms + monthly + + + https://gsdm.site/privacy + monthly + + + https://gsdm.site/ + weekly + 1.0 + + + https://gsdm.site/mock/gitsdm + monthly + 0.8 + + + https://gsdm.site/mock/todo-app + monthly + 0.8 + + diff --git a/scripts/prerender.tsx b/scripts/prerender.tsx new file mode 100644 index 0000000..9fb89e6 --- /dev/null +++ b/scripts/prerender.tsx @@ -0,0 +1,47 @@ +import { renderToStaticMarkup } from 'react-dom/server'; +import { StaticRouter } from 'react-router-dom'; +import { Providers } from '../src/app/providers'; +import { HomePage } from '../src/pages/HomePage'; +import { PrivacyPage } from '../src/pages/PrivacyPage'; +import { TermsPage } from '../src/pages/TermsPage'; +import { NotFoundPage } from '../src/pages/NotFoundPage'; +import { pageMetadata } from '../src/lib/page-metadata'; + +const shell = await Bun.file('dist/index.html').text(); +if (!shell.includes('

')) throw new Error('Prerender root marker missing'); +const appHtml = shell + .replace(/]*>/, '') + .replace(/]*>/, '') + .replace(/[^<]*<\/title>/, '<title>gitSdm — Repository Explorer'); +await Bun.write('dist/app.html', appHtml); +for (const [route, file, page] of [ + ['/', 'dist/index.html', ], + ['/privacy', 'dist/privacy.html', ], + ['/terms', 'dist/terms.html', ], + ['/not-found', 'dist/404.html', ], +] as const) { + const meta = pageMetadata(route); + const content = renderToStaticMarkup( + + {page} + , + ); + let html = shell + .replace('
', `
${content}
`) + .replace(/[^<]*<\/title>/, `<title>${meta.title}`) + .replace(/]*>/, meta.canonical ? `` : '') + .replace('', ``); + for (const [attribute, name, value] of [ + ['name', 'description', meta.description], + ['property', 'og:title', meta.title], + ['property', 'og:description', meta.description], + ['property', 'og:url', `https://gsdm.site${route}`], + ['name', 'twitter:title', meta.title], + ['name', 'twitter:description', meta.description], + ]) + html = html.replace( + new RegExp(`]*>`), + ``, + ); + await Bun.write(file, html); +} diff --git a/server/ai/chat-config.test.ts b/server/ai/chat-config.test.ts new file mode 100644 index 0000000..1201689 --- /dev/null +++ b/server/ai/chat-config.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from 'bun:test'; +import { chatIdentity, chatOverrides, readChatOverrides, resolveChatConfig } from './chat-config'; + +test('custom URLs require a user key and a valid HTTPS endpoint', () => { + const headers = new Headers({ + 'x-ai-provider': 'openai', + 'x-ai-base-url': 'https://api.openai.com/v1', + 'x-ai-model': 'custom-model', + }); + expect(() => readChatOverrides(headers)).toThrow('own API key'); + expect(readChatOverrides(headers, 'user-key').model).toBe('custom-model'); + for (const url of ['http://127.0.0.1/v1', 'https://api.openai.com/v1?redirect=1']) { + headers.set('x-ai-base-url', url); + expect(() => readChatOverrides(headers, 'user-key')).toThrow(); + } +}); + +test('concurrent request overrides stay isolated and distinguish cache identities', async () => { + const run = (model: string) => + chatOverrides.run({ provider: 'openai', model, baseURL: 'https://api.openai.com/v1' }, async () => { + await Promise.resolve(); + return { config: resolveChatConfig('custom-key'), identity: chatIdentity('custom-key') }; + }); + const [a, b] = await Promise.all([run('model-a'), run('model-b')]); + expect(a.config.model).toBe('model-a'); + expect(b.config.model).toBe('model-b'); + expect(a.identity).not.toBe(b.identity); + expect(a.identity).not.toContain('custom-key'); +}); + +test('endpoint and credential changes invalidate chat identity', () => { + const a = chatOverrides.run({ provider: 'openai', baseURL: 'https://one.example/v1' }, () => chatIdentity('key-one')); + const b = chatOverrides.run({ provider: 'openai', baseURL: 'https://two.example/v1' }, () => chatIdentity('key-one')); + const c = chatOverrides.run({ provider: 'openai', baseURL: 'https://one.example/v1' }, () => chatIdentity('key-two')); + expect(a).not.toBe(b); + expect(a).not.toBe(c); +}); diff --git a/server/ai/chat-config.ts b/server/ai/chat-config.ts new file mode 100644 index 0000000..5f579a3 --- /dev/null +++ b/server/ai/chat-config.ts @@ -0,0 +1,77 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { hashToken } from '../cache/lru'; +import { AppError } from '../utils/errors'; + +export interface ChatOverrides { + provider?: string; + model?: string; + baseURL?: string; +} +export const chatOverrides = new AsyncLocalStorage(); +const clean = (value?: string | null) => value?.trim() || undefined; + +export function readChatOverrides(headers: Headers, apiKey?: string): ChatOverrides { + const cleanKey = clean(apiKey); + const provider = clean(headers.get('x-ai-provider')); + const model = clean(headers.get('x-ai-model')); + const baseURL = clean(headers.get('x-ai-base-url')); + if (!provider && !model && !baseURL) return {}; + if (!cleanKey) throw new AppError(400, 'Custom AI settings require your own API key.', 'INVALID_AI_CONFIG'); + if (provider && !['openai', 'gemini', 'anthropic'].includes(provider)) + throw new AppError(400, 'Choose a supported AI provider.', 'INVALID_AI_CONFIG'); + if (model && !/^[\x21-\x7e]{1,200}$/.test(model)) + throw new AppError(400, 'Model must contain 1–200 printable characters without spaces.', 'INVALID_AI_CONFIG'); + if (baseURL) { + if (provider !== 'openai') + throw new AppError(400, 'Custom base URLs require OpenAI-compatible chat.', 'INVALID_AI_CONFIG'); + let url: URL; + try { + url = new URL(baseURL); + } catch { + throw new AppError(400, 'Enter a valid HTTPS base URL.', 'INVALID_AI_CONFIG'); + } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || baseURL.length > 2048) + throw new AppError(400, 'Enter a public HTTPS base URL without credentials, query, or fragment.', 'INVALID_AI_CONFIG'); + return { provider, model, baseURL: url.href.replace(/\/+$/, '') }; + } + return { provider, model }; +} + +export function resolveChatConfig(rawOverrideKey?: string) { + const overrideKey = clean(rawOverrideKey); + const custom = overrideKey ? chatOverrides.getStore() : undefined; + const provider = + custom?.provider ?? + (overrideKey + ? overrideKey!.trim().startsWith('sk-ant-') + ? 'anthropic' + : overrideKey!.trim().startsWith('sk-') + ? 'openai' + : 'gemini' + : (clean(process.env.AI_PROVIDER)?.toLowerCase() ?? + (clean(process.env.GEMINI_API_KEY) + ? 'gemini' + : clean(process.env.OPENAI_API_KEY) + ? 'openai' + : clean(process.env.ANTHROPIC_API_KEY) + ? 'anthropic' + : 'mock'))); + const prefix = provider.toUpperCase(); + const defaults: Record = { + openai: 'gpt-4o-mini', + gemini: 'gemini-2.5-flash', + anthropic: 'claude-3-5-haiku-latest', + mock: 'mock', + }; + return { + provider, + apiKey: overrideKey ?? clean(process.env[`${prefix}_API_KEY`]), + model: custom?.model ?? clean(process.env[`${prefix}_MODEL`]) ?? defaults[provider], + baseURL: custom?.baseURL ?? clean(process.env[`${prefix}_API_BASE`]), + apiVersion: clean(process.env.GEMINI_API_VERSION) ?? 'v1alpha', + }; +} + +export function chatIdentity(key?: string): string { + return hashToken(JSON.stringify(resolveChatConfig(key))); +} diff --git a/server/ai/provider-controls.test.ts b/server/ai/provider-controls.test.ts new file mode 100644 index 0000000..51c59d2 --- /dev/null +++ b/server/ai/provider-controls.test.ts @@ -0,0 +1,69 @@ +import { test, expect } from 'bun:test'; +import { protectAI } from './provider-controls'; +import { protectEmbeddings } from '../search/embedding-controls'; + +test('server-funded AI is blocked at zero budget, while a user key can still run', async () => { + const previous = process.env.SERVER_AI_DAILY_CALLS; + process.env.SERVER_AI_DAILY_CALLS = '0'; + let called = 0; + const provider = { + complete: async () => { + called++; + return 'answer'; + }, + }; + try { + await expect(protectAI(provider, true).complete([{ role: 'user', content: 'hello' }])).rejects.toMatchObject({ + code: 'USAGE_LIMIT_EXCEEDED', + }); + expect(called).toBe(0); + expect(await protectAI(provider, false).complete([{ role: 'user', content: 'hello' }])).toBe('answer'); + expect(called).toBe(1); + await expect( + protectAI(provider, false).complete([{ role: 'user', content: 'x'.repeat(64001) }]), + ).rejects.toMatchObject({ status: 413 }); + } finally { + if (previous === undefined) delete process.env.SERVER_AI_DAILY_CALLS; + else process.env.SERVER_AI_DAILY_CALLS = previous; + } +}); + +test('a pre-aborted caller signal rejects without invoking the provider', async () => { + let called = false; + const provider = { + complete: async () => { + called = true; + return 'answer'; + }, + }; + const controller = new AbortController(); + controller.abort(new Error('cancelled')); + await expect( + protectAI(provider, false).complete([{ role: 'user', content: 'hello' }], { signal: controller.signal }), + ).rejects.toThrow('cancelled'); + expect(called).toBe(false); +}); + +test('embedding budget is checked before invoking the provider', async () => { + const previous = process.env.SERVER_EMBEDDING_DAILY_BYTES; + process.env.SERVER_EMBEDDING_DAILY_BYTES = '0'; + let called = false; + const embed = async () => { + called = true; + return { vector: new Float32Array([1]), tokenCount: 1 }; + }; + const provider = protectEmbeddings({ + dimensions: 1, + maxTokens: 512, + providerName: 'test', + embed, + embedBatch: async () => [await embed()], + }); + try { + await expect(provider.embed('hello')).rejects.toMatchObject({ code: 'USAGE_LIMIT_EXCEEDED' }); + expect(called).toBe(false); + } finally { + if (previous === undefined) delete process.env.SERVER_EMBEDDING_DAILY_BYTES; + else process.env.SERVER_EMBEDDING_DAILY_BYTES = previous; + } +}); diff --git a/server/ai/provider-controls.ts b/server/ai/provider-controls.ts new file mode 100644 index 0000000..c29fa43 --- /dev/null +++ b/server/ai/provider-controls.ts @@ -0,0 +1,20 @@ +import type { AIProvider } from './provider'; +import { createBoundedQueue } from '../utils/bounded-queue'; +import { configuredLimit, reserveUsage } from '../utils/usage-limits'; +import { AppError } from '../utils/errors'; + +const queue = createBoundedQueue(); +export function protectAI(provider: AIProvider, serverFunded: boolean): AIProvider { + return { + complete(messages, options) { + if (messages.reduce((sum, message) => sum + message.content.length, 0) > 64000) + return Promise.reject(new AppError(413, 'AI prompt is too large.', 'PROMPT_TOO_LARGE')); + // Forward the caller signal so request aborts reach the provider instead of hanging a queue slot. + return queue(async (signal) => { + if (serverFunded) await reserveUsage('ai-daily', 1, configuredLimit('SERVER_AI_DAILY_CALLS', 500), 86400000); + if (signal.aborted) throw new AppError(504, 'Provider request timed out.', 'PROVIDER_TIMEOUT', true); + return provider.complete(messages, { ...options, signal }); + }, options?.signal); + }, + }; +} diff --git a/server/ai/provider.test.ts b/server/ai/provider.test.ts index 91c8abe..25607d3 100644 --- a/server/ai/provider.test.ts +++ b/server/ai/provider.test.ts @@ -45,12 +45,11 @@ describe('ai provider', () => { delete process.env.GEMINI_API_KEY; delete process.env.OPENAI_API_KEY; delete process.env.ANTHROPIC_API_KEY; - delete process.env.EDGEONE_API_KEY; - delete process.env.MAKERS_MODELS_KEY; delete process.env.GEMINI_MODEL; delete process.env.OPENAI_MODEL; delete process.env.ANTHROPIC_MODEL; - delete process.env.EDGEONE_MODEL; + delete process.env.OPENAI_API_BASE; + delete process.env.ANTHROPIC_API_BASE; }); afterEach(() => { @@ -101,17 +100,24 @@ describe('ai provider', () => { { role: 'user', content: 'now' }, ])).toBe('anthropic-ok'); - delete process.env.ANTHROPIC_API_KEY; - process.env.EDGEONE_API_KEY = 'sk-eo-env'; - process.env.EDGEONE_MODEL = 'deepseek-test'; - provider = await createProvider(); - expect(await provider.complete([{ role: 'user', content: 'now' }], { json: true })).toContain('openai'); + }); + + it('rejects env-configured non-HTTPS or private provider endpoints', async () => { + process.env.AI_PROVIDER = 'openai'; + process.env.OPENAI_API_KEY = 'sk-env'; + for (const base of ['http://openai.test', 'https://10.0.0.5', 'https://169.254.169.254']) { + process.env.OPENAI_API_BASE = base; + await expect(createProvider()).rejects.toMatchObject({ code: 'INVALID_AI_CONFIG' }); + } + process.env.AI_PROVIDER = 'anthropic'; + process.env.ANTHROPIC_API_KEY = 'sk-ant-env'; + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_API_BASE; + process.env.ANTHROPIC_API_BASE = 'http://anthropic.test'; + await expect(createProvider()).rejects.toMatchObject({ code: 'INVALID_AI_CONFIG' }); }); it('uses override key detection and validates missing env keys', async () => { - process.env.AI_PROVIDER = 'edgeone'; - expect(await (await createProvider('sk-eo-key')).complete([{ role: 'user', content: 'x' }])).toContain('openai'); - delete process.env.AI_PROVIDER; expect(await (await createProvider('sk-openai')).complete([{ role: 'user', content: 'x' }])).toContain('openai'); expect(await (await createProvider('sk-ant-anthropic')).complete([{ role: 'user', content: 'x' }])).toBe('anthropic-ok'); @@ -123,8 +129,6 @@ describe('ai provider', () => { process.env.AI_PROVIDER = 'openai'; await expect(createProvider()).rejects.toThrow('OPENAI_API_KEY'); - process.env.AI_PROVIDER = 'edgeone'; - await expect(createProvider()).rejects.toThrow('EDGEONE_API_KEY'); process.env.AI_PROVIDER = 'anthropic'; await expect(createProvider()).rejects.toThrow('ANTHROPIC_API_KEY'); @@ -147,4 +151,4 @@ describe('ai provider', () => { const b = await getAIProvider('gemini-key'); expect(a).not.toBe(b); }); -}); \ No newline at end of file +}); diff --git a/server/ai/provider.ts b/server/ai/provider.ts index a294431..6a84826 100644 --- a/server/ai/provider.ts +++ b/server/ai/provider.ts @@ -1,3 +1,9 @@ +import { fetchPublicChat } from './public-chat-fetch'; +import { chatOverrides } from './chat-config'; +import { resolveChatConfig, chatIdentity } from './chat-config'; +import { protectAI } from './provider-controls'; +import { isSafeRemoteUrl } from '../utils/url-guard'; +import { AppError } from '../utils/errors'; import { createMockProvider } from './mock-provider'; export interface Message { @@ -6,221 +12,165 @@ export interface Message { } export interface AIProvider { - complete(messages: Message[], options?: { json?: boolean }): Promise; + complete(messages: Message[], options?: { json?: boolean; signal?: AbortSignal }): Promise; } -function detectProviderType(key: string): 'gemini' | 'openai' | 'anthropic' | 'edgeone' { - const trimmed = key.trim(); - if (trimmed.startsWith('sk-ant-')) { - return 'anthropic'; - } - if (trimmed.startsWith('sk-')) { - if (process.env.AI_PROVIDER?.toLowerCase() === 'edgeone') { - return 'edgeone'; - } - return 'openai'; - } - return 'gemini'; -} - -export async function createProvider(overrideKey?: string): Promise { - if (overrideKey && overrideKey.trim()) { - const type = detectProviderType(overrideKey); - switch (type) { - case 'edgeone': - return createEdgeOneProvider(overrideKey); - case 'openai': - return createOpenAIProvider(overrideKey); - case 'anthropic': - return createAnthropicProvider(overrideKey); - case 'gemini': - default: - return createGeminiProvider(overrideKey); - } - } - - // Auto-detect provider based on available environment API keys - let providerType: 'gemini' | 'openai' | 'anthropic' | 'edgeone' | 'mock' = 'mock'; - - // AI_PROVIDER takes explicit precedence over key-based auto-detection - if (process.env.AI_PROVIDER) { - const envProvider = process.env.AI_PROVIDER.toLowerCase(); - if ( - envProvider === 'gemini' || - envProvider === 'openai' || - envProvider === 'anthropic' || - envProvider === 'edgeone' || - envProvider === 'mock' - ) { - providerType = envProvider as 'gemini' | 'openai' | 'anthropic' | 'edgeone' | 'mock'; - } - } else if (process.env.EDGEONE_API_KEY && process.env.EDGEONE_API_KEY.trim()) { - providerType = 'edgeone'; - } else if (process.env.MAKERS_MODELS_KEY && process.env.MAKERS_MODELS_KEY.trim()) { - providerType = 'edgeone'; - } else if (process.env.GEMINI_API_KEY && process.env.GEMINI_API_KEY.trim()) { - providerType = 'gemini'; - } else if (process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY.trim()) { - providerType = 'openai'; - } else if (process.env.ANTHROPIC_API_KEY && process.env.ANTHROPIC_API_KEY.trim()) { - providerType = 'anthropic'; - } - - switch (providerType) { - case 'edgeone': - return createEdgeOneProvider(); +export async function createProvider(rawOverrideKey?: string): Promise { + const overrideKey = rawOverrideKey?.trim() || undefined; + const { provider } = resolveChatConfig(overrideKey); + switch (provider) { case 'openai': - return createOpenAIProvider(); - case 'anthropic': - return createAnthropicProvider(); + return createOpenAIProvider(overrideKey); case 'gemini': - return createGeminiProvider(); - default: + return createGeminiProvider(overrideKey); + case 'anthropic': + return createAnthropicProvider(overrideKey); + case 'mock': return createMockProvider(); + default: + throw new Error('Unsupported AI_PROVIDER. Use gemini, openai, anthropic, or mock.'); } } async function createGeminiProvider(overrideKey?: string): Promise { const { GoogleGenAI } = await import('@google/genai'); - const apiKey = overrideKey ?? process.env.GEMINI_API_KEY; - const model = process.env.GEMINI_MODEL ?? 'gemini-2.5-flash'; - const apiVersion = process.env.GEMINI_API_VERSION ?? 'v1alpha'; + const { apiKey, model, apiVersion } = resolveChatConfig(overrideKey); if (!apiKey) { throw new Error('GEMINI_API_KEY is required when using Gemini provider'); } - const ai = new GoogleGenAI({ apiKey, apiVersion }); - - return { - async complete(messages, options) { - const systemMessage = messages.find((m) => m.role === 'system'); - const systemInstruction = systemMessage?.content; - - const contents = messages - .filter((m) => m.role !== 'system') - .map((m) => ({ - role: (m.role === 'assistant' ? 'model' : 'user') as 'model' | 'user', - parts: [{ text: m.content }], - })); - - const response = await ai.models.generateContent({ - model, - contents, - config: { - maxOutputTokens: 4096, - temperature: 0.2, - systemInstruction, - responseMimeType: options?.json ? 'application/json' : undefined, - }, - }); - - return response.text ?? ''; + const ai = new GoogleGenAI({ apiKey, apiVersion, httpOptions: { timeout: 30000, retryOptions: { attempts: 1 } } }); + + return protectAI( + { + async complete(messages, options) { + const systemMessage = messages.find((m) => m.role === 'system'); + const systemInstruction = systemMessage?.content; + + const contents = messages + .filter((m) => m.role !== 'system') + .map((m) => ({ + role: (m.role === 'assistant' ? 'model' : 'user') as 'model' | 'user', + parts: [{ text: m.content }], + })); + + const response = await ai.models.generateContent({ + model, + contents, + config: { + abortSignal: options?.signal, + maxOutputTokens: 4096, + temperature: 0.2, + systemInstruction, + responseMimeType: options?.json ? 'application/json' : undefined, + }, + }); + + return response.text ?? ''; + }, }, - }; + !overrideKey, + ); +} + +function assertPublicHttpsBaseURL(baseURL: string | undefined): asserts baseURL is string | undefined { + // ponytail: env-configured bases skip readChatOverrides() validation, so enforce the + // same public-HTTPS rule here before any SDK sends the key there. + if (baseURL && !isSafeRemoteUrl(baseURL)) + throw new AppError(400, 'AI endpoint must be a public HTTPS URL.', 'INVALID_AI_CONFIG'); } async function createOpenAIProvider(overrideKey?: string): Promise { const { default: OpenAI } = await import('openai'); - const apiKey = overrideKey ?? process.env.OPENAI_API_KEY; + const { apiKey, model, baseURL } = resolveChatConfig(overrideKey); if (!apiKey) { throw new Error('OPENAI_API_KEY is required when using OpenAI provider'); } - const client = new OpenAI({ apiKey }); - if (process.env.OPENAI_API_BASE) { - client.baseURL = process.env.OPENAI_API_BASE; + assertPublicHttpsBaseURL(baseURL); + const client = new OpenAI({ + apiKey, + timeout: 30000, + maxRetries: 0, + fetch: overrideKey && chatOverrides.getStore()?.baseURL + ? fetchPublicChat + : baseURL + ? fetchPublicChat + : (input, init) => fetch(input, { ...init, redirect: 'error' }), + }); + if (baseURL) { + client.baseURL = baseURL; } - const model = process.env.OPENAI_MODEL ?? 'gpt-4o-mini'; - - return { - async complete(messages, options) { - const response = await client.chat.completions.create({ - model, - messages: messages.map((m) => ({ role: m.role, content: m.content })), - max_tokens: 4096, - temperature: 0.2, - response_format: options?.json ? { type: 'json_object' } : undefined, - }); - return response.choices[0]?.message?.content ?? ''; - }, - }; -} -async function createEdgeOneProvider(overrideKey?: string): Promise { - // ponytail: EdgeOne Makers Models exposes OpenAI-compatible endpoint. Reuse openai SDK client. - const { default: OpenAI } = await import('openai'); - const apiKey = overrideKey ?? (process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim()); - if (!apiKey) { - throw new Error('EDGEONE_API_KEY or MAKERS_MODELS_KEY is required when using EdgeOne provider'); - } - const baseURL = process.env.EDGEONE_API_BASE ?? 'https://ai-gateway.edgeone.link/v1'; - const model = process.env.EDGEONE_MODEL ?? '@makers/deepseek-v4-flash'; - const client = new OpenAI({ apiKey, baseURL }); - - return { - async complete(messages, options) { - const response = await client.chat.completions.create({ - model, - messages: messages.map((m) => ({ role: m.role, content: m.content })), - max_tokens: 4096, - temperature: 0.2, - response_format: options?.json ? { type: 'json_object' } : undefined, - }); - return response.choices[0]?.message?.content ?? ''; + return protectAI( + { + async complete(messages, options) { + const response = await client.chat.completions.create( + { + model, + messages: messages.map((m) => ({ role: m.role, content: m.content })), + max_tokens: 4096, + temperature: 0.2, + response_format: options?.json ? { type: 'json_object' } : undefined, + }, + { signal: options?.signal }, + ); + return response.choices[0]?.message?.content ?? ''; + }, }, - }; + !overrideKey, + ); } async function createAnthropicProvider(overrideKey?: string): Promise { const Anthropic = (await import('@anthropic-ai/sdk')).default; - const apiKey = overrideKey ?? process.env.ANTHROPIC_API_KEY; + const { apiKey, model, baseURL } = resolveChatConfig(overrideKey); if (!apiKey) { throw new Error('ANTHROPIC_API_KEY is required when using Anthropic provider'); } - const client = new Anthropic({ apiKey }); - if (process.env.ANTHROPIC_API_BASE) { - client.baseURL = process.env.ANTHROPIC_API_BASE; + assertPublicHttpsBaseURL(baseURL); + const client = new Anthropic({ apiKey, timeout: 30000, maxRetries: 0, fetch: baseURL ? fetchPublicChat : undefined }); + if (baseURL) { + client.baseURL = baseURL; } - const model = process.env.ANTHROPIC_MODEL ?? 'claude-3-5-haiku-latest'; - - return { - async complete(messages) { - const system = messages.find((m) => m.role === 'system')?.content ?? ''; - const userMessages = messages.filter((m) => m.role !== 'system'); - - const response = await client.messages.create({ - model, - max_tokens: 4096, - system, - messages: userMessages.map((m) => ({ - role: m.role === 'assistant' ? 'assistant' : 'user', - content: m.content, - })), - }); - - const block = response.content[0]; - return block?.type === 'text' ? block.text : ''; + + return protectAI( + { + async complete(messages, options) { + const system = messages.find((m) => m.role === 'system')?.content ?? ''; + const userMessages = messages.filter((m) => m.role !== 'system'); + + const response = await client.messages.create( + { + model, + max_tokens: 4096, + system, + messages: userMessages.map((m) => ({ + role: m.role === 'assistant' ? 'assistant' : 'user', + content: m.content, + })), + }, + { signal: options?.signal }, + ); + + const block = response.content[0]; + return block?.type === 'text' ? block.text : ''; + }, }, - }; + !overrideKey, + ); } - let providerInstance: AIProvider | null = null; let providerInstanceKey: string | null = null; -export async function getAIProvider(overrideKey?: string): Promise { +export async function getAIProvider(rawOverrideKey?: string): Promise { + const overrideKey = rawOverrideKey?.trim() || undefined; // User-provided key: always create a fresh instance (no caching across users) if (overrideKey) { return createProvider(overrideKey); } - // Cache by the resolved provider type so changing AI_PROVIDER invalidates the cache. - // Mirror createProvider()'s auto-detection order (edgeone keys take precedence). - const currentKey = process.env.AI_PROVIDER - || ((process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim()) ? 'edgeone' - : process.env.GEMINI_API_KEY?.trim() ? 'gemini' - : process.env.OPENAI_API_KEY?.trim() ? 'openai' - : process.env.ANTHROPIC_API_KEY?.trim() ? 'anthropic' - : 'mock'); + const currentKey = chatIdentity(); if (!providerInstance || providerInstanceKey !== currentKey) { providerInstance = await createProvider(); providerInstanceKey = currentKey; diff --git a/server/ai/public-chat-fetch.test.ts b/server/ai/public-chat-fetch.test.ts new file mode 100644 index 0000000..ee06b30 --- /dev/null +++ b/server/ai/public-chat-fetch.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from 'bun:test'; +import { isPublicAddress, fetchPublicChat, readBoundedBody, EMBEDDING_REQUEST_BODY_LIMIT } from './public-chat-fetch'; + +test('embedding transport admits escaped input while retaining a strict body limit', async () => { + const body = JSON.stringify({ input: Array.from({ length: 32 }, () => '\u0000'.repeat(8000)) }); + const request = new Request('https://example.com', { method: 'POST', body }); + expect((await readBoundedBody(request.body, EMBEDDING_REQUEST_BODY_LIMIT)).length).toBe(Buffer.byteLength(body)); + const oversized = new Request('https://example.com', { method: 'POST', body: 'x'.repeat(EMBEDDING_REQUEST_BODY_LIMIT + 1) }); + await expect(readBoundedBody(oversized.body, EMBEDDING_REQUEST_BODY_LIMIT)).rejects.toMatchObject({ code: 'PROMPT_TOO_LARGE' }); +}); + +test('blocks local, private, reserved and mapped network addresses', () => { + for (const address of ['127.0.0.1', '10.0.0.1', '169.254.169.254', '172.16.0.1', '192.168.1.1', '100.64.0.1', '0.0.0.0', '224.0.0.1', '::1', '::ffff:127.0.0.1', 'fc00::1', 'fe80::1', '2002:7f00:1::', '2001:db8::1']) expect(isPublicAddress(address)).toBe(false); + expect(isPublicAddress('8.8.8.8')).toBe(true); + expect(isPublicAddress('2606:4700:4700::1111')).toBe(true); +}); + +test('rejects a private endpoint before sending a request', async () => { + await expect(fetchPublicChat('https://127.0.0.1/v1')).rejects.toMatchObject({ code: 'INVALID_AI_CONFIG' }); +}); + +test('rejects oversized request bodies before opening a connection', async () => { + const big = new Request('https://8.8.8.8/v1', { method: 'POST', body: 'x'.repeat(256 * 1024 + 1) }); + await expect(fetchPublicChat(big)).rejects.toMatchObject({ code: 'PROMPT_TOO_LARGE' }); +}); + +test('enforces the request-size limit while streaming chunked bodies', async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('x'.repeat(128 * 1024))); + controller.enqueue(new TextEncoder().encode('x'.repeat(128 * 1024 + 1))); + controller.close(); + }, + }); + const chunked = new Request('https://8.8.8.8/v1', { method: 'POST', body: stream, duplex: 'half' }); + await expect(fetchPublicChat(chunked)).rejects.toMatchObject({ code: 'PROMPT_TOO_LARGE' }); +}); diff --git a/server/ai/public-chat-fetch.ts b/server/ai/public-chat-fetch.ts new file mode 100644 index 0000000..db70ce3 --- /dev/null +++ b/server/ai/public-chat-fetch.ts @@ -0,0 +1,147 @@ +import { lookup } from 'node:dns/promises'; +import { request as httpsRequest } from 'node:https'; +import { BlockList, isIP } from 'node:net'; +import { AppError } from '../utils/errors'; + +const blocked = new BlockList(); +for (const [network, bits] of [ + ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], + ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24], + ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24], ['224.0.0.0', 3], +] as const) blocked.addSubnet(network, bits, 'ipv4'); + +export function isPublicAddress(address: string): boolean { + if (isIP(address) === 4) return !blocked.check(address, 'ipv4'); + if (isIP(address) !== 6) return false; + const global = new BlockList(); + global.addSubnet('2000::', 3, 'ipv6'); + const special = new BlockList(); + special.addSubnet('2001::', 23, 'ipv6'); + special.addSubnet('2001:db8::', 32, 'ipv6'); + special.addSubnet('2002::', 16, 'ipv6'); + special.addSubnet('3fff::', 20, 'ipv6'); + return global.check(address, 'ipv6') && !special.check(address, 'ipv6'); +} + +/** Read a request body enforcing maxBytes while streaming, never materializing beyond the cap. */ +export async function readBoundedBody(body: ReadableStream | null, maxBytes: number): Promise { + if (!body) return Buffer.alloc(0); + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > maxBytes) { + await reader.cancel().catch(() => {}); + throw new AppError(413, 'AI request body is too large.', 'PROMPT_TOO_LARGE'); + } + chunks.push(value); + } + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))); +} + +/** Resolve once and connect to the checked address, keeping the original TLS identity. */ +export async function fetchPublicChat(input: string | URL | Request, init?: RequestInit): Promise { + return fetchPublicProvider(input, init, 256 * 1024); +} + +/** A 256 KB embedding input can expand sixfold when JSON escapes control characters. */ +export const EMBEDDING_REQUEST_BODY_LIMIT = 2 * 1024 * 1024; +export async function fetchPublicEmbeddings(input: string | URL | Request, init?: RequestInit): Promise { + return fetchPublicProvider(input, init, EMBEDDING_REQUEST_BODY_LIMIT); +} + +async function fetchPublicProvider(input: string | URL | Request, init: RequestInit | undefined, maxBodyBytes: number): Promise { + const source = new Request(input, init); + const url = new URL(source.url); + if (url.protocol !== 'https:' || url.username || url.password) + throw new AppError(400, 'Use a public HTTPS AI endpoint.', 'INVALID_AI_CONFIG'); + const hostname = url.hostname.replace(/^\[|\]$/g, ''); + let timer: ReturnType | undefined; + const addresses = await Promise.race([ + lookup(hostname, { all: true, verbatim: true }), + new Promise((_, reject) => { timer = setTimeout(() => reject(new Error('AI endpoint DNS lookup timed out')), 5000); }), + ]).finally(() => clearTimeout(timer)); + if (!addresses.length || addresses.some(({ address }) => !isPublicAddress(address))) + throw new AppError(400, 'Local and private-network AI endpoints are not supported.', 'INVALID_AI_CONFIG'); + const MAX_BODY_BYTES = maxBodyBytes; + const declared = Number(source.headers.get('content-length')); + if (Number.isSafeInteger(declared) && declared > MAX_BODY_BYTES) + throw new AppError(413, 'AI request body is too large.', 'PROMPT_TOO_LARGE'); + // Bound while reading: arrayBuffer() would materialize an unbounded body first. + const body = await readBoundedBody(source.body, MAX_BODY_BYTES); + const headers = Object.fromEntries(source.headers); + headers.host = url.host; + headers['accept-encoding'] = 'identity'; + const overallSignal = AbortSignal.any([source.signal, AbortSignal.timeout(30000)]); + if (overallSignal.aborted) { + throw overallSignal.reason || new Error('Request aborted'); + } + let lastError: unknown; + for (let i = 0; i < addresses.length; i++) { + if (overallSignal.aborted) break; + try { + return await new Promise((resolve, reject) => { + const req = httpsRequest( + url, + { + hostname: addresses[i].address, + servername: isIP(hostname) ? undefined : hostname, + method: source.method, + headers, + signal: overallSignal, + }, + (res) => { + const chunks: Buffer[] = []; + let bytes = 0; + res.on('data', (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 10 * 1024 * 1024) { + res.destroy(new Error('AI response exceeds size limit')); + return; + } + chunks.push(chunk); + }); + res.on('error', reject); + res.on('end', () => { + const status = res.statusCode ?? 502; + if (status >= 300 && status < 400) { + reject(new Error('AI endpoint redirects are not supported')); + return; + } + const responseHeaders = new Headers(); + for (const [name, value] of Object.entries(res.headers)) { + if (value !== undefined) responseHeaders.set(name, Array.isArray(value) ? value.join(', ') : value); + } + resolve( + new Response([204, 205, 304].includes(status) ? null : Buffer.concat(chunks), { + status, + headers: responseHeaders, + }), + ); + }); + }, + ); + req.on('error', reject); + req.end(body); + }); + } catch (err) { + lastError = err; + const isSizeLimit = err instanceof Error && err.message === 'AI response exceeds size limit'; + const isRedirect = err instanceof Error && err.message === 'AI endpoint redirects are not supported'; + const isAppError = err instanceof AppError; + const isConnError = + err instanceof Error && + 'code' in err && + typeof (err as { code?: unknown }).code === 'string' && + ['ECONNREFUSED', 'ENETUNREACH', 'EHOSTUNREACH', 'EADDRNOTAVAIL'].includes((err as { code: string }).code); + + if (overallSignal.aborted || isSizeLimit || isRedirect || isAppError || !isConnError || i === addresses.length - 1) { + throw err; + } + } + } + throw lastError || (overallSignal.aborted ? overallSignal.reason : new Error('No reachable AI endpoint address')); +} diff --git a/server/ai/service.ts b/server/ai/service.ts index e3a03c8..e95ce90 100644 --- a/server/ai/service.ts +++ b/server/ai/service.ts @@ -1,76 +1,9 @@ +import { isMockRepo } from '../github/mock-data'; +import { chatIdentity } from './chat-config'; import { getAIProvider } from './provider'; -import { aiCacheKey, cache, hashToken } from '../cache/lru'; +import { aiCacheKey, cache } from '../cache/lru'; import { logApi } from '../utils/logger'; -// Concurrency queue to control/batch AI API requests -class PromiseQueue { - private queue: (() => Promise)[] = []; - private activeCount = 0; - constructor(private maxConcurrency = 2) { } - - add(fn: () => Promise): Promise { - return new Promise((resolve, reject) => { - this.queue.push(async () => { - try { - const res = await fn(); - resolve(res); - } catch (err) { - reject(err); - } - }); - this.next(); - }); - } - - private next() { - if (this.activeCount >= this.maxConcurrency || this.queue.length === 0) { - return; - } - this.activeCount++; - const fn = this.queue.shift()!; - fn().finally(() => { - this.activeCount--; - this.next(); - }); - } -} - -const aiQueue = new PromiseQueue(2); - -function getAiCacheDiscriminator(apiKey?: string): string { - const envProvider = process.env.AI_PROVIDER?.trim().toLowerCase(); - const provider = apiKey?.trim() - ? (apiKey.trim().startsWith('sk-ant-') - ? 'anthropic' - : apiKey.trim().startsWith('sk-') - ? (envProvider === 'edgeone' ? 'edgeone' : 'openai') - : 'gemini') - : (envProvider && envProvider.length > 0 - ? envProvider - : (process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim() - ? 'edgeone' - : process.env.GEMINI_API_KEY?.trim() - ? 'gemini' - : process.env.OPENAI_API_KEY?.trim() - ? 'openai' - : process.env.ANTHROPIC_API_KEY?.trim() - ? 'anthropic' - : 'mock')); - - const model = provider === 'edgeone' - ? (process.env.EDGEONE_MODEL ?? '@makers/deepseek-v4-flash') - : provider === 'gemini' - ? (process.env.GEMINI_MODEL ?? 'gemini-2.5-flash') - : provider === 'openai' - ? (process.env.OPENAI_MODEL ?? 'gpt-4o-mini') - : provider === 'anthropic' - ? (process.env.ANTHROPIC_MODEL ?? 'claude-3-5-haiku-latest') - : 'mock'; - - const keyScope = apiKey?.trim() ? `user-key:${hashToken(apiKey)}` : 'env-key'; - return `${provider}:${model}:${keyScope}`; -} - function safeParseJSON(raw: string): T { let cleaned = raw.trim(); @@ -84,15 +17,21 @@ function safeParseJSON(raw: string): T { // 2. Locate the first { or [ and last } or ] if there is outer noise const firstBrace = cleaned.indexOf('{'); const firstBracket = cleaned.indexOf('['); - const startIdx = (firstBrace !== -1 && firstBracket !== -1) - ? Math.min(firstBrace, firstBracket) - : (firstBrace !== -1 ? firstBrace : firstBracket); + const startIdx = + firstBrace !== -1 && firstBracket !== -1 + ? Math.min(firstBrace, firstBracket) + : firstBrace !== -1 + ? firstBrace + : firstBracket; const lastBrace = cleaned.lastIndexOf('}'); const lastBracket = cleaned.lastIndexOf(']'); - const endIdx = (lastBrace !== -1 && lastBracket !== -1) - ? Math.max(lastBrace, lastBracket) - : (lastBrace !== -1 ? lastBrace : lastBracket); + const endIdx = + lastBrace !== -1 && lastBracket !== -1 + ? Math.max(lastBrace, lastBracket) + : lastBrace !== -1 + ? lastBrace + : lastBracket; if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) { cleaned = cleaned.slice(startIdx, endIdx + 1); @@ -115,11 +54,11 @@ export interface AiTaskParams> { } export async function executeAiTask>( - params: AiTaskParams + params: AiTaskParams, ): Promise<{ data: T; cached: boolean }> { const providerEnv = process.env.AI_PROVIDER?.trim().toLowerCase(); const hasApiKey = params.apiKey?.trim(); - const isMock = !hasApiKey && (!providerEnv || providerEnv === 'mock'); + const isMock = (!hasApiKey && (!providerEnv || providerEnv === 'mock')) || (!hasApiKey && isMockRepo(params.owner)); if (isMock) { return { data: params.mockFallback(), cached: false }; @@ -131,21 +70,19 @@ export async function executeAiTask>( params.repo, params.sha, params.paramHash ?? 'v1', - getAiCacheDiscriminator(params.apiKey) + chatIdentity(params.apiKey), ); if (cache.has(key)) { return { data: cache.get(key)!, cached: true }; } const provider = await getAIProvider(params.apiKey); - const raw = await aiQueue.add(() => - provider.complete( - [ - { role: 'system', content: params.systemPrompt }, - { role: 'user', content: params.userPrompt }, - ], - { json: params.json } - ) + const raw = await provider.complete( + [ + { role: 'system', content: params.systemPrompt }, + { role: 'user', content: params.userPrompt }, + ], + { json: params.json }, ); let result: T; diff --git a/server/api-router.ts b/server/api-router.ts index 0b8087c..29e0edc 100644 --- a/server/api-router.ts +++ b/server/api-router.ts @@ -1,10 +1,10 @@ +import { chatOverrides, readChatOverrides } from './ai/chat-config'; import { getOctokit } from './github/client'; import type { RequestContext } from './utils/context'; import { fetchTrending } from './services/trending'; import { logApi, logError } from './utils/logger'; -import { clearAllCaches } from './cache/lru'; import { getPublicAppConfig } from './config/app-config'; -import { toErrorPayload } from './utils/errors'; +import { AppError, toErrorPayload } from './utils/errors'; import type { TrendingRepo } from '../src/types'; // Decoupled Router Submodules @@ -12,10 +12,48 @@ import { handleAiRoutes } from './router/ai-routes'; import { handleRepoRoutes } from './router/repo-routes'; import { handleSearchRoutes } from './router/search-routes'; import { addSecurityHeaders } from './utils/http'; +import { configuredLimit, reserveUsage } from './utils/usage-limits'; +import { createAdmissionLimit, limitRequestBody } from './utils/request-limits'; +import { limitClient } from './utils/client-limits'; -export async function handleApiRequest( - req: Request, -): Promise { +const admitRequest = createAdmissionLimit(8, Infinity); +const admitConnection = createAdmissionLimit(8, Infinity); + +// Endpoints registered in server/router/. Admission and usage buckets are only spent on +// these, so cheap 404s against unknown /api/* paths cannot exhaust the deployment budget. +export const registeredApiPaths = new Set([ + '/api/config', + '/api/trending', + '/api/repo/analyze', + '/api/repo/branches', + '/api/repo/churn', + '/api/repo/contributors', + '/api/repo/file', + '/api/repo/graph', + '/api/repo/health', + '/api/repo/tags', + '/api/repo/tree', + '/api/ai/explain', + '/api/ai/explain-lif', + '/api/ai/health', + '/api/ai/learning-path', + '/api/ai/mermaid', + '/api/ai/readme-enhance', + '/api/ai/refactor', + '/api/ai/roast', + '/api/search', + '/api/search/ask', + '/api/search/cancel', + '/api/search/index', + '/api/search/status', +]); + +/** True only for endpoints registered in server/router/; non-/api paths are handled elsewhere. */ +export function isRegisteredApiPath(pathname: string): boolean { + return registeredApiPaths.has(pathname); +} + +export async function handleApiRequest(req: Request, remoteAddress?: string): Promise { const start = Date.now(); const method = req.method; const url = new URL(req.url); @@ -27,18 +65,45 @@ export async function handleApiRequest( query[k] = v; }); - const userKey = req.headers.get('x-gemini-api-key') || undefined; + const userKey = req.headers.get('x-ai-api-key')?.trim() || req.headers.get('x-gemini-api-key')?.trim() || undefined; + // Reject unknown /api/* paths before any admission, client or usage budget is spent. + if (pathname.startsWith('/api/') && !isRegisteredApiPath(pathname)) { + return addSecurityHeaders( + Response.json({ error: 'Not found', code: 'NOT_FOUND', status: 404, retryable: false }, { status: 404 }), + ); + } const gitHubToken = req.headers.get('x-github-token') || undefined; const ctx: RequestContext = { octokit: getOctokit(gitHubToken), + gitHubToken, }; + const expensive = + pathname === '/api/trending' || + pathname.startsWith('/api/ai/') || + pathname === '/api/search' || + pathname.startsWith('/api/search/') || + pathname.startsWith('/api/repo/'); + let release: (() => void) | null | undefined; + let releaseConnection: (() => void) | null | undefined; try { + releaseConnection = expensive ? admitConnection() : undefined; + if (releaseConnection === null) + throw new AppError(429, 'Server is busy. Please try again shortly.', 'RATE_LIMIT_EXCEEDED', true); + if (expensive) await limitClient(req.headers, remoteAddress); + release = expensive ? admitRequest() : undefined; + if (release === null) + throw new AppError(429, 'Server is busy. Please try again shortly.', 'RATE_LIMIT_EXCEEDED', true); + if (expensive) await reserveUsage('api-minute', 1, configuredLimit('API_REQUESTS_PER_MINUTE', 600), 60000); + req = await limitRequestBody(req); // ── Global System Utilities ───────────────────────────────────── if (pathname === '/api/trending' && method === 'GET') { const repos: TrendingRepo[] = await fetchTrending(); - logApi('/api/trending', { durationMs: Date.now() - start, count: repos.length }); + logApi('/api/trending', { + durationMs: Date.now() - start, + count: repos.length, + }); return addSecurityHeaders(Response.json({ repos }, { status: 200 })); } @@ -46,28 +111,36 @@ export async function handleApiRequest( return addSecurityHeaders(Response.json(getPublicAppConfig(), { status: 200 })); } - if (pathname === '/api/cache/clear' && method === 'POST') { - clearAllCaches(); - logApi('/api/cache/clear', { durationMs: Date.now() - start }); - return addSecurityHeaders(Response.json({ cleared: true }, { status: 200 })); - } - // ── Repository Routes ─────────────────────────────────────────── const repoResponse = await handleRepoRoutes(pathname, req, query, ctx, start); if (repoResponse) return addSecurityHeaders(repoResponse); // ── AI Summary & Analysis Routes ──────────────────────────────── - const aiResponse = await handleAiRoutes(pathname, req, userKey, gitHubToken, ctx); + const aiResponse = await chatOverrides.run(readChatOverrides(req.headers, userKey), () => + handleAiRoutes(pathname, req, userKey, gitHubToken, ctx), + ); if (aiResponse) return addSecurityHeaders(aiResponse); // ── Semantic Search & Ingest Routes ───────────────────────────── - const searchResponse = await handleSearchRoutes(pathname, req, query, userKey, ctx, start); + const searchResponse = await chatOverrides.run( + pathname === '/api/search/ask' ? readChatOverrides(req.headers, userKey) : {}, + () => handleSearchRoutes(pathname, req, query, userKey, ctx, start), + ); if (searchResponse) return addSecurityHeaders(searchResponse); return null; } catch (error) { const payload = toErrorPayload(error); logError(pathname, error, { durationMs: Date.now() - start }); - return addSecurityHeaders(Response.json(payload, { status: payload.status })); + return addSecurityHeaders( + Response.json(payload, { + status: payload.status, + headers: + payload.status === 429 ? { 'Retry-After': String(payload.context?.retryAfterSeconds ?? 60) } : undefined, + }), + ); + } finally { + release?.(); + releaseConnection?.(); } } diff --git a/server/cache/hash-secret.test.ts b/server/cache/hash-secret.test.ts new file mode 100644 index 0000000..eb4b158 --- /dev/null +++ b/server/cache/hash-secret.test.ts @@ -0,0 +1,29 @@ +import { expect, test } from 'bun:test'; +import { cacheHashSecret } from './hash-secret'; +import { analyzeCacheKey } from './lru'; + +test('local fallback is random process-stable material and empty configuration uses it', () => { + const secret = cacheHashSecret({}); + expect(secret).toMatch(/^[a-f0-9]{64}$/); + expect(cacheHashSecret({ TOKEN_CACHE_HASH_SECRET: ' ' })).toBe(secret); +}); + +test('shared mode requires a configured secret, including implicit Redis mode', () => { + for (const env of [ + { RATE_LIMIT_MODE: 'shared' }, + { UPSTASH_REDIS_REST_URL: 'https://example.com' }, + { UPSTASH_REDIS_REST_TOKEN: 'test' }, + ]) { + expect(() => cacheHashSecret(env)).toThrow('TOKEN_CACHE_HASH_SECRET is required'); + expect(cacheHashSecret({ ...env, TOKEN_CACHE_HASH_SECRET: 'configured-test-secret' })).toBe( + 'configured-test-secret', + ); + } +}); + +test('analysis keys isolate credentials without exposing tokens', () => { + const first = analyzeCacheKey('owner', 'repo', 'sha', 'main', 'credential-a'); + expect(first).not.toContain('credential-a'); + expect(first).not.toBe(analyzeCacheKey('owner', 'repo', 'sha', 'main', 'credential-b')); + expect(first).toBe(analyzeCacheKey('owner', 'repo', 'sha', 'main', 'credential-a')); +}); diff --git a/server/cache/hash-secret.ts b/server/cache/hash-secret.ts new file mode 100644 index 0000000..9d7654d --- /dev/null +++ b/server/cache/hash-secret.ts @@ -0,0 +1,13 @@ +import { randomBytes } from 'node:crypto'; +import { AppError } from '../utils/errors'; + +const processSecret = randomBytes(32).toString('hex'); + +/** Read lazily: server environment loading can happen after module imports. */ +export function cacheHashSecret(env: Record = process.env): string { + const configured = env.TOKEN_CACHE_HASH_SECRET?.trim(); + if (configured) return configured; + const shared = env.RATE_LIMIT_MODE === 'shared' || !!env.UPSTASH_REDIS_REST_URL || !!env.UPSTASH_REDIS_REST_TOKEN; + if (shared) throw new AppError(503, 'TOKEN_CACHE_HASH_SECRET is required in shared mode.', 'CACHE_SECRET_REQUIRED'); + return processSecret; +} diff --git a/server/cache/lru.test.ts b/server/cache/lru.test.ts index 8d8f80b..03c139d 100644 --- a/server/cache/lru.test.ts +++ b/server/cache/lru.test.ts @@ -85,8 +85,8 @@ describe('cache/lru', () => { }); it('builds cache keys deterministically', () => { - expect(analyzeCacheKey('o', 'r', 's')).toBe('analyze:o/r@s'); - expect(analyzeCacheKey('o', 'r', 's', 'main')).toBe('analyze:o/r@s:main'); + expect(analyzeCacheKey('o', 'r', 's')).toBe(analyzeCacheKey('o', 'r', 's')); + expect(analyzeCacheKey('o', 'r', 's', 'main')).not.toBe(analyzeCacheKey('o', 'r', 's')); expect(aiCacheKey('summary', 'o', 'r', 's', 'ctx')).toBe('ai:summary:o/r@s:ctx'); }); }); @@ -94,9 +94,7 @@ describe('cache/lru', () => { describe('hashContext', () => { it('produces the correct SHA-256 hex string for a known input', () => { // "hello world" -> SHA256 -> b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 - expect(hashContext('hello world')).toBe( - 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9', - ); + expect(hashContext('hello world')).toBe('b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9'); }); it('produces different hashes for different strings', () => { @@ -108,9 +106,7 @@ describe('hashContext', () => { it('produces the correct hash for an empty string', () => { // "" -> SHA256 -> e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 - expect(hashContext('')).toBe( - 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', - ); + expect(hashContext('')).toBe('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'); }); }); @@ -125,4 +121,4 @@ describe('hashToken', () => { expect(hash1).not.toBe(hash2); expect(hash1).toBe(hashToken(key1)); // Deterministic }); -}); \ No newline at end of file +}); diff --git a/server/cache/lru.ts b/server/cache/lru.ts index 80cba32..3eb5b90 100644 --- a/server/cache/lru.ts +++ b/server/cache/lru.ts @@ -1,5 +1,6 @@ import { LRUCache } from 'lru-cache'; import crypto from 'crypto'; +import { cacheHashSecret } from './hash-secret'; export interface CacheStore { get(key: string): T | undefined; @@ -85,10 +86,9 @@ export function invalidateSearchCache(owner: string, repo: string): void { } } -export function analyzeCacheKey(owner: string, repo: string, sha: string, branch?: string): string { - return branch - ? `analyze:${owner}/${repo}@${sha}:${branch}` - : `analyze:${owner}/${repo}@${sha}`; +export function analyzeCacheKey(owner: string, repo: string, sha: string, branch?: string, token?: string): string { + const scope = hashToken(token || process.env.GITHUB_TOKEN?.trim() || 'anonymous'); + return `analyze:${owner}/${repo}@${sha}:${scope}:${JSON.stringify(branch ?? null)}`; } export function aiCacheKey( @@ -105,22 +105,15 @@ export function aiCacheKey( } export function churnCacheKey(owner: string, repo: string, branch?: string, days = 90): string { - return branch - ? `churn:${owner}/${repo}@${branch}:${days}d` - : `churn:${owner}/${repo}@default:${days}d`; + return branch ? `churn:${owner}/${repo}@${branch}:${days}d` : `churn:${owner}/${repo}@default:${days}d`; } export function hashContext(input: string): string { return crypto.createHash('sha256').update(input).digest('hex'); } -const TOKEN_CACHE_HASH_SECRET = process.env.TOKEN_CACHE_HASH_SECRET ?? 'token-cache-v1'; - // This is cache-key derivation, not password storage. -// HMAC avoids exposing raw tokens without blocking request handling. +// HMAC keeps raw tokens out of cache identities. export function hashToken(token: string): string { - return crypto - .createHmac('sha256', TOKEN_CACHE_HASH_SECRET) - .update(token) - .digest('hex'); + return crypto.createHmac('sha256', cacheHashSecret()).update(token).digest('hex'); } diff --git a/server/dev-api.ts b/server/dev-api.ts index 162307b..359c44c 100644 --- a/server/dev-api.ts +++ b/server/dev-api.ts @@ -1,8 +1,11 @@ +import fs from 'node:fs'; +import path from 'node:path'; import type { Plugin } from 'vite'; import { handleNodeRequest } from './utils/http'; import { logInfo } from './utils/logger'; import { loadServerEnv } from './env'; import { resetOctokit } from './github/client'; +import { isRepositoryPage } from './utils/page-route'; export function apiMiddleware(): Plugin { return { @@ -22,15 +25,44 @@ export function apiMiddleware(): Plugin { const url = req.url ?? ''; const pathname = url.split('?')[0]; - if (!pathname.startsWith('/api/')) { + if (pathname.startsWith('/api/')) { + const handled = await handleNodeRequest(req, res); + if (!handled) { + sendNotFound(res); + } + return; + } + + if (pathname.startsWith('/assets/')) { next(); return; } - const handled = await handleNodeRequest(req, res); - if (!handled) { - sendNotFound(res); + if ( + req.method === 'GET' && + req.headers.accept?.includes('text/html') && + !pathname.startsWith('/@') && + !pathname.startsWith('/src') && + !pathname.startsWith('/node_modules') && + !pathname.startsWith('/assets') && + !pathname.includes('.') && + pathname !== '/' && + !/^\/(privacy|terms)\/?$/.test(pathname) && + !isRepositoryPage(pathname) + ) { + try { + const template = fs.readFileSync(path.resolve('index.html'), 'utf8'); + const html = await server.transformIndexHtml(url, template); + res.statusCode = 404; + res.setHeader('Content-Type', 'text/html'); + res.end(html); + return; + } catch { + // fallback to default pipeline if transform fails + } } + + next(); }); }, }; diff --git a/server/github/client.ts b/server/github/client.ts index b715e8e..a703794 100644 --- a/server/github/client.ts +++ b/server/github/client.ts @@ -11,6 +11,7 @@ export function getOctokit(token?: string): Octokit { return new Octokit({ auth: token, userAgent: 'gitSdm/1.0', + request: { timeout: 15000 }, }); } if (!octokitInstance) { @@ -18,6 +19,7 @@ export function getOctokit(token?: string): Octokit { octokitInstance = new Octokit({ auth: envToken || undefined, userAgent: 'gitSdm/1.0', + request: { timeout: 15000 }, }); } return octokitInstance; diff --git a/server/github/fetch-tree.ts b/server/github/fetch-tree.ts index 710b4d1..26f2455 100644 --- a/server/github/fetch-tree.ts +++ b/server/github/fetch-tree.ts @@ -210,7 +210,16 @@ export async function fetchFlatTree( const isNoise = (p: string) => { const lower = p.toLowerCase(); // Match explicit noise folders - if (lower.includes('docs/') || lower.includes('test/') || lower.includes('tests/') || lower.includes('spec/') || lower.includes('dist/') || lower.includes('build/') || lower.includes('example/')) return true; + if ( + lower.includes('docs/') || + lower.includes('test/') || + lower.includes('tests/') || + lower.includes('spec/') || + lower.includes('dist/') || + lower.includes('build/') || + lower.includes('example/') + ) + return true; // Match any hidden folder starting with a dot (e.g., .github, .vscode, .husky) if (lower.includes('/.') || lower.startsWith('.')) return true; return false; @@ -218,8 +227,9 @@ export async function fetchFlatTree( const mapped = allBlobs.map((item) => { let priority = 0; - if (isManifest(item.path)) priority = -1; // Highest priority: never truncate manifests - else if (isNoise(item.path)) priority = 1; // Lowest priority: noise + if (isManifest(item.path)) + priority = -1; // Highest priority: never truncate manifests + else if (isNoise(item.path)) priority = 1; // Lowest priority: noise return { item, priority }; }); @@ -331,6 +341,7 @@ export async function fetchFileContents( paths: string[], ref: string, tokenOrCtx?: string | RequestContext, + maxBytes?: number, ): Promise> { if (isMockRepo(owner)) { return fetchMockFileContents(owner, repo, paths); @@ -350,10 +361,20 @@ export async function fetchFileContents( path, ref, }); - if (!Array.isArray(data) && data.type === 'file' && 'content' in data && data.content) { - result[path] = Buffer.from(data.content, 'base64').toString('utf-8').slice(0, 50000); + if (!Array.isArray(data) && data.type === 'file' && 'content' in data && typeof data.content === 'string') { + if ( + maxBytes !== undefined && + (data.size > maxBytes || data.content.length > Math.ceil(maxBytes * 1.4) + 1024) + ) + throw new Error('File exceeds the indexing byte limit'); + const decoded = Buffer.from(data.content, 'base64'); + if (maxBytes !== undefined && decoded.byteLength > maxBytes) + throw new Error('File exceeds the indexing byte limit'); + result[path] = + maxBytes === undefined ? decoded.toString('utf-8').slice(0, 50000) : decoded.toString('utf-8'); } - } catch { + } catch (error) { + if (maxBytes !== undefined) throw error; // skip unreadable files } }), @@ -479,4 +500,3 @@ export async function fetchTimeline( return []; } } - diff --git a/server/parser/file-classifier.ts b/server/parser/file-classifier.ts index 8c439fb..4daeb9d 100644 --- a/server/parser/file-classifier.ts +++ b/server/parser/file-classifier.ts @@ -1,3 +1,4 @@ +import type { TreeNode } from '../../src/types'; import type { FileClass } from '../../src/types'; const CONFIG_PATTERNS = [ @@ -56,7 +57,7 @@ export function classifyFile(path: string): FileClass { return 'other'; } -export function annotateTree(nodes: import('../../src/types').TreeNode[]): import('../../src/types').TreeNode[] { +export function annotateTree(nodes: TreeNode[]): TreeNode[] { return nodes.map((node) => ({ ...node, fileClass: node.type === 'file' ? classifyFile(node.path) : undefined, diff --git a/server/parser/manifest-parsers/index.test.ts b/server/parser/manifest-parsers/index.test.ts index 37a7069..56b8afc 100644 --- a/server/parser/manifest-parsers/index.test.ts +++ b/server/parser/manifest-parsers/index.test.ts @@ -167,6 +167,31 @@ describe('manifest parsers', () => { expect(deps[2].name).toBe('numpy'); }); + it('ignores dependencies arrays outside [project] and keeps versionless poetry inline tables', () => { + const toml = ` +[project] +name = "pkg" +dependencies = ["requests>=2.0"] + +[tool.black] +dependencies = ["should-not-appear>=1.0"] + +[tool.poetry.dependencies] +python = "^3.10" +local-pkg = { path = "../local-pkg" } +git-pkg = { git = "https://github.com/o/p.git" } +url-pkg = { url = "https://example.com/p.tar.gz" } +`; + const deps = parsePyproject(toml); + const names = deps.map((d) => d.name); + expect(names).toContain('requests'); + expect(names).not.toContain('should-not-appear'); + for (const name of ['local-pkg', 'git-pkg', 'url-pkg']) { + expect(names).toContain(name); + expect(deps.find((d) => d.name === name)?.version).toBeUndefined(); + } + }); + it('parses pyproject.toml dependencies', () => { const content = `[project.dependencies]\n"django>=4.0"\n"pandas"\n`; const deps = parsePyproject(content); @@ -176,6 +201,102 @@ describe('manifest parsers', () => { expect(deps[1].name).toBe('pandas'); }); + it('handles environment markers, package extras, optional-dependencies and poetry dev groups', () => { + const reqContent = `requests[security]>=2.28.0; python_version >= '3.8'\npytest<8.0 ; sys_platform == 'linux'\n`; + const reqDeps = parseRequirementsTxt(reqContent); + expect(reqDeps).toHaveLength(2); + expect(reqDeps[0]).toEqual({ name: 'requests', version: '2.28.0', type: 'prod', ecosystem: 'python' }); + expect(reqDeps[1]).toEqual({ name: 'pytest', version: '8.0', type: 'prod', ecosystem: 'python' }); + + const pyprojectContent = ` +[project] +name = "complex-pkg" +dependencies = [ + "requests[socks]>=2.0.0; os_name == 'posix'", + "urllib3<2" +] + +[project.optional-dependencies] +gui = [ + "PyQt5[core]>=5.15.0; sys_platform == 'win32'", + "Pillow" +] + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.0" +black = ">=22.0.0" + +[tool.poetry.dependencies] +fastapi = ">=0.95.0" +`; + const deps = parsePyproject(pyprojectContent); + expect(deps).toHaveLength(7); + const req = deps.find((d) => d.name === 'requests'); + expect(req?.version).toBe('2.0.0'); + expect(req?.type).toBe('prod'); + + const pyqt = deps.find((d) => d.name === 'PyQt5'); + expect(pyqt?.version).toBe('5.15.0'); + expect(pyqt?.type).toBe('prod'); + + const pytest = deps.find((d) => d.name === 'pytest'); + expect(pytest?.version).toBe('^7.2.0'); + expect(pytest?.type).toBe('dev'); + + const black = deps.find((d) => d.name === 'black'); + expect(black?.type).toBe('dev'); + + const fastapi = deps.find((d) => d.name === 'fastapi'); + expect(fastapi?.type).toBe('prod'); + }); + + it('handles brackets in comments inside PEP 621 arrays and dotted optional-dependencies keys', () => { + const toml = ` +[project] +name = "pkg-with-comments" +dependencies = [ + # comment with [brackets] inside + "requests>=2.0", + # another comment with [more] + "urllib3" +] +optional-dependencies.gui = [ + "PyQt5>=5.0" +] +`; + const deps = parsePyproject(toml); + expect(deps).toHaveLength(3); + expect(deps.map((d) => d.name)).toEqual(['requests', 'urllib3', 'PyQt5']); + }); + + it('parses standard PEP 621 dependencies array and Poetry tables in pyproject.toml', () => { + const pep621 = ` +[project] +name = "cypy" +dependencies = [ + "opencv-python", + "PyMuPDF", + "numpy>=1.20", + "Pillow" +] +`; + const deps1 = parsePyproject(pep621); + expect(deps1).toHaveLength(4); + expect(deps1.map((d) => d.name)).toEqual(['opencv-python', 'PyMuPDF', 'numpy', 'Pillow']); + expect(deps1.find((d) => d.name === 'numpy')?.version).toBe('1.20'); + + const poetry = ` +[tool.poetry.dependencies] +python = "^3.10" +requests = "^2.28.0" +pandas = { version = ">=1.3.0" } +`; + const deps2 = parsePyproject(poetry); + expect(deps2).toHaveLength(2); + expect(deps2.find((d) => d.name === 'requests')?.version).toBe('^2.28.0'); + expect(deps2.find((d) => d.name === 'pandas')?.version).toBe('>=1.3.0'); + }); + it('parses Cargo.toml dependencies', () => { const content = `[dependencies]\ntokio = "1.28.0"\nserde = { version = "1.0", features = ["derive"] }\n`; const deps = parseCargoToml(content); diff --git a/server/parser/manifest-parsers/pip-parser.ts b/server/parser/manifest-parsers/pip-parser.ts index 97e5cd8..52ed246 100644 --- a/server/parser/manifest-parsers/pip-parser.ts +++ b/server/parser/manifest-parsers/pip-parser.ts @@ -7,9 +7,10 @@ export function parseRequirementsTxt(content: string): Dependency[] { .map((l) => l.trim()) .filter((l) => l && !l.startsWith('#')) .map((line) => { - const match = line.match(/^([a-zA-Z0-9_-]+)([>=<~!]+.*)?$/); + const withoutMarker = line.split(';')[0].trim(); + const match = withoutMarker.match(/^([a-zA-Z0-9_.-]+)(?:\[.*?\])?(?:[>=<~!]+(.+))?$/); return { - name: match?.[1] ?? line.split(/[>=<]/)[0], + name: match?.[1] ?? withoutMarker.split(/[>=<]/)[0], version: match?.[2]?.replace(/^[=<>~!]+/, ''), type: 'prod' as const, ecosystem: 'python', @@ -24,21 +25,129 @@ export const requirementsParser: ManifestParser = { parse: parseRequirementsTxt, }; +function extractPythonPackageSpec(spec: string): { name: string; version?: string } | null { + const cleaned = spec.trim().replace(/^['"]+|['"]+$/g, '').trim(); + if (!cleaned || cleaned.startsWith('#')) return null; + const markerIdx = cleaned.indexOf(';'); + const baseSpec = (markerIdx >= 0 ? cleaned.slice(0, markerIdx) : cleaned).trim(); + if (!baseSpec) return null; + + const match = baseSpec.match(/^([a-zA-Z0-9_.-]+)(?:\[.*?\])?(?:[=<>~!^@\s]+(.+))?$/); + if (match) { + const name = match[1].trim(); + const version = match[2]?.trim().replace(/^[=<>~!^@\s]+/, '').replace(/['"]/g, '').trim(); + return { name, version: version || undefined }; + } + const parts = baseSpec.split(/[=<>~!^@\s]+/); + return parts[0]?.trim() ? { name: parts[0].trim(), version: parts[1]?.trim() } : null; +} + +function extractQuotedArrayItems(str: string, openBracketIndex: number): string[] { + let inDouble = false; + let inSingle = false; + let inComment = false; + let depth = 0; + let start = -1; + let arrayBody = ''; + for (let i = openBracketIndex; i < str.length; i++) { + const ch = str[i]; + if (inComment) { + if (ch === '\n') inComment = false; + continue; + } + if (ch === '\\' && (inDouble || inSingle)) { + i++; + continue; + } + if (ch === '"' && !inSingle) inDouble = !inDouble; + else if (ch === "'" && !inDouble) inSingle = !inSingle; + else if (!inDouble && !inSingle) { + if (ch === '#') { + inComment = true; + continue; + } + if (ch === '[') { + if (depth === 0) start = i + 1; + depth++; + } else if (ch === ']') { + depth--; + if (depth === 0 && start !== -1) { + arrayBody = str.slice(start, i); + break; + } + } + } + } + return arrayBody.match(/"[^"]+"|'[^']+'/g) || []; +} + export function parsePyproject(content: string): Dependency[] { const deps: Dependency[] = []; - const depSection = content.match(/\[project\.dependencies\]([\s\S]*?)(\[|$)/); - if (depSection) { - const lines = depSection[1].match(/"([^"]+)"/g) ?? []; + const seen = new Set(); + + const add = (name: string, version: string | undefined, type: 'prod' | 'dev') => { + const lower = name.toLowerCase(); + if (!name || lower === 'python' || seen.has(lower)) return; + seen.add(lower); + deps.push({ name, version, type, ecosystem: 'python' }); + }; + + // 1. PEP 621 `[project] dependencies = [ ... ]` — only when the assignment sits + // inside the [project] table (a [tool.*] or [project.optional-*] table with its + // own `dependencies = [` must not leak in as production dependencies). + const projectMatch = content.match(/\[project\]([\s\S]*?)(?=\n\s*\[|$)/); + if (projectMatch) { + const section = projectMatch[1]; + const depAssignMatches = section.matchAll(/(?:^|\n)\s*(?:dependencies|optional-dependencies(?:\.[a-zA-Z0-9_.-]+)?)\s*=\s*\[/g); + for (const m of depAssignMatches) { + const items = extractQuotedArrayItems(section, m.index + m[0].lastIndexOf('[')); + for (const item of items) { + const parsed = extractPythonPackageSpec(item); + if (parsed) add(parsed.name, parsed.version, 'prod'); + } + } + } + + // 2. PEP 621 [project.optional-dependencies] section + const optSectionMatch = content.match(/\[project\.optional-dependencies\]([\s\S]*?)(?=\n\s*\[|$)/); + if (optSectionMatch) { + const section = optSectionMatch[1]; + const groupMatches = section.matchAll(/[a-zA-Z0-9_.-]+\s*=\s*\[/g); + for (const gm of groupMatches) { + const items = extractQuotedArrayItems(section, gm.index + gm[0].lastIndexOf('[')); + for (const item of items) { + const parsed = extractPythonPackageSpec(item); + if (parsed) add(parsed.name, parsed.version, 'prod'); + } + } + } + + // 3. Table syntax: [project.dependencies], [tool.poetry.dependencies], [tool.poetry.dev-dependencies], [tool.poetry.group..dependencies] + const tableMatches = content.matchAll(/\[(project\.dependencies|tool\.poetry\.(?:group\.([a-zA-Z0-9_.-]+)\.)?dependencies|tool\.poetry\.dev-dependencies)\]([\s\S]*?)(?=\n\s*\[|$)/g); + for (const tm of tableMatches) { + const tableType = tm[1]; + const groupName = tm[2]; + const isDev = tableType === 'tool.poetry.dev-dependencies' || groupName === 'dev' || groupName === 'test'; + const depType = isDev ? 'dev' : 'prod'; + + const lines = tm[3].split('\n'); for (const line of lines) { - const pkg = line.replace(/"/g, ''); - const parts = pkg.split(/[=<>~!]+/); - const name = parts[0]?.trim(); - const version = parts[1]?.trim(); - if (name) { - deps.push({ name, version, type: 'prod', ecosystem: 'python' }); + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const kv = trimmed.match(/^([a-zA-Z0-9_.-]+)\s*=\s*(?:"([^"]+)"|'([^']+)'|\{.*?version\s*=\s*["']([^"']+)["'].*?\}|\{[^}]*\})/); + if (kv) { + // Key-only match keeps path/git/url inline tables (no version field) instead of dropping them. + add(kv[1], kv[2] || kv[3] || kv[4], depType); + continue; + } + const quoted = trimmed.match(/^["']([^"']+)["']/); + if (quoted) { + const parsed = extractPythonPackageSpec(quoted[1]); + if (parsed) add(parsed.name, parsed.version, depType); } } } + return deps; } diff --git a/server/prod-server.ts b/server/prod-server.ts index 3e7b82d..14e0c79 100644 --- a/server/prod-server.ts +++ b/server/prod-server.ts @@ -3,9 +3,9 @@ import { handleApiRequest } from './api-router'; import { resetOctokit } from './github/client'; import { addSecurityHeaders } from './utils/http'; import { logInfo } from './utils/logger'; +import { isRepositoryPage } from './utils/page-route'; const distDir = path.resolve(import.meta.dir, '../dist'); -const indexFile = path.join(distDir, 'index.html'); const port = Number(process.env.PORT ?? 3000); const host = process.env.HOST ?? '0.0.0.0'; @@ -14,12 +14,12 @@ resetOctokit(); Bun.serve({ port, hostname: host, - async fetch(req: Request): Promise { + async fetch(req: Request, server): Promise { const url = new URL(req.url); const pathname = url.pathname; if (pathname.startsWith('/api/')) { - const response = await handleApiRequest(req); + const response = await handleApiRequest(req, server.requestIP(req)?.address); if (response) return response; return new Response(JSON.stringify({ error: 'Not found' }), { status: 404, @@ -35,14 +35,29 @@ Bun.serve({ return new Response('Bad Request', { status: 400 }); } - const filePath = safeJoin(distDir, decodedPath); + const pageFile = + decodedPath === '/' + ? '/index.html' + : /^\/(privacy|terms)\/?$/.test(decodedPath) + ? `/${decodedPath.split('/')[1]}.html` + : decodedPath; + const filePath = safeJoin(distDir, pageFile); + if (!filePath) return addSecurityHeaders(new Response('Not found', { status: 404 })); const file = Bun.file(filePath); if (await file.exists()) { - return addSecurityHeaders(new Response(file)); + const res = new Response(file); + if (pathname.startsWith('/assets/')) { + res.headers.set('Cache-Control', 'public, max-age=31536000, immutable'); + } + return addSecurityHeaders(res); } - // SPA fallback - return addSecurityHeaders(new Response(Bun.file(indexFile))); + const validRoute = isRepositoryPage(decodedPath); + const fallbackRes = new Response(Bun.file(path.join(distDir, validRoute ? 'app.html' : '404.html')), { + status: validRoute ? 200 : 404, + }); + fallbackRes.headers.set('Cache-Control', 'no-cache'); + return addSecurityHeaders(fallbackRes); }, }); @@ -50,7 +65,7 @@ const hasToken = Boolean(process.env.GITHUB_TOKEN?.trim()); logInfo(`[gitSdm] listening on http://${host}:${port}`); logInfo(`[gitSdm] GitHub API: ${hasToken ? 'authenticated' : 'unauthenticated'}`); -function safeJoin(root: string, pathname: string): string { +function safeJoin(root: string, pathname: string): string | null { const normalized = pathname.replace(/^[/\\]+/, ''); const resolved = path.resolve(root, normalized); @@ -62,5 +77,5 @@ function safeJoin(root: string, pathname: string): string { return resolved; } - return indexFile; + return null; } diff --git a/server/router/admission.test.ts b/server/router/admission.test.ts new file mode 100644 index 0000000..84c9445 --- /dev/null +++ b/server/router/admission.test.ts @@ -0,0 +1,158 @@ +import { readFile } from 'node:fs/promises'; +import { promises as dnsPromises } from 'node:dns'; +import { test, expect, spyOn } from 'bun:test'; +import { handleApiRequest, isRegisteredApiPath, registeredApiPaths } from '../api-router'; +import { usageLimitTestHooks } from '../utils/usage-limits'; + +// The limiter resolves and re-validates its host before connecting; tests stub DNS so the +// fake 'limiter.invalid' endpoint resolves to a safe public address. +const stubLimiterDns = () => + spyOn(dnsPromises, 'lookup').mockImplementation( + (async () => [{ address: '93.184.216.34', family: 4 }]) as unknown as typeof dnsPromises.lookup, + ); + +// Dispatch literals extracted from the router sources, so this test fails if a route is +// added to a router without registering it — or if a registered path loses its dispatch. +const dispatchedPaths = [ + ...new Set( + ( + await Promise.all( + ['../api-router.ts', './ai-routes.ts', './repo-routes.ts', './search-routes.ts'].map((file) => + readFile(new URL(file, import.meta.url), 'utf8'), + ), + ) + ) + .join('\n') + .match(/pathname === '([^']+)'/g) + ?.map((match) => match.slice("pathname === '".length, -1)) ?? [], + ), +].sort(); + +test('isRegisteredApiPath covers exactly the paths the routers dispatch on', () => { + expect(dispatchedPaths.length).toBeGreaterThan(0); + expect([...registeredApiPaths].sort()).toEqual(dispatchedPaths); + for (const path of ['/api/search/xyz', '/api/ai/unknown', '/api/repo/analyze/extra', '/api/']) + expect(isRegisteredApiPath(path)).toBe(false); + // Non-/api paths are handled elsewhere (SPA/static middleware), never by the API router. + expect(isRegisteredApiPath('/mock/gitsdm')).toBe(false); + expect(isRegisteredApiPath('/')).toBe(false); +}); + +test('trending is rejected before GitHub work when the per-IP budget is exhausted', async () => { + const previous = process.env.API_REQUESTS_PER_IP_PER_MINUTE; + process.env.API_REQUESTS_PER_IP_PER_MINUTE = '0'; + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + const response = await handleApiRequest(new Request('http://localhost/api/trending'), '192.0.2.230'); + expect(response?.status).toBe(429); + const retryAfter = Number(response?.headers.get('Retry-After')); + expect(retryAfter).toBeGreaterThan(0); + expect(retryAfter).toBeLessThanOrEqual(60); + expect(await response?.json()).toMatchObject({ code: 'USAGE_LIMIT_EXCEEDED' }); + } finally { + log.mockRestore(); + if (previous === undefined) delete process.env.API_REQUESTS_PER_IP_PER_MINUTE; + else process.env.API_REQUESTS_PER_IP_PER_MINUTE = previous; + } +}); + +test('shared limiter work is bounded before Redis calls and releases slots after rejection', async () => { + const names = ['RATE_LIMIT_MODE', 'UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN']; + const previous = names.map((name) => process.env[name]); + process.env.RATE_LIMIT_MODE = 'shared'; + process.env.UPSTASH_REDIS_REST_URL = 'https://limiter.invalid'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'unit-test'; + let calls = 0; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + usageLimitTestHooks.postJson = async () => { + calls++; + await gate; + return { result: 0 }; + }; + const log = spyOn(console, 'error').mockImplementation(() => {}); + const lookupStub = stubLimiterDns(); + const work: Array> = []; + try { + for (let i = 0; i < 25; i++) work.push(handleApiRequest(new Request('http://localhost/api/search'), '192.0.2.231')); + // The limiter resolves and re-validates DNS before connecting, so the admitted + // requests reach the gated pinned POST a tick later — flush before asserting. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(calls).toBe(8); + release(); + expect((await Promise.all(work)).every((response) => response?.status === 429)).toBe(true); + await handleApiRequest(new Request('http://localhost/api/search'), '192.0.2.231'); + expect(calls).toBe(9); + } finally { + release(); + await Promise.allSettled(work); + usageLimitTestHooks.postJson = undefined; + lookupStub.mockRestore(); + log.mockRestore(); + names.forEach((name, i) => { + if (previous[i] === undefined) delete process.env[name]; + else process.env[name] = previous[i]; + }); + } +}); + +test('exact search endpoint is subject to usage admission before work starts', async () => { + const previous = process.env.API_REQUESTS_PER_MINUTE; + process.env.API_REQUESTS_PER_MINUTE = '0'; + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + const response = await handleApiRequest( + new Request('http://localhost/api/search', { + method: 'POST', + body: JSON.stringify({ owner: 'mock', repo: 'todo-app', query: 'hello' }), + }), + ); + expect(response?.status).toBe(429); + expect(await response?.json()).toMatchObject({ code: 'USAGE_LIMIT_EXCEEDED' }); + } finally { + log.mockRestore(); + if (previous === undefined) delete process.env.API_REQUESTS_PER_MINUTE; + else process.env.API_REQUESTS_PER_MINUTE = previous; + } +}); + +test('unregistered /api paths return 404 before spending admission or usage budget', async () => { + const names = ['RATE_LIMIT_MODE', 'UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN']; + const previous = names.map((name) => process.env[name]); + process.env.RATE_LIMIT_MODE = 'shared'; + process.env.UPSTASH_REDIS_REST_URL = 'https://limiter.invalid'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'unit-test'; + let pinnedCalls = 0; + usageLimitTestHooks.postJson = async () => { + pinnedCalls++; + return { result: 1 }; + }; + const fetchStub = spyOn(globalThis, 'fetch').mockImplementation(async () => Response.json({ result: 1 })); + const lookupStub = stubLimiterDns(); + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + const unknown = await handleApiRequest(new Request('http://localhost/api/search/xyz'), '192.0.2.232'); + expect(unknown?.status).toBe(404); + expect(await unknown?.json()).toMatchObject({ code: 'NOT_FOUND' }); + expect(pinnedCalls).toBe(0); + expect(fetchStub).not.toHaveBeenCalled(); + // Control: a registered path still reaches the shared limiter. + const known = await handleApiRequest( + new Request('http://localhost/api/search', { method: 'POST', body: '{}' }), + '192.0.2.232', + ); + expect(known?.status).toBe(400); + expect(pinnedCalls).toBeGreaterThan(0); + } finally { + usageLimitTestHooks.postJson = undefined; + fetchStub.mockRestore(); + lookupStub.mockRestore(); + log.mockRestore(); + names.forEach((name, i) => { + if (previous[i] === undefined) delete process.env[name]; + else process.env[name] = previous[i]; + }); + } +}); diff --git a/server/router/repo-routes.ts b/server/router/repo-routes.ts index dca1eb3..e9d57cd 100644 --- a/server/router/repo-routes.ts +++ b/server/router/repo-routes.ts @@ -7,6 +7,14 @@ import { getRepoFileContent } from '../services/get-file'; import { parseRepoParams } from '../github/parse-url'; import { logApi } from '../utils/logger'; import type { RepoAnalysis } from '../../src/types'; +import { enrichRepository } from '../services/repo-enrichment'; +import { z } from 'zod'; + +const continuationSchema = z.object({ + completed: z.array(z.string().max(1000)).max(200).optional(), + pending: z.array(z.string().max(1000)).max(200).optional(), + scope: z.string().max(128).optional(), +}); export async function handleRepoRoutes( pathname: string, @@ -15,6 +23,15 @@ export async function handleRepoRoutes( ctx: RequestContext, start: number, ): Promise { + if (pathname === '/api/repo/churn' || pathname === '/api/repo/health') { + const parsed = repoQuerySchema.safeParse(query); + if (!parsed.success) throw new AppError(400, 'Invalid repository', 'INVALID_PARAMS'); + const continuation = continuationSchema.safeParse(req.method === 'POST' ? await req.json().catch(() => null) : {}); + if (!continuation.success) throw new AppError(400, 'Invalid churn continuation', 'INVALID_PARAMS'); + return Response.json( + await enrichRepository(parsed.data, pathname.endsWith('/churn') ? 'churn' : 'health', ctx, continuation.data), + ); + } if (pathname === '/api/repo/analyze') { const body = await req.json().catch(() => ({})); const parsed = analyzeBodySchema.safeParse(body); @@ -71,11 +88,14 @@ export async function handleRepoRoutes( throw new AppError(400, 'Invalid owner/repo', 'INVALID_PARAMS'); } const analysis: RepoAnalysis = await analyzeRepository(q.data, ctx); - return Response.json({ - tree: analysis.tree, - truncated: analysis.treeTruncated, - importantFiles: analysis.importantFiles, - }, { status: 200 }); + return Response.json( + { + tree: analysis.tree, + truncated: analysis.treeTruncated, + importantFiles: analysis.importantFiles, + }, + { status: 200 }, + ); } if (pathname === '/api/repo/file') { @@ -83,7 +103,13 @@ export async function handleRepoRoutes( if (!q.success) { throw new AppError(400, 'Invalid owner, repo, or path', 'INVALID_PARAMS'); } - const file: { content: string } = await getRepoFileContent(q.data.owner, q.data.repo, q.data.path, q.data.branch, ctx); + const file: { content: string } = await getRepoFileContent( + q.data.owner, + q.data.repo, + q.data.path, + q.data.branch, + ctx, + ); return Response.json(file, { status: 200 }); } diff --git a/server/router/schemas.test.ts b/server/router/schemas.test.ts new file mode 100644 index 0000000..350e2e8 --- /dev/null +++ b/server/router/schemas.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'bun:test'; +import { indexBodySchema } from './schemas'; + +const parseScope = (includePaths?: string[], excludePaths?: string[]) => + indexBodySchema.safeParse({ owner: 'o', repo: 'r', query: 'abc', includePaths, excludePaths }); + +describe('index scope path validation', () => { + it('rejects whole `..` path components in both arrays', () => { + for (const path of ['..', 'a/../b', '../x', 'x/..', 'a/b/../../c']) { + expect(parseScope([path]).success).toBe(false); + expect(parseScope(undefined, [path]).success).toBe(false); + } + }); + + it('accepts names that merely contain `..` inside a component', () => { + for (const path of ['a..b.ts', 'foo..bar', 'src/a..b/x.ts', '..foo/bar']) { + expect(parseScope([path]).success).toBe(true); + expect(parseScope(undefined, [path]).success).toBe(true); + } + }); + + it('keeps the structural rules: no leading slash, no trailing slash, no wildcards', () => { + for (const path of ['/src', 'src/', 'src/**', '']) { + expect(parseScope([path]).success).toBe(false); + } + }); +}); diff --git a/server/router/schemas.ts b/server/router/schemas.ts index 8d28f38..b8a7c5b 100644 --- a/server/router/schemas.ts +++ b/server/router/schemas.ts @@ -2,21 +2,39 @@ import { z } from 'zod'; export const analyzeBodySchema = z.object({ url: z.string().optional(), - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/).optional(), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/).optional(), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/) + .optional(), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/) + .optional(), branch: z.string().optional(), }); export const aiExplainSchema = z .object({ - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), sha: z.string().optional(), scope: z.enum(['repo', 'node', 'file']).default('repo'), nodeId: z.string().optional(), filePath: z.string().optional(), - fileSnippet: z.string().optional(), + fileSnippet: z.string().max(100000).optional(), context: z.string().max(10000).optional(), eli5: z.boolean().optional(), }) @@ -28,12 +46,20 @@ export const aiExplainSchema = z }, { message: 'nodeId is required when scope is "node", filePath is required when scope is "file"', - } + }, ); export const repoQuerySchema = z.object({ - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), goal: z.string().min(1).max(500).optional(), }); @@ -42,22 +68,61 @@ export const fileQuerySchema = repoQuerySchema.extend({ path: z.string().min(1).max(500), }); -export const searchBodySchema = z.object({ +// Scope paths are matched literally against repo file paths (see matchesPath in +// server/search/build-snapshot.ts): repo-relative, no leading slash, no wildcards, +// no trailing slash, and no `..` path component (`a..b` inside a name is fine). +const scopePathSchema = z + .string() + .min(1) + .max(200) + .regex(/^(?!\/)(?!(?:.*\/)?\.\.(?:\/|$))(?:[^/*]+\/)*[^/*]+$/); + +const indexScopeSchema = z.object({ + includePaths: z.array(scopePathSchema).max(16).optional(), + excludePaths: z.array(scopePathSchema).max(16).optional(), +}); + +export const searchBodySchema = indexScopeSchema.extend({ query: z.string().min(3).max(500), - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), }); -export const askBodySchema = z.object({ +export const askBodySchema = indexScopeSchema.extend({ question: z.string().min(3).max(500), - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), }); -export const indexBodySchema = z.object({ - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), +export const indexBodySchema = indexScopeSchema.extend({ + buildId: z.string().uuid().optional(), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), }); diff --git a/server/router/search-cancel.test.ts b/server/router/search-cancel.test.ts new file mode 100644 index 0000000..9b67e58 --- /dev/null +++ b/server/router/search-cancel.test.ts @@ -0,0 +1,135 @@ +import { test, expect, spyOn } from 'bun:test'; +import { Octokit } from '@octokit/rest'; +import { handleSearchRoutes } from './search-routes'; +import { getIndexingPipeline } from '../search/indexing-pipeline'; +import { AppError } from '../utils/errors'; + +test('Cancel still reaches a paused build after Resume is rejected as busy', async () => { + const pipeline = getIndexingPipeline(); + const status = spyOn(pipeline, 'getStatus').mockReturnValue({ state: 'paused', progress: 0 }); + const start = spyOn(pipeline, 'startIndexing').mockRejectedValue(new AppError(429, 'Busy', 'INDEXING_BUSY', true)); + const cancel = spyOn(pipeline, 'cancelIndexing').mockImplementation(() => {}); + try { + const ctx = { + gitHubToken: crypto.randomUUID(), + octokit: new Octokit({ + request: { + fetch: async (input: string | URL | Request) => + String(input).includes('/commits/') + ? Response.json({ sha: 'paused-snapshot' }) + : Response.json({ default_branch: 'main', full_name: 'review/busy', topics: [] }), + }, + }), + }; + const body = { owner: 'review', repo: 'busy', buildId: crypto.randomUUID() }; + const call = (path: string) => + handleSearchRoutes( + path, + new Request('http://localhost' + path, { method: 'POST', body: JSON.stringify(body) }), + {}, + undefined, + ctx, + Date.now(), + ); + await expect(call('/api/search/index')).rejects.toMatchObject({ code: 'INDEXING_BUSY' }); + await call('/api/search/cancel'); + expect(cancel).toHaveBeenCalledTimes(1); + } finally { + status.mockRestore(); + start.mockRestore(); + cancel.mockRestore(); + } +}); + +test('Cancel prevents a metadata-delayed build from starting, with and without a build ID', async () => { + const previous = process.env.EMBEDDING_PROVIDER; + process.env.EMBEDDING_PROVIDER = 'mock'; + try { + for (const buildId of [crypto.randomUUID(), undefined]) { + let release!: () => void; + let entered!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const waiting = new Promise((resolve) => { + entered = resolve; + }); + let metadata = 0; + let trees = 0; + const octokit = new Octokit({ + request: { + fetch: async (input: string | URL | Request) => { + const url = String(input); + if (url.endsWith('/repos/review/cancel-order')) { + if (++metadata === 1) { + entered(); + await gate; + } + return Response.json({ default_branch: 'main', full_name: 'review/cancel-order', topics: [] }); + } + if (url.includes('/commits/')) return Response.json({ sha: 'snapshot' }); + if (url.includes('/git/trees/')) { + trees++; + return Response.json({ tree: [], truncated: false }); + } + throw new Error('Unexpected mocked request'); + }, + }, + }); + const ctx = { octokit, gitHubToken: crypto.randomUUID() }; + const call = (path: string) => + handleSearchRoutes( + path, + new Request('http://localhost' + path, { + method: 'POST', + body: JSON.stringify({ owner: 'review', repo: 'cancel-order', buildId }), + }), + {}, + undefined, + ctx, + Date.now(), + ); + const start = call('/api/search/index'); + await waiting; + expect(await (await call('/api/search/cancel'))?.json()).toMatchObject({ state: 'idle' }); + release(); + expect(await (await start)?.json()).toMatchObject({ state: 'idle' }); + expect(trees).toBe(0); + if (!buildId) { + expect(await (await call('/api/search/index'))?.json()).toMatchObject({ state: 'complete' }); + expect(trees).toBe(1); + } + } + } finally { + if (previous === undefined) delete process.env.EMBEDDING_PROVIDER; + else process.env.EMBEDDING_PROVIDER = previous; + } +}); + +test('a Cancel arriving before Start is remembered without GitHub requests', async () => { + const ctx = { + octokit: new Octokit({ + request: { + fetch: async () => { + throw new Error('No GitHub work expected'); + }, + }, + }), + gitHubToken: 'review-before-start', + }; + const body = { owner: 'review', repo: 'cancel-order', buildId: crypto.randomUUID() }; + for (const path of ['/api/search/cancel', '/api/search/index']) { + const response = await handleSearchRoutes( + path, + new Request('http://localhost' + path, { + method: 'POST', + body: JSON.stringify(body), + }), + {}, + undefined, + ctx, + Date.now(), + ); + expect(await response?.json()).toMatchObject({ state: 'idle' }); + } +}); diff --git a/server/router/search-routes.ts b/server/router/search-routes.ts index 9683c6d..cc0f83c 100644 --- a/server/router/search-routes.ts +++ b/server/router/search-routes.ts @@ -1,11 +1,14 @@ import type { RequestContext } from '../utils/context'; import { AppError } from '../utils/errors'; -import { searchBodySchema, askBodySchema, indexBodySchema } from './schemas'; +import { searchBodySchema, askBodySchema, indexBodySchema, repoQuerySchema } from './schemas'; import { fetchRepoInfo } from '../github/fetch-tree'; import { getSearchEngine } from '../search/search-engine'; import { getQAEngine } from '../search/qa-engine'; import { getIndexingPipeline } from '../search/indexing-pipeline'; import { logApi } from '../utils/logger'; +import { searchIndexKey, indexRequestKey } from '../search/index-identity'; +import { indexRequests } from '../search/index-requests'; +import { hashContext } from '../cache/lru'; export async function handleSearchRoutes( pathname: string, @@ -28,8 +31,14 @@ export async function handleSearchRoutes( owner: parsed.data.owner, repo: parsed.data.repo, commitSha: info.sha, + gitHubToken: ctx.gitHubToken, + includePaths: parsed.data.includePaths, + excludePaths: parsed.data.excludePaths, + }); + logApi('/api/search', { + durationMs: Date.now() - start, + results: result.results.length, }); - logApi('/api/search', { durationMs: Date.now() - start, results: result.results.length }); return Response.json(result, { status: 200 }); } @@ -47,48 +56,150 @@ export async function handleSearchRoutes( repo: parsed.data.repo, commitSha: info.sha, apiKey: userKey, + gitHubToken: ctx.gitHubToken, + includePaths: parsed.data.includePaths, + excludePaths: parsed.data.excludePaths, + }); + logApi('/api/search/ask', { + durationMs: Date.now() - start, + citations: result.citations.length, }); - logApi('/api/search/ask', { durationMs: Date.now() - start, citations: result.citations.length }); return Response.json(result, { status: 200 }); } - if (pathname === '/api/search/index') { + if (pathname === '/api/search/index' || pathname === '/api/search/cancel') { const body = await req.json().catch(() => ({})); const parsed = indexBodySchema.safeParse(body); if (!parsed.success) { throw new AppError(400, 'Invalid index request', 'VALIDATION_ERROR', false, parsed.error.flatten()); } - const info = await fetchRepoInfo(parsed.data.owner, parsed.data.repo, parsed.data.branch, ctx); - const pipeline = getIndexingPipeline(); - + const requestKey = parsed.data.buildId + ? indexRequestKey(parsed.data.owner, parsed.data.repo, parsed.data.buildId, ctx) + : undefined; + if (pathname === '/api/search/cancel' && requestKey) { + indexRequests.cancel(requestKey); + return Response.json({ state: 'idle' }); + } + const legacyKey = indexRequestKey( + parsed.data.owner, + parsed.data.repo, + hashContext(JSON.stringify([parsed.data.branch, parsed.data.includePaths, parsed.data.excludePaths])), + ctx, + ); + if (pathname === '/api/search/cancel' && !requestKey) { + // Legacy builds carry no buildId: cancelPending misses once the registry + // entry settles, so remove the registration and fall through to the + // pipeline cancel below instead of leaking it. + if (indexRequests.cancelPending(legacyKey)) return Response.json({ state: 'idle' }); + indexRequests.cancel(legacyKey); + } + const registration = + pathname === '/api/search/index' ? indexRequests.begin(requestKey ?? legacyKey, parsed.data.buildId) : undefined; + let resumable = false; try { - await pipeline.startIndexing({ - owner: parsed.data.owner, - repo: parsed.data.repo, - branch: parsed.data.branch, - commitSha: info.sha, - }, ctx); - } catch (err) { - if (err instanceof AppError && err.status === 409) { - return Response.json({ status: 'rejected', error: 'Indexing already in progress' }, { status: 409 }); + if (registration?.cancelled()) return Response.json({ state: 'idle' }); + const info = await fetchRepoInfo(parsed.data.owner, parsed.data.repo, parsed.data.branch, ctx); + if (registration?.cancelled()) return Response.json({ state: 'idle' }); + const pipeline = getIndexingPipeline(); + const snapshotKey = searchIndexKey( + parsed.data.owner, + parsed.data.repo, + info.sha, + ctx, + parsed.data.includePaths, + parsed.data.excludePaths, + ); + if (pathname === '/api/search/index' && pipeline.getStatus(snapshotKey).state === 'indexing') + throw new AppError(409, 'Indexing is already in progress.', 'INDEXING_IN_PROGRESS'); + registration?.attach(snapshotKey, () => pipeline.cancelIndexing(snapshotKey)); + if (pathname === '/api/search/cancel') { + const key = searchIndexKey( + parsed.data.owner, + parsed.data.repo, + info.sha, + ctx, + parsed.data.includePaths, + parsed.data.excludePaths, + ); + pipeline.cancelIndexing(key); + return Response.json(pipeline.getStatus(key)); } - throw err; - } - logApi('/api/search/index', { durationMs: Date.now() - start, repo: `${parsed.data.owner}/${parsed.data.repo}` }); - return Response.json({ status: 'started' }, { status: 200 }); + try { + await pipeline.startIndexing( + { + owner: parsed.data.owner, + repo: parsed.data.repo, + branch: parsed.data.branch, + commitSha: info.sha, + includePaths: parsed.data.includePaths, + excludePaths: parsed.data.excludePaths, + }, + ctx, + ); + } catch (err) { + if (err instanceof AppError && err.status === 409) { + return Response.json({ status: 'rejected', error: 'Indexing already in progress' }, { status: 409 }); + } + throw err; + } finally { + resumable = pipeline.getStatus(snapshotKey).state === 'paused'; + } + + logApi('/api/search/index', { + durationMs: Date.now() - start, + repo: `${parsed.data.owner}/${parsed.data.repo}`, + }); + return Response.json( + pipeline.getStatus( + searchIndexKey( + parsed.data.owner, + parsed.data.repo, + info.sha, + ctx, + parsed.data.includePaths, + parsed.data.excludePaths, + ), + ), + { + status: 200, + }, + ); + } finally { + registration?.finish(resumable, !!requestKey); + } } if (pathname === '/api/search/status') { - const owner = query.owner; - const repo = query.repo; - if (!owner || !repo) { + const parsed = repoQuerySchema.safeParse(query); + if (!parsed.success) { throw new AppError(400, 'owner and repo query params required', 'INVALID_PARAMS'); } + const { owner, repo, branch } = parsed.data; const pipeline = getIndexingPipeline(); - const status = pipeline.getStatus(`${owner}/${repo}`); - return Response.json(status, { status: 200 }); + const info = await fetchRepoInfo(owner, repo, branch, ctx); + const includePaths = splitPaths(query.includePaths); + const excludePaths = splitPaths(query.excludePaths); + const status = pipeline.getStatus(searchIndexKey(owner, repo, info.sha, ctx, includePaths, excludePaths)); + return Response.json( + { + ...status, + buildId: indexRequests.buildId(searchIndexKey(owner, repo, info.sha, ctx, includePaths, excludePaths)), + }, + { status: 200 }, + ); } return null; } + +function splitPaths(value: string | undefined): string[] { + if (!value) return []; + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed)) return parsed.map(String).filter(Boolean); + } catch { + // legacy comma-separated string + } + return value.split(',').filter(Boolean); +} diff --git a/server/search/build-snapshot.ts b/server/search/build-snapshot.ts new file mode 100644 index 0000000..d819e32 --- /dev/null +++ b/server/search/build-snapshot.ts @@ -0,0 +1,166 @@ +import type { IndexingOptions, IndexingStatus, IndexedChunk, Chunk, RequestContext } from './types'; +import { SUPPORTED_EXTENSIONS, extToLanguage } from './constants'; +import { createEmbeddingProvider } from './embedding-provider'; +import { createChunker } from './chunker'; +import { getVectorStore, chunkBytes } from './vector-store'; +import { createCheckpoints } from './checkpoints'; +import { fetchFlatTree, fetchFileContents } from '../github/fetch-tree'; +import { AppError } from '../utils/errors'; +import { SEARCH_LIMITS } from './limits'; + +export const indexingDependencies = { + fetchFlatTree, + fetchFileContents, + createEmbeddingProvider, + createChunker, + getVectorStore, +}; +export async function buildSnapshot( + deps: typeof indexingDependencies, + limits: typeof SEARCH_LIMITS, + key: string, + options: IndexingOptions, + ctx: RequestContext, + checkpoints: ReturnType, + check: () => void, + report: (status: IndexingStatus) => void, +) { + const { owner, repo, commitSha, includePaths = [], excludePaths = [] } = options; + const tree = await deps.fetchFlatTree(owner, repo, commitSha, ctx); + check(); + const files = tree.items.filter( + (item) => + SUPPORTED_EXTENSIONS.has(extension(item.path)) && + (includePaths.length === 0 || includePaths.some((prefix) => matchesPath(item.path, prefix))) && + !excludePaths.some((prefix) => matchesPath(item.path, prefix)), + ); + if (tree.truncated || files.length > limits.files || files.some((file) => (file.size ?? 0) > limits.fileBytes)) + throw new AppError(413, 'Repository exceeds search indexing file limits.', 'INDEX_TOO_LARGE'); + const provider = await deps.createEmbeddingProvider(); + const chunker = deps.createChunker(); + const checkpoint = checkpoints.get(key); + const indexed: IndexedChunk[] = [...(checkpoint?.chunks ?? [])]; + const completed = new Set(checkpoint?.completed ?? []); + const embedded = new Set(indexed.map((chunk) => chunk.metadata.filePath + ':' + chunk.metadata.chunkIndex)); + let bytes = checkpoint?.bytes ?? 0; + let processed = completed.size; + let pending: Array<{ chunk: Chunk; lastInFile: boolean }> = []; + let pendingBytes = 0; + let pendingMemory = 0; + const reportProgress = () => + report({ + state: 'indexing', + progress: files.length ? Math.round((processed / files.length) * 100) : 0, + filesProcessed: processed, + totalFiles: files.length, + }); + const estimateChunk = (chunk: Chunk) => + provider.dimensions * 4 + 2 * (chunk.content.length + key.length * 2 + chunk.filePath.length * 2) + 1024; + const flush = async () => { + if (!pending.length) return; + check(); + const embeddings = await provider.embedBatch(pending.map(({ chunk }) => chunk.content)); + check(); + if (embeddings.length !== pending.length) + throw new AppError(502, 'Embedding provider returned an incomplete batch.', 'INDEXING_FAILED', true); + if (embeddings.some(({ vector }) => vector.length !== provider.dimensions || !vector.every(Number.isFinite))) + throw new AppError(502, 'Embedding provider returned an invalid vector.', 'INDEXING_FAILED', true); + for (let i = 0; i < pending.length; i++) { + const { chunk, lastInFile } = pending[i]; + const vector = embeddings[i].vector; + if (vector.length !== provider.dimensions || !vector.every(Number.isFinite)) + throw new AppError(502, 'Embedding provider returned an invalid vector.', 'INDEXING_FAILED', true); + const entry: IndexedChunk = { + id: key + ':' + chunk.filePath + ':' + chunk.chunkIndex, + vector, + metadata: { + filePath: chunk.filePath, + startLine: chunk.startLine, + endLine: chunk.endLine, + chunkIndex: chunk.chunkIndex, + language: chunk.language, + content: chunk.content.slice(0, 2000), + repoKey: key, + commitSha, + }, + }; + bytes += chunkBytes(entry); + if (bytes > limits.indexBytes) throw new AppError(413, 'Search index memory limit exceeded.', 'INDEX_TOO_LARGE'); + indexed.push(entry); + if (lastInFile) { + completed.add(chunk.filePath); + processed++; + } + } + checkpoints.save(key, indexed, completed, files.length); + pending = []; + pendingBytes = 0; + pendingMemory = 0; + reportProgress(); + }; + reportProgress(); + checkpoints.save(key, indexed, completed, files.length); + // Four files in flight, at most 1 MiB of source per group with the default file cap. + for (let offset = 0; offset < files.length; offset += 4) { + const group = files.slice(offset, offset + 4).filter((file) => !completed.has(file.path)); + if (!group.length) continue; + check(); + const contents = await deps.fetchFileContents( + owner, + repo, + group.map((file) => file.path), + commitSha, + ctx, + limits.fileBytes, + ); + check(); + for (const file of group) { + const content = contents[file.path]; + if (content === undefined) + throw new AppError(502, 'A repository file could not be read.', 'INDEXING_FAILED', true); + if (Buffer.byteLength(content, 'utf8') > limits.fileBytes) + throw new AppError(413, 'A file exceeds the search indexing size limit.', 'INDEX_TOO_LARGE'); + const chunks = chunker + .chunkFile(content, file.path, extToLanguage(extension(file.path)), { + maxChunks: limits.chunks, + maxBytes: limits.indexBytes, + bytesPerChunk: provider.dimensions * 4 + 4 * (key.length + file.path.length) + 1024, + }) + .filter((chunk) => !embedded.has(chunk.filePath + ':' + chunk.chunkIndex)); + if (indexed.length + pending.length + chunks.length > limits.chunks) + throw new AppError(413, 'Repository exceeds the search chunk limit.', 'INDEX_TOO_LARGE'); + // Reserve conservatively before paying for embeddings or retaining their vectors. + const estimated = chunks.reduce((sum, chunk) => sum + estimateChunk(chunk), 0); + if (bytes + pendingMemory + estimated > limits.indexBytes) + throw new AppError(413, 'Repository exceeds the search index memory limit.', 'INDEX_TOO_LARGE'); + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const size = Buffer.byteLength(chunk.content, 'utf8'); + if (size > 256000) throw new AppError(413, 'Embedding chunk is too large.', 'INDEX_TOO_LARGE'); + if (pending.length && (pending.length >= 32 || pendingBytes + size > 256000)) await flush(); + pending.push({ chunk, lastInFile: i === chunks.length - 1 }); + pendingBytes += size; + pendingMemory += estimateChunk(chunk); + if (pending.length === 32) await flush(); + } + if (!chunks.length) { + completed.add(file.path); + processed++; + checkpoints.save(key, indexed, completed, files.length); + reportProgress(); + } + } + } + await flush(); + check(); + return { indexed, totalFiles: files.length }; +} +function extension(path: string): string { + const base = path.split('/').pop()?.toLowerCase(); + if (base === 'dockerfile') return '.dockerfile'; + const dot = path.lastIndexOf('.'); + return dot < 0 ? '' : path.slice(dot).toLowerCase(); +} +function matchesPath(path: string, prefix: string): boolean { + return path === prefix || path.startsWith(`${prefix}/`); +} diff --git a/server/search/checkpoint-cleanup.test.ts b/server/search/checkpoint-cleanup.test.ts new file mode 100644 index 0000000..be9610e --- /dev/null +++ b/server/search/checkpoint-cleanup.test.ts @@ -0,0 +1,53 @@ +import { test, expect, spyOn } from 'bun:test'; +import { Octokit } from '@octokit/rest'; +import { createIndexingPipeline } from './indexing-pipeline'; +import { createVectorStore } from './vector-store'; +import { SEARCH_LIMITS } from './limits'; +import { searchIndexKey } from './index-identity'; + +test('terminal failures release capacity for unrelated repositories without deleting complete indexes', async () => { + const limits = { ...SEARCH_LIMITS, chunks: 1, indices: 1 }; + const store = createVectorStore(limits); + const ctx = { octokit: new Octokit(), gitHubToken: 'test' }; + const pipeline = createIndexingPipeline( + { + getVectorStore: () => store, + fetchFlatTree: async () => ({ + items: [{ path: 'a.ts', type: 'blob', sha: 'x', size: 10 }], + truncated: false, + totalFiles: 1, + }), + fetchFileContents: async (_owner, repo) => ({ 'a.ts': repo }), + createChunker: () => ({ + chunkFile: (content) => + Array.from({ length: content === 'oversize' ? 2 : 1 }, (_, chunkIndex) => ({ + content: 'x', + filePath: 'a.ts', + chunkIndex, + startLine: 1, + endLine: 1, + language: 'ts', + })), + }), + createEmbeddingProvider: async () => ({ + dimensions: 1, + maxTokens: 10, + providerName: 'test', + embed: async () => ({ vector: new Float32Array([1]), tokenCount: 1 }), + embedBatch: async (texts) => texts.map(() => ({ vector: new Float32Array([1]), tokenCount: 1 })), + }), + }, + limits, + ); + const options = { owner: 'test', repo: 'valid', commitSha: 'one' }; + await pipeline.startIndexing(options, ctx); + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + await expect(pipeline.startIndexing({ ...options, repo: 'oversize' }, ctx)).rejects.toMatchObject({ status: 413 }); + expect(store.hasIndex(searchIndexKey('test', 'valid', 'one', ctx))).toBe(true); + await pipeline.startIndexing({ ...options, repo: 'another' }, ctx); + expect(pipeline.getStatus(searchIndexKey('test', 'another', 'one', ctx)).state).toBe('complete'); + } finally { + log.mockRestore(); + } +}); diff --git a/server/search/checkpoints.ts b/server/search/checkpoints.ts new file mode 100644 index 0000000..71899de --- /dev/null +++ b/server/search/checkpoints.ts @@ -0,0 +1,48 @@ +import type { IndexedChunk } from './types'; +import { chunkBytes } from './vector-store'; +import { SEARCH_LIMITS } from './limits'; +import { AppError } from '../utils/errors'; + +export interface Checkpoint { + chunks: IndexedChunk[]; + completed: Set; + totalFiles: number; + bytes: number; + expires: number; +} + +/** Separate, bounded staging area; published indexes are never overwritten by partial data. */ +export function createCheckpoints(limits = SEARCH_LIMITS, now = Date.now) { + const entries = new Map(); + const prune = () => { + for (const [key, value] of entries) if (value.expires <= now()) entries.delete(key); + }; + return { + get(key: string) { + prune(); + return entries.get(key); + }, + remove(key: string) { + entries.delete(key); + }, + save(key: string, chunks: IndexedChunk[], completed: Set, totalFiles: number) { + prune(); + const bytes = chunks.reduce((sum, chunk) => sum + chunkBytes(chunk), 0); + const others = [...entries].filter(([id]) => id !== key); + if ( + bytes > limits.indexBytes || + chunks.length > limits.chunks || + others.reduce((sum, [, value]) => sum + value.bytes, 0) + bytes > limits.totalBytes || + (!entries.has(key) && entries.size >= limits.indices) + ) + throw new AppError(413, 'Saved indexing progress is at capacity.', 'INDEX_TOO_LARGE'); + entries.set(key, { + chunks: [...chunks], + completed: new Set(completed), + totalFiles, + bytes, + expires: now() + Math.max(limits.ttlMs, 25 * 60 * 60 * 1000), + }); + }, + }; +} diff --git a/server/search/chunker.test.ts b/server/search/chunker.test.ts index e288309..d6e6b60 100644 --- a/server/search/chunker.test.ts +++ b/server/search/chunker.test.ts @@ -7,6 +7,24 @@ const MAX_CHUNK_CHARS = MAX_CHUNK_TOKENS * 4; describe('createChunker', () => { const chunker = createChunker(); + it('does not duplicate nested blocks', () => { + const content = 'function f(){\n'.repeat(1000) + '}\n'.repeat(1000); + const chunks = chunker.chunkFile(content, 'nested.ts', 'typescript'); + expect(chunks.length).toBeLessThan(20); + expect(chunks.reduce((sum, chunk) => sum + chunk.content.length, 0)).toBeLessThanOrEqual(content.length); + }); + + it('stops emitting when count or byte budgets are exhausted', () => { + const content = Array.from({ length: 1000 }, (_, i) => `function f${i}() { return ${i}; }`).join('\n'); + for (const budget of [ + { maxChunks: 2, maxBytes: 100000, bytesPerChunk: 100 }, + { maxChunks: 1000, maxBytes: 100, bytesPerChunk: 100 }, + ]) { + expect(() => chunker.chunkFile(content, 'many.ts', 'typescript', budget)).toThrow('search chunk or memory limit'); + expect(() => chunker.chunkFile(content, 'many.txt', 'text', budget)).toThrow('search chunk or memory limit'); + } + }); + describe('chunkFile', () => { it('returns empty array for empty content', () => { expect(chunker.chunkFile('', 'src/a.ts', 'typescript')).toEqual([]); @@ -98,7 +116,9 @@ describe('createChunker', () => { }); it('sub-splits oversized AST blocks', () => { - const content = `function huge() {\n${Array(4).fill(' ' + 'x'.repeat(MAX_CHUNK_CHARS)).join('\n')}\n}`; + const content = `function huge() {\n${Array(4) + .fill(' ' + 'x'.repeat(MAX_CHUNK_CHARS)) + .join('\n')}\n}`; const chunks = chunker.chunkFile(content, 'huge.ts', 'typescript'); expect(chunks.length).toBeGreaterThan(1); }); @@ -141,7 +161,9 @@ describe('createChunker', () => { it('splits large content across multiple chunks', () => { const line = 'x'.repeat(100); - const content = Array(Math.ceil((MAX_CHUNK_CHARS / 100) * 3)).fill(line).join('\n'); + const content = Array(Math.ceil((MAX_CHUNK_CHARS / 100) * 3)) + .fill(line) + .join('\n'); const chunks = chunker.chunkFile(content, 'big.go', 'go'); expect(chunks.length).toBeGreaterThanOrEqual(2); }); diff --git a/server/search/chunker.ts b/server/search/chunker.ts index 3884bea..7367520 100644 --- a/server/search/chunker.ts +++ b/server/search/chunker.ts @@ -1,26 +1,26 @@ import type { Chunk, Chunker } from './types'; +import { AppError } from '../utils/errors'; import { MAX_CHUNK_TOKENS, AST_SUPPORTED_LANGUAGES } from './constants'; const MAX_CHUNK_CHARS = MAX_CHUNK_TOKENS * 4; // ~4 chars per token export function createChunker(): Chunker { return { - chunkFile(content: string, filePath: string, language: string): Chunk[] { + chunkFile(content, filePath, language, budget): Chunk[] { if (!content || content.trim().length === 0) return []; - - const useAst = AST_SUPPORTED_LANGUAGES.has(language); - const rawChunks = useAst - ? astAwareChunk(content, language) - : slidingWindowChunk(content); - - return rawChunks.map((c, i) => ({ - content: c.content, - startLine: c.startLine, - endLine: c.endLine, - chunkIndex: i, - language, - filePath, - })); + const chunks: Chunk[] = []; + let bytes = 0; + const emit = (chunk: RawChunk) => { + const nextBytes = bytes + chunk.content.length * 2 + (budget?.bytesPerChunk ?? 512); + if (budget && (chunks.length >= budget.maxChunks || nextBytes > budget.maxBytes)) { + throw new AppError(413, 'Repository exceeds the search chunk or memory limit.', 'INDEX_TOO_LARGE'); + } + bytes = nextBytes; + chunks.push({ ...chunk, chunkIndex: chunks.length, language, filePath }); + }; + if (AST_SUPPORTED_LANGUAGES.has(language)) astAwareChunk(content, language, emit); + else slidingWindowChunk(content, emit); + return chunks; }, }; } @@ -33,9 +33,8 @@ interface RawChunk { endLine: number; } -function slidingWindowChunk(content: string): RawChunk[] { +function slidingWindowChunk(content: string, emit: (chunk: RawChunk) => void): void { const lines = content.split('\n'); - const chunks: RawChunk[] = []; let current = ''; let startLine = 1; @@ -50,7 +49,7 @@ function slidingWindowChunk(content: string): RawChunk[] { const before = current.slice(0, breakAt); const after = current.slice(breakAt + 1); const beforeLines = before.split('\n'); - chunks.push({ + emit({ content: before, startLine: startLine, endLine: startLine + beforeLines.length - 1, @@ -60,7 +59,7 @@ function slidingWindowChunk(content: string): RawChunk[] { } else { // Hard split const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine: startLine, endLine: startLine + currentLines.length - 1, @@ -75,14 +74,12 @@ function slidingWindowChunk(content: string): RawChunk[] { if (current.length > 0) { const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine, endLine: startLine + currentLines.length - 1, }); } - - return chunks.length > 0 ? chunks : [{ content, startLine: 1, endLine: lines.length }]; } function findLastBlankLineBreak(text: string): number { @@ -106,15 +103,15 @@ function findLastBlankLineBreak(text: string): number { * Lightweight regex-based chunking that splits at function/class boundaries * for TypeScript, JavaScript, and Python files. */ -function astAwareChunk(content: string, language: string): RawChunk[] { +function astAwareChunk(content: string, language: string, emit: (chunk: RawChunk) => void): void { const lines = content.split('\n'); const boundaries = findBoundaries(lines, language); if (boundaries.length === 0) { - return slidingWindowChunk(content); + slidingWindowChunk(content, emit); + return; } - const chunks: RawChunk[] = []; let prevEnd = 0; for (const boundary of boundaries) { @@ -122,21 +119,21 @@ function astAwareChunk(content: string, language: string): RawChunk[] { if (boundary.startLine > prevEnd) { const before = lines.slice(prevEnd, boundary.startLine).join('\n'); if (before.trim().length > 0) { - addSizedChunks(chunks, before, prevEnd + 1); + addSizedChunks(emit, before, prevEnd + 1); } } // Add the boundary block (function/class) const block = lines.slice(boundary.startLine, boundary.endLine + 1).join('\n'); if (block.length <= MAX_CHUNK_CHARS) { - chunks.push({ + emit({ content: block, startLine: boundary.startLine + 1, endLine: boundary.endLine + 1, }); } else { // Sub-split large blocks - addSizedChunks(chunks, block, boundary.startLine + 1); + addSizedChunks(emit, block, boundary.startLine + 1); } prevEnd = boundary.endLine + 1; @@ -146,11 +143,9 @@ function astAwareChunk(content: string, language: string): RawChunk[] { if (prevEnd < lines.length) { const remaining = lines.slice(prevEnd).join('\n'); if (remaining.trim().length > 0) { - addSizedChunks(chunks, remaining, prevEnd + 1); + addSizedChunks(emit, remaining, prevEnd + 1); } } - - return chunks.length > 0 ? chunks : slidingWindowChunk(content); } interface Boundary { @@ -168,6 +163,7 @@ function findBoundaries(lines: string[], language: string): Boundary[] { if (pattern.test(trimmed)) { const end = findBlockEnd(lines, i, language); boundaries.push({ startLine: i, endLine: end }); + i = end; // Nested blocks are already included in the outer block. break; } } @@ -187,10 +183,7 @@ const JS_PATTERNS: RegExp[] = [ /^(export\s+)?abstract\s+class\s+/, ]; -const PY_PATTERNS: RegExp[] = [ - /^(async\s+)?def\s+/, - /^class\s+/, -]; +const PY_PATTERNS: RegExp[] = [/^(async\s+)?def\s+/, /^class\s+/]; const PATTERN_MAP: Record = { typescript: JS_PATTERNS, @@ -248,7 +241,7 @@ function findPythonBlockEnd(lines: string[], startLine: number): number { return end; } -function addSizedChunks(chunks: RawChunk[], content: string, baseLine: number): void { +function addSizedChunks(emit: (chunk: RawChunk) => void, content: string, baseLine: number): void { const lines = content.split('\n'); let current = ''; let startLine = baseLine; @@ -259,7 +252,7 @@ function addSizedChunks(chunks: RawChunk[], content: string, baseLine: number): if (test.length > MAX_CHUNK_CHARS && current.length > 0) { const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine, endLine: startLine + currentLines.length - 1, @@ -273,7 +266,7 @@ function addSizedChunks(chunks: RawChunk[], content: string, baseLine: number): if (current.length > 0) { const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine, endLine: startLine + currentLines.length - 1, diff --git a/server/search/embedding-config.test.ts b/server/search/embedding-config.test.ts new file mode 100644 index 0000000..01058d3 --- /dev/null +++ b/server/search/embedding-config.test.ts @@ -0,0 +1,62 @@ +import { expect, test } from 'bun:test'; +import { embeddingConfig } from './embedding-config'; + +test('blank dedicated settings use nonempty legacy settings', () => { + expect( + embeddingConfig({ + EMBEDDING_PROVIDER: ' ', + AI_PROVIDER: 'openai', + EMBEDDING_API_KEY: '', + EMBEDDING_API_BASE: ' ', + EMBEDDING_MODEL: '', + OPENAI_API_KEY: ' key ', + OPENAI_API_BASE: 'https://legacy.example/v1', + OPENAI_EMBEDDING_MODEL: 'model', + }), + ).toMatchObject({ provider: 'openai', apiKey: 'key', baseURL: 'https://legacy.example/v1', model: 'model' }); +}); + +test('explicit embedding configuration is independent of chat settings', () => { + const env = { + EMBEDDING_PROVIDER: 'openai', + EMBEDDING_API_KEY: 'embedding-key', + EMBEDDING_API_BASE: 'https://embedding.example/v1', + EMBEDDING_MODEL: 'embed-model', + }; + expect( + embeddingConfig({ + ...env, + AI_PROVIDER: 'anthropic', + OPENAI_API_KEY: 'chat-key', + OPENAI_API_BASE: 'https://chat.example/v1', + }), + ).toEqual(embeddingConfig({ ...env, AI_PROVIDER: 'gemini' })); +}); + +test('existing OpenAI settings remain supported', () => { + expect( + embeddingConfig({ + AI_PROVIDER: 'openai', + OPENAI_API_KEY: 'key', + OPENAI_API_BASE: 'https://legacy.example/v1', + OPENAI_EMBEDDING_MODEL: 'legacy-model', + }), + ).toEqual({ provider: 'openai', apiKey: 'key', baseURL: 'https://legacy.example/v1', model: 'legacy-model' }); +}); + +test('non-HTTPS or private embedding endpoints fail closed before the key is used', () => { + for (const base of ['http://embeddings.example/v1', 'https://10.0.0.5/v1', 'https://169.254.169.254/v1']) { + for (const env of [ + { AI_PROVIDER: 'openai', OPENAI_API_KEY: 'key', OPENAI_API_BASE: base }, + { EMBEDDING_PROVIDER: 'openai', EMBEDDING_API_KEY: 'key', EMBEDDING_API_BASE: base }, + ]) { + let error: unknown; + try { + embeddingConfig(env); + } catch (err) { + error = err; + } + expect(error).toMatchObject({ code: 'EMBEDDING_PROVIDER_UNAVAILABLE' }); + } + } +}); diff --git a/server/search/embedding-config.ts b/server/search/embedding-config.ts new file mode 100644 index 0000000..56ad1d1 --- /dev/null +++ b/server/search/embedding-config.ts @@ -0,0 +1,31 @@ +import { AppError } from '../utils/errors'; +import { isSafeRemoteUrl } from '../utils/url-guard'; + +/** Explicit embedding settings take precedence; legacy provider settings remain compatible. */ +export function embeddingConfig(env: Record = process.env) { + const value = (name: string) => env[name]?.trim() || undefined; + const explicit = value('EMBEDDING_PROVIDER')?.toLowerCase(); + const requested = (explicit ?? value('AI_PROVIDER') ?? 'mock').toLowerCase(); + const provider = + !explicit && requested === 'anthropic' + ? value('GEMINI_API_KEY') + ? 'gemini' + : value('OPENAI_API_KEY') + ? 'openai' + : requested + : requested; + const baseURL = value('EMBEDDING_API_BASE') ?? value('OPENAI_API_BASE'); + // Fail closed before the key leaves the process: only public HTTPS endpoints allowed. + if (baseURL && !isSafeRemoteUrl(baseURL)) + throw new AppError(400, 'Embedding endpoint must be a public HTTPS URL.', 'EMBEDDING_PROVIDER_UNAVAILABLE'); + return { + provider, + apiKey: value('EMBEDDING_API_KEY') ?? (provider === 'gemini' ? value('GEMINI_API_KEY') : value('OPENAI_API_KEY')), + baseURL, + model: + value('EMBEDDING_MODEL') ?? + (provider === 'gemini' + ? (value('GEMINI_EMBEDDING_MODEL') ?? 'gemini-embedding-001') + : (value('OPENAI_EMBEDDING_MODEL') ?? 'openrouter/openai/text-embedding-3-large')), + }; +} diff --git a/server/search/embedding-controls.test.ts b/server/search/embedding-controls.test.ts new file mode 100644 index 0000000..e9fc211 --- /dev/null +++ b/server/search/embedding-controls.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from 'bun:test'; +import { protectEmbeddings } from './embedding-controls'; +import type { EmbeddingProvider } from './types'; +import { AppError } from '../utils/errors'; + +test('embedding budget charges actual attempts instead of hypothetical retries', async () => { + const charges: number[] = []; + let attempts = 0; + const provider: EmbeddingProvider = { + providerName: 'test', + dimensions: 1, + maxTokens: 100, + embed: async () => { + if (++attempts === 2) throw new Error('timeout'); + return { vector: new Float32Array([1]), tokenCount: 1 }; + }, + embedBatch: async () => [], + }; + const protectedProvider = protectEmbeddings(provider, async (_kind, bytes) => { + charges.push(bytes); + }); + await protectedProvider.embed('hello'); + expect(charges).toEqual([5]); + await protectedProvider.embed('hello'); + expect(attempts).toBe(3); + expect(charges).toEqual([5, 5, 5]); +}); + +test('a timeout retry honors cooldown from another request without reserving more budget', async () => { + let releaseTimeout!: () => void; + const timeoutGate = new Promise((resolve) => { + releaseTimeout = resolve; + }); + let attempts = 0; + let reservations = 0; + const provider: EmbeddingProvider = { + providerName: 'test', + dimensions: 1, + maxTokens: 100, + embed: async (text) => { + if (text === 'timeout') { + attempts++; + await timeoutGate; + throw new Error('timeout'); + } + throw new AppError(429, 'Cooling down', 'EMBEDDING_RATE_LIMITED', true, { retryAfterSeconds: 2 }); + }, + embedBatch: async () => [], + }; + const protectedProvider = protectEmbeddings(provider, async () => { + reservations++; + }); + const pending = protectedProvider.embed('timeout').then( + () => null, + (error: unknown) => error, + ); + await expect(protectedProvider.embed('limited')).rejects.toMatchObject({ code: 'EMBEDDING_RATE_LIMITED' }); + releaseTimeout(); + expect(await pending).toMatchObject({ code: 'EMBEDDING_RATE_LIMITED' }); + expect(attempts).toBe(1); + expect(reservations).toBe(2); + await new Promise((resolve) => setTimeout(resolve, 2100)); +}, 10000); + +test('a delayed reservation cannot dispatch after another request starts a cooldown', async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let reservations = 0; + const dispatched: string[] = []; + const provider: EmbeddingProvider = { + providerName: 'test', + dimensions: 1, + maxTokens: 100, + embed: async (text) => { + dispatched.push(text); + throw new AppError(429, 'Cooling down', 'EMBEDDING_RATE_LIMITED', true, { retryAfterSeconds: 2 }); + }, + embedBatch: async () => [], + }; + const protectedProvider = protectEmbeddings(provider, async () => { + if (++reservations === 1) await gate; + }); + const delayed = protectedProvider.embed('delayed').then( + () => null, + (error: unknown) => error, + ); + await expect(protectedProvider.embed('limited')).rejects.toMatchObject({ code: 'EMBEDDING_RATE_LIMITED' }); + release(); + expect(await delayed).toMatchObject({ code: 'EMBEDDING_RATE_LIMITED' }); + expect(dispatched).toEqual(['limited']); + await new Promise((resolve) => setTimeout(resolve, 2100)); +}, 10000); diff --git a/server/search/embedding-controls.ts b/server/search/embedding-controls.ts new file mode 100644 index 0000000..c0646c2 --- /dev/null +++ b/server/search/embedding-controls.ts @@ -0,0 +1,89 @@ +import type { EmbeddingProvider } from './types'; +import { createBoundedQueue } from '../utils/bounded-queue'; +import { configuredLimit, reserveUsage } from '../utils/usage-limits'; +import { AppError } from '../utils/errors'; +import { withEmbeddingRetry } from './embedding-retry'; + +const queue = createBoundedQueue(2, 8, 10000, 30000); +let nextStart = 0; +let cooldownUntil = 0; + +function rateLimitError(): AppError { + const retryAfterSeconds = Math.max(1, Math.ceil((cooldownUntil - Date.now()) / 1000)); + const error = new AppError( + 429, + `Embedding provider is cooling down. Retry after ${retryAfterSeconds} seconds.`, + 'EMBEDDING_RATE_LIMITED', + true, + { retryAfterSeconds }, + ); + // Marker distinguishes locally-raised cooldown errors from provider-originated + // 429s so retries preserve the existing deadline instead of extending it. + (error.context as Record).localCooldown = true; + return error; +} + +function isLocalCooldownError(error: AppError): boolean { + return error.context?.localCooldown === true; +} + +async function pace(signal: AbortSignal): Promise { + if (Date.now() < cooldownUntil) throw rateLimitError(); + const start = Math.max(Date.now(), nextStart); + nextStart = start + configuredLimit('EMBEDDING_REQUEST_INTERVAL_MS', 1000); + const delay = start - Date.now(); + if (delay > 0) + await new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer); + reject(new AppError(504, 'Provider request timed out.', 'PROVIDER_TIMEOUT', true)); + }; + const timer = setTimeout(() => { + signal.removeEventListener('abort', abort); + resolve(); + }, delay); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + }); + if (Date.now() < cooldownUntil) throw rateLimitError(); +} +export function protectEmbeddings(provider: EmbeddingProvider, reserve = reserveUsage): EmbeddingProvider { + const run = (texts: string[], work: (signal: AbortSignal) => Promise, callerSignal?: AbortSignal) => { + const bytes = texts.reduce((sum, text) => sum + Buffer.byteLength(text), 0); + if (bytes > 256000 || texts.length > 100) + return Promise.reject(new AppError(413, 'Embedding batch is too large.', 'PROMPT_TOO_LARGE')); + return queue(async (queueSignal) => { + const signal = callerSignal ? AbortSignal.any([callerSignal, queueSignal]) : queueSignal; + const attempt = async () => { + if (Date.now() < cooldownUntil) throw rateLimitError(); + if (signal.aborted) throw new AppError(504, 'Provider request timed out.', 'PROVIDER_TIMEOUT', true); + // Reserve for this attempt, not hypothetical retries. Reservations are not refunded, + // including when a later cooldown or timeout prevents dispatch. + await reserve('embedding-daily', bytes, configuredLimit('SERVER_EMBEDDING_DAILY_BYTES', 300000000), 86400000); + // Reservations may await Redis; schedule the actual dispatch only after they settle. + await pace(signal); + if (signal.aborted) throw new AppError(504, 'Provider request timed out.', 'PROVIDER_TIMEOUT', true); + return work(signal); + }; + try { + return await withEmbeddingRetry(attempt); + } catch (error) { + if (error instanceof AppError && error.code === 'EMBEDDING_RATE_LIMITED') { + // Only a provider-originated 429 may arm/extend the cooldown; a + // locally-raised cooldown error must preserve the existing deadline, + // otherwise repeated retries plus ceil rounding cause indefinite outage. + if (!isLocalCooldownError(error)) { + const seconds = Number(error.context?.retryAfterSeconds) || 60; + cooldownUntil = Math.max(cooldownUntil, Date.now() + seconds * 1000); + } + } + throw error; + } + }, callerSignal); + }; + return { + ...provider, + embed: (text, signal) => run([text], (s) => provider.embed(text, s), signal), + embedBatch: (texts, signal) => run(texts, (s) => provider.embedBatch(texts, s), signal), + }; +} diff --git a/server/search/embedding-provider.test.ts b/server/search/embedding-provider.test.ts index 26defb8..6bcd30a 100644 --- a/server/search/embedding-provider.test.ts +++ b/server/search/embedding-provider.test.ts @@ -1,11 +1,41 @@ import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from 'bun:test'; import { createEmbeddingProvider } from './embedding-provider'; +import { fetchPublicEmbeddings } from '../ai/public-chat-fetch'; + +const realSetTimeout = globalThis.setTimeout; +const originalEmbeddingProvider = process.env.EMBEDDING_PROVIDER; +const dedicatedNames = ['EMBEDDING_API_KEY', 'EMBEDDING_API_BASE', 'EMBEDDING_MODEL'] as const; +const dedicatedValues = dedicatedNames.map(name => process.env[name]); +const originalEmbeddingInterval = process.env.EMBEDDING_REQUEST_INTERVAL_MS; let openAIError: Error | null = null; let geminiFailures = 0; +let geminiRequests = 0; +let openAIConfig: { apiKey: string; baseURL?: string; fetch?: typeof fetchPublicEmbeddings } | undefined; describe('createEmbeddingProvider', () => { + it('uses the normalized base URL for single and batch embeddings', async () => { + const originalBase = process.env.OPENAI_API_BASE; + try { + process.env.AI_PROVIDER = 'openai'; + process.env.OPENAI_API_KEY = 'test-key'; + process.env.OPENAI_API_BASE = ' '; + const provider = await createEmbeddingProvider(); + await provider.embed('one'); + expect(openAIConfig?.baseURL).toBeUndefined(); + expect(openAIConfig?.fetch).toBe(fetchPublicEmbeddings); + await provider.embedBatch(['two']); + expect(openAIConfig?.baseURL).toBeUndefined(); + expect(openAIConfig?.fetch).toBe(fetchPublicEmbeddings); + } finally { + if (originalBase === undefined) delete process.env.OPENAI_API_BASE; + else process.env.OPENAI_API_BASE = originalBase; + } + }); beforeEach(() => { + process.env.EMBEDDING_REQUEST_INTERVAL_MS = '0'; + delete process.env.EMBEDDING_PROVIDER; + dedicatedNames.forEach(name => { delete process.env[name]; }); mock.module('openai', () => { class OpenAI { baseURL = ''; @@ -18,7 +48,9 @@ describe('createEmbeddingProvider', () => { }), }; - constructor(_config: { apiKey: string }) {} + constructor(config: { apiKey: string; baseURL?: string }) { + openAIConfig = config; + } } return { default: OpenAI }; @@ -27,13 +59,15 @@ describe('createEmbeddingProvider', () => { mock.module('@google/genai', () => ({ GoogleGenAI: class { models = { - embedContent: mock(async ({ contents }: { contents: string }) => { + embedContent: mock(async ({ contents }: { contents: string | string[] }) => { + geminiRequests++; if (geminiFailures > 0) { geminiFailures--; throw new Error('timeout'); } + const texts = Array.isArray(contents) ? contents : [contents]; return { - embeddings: contents === 'missing' ? [] : [{ values: [0, 3, 4] }], + embeddings: texts.includes('missing') ? [] : texts.map(() => ({ values: [0, 3, 4] })), }; }), }; @@ -46,18 +80,73 @@ describe('createEmbeddingProvider', () => { delete process.env.GEMINI_API_KEY; delete process.env.OPENAI_API_KEY; delete process.env.ANTHROPIC_API_KEY; - delete process.env.EDGEONE_API_KEY; - delete process.env.MAKERS_MODELS_KEY; delete process.env.OPENAI_EMBEDDING_MODEL; delete process.env.GEMINI_EMBEDDING_MODEL; openAIError = null; geminiFailures = 0; + geminiRequests = 0; }); afterEach(() => { + dedicatedNames.forEach((name, i) => { if (dedicatedValues[i] === undefined) delete process.env[name]; else process.env[name] = dedicatedValues[i]; }); + if (originalEmbeddingInterval === undefined) delete process.env.EMBEDDING_REQUEST_INTERVAL_MS; + else process.env.EMBEDDING_REQUEST_INTERVAL_MS = originalEmbeddingInterval; + if (originalEmbeddingProvider === undefined) delete process.env.EMBEDDING_PROVIDER; + else process.env.EMBEDDING_PROVIDER = originalEmbeddingProvider; mock.restore(); }); + it('uses dedicated embedding credentials and endpoint instead of chat settings', async () => { + const names = ['EMBEDDING_API_KEY', 'EMBEDDING_API_BASE', 'EMBEDDING_MODEL', 'OPENAI_API_BASE'] as const; + const saved = names.map((name) => process.env[name]); + try { + process.env.EMBEDDING_PROVIDER = 'openai'; + process.env.EMBEDDING_API_KEY = 'embedding-only'; + process.env.EMBEDDING_API_BASE = 'https://embedding.example/v1'; + process.env.EMBEDDING_MODEL = 'embed-test'; + process.env.OPENAI_API_BASE = 'https://chat.example/v1'; + const provider = await createEmbeddingProvider(); + await provider.embed('example'); + expect(openAIConfig).toMatchObject({ apiKey: 'embedding-only', baseURL: 'https://embedding.example/v1' }); + } finally { + names.forEach((name, index) => { + if (saved[index] === undefined) delete process.env[name]; + else process.env[name] = saved[index]; + }); + } + }); + + it('sends a Gemini chunk batch in one provider request', async () => { + process.env.AI_PROVIDER = 'gemini'; + process.env.GEMINI_API_KEY = 'gemini-batch-regression'; + const provider = await createEmbeddingProvider(); + const results = await provider.embedBatch(Array.from({ length: 32 }, (_, i) => `chunk ${i}`)); + expect(results).toHaveLength(32); + expect(geminiRequests).toBe(1); + }); + + it('uses Featherless credentials and endpoint independently of the AI provider', async () => { + const previous = process.env.OPENAI_API_BASE; + try { + process.env.EMBEDDING_PROVIDER = 'openai'; + process.env.AI_PROVIDER = 'anthropic'; + process.env.OPENAI_API_KEY = 'unit-featherless-key'; + process.env.OPENAI_API_BASE = 'https://api.featherless.ai/v1'; + const provider = await createEmbeddingProvider(); + expect(provider.providerName).toBe('openai'); + await provider.embedBatch(['search']); + expect(openAIConfig).toEqual( + expect.objectContaining({ + apiKey: 'unit-featherless-key', + baseURL: 'https://api.featherless.ai/v1', + }), + ); + } finally { + if (previous === undefined) delete process.env.OPENAI_API_BASE; + else process.env.OPENAI_API_BASE = previous; + } + }); + it('creates deterministic mock embeddings and caches provider', async () => { const p1 = await createEmbeddingProvider(); const p2 = await createEmbeddingProvider(); @@ -121,28 +210,6 @@ describe('createEmbeddingProvider', () => { delete process.env.OPENAI_API_BASE; }); - it('uses edgeone embeddings', async () => { - process.env.AI_PROVIDER = 'edgeone'; - process.env.EDGEONE_API_KEY = 'sk-eo-test'; - - const provider = await createEmbeddingProvider(); - expect(provider.providerName).toBe('edgeone'); - expect(provider.maxTokens).toBe(8191); - - const one = await provider.embed('hello'); - expect(one.tokenCount).toBe(2); - delete process.env.EDGEONE_API_KEY; - const batch = await provider.embedBatch(['a', 'b']); - expect(batch).toHaveLength(2); - - delete process.env.AI_PROVIDER; - delete process.env.EDGEONE_API_KEY; - delete process.env.MAKERS_MODELS_KEY; - process.env.AI_PROVIDER = 'edgeone'; - await expect(createEmbeddingProvider()).rejects.toThrow('EDGEONE_API_KEY or MAKERS_MODELS_KEY is required for EdgeOne embeddings.'); - - }); - it('uses gemini embeddings and batch path', async () => { process.env.AI_PROVIDER = 'gemini'; process.env.GEMINI_API_KEY = 'gemini-key'; @@ -199,7 +266,8 @@ describe('createEmbeddingProvider', () => { geminiFailures = 1; const provider = await createEmbeddingProvider(); - spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void) => { + spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, delay = 0) => { + if (delay >= 10000) return realSetTimeout(callback, delay); callback(); return 0; }) as typeof setTimeout); @@ -224,7 +292,8 @@ describe('createEmbeddingProvider', () => { geminiFailures = 3; const provider = await createEmbeddingProvider(); - spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void) => { + spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, delay = 0) => { + if (delay >= 10000) return realSetTimeout(callback, delay); callback(); return 0; }) as typeof setTimeout); diff --git a/server/search/embedding-provider.ts b/server/search/embedding-provider.ts index 928b2e3..eea372d 100644 --- a/server/search/embedding-provider.ts +++ b/server/search/embedding-provider.ts @@ -1,6 +1,11 @@ +import { fetchPublicEmbeddings } from '../ai/public-chat-fetch'; +import { embeddingConfig } from './embedding-config'; +import { protectEmbeddings } from './embedding-controls'; import type { EmbeddingProvider, EmbeddingResult } from './types'; import { EMBEDDING_DIMENSIONS } from './constants'; import { AppError } from '../utils/errors'; +import { hashToken } from '../cache/lru'; +import { embeddingIdentity } from './index-identity'; // ── Factory ──────────────────────────────────────────────────────────── @@ -8,8 +13,10 @@ let cachedProvider: EmbeddingProvider | null = null; let cachedProviderKey: string | null = null; export async function createEmbeddingProvider(): Promise { - const envProvider = (process.env.AI_PROVIDER ?? 'mock').toLowerCase(); - const cacheKey = `${envProvider}:${process.env.GEMINI_API_KEY ? 'g' : ''}${process.env.OPENAI_API_KEY ? 'o' : ''}${process.env.EDGEONE_API_KEY || process.env.MAKERS_MODELS_KEY ? 'e' : ''}`; + const envProvider = embeddingConfig().provider; + const cacheKey = + embeddingIdentity() + + hashToken(JSON.stringify([process.env.EMBEDDING_API_KEY, process.env.OPENAI_API_KEY, process.env.GEMINI_API_KEY])); if (cachedProvider && cachedProviderKey === cacheKey) { return cachedProvider; @@ -19,29 +26,19 @@ export async function createEmbeddingProvider(): Promise { if (envProvider === 'openai') { provider = createOpenAIEmbeddingProvider(); - } else if (envProvider === 'edgeone') { - provider = createEdgeOneEmbeddingProvider(); } else if (envProvider === 'gemini') { provider = await createGeminiEmbeddingProvider(); - } else if (envProvider === 'anthropic') { - // Anthropic has no native embedding API – prefer Gemini, then try OpenAI - if (process.env.GEMINI_API_KEY) { - provider = await createGeminiEmbeddingProvider(); - } else if (process.env.OPENAI_API_KEY) { - provider = createOpenAIEmbeddingProvider(); - } else if (process.env.EDGEONE_API_KEY || process.env.MAKERS_MODELS_KEY) { - provider = createEdgeOneEmbeddingProvider(); - } else { - throw new AppError( - 400, - 'Anthropic does not support embeddings. Configure OPENAI_API_KEY, EDGEONE_API_KEY, MAKERS_MODELS_KEY, or GEMINI_API_KEY as fallback.', - 'EMBEDDING_PROVIDER_UNAVAILABLE', - ); - } - } else { + } else if (envProvider === 'mock') { provider = createMockEmbeddingProvider(); + } else { + throw new AppError( + 400, + 'Unsupported embedding provider. Configure openai, gemini, or mock.', + 'EMBEDDING_PROVIDER_UNAVAILABLE', + ); } + provider = provider.providerName === 'mock' ? provider : protectEmbeddings(provider); cachedProvider = provider; cachedProviderKey = cacheKey; return provider; @@ -108,22 +105,25 @@ function createOpenAICompatibleEmbeddingProvider(config: OpenAICompatibleConfig) maxTokens, providerName, - async embed(text: string): Promise { + async embed(text: string, signal?: AbortSignal): Promise { const truncated = truncateText(text, maxTokens); - const vector = await openAIEmbed(apiKey, model, truncated, baseURL); + const vector = await openAIEmbed(apiKey, model, truncated, baseURL, signal); return { vector, tokenCount: estimateTokens(truncated) }; }, - async embedBatch(texts: string[]): Promise { + async embedBatch(texts: string[], signal?: AbortSignal): Promise { const truncated = texts.map((t) => truncateText(t, maxTokens)); - const vectors = await openAIEmbedBatch(apiKey, model, truncated, baseURL); - return vectors.map((v, i) => ({ vector: v, tokenCount: estimateTokens(truncated[i]) })); + const vectors = await openAIEmbedBatch(apiKey, model, truncated, baseURL, signal); + return vectors.map((v, i) => ({ + vector: v, + tokenCount: estimateTokens(truncated[i]), + })); }, }; } function createOpenAIEmbeddingProvider(): EmbeddingProvider { - const apiKey = process.env.OPENAI_API_KEY; + const apiKey = embeddingConfig().apiKey; if (!apiKey) { throw new AppError(401, 'OPENAI_API_KEY is required for OpenAI embeddings.', 'MISSING_API_KEY'); } @@ -131,57 +131,69 @@ function createOpenAIEmbeddingProvider(): EmbeddingProvider { return createOpenAICompatibleEmbeddingProvider({ providerName: 'openai', apiKey, - model: process.env.OPENAI_EMBEDDING_MODEL ?? 'openrouter/openai/text-embedding-3-large', + model: embeddingConfig().model, + baseURL: embeddingConfig().baseURL, }); } -function createEdgeOneEmbeddingProvider(): EmbeddingProvider { - // ponytail: EdgeOne Makers Models embedding endpoint is OpenAI-compatible - const apiKey = process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim(); - if (!apiKey) { - throw new AppError( - 401, - 'EDGEONE_API_KEY or MAKERS_MODELS_KEY is required for EdgeOne embeddings.', - 'MISSING_API_KEY', - ); - } - - return createOpenAICompatibleEmbeddingProvider({ - providerName: 'edgeone', +async function openAIEmbed( + apiKey: string, + model: string, + text: string, + baseURL?: string, + signal?: AbortSignal, +): Promise { + const { default: OpenAI } = await import('openai'); + const client = new OpenAI({ apiKey, - baseURL: process.env.EDGEONE_API_BASE ?? 'https://ai-gateway.edgeone.link/v1', - model: process.env.EDGEONE_EMBEDDING_MODEL ?? process.env.OPENAI_EMBEDDING_MODEL ?? 'openrouter/openai/text-embedding-3-large', + baseURL, + fetch: fetchPublicEmbeddings, + timeout: 30000, + maxRetries: 0, }); -} - -async function openAIEmbed(apiKey: string, model: string, text: string, baseURL?: string): Promise { - const { default: OpenAI } = await import('openai'); - const client = new OpenAI({ apiKey, baseURL: baseURL ?? process.env.OPENAI_API_BASE }); - const response = await withRetry(() => - client.embeddings.create({ + const response = await client.embeddings.create( + { model, + dimensions: EMBEDDING_DIMENSIONS, input: text, - }), + encoding_format: 'float', + }, + { signal }, ); return normalizeVector(new Float32Array(response.data[0].embedding)); } -async function openAIEmbedBatch(apiKey: string, model: string, texts: string[], baseURL?: string): Promise { +async function openAIEmbedBatch( + apiKey: string, + model: string, + texts: string[], + baseURL?: string, + signal?: AbortSignal, +): Promise { if (texts.length === 0) return []; const { default: OpenAI } = await import('openai'); - const client = new OpenAI({ apiKey, baseURL: baseURL ?? process.env.OPENAI_API_BASE }); + const client = new OpenAI({ + apiKey, + baseURL, + fetch: fetchPublicEmbeddings, + timeout: 30000, + maxRetries: 0, + }); // OpenAI supports batch – send in chunks of 100 const results: Float32Array[] = []; for (let i = 0; i < texts.length; i += 100) { const batch = texts.slice(i, i + 100); - const response = await withRetry(() => - client.embeddings.create({ + const response = await client.embeddings.create( + { model, + dimensions: EMBEDDING_DIMENSIONS, input: batch, - }), + encoding_format: 'float', + }, + { signal }, ); for (const item of response.data) { results.push(normalizeVector(new Float32Array(item.embedding))); @@ -193,14 +205,17 @@ async function openAIEmbedBatch(apiKey: string, model: string, texts: string[], // ── Gemini Provider ──────────────────────────────────────────────────── async function createGeminiEmbeddingProvider(): Promise { - const apiKey = process.env.GEMINI_API_KEY; + const apiKey = embeddingConfig().apiKey; if (!apiKey) { throw new AppError(401, 'GEMINI_API_KEY is required for Gemini embeddings.', 'MISSING_API_KEY'); } const { GoogleGenAI } = await import('@google/genai'); - const ai = new GoogleGenAI({ apiKey }); - const model = process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001'; + const ai = new GoogleGenAI({ + apiKey, + httpOptions: { timeout: 30000, retryOptions: { attempts: 1 } }, + }); + const model = embeddingConfig().model; const maxTokens = 2048; return { @@ -208,15 +223,16 @@ async function createGeminiEmbeddingProvider(): Promise { maxTokens, providerName: 'gemini', - async embed(text: string): Promise { + async embed(text: string, signal?: AbortSignal): Promise { const truncated = truncateText(text, maxTokens); - const response = await withRetry(() => - ai.models.embedContent({ - model, - contents: truncated, - config: { outputDimensionality: EMBEDDING_DIMENSIONS }, - }), - ); + const response = await ai.models.embedContent({ + model, + contents: truncated, + config: { + outputDimensionality: EMBEDDING_DIMENSIONS, + abortSignal: signal, + }, + }); const embedding = response.embeddings?.[0]?.values; if (!embedding) throw new AppError(503, 'Gemini returned no embedding.', 'EMBEDDING_FAILURE', true); return { @@ -225,30 +241,34 @@ async function createGeminiEmbeddingProvider(): Promise { }; }, - async embedBatch(texts: string[]): Promise { - const concurrency = 5; + async embedBatch(texts: string[], signal?: AbortSignal): Promise { + // Gemini accepts up to 100 contents per request; avoid one request per chunk. + const batchSize = 100; const results: EmbeddingResult[] = []; - for (let i = 0; i < texts.length; i += concurrency) { - const chunk = texts.slice(i, i + concurrency); - const chunkResults = await Promise.all( - chunk.map(async (text) => { - const truncated = truncateText(text, maxTokens); - const response = await withRetry(() => - ai.models.embedContent({ - model, - contents: truncated, - config: { outputDimensionality: EMBEDDING_DIMENSIONS }, - }), - ); - const embedding = response.embeddings?.[0]?.values; - if (!embedding) throw new AppError(503, 'Gemini returned no embedding.', 'EMBEDDING_FAILURE', true); - return { - vector: normalizeVector(new Float32Array(embedding)), - tokenCount: estimateTokens(truncated), - }; - }), - ); - results.push(...chunkResults); + for (let i = 0; i < texts.length; i += batchSize) { + const truncated = texts.slice(i, i + batchSize).map((text) => truncateText(text, maxTokens)); + const response = await ai.models.embedContent({ + model, + contents: truncated, + config: { + outputDimensionality: EMBEDDING_DIMENSIONS, + abortSignal: signal, + }, + }); + const embeddings = response.embeddings; + if (!embeddings || embeddings.length !== truncated.length) { + throw new AppError(503, 'Gemini returned an incomplete embedding batch.', 'EMBEDDING_FAILURE', true); + } + for (let j = 0; j < embeddings.length; j++) { + const embedding = embeddings[j].values; + if (!embedding) { + throw new AppError(503, 'Gemini returned no embedding.', 'EMBEDDING_FAILURE', true); + } + results.push({ + vector: normalizeVector(new Float32Array(embedding)), + tokenCount: estimateTokens(truncated[j]), + }); + } } return results; }, @@ -276,22 +296,3 @@ function truncateText(text: string, maxTokens: number): string { if (text.length <= approxChars) return text; return text.slice(0, approxChars); } - -/** Exponential backoff retry for rate limits and transient errors. */ -async function withRetry(fn: () => Promise, maxRetries = 3): Promise { - let lastError: unknown; - for (let attempt = 0; attempt < maxRetries; attempt++) { - try { - return await fn(); - } catch (err) { - lastError = err; - const msg = err instanceof Error ? err.message : String(err); - const isRateLimit = msg.includes('429') || msg.toLowerCase().includes('rate limit'); - const isTimeout = msg.toLowerCase().includes('timeout'); - if (!isRateLimit && !isTimeout) throw err; - const delay = Math.min(1000 * Math.pow(2, attempt), 10000); - await new Promise((r) => setTimeout(r, delay)); - } - } - throw lastError; -} diff --git a/server/search/embedding-retry.test.ts b/server/search/embedding-retry.test.ts new file mode 100644 index 0000000..f1d831a --- /dev/null +++ b/server/search/embedding-retry.test.ts @@ -0,0 +1,59 @@ +import { expect, test } from 'bun:test'; +import { embeddingRetryAfter, isEmbeddingRateLimit, withEmbeddingRetry } from './embedding-retry'; +import { toErrorPayload } from '../utils/errors'; + +test('recognizes SDK rate-limit wording and structured status', () => { + expect(isEmbeddingRateLimit(new Error('Retryable HTTP Error: Too Many Requests'))).toBe(true); + expect(isEmbeddingRateLimit({ status: 429 })).toBe(true); + expect(isEmbeddingRateLimit(new Error('RESOURCE_EXHAUSTED'))).toBe(true); + expect(isEmbeddingRateLimit(new Error('invalid credentials'))).toBe(false); +}); + +test('rate limits return a safe 429 without spending more requests during cooldown', async () => { + let attempts = 0; + const delays: number[] = []; + try { + await withEmbeddingRetry(async () => { + attempts++; + throw new Error('Retryable HTTP Error: Too Many Requests'); + }, async (ms) => { delays.push(ms); }); + throw new Error('Expected rejection'); + } catch (error) { + expect(toErrorPayload(error)).toMatchObject({ status: 429, code: 'EMBEDDING_RATE_LIMITED', retryable: true }); + } + expect(attempts).toBe(1); + expect(delays).toEqual([]); +}); + +test('transient timeout recovers without unnecessary retries', async () => { + let attempts = 0; + const result = await withEmbeddingRetry(async () => { + if (++attempts === 1) throw new Error('timeout'); + return 'ok'; + }, async () => {}); + expect(result).toBe('ok'); + expect(attempts).toBe(2); +}); + +test('SDK client timeouts ("Request timed out.") are retried with backoff', async () => { + let attempts = 0; + const delays: number[] = []; + const result = await withEmbeddingRetry(async () => { + if (++attempts === 1) throw new Error('Request timed out.'); + return 'ok'; + }, async (ms) => { + delays.push(ms); + }); + expect(result).toBe('ok'); + expect(attempts).toBe(2); + expect(delays).toEqual([1000]); +}); + +test('provider retry-after headers survive rate-limit conversion', async () => { + const error = Object.assign(new Error('Too Many Requests'), { headers: new Headers({ 'Retry-After': '120' }) }); + expect(embeddingRetryAfter(error)).toBe(120); + await expect(withEmbeddingRetry(async () => { throw error; })).rejects.toMatchObject({ + status: 429, context: { retryAfterSeconds: 120 }, + }); + expect(embeddingRetryAfter({ headers: { 'retry-after': 'invalid' } })).toBe(60); +}); diff --git a/server/search/embedding-retry.ts b/server/search/embedding-retry.ts new file mode 100644 index 0000000..263ea07 --- /dev/null +++ b/server/search/embedding-retry.ts @@ -0,0 +1,44 @@ +import { AppError } from '../utils/errors'; + +export function isEmbeddingRateLimit(error: unknown): boolean { + if (typeof error === 'object' && error !== null && 'status' in error && Number(error.status) === 429) return true; + const message = error instanceof Error ? error.message : String(error); + return /429|rate.?limit|too many requests|resource_exhausted/i.test(message); +} + +export function embeddingRetryAfter(error: unknown): number { + if (typeof error !== 'object' || error === null || !('headers' in error)) return 60; + const headers = error.headers; + const value = headers instanceof Headers ? headers.get('retry-after') : + typeof headers === 'object' && headers !== null && 'retry-after' in headers ? String(headers['retry-after']) : null; + if (!value) return 60; + const seconds = Number(value); + const delay = Number.isFinite(seconds) ? seconds : (Date.parse(value) - Date.now()) / 1000; + return Number.isFinite(delay) && delay > 0 ? Math.min(3600, Math.max(1, Math.ceil(delay))) : 60; +} + +export async function withEmbeddingRetry( + work: () => Promise, + sleep: (ms: number) => Promise = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +): Promise { + for (let attempt = 0; ; attempt++) { + try { + return await work(); + } catch (error) { + // Application admission failures already carry their own reason and reset time. + if (error instanceof AppError) throw error; + const rateLimited = isEmbeddingRateLimit(error); + if (rateLimited) { + const retryAfterSeconds = embeddingRetryAfter(error); + throw new AppError(429, `Embedding provider rate limit reached. Retry after ${retryAfterSeconds} seconds; indexing a smaller scope may help.`, 'EMBEDDING_RATE_LIMITED', true, { retryAfterSeconds }); + } + // Covers "timeout" as well as the OpenAI SDK's "Request timed out." client timeout. + const timeout = error instanceof Error && /(?:timeout|timed\s*out)/i.test(error.message); + if (!rateLimited && !timeout) throw error; + if (attempt === 2) { + throw error; + } + await sleep(1000 * 2 ** attempt); + } + } +} diff --git a/server/search/index-availability.ts b/server/search/index-availability.ts new file mode 100644 index 0000000..20031d1 --- /dev/null +++ b/server/search/index-availability.ts @@ -0,0 +1,60 @@ +import type { SearchCoverage } from '../../src/types'; +import type { VectorStore } from './types'; +import { createVectorStore } from './vector-store'; +import { createCheckpoints } from './checkpoints'; +import { SEARCH_LIMITS } from './limits'; +import { searchIndexIdentity } from './index-identity'; + +export function createIndexAvailability( + deps: { getVectorStore(): VectorStore }, + checkpoints: ReturnType, + limits = SEARCH_LIMITS, + now = Date.now, +) { + const published = new Map(); + const available = (key: string) => { + const store = deps.getVectorStore(); + if (store.hasIndex(key)) + return { + store, + key, + coverage: published.get(key) ?? { + kind: 'complete' as const, + commitSha: searchIndexIdentity(key).snapshotSha, + indexedFiles: 0, + totalFiles: 0, + }, + }; + for (const [id, coverage] of [...published].reverse()) { + if (!store.hasIndex(id)) { + published.delete(id); + continue; + } + if (searchIndexIdentity(id).family === searchIndexIdentity(key).family) + return { store, key: id, coverage: { ...coverage, kind: 'previous' as const } }; + } + const checkpoint = checkpoints.get(key); + if (!checkpoint?.completed.size) return undefined; + const chunks = checkpoint.chunks.filter((chunk) => checkpoint.completed.has(chunk.metadata.filePath)); + if (!chunks.length) return undefined; + const partial = createVectorStore(limits, now); + partial.replaceIndex(key, chunks); + return { + store: partial, + key, + coverage: { + kind: 'partial' as const, + commitSha: chunks[0].metadata.commitSha, + indexedFiles: checkpoint.completed.size, + totalFiles: checkpoint.totalFiles, + }, + }; + }; + return { + available, + publish(key: string, coverage: SearchCoverage) { + published.set(key, coverage); + for (const id of published.keys()) if (!deps.getVectorStore().hasIndex(id)) published.delete(id); + }, + }; +} diff --git a/server/search/index-identity.test.ts b/server/search/index-identity.test.ts new file mode 100644 index 0000000..a06b540 --- /dev/null +++ b/server/search/index-identity.test.ts @@ -0,0 +1,41 @@ +import { test, expect } from 'bun:test'; +import { searchIndexKey, searchIndexIdentity } from './index-identity'; + +test('snapshot families retain credential and path isolation', () => { + const key = (sha: string, token = 'one', paths: string[] = []) => + searchIndexKey('org', 'repo', sha, { gitHubToken: token }, paths); + expect(searchIndexIdentity(key('first')).snapshotSha).toBe('first'); + expect(searchIndexIdentity(key('first')).family).toBe(searchIndexIdentity(key('second')).family); + expect(searchIndexIdentity(key('first')).family).not.toBe(searchIndexIdentity(key('first', 'other')).family); + expect(searchIndexIdentity(key('first')).family).not.toBe(searchIndexIdentity(key('first', 'one', ['src'])).family); + expect(() => searchIndexIdentity('invalid')).toThrow(); +}); + +test('index identity isolates credentials, snapshots and embedding models without exposing tokens', () => { + const key = searchIndexKey('Org', 'Repo', 'sha', { gitHubToken: 'private-token' }); + expect(key).not.toContain('private-token'); + expect(key).toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'private-token' })); + expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha2', { gitHubToken: 'private-token' })); + expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'other' })); + const original = process.env.EMBEDDING_MODEL; + try { + process.env.EMBEDDING_MODEL = 'different-model'; + expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'private-token' })); + } finally { + if (original === undefined) delete process.env.EMBEDDING_MODEL; + else process.env.EMBEDDING_MODEL = original; + } +}); + +test('Featherless model changes invalidate existing search indexes', () => { + const previous = process.env.EMBEDDING_MODEL; + try { + process.env.EMBEDDING_MODEL = 'Qwen/Qwen3-Embedding-8B'; + const key = searchIndexKey('org', 'repo', 'sha'); + process.env.EMBEDDING_MODEL = 'different-model'; + expect(searchIndexKey('org', 'repo', 'sha')).not.toBe(key); + } finally { + if (previous === undefined) delete process.env.EMBEDDING_MODEL; + else process.env.EMBEDDING_MODEL = previous; + } +}); diff --git a/server/search/index-identity.ts b/server/search/index-identity.ts new file mode 100644 index 0000000..db61587 --- /dev/null +++ b/server/search/index-identity.ts @@ -0,0 +1,48 @@ +import { embeddingConfig } from './embedding-config'; +import { hashToken, hashContext } from '../cache/lru'; +import type { RequestContext } from '../utils/context'; + +/** Decode the canonical key only here; callers must retain every authorization/model/path field. */ +export function searchIndexIdentity(key: string): { snapshotSha: string; family: string } { + const separator = key.indexOf('@'); + const end = key.indexOf(':', separator + 1); + if (separator < 1 || end <= separator + 1) throw new Error('Invalid search index identity'); + return { snapshotSha: key.slice(separator + 1, end), family: key.slice(0, separator + 1) + key.slice(end) }; +} + +export function indexRequestKey( + owner: string, + repo: string, + buildId: string, + ctx: Pick, +): string { + return `${owner.toLowerCase()}/${repo.toLowerCase()}:${hashToken(ctx.gitHubToken || process.env.GITHUB_TOKEN?.trim() || 'anonymous')}:${buildId}`; +} + +/** Include model and endpoint configuration so incompatible embeddings never mix. */ +export function embeddingIdentity(): string { + return hashContext( + JSON.stringify([ + embeddingConfig().provider, + embeddingConfig().model, + embeddingConfig().provider === 'openai' ? embeddingConfig().baseURL : undefined, + process.env.EMBEDDING_DIMENSIONS ?? '3072', + ]), + ); +} + +export function searchIndexKey( + owner: string, + repo: string, + sha: string, + ctx?: Pick, + includePaths: string[] = [], + excludePaths: string[] = [], +): string { + const scope = hashToken(ctx?.gitHubToken || process.env.GITHUB_TOKEN?.trim() || 'anonymous'); + const paths = JSON.stringify({ + includePaths: [...includePaths].sort(), + excludePaths: [...excludePaths].sort(), + }); + return `${owner.toLowerCase()}/${repo.toLowerCase()}@${sha}:${scope}:${embeddingIdentity()}:${hashContext(paths)}`; +} diff --git a/server/search/index-requests.test.ts b/server/search/index-requests.test.ts new file mode 100644 index 0000000..032ada0 --- /dev/null +++ b/server/search/index-requests.test.ts @@ -0,0 +1,71 @@ +import { test, expect } from 'bun:test'; +import { createIndexRequests } from './index-requests'; +import { indexRequestKey } from './index-identity'; + +test('a cancelled build ID cannot start and completed registrations release capacity', () => { + const registry = createIndexRequests(Date.now, 1); + registry.cancel('early'); + const early = registry.begin('early'); + expect(early.cancelled()).toBe(true); + early.finish(false, false); + const next = registry.begin('next'); + expect(next.cancelled()).toBe(false); + next.finish(); + expect(registry.begin('another').cancelled()).toBe(false); +}); +test('late cancellation of an old attempt cannot cancel its replacement', () => { + const registry = createIndexRequests(); + let cancelled = 0; + const old = registry.begin('old'); + old.attach('snapshot', () => { + cancelled++; + }); + old.finish(true); + const current = registry.begin('current'); + current.attach('snapshot', () => { + cancelled++; + }); + registry.cancel('old'); + expect(cancelled).toBe(0); + registry.cancel('current'); + expect(cancelled).toBe(1); +}); +test('cancellation identities isolate credentials and repositories', () => { + const first = indexRequestKey('owner', 'repo', 'id', { gitHubToken: 'a' }); + expect(first).not.toBe(indexRequestKey('owner', 'repo', 'id', { gitHubToken: 'b' })); + expect(first).not.toBe(indexRequestKey('owner', 'another', 'id', { gitHubToken: 'a' })); +}); + +test('a failed Resume metadata lookup retains cancellation of the saved build', () => { + const registry = createIndexRequests(); + let cancelled = false; + const first = registry.begin('build'); + first.attach('snapshot', () => { + cancelled = true; + }); + first.finish(true); + registry.begin('build').finish(); // Metadata failed before reattaching to the pipeline. + registry.cancel('build'); + expect(cancelled).toBe(true); +}); + +test('unknown cancellations and cancelled builds do not reserve indexing capacity', () => { + const registry = createIndexRequests(Date.now, 1); + for (let i = 0; i < 300; i++) registry.cancel(`unknown-${i}`); + const build = registry.begin('legitimate'); + build.attach('snapshot', () => {}); + build.finish(true); + registry.cancel('legitimate'); + expect(registry.begin('next').cancelled()).toBe(false); +}); + +test('early cancellation expires after the bounded arrival window', () => { + let now = 1000; + const registry = createIndexRequests(() => now); + registry.cancel('early'); + const early = registry.begin('early'); + expect(early.cancelled()).toBe(true); + early.finish(); + now += 2 * 60 * 1000 + 1; + expect(registry.begin('early').cancelled()).toBe(false); +}); diff --git a/server/search/index-requests.ts b/server/search/index-requests.ts new file mode 100644 index 0000000..b148be3 --- /dev/null +++ b/server/search/index-requests.ts @@ -0,0 +1,98 @@ +import { AppError } from '../utils/errors'; + +interface IndexRequest { + cancelled: boolean; + active: boolean; + expires: number; + cancelWork?: () => void; + workKey?: string; + buildId?: string; +} + +/** Bounded cancellation records also cover requests whose metadata lookup has not completed. */ +export function createIndexRequests(now = Date.now, capacity = 64) { + const requests = new Map(); + const owners = new Map(); + // Cancellation-before-start protection must never reserve build capacity. + const cancellations = new Map(); + const rememberCancellation = (key: string) => { + for (const [id, expires] of cancellations) if (expires <= now()) cancellations.delete(id); + cancellations.delete(key); + while (cancellations.size >= 256) cancellations.delete(cancellations.keys().next().value!); + cancellations.set(key, now() + 2 * 60 * 1000); + }; + const remove = (key: string, request: IndexRequest) => { + requests.delete(key); + if (request.workKey && owners.get(request.workKey) === request) owners.delete(request.workKey); + }; + const allocate = (key: string) => { + for (const [id, request] of requests) if (!request.active && request.expires <= now()) remove(id, request); + let request = requests.get(key); + if (!request) { + if (requests.size >= capacity) + throw new AppError(429, 'Too many indexing requests. Retry later.', 'INDEXING_BUSY', true); + request = { + cancelled: (cancellations.get(key) ?? 0) > now(), + active: false, + expires: now() + 25 * 60 * 60 * 1000, + }; + requests.set(key, request); + } + return request; + }; + return { + begin(key: string, buildId?: string) { + const request = allocate(key); + request.buildId = buildId; + if (request.active) throw new AppError(409, 'Indexing is already in progress.', 'INDEXING_IN_PROGRESS'); + request.active = true; + const resuming = !!request.workKey && owners.get(request.workKey) === request; + let attached = false; + return { + cancelled: () => request.cancelled, + attach(workKey: string, cancelWork: () => void) { + attached = true; + request.workKey = workKey; + owners.set(workKey, request); + request.cancelWork = () => { + if (owners.get(workKey) === request) cancelWork(); + }; + if (request.cancelled) request.cancelWork(); + }, + finish(keep = false, retainCancellation = true) { + keep ||= resuming && !attached && !request.cancelled; + keep &&= !request.cancelled; + request.active = false; + if (!keep) { + request.cancelWork = undefined; + if (request.workKey && owners.get(request.workKey) === request) owners.delete(request.workKey); + } + if (!keep) { + if (request.cancelled && retainCancellation) rememberCancellation(key); + remove(key, request); + } else request.expires = now() + 25 * 60 * 60 * 1000; + }, + }; + }, + cancel(key: string) { + rememberCancellation(key); + const request = requests.get(key); + if (!request) return; + request.cancelled = true; + request.cancelWork?.(); + if (!request.active) remove(key, request); + }, + cancelPending(key: string): boolean { + const request = requests.get(key); + if (!request?.active) return false; + request.cancelled = true; + request.cancelWork?.(); + return true; + }, + buildId(workKey: string) { + return owners.get(workKey)?.buildId; + }, + }; +} + +export const indexRequests = createIndexRequests(); diff --git a/server/search/indexing-pipeline.test.ts b/server/search/indexing-pipeline.test.ts index cea67aa..eb0eda3 100644 --- a/server/search/indexing-pipeline.test.ts +++ b/server/search/indexing-pipeline.test.ts @@ -1,295 +1,277 @@ -import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; -import * as lruModule from '../cache/lru'; -import * as loggerModule from '../utils/logger'; - -const realLoggerExports = { ...loggerModule }; - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - if (a.length !== b.length) return 0; - let dot = 0, magA = 0, magB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - magA += a[i] * a[i]; - magB += b[i] * b[i]; - } - const denom = Math.sqrt(magA) * Math.sqrt(magB); - if (denom === 0) return 0; - return Math.max(0, Math.min(1, dot / denom)); -} - -const vectorStore = { - chunks: [] as any[], - hasIndexValue: false, - addChunks: mock((chunks: any[]) => vectorStore.chunks.push(...chunks)), - removeByRepo: mock(() => { - vectorStore.chunks = []; - }), - removeByFile: mock((repoKey: string, filePath: string) => { - vectorStore.chunks = vectorStore.chunks.filter((chunk) => chunk.metadata.repoKey !== repoKey || chunk.metadata.filePath !== filePath); - }), - getChunkCount: mock((repoKey: string) => vectorStore.chunks.filter((chunk) => chunk.metadata.repoKey === repoKey).length), - hasIndex: mock(() => vectorStore.hasIndexValue || vectorStore.chunks.length > 0), - search: mock((queryVector: Float32Array, repoKey: string, topK: number, minScore: number) => { - return vectorStore.chunks - .filter((c) => c.metadata.repoKey === repoKey) - .map((c) => ({ chunk: c.metadata, score: cosineSimilarity(queryVector, c.vector) })) - .filter((r) => r.score >= minScore) - .sort((a, b) => b.score - a.score) - .slice(0, topK); - }), -}; - -let repoInfoSha = 'sha-current'; -let flatTrees: Record = {}; -let fileContents: Record = {}; -let fetchFileContentsError = false; -let embedBatchError = false; -let chunkFileResult: any[] | null = null; - -const fetchRepoInfo = mock(async () => ({ sha: repoInfoSha })); -const fetchFlatTree = mock(async (_owner: string, _repo: string, sha: string) => ({ items: flatTrees[sha] ?? [] })); -const fetchFileContents = mock(async (_owner: string, _repo: string, paths: string[]) => { - if (fetchFileContentsError) throw new Error('fetch failed'); - return Object.fromEntries(paths.map((path) => [path, fileContents[path]])); -}); -const embedBatch = mock(async (texts: string[]) => { - if (embedBatchError) throw new Error('embed failed'); - return texts.map(() => ({ vector: new Float32Array([1, 0]), tokenCount: 1 })); -}); -const chunkFile = mock((content: string, filePath: string, language: string) => chunkFileResult ?? [{ - content, - filePath, - language, - startLine: 1, - endLine: 1, - chunkIndex: 0, -}]); -const invalidateSearchCache = mock(() => undefined); -const logError = mock(() => undefined); - -const { createIndexingPipeline, getIndexingPipeline } = await import('./indexing-pipeline'); -const { AppError } = await import('../utils/errors'); - -const options = { owner: 'o', repo: 'r', commitSha: 'sha-current' }; -const key = 'o/r'; - -function setupModuleMocks() { - mock.module('./vector-store', () => ({ getVectorStore: () => vectorStore })); - mock.module('../github/fetch-tree', () => ({ fetchRepoInfo, fetchFlatTree, fetchFileContents })); - mock.module('./embedding-provider', () => ({ - createEmbeddingProvider: async () => ({ - embed: async (_text: string) => ({ vector: new Float32Array([1, 0]), tokenCount: 1 }), - embedBatch, - }), +import { test, expect, mock, spyOn } from 'bun:test'; +import { Octokit } from '@octokit/rest'; +import { createIndexingPipeline } from './indexing-pipeline'; +import { createVectorStore } from './vector-store'; +import { createChunker } from './chunker'; +import { searchIndexKey } from './index-identity'; +import { SEARCH_LIMITS } from './limits'; +import type { EmbeddingProvider } from './types'; +import { AppError } from '../utils/errors'; +const ctx = { octokit: new Octokit(), gitHubToken: 'test-token' }; +const options = { owner: 'owner', repo: 'repo', commitSha: 'first' }; + +test('combines small files into bounded batches and preserves their file paths', async () => { + const s = setup(); + s.fetchFlatTree.mockResolvedValue({ + items: Array.from({ length: 65 }, (_, i) => ({ + path: `file${i}.ts`, + type: 'blob' as const, + sha: 'blob', + size: 20, + })), + truncated: false, + totalFiles: 65, + }); + s.fetchFileContents.mockImplementation(async (_owner, _repo, paths: string[]) => ({ + 'a.ts': '', + ...Object.fromEntries(paths.map((path) => [path, 'export const value = 1;'])), })); - mock.module('./chunker', () => ({ createChunker: () => ({ chunkFile }) })); - mock.module('../cache/lru', () => ({ ...lruModule, invalidateSearchCache })); - mock.module('../utils/logger', () => ({ ...loggerModule, logError })); -} - -beforeEach(() => { - mock.restore(); - setupModuleMocks(); - for (const fn of [ - vectorStore.addChunks, - vectorStore.removeByRepo, - vectorStore.removeByFile, - vectorStore.getChunkCount, - vectorStore.hasIndex, - vectorStore.search, - fetchRepoInfo, - fetchFlatTree, - fetchFileContents, + await s.pipeline.startIndexing(options, ctx); + expect(s.embedBatch.mock.calls.map(([texts]) => texts.length)).toEqual([32, 32, 1]); + expect(s.fetchFileContents.mock.calls.every((call) => call[2].length <= 4)).toBe(true); + expect(s.fetchFileContents).toHaveBeenCalledTimes(17); + const key = searchIndexKey('owner', 'repo', 'first', ctx); + expect(s.store.getChunkCount(key)).toBe(65); + const results = s.store.search(new Float32Array([1, 0]), key, 100, 0); + expect(new Set(results.map((result) => result.chunk.filePath)).size).toBe(65); +}); +function setup(overrides: Partial = {}, now = Date.now) { + const limits = { ...SEARCH_LIMITS, ...overrides }; + const store = createVectorStore(limits); + const embedBatch = mock(async (texts: string[]) => + texts.map(() => ({ vector: new Float32Array([1, 0]), tokenCount: 1 })), + ); + const provider: EmbeddingProvider = { + dimensions: 2, + maxTokens: 512, + providerName: 'test', embedBatch, - chunkFile, - invalidateSearchCache, - logError, - ]) { - fn.mockClear(); - } - vectorStore.chunks = []; - vectorStore.hasIndexValue = false; - repoInfoSha = 'sha-current'; - flatTrees = { - 'sha-current': [ - { path: 'src/a.ts', type: 'blob', sha: 'a1' }, - { path: 'README.md', type: 'blob', sha: 'readme' }, - { path: 'image.png', type: 'blob', sha: 'png' }, - ], - 'sha-prev': [ - { path: 'src/a.ts', type: 'blob', sha: 'old-a' }, - { path: 'src/deleted.ts', type: 'blob', sha: 'deleted' }, - ], - }; - fileContents = { - 'src/a.ts': 'export const a = 1;', - 'README.md': '# readme', + embed: async () => ({ vector: new Float32Array([1, 0]), tokenCount: 1 }), }; - fetchFileContentsError = false; - embedBatchError = false; - chunkFileResult = null; -}); - -afterEach(() => { - mock.restore(); - mock.module('../utils/logger', () => realLoggerExports); -}); - -describe('indexing pipeline', () => { - it('returns idle status and can cancel to idle', () => { - const pipeline = createIndexingPipeline(); + const fetchFlatTree = mock(async () => ({ + items: [{ path: 'a.ts', type: 'blob' as const, sha: 'blob', size: 20 }], + truncated: false, + totalFiles: 1, + })); + const fetchFileContents = mock(async () => ({ + 'a.ts': 'export const value = 1;', + })); + const pipeline = createIndexingPipeline( + { + fetchFlatTree, + fetchFileContents, + createChunker, + getVectorStore: () => store, + createEmbeddingProvider: async () => provider, + }, + limits, + now, + ); + return { store, pipeline, embedBatch, fetchFlatTree, fetchFileContents }; +} - expect(pipeline.getStatus(key)).toEqual({ state: 'idle' }); - pipeline.cancelIndexing(key); - expect(pipeline.getStatus(key)).toEqual({ state: 'idle' }); +test('rate limit preserves completed batches and resumes only missing embeddings after cooldown', async () => { + let now = 1000; + const s = setup({}, () => now); + s.fetchFlatTree.mockResolvedValue({ + items: Array.from({ length: 40 }, (_, i) => ({ + path: `file${i}.ts`, + type: 'blob' as const, + sha: 'blob', + size: 20, + })), + truncated: false, + totalFiles: 40, }); - - it('indexes supported files and stores chunks', async () => { - const pipeline = createIndexingPipeline(); - - await pipeline.startIndexing(options, {} as any); - - expect(fetchRepoInfo).toHaveBeenCalledWith('o', 'r', undefined, {}); - expect(fetchFlatTree).toHaveBeenCalledWith('o', 'r', 'sha-current', {}); - expect(vectorStore.removeByRepo).toHaveBeenCalledWith(key); - expect(fetchFileContents).toHaveBeenCalledWith('o', 'r', ['src/a.ts', 'README.md'], 'sha-current', {}); - expect(vectorStore.addChunks).toHaveBeenCalledTimes(1); - expect(vectorStore.chunks.map((chunk) => chunk.metadata.filePath)).toEqual(['src/a.ts', 'README.md']); - expect(invalidateSearchCache).toHaveBeenCalledWith('o', 'r'); - expect(pipeline.getStatus(key).state).toBe('complete'); + s.fetchFileContents.mockImplementation(async (_owner, _repo, paths: string[]) => ({ + 'a.ts': '', + ...Object.fromEntries(paths.map((path) => [path, 'export const value = 1;'])), + })); + let calls = 0; + s.embedBatch.mockImplementation(async (texts) => { + if (++calls === 2) + throw new AppError(429, 'Provider cooldown', 'EMBEDDING_RATE_LIMITED', true, { retryAfterSeconds: 60 }); + return texts.map(() => ({ vector: new Float32Array([1, 0]), tokenCount: 1 })); }); - - it('uses repo info sha when commitSha is empty', async () => { - const pipeline = createIndexingPipeline(); - repoInfoSha = 'resolved-sha'; - flatTrees['resolved-sha'] = flatTrees['sha-current']; - - await pipeline.startIndexing({ ...options, commitSha: '' }, {} as any); - - expect(fetchFlatTree).toHaveBeenCalledWith('o', 'r', 'resolved-sha', {}); + const key = searchIndexKey('owner', 'repo', 'first', ctx); + await s.pipeline.startIndexing(options, ctx); + expect(s.pipeline.getStatus(key)).toMatchObject({ + state: 'paused', + filesProcessed: 32, + totalFiles: 40, + retryAt: 61000, }); + expect(s.store.hasIndex(key)).toBe(false); + const partial = s.pipeline.available(key)!; + expect(partial.coverage).toMatchObject({ kind: 'partial', indexedFiles: 32 }); + expect(partial.store.getChunkCount(key)).toBe(32); + expect(s.pipeline.available(searchIndexKey('owner', 'repo', 'first', { gitHubToken: 'other' }))).toBeUndefined(); + expect(s.pipeline.available(searchIndexKey('owner', 'repo', 'first', ctx, ['src']))).toBeUndefined(); + await s.pipeline.startIndexing(options, ctx); + expect(calls).toBe(2); + now = 61000; + await s.pipeline.startIndexing(options, ctx); + expect(s.embedBatch.mock.calls.map(([texts]) => texts.length)).toEqual([32, 8, 8]); + expect(s.store.getChunkCount(key)).toBe(40); + expect(s.pipeline.getStatus(key).state).toBe('complete'); +}); - it('skips indexing when repo already has chunks and no previousSha', async () => { - const pipeline = createIndexingPipeline(); - vectorStore.hasIndexValue = true; - vectorStore.chunks = [{ metadata: { repoKey: key } }]; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchRepoInfo).not.toHaveBeenCalled(); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'complete', chunkCount: 1 }); +test('cancelling a paused build discards its partial results', async () => { + const s = setup(); + s.embedBatch.mockImplementation(async () => { + throw new AppError(429, 'Limit', 'USAGE_LIMIT_EXCEEDED', true, { retryAfterSeconds: 86400 }); }); - - it('still reindexes when index exists but chunk count is zero', async () => { - const pipeline = createIndexingPipeline(); - vectorStore.hasIndexValue = true; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchRepoInfo).toHaveBeenCalled(); + await s.pipeline.startIndexing(options, ctx); + const key = searchIndexKey('owner', 'repo', 'first', ctx); + expect(s.pipeline.getStatus(key).state).toBe('paused'); + s.pipeline.cancelIndexing(key); + expect(s.pipeline.getStatus(key).state).toBe('idle'); + expect(s.pipeline.available(key)).toBeUndefined(); +}); +test('same snapshot is reused; changed SHA and credential build separate indices', async () => { + const s = setup(); + await s.pipeline.startIndexing(options, ctx); + await s.pipeline.startIndexing(options, ctx); + expect(s.embedBatch).toHaveBeenCalledTimes(1); + await s.pipeline.startIndexing({ ...options, commitSha: 'second' }, ctx); + await s.pipeline.startIndexing(options, { ...ctx, gitHubToken: 'other' }); + expect(s.embedBatch).toHaveBeenCalledTimes(3); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'first', ctx))).toBe(true); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'second', ctx))).toBe(true); +}); +test('failed replacement pauses and preserves the published old snapshot', async () => { + const s = setup(); + await s.pipeline.startIndexing(options, ctx); + s.embedBatch.mockImplementation(async () => { + throw new Error('secret provider detail'); }); - - it('does incremental indexing and removes deleted files', async () => { - const pipeline = createIndexingPipeline(); - vectorStore.hasIndexValue = true; - vectorStore.chunks = [{ metadata: { repoKey: key, filePath: 'src/deleted.ts' } }]; - - await pipeline.startIndexing({ ...options, previousSha: 'sha-prev' }, {} as any); - - expect(fetchFlatTree).toHaveBeenCalledWith('o', 'r', 'sha-prev', {}); - expect(vectorStore.removeByFile).toHaveBeenCalledWith(key, 'src/deleted.ts'); - expect(fetchFileContents).toHaveBeenCalledWith('o', 'r', ['src/a.ts', 'README.md'], 'sha-current', {}); + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + await s.pipeline.startIndexing({ ...options, commitSha: 'new' }, ctx); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'first', ctx))).toBe(true); + const key = searchIndexKey('owner', 'repo', 'new', ctx); + expect(s.store.hasIndex(key)).toBe(false); + expect(s.pipeline.getStatus(key)).toMatchObject({ + state: 'paused', + coverage: { kind: 'previous', commitSha: 'first' }, + error: 'Repository indexing failed. Please retry.', + }); + } finally { + log.mockRestore(); + } +}); +test('global job cap rejects other repositories and duplicate jobs without queuing', async () => { + const s = setup({ concurrentJobs: 1 }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); - - it('handles no supported files', async () => { - const pipeline = createIndexingPipeline(); - flatTrees['sha-current'] = [{ path: 'asset.png', type: 'blob', sha: 'png' }]; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchFileContents).not.toHaveBeenCalled(); - expect(vectorStore.addChunks).toHaveBeenCalledWith([]); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'complete', chunkCount: 0 }); + s.fetchFileContents.mockImplementation(async () => { + await gate; + return { 'a.ts': 'const a = 1;' }; }); - - it('fetches file contents in batches of at most five paths', async () => { - const pipeline = createIndexingPipeline(); - flatTrees['sha-current'] = Array.from({ length: 12 }, (_, i) => ({ path: `src/file-${i}.ts`, type: 'blob', sha: `sha-${i}` })); - fileContents = Object.fromEntries(flatTrees['sha-current'].map((item) => [item.path, `export const n = ${item.sha};`])) as Record; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchFileContents.mock.calls.map((call) => call[2].length)).toEqual([5, 5, 2]); + const first = s.pipeline.startIndexing(options, ctx); + try { + await expect(s.pipeline.startIndexing(options, ctx)).rejects.toMatchObject({ + status: 409, + }); + await expect(s.pipeline.startIndexing({ ...options, repo: 'other' }, ctx)).rejects.toMatchObject({ status: 429 }); + } finally { + release(); + await first; + } +}); +test('file and chunk limits reject before embedding', async () => { + for (const limits of [{ fileBytes: 10 }, { chunks: 0 }, { indexBytes: 1 }]) { + const s = setup(limits); + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + await expect(s.pipeline.startIndexing(options, ctx)).rejects.toMatchObject({ status: 413 }); + expect(s.embedBatch).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + } +}); +test('indexes only included paths after excluded paths are removed', async () => { + const s = setup(); + s.fetchFlatTree.mockResolvedValue({ + items: [ + { + path: 'packages/app/src/main.ts', + type: 'blob' as const, + sha: 'main', + size: 20, + }, + { + path: 'packages/app/test/main.test.ts', + type: 'blob' as const, + sha: 'test', + size: 20, + }, + { path: 'docs/guide.md', type: 'blob' as const, sha: 'docs', size: 20 }, + ], + truncated: false, + totalFiles: 3, }); + s.fetchFileContents.mockImplementation(async (_owner, _repo, paths: string[]) => ({ + [paths[0]]: 'export const value = 1;', + })); - it('counts missing content as failed and aborts over threshold', async () => { - const pipeline = createIndexingPipeline(); - fileContents = {}; - - await pipeline.startIndexing(options, {} as any); + const scope = { + includePaths: ['packages/app'], + excludePaths: ['packages/app/test'], + }; + await s.pipeline.startIndexing({ ...options, ...scope }, ctx); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', failedFiles: 2 }); - expect(vectorStore.addChunks).not.toHaveBeenCalled(); + expect(s.fetchFileContents).toHaveBeenCalledTimes(1); + expect(s.fetchFileContents.mock.calls[0][2]).toEqual(['packages/app/src/main.ts']); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'first', ctx, scope.includePaths, scope.excludePaths))).toBe( + true, + ); +}); +test('empty repository is an actual completed index', async () => { + const s = setup(); + s.fetchFlatTree.mockResolvedValue({ + items: [], + truncated: false, + totalFiles: 0, }); - - it('skips files with no chunks', async () => { - const pipeline = createIndexingPipeline(); - chunkFileResult = []; - - await pipeline.startIndexing(options, {} as any); - - expect(embedBatch).not.toHaveBeenCalled(); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'complete', chunkCount: 0 }); + await s.pipeline.startIndexing(options, ctx); + expect(s.pipeline.getStatus(searchIndexKey('owner', 'repo', 'first', ctx))).toMatchObject({ + state: 'complete', + chunkCount: 0, }); - - it('counts fetch content failures and aborts over threshold', async () => { - const pipeline = createIndexingPipeline(); - fetchFileContentsError = true; - - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', failedFiles: 2 }); +}); +test('status becomes idle when its index is evicted', async () => { + const s = setup({ indices: 1 }); + await s.pipeline.startIndexing(options, ctx); + await s.pipeline.startIndexing({ ...options, commitSha: 'new' }, ctx); + expect(s.pipeline.getStatus(searchIndexKey('owner', 'repo', 'first', ctx))).toMatchObject({ + state: 'idle', + coverage: { kind: 'previous' }, }); - - it('counts embedding failures and aborts over threshold', async () => { - const pipeline = createIndexingPipeline(); - embedBatchError = true; - - await pipeline.startIndexing(options, {} as any); - - expect(logError).toHaveBeenCalledWith('indexing:embed', expect.any(Error), { file: expect.any(String) }); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', failedFiles: 2 }); +}); +test('cancellation keeps the concurrency slot until the pending operation settles', async () => { + const s = setup({ concurrentJobs: 1 }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); - - it('records failed status when setup throws', async () => { - const pipeline = createIndexingPipeline(); - fetchRepoInfo.mockImplementationOnce(async () => { - throw new Error('boom'); - }); - - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', error: 'boom', failedFiles: 0 }); - expect(logError).toHaveBeenCalledWith('/api/search/index', expect.any(Error), { repo: key }); + s.fetchFileContents.mockImplementation(async () => { + await gate; + return { 'a.ts': 'const a = 1;' }; }); - - it('rejects concurrent indexing and then clears active flag', async () => { - const pipeline = createIndexingPipeline(); - let release!: () => void; - fetchRepoInfo.mockImplementationOnce(async () => new Promise((resolve) => { - release = () => resolve({ sha: 'sha-current' }); - })); - - const first = pipeline.startIndexing(options, {} as any); - await expect(pipeline.startIndexing(options, {} as any)).rejects.toBeInstanceOf(AppError); + const log = spyOn(console, 'error').mockImplementation(() => {}); + const work = s.pipeline.startIndexing(options, ctx); + const rejection = work.catch((error) => error); + s.pipeline.cancelIndexing(searchIndexKey('owner', 'repo', 'first', ctx)); + try { + await expect(s.pipeline.startIndexing({ ...options, repo: 'other' }, ctx)).rejects.toMatchObject({ status: 429 }); + } finally { release(); - await first; - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key).state).toBe('complete'); - }); - - it('returns singleton pipeline', () => { - expect(getIndexingPipeline()).toBe(getIndexingPipeline()); - }); -}); \ No newline at end of file + expect(await rejection).toBeUndefined(); + expect(s.pipeline.getStatus(searchIndexKey('owner', 'repo', 'first', ctx)).state).toBe('idle'); + log.mockRestore(); + } +}); diff --git a/server/search/indexing-pipeline.ts b/server/search/indexing-pipeline.ts index 1eadc41..e761934 100644 --- a/server/search/indexing-pipeline.ts +++ b/server/search/indexing-pipeline.ts @@ -1,233 +1,140 @@ -import type { IndexingOptions, IndexingStatus, IndexingPipeline, RequestContext, IndexedChunk } from './types'; -import { SUPPORTED_EXTENSIONS, extToLanguage } from './constants'; -import { createEmbeddingProvider } from './embedding-provider'; -import { createChunker } from './chunker'; -import { getVectorStore } from './vector-store'; -import { fetchFlatTree, fetchFileContents, fetchRepoInfo } from '../github/fetch-tree'; -import { invalidateSearchCache } from '../cache/lru'; -import { AppError } from '../utils/errors'; +import type { IndexingStatus, IndexingPipeline } from './types'; +import { buildSnapshot, indexingDependencies as defaults } from './build-snapshot'; +import { createCheckpoints } from './checkpoints'; +import { createIndexAvailability } from './index-availability'; +import { AppError, toErrorPayload } from '../utils/errors'; import { logError } from '../utils/logger'; - -function repoKey(owner: string, repo: string): string { - return `${owner}/${repo}`; -} - -/** Tracks per-repo indexing status. */ -const statusMap = new Map(); -const activeIndexing = new Set(); // prevents concurrent indexing - -export function createIndexingPipeline(): IndexingPipeline { +import { searchIndexKey, searchIndexIdentity } from './index-identity'; +import { SEARCH_LIMITS } from './limits'; + +export function createIndexingPipeline(deps = defaults, limits = SEARCH_LIMITS, now = Date.now): IndexingPipeline { + const statuses = new Map(); + const active = new Map(); + const checkpoints = createCheckpoints(limits, now); + const { available, publish } = createIndexAvailability(deps, checkpoints, limits, now); + const save = (key: string, value: IndexingStatus) => { + value = { ...value, snapshotSha: searchIndexIdentity(key).snapshotSha }; + for (const [id, entry] of statuses) if (entry.expires <= now() && !active.has(id)) statuses.delete(id); + statuses.delete(key); + while (statuses.size >= limits.statuses) { + const oldest = [...statuses.keys()].find((id) => !active.has(id)); + if (!oldest) break; + statuses.delete(oldest); + } + statuses.set(key, { value, expires: now() + (value.state === 'paused' ? 25 * 60 * 60 * 1000 : limits.ttlMs) }); + }; return { - async startIndexing(options: IndexingOptions, ctx: RequestContext): Promise { - const key = repoKey(options.owner, options.repo); - - // Prevent concurrent indexing - if (activeIndexing.has(key)) { - throw new AppError(409, 'Indexing already in progress for this repository.', 'INDEXING_IN_PROGRESS'); - } - - activeIndexing.add(key); - statusMap.set(key, { state: 'indexing', progress: 0, filesProcessed: 0, totalFiles: 0 }); - + available, + async startIndexing(options, ctx) { + const { owner, repo, commitSha, includePaths = [], excludePaths = [] } = options; + const key = searchIndexKey(owner, repo, commitSha, ctx, includePaths, excludePaths); + const prior = statuses.get(key)?.value; + if (prior?.state === 'paused' && prior.retryAt && prior.retryAt > now()) return; + if (active.has(key)) throw new AppError(409, 'Indexing is already in progress.', 'INDEXING_IN_PROGRESS'); + if (active.size >= limits.concurrentJobs) + throw new AppError(429, 'Search indexing is busy. Please retry later.', 'INDEXING_BUSY', true); + const job = { cancelled: false }; + active.set(key, job); + const check = () => { + if (job.cancelled) throw new AppError(409, 'Indexing was cancelled.', 'INDEXING_CANCELLED'); + }; + save(key, { + state: 'indexing', + progress: 0, + filesProcessed: 0, + totalFiles: 0, + }); try { - await runIndexing(options, ctx, key); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - statusMap.set(key, { state: 'failed', error: msg, failedFiles: 0 }); - logError('/api/search/index', err, { repo: key }); + const store = deps.getVectorStore(); + if (store.hasIndex(key)) { + save(key, { + state: 'complete', + chunkCount: store.getChunkCount(key), + timestamp: now(), + }); + return; + } + const { indexed, totalFiles } = await buildSnapshot( + deps, + limits, + key, + options, + ctx, + checkpoints, + check, + (status) => save(key, status), + ); + store.replaceIndex(key, indexed); + checkpoints.remove(key); + publish(key, { kind: 'complete', commitSha, indexedFiles: totalFiles, totalFiles }); + save(key, { + state: 'complete', + chunkCount: indexed.length, + timestamp: now(), + }); + } catch (error) { + const failure = + error instanceof AppError + ? error + : new AppError(502, 'Repository indexing failed. Please retry.', 'INDEXING_FAILED', true); + const checkpoint = checkpoints.get(key); + if (job.cancelled) { + checkpoints.remove(key); + save(key, { state: 'idle' }); + return; + } + if (failure.retryable && (checkpoint || failure.status === 429)) { + const seconds = failure.context?.retryAfterSeconds; + save(key, { + state: 'paused', + error: failure.message, + reason: failure.code, + retryAt: typeof seconds === 'number' ? now() + seconds * 1000 : undefined, + filesProcessed: checkpoint?.completed.size ?? 0, + totalFiles: checkpoint?.totalFiles ?? 0, + progress: checkpoint?.totalFiles + ? Math.round((checkpoint.completed.size / checkpoint.totalFiles) * 100) + : 0, + }); + return; + } + checkpoints.remove(key); + save(key, { state: 'failed', error: failure.message, failedFiles: 0 }); + // Log the sanitized AppError payload, not the raw provider/network + // error — logError serializes error.message/stack verbatim (URLs, keys, fragments). + logError('/api/search/index', toErrorPayload(failure), { repo: owner + '/' + repo }); + throw failure; } finally { - activeIndexing.delete(key); + active.delete(key); } }, - - getStatus(key: string): IndexingStatus { - return statusMap.get(key) ?? { state: 'idle' }; + getStatus(key) { + const entry = statuses.get(key); + if (entry && (active.has(key) || entry.expires > now())) { + if (entry.value.state !== 'complete' || deps.getVectorStore().hasIndex(key)) + return { ...entry.value, coverage: available(key)?.coverage }; + } + statuses.delete(key); + if (deps.getVectorStore().hasIndex(key)) + return { + state: 'complete', + chunkCount: deps.getVectorStore().getChunkCount(key), + timestamp: now(), + }; + return { state: 'idle', coverage: available(key)?.coverage }; }, - - cancelIndexing(key: string): void { - activeIndexing.delete(key); - statusMap.set(key, { state: 'idle' }); + cancelIndexing(key) { + const job = active.get(key); + if (job) + job.cancelled = true; // Keep the slot until the awaited operation settles. + else { + checkpoints.remove(key); + statuses.delete(key); + } }, }; } - -async function runIndexing(options: IndexingOptions, ctx: RequestContext, key: string): Promise { - const { owner, repo, commitSha, previousSha } = options; - const vectorStore = getVectorStore(); - - // Idempotent: if same SHA already indexed, skip - if (vectorStore.hasIndex(key) && !previousSha) { - // Check if existing index matches this SHA – we can't easily verify, so re-index only if not present - const existingCount = vectorStore.getChunkCount(key); - if (existingCount > 0) { - statusMap.set(key, { state: 'complete', chunkCount: existingCount, timestamp: Date.now() }); - return; - } - } - - // 1. Fetch the file tree first so we can report totalFiles early - const info = await fetchRepoInfo(owner, repo, options.branch, ctx); - const targetSha = commitSha || info.sha; - const { items } = await fetchFlatTree(owner, repo, targetSha, ctx); - - // 2. Filter to supported extensions - const sourceFiles = items.filter((item) => { - const ext = getExtension(item.path); - return SUPPORTED_EXTENSIONS.has(ext); - }); - - const totalFiles = sourceFiles.length; - statusMap.set(key, { state: 'indexing', progress: 0, filesProcessed: 0, totalFiles }); - - // 3. Create embedding provider (may fail on proxy issues) - const provider = await createEmbeddingProvider(); - const chunker = createChunker(); - - // 4. Determine files to process (incremental: only changed files) - let filesToProcess = sourceFiles; - - if (previousSha && vectorStore.hasIndex(key)) { - const { items: prevItems } = await fetchFlatTree(owner, repo, previousSha, ctx); - const prevFiles = new Set( - prevItems.filter((i) => SUPPORTED_EXTENSIONS.has(getExtension(i.path))).map((i) => i.path), - ); - const currFiles = new Set(sourceFiles.map((i) => i.path)); - - // Files in current but not prev, or SHA changed → treat as modified - // Files in prev but not current → delete - const filesToDelete = [...prevFiles].filter((f) => !currFiles.has(f)); - const prevItemsMap = new Map(prevItems.map((p) => [p.path, p])); - const modifiedOrAdded = sourceFiles.filter((item) => { - const prevItem = prevItemsMap.get(item.path); - return !prevItem || prevItem.sha !== item.sha; // SHA changed or new file - }); - filesToProcess = modifiedOrAdded; - - // Remove deleted files from store - for (const filePath of filesToDelete) { - vectorStore.removeByFile(key, filePath); - } - } else { - // Full re-index: clear existing - vectorStore.removeByRepo(key); - } - - const BATCH_SIZE = 5; - let filesProcessed = 0; - let failedChunks = 0; - let totalChunks = 0; - const allIndexedChunks: IndexedChunk[] = []; - - const batches: (typeof filesToProcess)[] = []; - for (let i = 0; i < filesToProcess.length; i += BATCH_SIZE) { - batches.push(filesToProcess.slice(i, i + BATCH_SIZE)); - } - - for (const batch of batches) { - const paths = batch.map((b) => b.path); - - let contents: Record; - try { - contents = await fetchFileContents(owner, repo, paths, targetSha, ctx); - } catch { - failedChunks += batch.length; - filesProcessed += batch.length; - continue; - } - - for (const item of batch) { - const content = contents[item.path]; - if (!content) { - failedChunks++; - filesProcessed++; - continue; - } - - const ext = getExtension(item.path); - const language = extToLanguage(ext); - const chunks = chunker.chunkFile(content, item.path, language); - - if (chunks.length === 0) { - filesProcessed++; - continue; - } - - totalChunks += chunks.length; - - try { - const chunkContents = new Array(chunks.length); - for (let j = 0; j < chunks.length; j++) { - chunkContents[j] = chunks[j].content; - } - const embeddings = await provider.embedBatch(chunkContents); - - for (let j = 0; j < chunks.length; j++) { - const chunk = chunks[j]; - const embedding = embeddings[j]; - allIndexedChunks.push({ - id: `${key}:${item.path}:${chunk.chunkIndex}`, - vector: embedding.vector, - metadata: { - filePath: item.path, - startLine: chunk.startLine, - endLine: chunk.endLine, - chunkIndex: chunk.chunkIndex, - language: chunk.language, - content: chunk.content.slice(0, 2000), - repoKey: key, - commitSha: targetSha, - }, - }); - } - } catch (err) { - failedChunks++; - logError('indexing:embed', err, { file: item.path }); - } - - filesProcessed++; - } - - const progress = totalFiles > 0 ? Math.round((filesProcessed / totalFiles) * 100) : 100; - statusMap.set(key, { state: 'indexing', progress, filesProcessed, totalFiles }); - } - - // 6. Check failure threshold - const totalAttempted = totalChunks + failedChunks; - if (totalAttempted > 0 && failedChunks / totalAttempted > 0.1) { - statusMap.set(key, { - state: 'failed', - error: `Indexing aborted: ${failedChunks} of ${totalAttempted} chunks failed (>10% threshold)`, - failedFiles: failedChunks, - }); - return; - } - - // 7. Store all chunks - vectorStore.addChunks(allIndexedChunks); - - // 8. Invalidate search cache for this repo (SHA changed) - invalidateSearchCache(owner, repo); - - statusMap.set(key, { - state: 'complete', - chunkCount: vectorStore.getChunkCount(key), - timestamp: Date.now(), - }); -} - -function getExtension(path: string): string { - const dot = path.lastIndexOf('.'); - if (dot < 0) return ''; - return path.slice(dot).toLowerCase(); -} - -// ── Singleton ────────────────────────────────────────────────────────── - let globalPipeline: IndexingPipeline | null = null; - export function getIndexingPipeline(): IndexingPipeline { - if (!globalPipeline) { - globalPipeline = createIndexingPipeline(); - } - return globalPipeline; + return (globalPipeline ??= createIndexingPipeline()); } diff --git a/server/search/limits.ts b/server/search/limits.ts new file mode 100644 index 0000000..478814b --- /dev/null +++ b/server/search/limits.ts @@ -0,0 +1,11 @@ +export const SEARCH_LIMITS = { + concurrentJobs: 2, + files: 2000, + fileBytes: 256 * 1024, + chunks: 4000, + indexBytes: 32 * 1024 * 1024, + totalBytes: 128 * 1024 * 1024, + indices: 16, + statuses: 64, + ttlMs: 30 * 60 * 1000, +}; diff --git a/server/search/qa-engine.test.ts b/server/search/qa-engine.test.ts index 232a3b2..9152255 100644 --- a/server/search/qa-engine.test.ts +++ b/server/search/qa-engine.test.ts @@ -1,3 +1,4 @@ +import { searchIndexKey } from './index-identity'; import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test'; import { createQAEngine, getQAEngine } from './qa-engine'; import { clearAllCaches } from '../cache/lru'; @@ -14,7 +15,7 @@ mock.module('./embedding-provider', () => ({ }), })); -const repoKey = 'owner/repo'; +const repoKey = () => searchIndexKey('owner', 'repo', 'sha'); function indexedChunk(id: string, scoreVector: Float32Array): IndexedChunk { return { @@ -27,7 +28,7 @@ function indexedChunk(id: string, scoreVector: Float32Array): IndexedChunk { chunkIndex: 0, language: 'typescript', content: `export const ${id} = true;`, - repoKey, + repoKey: repoKey(), commitSha: 'sha', }, }; @@ -36,7 +37,7 @@ function indexedChunk(id: string, scoreVector: Float32Array): IndexedChunk { describe('createQAEngine', () => { beforeEach(() => { clearAllCaches(); - getVectorStore().removeByRepo(repoKey); + getVectorStore().removeByRepo(repoKey()); process.env.AI_PROVIDER = 'mock'; }); @@ -47,16 +48,9 @@ describe('createQAEngine', () => { const qa = createQAEngine(); it('returns not-available answer when no index exists', async () => { - const response = await qa.ask({ - question: 'Where is the auth middleware?', - owner: 'nobody', - repo: 'ghost', - commitSha: 'abc', - }); - - expect(response.answer).toContain('could not find'); - expect(response.citations).toEqual([]); - expect(typeof response.cached).toBe('boolean'); + await expect( + qa.ask({ question: 'Where is the auth middleware?', owner: 'nobody', repo: 'ghost', commitSha: 'abc' }), + ).rejects.toMatchObject({ status: 404 }); }); it('returns empty citations when scores are below threshold', async () => { diff --git a/server/search/qa-engine.ts b/server/search/qa-engine.ts index a6632a1..199682e 100644 --- a/server/search/qa-engine.ts +++ b/server/search/qa-engine.ts @@ -1,4 +1,5 @@ import type { QAEngine, QAOptions, QAResponse, Citation } from './types'; +import type { SearchCoverage } from '../../src/types'; import { DEFAULT_MIN_SCORE } from './constants'; import { getSearchEngine } from './search-engine'; import { getAIProvider } from '../ai/provider'; @@ -7,6 +8,17 @@ const QA_TOP_K = 5; const NOT_AVAILABLE_MESSAGE = 'I could not find relevant information in the indexed codebase to answer this question. Try indexing the repository first or rephrasing your question.'; +// Local copy of the coverage notice formatter: runtime code cannot be shared +// across the server/frontend boundary (only src/types is in both tsconfigs), +// so this intentionally mirrors coverageMessage in src/lib/search-coverage.ts. +function coverageMessage(coverage: SearchCoverage): string { + if (coverage.kind === 'previous') + return `Showing results from the previous index (${coverage.commitSha.slice(0, 7)}).`; + if (coverage.kind === 'partial') + return `Partial results — ${coverage.indexedFiles} of ${coverage.totalFiles} files indexed.`; + return 'Showing results from the complete index.'; +} + export function createQAEngine(): QAEngine { const searchEngine = getSearchEngine(); @@ -15,27 +27,27 @@ export function createQAEngine(): QAEngine { const { question, owner, repo, commitSha, apiKey } = options; // Retrieve top 5 chunks via semantic search - let searchResponse; - try { - searchResponse = await searchEngine.search({ - query: question, - owner, - repo, - commitSha, - topK: QA_TOP_K, - minScore: DEFAULT_MIN_SCORE, - }); - } catch { - // If search fails (no index, etc.) return not-available - return { answer: NOT_AVAILABLE_MESSAGE, citations: [], cached: false }; - } + const searchResponse = await searchEngine.search({ + query: question, + owner, + repo, + commitSha, + gitHubToken: options.gitHubToken, + includePaths: options.includePaths, + excludePaths: options.excludePaths, + topK: QA_TOP_K, + minScore: DEFAULT_MIN_SCORE, + }); - const relevantResults = searchResponse.results.filter( - (r) => r.score >= DEFAULT_MIN_SCORE, - ); + const relevantResults = searchResponse.results.filter((r) => r.score >= DEFAULT_MIN_SCORE); if (relevantResults.length < 1) { - return { answer: NOT_AVAILABLE_MESSAGE, citations: [], cached: searchResponse.cached }; + return { + answer: NOT_AVAILABLE_MESSAGE, + citations: [], + cached: searchResponse.cached, + coverage: searchResponse.coverage, + }; } // Build context from retrieved chunks @@ -48,8 +60,17 @@ export function createQAEngine(): QAEngine { // Generate answer using the configured AI provider const aiProvider = await getAIProvider(apiKey); + const coverage = searchResponse.coverage; + const notice = coverage ? coverageMessage(coverage) : ''; const answer = await aiProvider.complete([ - { role: 'system', content: SYSTEM_PROMPT }, + { + role: 'system', + content: + SYSTEM_PROMPT + + '\n' + + notice + + '\nAnswers cover only indexed files. Never infer repository-wide absence or completeness from the retrieved excerpts.', + }, { role: 'user', content: prompt }, ]); @@ -61,7 +82,8 @@ export function createQAEngine(): QAEngine { })); return { - answer, + answer: coverage && coverage.kind !== 'complete' ? notice + '\n\n' + answer : answer, + coverage, citations, cached: searchResponse.cached, }; diff --git a/server/search/resumable-search.test.ts b/server/search/resumable-search.test.ts new file mode 100644 index 0000000..ff864e3 --- /dev/null +++ b/server/search/resumable-search.test.ts @@ -0,0 +1,63 @@ +import { test, expect } from 'bun:test'; +import { createCheckpoints } from './checkpoints'; +import { createIndexAvailability } from './index-availability'; +import { createVectorStore } from './vector-store'; +import { createSearchEngine } from './search-engine'; +import { searchIndexKey } from './index-identity'; +import { SEARCH_LIMITS } from './limits'; +import type { IndexedChunk, EmbeddingProvider } from './types'; + +test('partial search is not cached and switches to the completed snapshot', async () => { + const store = createVectorStore(); + const checkpoints = createCheckpoints(); + const { available, publish } = createIndexAvailability({ getVectorStore: () => store }, checkpoints); + const options = { owner: 'partial', repo: 'search', commitSha: 'one', query: 'find example' }; + const key = searchIndexKey(options.owner, options.repo, options.commitSha); + const chunk: IndexedChunk = { + id: key + ':a', + vector: new Float32Array([1, 0]), + metadata: { + repoKey: key, + commitSha: 'one', + filePath: 'a.ts', + startLine: 1, + endLine: 1, + chunkIndex: 0, + language: 'typescript', + content: 'example', + }, + }; + checkpoints.save(key, [chunk], new Set(['a.ts']), 2); + const provider: EmbeddingProvider = { + providerName: 'test', + dimensions: 2, + maxTokens: 100, + embed: async () => ({ vector: new Float32Array([1, 0]), tokenCount: 1 }), + embedBatch: async () => [], + }; + const engine = createSearchEngine( + { getVectorStore: () => store, createEmbeddingProvider: async () => provider }, + available, + ); + expect((await engine.search(options)).coverage?.kind).toBe('partial'); + expect((await engine.search(options)).cached).toBe(false); + store.replaceIndex(key, [chunk]); + checkpoints.remove(key); + publish(key, { kind: 'complete', commitSha: 'one', indexedFiles: 2, totalFiles: 2 }); + expect((await engine.search(options)).coverage?.kind).toBe('complete'); + const previous = await engine.search({ ...options, commitSha: 'two' }); + expect(previous.coverage?.kind).toBe('previous'); + expect(previous.results[0].chunk.commitSha).toBe('one'); + expect(available(searchIndexKey('partial', 'search', 'two', { gitHubToken: 'different' }))).toBeUndefined(); +}); + +test('checkpoint capacity rejects excess entries and expiry releases them', () => { + let now = 0; + const checkpoints = createCheckpoints({ ...SEARCH_LIMITS, indices: 1 }, () => now); + checkpoints.save('one', [], new Set(), 1); + expect(() => checkpoints.save('two', [], new Set(), 1)).toThrow('capacity'); + now = 26 * 60 * 60 * 1000; + expect(checkpoints.get('one')).toBeUndefined(); + checkpoints.save('two', [], new Set(), 1); + expect(checkpoints.get('two')).toBeDefined(); +}); diff --git a/server/search/search-engine.test.ts b/server/search/search-engine.test.ts index aeed866..f4bf2d5 100644 --- a/server/search/search-engine.test.ts +++ b/server/search/search-engine.test.ts @@ -1,10 +1,11 @@ -import { beforeEach, describe, expect, it, mock } from 'bun:test'; +import { searchIndexKey } from './index-identity'; +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; import { cache, clearAllCaches, hashContext } from '../cache/lru'; import { createSearchEngine, getSearchEngine } from './search-engine'; import { getVectorStore } from './vector-store'; import type { IndexedChunk, SearchEngine, SearchResponse } from './types'; -const repoKey = 'owner/repo'; +const repoKey = () => searchIndexKey('owner', 'repo', 'sha'); function chunk(id: string, vector: Float32Array): IndexedChunk { return { @@ -17,7 +18,7 @@ function chunk(id: string, vector: Float32Array): IndexedChunk { chunkIndex: 0, language: 'typescript', content: `content ${id}`, - repoKey, + repoKey: repoKey(), commitSha: 'sha', }, }; @@ -27,16 +28,16 @@ describe('createSearchEngine — input validation', () => { const engine: SearchEngine = createSearchEngine(); it('rejects query shorter than 3 chars', async () => { - await expect( - engine.search({ query: 'ab', owner: 'o', repo: 'r', commitSha: 'sha' }), - ).rejects.toMatchObject({ status: 400 }); + await expect(engine.search({ query: 'ab', owner: 'o', repo: 'r', commitSha: 'sha' })).rejects.toMatchObject({ + status: 400, + }); }); it('rejects query longer than 500 chars', async () => { const longQuery = 'x'.repeat(501); - await expect( - engine.search({ query: longQuery, owner: 'o', repo: 'r', commitSha: 'sha' }), - ).rejects.toMatchObject({ status: 400 }); + await expect(engine.search({ query: longQuery, owner: 'o', repo: 'r', commitSha: 'sha' })).rejects.toMatchObject({ + status: 400, + }); }); it('rejects when no index exists for repo', async () => { @@ -52,19 +53,22 @@ describe('createSearchEngine — input validation', () => { }); describe('createSearchEngine — success/cache paths', () => { + const originalEmbeddingProvider = process.env.EMBEDDING_PROVIDER; + afterEach(() => { + if (originalEmbeddingProvider === undefined) delete process.env.EMBEDDING_PROVIDER; + else process.env.EMBEDDING_PROVIDER = originalEmbeddingProvider; + }); beforeEach(() => { clearAllCaches(); - getVectorStore().removeByRepo(repoKey); + getVectorStore().removeByRepo(repoKey()); process.env.AI_PROVIDER = 'mock'; + process.env.EMBEDDING_PROVIDER = 'mock'; delete process.env.OPENAI_API_KEY; delete process.env.GEMINI_API_KEY; }); it('searches indexed chunks and caches response', async () => { - getVectorStore().addChunks([ - chunk('a', new Float32Array([1, 0, 0])), - chunk('b', new Float32Array([0, 1, 0])), - ]); + getVectorStore().addChunks([chunk('a', new Float32Array([1, 0, 0])), chunk('b', new Float32Array([0, 1, 0]))]); const engine = createSearchEngine(); const first = await engine.search({ @@ -105,7 +109,7 @@ describe('createSearchEngine — success/cache paths', () => { query: 'cached query', cached: false, }; - cache.set(`search:owner/repo@sha:${hashContext('cached query')}`, cached); + cache.set(`search:${repoKey()}:${hashContext('cached query')}:10:0.3`, cached); const result = await createSearchEngine().search({ query: 'cached query', @@ -124,7 +128,9 @@ describe('createSearchEngine — success/cache paths', () => { // Mock createEmbeddingProvider to return a provider that throws on embed mock.module('./embedding-provider', () => ({ createEmbeddingProvider: async () => ({ - embed: async () => { throw new Error('embedding service down'); }, + embed: async () => { + throw new Error('embedding service down'); + }, embedBatch: async () => [], tokenCount: (text: string) => text.split(/\s+/).length, }), @@ -139,10 +145,10 @@ describe('createSearchEngine — success/cache paths', () => { query: 'some failing query', owner: 'owner', repo: 'repo', - commitSha: 'sha2', + commitSha: 'sha', }), ).rejects.toMatchObject({ status: 503 }); mock.restore(); }); -}); \ No newline at end of file +}); diff --git a/server/search/search-engine.ts b/server/search/search-engine.ts index 546bacd..264b543 100644 --- a/server/search/search-engine.ts +++ b/server/search/search-engine.ts @@ -1,71 +1,73 @@ import type { SearchEngine, SearchOptions, SearchResponse } from './types'; -import { DEFAULT_TOP_K, DEFAULT_MIN_SCORE, searchCacheKey } from './constants'; +import { DEFAULT_TOP_K, DEFAULT_MIN_SCORE } from './constants'; +import { searchIndexKey } from './index-identity'; import { createEmbeddingProvider } from './embedding-provider'; import { getVectorStore } from './vector-store'; import { cache } from '../cache/lru'; import { hashContext } from '../cache/lru'; import { AppError } from '../utils/errors'; +import { getIndexingPipeline } from './indexing-pipeline'; -export function createSearchEngine(): SearchEngine { +export function createSearchEngine( + deps = { createEmbeddingProvider, getVectorStore }, + resolveAvailable = (key: string) => getIndexingPipeline().available(key), +): SearchEngine { return { async search(options: SearchOptions): Promise { const { query, owner, repo, commitSha } = options; const topK = options.topK ?? DEFAULT_TOP_K; const minScore = options.minScore ?? DEFAULT_MIN_SCORE; - const repoKey = `${owner}/${repo}`; + const requestedKey = searchIndexKey(owner, repo, commitSha, options, options.includePaths, options.excludePaths); // Validate query length if (query.length < 3 || query.length > 500) { - throw new AppError( - 400, - 'Query must be between 3 and 500 characters.', - 'INVALID_QUERY_LENGTH', - ); + throw new AppError(400, 'Query must be between 3 and 500 characters.', 'INVALID_QUERY_LENGTH'); } // Check if index exists - const vectorStore = getVectorStore(); + const available = resolveAvailable(requestedKey); + const vectorStore = deps.getVectorStore().hasIndex(requestedKey) + ? deps.getVectorStore() + : (available?.store ?? deps.getVectorStore()); + const repoKey = vectorStore === available?.store ? available.key : requestedKey; + const coverage = available?.coverage; if (!vectorStore.hasIndex(repoKey)) { - throw new AppError( - 404, - 'No index found. Please index the repository first.', - 'INDEX_NOT_FOUND', - ); + throw new AppError(404, 'No index found. Please index the repository first.', 'INDEX_NOT_FOUND'); } // Check cache const queryHash = hashContext(query); - const cacheKey = searchCacheKey(owner, repo, commitSha, queryHash); - const cached = cache.get(cacheKey); + const cacheKey = `search:${repoKey}${repoKey === requestedKey ? '' : ':' + requestedKey}:${queryHash}:${topK}:${minScore}`; + const cached = coverage?.kind === 'partial' ? undefined : cache.get(cacheKey); if (cached) { - return { ...cached, cached: true }; + return { ...cached, coverage, cached: true }; } // Generate embedding for query - const provider = await createEmbeddingProvider(); + const provider = await deps.createEmbeddingProvider(); let embeddingResult; try { embeddingResult = await provider.embed(query); - } catch { - throw new AppError( - 503, - 'Search temporarily unavailable. Please retry.', - 'EMBEDDING_FAILURE', - true, - ); + } catch (error) { + if (error instanceof AppError) throw error; + throw new AppError(503, 'Search temporarily unavailable. Please retry.', 'EMBEDDING_FAILURE', true); } // Search vector store + if (!vectorStore.hasIndex(repoKey)) { + throw new AppError(404, 'Search index expired. Please rebuild it.', 'INDEX_NOT_FOUND'); + } const results = vectorStore.search(embeddingResult.vector, repoKey, topK, minScore); const response: SearchResponse = { results, query, cached: false, + coverage, }; // Store in cache - cache.set(cacheKey, response); + if (coverage?.kind !== 'partial') cache.set(cacheKey, response); return response; }, diff --git a/server/search/search-eviction.test.ts b/server/search/search-eviction.test.ts new file mode 100644 index 0000000..895d168 --- /dev/null +++ b/server/search/search-eviction.test.ts @@ -0,0 +1,44 @@ +import { test, expect } from 'bun:test'; +import { createSearchEngine } from './search-engine'; +import { createVectorStore } from './vector-store'; +import { searchIndexKey } from './index-identity'; +import type { EmbeddingProvider, IndexedChunk } from './types'; + +test('eviction during embedding rejects without caching an empty result', async () => { + const store = createVectorStore(); + const key = searchIndexKey('eviction', 'test', 'sha'); + const entry: IndexedChunk = { + id: 'a', + vector: new Float32Array([1]), + metadata: { + repoKey: key, + commitSha: 'sha', + filePath: 'a.ts', + language: 'typescript', + content: 'hello', + startLine: 1, + endLine: 1, + chunkIndex: 0, + }, + }; + store.replaceIndex(key, [entry]); + let calls = 0; + const provider: EmbeddingProvider = { + dimensions: 1, + maxTokens: 512, + providerName: 'test', + embedBatch: async () => [], + embed: async () => { + if (++calls === 1) store.removeByRepo(key); + return { vector: new Float32Array([1]), tokenCount: 1 }; + }, + }; + const engine = createSearchEngine({ getVectorStore: () => store, createEmbeddingProvider: async () => provider }); + const options = { owner: 'eviction', repo: 'test', commitSha: 'sha', query: 'hello eviction' }; + await expect(engine.search(options)).rejects.toMatchObject({ code: 'INDEX_NOT_FOUND' }); + store.replaceIndex(key, [entry]); + const result = await engine.search(options); + expect(result.cached).toBe(false); + expect(result.results).toHaveLength(1); + expect(calls).toBe(2); +}); diff --git a/server/search/types.ts b/server/search/types.ts index 7fc7c02..b771765 100644 --- a/server/search/types.ts +++ b/server/search/types.ts @@ -1,4 +1,7 @@ import type { RequestContext } from '../utils/context'; +import type { IndexingStatus, SearchCoverage } from '../../src/types'; + +export type { IndexingStatus }; // ── Embedding ────────────────────────────────────────────────────────── @@ -8,8 +11,8 @@ export interface EmbeddingResult { } export interface EmbeddingProvider { - embed(text: string): Promise; - embedBatch(texts: string[]): Promise; + embed(text: string, signal?: AbortSignal): Promise; + embedBatch(texts: string[], signal?: AbortSignal): Promise; readonly dimensions: number; readonly maxTokens: number; readonly providerName: string; @@ -40,15 +43,11 @@ export interface SearchResult { } export interface VectorStore { + replaceIndex(repoKey: string, chunks: IndexedChunk[]): void; addChunks(chunks: IndexedChunk[]): void; removeByRepo(repoKey: string): void; removeByFile(repoKey: string, filePath: string): void; - search( - queryVector: Float32Array, - repoKey: string, - topK: number, - minScore: number, - ): SearchResult[]; + search(queryVector: Float32Array, repoKey: string, topK: number, minScore: number): SearchResult[]; getChunkCount(repoKey: string): number; hasIndex(repoKey: string): boolean; } @@ -61,37 +60,33 @@ export interface IndexingOptions { branch?: string; commitSha: string; previousSha?: string; // for incremental re-indexing + includePaths?: string[]; + excludePaths?: string[]; } -export type IndexingStatus = - | { state: 'idle' } - | { - state: 'indexing'; - progress: number; - filesProcessed: number; - totalFiles: number; - } - | { state: 'complete'; chunkCount: number; timestamp: number } - | { state: 'failed'; error: string; failedFiles: number }; - export interface IndexingPipeline { startIndexing(options: IndexingOptions, ctx: RequestContext): Promise; getStatus(repoKey: string): IndexingStatus; cancelIndexing(repoKey: string): void; + available(repoKey: string): { store: VectorStore; key: string; coverage: SearchCoverage } | undefined; } // ── Search Engine ────────────────────────────────────────────────────── export interface SearchOptions { + gitHubToken?: string; query: string; owner: string; repo: string; commitSha: string; topK?: number; // default 10 minScore?: number; // default 0.3 + includePaths?: string[]; + excludePaths?: string[]; } export interface SearchResponse { + coverage?: SearchCoverage; results: SearchResult[]; query: string; cached: boolean; @@ -104,11 +99,14 @@ export interface SearchEngine { // ── QA Engine ────────────────────────────────────────────────────────── export interface QAOptions { + gitHubToken?: string; question: string; owner: string; repo: string; commitSha: string; apiKey?: string; // user override key + includePaths?: string[]; + excludePaths?: string[]; } export interface Citation { @@ -118,6 +116,7 @@ export interface Citation { } export interface QAResponse { + coverage?: SearchCoverage; answer: string; // markdown-formatted citations: Citation[]; cached: boolean; @@ -139,7 +138,16 @@ export interface Chunk { } export interface Chunker { - chunkFile(content: string, filePath: string, language: string): Chunk[]; + chunkFile( + content: string, + filePath: string, + language: string, + budget?: { + maxChunks: number; + maxBytes: number; + bytesPerChunk: number; + }, + ): Chunk[]; } export type { RequestContext }; diff --git a/server/search/vector-store.test.ts b/server/search/vector-store.test.ts index 8c8ea67..57e5598 100644 --- a/server/search/vector-store.test.ts +++ b/server/search/vector-store.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach } from 'bun:test'; import { createVectorStore } from './vector-store'; import type { IndexedChunk } from './types'; +import { SEARCH_LIMITS } from './limits'; function makeVector(size: number, val: number): Float32Array { const v = new Float32Array(size).fill(0); @@ -9,12 +10,7 @@ function makeVector(size: number, val: number): Float32Array { return v.map((x) => x / mag); } -function makeChunk( - id: string, - repoKey: string, - filePath: string, - vector: Float32Array, -): IndexedChunk { +function makeChunk(id: string, repoKey: string, filePath: string, vector: Float32Array): IndexedChunk { return { id, vector, @@ -32,6 +28,33 @@ function makeChunk( } describe('createVectorStore', () => { + it('evicts least recently used indices and expires snapshots', () => { + let time = 0; + const bounded = createVectorStore({ ...SEARCH_LIMITS, indices: 2, ttlMs: 10 }, () => time); + const add = (key: string) => bounded.addChunks([makeChunk(key, key, 'a.ts', new Float32Array([1]))]); + add('one'); + add('two'); + bounded.hasIndex('one'); + add('three'); + expect(bounded.hasIndex('two')).toBe(false); + expect(bounded.hasIndex('one')).toBe(true); + time = 11; + expect(bounded.hasIndex('one')).toBe(false); + }); + it('rejects oversized replacements without destroying the old index', () => { + const bounded = createVectorStore({ ...SEARCH_LIMITS, chunks: 1 }); + const one = makeChunk('one', 'repo', 'a.ts', new Float32Array([1])); + bounded.replaceIndex('repo', [one]); + expect(() => bounded.replaceIndex('repo', [one, one])).toThrow(); + expect(bounded.getChunkCount('repo')).toBe(1); + }); + it('enforces the global byte budget', () => { + const bounded = createVectorStore({ ...SEARCH_LIMITS, totalBytes: 1000 }); + bounded.addChunks([makeChunk('one', 'one', 'a.ts', new Float32Array([1]))]); + bounded.addChunks([makeChunk('two', 'two', 'a.ts', new Float32Array([1]))]); + expect(bounded.hasIndex('one')).toBe(false); + expect(bounded.hasIndex('two')).toBe(true); + }); let store: ReturnType; beforeEach(() => { @@ -83,10 +106,7 @@ describe('createVectorStore', () => { describe('removeByFile', () => { it('removes only chunks for the specified file', () => { const vec = makeVector(4, 1); - store.addChunks([ - makeChunk('c1', 'owner/repo', 'a.ts', vec), - makeChunk('c2', 'owner/repo', 'b.ts', vec), - ]); + store.addChunks([makeChunk('c1', 'owner/repo', 'a.ts', vec), makeChunk('c2', 'owner/repo', 'b.ts', vec)]); store.removeByFile('owner/repo', 'a.ts'); expect(store.getChunkCount('owner/repo')).toBe(1); }); @@ -147,9 +167,7 @@ describe('createVectorStore', () => { const dim = 4; const vec = new Float32Array(dim); vec[0] = 1; - const chunks = Array.from({ length: 10 }, (_, i) => - makeChunk(`c${i}`, 'owner/repo', `${i}.ts`, vec), - ); + const chunks = Array.from({ length: 10 }, (_, i) => makeChunk(`c${i}`, 'owner/repo', `${i}.ts`, vec)); store.addChunks(chunks); const results = store.search(vec, 'owner/repo', 3, 0); expect(results).toHaveLength(3); diff --git a/server/search/vector-store.ts b/server/search/vector-store.ts index f8eff5a..220e272 100644 --- a/server/search/vector-store.ts +++ b/server/search/vector-store.ts @@ -1,97 +1,84 @@ import type { IndexedChunk, SearchResult, VectorStore } from './types'; +import { SEARCH_LIMITS } from './limits'; +import { AppError } from '../utils/errors'; -interface RepoIndex { - chunks: IndexedChunk[]; - commitSha: string; - createdAt: number; +export function chunkBytes(chunk: IndexedChunk): number { + return chunk.vector.byteLength + 2 * (chunk.id.length + JSON.stringify(chunk.metadata).length) + 256; } -export function createVectorStore(): VectorStore { - const indices = new Map(); - +export function createVectorStore(limits = SEARCH_LIMITS, now = Date.now): VectorStore { + const indices = new Map(); + const prune = () => { + for (const [key, index] of indices) if (index.expires <= now()) indices.delete(key); + }; + const get = (key: string) => { + prune(); + const index = indices.get(key); + if (index) { + indices.delete(key); + indices.set(key, index); + } + return index; + }; + const replaceIndex = (key: string, chunks: IndexedChunk[]) => { + const bytes = chunks.reduce((sum, chunk) => sum + chunkBytes(chunk), 0); + if (chunks.length > limits.chunks || bytes > limits.indexBytes || bytes > limits.totalBytes) + throw new AppError(413, 'Search index exceeds the repository size limit.', 'INDEX_TOO_LARGE'); + if (chunks.some((chunk) => chunk.metadata.repoKey !== key)) + throw new AppError(400, 'Index identity mismatch.', 'INVALID_INDEX'); + prune(); + // Validate before touching the published index. + indices.delete(key); + let total = [...indices.values()].reduce((sum, index) => sum + index.bytes, 0); + while (indices.size && (total + bytes > limits.totalBytes || indices.size >= limits.indices)) { + const oldest = indices.keys().next().value!; + total -= indices.get(oldest)!.bytes; + indices.delete(oldest); + } + indices.set(key, { chunks: [...chunks], bytes, expires: now() + limits.ttlMs }); + }; return { - addChunks(chunks: IndexedChunk[]): void { - if (chunks.length === 0) return; - const repoKey = chunks[0].metadata.repoKey; - const existing = indices.get(repoKey); - - if (existing) { - existing.chunks.push(...chunks); - } else { - indices.set(repoKey, { - chunks: [...chunks], - commitSha: chunks[0].metadata.commitSha, - createdAt: Date.now(), - }); - } + replaceIndex, + addChunks(chunks) { + if (!chunks.length) return; + const key = chunks[0].metadata.repoKey; + replaceIndex(key, [...(get(key)?.chunks ?? []), ...chunks]); }, - - removeByRepo(repoKey: string): void { - indices.delete(repoKey); + removeByRepo(key) { + indices.delete(key); }, - - removeByFile(repoKey: string, filePath: string): void { - const index = indices.get(repoKey); + removeByFile(key, path) { + const index = get(key); if (!index) return; - index.chunks = index.chunks.filter((c) => c.metadata.filePath !== filePath); - if (index.chunks.length === 0) { - indices.delete(repoKey); - } + const chunks = index.chunks.filter((chunk) => chunk.metadata.filePath !== path); + if (!chunks.length) indices.delete(key); + else replaceIndex(key, chunks); }, - - search( - queryVector: Float32Array, - repoKey: string, - topK: number, - minScore: number, - ): SearchResult[] { - const index = indices.get(repoKey); - if (!index || index.chunks.length === 0) return []; - + search(vector, key, topK, minScore): SearchResult[] { + const index = get(key); + if (!index) return []; const scored: SearchResult[] = []; - for (const chunk of index.chunks) { - const score = cosineSimilarity(queryVector, chunk.vector); - if (score >= minScore) { - scored.push({ chunk: chunk.metadata, score }); - } + const score = cosineSimilarity(vector, chunk.vector); + if (score >= minScore) scored.push({ chunk: chunk.metadata, score }); } - - // Sort descending by score - scored.sort((a, b) => b.score - a.score); - - return scored.slice(0, topK); + return scored.sort((a, b) => b.score - a.score).slice(0, topK); }, - - getChunkCount(repoKey: string): number { - return indices.get(repoKey)?.chunks.length ?? 0; + getChunkCount(key) { + return get(key)?.chunks.length ?? 0; }, - - hasIndex(repoKey: string): boolean { - return indices.has(repoKey); + hasIndex(key) { + return !!get(key); }, }; } - -/** Compute cosine similarity between two unit vectors. */ function cosineSimilarity(a: Float32Array, b: Float32Array): number { if (a.length !== b.length) return 0; let dot = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - } - // Vectors are pre-normalized, so dot product = cosine similarity - // Clamp to [0, 1] range (negative similarity → 0) + for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; return Math.max(0, Math.min(1, dot)); } - -// ── Singleton ────────────────────────────────────────────────────────── - let globalStore: VectorStore | null = null; - export function getVectorStore(): VectorStore { - if (!globalStore) { - globalStore = createVectorStore(); - } - return globalStore; + return (globalStore ??= createVectorStore()); } diff --git a/server/services/analyze-repo.test.ts b/server/services/analyze-repo.test.ts index d47a3c3..b1eac7a 100644 --- a/server/services/analyze-repo.test.ts +++ b/server/services/analyze-repo.test.ts @@ -1,87 +1,130 @@ -import { afterEach, describe, expect, it, mock, beforeEach } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; import { analyzeRepository } from './analyze-repo'; +import { enrichRepository } from './repo-enrichment'; import { clearAllCaches } from '../cache/lru'; -import type { GraphBuildInput } from '../graph/graph-builder'; import type { RequestContext } from '../utils/context'; -const mockRepoInfo = { - fullName: 'test-owner/test-repo', +const mockRepoMeta = { + full_name: 'test-owner/test-repo', + html_url: 'https://github.com/test-owner/test-repo', description: 'test-desc', - stars: 10, - forks: 5, + stargazers_count: 10, + forks_count: 5, language: 'TypeScript', - defaultBranch: 'main', - sha: 'test-sha', - url: 'https://github.com/test-owner/test-repo', + default_branch: 'main', topics: [], - license: 'MIT', - createdAt: '2026-01-01T00:00:00Z', - updatedAt: '2026-01-02T00:00:00Z', + license: { spdx_id: 'MIT' }, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-02T00:00:00Z', }; const workspaceFileContents = { - 'package.json': JSON.stringify({ name: '@repo/root', workspaces: ['packages/*'] }), - 'packages/a/package.json': JSON.stringify({ name: '@repo/a', dependencies: { '@repo/b': 'workspace:*' } }), + 'package.json': JSON.stringify({ + name: '@repo/root', + workspaces: ['packages/*'], + }), + 'packages/a/package.json': JSON.stringify({ + name: '@repo/a', + dependencies: { '@repo/b': 'workspace:*' }, + }), 'packages/b/package.json': JSON.stringify({ name: '@repo/b' }), 'src/main.ts': 'export const value = 1', }; const plainFileContents = { - 'package.json': JSON.stringify({ name: 'plain-app', description: 'no workspaces here', dependencies: { react: '^19' } }), + 'package.json': JSON.stringify({ + name: 'plain-app', + description: 'no workspaces here', + dependencies: { react: '^19' }, + }), 'src/main.ts': 'export const value = 1', }; const excludedWorkspaceFileContents = { - 'package.json': JSON.stringify({ name: '@repo/root', packageManager: 'yarn@4.0.0', workspaces: ['packages/*', '!packages/excluded'] }), + 'package.json': JSON.stringify({ + name: '@repo/root', + packageManager: 'yarn@4.0.0', + workspaces: ['packages/*', '!packages/excluded'], + }), 'packages/a/package.json': JSON.stringify({ name: '@repo/a' }), 'packages/excluded/package.json': JSON.stringify({ name: '@repo/excluded' }), 'src/main.ts': 'export const value = 1', }; -let activeFileContents: Record = workspaceFileContents; -const buildGraphMock = mock((input: GraphBuildInput) => ({ - nodes: input.workspacePackages?.map((pkg) => ({ - id: `package:${pkg.rootPath}`, - type: 'package' as const, - position: { x: 0, y: 0 }, - data: { label: pkg.name ?? pkg.rootPath }, - })) ?? [], - edges: input.scopedDependencies - ?.filter((dep) => dep.name === '@repo/b') - .map((dep) => ({ - id: `e:${dep.manifestPath}->${dep.name}`, - source: 'package:packages/a', - target: 'package:packages/b', - type: 'depends_on' as const, - })) ?? [], - layout: 'dagre' as const, -})); +// Mutable stub behavior, reset in beforeEach. The stub octokit below serves +// every GitHub endpoint the analysis pipeline touches, so no mock.module() +// on shared internal modules is needed (see AGENTS.md testing gotchas). +const stubState: { files: Record; denyAccess: boolean; calls: Record } = { + files: workspaceFileContents, + denyAccess: false, + calls: {}, +}; + +function record(name: string): void { + stubState.calls[name] = (stubState.calls[name] ?? 0) + 1; +} -const originalFetch = globalThis.fetch; const stubOctokit = { repos: { - listCommits: async () => ({ - data: [{ commit: { author: { name: 'test-user', date: '2026-01-01T00:00:00Z' } } }], - headers: {}, - }), + get: async () => { + record('repos.get'); + return { data: mockRepoMeta }; + }, + getCommit: async () => { + record('repos.getCommit'); + if (stubState.denyAccess) throw new Error('Repository access denied'); + return { data: { sha: 'test-sha' } }; + }, + listContributors: async () => { + record('repos.listContributors'); + return { data: [] }; + }, + listCommits: async () => { + record('repos.listCommits'); + return { data: [], headers: {} }; + }, + getContent: async ({ path }: { path: string }) => { + record('repos.getContent'); + const content = stubState.files[path]; + if (content === undefined) throw { status: 404 }; + return { data: { type: 'file', content: Buffer.from(content).toString('base64') } }; + }, + }, + git: { + getTree: async () => { + record('git.getTree'); + return { + data: { + tree: Object.keys(stubState.files).map((path) => ({ path, type: 'blob', sha: `sha-${path}` })), + }, + }; + }, }, }; -const mockCtx = { octokit: stubOctokit } as RequestContext; +const mockCtx = { octokit: stubOctokit } as unknown as RequestContext; + +const originalFetch = globalThis.fetch; const fetchMock = mock(async (input: RequestInfo | URL) => { const packageName = decodeURIComponent(new URL(String(input)).pathname.slice(1)); if (packageName === '@repo/b') { - return new Response(JSON.stringify({ - 'dist-tags': { latest: 'workspace:*' }, - license: 'MIT', - }), { status: 200 }); + return new Response( + JSON.stringify({ + 'dist-tags': { latest: 'workspace:*' }, + license: 'MIT', + }), + { status: 200 }, + ); } if (packageName === 'react') { - return new Response(JSON.stringify({ - 'dist-tags': { latest: '19.1.0' }, - license: 'MIT', - }), { status: 200 }); + return new Response( + JSON.stringify({ + 'dist-tags': { latest: '19.1.0' }, + license: 'MIT', + }), + { status: 200 }, + ); } throw new TypeError(`unexpected fetch for ${packageName}`); @@ -89,99 +132,105 @@ const fetchMock = mock(async (input: RequestInfo | URL) => { describe('services/analyze-repo', () => { beforeEach(() => { - mock.module('../github/fetch-tree', () => ({ - fetchRepoInfo: async () => mockRepoInfo, - fetchFlatTree: async () => ({ - items: Object.keys(activeFileContents).map((path) => ({ path, type: 'file' })), - truncated: false, - }), - fetchContributors: async () => [], - fetchTimeline: async () => [], - fetchTotalCommits: async () => 42, - buildTreeFromPaths: (items: readonly { readonly path: string; readonly type: string }[]) => items, - findManifestPaths: () => ['package.json', 'packages/a/package.json', 'packages/b/package.json'], - fetchFileContents: async () => activeFileContents, - })); - - mock.module('../github/parse-url', () => ({ - parseGitHubUrl: (input: string) => { - if (input.includes('invalid')) return null; - return { owner: 'test-owner', repo: 'test-repo' }; - }, - })); - - mock.module('../parser/file-classifier', () => ({ - annotateTree: (tree: readonly { readonly path: string; readonly type: string }[]) => tree, - findImportantFiles: () => ['src/main.ts'], - })); - - mock.module('../graph/graph-builder', () => ({ - buildGraph: buildGraphMock, - })); - + stubState.files = workspaceFileContents; + stubState.denyAccess = false; + stubState.calls = {}; clearAllCaches(); - activeFileContents = workspaceFileContents; - buildGraphMock.mockClear(); fetchMock.mockClear(); globalThis.fetch = fetchMock as typeof fetch; }); afterEach(() => { globalThis.fetch = originalFetch; - mock.restore(); }); it('throws error for invalid repo url', () => { expect(analyzeRepository('invalid-url')).rejects.toThrow('Invalid GitHub repository URL'); }); + it('isolates analysis and SHA aliases by credentials and rechecks access on cache hits', async () => { + const input = { owner: 'test-owner', repo: 'test-repo', branch: 'main' }; + const first = { ...mockCtx, gitHubToken: 'first-credential' }; + const second = { ...mockCtx, gitHubToken: 'second-credential' }; + const original = await analyzeRepository(input, first); + expect(await analyzeRepository({ ...input, branch: 'test-sha' }, first)).toBe(original); + const getCommitCalls = stubState.calls['repos.getCommit'] ?? 0; + const recomputed = await analyzeRepository({ ...input, branch: 'test-sha' }, second); + expect(recomputed).not.toBe(original); + expect(stubState.calls['repos.getCommit']).toBeGreaterThan(getCommitCalls); + stubState.denyAccess = true; + await expect(analyzeRepository(input, first)).rejects.toThrow('Repository access denied'); + }); + it('runs the full repository analysis pipeline and caches the result', async () => { const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); expect(analysis.meta.fullName).toBe('test-owner/test-repo'); - expect(analysis.importantFiles).toEqual(['src/main.ts']); + expect(analysis.importantFiles).toEqual([ + 'package.json', + 'src/main.ts', + 'packages/a/package.json', + 'packages/b/package.json', + ]); expect(analysis.treeTruncated).toBe(false); expect(analysis.workspacePackages?.map((pkg) => pkg.rootPath)).toEqual(['', 'packages/a', 'packages/b']); expect(analysis.dependencies).toEqual([ - { name: '@repo/b', version: 'workspace:*', type: 'prod', ecosystem: 'npm' }, + { + name: '@repo/b', + version: 'workspace:*', + type: 'prod', + ecosystem: 'npm', + }, + ]); + expect(analysis.scopedDependencies).toEqual([ + expect.objectContaining({ + manifestPath: 'packages/a/package.json', + name: '@repo/b', + }), ]); expect(analysis.graph.nodes.some((node) => node.id === 'package:packages/a')).toBe(true); - expect(analysis.graph.edges.some((edge) => edge.source === 'package:packages/a' && edge.target === 'package:packages/b')).toBe(true); - expect(analysis.dependencyHealth).toEqual(expect.objectContaining({ - summary: expect.objectContaining({ total: 1, current: 1, unsupported: 0 }), - })); - expect(buildGraphMock).toHaveBeenCalledWith(expect.objectContaining({ - workspacePackages: expect.arrayContaining([ - expect.objectContaining({ rootPath: 'packages/a', name: '@repo/a' }), - expect.objectContaining({ rootPath: 'packages/b', name: '@repo/b' }), - ]), - scopedDependencies: expect.arrayContaining([ - expect.objectContaining({ manifestPath: 'packages/a/package.json', name: '@repo/b' }), - ]), - })); + expect(analysis.graph.nodes.some((node) => node.id === 'package:packages/b')).toBe(true); + expect( + analysis.graph.edges.some( + (edge) => edge.source === 'package:packages/a' && edge.target === 'file:packages/a/package.json', + ), + ).toBe(true); + expect(analysis.dependencyHealth).toBeUndefined(); + expect(await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx)).toEqual( + expect.objectContaining({ + summary: expect.objectContaining({ + total: 1, + current: 1, + unsupported: 0, + }), + }), + ); expect(fetchMock).toHaveBeenCalledTimes(0); - // Call again, should return cached - const cachedAnalysis = await analyzeRepository('https://github.com/test-owner/test-repo'); + // Call again with the same credentials, should return cached. + // A context is required: the info fetch precedes the cache lookup. + const cachedAnalysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); expect(cachedAnalysis).toBe(analysis); expect(fetchMock).toHaveBeenCalledTimes(0); }); it('keeps no-workspace repository analysis backward-compatible', async () => { - activeFileContents = plainFileContents; + stubState.files = plainFileContents; const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); - expect(analysis.dependencies).toEqual([ - { name: 'react', version: '^19', type: 'prod', ecosystem: 'npm' }, - ]); + expect(analysis.dependencies).toEqual([{ name: 'react', version: '^19', type: 'prod', ecosystem: 'npm' }]); + expect(fetchMock).toHaveBeenCalledTimes(0); + await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx); expect(fetchMock).toHaveBeenCalledTimes(1); expect(String(fetchMock.mock.calls[0]?.[0])).toContain('/react'); expect(analysis.workspacePackages).toEqual([]); - expect(analysis.graph.edges.some((edge) => edge.source.startsWith('package:') && edge.target.startsWith('package:'))).toBe(false); + expect( + analysis.graph.edges.some((edge) => edge.source.startsWith('package:') && edge.target.startsWith('package:')), + ).toBe(false); }); it('skips registry metadata for non-npm dependencies', async () => { - activeFileContents = { + stubState.files = { 'go.mod': 'module example.com/app\n\nrequire github.com/gin-gonic/gin v1.9.1', 'src/main.go': 'package main', }; @@ -189,16 +238,30 @@ describe('services/analyze-repo', () => { const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); expect(fetchMock).toHaveBeenCalledTimes(0); - expect(analysis.dependencyHealth).toEqual(expect.objectContaining({ - summary: expect.objectContaining({ total: 1, unsupported: 1, current: 0, outdated: 0, unknown: 0, errors: 0 }), - items: expect.arrayContaining([ - expect.objectContaining({ ecosystem: 'go', name: 'github.com/gin-gonic/gin', state: 'unsupported' }), - ]), - })); + expect(analysis.dependencyHealth).toBeUndefined(); + expect(await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx)).toEqual( + expect.objectContaining({ + summary: expect.objectContaining({ + total: 1, + unsupported: 1, + current: 0, + outdated: 0, + unknown: 0, + errors: 0, + }), + items: expect.arrayContaining([ + expect.objectContaining({ + ecosystem: 'go', + name: 'github.com/gin-gonic/gin', + state: 'unsupported', + }), + ]), + }), + ); }); it('honors negated workspace globs and explicit yarn packageManager', async () => { - activeFileContents = excludedWorkspaceFileContents; + stubState.files = excludedWorkspaceFileContents; const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); @@ -207,4 +270,29 @@ describe('services/analyze-repo', () => { expect.objectContaining({ rootPath: 'packages/a', manager: 'yarn' }), ]); }); + + it('serves health from a dependency-focused path without contributor, timeline, or commit endpoints', async () => { + const report = await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx); + expect(report.summary).toEqual(expect.objectContaining({ total: 1, current: 1 })); + expect(stubState.calls['git.getTree'] ?? 0).toBeGreaterThan(0); + expect(stubState.calls['repos.listContributors'] ?? 0).toBe(0); + expect(stubState.calls['repos.listCommits'] ?? 0).toBe(0); + }); + + it('fetches registry metadata for every npm dependency instead of truncating at 100', async () => { + const many = Object.fromEntries(Array.from({ length: 120 }, (_, i) => [`pkg-${i}`, '^1.0.0'])); + stubState.files = { + 'package.json': JSON.stringify({ name: 'big-app', dependencies: many }), + }; + const seen = new Set(); + globalThis.fetch = (async (input: RequestInfo | URL) => { + seen.add(decodeURIComponent(new URL(String(input)).pathname.slice(1))); + return new Response(JSON.stringify({ 'dist-tags': { latest: '1.0.0' }, license: 'MIT' }), { + status: 200, + }); + }) as typeof fetch; + const report = await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx); + expect(report.summary.total).toBe(120); + expect(seen.size).toBe(120); + }); }); diff --git a/server/services/analyze-repo.ts b/server/services/analyze-repo.ts index b6a8b05..91a4231 100644 --- a/server/services/analyze-repo.ts +++ b/server/services/analyze-repo.ts @@ -11,12 +11,13 @@ import { } from '../github/fetch-tree'; import { parseGitHubUrl } from '../github/parse-url'; import { buildGraph } from '../graph/graph-builder'; -import { analyzeDependencies, analyzeManifestDependencies, analyzeWorkspacePackages } from '../parser/dependency-analyzer'; +import { + analyzeDependencies, + analyzeManifestDependencies, + analyzeWorkspacePackages, +} from '../parser/dependency-analyzer'; import { annotateTree, findImportantFiles } from '../parser/file-classifier'; import { analyzeAllFileComplexity } from '../parser/complexity-analyzer'; -import { fetchRepoChurn } from './churn-service'; -import { buildDependencyHealthReport } from './dependency-health'; -import { fetchNpmDependencyMetadataBatch } from './npm-registry'; import type { RepoAnalysis } from '../../src/types'; import type { RequestContext } from '../utils/context'; @@ -24,10 +25,7 @@ export async function analyzeRepository( input: string | { owner: string; repo: string; branch?: string }, tokenOrCtx?: string | RequestContext, ): Promise { - const parsed = - typeof input === 'string' - ? parseGitHubUrl(input) - : input; + const parsed = typeof input === 'string' ? parseGitHubUrl(input) : input; if (!parsed) { throw new Error('Invalid GitHub repository URL'); @@ -36,7 +34,8 @@ export async function analyzeRepository( const { owner, repo } = parsed; const branch = typeof parsed === 'object' && 'branch' in parsed ? (parsed as { branch?: string }).branch : undefined; const info = await fetchRepoInfo(owner, repo, branch, tokenOrCtx); - const cacheKey = analyzeCacheKey(owner, repo, info.sha, branch); + const token = typeof tokenOrCtx === 'string' ? tokenOrCtx : tokenOrCtx?.gitHubToken; + const cacheKey = analyzeCacheKey(owner, repo, info.sha, branch, token); const cached = cache.get(cacheKey); if (cached) return cached; @@ -44,25 +43,22 @@ export async function analyzeRepository( const [{ items, truncated, totalFiles }, contributors, timeline, totalCommits] = await Promise.all([ fetchFlatTree(owner, repo, info.sha, tokenOrCtx), fetchContributors(owner, repo, tokenOrCtx), - fetchTimeline(owner, repo, branch, tokenOrCtx), - fetchTotalCommits(owner, repo, branch, tokenOrCtx), + fetchTimeline(owner, repo, info.sha, tokenOrCtx), + fetchTotalCommits(owner, repo, info.sha, tokenOrCtx), ]); const tree = annotateTree(buildTreeFromPaths(items)); const manifestPaths = findManifestPaths(items); const importantFiles = findImportantFiles(items.map((i) => i.path)); - + // Extract up to 20 important code files to parse internal imports const sourceExtensions = ['.ts', '.tsx', '.js', '.jsx', '.py', '.go']; - const importantSourceFiles = importantFiles.filter((path) => - sourceExtensions.some((ext) => path.endsWith(ext)) - ); + const importantSourceFiles = importantFiles.filter((path) => sourceExtensions.some((ext) => path.endsWith(ext))); - // All source files from the tree — used for churn (API endpoint: listCommits) - // and to expand content fetching for complexity coverage + // Expand content fetching for complexity coverage. const allSourceFilePaths = items - .filter(i => i.type === 'blob' && sourceExtensions.some(ext => i.path.endsWith(ext))) - .map(i => i.path); + .filter((i) => i.type === 'blob' && sourceExtensions.some((ext) => i.path.endsWith(ext))) + .map((i) => i.path); const pathsToFetch = Array.from(new Set([...manifestPaths, ...importantSourceFiles])); @@ -77,34 +73,17 @@ export async function analyzeRepository( // Compute complexity from file content (synchronous, no API calls) const complexityData = analyzeAllFileComplexity(fileContents); - // For churn: use all source file paths from the tree (not just important ones) - // so the overlay covers all rendered graph file nodes. Cap at 200 to - // avoid excessive API calls on large repos. - const churnFetchPaths = allSourceFilePaths.length > 0 - ? allSourceFilePaths.slice(0, 200) - : pathsToFetch; - - // Fetch churn (async API calls) and npm metadata in parallel - const [npmDependencyMetadata, churnData] = await Promise.all([ - fetchNpmDependencyMetadataBatch( - dependencies.filter((dependency) => dependency.ecosystem === 'npm'), - ), - fetchRepoChurn(owner, repo, churnFetchPaths, branch, tokenOrCtx), - ]); - - const graph = await buildGraph({ - owner, - repo, - tree, - dependencies, - contributors, - fileContents, - workspacePackages, - scopedDependencies, - churnData, - complexityData, - }); - const dependencyHealth = buildDependencyHealthReport(dependencies, scopedDependencies, npmDependencyMetadata); + const graph = await buildGraph({ + owner, + repo, + tree, + dependencies, + contributors, + fileContents, + workspacePackages, + scopedDependencies, + complexityData, + }); const analysis: RepoAnalysis = { meta: { @@ -127,15 +106,16 @@ export async function analyzeRepository( treeTruncated: truncated, dependencies, workspacePackages, + scopedDependencies, graph, contributors, timeline, importantFiles, totalFiles, totalCommits, - dependencyHealth, }; cache.set(cacheKey, analysis); + cache.set(analyzeCacheKey(owner, repo, info.sha, info.sha, token), analysis); return analysis; } diff --git a/server/services/churn-service.test.ts b/server/services/churn-service.test.ts index d416d9a..649baac 100644 --- a/server/services/churn-service.test.ts +++ b/server/services/churn-service.test.ts @@ -1,107 +1,220 @@ -import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { Octokit } from '@octokit/rest'; import { fetchRepoChurn } from './churn-service'; import { clearAllCaches } from '../cache/lru'; -describe('churn-service', () => { - beforeEach(() => { - clearAllCaches(); +test('successful empty histories count as checked and batches resume without repeating files', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return Response.json([]); + }, + }, }); - - afterEach(() => { - mock.restore(); + const paths = Array.from({ length: 10 }, (_, i) => `${i}.ts`); + const first = await fetchRepoChurn('owner', 'repo', paths, 'sha', { + octokit, }); - - it('handles mock repos via fetchMockChurn', async () => { - const res = await fetchRepoChurn('mock', 'mock-repo', ['src/index.ts', 'src/app.ts']); - expect(res['src/index.ts']).toBeDefined(); - expect(res['src/index.ts'].commitCount).toBeGreaterThanOrEqual(1); - expect(res['src/app.ts'].churnScore).toBeGreaterThanOrEqual(0); + expect(first.checked).toBe(7); + expect(first.complete).toBe(false); + expect(first.files['0.ts'].commitCount).toBe(0); + expect(first.files['0.ts'].lastModified).toBeUndefined(); + const second = await fetchRepoChurn('owner', 'repo', paths, 'sha', { + octokit, }); + expect(second.checked).toBe(10); + expect(second.complete).toBe(true); + expect(calls).toBe(10); + await fetchRepoChurn('owner', 'repo', paths, 'sha', { octokit }); + expect(calls).toBe(10); +}); - it('fetches churn data from octokit, caches result, parses link header and author info', async () => { - const listCommitsMock = mock(async ({ path }: { path: string }) => { - if (path === 'err.ts') { - throw new Error('Github API rate limited'); - } - if (path === 'empty.ts') { - return { data: [], headers: {} }; - } - return { - data: [ - { - commit: { - author: { name: 'Alice', date: '2025-01-01T00:00:00Z' }, - }, - author: { login: 'Alice' }, - }, - { - commit: { - author: { name: 'Bob', date: '2024-12-01T00:00:00Z' }, - }, - author: { login: 'Bob' }, - }, - ], - headers: { - link: '; rel="last"', - }, - }; - }); - - const octokitMock = { - repos: { - listCommits: listCommitsMock, +test('partial results survive failure and cooldown prevents immediate retry', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return calls === 2 + ? Response.json({ message: 'limited' }, { status: 429, headers: { 'retry-after': '120' } }) + : Response.json([]); }, - }; - - const consoleErrorSpy = spyOn(console, 'error').mockImplementation(() => {}); - - const res = await fetchRepoChurn( - 'real-owner', - 'real-repo', - ['file1.ts', 'empty.ts', 'err.ts'], - 'main', - { octokit: octokitMock } as any, - ); + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const paths = ['a.ts', 'b.ts', 'c.ts']; + const result = await fetchRepoChurn('owner', 'repo', paths, 'sha', { + octokit, + }); + expect(result.checked).toBe(2); + expect(result.issue).toBe('rate-limit'); + expect(result.files['b.ts']).toBeUndefined(); + expect(result.retryAt).toBeGreaterThan(Date.now() + 110000); + await fetchRepoChurn('owner', 'repo', paths, 'sha', { octokit }); + expect(calls).toBe(3); + const other = await fetchRepoChurn('owner', 'repo', paths, 'other-sha', { + octokit, + }); + expect(other.complete).toBe(true); + expect(calls).toBe(6); +}); - expect(res['file1.ts']).toBeDefined(); - expect(res['file1.ts'].commitCount).toBe(300); // 3 * 100 from Link header - expect(res['file1.ts'].authorCount).toBe(2); - expect(res['file1.ts'].churnScore).toBe(1); // max commits +test('concurrent cold requests share in-flight work', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + await new Promise((resolve) => setTimeout(resolve, 5)); + return Response.json([]); + }, + }, + }); + const run = () => fetchRepoChurn('owner', 'concurrent', ['a.ts', 'b.ts'], 'sha', { octokit }); + const results = await Promise.all([run(), run()]); + expect(calls).toBe(2); + expect(results.map((result) => result.checked)).toEqual([2, 2]); +}); - expect(res['empty.ts']).toBeUndefined(); - expect(res['err.ts']).toBeUndefined(); +test('credential-specific failures do not block another caller', async () => { + clearAllCaches(); + const failed = new Octokit({ + request: { + fetch: async () => Response.json({ message: 'Forbidden' }, { status: 403 }), + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const good = new Octokit({ + request: { fetch: async () => Response.json([]) }, + }); + expect( + ( + await fetchRepoChurn('owner', 'scope', ['a.ts'], 'sha', { + octokit: failed, + gitHubToken: 'test-limited', + }) + ).issue, + ).toBe('access'); + expect( + ( + await fetchRepoChurn('owner', 'scope', ['a.ts'], 'sha', { + octokit: good, + gitHubToken: 'test-valid', + }) + ).complete, + ).toBe(true); +}); - // Verify cache hit - const cachedRes = await fetchRepoChurn( - 'real-owner', - 'real-repo', - ['file1.ts'], - 'main', - { octokit: octokitMock } as any, - ); - expect(cachedRes).toEqual(res); +test('explicit continuation resumes after losing server cache', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return Response.json([]); + }, + }, + }); + const paths = Array.from({ length: 10 }, (_, i) => `${i}.ts`); + const first = await fetchRepoChurn('owner', 'cold', paths, 'sha', { + octokit, + }); + clearAllCaches(); + const second = await fetchRepoChurn('owner', 'cold', paths, 'sha', { octokit }, 90, { + completed: Object.keys(first.files), + pending: first.remaining, + }); + expect(second.complete).toBe(true); + expect(calls).toBe(10); +}); - // Test 403/429 silent error handling (console.error still suppressed so the - // whole-batch-failure warning does not leak into the test output) - const rateLimitMock = { - repos: { - listCommits: mock(async () => { - const error: any = new Error('Rate limit'); - error.status = 403; - throw error; - }), +test('failed first batch does not prevent checking later files', async () => { clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return calls <= 7 ? Response.json({ message: 'Unavailable' }, { status: 500 }) : Response.json([]); }, - }; - const rateLimitRes = await fetchRepoChurn( - 'rate-limit-owner', - 'rate-limit-repo', - ['blocked.ts'], - 'main', - { octokit: rateLimitMock } as any, - ); - expect(rateLimitRes).toEqual({}); + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const paths = Array.from({ length: 10 }, (_, i) => `${i}.ts`); + await fetchRepoChurn('owner', 'later', paths, 'sha', { octokit }); + const second = await fetchRepoChurn('owner', 'later', paths, 'sha', { + octokit, + }); + expect(second.checked).toBe(3); + expect(second.files['7.ts']).toBeDefined(); +}); - consoleErrorSpy.mockRestore(); +test('stale continuation from another credential is refetched, not trusted', async () => { + clearAllCaches(); + const octokit = new Octokit({ + request: { fetch: async () => Response.json([]) }, }); + const paths = ['a.ts', 'b.ts', 'c.ts']; + const first = await fetchRepoChurn('owner', 'rescope', paths, 'sha', { + octokit, + gitHubToken: 'token-a', + }); + expect(first.complete).toBe(true); + expect(first.scope).toBeDefined(); + clearAllCaches(); + let calls = 0; + const counting = new Octokit({ + request: { + fetch: async () => { + calls++; + return Response.json([]); + }, + }, + }); + const second = await fetchRepoChurn('owner', 'rescope', paths, 'sha', { + octokit: counting, + gitHubToken: 'token-b', + }, 90, { completed: Object.keys(first.files), scope: first.scope }); + expect(calls).toBe(3); + expect(second.complete).toBe(true); + expect(Object.keys(second.files)).toHaveLength(3); }); +test('login and git author name count as one contributor', async () => { + clearAllCaches(); + const octokit = new Octokit({ + request: { + fetch: async () => + Response.json([ + { + author: { login: 'octocat' }, + commit: { author: { name: 'Mona Octocat', date: '2026-01-01T00:00:00Z' } }, + }, + ]), + }, + }); + const result = await fetchRepoChurn('owner', 'authors', ['a.ts'], 'sha', { octokit }); + expect(result.files['a.ts'].authorCount).toBe(1); +}); +test('per-file 404 does not halt the batch as a repo-wide access failure', async () => { + clearAllCaches(); + const octokit = new Octokit({ + request: { + fetch: async (input: RequestInfo | URL) => + String(input).includes('path=missing.ts') + ? Response.json({ message: 'Not Found' }, { status: 404 }) + : Response.json([]), + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const result = await fetchRepoChurn('owner', 'notfound', ['missing.ts', 'b.ts'], 'sha', { octokit }); + expect(result.issue).toBe('timeout-or-network'); + expect(result.complete).toBe(false); + expect(result.remaining).toEqual(['missing.ts']); + expect(result.files['b.ts']).toBeDefined(); +}); diff --git a/server/services/churn-service.ts b/server/services/churn-service.ts index ff20db4..9c021e4 100644 --- a/server/services/churn-service.ts +++ b/server/services/churn-service.ts @@ -1,122 +1,142 @@ +import { createHash } from 'node:crypto'; import { cache, churnCacheKey } from '../cache/lru'; import { getOctokit } from '../github/client'; import { isMockRepo, fetchMockChurn } from '../github/mock-data'; import type { RequestContext } from '../utils/context'; +import type { ChurnResponse, ChurnContinuation, FileChurnData } from '../../src/types/churn'; +export type { FileChurnData } from '../../src/types/churn'; -export interface FileChurnData { - commitCount: number; - authorCount: number; - lastModified: string; - churnScore: number; -} +type Failure = ChurnResponse['failures'][string]; +type Stored = { + files: Record; + failures: Record; +}; +const inFlight = new Map>(); -/** - * Fetch per-file churn data: commit count, distinct authors, last-modified date. - * For each file path, calls listCommits with the path filter (last N days). - * Processes paths in concurrent batches. - * - * Note on commitCount precision: - * When the response is paginated (Link header contains a rel="last" page), - * commitCount is approximated as `lastPage * 100`. This overestimates when - * the last page has fewer than 100 commits. For example, 101 commits → 200, - * 201 commits → 300. The error is non-uniform across files and compounds - * in the churnScore normalization step. This is an intentional trade-off - * to avoid O(repos) extra API calls. If precise counts are needed, callers - * should paginate fully. - */ +/** Credential-scoped cached results plus explicit continuation work across server instances. */ export async function fetchRepoChurn( owner: string, repo: string, - paths: string[], + inputPaths: string[], branch?: string, tokenOrCtx?: string | RequestContext, days = 90, -): Promise> { - if (isMockRepo(owner)) { - return fetchMockChurn(paths); - } - - const cacheKey = churnCacheKey(owner, repo, branch, days); - const cached = cache.get>(cacheKey); - if (cached) return cached; - - const octokit = (tokenOrCtx && typeof tokenOrCtx === 'object' && 'octokit' in tokenOrCtx) - ? tokenOrCtx.octokit - : getOctokit(tokenOrCtx as string | undefined); - + continuation: ChurnContinuation = {}, +): Promise { + const paths = [...new Set(inputPaths)].slice(0, 200); + const token = typeof tokenOrCtx === 'string' ? tokenOrCtx : tokenOrCtx?.gitHubToken; + const scope = createHash('sha256') + .update(token || process.env.GITHUB_TOKEN || 'anonymous') + .digest('hex'); + if (isMockRepo(owner)) + return { + files: await fetchMockChurn(paths), + checked: paths.length, + total: paths.length, + complete: true, + remaining: [], + failures: {}, + scope, + }; const since = new Date(); - since.setDate(since.getDate() - days); - - const result: Record = {}; - const CONCURRENCY = 7; - - for (let i = 0; i < paths.length; i += CONCURRENCY) { - const chunk = paths.slice(i, i + CONCURRENCY); - const entries = await Promise.all( - chunk.map(async (path) => { + since.setUTCHours(0, 0, 0, 0); + since.setUTCDate(since.getUTCDate() - days); + const key = `${churnCacheKey(owner, repo, branch, days)}:incremental:${since.toISOString()}:${scope}`; + const state = cache.get(key) ?? { files: {}, failures: {} }; + cache.set(key, state); + // Credential-scoped continuation: client progress is only trusted when it echoes + // this credential's scope. After a token change the cache is empty AND stale + // completed paths are discarded, so the batch refetches instead of reporting + // complete with no data. Older clients omit scope and keep legacy behavior. + const cont: ChurnContinuation = + continuation.scope === undefined || continuation.scope === scope ? continuation : {}; + const completed = new Set((cont.completed ?? []).filter((path) => paths.includes(path))); + const order = [...new Set([...(cont.pending ?? []).filter((path) => paths.includes(path)), ...paths])]; + const pending = order + .filter( + (path) => + (!cont.pending || cont.pending.includes(path)) && + !completed.has(path) && + !state.files[path] && + !(state.failures[path]?.retryAt > Date.now()), + ) + .slice(0, 7); + const octokit = tokenOrCtx && typeof tokenOrCtx === 'object' ? tokenOrCtx.octokit : getOctokit(tokenOrCtx); + await Promise.all( + pending.map((path) => { + const flightKey = `${key}:${path}`; + const existing = inFlight.get(flightKey); + if (existing) return existing; + const task = (async () => { try { const { data, headers } = await octokit.repos.listCommits({ owner, repo, - sha: branch || undefined, + sha: branch, path, since: since.toISOString(), per_page: 100, + request: { signal: AbortSignal.timeout(5000) }, }); - - if (!data || data.length === 0) return null; - - // Count from Link header if paginated, else data length - const link = headers?.link ?? ''; - const lastPage = /[?&]page=(\d+)>;\s*rel="last"/.exec(link)?.[1]; - const commitCount = lastPage ? Number(lastPage) * 100 : data.length; - - // Distinct authors + const lastPage = /[?&]page=(\d+)>;\s*rel="last"/.exec(headers.link ?? '')?.[1]; + // Login and git author name describe the same contributor. const authors = new Set(); - for (const c of data) { - if (c.commit?.author?.name) authors.add(c.commit.author.name); - if (c.author?.login) authors.add(c.author.login); - } - - const lastCommit = data[0]; - const lastModified = - lastCommit?.commit?.author?.date ?? - lastCommit?.commit?.committer?.date ?? - new Date().toISOString(); - - return { path, commitCount, authorCount: authors.size, lastModified }; - } catch (err) { - const status = err && typeof err === 'object' && 'status' in err ? (err as { status: number }).status : undefined; - if (status !== 403 && status !== 429) { - console.error(`[churn-service] Failed to fetch churn for ${path}:`, err instanceof Error ? err.message : String(err)); + for (const commit of data) { + const identity = commit.author?.login ?? commit.commit.author?.name; + if (identity) authors.add(identity); } - return null; + state.files[path] = { + commitCount: lastPage ? Number(lastPage) * 100 : data.length, + authorCount: authors.size, + lastModified: data[0]?.commit.author?.date ?? data[0]?.commit.committer?.date ?? undefined, + churnScore: 0, + }; + delete state.failures[path]; + } catch (error) { + const err = error as { + status?: number; + response?: { headers?: Record }; + }; + const headers = err.response?.headers ?? {}; + const limited = + err.status === 429 || + (err.status === 403 && (headers['x-ratelimit-remaining'] === '0' || !!headers['retry-after'])); + // 404 is per-file (renamed/missing path), not repo-wide — only + // 401/403 halt the whole batch in churnInterval(); keep polling the rest. + const issue = limited ? 'rate-limit' : err.status === 401 || err.status === 403 ? 'access' : 'timeout-or-network'; + const seconds = Number(headers['retry-after']); + const reset = Number(headers['x-ratelimit-reset']) * 1000; + const retryAt = limited + ? Math.max( + Date.now() + 60_000, + Number.isFinite(seconds) ? Date.now() + seconds * 1000 : 0, + Number.isFinite(reset) ? reset : 0, + ) + : Date.now() + 10_000; + state.failures[path] = { issue, retryAt }; } - }), - ); - - for (const entry of entries) { - if (!entry) continue; - result[entry.path] = { - commitCount: entry.commitCount, - authorCount: entry.authorCount, - lastModified: entry.lastModified, - churnScore: 0, // normalized later - }; - } - - // Warn if an entire batch failed — possible auth/rate-limit issue - if (chunk.length > 0 && entries.every((e) => e === null)) { - console.error(`[churn-service] All ${chunk.length} paths in batch failed for ${owner}/${repo} — possible auth/rate-limit issue`); - } - } - - // Normalize churnScore: find max commit count - const maxCommits = Math.max(1, ...Object.values(result).map((d) => d.commitCount)); - for (const data of Object.values(result)) { - data.churnScore = +(data.commitCount / maxCommits).toFixed(4); - } - - cache.set(cacheKey, result); - return result; + })().finally(() => inFlight.delete(flightKey)); + inFlight.set(flightKey, task); + return task; + }), + ); + const entries = paths.flatMap((path) => (state.files[path] ? [[path, state.files[path]] as const] : [])); + entries.forEach(([path]) => completed.add(path)); + const remaining = order.filter((path) => !completed.has(path)); + const failures = Object.fromEntries( + remaining.flatMap((path) => (state.failures[path] ? [[path, state.failures[path]]] : [])), + ); + const stopped = Object.values(failures).sort((a, b) => a.retryAt - b.retryAt)[0]; + const max = Math.max(1, ...entries.map(([, file]) => file.commitCount)); + return { + files: Object.fromEntries(entries.map(([path, file]) => [path, { ...file, churnScore: file.commitCount / max }])), + checked: completed.size, + total: paths.length, + complete: remaining.length === 0, + remaining, + failures, + issue: stopped?.issue, + retryAt: stopped?.retryAt, + scope, + }; } diff --git a/server/services/get-file.test.ts b/server/services/get-file.test.ts index 7a767e1..52a1685 100644 --- a/server/services/get-file.test.ts +++ b/server/services/get-file.test.ts @@ -1,121 +1,94 @@ -import { afterEach, describe, expect, it, mock, beforeEach } from 'bun:test'; -import * as mockDataModule from '../github/mock-data'; +import { beforeEach, describe, expect, it } from 'bun:test'; import { getRepoFileContent } from './get-file'; - -const realMockDataExports = { ...mockDataModule }; - -const mockGetContent = mock(async () => ({ - data: { - type: 'file', - content: Buffer.from('hello world').toString('base64'), - }, -})); - -function setupModuleMocks() { - mock.module('../github/client', () => ({ - getOctokit: () => ({ - repos: { - getContent: mockGetContent, +import { fetchMockFileContents, fetchMockRepoInfo } from '../github/mock-data'; +import type { RequestContext } from '../utils/context'; + +// Stub behavior for the non-mock (real API) path, served through +// RequestContext.octokit so no mock.module() on shared internal modules is +// needed (see AGENTS.md testing gotchas). +let getContentImpl: (path: string) => Promise<{ type: string; content?: string }>; +let getContentCalls = 0; +const stubOctokit = { + repos: { + get: async () => ({ + data: { + full_name: 'real-owner/repo', + html_url: 'https://github.com/real-owner/repo', + description: null, + stargazers_count: 0, + forks_count: 0, + language: null, + default_branch: 'main', + topics: [], + license: null, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', }, }), - handleOctokitError: (err: any) => { - throw err; + getCommit: async () => ({ data: { sha: 'test-sha' } }), + getContent: async ({ path }: { path: string }) => { + getContentCalls++; + return { data: await getContentImpl(path) }; }, - })); - - mock.module('../github/fetch-tree', () => ({ - fetchRepoInfo: async () => ({ - sha: 'test-sha', - }), - })); + }, +}; +const mockCtx = { octokit: stubOctokit } as unknown as RequestContext; - mock.module('../github/mock-data', () => ({ - isMockRepo: (owner: string) => owner === 'mock-owner', - fetchMockFileContents: async (owner: string, repo: string, paths: string[]) => ({ - [paths[0]]: 'mock file content', - }), - })); -} +const helloContent = Buffer.from('hello world').toString('base64'); describe('services/get-file', () => { beforeEach(() => { - setupModuleMocks(); - mockGetContent.mockClear(); - }); - - afterEach(() => { - mock.restore(); - mock.module('../github/mock-data', () => realMockDataExports); + getContentCalls = 0; + getContentImpl = async () => ({ type: 'file', content: helloContent }); }); it('fetches mock file content when owner is mock-owner', async () => { const res = await getRepoFileContent('mock-owner', 'repo', 'src/main.ts'); + const expected = await fetchMockFileContents('mock-owner', 'repo', ['src/main.ts']); + const info = await fetchMockRepoInfo('mock-owner', 'repo'); expect(res).toEqual({ path: 'src/main.ts', - content: 'mock file content', - sha: 'test-sha', + content: expected['src/main.ts'], + sha: info.sha, }); - expect(mockGetContent).not.toHaveBeenCalled(); + expect(getContentCalls).toBe(0); }); it('fetches real file content from github when owner is not mock', async () => { - const res = await getRepoFileContent('real-owner', 'repo', 'src/main.ts'); + const res = await getRepoFileContent('real-owner', 'repo', 'src/main.ts', undefined, mockCtx); expect(res).toEqual({ path: 'src/main.ts', content: 'hello world', sha: 'test-sha', }); - expect(mockGetContent).toHaveBeenCalled(); - }); - - it('throws error when file is not found in mock repo', async () => { - mock.module('../github/mock-data', () => ({ - isMockRepo: (owner: string) => owner === 'mock-owner', - fetchMockFileContents: async () => ({}), - })); - - expect(getRepoFileContent('mock-owner', 'repo', 'nonexistent.ts')).rejects.toThrow('File not found: nonexistent.ts'); + expect(getContentCalls).toBe(1); }); it('throws error when file is not found in github repo (404 status)', async () => { - // Restore basic mock data module first - mock.module('../github/mock-data', () => ({ - isMockRepo: (owner: string) => owner === 'mock-owner', - fetchMockFileContents: async (owner: string, repo: string, paths: string[]) => ({ - [paths[0]]: 'mock file content', - }), - })); - - mockGetContent.mockImplementationOnce(async () => { + getContentImpl = async () => { throw { status: 404 }; - }); + }; - expect(getRepoFileContent('real-owner', 'repo', 'nonexistent.ts')).rejects.toThrow('File not found: nonexistent.ts'); + await expect(getRepoFileContent('real-owner', 'repo', 'nonexistent.ts', undefined, mockCtx)).rejects.toThrow( + 'File not found: nonexistent.ts', + ); }); it('throws error when file is not a regular file (e.g. is a directory)', async () => { - // Restore basic mock data module first - mock.module('../github/mock-data', () => ({ - isMockRepo: (owner: string) => owner === 'mock-owner', - fetchMockFileContents: async (owner: string, repo: string, paths: string[]) => ({ - [paths[0]]: 'mock file content', - }), - })); + getContentImpl = async () => ({ type: 'dir' }); - mockGetContent.mockImplementationOnce(async () => ({ - data: { - type: 'dir', - }, - })); - - expect(getRepoFileContent('real-owner', 'repo', 'src/folder')).rejects.toThrow('File is not a regular file or too large to display.'); + await expect(getRepoFileContent('real-owner', 'repo', 'src/folder', undefined, mockCtx)).rejects.toThrow( + 'File is not a regular file or too large to display.', + ); }); it('delegates other API errors to handleOctokitError', async () => { - mockGetContent.mockImplementationOnce(async () => { + getContentImpl = async () => { throw new Error('500 internal server error'); - }); + }; - expect(getRepoFileContent('real-owner', 'repo', 'src/main.ts')).rejects.toThrow('500 internal server error'); + await expect(getRepoFileContent('real-owner', 'repo', 'src/main.ts', undefined, mockCtx)).rejects.toThrow( + '500 internal server error', + ); }); }); diff --git a/server/services/repo-enrichment.ts b/server/services/repo-enrichment.ts new file mode 100644 index 0000000..e82be65 --- /dev/null +++ b/server/services/repo-enrichment.ts @@ -0,0 +1,52 @@ +import { analyzeRepository } from './analyze-repo'; +import { fetchRepoChurn } from './churn-service'; +import { fetchNpmDependencyMetadataBatch } from './npm-registry'; +import { buildDependencyHealthReport } from './dependency-health'; +import { fetchFileContents, fetchFlatTree, fetchRepoInfo, findManifestPaths } from '../github/fetch-tree'; +import { analyzeDependencies, analyzeManifestDependencies } from '../parser/dependency-analyzer'; +import { findImportantFiles } from '../parser/file-classifier'; +import type { RequestContext } from '../utils/context'; +import type { ChurnContinuation } from '../../src/types/churn'; + +export async function enrichRepository( + input: { owner: string; repo: string; branch?: string }, + kind: 'churn' | 'health', + ctx: RequestContext, + continuation: ChurnContinuation = {}, +) { + if (kind === 'health') return fetchDependencyHealth(input, ctx); + const analysis = await analyzeRepository(input, ctx); + const paths = analysis.graph.nodes + .flatMap((node) => + node.type === 'file' && node.data.path && /\.(tsx?|jsx?|py|go)$/.test(node.data.path) ? [node.data.path] : [], + ) + .slice(0, 200); + return fetchRepoChurn(input.owner, input.repo, paths, analysis.meta.sha, ctx, 90, continuation); +} + +// Dependency-focused health path: a cold /api/repo/health must not pay for the +// full analyzeRepository (contributors, timeline, commit counts, graph build) +// when only manifest/source contents are needed. Content selection mirrors +// analyze-repo so reports stay identical; analyzeRepository's cache still serves +// callers that need the full analysis. +async function fetchDependencyHealth( + input: { owner: string; repo: string; branch?: string }, + ctx: RequestContext, +) { + const info = await fetchRepoInfo(input.owner, input.repo, input.branch, ctx); + const { items } = await fetchFlatTree(input.owner, input.repo, info.sha, ctx); + const manifestPaths = findManifestPaths(items); + const sourceExtensions = ['.ts', '.tsx', '.js', '.jsx', '.py', '.go']; + const importantSourceFiles = findImportantFiles(items.map((item) => item.path)).filter((path) => + sourceExtensions.some((ext) => path.endsWith(ext)), + ); + const contentFetchPaths = Array.from(new Set([...manifestPaths, ...importantSourceFiles])).slice(0, 50); + const fileContents = await fetchFileContents(input.owner, input.repo, contentFetchPaths, info.sha, ctx); + const dependencies = analyzeDependencies(fileContents); + const scopedDependencies = analyzeManifestDependencies(fileContents); + // No truncation: every npm dependency gets registry metadata. Capping the + // batch would silently report omitted deps as 'unknown', indistinguishable + // from missing or failed lookups. Batching + the 24h registry cache bound cost. + const metadata = await fetchNpmDependencyMetadataBatch(dependencies.filter((dep) => dep.ecosystem === 'npm')); + return buildDependencyHealthReport(dependencies, scopedDependencies, metadata); +} diff --git a/server/services/trending.test.ts b/server/services/trending.test.ts index 18c40b0..11bf7f1 100644 --- a/server/services/trending.test.ts +++ b/server/services/trending.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, mock, beforeEach } from 'bun:test'; -import { fetchTrending } from './trending'; +import { createTrendingFetcher, loadTrending } from './trending'; const mockSearchRepos = mock(async () => ({ data: { @@ -25,7 +25,7 @@ describe('services/trending', () => { repos: mockSearchRepos, }, }), - handleOctokitError: (err: any) => { + handleOctokitError: (err: unknown) => { throw err; }, })); @@ -37,7 +37,7 @@ describe('services/trending', () => { }); it('fetches and maps trending repositories successfully', async () => { - const repos = await fetchTrending(); + const repos = await loadTrending(); expect(repos).toHaveLength(1); expect(repos[0]).toEqual({ owner: 'test-owner', @@ -56,7 +56,7 @@ describe('services/trending', () => { throw new Error('GitHub API Error'); }); - const repos = await fetchTrending(); + const repos = await loadTrending(); expect(repos.length).toBeGreaterThan(0); expect(repos[0].owner).toBe('facebook'); }); @@ -76,7 +76,7 @@ describe('services/trending', () => { handleOctokitError: () => {}, })); - const repos = await fetchTrending(); + const repos = await loadTrending(); expect(repos.length).toBeGreaterThan(0); expect(repos[0].owner).toBe('facebook'); @@ -87,9 +87,22 @@ describe('services/trending', () => { repos: mockSearchRepos, }, }), - handleOctokitError: (err: any) => { + handleOctokitError: (err: unknown) => { throw err; }, })); }); + + it('coalesces concurrent lookups and serves the cached snapshot', async () => { + let calls = 0; + const fetcher = createTrendingFetcher(async () => { + calls++; + return []; + }); + + await Promise.all([fetcher(), fetcher()]); + await fetcher(); + + expect(calls).toBe(1); + }); }); diff --git a/server/services/trending.ts b/server/services/trending.ts index 062268e..9a11569 100644 --- a/server/services/trending.ts +++ b/server/services/trending.ts @@ -10,7 +10,24 @@ const FALLBACK: TrendingRepo[] = [ { owner: 'rust-lang', repo: 'rust', fullName: 'rust-lang/rust', description: 'Empowering everyone to build reliable software', stars: 98000, language: 'Rust', url: 'https://github.com/rust-lang/rust' }, ]; -export async function fetchTrending(): Promise { +/** One public snapshot and one in-flight lookup per process, including fallback results. */ +export function createTrendingFetcher(load: () => Promise, now = Date.now) { + let cached: { repos: TrendingRepo[]; expires: number } | undefined; + let pending: Promise | undefined; + return (): Promise => { + if (cached && cached.expires > now()) return Promise.resolve(cached.repos); + if (pending) return pending; + pending = Promise.resolve().then(load).then((repos) => { + cached = { repos, expires: now() + 300000 }; + return repos; + }).finally(() => { pending = undefined; }); + return pending; + }; +} + +export const fetchTrending = createTrendingFetcher(loadTrending); + +export async function loadTrending(): Promise { try { const octokit = getOctokit(); const { data } = await octokit.search.repos({ diff --git a/server/utils/bounded-queue.test.ts b/server/utils/bounded-queue.test.ts new file mode 100644 index 0000000..d64dc3b --- /dev/null +++ b/server/utils/bounded-queue.test.ts @@ -0,0 +1,132 @@ +import { test, expect, spyOn } from 'bun:test'; +import { createBoundedQueue } from './bounded-queue'; + +test('queue rejects overflow and runs queued work after a slot is released', async () => { + const queue = createBoundedQueue(1, 1, 1000, 1000); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const first = queue(async () => { + await gate; + return 1; + }); + const second = queue(async () => 2); + await expect(queue(async () => 3)).rejects.toMatchObject({ status: 429 }); + release(); + expect(await first).toBe(1); + expect(await second).toBe(2); +}); + +test('expired waiting jobs never execute', async () => { + const queue = createBoundedQueue(1, 1, 10, 1000); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const first = queue(async () => { + await gate; + }); + let ran = false; + await expect( + queue(async () => { + ran = true; + }), + ).rejects.toMatchObject({ status: 429 }); + release(); + await first; + expect(ran).toBe(false); +}); + +test('execution deadline aborts but does not release a still-running operation', async () => { + const queue = createBoundedQueue(1, 0, 1000, 10); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let signal: AbortSignal | undefined; + const first = queue(async (current) => { + signal = current; + await gate; + }); + await expect(first).rejects.toMatchObject({ status: 504 }); + expect(signal?.aborted).toBe(true); + await expect(queue(async () => {})).rejects.toMatchObject({ status: 429 }); + release(); +}); + +test('caller signal aborts and removes waiting job immediately', async () => { + const queue = createBoundedQueue(1, 4, 10000, 30000); + let release = () => {}; + const held = new Promise((r) => { release = r; }); + const first = queue(() => held); + + const controller = new AbortController(); + const second = queue(() => Promise.resolve('second'), controller.signal); + + controller.abort(); + await expect(second).rejects.toThrow(); + + release(); + await first; +}); + +test('caller signal aborts running job, aborts work signal, rejects caller immediately, and retains slot until work settles', async () => { + const queue = createBoundedQueue(1, 0, 10000, 30000); + let releaseWork = () => {}; + const workHeld = new Promise((r) => { releaseWork = r; }); + let workSignal: AbortSignal | undefined; + + const controller = new AbortController(); + const job = queue(async (sig) => { + workSignal = sig; + await workHeld; + return 'done'; + }, controller.signal); + + await new Promise((r) => setTimeout(r, 5)); + expect(workSignal).toBeDefined(); + expect(workSignal?.aborted).toBe(false); + + controller.abort(new Error('caller cancelled')); + + await expect(job).rejects.toThrow('caller cancelled'); + expect(workSignal?.aborted).toBe(true); + + await expect(queue(async () => 'next')).rejects.toMatchObject({ status: 429 }); + + releaseWork(); + await new Promise((r) => setTimeout(r, 10)); + + const nextJob = await queue(async () => 'free'); + expect(nextJob).toBe('free'); +}); + +test('abort right after queue() never invokes work and releases the slot', async () => { + const queue = createBoundedQueue(1, 0, 1000, 30000); + const controller = new AbortController(); + let ran = false; + const job = queue(async () => { + ran = true; + return 'x'; + }, controller.signal); + controller.abort(new Error('too late')); + await expect(job).rejects.toThrow('too late'); + await new Promise((r) => setTimeout(r, 10)); + expect(ran).toBe(false); + expect(await queue(async () => 'free')).toBe('free'); +}); + +test('runMs deadline detaches the caller-abort listener even when work never settles', async () => { + const queue = createBoundedQueue(1, 0, 1000, 10); + const controller = new AbortController(); + const addSpy = spyOn(controller.signal, 'addEventListener'); + const removeSpy = spyOn(controller.signal, 'removeEventListener'); + const hung = queue(() => new Promise(() => {}), controller.signal); + await expect(hung).rejects.toMatchObject({ status: 504 }); + const registered = addSpy.mock.calls.find(([type]) => type === 'abort')?.[1]; + expect(registered).toBeTypeOf('function'); + expect(removeSpy).toHaveBeenCalledWith('abort', registered); + removeSpy.mockRestore(); + addSpy.mockRestore(); +}); diff --git a/server/utils/bounded-queue.ts b/server/utils/bounded-queue.ts new file mode 100644 index 0000000..5788073 --- /dev/null +++ b/server/utils/bounded-queue.ts @@ -0,0 +1,114 @@ +import { AppError } from './errors'; + +const timeoutError = () => new AppError(504, 'Provider request timed out.', 'PROVIDER_TIMEOUT', true); + +/** Deadlines reject callers promptly, but running work holds its slot until it actually settles. */ +export function createBoundedQueue(concurrency = 2, maxWaiting = 8, waitMs = 10000, runMs = 30000) { + let active = 0; + const waiting: Array<() => void> = []; + return (work: (signal: AbortSignal) => Promise, callerSignal?: AbortSignal): Promise => + new Promise((resolve, reject) => { + if (callerSignal?.aborted) { + reject(callerSignal.reason || timeoutError()); + return; + } + if (active >= concurrency && waiting.length >= maxWaiting) { + reject(new AppError(429, 'Provider queue is full. Please retry later.', 'PROVIDER_BUSY', true)); + return; + } + let waitTimer: ReturnType | undefined; + let onWaitAbort: (() => void) | undefined; + const cleanupWait = () => { + clearTimeout(waitTimer); + if (onWaitAbort && callerSignal) callerSignal.removeEventListener('abort', onWaitAbort); + }; + const start = () => { + cleanupWait(); + if (callerSignal?.aborted) { + waiting.shift()?.(); + reject(callerSignal.reason || timeoutError()); + return; + } + active++; + let settled = false; + const controller = new AbortController(); + + const onActiveAbort = () => { + if (settled) return; + settled = true; + controller.abort(callerSignal?.reason); + reject(callerSignal?.reason || timeoutError()); + }; + const detachCallerAbort = () => { + if (callerSignal) callerSignal.removeEventListener('abort', onActiveAbort); + }; + + const timer = setTimeout(() => { + if (settled) return; + settled = true; + controller.abort(); + // The caller is gone even though the slot stays held until work settles. + detachCallerAbort(); + reject(timeoutError()); + }, runMs); + + if (callerSignal) { + callerSignal.addEventListener('abort', onActiveAbort, { once: true }); + } + + const releaseSlot = () => { + clearTimeout(timer); + detachCallerAbort(); + active--; + waiting.shift()?.(); + }; + + Promise.resolve() + .then(() => { + // Abort won the race before work started — release the slot + // without invoking side-effecting work. + if (settled) { + releaseSlot(); + return new Promise(() => {}); + } + return work(controller.signal); + }) + .then( + (val) => { + releaseSlot(); + if (!settled) { + settled = true; + resolve(val); + } + }, + (err) => { + releaseSlot(); + if (!settled) { + settled = true; + reject(err); + } + }, + ); + }; + + if (active < concurrency) start(); + else { + waiting.push(start); + if (callerSignal) { + onWaitAbort = () => { + const index = waiting.indexOf(start); + if (index >= 0) waiting.splice(index, 1); + cleanupWait(); + reject(callerSignal.reason || timeoutError()); + }; + callerSignal.addEventListener('abort', onWaitAbort, { once: true }); + } + waitTimer = setTimeout(() => { + const index = waiting.indexOf(start); + if (index >= 0) waiting.splice(index, 1); + cleanupWait(); + reject(new AppError(429, 'Provider queue wait expired. Please retry.', 'PROVIDER_BUSY', true)); + }, waitMs); + } + }); +} diff --git a/server/utils/client-limits.test.ts b/server/utils/client-limits.test.ts new file mode 100644 index 0000000..33b89c8 --- /dev/null +++ b/server/utils/client-limits.test.ts @@ -0,0 +1,55 @@ +import { afterEach, expect, test } from 'bun:test'; +import { limitClient, resolveClientAddress } from './client-limits'; + +const originalProxy = process.env.TRUSTED_PROXY_IPS; +test('mapped IPv6 spellings share the IPv4 identity and trusted proxy matching', () => { + process.env.TRUSTED_PROXY_IPS = '127.0.0.1'; + for (const peer of ['::ffff:7f00:1', '0:0:0:0:0:ffff:7f00:1', '::ffff:127.0.0.1']) { + expect(resolveClientAddress(new Headers(), peer)).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '1.2.3.4' }), peer)).toBe('1.2.3.4'); + } +}); +const originalLimit = process.env.API_REQUESTS_PER_IP_PER_MINUTE; +afterEach(() => { + if (originalProxy === undefined) delete process.env.TRUSTED_PROXY_IPS; + else process.env.TRUSTED_PROXY_IPS = originalProxy; + if (originalLimit === undefined) delete process.env.API_REQUESTS_PER_IP_PER_MINUTE; + else process.env.API_REQUESTS_PER_IP_PER_MINUTE = originalLimit; +}); + +test('untrusted connections cannot spoof client identity', () => { + delete process.env.TRUSTED_PROXY_IPS; + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '1.2.3.4' }), '::ffff:192.0.2.1')).toBe('192.0.2.1'); + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '1.2.3.4' }))).toBe('unknown'); +}); + +test('trusted chain stops at nearest untrusted hop and rejects malformed addresses', () => { + process.env.TRUSTED_PROXY_IPS = '127.0.0.1, ::1'; + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '1.2.3.4, 192.0.2.2, ::1' }), '127.0.0.1')).toBe('192.0.2.2'); + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': 'invalid' }), '127.0.0.1')).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers(), '2001:0db8:0:0:0:0:0:1')).toBe('2001:db8::1'); +}); + +test('hex and octal IPv4 forms canonicalize to the same bucket as dotted decimal', () => { + delete process.env.TRUSTED_PROXY_IPS; + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '0x7f.0.0.1' }), '9.9.9.9')).toBe('9.9.9.9'); + process.env.TRUSTED_PROXY_IPS = '9.9.9.9'; + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '0x7f.0.0.1' }), '9.9.9.9')).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers({ 'x-forwarded-for': '0177.0.0.1' }), '9.9.9.9')).toBe('127.0.0.1'); +}); + +test('hex and octal IPv4-mapped IPv6 forms canonicalize from the 32-bit payload', () => { + delete process.env.TRUSTED_PROXY_IPS; + expect(resolveClientAddress(new Headers(), '::ffff:0x7f.0.0.1')).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers(), '::ffff:0177.0.0.1')).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers(), '::ffff:0x7f000001')).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers(), '::ffff:2130706433')).toBe('127.0.0.1'); + expect(resolveClientAddress(new Headers(), '::ffff:garbage')).toBe('unknown'); +}); + +test('same IP shares quota regardless of credentials while other IP has its own allowance', async () => { + process.env.API_REQUESTS_PER_IP_PER_MINUTE = '1'; + await limitClient(new Headers(), '192.0.2.110'); + await expect(limitClient(new Headers({ 'x-gemini-api-key': 'user-key', 'x-github-token': 'pat' }), '192.0.2.110')).rejects.toMatchObject({ status: 429 }); + await limitClient(new Headers(), '192.0.2.111'); +}); diff --git a/server/utils/client-limits.ts b/server/utils/client-limits.ts new file mode 100644 index 0000000..1cff859 --- /dev/null +++ b/server/utils/client-limits.ts @@ -0,0 +1,79 @@ +import { createHmac } from 'node:crypto'; +import { isIP } from 'node:net'; +import { cacheHashSecret } from '../cache/hash-secret'; +import { configuredLimit, reserveUsage } from './usage-limits'; + +function parseDottedQuad(text: string): string | undefined { + const dotted = text.toLowerCase().match(/^([^:]+):([^:]+):([^:]+):([^:]+)$/); + const parts = dotted ? [dotted[1], dotted[2], dotted[3], dotted[4]] : text.includes('.') && !text.includes(':') ? text.split('.') : null; + if (!parts || parts.length !== 4) return undefined; + const octets: number[] = []; + for (const part of parts) { + const p = part.trim().toLowerCase(); + if (!/^(0[xX][0-9a-f]+|0[oO][0-7]+|0[0-7]*|[0-9]+)$/.test(p)) return undefined; + const n = p.startsWith('0x') || p.startsWith('0X') ? parseInt(p, 16) : p.startsWith('0o') || p.startsWith('0O') ? parseInt(p.slice(2), 8) : p.length > 1 && p.startsWith('0') ? parseInt(p, 8) : parseInt(p, 10); + if (!Number.isInteger(n) || n < 0 || n > 255) return undefined; + octets.push(n); + } + return octets.join('.'); +} + +function parseSingleU32(text: string): string | undefined { + const p = text.trim().toLowerCase(); + if (!/^(0[xX][0-9a-f]+|0[oO][0-7]+|0[0-7]*|[0-9]+)$/.test(p)) return undefined; + const n = p.startsWith('0x') || p.startsWith('0X') ? parseInt(p, 16) : p.startsWith('0o') || p.startsWith('0O') ? parseInt(p.slice(2), 8) : p.length > 1 && p.startsWith('0') ? parseInt(p, 8) : parseInt(p, 10); + if (!Number.isInteger(n) || n < 0 || n > 0xffffffff) return undefined; + return [(n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255].join('.'); +} + +function normalize(value: string): string | undefined { + const raw = value.trim(); + if (!raw) return undefined; + // ponytail: parse hex/octal/decimal IPv4 forms to their 32-bit payload so forwarded + // values cannot split buckets (0x7f.0.0.1 and 127.0.0.1 hash together). + const quad = parseDottedQuad(raw); + if (quad) return quad; + const address = raw.toLowerCase(); + if (isIP(address) === 6) { + const canonical = new URL(`http://[${address}]/`).hostname.slice(1, -1); + const mapped = /^::ffff:([0-9a-f]+):([0-9a-f]+)$/.exec(canonical); + if (mapped) { + const high = parseInt(mapped[1], 16); + const low = parseInt(mapped[2], 16); + return [high >> 8, high & 255, low >> 8, low & 255].join('.'); + } + return canonical; + } + // IPv4-mapped IPv6 must canonicalize from the embedded 32-bit payload, including + // hex/octal forms that isIP() rejects (e.g. ::ffff:0x7f.0.0.1, ::ffff:0x7f000001). + if (address.startsWith('::ffff:')) { + const tail = address.slice(7); + return parseDottedQuad(tail) ?? parseSingleU32(tail) ?? undefined; + } + if (!isIP(address)) return undefined; + return isIP(address) === 6 ? new URL(`http://[${address}]/`).hostname.slice(1, -1) : address; +} + +/** Only explicitly configured transport peers may supply a forwarded chain. */ +export function resolveClientAddress(headers: Headers, remoteAddress?: string): string { + const peer = remoteAddress && normalize(remoteAddress); + if (!peer) return 'unknown'; + const trusted = new Set((process.env.TRUSTED_PROXY_IPS ?? '').split(',').map(normalize).filter(Boolean)); + if (!trusted.has(peer)) return peer; + const chain = (headers.get('x-forwarded-for') ?? '').split(','); + let address = peer; + for (let i = chain.length - 1; i >= 0 && trusted.has(address); i--) { + const candidate = normalize(chain[i]); + if (!candidate) return peer; + address = candidate; + } + return address; +} + +export async function limitClient(headers: Headers, remoteAddress?: string): Promise { + const address = resolveClientAddress(headers, remoteAddress); + // ponytail: HMAC with the server secret (not unkeyed SHA-256) so readable Redis keys + // do not let IPv4 enumeration recover the client mapping. Missing transport shares one bucket. + const scope = createHmac('sha256', cacheHashSecret()).update(address).digest('hex'); + await reserveUsage(`ip-minute:${scope}`, 1, configuredLimit('API_REQUESTS_PER_IP_PER_MINUTE', 120), 60000); +} diff --git a/server/utils/http.test.ts b/server/utils/http.test.ts index 80c6df9..8aded0d 100644 --- a/server/utils/http.test.ts +++ b/server/utils/http.test.ts @@ -1,5 +1,7 @@ import { describe, it, expect } from 'bun:test'; -import { addSecurityHeaders } from './http'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { addSecurityHeaders, handleNodeRequest } from './http'; +import { MAX_BODY_BYTES } from './request-limits'; describe('addSecurityHeaders', () => { it('adds security headers to the response', () => { @@ -9,6 +11,83 @@ describe('addSecurityHeaders', () => { expect(secureResponse.headers.get('X-Content-Type-Options')).toBe('nosniff'); expect(secureResponse.headers.get('X-Frame-Options')).toBe('DENY'); expect(secureResponse.headers.get('Strict-Transport-Security')).toBe('max-age=31536000; includeSubDomains'); - expect(secureResponse.headers.get('Content-Security-Policy')).toBe("default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-mhDq8RP/TAuNwiFSk7hwZsZ3tIWH410AupSuJ9xEhZg=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); + expect(secureResponse.headers.get('Content-Security-Policy')).toBe("default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-1HwijBPaKaOtZGvlT2Hw0JpmgDixxpRo6XKpkDG9xvs=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); + }); +}); + +/** Minimal Node req/res contract used by handleNodeRequest's body-reading path. */ +function fakeNode(options: { headers?: Record; chunks?: Buffer[]; hang?: boolean }) { + const state = { destroyed: false, status: 0, headers: {} as Record, body: '', order: [] as string[] }; + const req = { + method: 'POST', + url: '/api/search', + headers: options.headers ?? {}, + socket: {}, + destroy() { + state.destroyed = true; + state.order.push('destroy'); + }, + iterator: options.hang + ? async function* () { + await new Promise(() => {}); + yield Buffer.alloc(0); + } + : options.chunks + ? async function* () { + for (const chunk of options.chunks!) yield chunk; + } + : undefined, + } as unknown as IncomingMessage; + const res = { + get statusCode() { + return state.status; + }, + set statusCode(value: number) { + state.status = value; + }, + setHeader(key: string, value: string) { + state.headers[key.toLowerCase()] = value; + }, + end(chunk?: unknown, cb?: () => void) { + if (typeof chunk === 'string') state.body = chunk; + state.order.push('end'); + cb?.(); + }, + } as unknown as ServerResponse; + return { req, res, state }; +} + +describe('handleNodeRequest body-limit teardown', () => { + it('responds 413 to an oversized Content-Length and destroys the request instead of draining', async () => { + const node = fakeNode({ headers: { 'content-length': String(MAX_BODY_BYTES + 1) } }); + const handled = await handleNodeRequest(node.req, node.res); + + expect(handled).toBe(true); + expect(node.state.status).toBe(413); + expect(node.state.headers['connection']).toBe('close'); + expect(node.state.destroyed).toBe(true); + }); + + it('responds 413 when a chunked body crosses the limit mid-stream and tears the request down', async () => { + const chunk = Buffer.alloc(200 * 1024, 1); + const node = fakeNode({ chunks: [chunk, chunk, chunk] }); + const handled = await handleNodeRequest(node.req, node.res); + + expect(handled).toBe(true); + expect(node.state.status).toBe(413); + expect(node.state.headers['connection']).toBe('close'); + expect(node.state.destroyed).toBe(true); + }); + + it('responds typed 408 when the body stalls, keeping the connection open for the error write', async () => { + const node = fakeNode({ hang: true }); + const handled = await handleNodeRequest(node.req, node.res, 50); + + expect(handled).toBe(true); + expect(node.state.status).toBe(408); + expect(JSON.parse(node.state.body)).toMatchObject({ status: 408, code: 'REQUEST_TIMEOUT' }); + expect(node.state.headers['connection']).toBe('close'); + // Destroy happens only in the end callback (after flush), never before the write. + expect(node.state.order).toEqual(['end', 'destroy']); }); }); diff --git a/server/utils/http.ts b/server/utils/http.ts index 70d0107..db1347b 100644 --- a/server/utils/http.ts +++ b/server/utils/http.ts @@ -1,6 +1,8 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import type { TLSSocket } from 'node:tls'; import { handleApiRequest } from '../api-router'; +import { checkBodySize } from './request-limits'; +import { AppError, toErrorPayload } from './errors'; /** * Adapter to handle Node-compatible requests (Vite dev server, Vercel) @@ -9,6 +11,7 @@ import { handleApiRequest } from '../api-router'; export async function handleNodeRequest( nodeReq: IncomingMessage, nodeRes: ServerResponse, + bodyTimeoutMs = 15000, ): Promise { const headers = new Headers(); for (const [key, value] of Object.entries(nodeReq.headers)) { @@ -26,16 +29,50 @@ export async function handleNodeRequest( const url = `${protocol}://${host}${nodeReq.url}`; let body: BodyInit | undefined; - const requestWithBody = nodeReq as IncomingMessage & { body?: unknown }; - if (requestWithBody.body !== undefined) { - const parsedBody = requestWithBody.body; - body = typeof parsedBody === 'string' ? parsedBody : JSON.stringify(parsedBody); - } else if (nodeReq.method !== 'GET' && nodeReq.method !== 'HEAD') { - const chunks: Buffer[] = []; - for await (const chunk of nodeReq) { - chunks.push(chunk as Buffer); + // A stalled body must abort the read without destroying the connection: the + // race below rejects with a typed 408 while the socket stays open for the + // error write (closed after flush in the catch block). Destroying the + // request up front would surface a plain Error -> generic 500 instead. + const requestTimeoutError = () => new AppError(408, 'Request body timed out.', 'REQUEST_TIMEOUT', true); + let bodyTimer: ReturnType | undefined; + const bodyDeadline = new Promise((_resolve, reject) => { + bodyTimer = setTimeout(() => reject(requestTimeoutError()), bodyTimeoutMs); + }); + try { + checkBodySize(Number(nodeReq.headers['content-length'] ?? 0)); + const requestWithBody = nodeReq as IncomingMessage & { body?: unknown }; + if (requestWithBody.body !== undefined) { + const parsedBody = requestWithBody.body; + body = typeof parsedBody === 'string' ? parsedBody : JSON.stringify(parsedBody); + checkBodySize(Buffer.byteLength(body)); + } else if (nodeReq.method !== 'GET' && nodeReq.method !== 'HEAD') { + const chunks: Buffer[] = []; + let size = 0; + const reader = (async () => { + for await (const chunk of nodeReq.iterator({ destroyOnReturn: false })) { + size += Buffer.byteLength(chunk); + checkBodySize(size); + chunks.push(chunk as Buffer); + } + })(); + // Swallow the abandoned reader's late settlement; teardown happens in + // the catch block after the error response is flushed. + void reader.catch(() => undefined); + await Promise.race([reader, bodyDeadline]); + body = Buffer.concat(chunks); } - body = Buffer.concat(chunks); + } catch (error) { + // The request stream is intentionally left unconsumed: draining an oversized body can + // tie the connection up indefinitely. Flush the error response first, then tear down. + const payload = toErrorPayload(error); + const response = addSecurityHeaders(Response.json(payload, { status: payload.status })); + nodeRes.statusCode = response.status; + response.headers.forEach((value, key) => nodeRes.setHeader(key, value)); + nodeRes.setHeader('Connection', 'close'); + nodeRes.end(await response.text(), () => nodeReq.destroy()); + return true; + } finally { + clearTimeout(bodyTimer); } const webReq = new Request(url, { @@ -44,7 +81,7 @@ export async function handleNodeRequest( body, }); - const webRes = await handleApiRequest(webReq); + const webRes = await handleApiRequest(webReq, nodeReq.socket.remoteAddress); if (!webRes) { return false; } @@ -64,6 +101,9 @@ export function addSecurityHeaders(response: Response): Response { response.headers.set('X-Frame-Options', 'DENY'); // X-XSS-Protection is deprecated; use CSP instead response.headers.set('Strict-Transport-Security', 'max-age=31536000; includeSubDomains'); - response.headers.set('Content-Security-Policy', "default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-mhDq8RP/TAuNwiFSk7hwZsZ3tIWH410AupSuJ9xEhZg=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); + response.headers.set( + 'Content-Security-Policy', + "default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-1HwijBPaKaOtZGvlT2Hw0JpmgDixxpRo6XKpkDG9xvs=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;", + ); return response; } diff --git a/server/utils/page-route.test.ts b/server/utils/page-route.test.ts new file mode 100644 index 0000000..fa985c2 --- /dev/null +++ b/server/utils/page-route.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from 'bun:test'; +import { isRepositoryPage } from './page-route'; + +test('repository routes support deep links without treating missing assets as pages', () => { + for (const path of ['/mock/todo-app', '/mbayue/gitSdm/search', '/owner/repo/']) expect(isRepositoryPage(path)).toBe(true); + for (const path of ['/missing', '/assets/missing.js', '/api/missing', '/owner/repo/unknown', '/a/b/c/d']) expect(isRepositoryPage(path)).toBe(false); +}); diff --git a/server/utils/page-route.ts b/server/utils/page-route.ts new file mode 100644 index 0000000..ecfdaed --- /dev/null +++ b/server/utils/page-route.ts @@ -0,0 +1,4 @@ +/** Valid SPA paths; repository existence is resolved by the repository API. */ +export function isRepositoryPage(path: string): boolean { + return /^\/(?!api\/|assets\/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38}\/[a-zA-Z0-9_.-]{1,100}(?:\/search)?\/?$/.test(path); +} diff --git a/server/utils/request-limits.test.ts b/server/utils/request-limits.test.ts new file mode 100644 index 0000000..a45f3c2 --- /dev/null +++ b/server/utils/request-limits.test.ts @@ -0,0 +1,78 @@ +import { expect, test } from 'bun:test'; +import { createAdmissionLimit, limitRequestBody, MAX_BODY_BYTES } from './request-limits'; + +test('a stalled body is cancelled at its deadline', async () => { + let cancelled = false; + const body = new ReadableStream({ + cancel() { + cancelled = true; + }, + }); + await expect( + limitRequestBody(new Request('http://localhost/api', { method: 'POST', body }), 10), + ).rejects.toMatchObject({ status: 408 }); + expect(cancelled).toBe(true); +}); + +test('admission rejects overload, releases once, and resets the time window', () => { + let time = 0; + const admit = createAdmissionLimit(1, 2, () => time); + const release = admit(); + expect(release).not.toBeNull(); + expect(admit()).toBeNull(); + release?.(); + release?.(); + const second = admit(); + expect(second).not.toBeNull(); + expect(admit()).toBeNull(); + second?.(); + expect(admit()).toBeNull(); + time = 60_000; + expect(admit()).not.toBeNull(); +}); + +test('body limit preserves JSON at the byte boundary', async () => { + const body = 'a'.repeat(MAX_BODY_BYTES); + const request = await limitRequestBody(new Request('http://localhost/api', { method: 'POST', body })); + expect(await request.text()).toBe(body); +}); + +test('body limit rejects streamed excess even with a false content length', async () => { + let cancelled = false; + const stream = new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(MAX_BODY_BYTES)); + }, + cancel() { + cancelled = true; + }, + }); + const request = new Request('http://localhost/api', { + method: 'POST', + body: stream, + headers: { 'content-length': '1' }, + }); + await expect(limitRequestBody(request)).rejects.toMatchObject({ + status: 413, + }); + expect(cancelled).toBe(true); +}); + +test('body limit counts UTF-8 bytes and rejects oversized declared lengths', async () => { + await expect( + limitRequestBody( + new Request('http://localhost/api', { + method: 'POST', + body: 'é'.repeat(MAX_BODY_BYTES), + }), + ), + ).rejects.toMatchObject({ status: 413 }); + await expect( + limitRequestBody( + new Request('http://localhost/api', { + method: 'POST', + headers: { 'content-length': String(MAX_BODY_BYTES + 1) }, + }), + ), + ).rejects.toMatchObject({ status: 413 }); +}); diff --git a/server/utils/request-limits.ts b/server/utils/request-limits.ts new file mode 100644 index 0000000..6fbb594 --- /dev/null +++ b/server/utils/request-limits.ts @@ -0,0 +1,71 @@ +import { AppError } from './errors'; + +export const MAX_BODY_BYTES = 256 * 1024; + +export function checkBodySize(bytes: number): void { + if (bytes > MAX_BODY_BYTES) throw new AppError(413, 'Request body is too large', 'PAYLOAD_TOO_LARGE'); +} + +/** Bound the stream before JSON parsing, including requests without Content-Length. */ +export async function limitRequestBody(req: Request, timeoutMs = 15000): Promise { + checkBodySize(Number(req.headers.get('content-length') ?? 0)); + if (!req.body) return req; + const reader = req.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + let timer: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new AppError(408, 'Request body timed out.', 'REQUEST_TIMEOUT', true)), timeoutMs); + }); + try { + while (true) { + const { done, value } = await Promise.race([reader.read(), deadline]); + if (done) break; + size += value.byteLength; + checkBodySize(size); + chunks.push(value); + } + } catch (error) { + void reader.cancel().catch(() => undefined); + throw error; + } finally { + clearTimeout(timer); + reader.releaseLock(); + } + const body = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return new Request(req.url, { + method: req.method, + headers: req.headers, + body, + signal: req.signal, + }); +} + +/** Process-wide ceiling; no growing client map and no waiting queue. */ +export function createAdmissionLimit(maxActive = 8, maxPerMinute = 120, now = Date.now) { + let active = 0; + let started = now(); + let count = 0; + return () => { + const time = now(); + if (time - started >= 60_000) { + started = time; + count = 0; + } + if (active >= maxActive || count >= maxPerMinute) return null; + active++; + count++; + let released = false; + return () => { + if (!released) { + active--; + released = true; + } + }; + }; +} diff --git a/server/utils/url-guard.test.ts b/server/utils/url-guard.test.ts new file mode 100644 index 0000000..e835f8a --- /dev/null +++ b/server/utils/url-guard.test.ts @@ -0,0 +1,112 @@ +import { test, expect } from 'bun:test'; +import { isSafeRemoteUrl, expandIpv6Groups } from './url-guard'; + +test('allows public https remotes', () => { + expect(isSafeRemoteUrl('https://api.upstash.com/v2')).toBe(true); + expect(isSafeRemoteUrl('https://169.63.10.1')).toBe(true); + expect(isSafeRemoteUrl('https://8.8.8.8')).toBe(true); + expect(isSafeRemoteUrl('https://192.0.1.5')).toBe(true); + expect(isSafeRemoteUrl('https://100.128.0.1')).toBe(true); + expect(isSafeRemoteUrl('https://172.32.1.1')).toBe(true); + expect(isSafeRemoteUrl('https://198.51.200.1')).toBe(true); + expect(isSafeRemoteUrl('https://203.0.114.1')).toBe(true); + expect(isSafeRemoteUrl('https://[2606:4700::1]/v2')).toBe(true); + expect(isSafeRemoteUrl('https://[64:ff9b::102:304]/v2')).toBe(true); + expect(isSafeRemoteUrl('https://[2002:100:2::]/v2')).toBe(true); +}); + +test('rejects non-http schemes and malformed urls', () => { + expect(isSafeRemoteUrl('ftp://example.com')).toBe(false); + expect(isSafeRemoteUrl('file:///etc/passwd')).toBe(false); + expect(isSafeRemoteUrl('not a url')).toBe(false); + expect(isSafeRemoteUrl('')).toBe(false); +}); + +test('rejects plaintext http so credentials never cross the network without TLS', () => { + expect(isSafeRemoteUrl('http://example.com')).toBe(false); + expect(isSafeRemoteUrl('http://8.8.8.8')).toBe(false); +}); + +test('rejects loopback, private, link-local, and reserved hosts', () => { + expect(isSafeRemoteUrl('https://localhost')).toBe(false); + expect(isSafeRemoteUrl('https://api.localhost')).toBe(false); + expect(isSafeRemoteUrl('https://127.0.0.1')).toBe(false); + expect(isSafeRemoteUrl('https://10.1.2.3')).toBe(false); + expect(isSafeRemoteUrl('https://172.16.0.9')).toBe(false); + expect(isSafeRemoteUrl('https://172.31.255.1')).toBe(false); + expect(isSafeRemoteUrl('https://192.168.1.1')).toBe(false); + expect(isSafeRemoteUrl('https://169.254.169.254')).toBe(false); + expect(isSafeRemoteUrl('https://0.0.0.0')).toBe(false); + expect(isSafeRemoteUrl('https://100.100.1.1')).toBe(false); + expect(isSafeRemoteUrl('https://224.0.0.1')).toBe(false); + expect(isSafeRemoteUrl('https://192.0.2.1')).toBe(false); + expect(isSafeRemoteUrl('https://198.51.100.7')).toBe(false); + expect(isSafeRemoteUrl('https://203.0.113.7')).toBe(false); + expect(isSafeRemoteUrl('https://192.0.0.1')).toBe(false); + expect(isSafeRemoteUrl('https://[64:ff9b::127.0.0.1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[64:ff9b::7f00:1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[2002:7f00:1::]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[2002::ac10:1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[64:ff9b::]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[2002::]/v2')).toBe(false); + // Transition-prefix precision pins: NAT64 /96 decodes only the well-known form. + expect(isSafeRemoteUrl('https://[64:ff9b:1::7f00:1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[64:ff9b:1::102:304]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[64:ff9b:1::801:8]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[64:ff9b:0:0:0:1:102:304]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[::1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[::]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[::ffff:127.0.0.1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[::ffff:7f00:1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[::127.0.0.1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[::10.0.0.1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[fc00::1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[fe80::1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[fe80::1%25eth0]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[ff02::1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[2001:db8::1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://redis.internal')).toBe(false); + expect(isSafeRemoteUrl('https://box.local')).toBe(false); + expect(isSafeRemoteUrl('https://internal')).toBe(false); + expect(isSafeRemoteUrl('https://local')).toBe(false); +}); + +test('rejects deprecated site-local IPv6 range', () => { + expect(isSafeRemoteUrl('https://[fec0::1]/v2')).toBe(false); + expect(isSafeRemoteUrl('https://[feff::1]/v2')).toBe(false); + // fe80::/10 link-local stays rejected; fe90::/10 and later fec-range forms too + expect(isSafeRemoteUrl('https://[fe90::1]/v2')).toBe(false); +}); + +test('canonicalization tricks resolve to the checked hostname', () => { + expect(isSafeRemoteUrl('https://0x7f.0.0.1')).toBe(false); + expect(isSafeRemoteUrl('https://0177.0.0.1')).toBe(false); + expect(isSafeRemoteUrl('https://2130706433')).toBe(false); + expect(isSafeRemoteUrl('https://user@127.0.0.1/')).toBe(false); + expect(isSafeRemoteUrl('https://127.0.0.1./v2')).toBe(false); + // Octal tricks are neutralized by the URL parser itself (010.0.0.1 -> 8.0.0.1, public). + expect(isSafeRemoteUrl('https://010.0.0.1')).toBe(true); +}); + +test('expandIpv6Groups expands and validates canonical and compressed IPv6 forms', () => { + // Rejects compressed forms with zero-length runs (missing <= 0) + expect(expandIpv6Groups('1:2:3:4:5:6:7:8::')).toEqual([]); + expect(expandIpv6Groups('::1:2:3:4:5:6:7:8')).toEqual([]); + expect(expandIpv6Groups('1:2:3:4::5:6:7:8')).toEqual([]); + expect(expandIpv6Groups('1:2:3:4::5:6:7:8:9')).toEqual([]); + + // Rejects malformed or out-of-range hex groups + expect(expandIpv6Groups('2002:12345::1')).toEqual([]); + expect(expandIpv6Groups('2002:12g4::1')).toEqual([]); + expect(expandIpv6Groups('2002:-1::1')).toEqual([]); + expect(expandIpv6Groups('1:::2')).toEqual([]); + expect(expandIpv6Groups('1::2:')).toEqual([]); + expect(expandIpv6Groups(':1::2')).toEqual([]); + + // Valid forms + expect(expandIpv6Groups('1:2:3:4:5:6:7:8')).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + expect(expandIpv6Groups('2002::1')).toEqual([0x2002, 0, 0, 0, 0, 0, 0, 1]); + expect(expandIpv6Groups('2606:4700::1')).toEqual([0x2606, 0x4700, 0, 0, 0, 0, 0, 1]); + expect(expandIpv6Groups('::')).toEqual([0, 0, 0, 0, 0, 0, 0, 0]); + expect(expandIpv6Groups('::1')).toEqual([0, 0, 0, 0, 0, 0, 0, 1]); +}); diff --git a/server/utils/url-guard.ts b/server/utils/url-guard.ts new file mode 100644 index 0000000..127a869 --- /dev/null +++ b/server/utils/url-guard.ts @@ -0,0 +1,93 @@ +function ipv4Forbidden([a, b, c]: number[]): boolean { + return ( + a === 0 || a === 10 || a === 127 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 192 && b === 0 && c === 0) || + (a === 192 && b === 0 && c === 2) || + (a === 198 && b === 51 && c === 100) || + (a === 198 && (b === 18 || b === 19)) || + (a === 203 && b === 0 && c === 113) || + a >= 224 + ); +} + +/** Expand an IPv6 hostname into its eight 16-bit groups; malformed forms return an empty list. */ +export function expandIpv6Groups(host: string): number[] { + const sides = host.split('::'); + if (sides.length > 2) return []; + const head = sides[0] ? sides[0].split(':') : []; + const tail = sides.length === 2 && sides[1] ? sides[1].split(':') : []; + const missing = 8 - head.length - tail.length; + if (sides.length === 2 ? missing <= 0 : head.length !== 8) return []; + const pieces = sides.length === 2 ? [...head, ...Array(missing).fill('0'), ...tail] : head; + if (pieces.length !== 8 || !pieces.every((group) => /^[0-9a-fA-F]{1,4}$/.test(group))) return []; + return pieces.map((group) => parseInt(group, 16)); +} + +function hostAllowed(host: string): boolean { + const h = host.toLowerCase().replace(/\.$/, ''); + if (!h) return false; + if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.internal') || h.endsWith('.local')) return false; + const ipv4 = h.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); + if (ipv4) { + const octets = ipv4.slice(1).map(Number); + return !octets.some((octet) => octet > 255) && !ipv4Forbidden(octets); + } + const hostWithoutBrackets = h.replace(/^\[|\]$/g, '').split('%')[0]; + if (hostWithoutBrackets.includes(':')) { + // Leading-zero compression only exists in ::/8 — unspecified, loopback, and the + // IPv4-compatible/mapped forms (::127.0.0.1, ::ffff:127.0.0.1, and their hex forms). + if (hostWithoutBrackets.startsWith('::')) return false; + const groups = expandIpv6Groups(hostWithoutBrackets); + if (groups.length !== 8) return false; + if (groups[0] === 0x64 && groups[1] === 0xff9b) { + // 64:ff9b::/96 (well-known NAT64) embeds the IPv4 destination in groups 6..7 and + // requires groups 2..5 to be zero; every other 64:ff9b form — the local-use + // variant 64:ff9b:1::/48 included — is reserved transition space → reject. + if (groups[2] !== 0 || groups[3] !== 0 || groups[4] !== 0 || groups[5] !== 0) return false; + return !ipv4Forbidden([ + (groups[6] >> 8) & 255, + groups[6] & 255, + (groups[7] >> 8) & 255, + groups[7] & 255, + ]); + } + if (groups[0] === 0x2002) { + // 6to4 (2002::/16) embeds the IPv4 destination in bits 16..48 (groups 1..2) — + // positional reads need the full expansion, not filtered groups. + return !ipv4Forbidden([ + (groups[1] >> 8) & 255, + groups[1] & 255, + (groups[2] >> 8) & 255, + groups[2] & 255, + ]); + } + if (/^ff/.test(hostWithoutBrackets)) return false; // ff00::/8 multicast + if (/^f[cd]/.test(hostWithoutBrackets)) return false; // fc00::/7 unique-local + // fec0::/10 site-local (deprecated, still private): top 10 bits 1111111011 + if ((groups[0] & 0xffc0) === 0xfec0) return false; + if ((groups[0] & 0xffc0) === 0xfe80) return false; // fe80::/10 link-local + if (hostWithoutBrackets.startsWith('2001:db8')) return false; // documentation + return true; + } + // Single-label names resolve through private search domains in practice. + return h.includes('.'); +} + +// ponytail: hostname-only matching cannot catch DNS rebinding (a public name resolving to a +// private IP). Ceiling named; upgrade path is an async dns.lookup re-validation of the resolved +// addresses before the fetch. +/** Trust-boundary guard for server-side outbound fetches: only public HTTPS, never loopback/private/reserved hosts. */ +export function isSafeRemoteUrl(rawUrl: string): boolean { + try { + const url = new URL(rawUrl); + // ponytail: http is rejected here — API keys must never cross the network without + // TLS. Local test servers needing plaintext must bypass this guard explicitly. + return url.protocol === 'https:' && hostAllowed(url.hostname); + } catch { + return false; + } +} diff --git a/server/utils/usage-limits.test.ts b/server/utils/usage-limits.test.ts new file mode 100644 index 0000000..00be6bd --- /dev/null +++ b/server/utils/usage-limits.test.ts @@ -0,0 +1,192 @@ +import { test, expect } from 'bun:test'; +import { reserveUsage, configuredLimit } from './usage-limits'; + +test('local reservations enforce budget and reset at window boundary', async () => { + let now = 0; + const deps = { fetch, now: () => now }; + await reserveUsage('unit-local', 2, 3, 1000, deps); + await expect(reserveUsage('unit-local', 2, 3, 1000, deps)).rejects.toMatchObject({ status: 429 }); + await reserveUsage('unit-local', 1, 3, 1000, deps); + now = 1000; + await reserveUsage('unit-local', 3, 3, 1000, deps); +}); + +test('negative and non-integer amounts fail closed instead of replenishing quota', async () => { + const deps = { fetch, now: () => 0 }; + await reserveUsage('unit-nonpositive', 2, 3, 1000, deps); + for (const amount of [-1, 1.5, Number.NaN]) { + await expect(reserveUsage('unit-nonpositive', amount, 3, 1000, deps)).rejects.toMatchObject({ + code: 'INVALID_USAGE_AMOUNT', + }); + await expect( + reserveUsage('unit-nonpositive-shared', amount, 3, 1000, { + ...deps, + lookup: async () => ['93.184.216.34'], + }), + ).rejects.toMatchObject({ code: 'INVALID_USAGE_AMOUNT' }); + } + await expect(reserveUsage('unit-nonpositive', 2, 3, 1000, deps)).rejects.toMatchObject({ status: 429 }); +}); + +test('zero-byte batches reserve nothing and fail closed on negative quota instead', async () => { + const deps = { fetch, now: () => 0 }; + await reserveUsage('unit-zero', 0, 3, 1000, deps); + await reserveUsage('unit-zero', 3, 3, 1000, deps); + await expect(reserveUsage('unit-zero', 1, 3, 1000, deps)).rejects.toMatchObject({ status: 429 }); +}); + +test('invalid configured limits fail closed', () => { + process.env.UNIT_LIMIT = '-1'; + try { + expect(() => configuredLimit('UNIT_LIMIT', 100)).toThrow(); + } finally { + delete process.env.UNIT_LIMIT; + } +}); + +test('IPv6-literal limiter URLs resolve without brackets and enforce the budget', async () => { + const previous = [ + process.env.RATE_LIMIT_MODE, + process.env.UPSTASH_REDIS_REST_URL, + process.env.UPSTASH_REDIS_REST_TOKEN, + ]; + process.env.RATE_LIMIT_MODE = 'shared'; + process.env.UPSTASH_REDIS_REST_URL = 'https://[2606:4700::1]'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'unit-only'; + try { + let seenUrl: string | undefined; + let seenAddresses: string[] | undefined; + const postJson = async (args: { url: string; payload: string; addresses: string[] }) => { + seenUrl = args.url; + seenAddresses = args.addresses; + expect(JSON.parse(args.payload)[0]).toBe('EVAL'); + return { result: 0 }; + }; + // Uses the real dns.lookup default: IP literals resolve locally, no network needed. + await expect( + reserveUsage('unit-shared-v6', 1, 1, 1000, { fetch, now: () => 0, postJson }), + ).rejects.toMatchObject({ status: 429 }); + expect(seenUrl).toBe('https://[2606:4700::1]'); + expect(seenAddresses).toContain('2606:4700::1'); + } finally { + ['RATE_LIMIT_MODE', 'UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN'].forEach((key, i) => { + if (previous[i] === undefined) delete process.env[key]; + else process.env[key] = previous[i]; + }); + } +}); + +test('shared limiter fails closed when DNS resolution hangs', async () => { + const previous = [ + process.env.RATE_LIMIT_MODE, + process.env.UPSTASH_REDIS_REST_URL, + process.env.UPSTASH_REDIS_REST_TOKEN, + ]; + process.env.RATE_LIMIT_MODE = 'shared'; + process.env.UPSTASH_REDIS_REST_URL = 'https://limiter.invalid'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'unit-only'; + try { + let postCalled = false; + const postJson = async () => { + postCalled = true; + return { result: 1 }; + }; + await expect( + reserveUsage('unit-shared', 1, 2, 1000, { + fetch, + now: () => 0, + postJson, + lookup: () => new Promise(() => {}), + lookupTimeoutMs: 10, + }), + ).rejects.toMatchObject({ status: 503, code: 'LIMITER_UNAVAILABLE' }); + expect(postCalled).toBe(false); + } finally { + ['RATE_LIMIT_MODE', 'UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN'].forEach((key, i) => { + if (previous[i] === undefined) delete process.env[key]; + else process.env[key] = previous[i]; + }); + } +}); + +test('shared limiter fails closed without contacting the backend when a resolved address is unsafe', async () => { + const previous = [ + process.env.RATE_LIMIT_MODE, + process.env.UPSTASH_REDIS_REST_URL, + process.env.UPSTASH_REDIS_REST_TOKEN, + ]; + process.env.RATE_LIMIT_MODE = 'shared'; + process.env.UPSTASH_REDIS_REST_URL = 'https://limiter.invalid'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'unit-only'; + try { + let postCalled = false; + const postJson = async () => { + postCalled = true; + return { result: 1 }; + }; + await expect( + reserveUsage('unit-shared', 1, 2, 1000, { + fetch, + now: () => 0, + postJson, + lookup: async () => ['10.0.0.5', '93.184.216.34'], + }), + ).rejects.toMatchObject({ status: 503, code: 'LIMITER_UNAVAILABLE' }); + expect(postCalled).toBe(false); + } finally { + ['RATE_LIMIT_MODE', 'UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN'].forEach((key, i) => { + if (previous[i] === undefined) delete process.env[key]; + else process.env[key] = previous[i]; + }); + } +}); + +test('shared counter enforces rejection and fails closed on backend error', async () => { + const previous = [ + process.env.RATE_LIMIT_MODE, + process.env.UPSTASH_REDIS_REST_URL, + process.env.UPSTASH_REDIS_REST_TOKEN, + ]; + process.env.RATE_LIMIT_MODE = 'shared'; + process.env.UPSTASH_REDIS_REST_URL = 'https://limiter.invalid'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'unit-only'; + try { + const postJson = async (args: { url: string; token: string; payload: string; addresses: string[] }) => { + const command = JSON.parse(args.payload); + expect(command[0]).toBe('EVAL'); + expect(command[3]).toContain('unit-shared'); + expect(args.url).toBe('https://limiter.invalid'); + expect(args.token).toBe('unit-only'); + expect(args.addresses).toEqual(['93.184.216.34']); + return { result: 0 }; + }; + await expect( + reserveUsage('unit-shared', 1, 2, 1000, { + fetch, + now: () => 0, + postJson, + lookup: async () => ['93.184.216.34'], + }), + ).rejects.toMatchObject({ + status: 429, + }); + const failingPost = async () => { + throw new Error('offline'); + }; + await expect( + reserveUsage('unit-shared', 1, 2, 1000, { + fetch, + now: () => 0, + postJson: failingPost, + lookup: async () => ['93.184.216.34'], + }), + ).rejects.toMatchObject({ + status: 503, + }); + } finally { + ['RATE_LIMIT_MODE', 'UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN'].forEach((key, i) => { + if (previous[i] === undefined) delete process.env[key]; + else process.env[key] = previous[i]; + }); + } +}); diff --git a/server/utils/usage-limits.ts b/server/utils/usage-limits.ts new file mode 100644 index 0000000..2138e39 --- /dev/null +++ b/server/utils/usage-limits.ts @@ -0,0 +1,196 @@ +import { promises as dnsPromises } from 'node:dns'; +import { request as httpsRequest } from 'node:https'; +import { isIP } from 'node:net'; +import { AppError } from './errors'; +import { isSafeRemoteUrl } from './url-guard'; + +const defaultLookup = async (hostname: string): Promise => + (await dnsPromises.lookup(hostname, { all: true })).map(({ address }) => address); + +export interface PinnedPostArgs { + url: string; + token: string; + payload: string; + addresses: string[]; + timeoutMs: number; +} + +export interface UsageLimitDependencies { + fetch: typeof fetch; + now: () => number; + lookup?: (hostname: string) => Promise; + lookupTimeoutMs?: number; + /** Pinned POST seam: defaults to connecting only to validated addresses (no re-resolve). */ + postJson?: (args: PinnedPostArgs) => Promise; +} + +/** Module-level override for integration tests that go through handleApiRequest (which builds its own dependencies). Production defaults to the pinned implementation. */ +export const usageLimitTestHooks: { postJson?: (args: PinnedPostArgs) => Promise } = {}; + +/** POST JSON to a validated address with the original TLS identity — never re-resolves DNS. */ +async function postPinnedJson({ url, token, payload, addresses, timeoutMs }: PinnedPostArgs): Promise { + const target = new URL(url); + const hostname = target.hostname.replace(/^\[|\]$/g, ''); + const sni = isIP(hostname) ? undefined : hostname; + const path = `${target.pathname}${target.search}`; + // Never replay a reservation: a lost response may follow a successful charge. + const address = addresses[0]; + if (!address) throw new Error('No reachable limiter address'); + const text = await new Promise((resolve, reject) => { + const req = httpsRequest( + { + hostname: address, + servername: sni, + port: Number(target.port) || 443, + path, + method: 'POST', + headers: { + host: target.host, + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(payload), + 'accept-encoding': 'identity', + }, + signal: AbortSignal.timeout(timeoutMs), + }, + (res) => { + const status = res.statusCode ?? 503; + if (status >= 300 && status < 400) { + res.resume(); + reject(new Error('Limiter redirects are not supported')); + return; + } + const chunks: Buffer[] = []; + let bytes = 0; + res.on('data', (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 64 * 1024) { + res.destroy(new Error('Limiter response exceeds size limit')); + return; + } + chunks.push(chunk); + }); + res.on('error', reject); + res.on('end', () => { + if (status < 200 || status >= 300) { + reject(new Error('Limiter unavailable')); + return; + } + resolve(Buffer.concat(chunks).toString('utf8')); + }); + }, + ); + req.on('error', reject); + req.end(payload); + }); + return JSON.parse(text) as unknown; + +} + +const local = new Map(); +const script = `local n = tonumber(redis.call('GET', KEYS[1]) or '0') +if n + tonumber(ARGV[1]) > tonumber(ARGV[2]) then return 0 end +redis.call('INCRBY', KEYS[1], ARGV[1]) +redis.call('PEXPIRE', KEYS[1], ARGV[3]) +return 1`; + +export function configuredLimit(name: string, fallback: number): number { + const raw = process.env[name]; + if (raw === undefined) return fallback; + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < 0) + throw new AppError(503, 'Invalid usage limit configuration.', 'LIMIT_CONFIG_INVALID'); + return value; +} + +/** Atomic fixed-window reservations; failures are not refunded because providers may have charged. */ +export async function reserveUsage( + kind: string, + amount: number, + limit: number, + windowMs: number, + dependencies: UsageLimitDependencies = { fetch, now: Date.now, lookup: defaultLookup }, +): Promise { + // ponytail: negative amounts would replenish quota via INCRBY/entry.used — fail closed. + if (!Number.isSafeInteger(amount) || amount < 0) + throw new AppError(400, 'Usage amount must be a non-negative integer.', 'INVALID_USAGE_AMOUNT'); + const now = dependencies.now(); + const lookup = dependencies.lookup ?? defaultLookup; + const lookupTimeoutMs = dependencies.lookupTimeoutMs ?? 3000; + const bucket = Math.floor(now / windowMs); + const ttl = windowMs - (now % windowMs); + const key = `gitsdm:limits:${kind}:${bucket}`; + const url = process.env.UPSTASH_REDIS_REST_URL; + const token = process.env.UPSTASH_REDIS_REST_TOKEN; + const shared = process.env.RATE_LIMIT_MODE === 'shared' || !!url || !!token; + let allowed: boolean; + if (shared) { + if (!url || !token) + throw new AppError(503, 'Shared usage limiter is not configured.', 'LIMITER_UNAVAILABLE', true); + if (!url.startsWith('https://') || !isSafeRemoteUrl(url)) + throw new AppError(503, 'Shared usage limiter URL is not permitted.', 'LIMITER_UNAVAILABLE', true); + // Hostname checks cannot catch DNS rebinding: resolve and re-validate every address, + // then connect only to a validated address with the original TLS identity (no re-resolve, + // so the bearer cannot leak to a re-bound private address). + let addresses: string[]; + let lookupTimer: ReturnType | undefined; + try { + // WHATWG hostnames keep brackets on IPv6 literals; dns.lookup needs them stripped. + // dns.promises.lookup has no deadline of its own — bound it so a hung resolver + // cannot hold admission slots past the limiter's own request timeout. + addresses = await Promise.race([ + lookup(new URL(url).hostname.replace(/^\[|\]$/g, '')), + new Promise((_, rejectLookup) => { + lookupTimer = setTimeout(() => rejectLookup(new Error('DNS lookup timed out')), lookupTimeoutMs); + lookupTimer.unref?.(); + }), + ]); + } catch { + throw new AppError(503, 'Shared usage limiter is unavailable.', 'LIMITER_UNAVAILABLE', true); + } finally { + clearTimeout(lookupTimer); + } + if ( + !addresses.length || + !addresses.every((address) => + isSafeRemoteUrl(address.includes(':') ? `https://[${address}]` : `https://${address}`), + ) + ) + throw new AppError(503, 'Shared usage limiter address is not permitted.', 'LIMITER_UNAVAILABLE', true); + try { + const postJson = dependencies.postJson ?? usageLimitTestHooks.postJson ?? postPinnedJson; + const result: unknown = await postJson({ + url, + token, + payload: JSON.stringify(['EVAL', script, 1, key, amount, limit, ttl]), + addresses, + timeoutMs: 3000, + }); + if ( + !result || + typeof result !== 'object' || + !('result' in result) || + (result.result !== 0 && result.result !== 1) + ) + throw new Error('Invalid limiter response'); + allowed = result.result === 1; + } catch { + throw new AppError(503, 'Shared usage limiter is unavailable.', 'LIMITER_UNAVAILABLE', true); + } + } else { + for (const [id, entry] of local) if (entry.expires <= now) local.delete(id); + // Never evict live counters: doing so would let identity churn reset quotas. + if (!local.has(key) && local.size >= 10000) + throw new AppError(503, 'Usage limiter is at capacity. Please retry shortly.', 'LIMITER_UNAVAILABLE', true); + const entry = local.get(key) ?? { used: 0, expires: now + ttl }; + allowed = entry.used + amount <= limit; + if (allowed) { + entry.used += amount; + local.set(key, entry); + } + } + if (!allowed) + throw new AppError(429, 'Usage limit reached. Please retry after the limit resets.', 'USAGE_LIMIT_EXCEEDED', true, { + retryAfterSeconds: Math.ceil(ttl / 1000), + }); +} diff --git a/src/app/providers.tsx b/src/app/providers.tsx index d2028d2..3c8fcb3 100644 --- a/src/app/providers.tsx +++ b/src/app/providers.tsx @@ -1,14 +1,19 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; -import type { ReactNode } from 'react'; +import { useEffect, type ReactNode } from 'react'; +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { ThemeSync } from '@/components/theme/ThemeSync'; import { TooltipProvider } from '@/components/ui/tooltip'; +import { MotionConfig } from 'framer-motion'; const queryClient = new QueryClient({ defaultOptions: { queries: { staleTime: 1000 * 60 * 5, retry: (failureCount, error) => { - if (error instanceof Error && (error.message.includes('404') || error.message.toLowerCase().includes('not found'))) { + if ( + error instanceof Error && + (error.message.includes('404') || error.message.toLowerCase().includes('not found')) + ) { return false; } return failureCount < 1; @@ -18,12 +23,17 @@ const queryClient = new QueryClient({ }); export function Providers({ children }: { children: ReactNode }) { + const reducedMotion = useMotionPreference(); + useEffect(() => { + document.documentElement.dataset.reducedMotion = String(reducedMotion); + }, [reducedMotion]); return ( - - {children} - + {/* Components apply the reactive preference; Motion's built-in flag is mount-only. */} + + {children} + ); } diff --git a/src/app/router.tsx b/src/app/router.tsx index 30fac6e..22e8693 100644 --- a/src/app/router.tsx +++ b/src/app/router.tsx @@ -1,33 +1,67 @@ -import { lazy, Suspense } from 'react'; +import { lazy, Suspense, useLayoutEffect, type ComponentType } from 'react'; +import { PageMetadata } from '@/components/layout/PageMetadata'; import { BrowserRouter, Route, Routes, useLocation } from 'react-router-dom'; import { ErrorBoundary } from '@/components/ErrorBoundary'; const HomePage = lazy(() => import('@/pages/HomePage').then((module) => ({ default: module.HomePage }))); +const TermsPage = lazy(() => import('@/pages/TermsPage').then((module) => ({ default: module.TermsPage }))); const VizPage = lazy(() => import('@/pages/VizPage').then((module) => ({ default: module.VizPage }))); -const SearchPage = lazy(() => import('@/pages/SearchPage').then((module) => ({ default: module.SearchPage }))); -const NotFoundPage = lazy(() => import('@/pages/NotFoundPage').then((module) => ({ default: module.NotFoundPage }))); +const SearchPage = lazy(() => + import('@/pages/SearchPage').then((module) => ({ + default: module.SearchPage, + })), +); +const NotFoundPage = lazy(() => + import('@/pages/NotFoundPage').then((module) => ({ + default: module.NotFoundPage, + })), +); +const PrivacyPage = lazy(() => + import('@/pages/PrivacyPage').then((module) => ({ + default: module.PrivacyPage, + })), +); -function RouterContent() { +function PageScrollReset() { + const { pathname } = useLocation(); + useLayoutEffect(() => { + if (!window.location.hash) window.scrollTo({ top: 0, left: 0, behavior: 'instant' }); + }, [pathname]); + return null; +} + +function RouterContent({ initialPage, initialPath }: { initialPage: ComponentType; initialPath: string }) { const location = useLocation(); + const InitialPage = initialPage; return ( + }> + - } /> - } /> - } /> - } /> + : } /> + : } /> + : } /> + : } + /> + : } + /> + : } /> ); } -export function AppRouter() { +export function AppRouter(props: { initialPage: ComponentType; initialPath: string }) { return ( - + ); } diff --git a/src/components/contributors/ContributorsView.test.ts b/src/components/contributors/ContributorsView.test.ts index 1227bdc..684c53c 100644 --- a/src/components/contributors/ContributorsView.test.ts +++ b/src/components/contributors/ContributorsView.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { CONTRIBUTOR_TIMELINE_HEIGHT, getContributorTimelineWrapperStyle, -} from './ContributorsView'; +} from './timeline-layout'; // Responsive-height contract: the commit activity timeline renders inside a // sized wrapper and recharts' ResponsiveContainer needs a sized parent (it diff --git a/src/components/contributors/ContributorsView.tsx b/src/components/contributors/ContributorsView.tsx index a584571..a0ad4e0 100644 --- a/src/components/contributors/ContributorsView.tsx +++ b/src/components/contributors/ContributorsView.tsx @@ -1,3 +1,4 @@ +import { CONTRIBUTOR_TIMELINE_HEIGHT, getContributorTimelineWrapperStyle } from "./timeline-layout"; import { TooltipHint } from '@/components/ui/tooltip'; import { useMemo } from 'react'; import { Users, Calendar, Flame, TrendingUp, GitCommit, ExternalLink, Trophy } from 'lucide-react'; @@ -13,26 +14,6 @@ interface ContributorsViewProps { const statLabelClass = "block text-xs leading-5 text-muted-foreground font-medium"; const statValueClass = "text-2xl leading-8 font-bold text-foreground mt-0.5 truncate"; -/** - * Fixed chart height for the commit activity timeline. - * Responsive-height contract: the wrapper's min-height (see - * {@link getContributorTimelineWrapperStyle}) must match this value so the - * recharts `ResponsiveContainer` always has a sized parent (it renders - * nothing when the parent height is 0). Width stays fluid. - */ -export const CONTRIBUTOR_TIMELINE_HEIGHT = 280; - -/** - * Wrapper style for the commit activity timeline. Derived from - * {@link CONTRIBUTOR_TIMELINE_HEIGHT} (inline style — Tailwind cannot - * interpolate a TS constant into an arbitrary-value class, so a hardcoded class would - * silently drift from the chart height). Changing the constant changes - * rendering. - */ -export function getContributorTimelineWrapperStyle(): { minHeight: number } { - return { minHeight: CONTRIBUTOR_TIMELINE_HEIGHT }; -} - export function ContributorsView({ analysis, owner, repo }: ContributorsViewProps) { const { contributors, timeline } = analysis; @@ -140,7 +121,7 @@ export function ContributorsView({ analysis, owner, repo }: ContributorsViewProp

- Leaderboard & Commit Share + Leaderboard & Commit Share

{sortedContributors.map((c, idx) => { diff --git a/src/components/contributors/timeline-layout.ts b/src/components/contributors/timeline-layout.ts new file mode 100644 index 0000000..f6e37e1 --- /dev/null +++ b/src/components/contributors/timeline-layout.ts @@ -0,0 +1,20 @@ +/** + * Fixed chart height for the commit activity timeline. + * Responsive-height contract: the wrapper's min-height (see + * {@link getContributorTimelineWrapperStyle}) must match this value so the + * recharts `ResponsiveContainer` always has a sized parent (it renders + * nothing when the parent height is 0). Width stays fluid. + */ +export const CONTRIBUTOR_TIMELINE_HEIGHT = 280; + +/** + * Wrapper style for the commit activity timeline. Derived from + * {@link CONTRIBUTOR_TIMELINE_HEIGHT} (inline style — Tailwind cannot + * interpolate a TS constant into an arbitrary-value class, so a hardcoded class would + * silently drift from the chart height). Changing the constant changes + * rendering. + */ +export function getContributorTimelineWrapperStyle(): { minHeight: number } { + return { minHeight: CONTRIBUTOR_TIMELINE_HEIGHT }; +} + diff --git a/src/components/home/CapabilityGroups.tsx b/src/components/home/CapabilityGroups.tsx index 30c8df3..6a47159 100644 --- a/src/components/home/CapabilityGroups.tsx +++ b/src/components/home/CapabilityGroups.tsx @@ -44,9 +44,9 @@ export function CapabilityGroups() {

- From your first look at a repository -
to your next architectural - decision. + Explore codebase structure and dependencies +
to make architectural decisions + with confidence.

diff --git a/src/components/home/HomeGraphPreview.tsx b/src/components/home/HomeGraphPreview.tsx index cb1256b..dd78aa5 100644 --- a/src/components/home/HomeGraphPreview.tsx +++ b/src/components/home/HomeGraphPreview.tsx @@ -53,7 +53,7 @@ export function HomeGraphPreview() {
} > - Explore files and dependencies in the interactive map when JavaScript is enabled.

: + />}
Drag to explore · Scroll to zoom diff --git a/src/components/home/RepoInput.test.ts b/src/components/home/RepoInput.test.ts index efd28eb..5f3f172 100644 --- a/src/components/home/RepoInput.test.ts +++ b/src/components/home/RepoInput.test.ts @@ -4,7 +4,7 @@ import { resolveOpenRepository, resolvePresetNavigation, resolveRepoNavigation, -} from './RepoInput'; +} from './repo-navigation'; // Locks the intended preset-click behavior: presets navigate immediately to // `/owner/repo` (they don't just fill the input). These pure helpers drive diff --git a/src/components/home/RepoInput.tsx b/src/components/home/RepoInput.tsx index a9859b8..9286b31 100644 --- a/src/components/home/RepoInput.tsx +++ b/src/components/home/RepoInput.tsx @@ -1,3 +1,4 @@ +import { resolveOpenRepository, resolvePresetNavigation, type RepoNavigation } from "./repo-navigation"; import { useState, useMemo, useRef } from "react"; import { useNavigate } from "react-router-dom"; import { useQuery } from "@tanstack/react-query"; @@ -6,75 +7,12 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/Input"; import { getVisibleRepoPresets } from "@/components/home/repoPresets"; import { fetchAppConfig } from "@/lib/apiClient"; -import { parseRepoFromUrl, LAST_REPO_KEY } from "@/lib/utils"; +import { LAST_REPO_KEY } from "@/lib/utils"; interface RepoInputProps { initialUrl?: string; } -export { REPO_PRESETS as PRESETS } from "@/components/home/repoPresets"; - -export interface RepoNavigation { - owner: string; - repo: string; - route: string; - pendingUrl: string; -} - -/** Build the canonical GitHub URL for an `owner/repo` preset slug. */ -export function getPresetUrl(repo: string): string { - return `https://github.com/${repo}`; -} - -/** - * Resolve a raw input (URL or `owner/repo`) to its navigation target. - * Returns `null` when the input is not a valid GitHub repo reference. - * Pure helper so the preset-click → navigate contract is unit-testable. - */ -export function resolveRepoNavigation(value: string): RepoNavigation | null { - const trimmed = value.trim(); - const parsed = parseRepoFromUrl(trimmed); - if (!parsed) return null; - return { - owner: parsed.owner, - repo: parsed.repo, - route: `/${parsed.owner}/${parsed.repo}`, - pendingUrl: trimmed, - }; -} - -export type OpenRepositoryResult = - | { ok: true; nav: RepoNavigation } - | { ok: false; error: string }; - -/** - * Pure decision logic for opening a repository from raw input. - * Returns the navigation target or the error message to display — - * no side effects, so both the submit flow and the preset-click flow - * (`handlePreset` → `openRepository`) are unit-testable. - */ -export function resolveOpenRepository(value: string): OpenRepositoryResult { - const nav = resolveRepoNavigation(value); - if (!nav) { - return { - ok: false, - error: 'Enter a valid GitHub URL or owner/repo (e.g. facebook/react)', - }; - } - return { ok: true, nav }; -} - -/** - * Resolve a preset slug (e.g. `facebook/react`) to the navigation that a - * preset click performs. Preset clicks navigate immediately — they don't - * just fill the input — so this composes `getPresetUrl` with the same - * resolution `openRepository` uses, and is covered by interaction-contract - * tests in `RepoInput.test.ts`. - */ -export function resolvePresetNavigation(repo: string): OpenRepositoryResult { - return resolveOpenRepository(getPresetUrl(repo)); -} - export function RepoInput({ initialUrl = "" }: RepoInputProps) { const [url, setUrl] = useState(initialUrl); const [error, setError] = useState(""); diff --git a/src/components/home/repo-navigation.ts b/src/components/home/repo-navigation.ts new file mode 100644 index 0000000..8fd7fa2 --- /dev/null +++ b/src/components/home/repo-navigation.ts @@ -0,0 +1,81 @@ +import { parseRepoFromUrl } from "@/lib/utils"; +export { REPO_PRESETS as PRESETS } from "@/components/home/repoPresets"; + +export interface RepoNavigation { + owner: string; + repo: string; + route: string; + pendingUrl: string; +} + +/** Build the canonical GitHub URL for an `owner/repo` preset slug. */ +export function getPresetUrl(repo: string): string { + return `https://github.com/${repo}`; +} + +/** + * Resolve a raw input (URL or `owner/repo`) to its navigation target. + * Returns `null` when the input is not a valid GitHub repo reference. + * Pure helper so the preset-click → navigate contract is unit-testable. + */ +const GITHUB_OWNER_RE = /^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?$/; +const GITHUB_REPO_RE = /^[a-zA-Z0-9-._]+$/; + +export function isValidRepoIdentifier(owner: string, repo: string): boolean { + return ( + owner.length >= 1 && + owner.length <= 39 && + !owner.includes('--') && + !owner.startsWith('-') && + !owner.endsWith('-') && + GITHUB_OWNER_RE.test(owner) && + repo.length >= 1 && + repo.length <= 100 && + GITHUB_REPO_RE.test(repo) + ); +} + +export function resolveRepoNavigation(value: string): RepoNavigation | null { + const trimmed = value.trim(); + const parsed = parseRepoFromUrl(trimmed); + if (!parsed || !isValidRepoIdentifier(parsed.owner, parsed.repo)) return null; + return { + owner: parsed.owner, + repo: parsed.repo, + route: `/${parsed.owner}/${parsed.repo}`, + pendingUrl: trimmed, + }; +} + +export type OpenRepositoryResult = + | { ok: true; nav: RepoNavigation } + | { ok: false; error: string }; + +/** + * Pure decision logic for opening a repository from raw input. + * Returns the navigation target or the error message to display — + * no side effects, so both the submit flow and the preset-click flow + * (`handlePreset` → `openRepository`) are unit-testable. + */ +export function resolveOpenRepository(value: string): OpenRepositoryResult { + const nav = resolveRepoNavigation(value); + if (!nav) { + return { + ok: false, + error: 'Enter a valid GitHub URL or owner/repo (e.g. facebook/react)', + }; + } + return { ok: true, nav }; +} + +/** + * Resolve a preset slug (e.g. `facebook/react`) to the navigation that a + * preset click performs. Preset clicks navigate immediately — they don't + * just fill the input — so this composes `getPresetUrl` with the same + * resolution `openRepository` uses, and is covered by interaction-contract + * tests in `RepoInput.test.ts`. + */ +export function resolvePresetNavigation(repo: string): OpenRepositoryResult { + return resolveOpenRepository(getPresetUrl(repo)); +} + diff --git a/src/components/layout/InfoPageLayout.tsx b/src/components/layout/InfoPageLayout.tsx new file mode 100644 index 0000000..746c2ee --- /dev/null +++ b/src/components/layout/InfoPageLayout.tsx @@ -0,0 +1,13 @@ +import { SiteFooter } from '@/components/layout/SiteFooter'; +import type { ReactNode } from 'react'; +import { Navbar } from './Navbar'; + +export function InfoPageLayout({ children }: { children: ReactNode }) { + return ( +
+ +
{children}
+ +
+ ); +} diff --git a/src/components/layout/Navbar.tsx b/src/components/layout/Navbar.tsx index 51e09bf..f85bcdb 100644 --- a/src/components/layout/Navbar.tsx +++ b/src/components/layout/Navbar.tsx @@ -1,3 +1,4 @@ +import { prefersReducedMotion } from '@/lib/motion-preference'; import { TooltipHint } from '@/components/ui/tooltip'; import { Link, useLocation, useNavigate } from "react-router-dom"; import { ArrowUpRight, GitBranch, Moon, Sun } from "lucide-react"; @@ -15,7 +16,7 @@ export function Navbar() { const handleSectionClick = (id: string) => (event: React.MouseEvent) => { event.preventDefault(); const scrollNow = () => { - document.getElementById(id)?.scrollIntoView({ behavior: "smooth", block: "start" }); + document.getElementById(id)?.scrollIntoView({ behavior: prefersReducedMotion() ? "instant" : "smooth", block: "start" }); }; if (location.pathname !== "/") { void navigate("/"); diff --git a/src/components/layout/PageMetadata.tsx b/src/components/layout/PageMetadata.tsx new file mode 100644 index 0000000..827ae90 --- /dev/null +++ b/src/components/layout/PageMetadata.tsx @@ -0,0 +1,32 @@ +import { useEffect } from 'react'; +import { pageMetadata } from '@/lib/page-metadata'; + +export function PageMetadata({ path }: { path: string }) { + useEffect(() => { + const meta = pageMetadata(path); + document.title = meta.title; + for (const [attribute, name, value] of [ + ['name', 'description', meta.description], + ['name', 'robots', meta.robots], + ['property', 'og:title', meta.title], + ['property', 'og:description', meta.description], + ['property', 'og:url', meta.canonical ?? new URL('/' + path.replace(/^\/+/g, ''), 'https://gsdm.site').href], + ['name', 'twitter:title', meta.title], + ['name', 'twitter:description', meta.description], + ]) { + const element = + document.head.querySelector(`meta[${attribute}="${name}"]`) ?? + document.head.appendChild(document.createElement('meta')); + element.setAttribute(attribute, name); + element.setAttribute('content', value); + } + const canonical = document.head.querySelector('link[rel="canonical"]'); + if (!meta.canonical) canonical?.remove(); + else { + const link = canonical ?? document.head.appendChild(document.createElement('link')); + link.setAttribute('rel', 'canonical'); + link.setAttribute('href', meta.canonical); + } + }, [path]); + return null; +} diff --git a/src/components/layout/SiteFooter.tsx b/src/components/layout/SiteFooter.tsx new file mode 100644 index 0000000..dc0e16c --- /dev/null +++ b/src/components/layout/SiteFooter.tsx @@ -0,0 +1,43 @@ +import { Link, useLocation } from 'react-router-dom'; +import { prefersReducedMotion } from '@/lib/motion-preference'; + +const links = [ + ['Home', '/'], + ['Privacy', '/privacy'], + ['Terms', '/terms'], +] as const; + +export function SiteFooter() { + const { pathname } = useLocation(); + return ( +
+
+
+ gitSdm + © 2026 +
+ +
+
+ ); +} diff --git a/src/components/ui/GlassCard.tsx b/src/components/ui/GlassCard.tsx index 170b773..0fbbc55 100644 --- a/src/components/ui/GlassCard.tsx +++ b/src/components/ui/GlassCard.tsx @@ -1,6 +1,7 @@ import { motion } from 'framer-motion'; import type { ReactNode } from 'react'; import { cn } from '@/lib/utils'; +import { useMotionPreference } from '@/hooks/useMotionPreference'; interface GlassCardProps { children: ReactNode; @@ -10,6 +11,7 @@ interface GlassCardProps { } export function GlassCard({ children, className, hover, onClick }: GlassCardProps) { + const reducedMotion = useMotionPreference(); const classes = cn( 'glass rounded-xl p-5 shadow-glow', onClick && 'cursor-pointer', @@ -20,8 +22,10 @@ export function GlassCard({ children, className, hover, onClick }: GlassCardProp return ( {children} diff --git a/src/components/ui/GlowButton.tsx b/src/components/ui/GlowButton.tsx index a820b7f..a46fbcf 100644 --- a/src/components/ui/GlowButton.tsx +++ b/src/components/ui/GlowButton.tsx @@ -1,6 +1,7 @@ import { motion, HTMLMotionProps } from 'framer-motion'; import type { ReactNode } from 'react'; import { cn } from '@/lib/utils'; +import { useMotionPreference } from '@/hooks/useMotionPreference'; interface GlowButtonProps extends HTMLMotionProps<"button"> { children: ReactNode; @@ -16,10 +17,9 @@ export function GlowButton({ disabled, ...props }: GlowButtonProps) { + const reducedMotion = useMotionPreference(); return ( {loading && ( diff --git a/src/components/ui/badge.tsx b/src/components/ui/badge.tsx index b20959d..10d4bbe 100644 --- a/src/components/ui/badge.tsx +++ b/src/components/ui/badge.tsx @@ -49,4 +49,4 @@ function Badge({ }) } -export { Badge, badgeVariants } +export { Badge } diff --git a/src/components/ui/button-variants.ts b/src/components/ui/button-variants.ts new file mode 100644 index 0000000..88edf34 --- /dev/null +++ b/src/components/ui/button-variants.ts @@ -0,0 +1,38 @@ +import { cva } from "class-variance-authority"; +export const buttonVariants = cva( + "group/button inline-flex shrink-0 items-center justify-center rounded-lg border border-transparent bg-clip-padding text-sm font-medium whitespace-nowrap transition-all outline-none select-none focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 active:not-aria-[haspopup]:translate-y-px disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground hover:bg-primary/80", + outline: + "border-border bg-background hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:border-input dark:bg-input/30 dark:hover:bg-input/50", + secondary: + "bg-secondary text-secondary-foreground hover:bg-[color-mix(in_oklch,var(--secondary),var(--foreground)_5%)] aria-expanded:bg-secondary aria-expanded:text-secondary-foreground", + ghost: + "hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:hover:bg-muted/50", + destructive: + "bg-destructive/10 text-destructive hover:bg-destructive/20 focus-visible:border-destructive/40 focus-visible:ring-destructive/20 dark:bg-destructive/20 dark:hover:bg-destructive/30 dark:focus-visible:ring-destructive/40", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: + "h-8 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", + xs: "h-6 gap-1 rounded-[min(var(--radius-md),10px)] px-2 text-xs in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3", + sm: "h-7 gap-1 rounded-[min(var(--radius-md),12px)] px-2.5 text-[0.8rem] in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3.5", + lg: "h-9 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", + icon: "size-8", + "icon-xs": + "size-6 rounded-[min(var(--radius-md),10px)] in-data-[slot=button-group]:rounded-lg [&_svg:not([class*='size-'])]:size-3", + "icon-sm": + "size-7 rounded-[min(var(--radius-md),12px)] in-data-[slot=button-group]:rounded-lg", + "icon-lg": "size-9", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + diff --git a/src/components/ui/button.tsx b/src/components/ui/button.tsx index b033601..df64257 100644 --- a/src/components/ui/button.tsx +++ b/src/components/ui/button.tsx @@ -1,45 +1,9 @@ +import { buttonVariants } from "./button-variants"; import { Button as ButtonPrimitive } from "@base-ui/react/button" -import { cva, type VariantProps } from "class-variance-authority" +import { type VariantProps } from "class-variance-authority" import { cn } from "@/lib/utils" -const buttonVariants = cva( - "group/button inline-flex shrink-0 items-center justify-center rounded-lg border border-transparent bg-clip-padding text-sm font-medium whitespace-nowrap transition-all outline-none select-none focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 active:not-aria-[haspopup]:translate-y-px disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", - { - variants: { - variant: { - default: "bg-primary text-primary-foreground hover:bg-primary/80", - outline: - "border-border bg-background hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:border-input dark:bg-input/30 dark:hover:bg-input/50", - secondary: - "bg-secondary text-secondary-foreground hover:bg-[color-mix(in_oklch,var(--secondary),var(--foreground)_5%)] aria-expanded:bg-secondary aria-expanded:text-secondary-foreground", - ghost: - "hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:hover:bg-muted/50", - destructive: - "bg-destructive/10 text-destructive hover:bg-destructive/20 focus-visible:border-destructive/40 focus-visible:ring-destructive/20 dark:bg-destructive/20 dark:hover:bg-destructive/30 dark:focus-visible:ring-destructive/40", - link: "text-primary underline-offset-4 hover:underline", - }, - size: { - default: - "h-8 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", - xs: "h-6 gap-1 rounded-[min(var(--radius-md),10px)] px-2 text-xs in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3", - sm: "h-7 gap-1 rounded-[min(var(--radius-md),12px)] px-2.5 text-[0.8rem] in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3.5", - lg: "h-9 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", - icon: "size-8", - "icon-xs": - "size-6 rounded-[min(var(--radius-md),10px)] in-data-[slot=button-group]:rounded-lg [&_svg:not([class*='size-'])]:size-3", - "icon-sm": - "size-7 rounded-[min(var(--radius-md),12px)] in-data-[slot=button-group]:rounded-lg", - "icon-lg": "size-9", - }, - }, - defaultVariants: { - variant: "default", - size: "default", - }, - } -) - function Button({ className, variant = "default", @@ -55,4 +19,4 @@ function Button({ ) } -export { Button, buttonVariants } +export { Button } diff --git a/src/components/ui/sidebar.tsx b/src/components/ui/sidebar.tsx index 284b317..fdef713 100644 --- a/src/components/ui/sidebar.tsx +++ b/src/components/ui/sidebar.tsx @@ -717,5 +717,4 @@ export { SidebarRail, SidebarSeparator, SidebarTrigger, - useSidebar, } diff --git a/src/components/ui/tabs.tsx b/src/components/ui/tabs.tsx index 2adaeb6..8200e66 100644 --- a/src/components/ui/tabs.tsx +++ b/src/components/ui/tabs.tsx @@ -77,4 +77,4 @@ function TabsContent({ className, ...props }: TabsPrimitive.Panel.Props) { ) } -export { Tabs, TabsList, TabsTrigger, TabsContent, tabsListVariants } +export { Tabs, TabsList, TabsTrigger, TabsContent } diff --git a/src/components/viz/AIErrorCard.tsx b/src/components/viz/AIErrorCard.tsx index 19190fb..973c5ce 100644 --- a/src/components/viz/AIErrorCard.tsx +++ b/src/components/viz/AIErrorCard.tsx @@ -21,12 +21,11 @@ export function AIErrorCard({ error, message, onRetry, title = "AI Request Faile } return ( -
-
+
- +
-
+
{title}
{errorCode && ( @@ -34,13 +33,14 @@ export function AIErrorCard({ error, message, onRetry, title = "AI Request Faile )}
-

+

{displayMessage}

{onRetry && (
+ )} +
+ ); +} diff --git a/src/components/viz/DependencyHealthTab.tsx b/src/components/viz/DependencyHealthTab.tsx index bb9f5bf..2102af1 100644 --- a/src/components/viz/DependencyHealthTab.tsx +++ b/src/components/viz/DependencyHealthTab.tsx @@ -1,8 +1,20 @@ import { TooltipHint } from '@/components/ui/tooltip'; import { useState } from 'react'; -import { AlertCircle, CheckCircle2, Clock, HelpCircle, PackageX, ChevronDown, ChevronRight, RefreshCw, XCircle, Info } from 'lucide-react'; +import { + AlertCircle, + CheckCircle2, + Clock, + HelpCircle, + PackageX, + ChevronDown, + ChevronRight, + RefreshCw, + XCircle, + Info, +} from 'lucide-react'; import { cn } from '@/lib/utils'; import type { RepoAnalysis, DependencyHealthItem } from '@/types'; +import { useRepoHealth } from '@/hooks/useRepoHealth'; interface DependencyHealthTabProps { analysis: RepoAnalysis; @@ -11,7 +23,8 @@ interface DependencyHealthTabProps { type FilterType = 'all' | 'current' | 'outdated' | 'unknown' | 'unsupported'; export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { - const health = analysis.dependencyHealth; + const query = useRepoHealth(analysis.meta.owner, analysis.meta.repo, analysis.meta.sha, true); + const health = query.data; const [filter, setFilter] = useState('all'); const [expandedRows, setExpandedRows] = useState>(new Set()); @@ -19,10 +32,17 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { return (
-
- -
-

Health data not found or still analyzing...

+
+ +
+

+ {query.isError ? 'Dependency health could not be loaded.' : 'Loading dependency health…'} +

+ {query.isError && ( + + )}
); @@ -33,17 +53,17 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { if (items.length === 0) { return (
-
-
- -
-

No dependencies found in supported manifests.

+
+
+ +
+

No dependencies found in supported manifests.

); } - const filteredItems = items.filter(item => { + const filteredItems = items.filter((item) => { if (filter === 'all') return true; if (filter === 'current') return item.state === 'current'; if (filter === 'outdated') return item.state === 'outdated'; @@ -56,7 +76,7 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { }); const toggleRow = (id: string) => { - setExpandedRows(prev => { + setExpandedRows((prev) => { const next = new Set(prev); if (next.has(id)) next.delete(id); else next.add(id); @@ -70,8 +90,8 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('all')} > @@ -80,8 +100,8 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('current')} > @@ -90,8 +110,8 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('outdated')} > @@ -103,22 +123,26 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('unknown')} > -
{summary.unknown + summary.errors}
+
+ {summary.unknown + summary.errors} +
Unknown
setFilter('unsupported')} > -
{summary.unsupported}
+
+ {summary.unsupported} +
Unsupported
@@ -127,117 +151,130 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {

- Freshness checks currently support npm only. Other ecosystems are inventoried but versions remain unchecked. + Freshness checks currently support npm only. Other ecosystems are inventoried but versions + remain unchecked.

-
-

- Dependencies -

- - {filteredItems.length} {filter !== 'all' && `(${filter})`} - -
- -
- {filteredItems.map(item => { - const id = `${item.ecosystem}-${item.name}-${item.type}`; - const isExpanded = expandedRows.has(id); - return ( -
-
toggleRow(id)} - > -
- {isExpanded ? : } - -
-
{item.name}
-
- {item.ecosystem} - - {item.currentVersion || 'unknown'} - {item.latestVersion && item.latestVersion !== item.currentVersion && ( - <> - - → {item.latestVersion} - - )} -
-
-
-
- - {item.type} - - {item.manifestPaths.length > 1 && ( - - x{item.manifestPaths.length} - - )} -
-
- - {isExpanded && ( -
-
-
Manifests
-
- {item.manifestPaths.map((p, i) => ( -
{p}
 - ))} -
-
- {item.packageNames.length > 0 && item.packageNames.some(n => n !== item.name) && ( -
-
Imported By
-
- {item.packageNames.filter(n => n !== item.name).map((name, i) => ( -
{name}
 +
+

Dependencies

+ + {filteredItems.length} {filter !== 'all' && `(${filter})`} + +
+ +
+ {filteredItems.map((item) => { + const id = `${item.ecosystem}-${item.name}-${item.type}`; + const isExpanded = expandedRows.has(id); + return ( +
+
toggleRow(id)} + > +
+ {isExpanded ? ( + + ) : ( + + )} + +
+
{item.name}
+
+ {item.ecosystem} + + {item.currentVersion || 'unknown'} + {item.latestVersion && item.latestVersion !== item.currentVersion && ( + <> + + → {item.latestVersion} + + )} +
+
+
+
+ + {item.type} + + {item.manifestPaths.length > 1 && ( + + x{item.manifestPaths.length} + + )} +
+
+ + {isExpanded && ( +
+
+
Manifests
+
+ {item.manifestPaths.map((p, i) => ( + +
{p}
+ + ))} +
+
+ {item.packageNames.length > 0 && item.packageNames.some((n) => n !== item.name) && ( +
+
Imported By
+
+ {item.packageNames + .filter((n) => n !== item.name) + .map((name, i) => ( + +
{name}
+ ))} -
- )} - {item.license && ( -
-
License
-
{item.license}
-
- )} - {item.checkedAt && ( -
-
Checked
-
{new Date(item.checkedAt).toLocaleString()}
-
- )} - {item.error && ( -
-
Error
-
{item.error}
-
- )} -
- )} -
- ); - })} -
+
+ )} + {item.license && ( +
+
License
+
{item.license}
+
+ )} + {item.checkedAt && ( +
+
Checked
+
{new Date(item.checkedAt).toLocaleString()}
+
+ )} + {item.error && ( +
+
Error
+
{item.error}
+
+ )} +
+ )} +
+ ); + })} +
-
); } function StateIcon({ state }: { state: DependencyHealthItem['state'] }) { switch (state) { - case 'current': return ; - case 'outdated': return ; - case 'error': return ; - case 'unknown': return ; - default: return ; + case 'current': + return ; + case 'outdated': + return ; + case 'error': + return ; + case 'unknown': + return ; + default: + return ; } } diff --git a/src/components/viz/LearningPathTab.tsx b/src/components/viz/LearningPathTab.tsx index c243e34..a28c49a 100644 --- a/src/components/viz/LearningPathTab.tsx +++ b/src/components/viz/LearningPathTab.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useState, useMemo } from 'react'; import { useLearningPath } from '@/features/ai/useAiTasks'; import { useVizStore } from '@/stores/vizStore'; @@ -12,6 +13,7 @@ import { AIErrorCard } from './AIErrorCard'; // Decoupled subcomponents export function LearningPathTab({ analysis }: { analysis: RepoAnalysis }) { + const reducedMotion = useMotionPreference(); const { owner, repo } = analysis.meta; const selectedBranch = useVizStore((s) => s.selectedBranch); const activeFocusLayer = useVizStore((s) => s.activeFocusLayer); @@ -73,13 +75,11 @@ export function LearningPathTab({ analysis }: { analysis: RepoAnalysis }) { if (lp.isError) { return ( -
-
); } @@ -170,7 +170,7 @@ export function LearningPathTab({ analysis }: { analysis: RepoAnalysis }) { key={item.path} initial={{ opacity: 0, y: 8 }} animate={{ opacity: 1, y: 0 }} - transition={{ delay: idx * 0.05 }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { delay: idx * 0.05 }} role="group" aria-label={`Learning step ${idx + 1}: ${item.path}`} className={`group flex gap-3 items-start rounded-md border p-3 transition-all duration-200 ${ diff --git a/src/components/viz/MotionSettings.tsx b/src/components/viz/MotionSettings.tsx new file mode 100644 index 0000000..4dc2e05 --- /dev/null +++ b/src/components/viz/MotionSettings.tsx @@ -0,0 +1,28 @@ +import { useMotionStore, type MotionPreference } from '@/stores/motionStore'; + +const options: { value: MotionPreference; label: string }[] = [ + { value: 'system', label: 'System' }, + { value: 'full', label: 'Full' }, + { value: 'reduced', label: 'Reduced' }, +]; + +export function MotionSettings() { + const { preference, setPreference } = useMotionStore(); + return ( +
+ Motion +
+ {options.map(({ value, label }) => ( + + ))} +
+ + Reduce interface and graph animations. Changes are saved automatically. + +
+ ); +} diff --git a/src/components/viz/OverviewTab.tsx b/src/components/viz/OverviewTab.tsx index 9c8222d..7545805 100644 --- a/src/components/viz/OverviewTab.tsx +++ b/src/components/viz/OverviewTab.tsx @@ -2,9 +2,20 @@ import { TooltipHint } from '@/components/ui/tooltip'; import { useCallback, useMemo } from 'react'; import { useVizStore } from '@/stores/vizStore'; import type { RepoAnalysis } from '@/types'; +import { useRepoChurn } from '@/hooks/useRepoChurn'; import { - Plus, Activity, AlertTriangle, GitBranch, - FileCode, Folder, Package, Users, Code2, ShieldAlert, Info, Flame + Plus, + Activity, + AlertTriangle, + GitBranch, + FileCode, + Folder, + Package, + Users, + Code2, + ShieldAlert, + Info, + Flame, } from 'lucide-react'; const GRAPH_FILE_NODE_CAP = 1200; @@ -21,6 +32,8 @@ interface OverviewTabProps { } export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTabProps) { + const churn = useRepoChurn(analysis.meta.owner, analysis.meta.repo, analysis.meta.sha, true); + const hasChurn = analysis.graph.nodes.some((node) => node.data.churnScore != null); // Direct graph centering helper — works for both file and folder nodes const focusOnNode = useCallback((nodeId: string, filePath?: string | null) => { useVizStore.getState().setSelectedNodeId(nodeId); @@ -40,40 +53,50 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab nodeById, maintenanceHotspots, } = useMemo(() => { - const fileCount = analysis.graph.nodes.filter(n => n.type === 'file').length; - const folderCount = analysis.graph.nodes.filter(n => n.type === 'folder').length; + const fileCount = analysis.graph.nodes.filter((n) => n.type === 'file').length; + const folderCount = analysis.graph.nodes.filter((n) => n.type === 'folder').length; const depCount = analysis.dependencies.length; const contributorCount = analysis.contributors.length; // Calculate High Coupling const degrees: Record = {}; - analysis.graph.edges.forEach(e => { + analysis.graph.edges.forEach((e) => { degrees[e.source] = (degrees[e.source] || 0) + 1; degrees[e.target] = (degrees[e.target] || 0) + 1; }); const highCoupling = analysis.graph.nodes - .filter(n => n.type === 'file' || n.type === 'folder') + .filter((n) => n.type === 'file' || n.type === 'folder') .sort((a, b) => (degrees[b.id] || 0) - (degrees[a.id] || 0)) .slice(0, 5); - const entryPoints = analysis.graph.nodes - .filter(n => n.data.fileClass === 'entry') - .slice(0, 5); + const entryPoints = analysis.graph.nodes.filter((n) => n.data.fileClass === 'entry').slice(0, 5); - const nodeById = new Map(analysis.graph.nodes.map(n => [n.id, n])); + const nodeById = new Map(analysis.graph.nodes.map((n) => [n.id, n])); - // Compute Maintenance Hotspots: files with churnScore or complexityScore, - // ranked by combined score - const maintenanceHotspots = analysis.graph.nodes - .filter(n => n.type === 'file' && (n.data.churnScore != null || n.data.complexityScore != null)) - .map(n => ({ - node: n, - combinedScore: (n.data.churnScore ?? 0) * 0.5 + (n.data.complexityScore ?? 0) * 0.5, - })) - .sort((a, b) => b.combinedScore - a.combinedScore) - .slice(0, 5); + // Compute Maintenance Hotspots: files with churnScore or complexityScore, + // ranked by combined score + const maintenanceHotspots = analysis.graph.nodes + .filter((n) => n.type === 'file' && (n.data.churnScore != null || n.data.complexityScore != null)) + .map((n) => ({ + node: n, + combinedScore: + ((n.data.churnScore ?? 0) + (n.data.complexityScore ?? 0)) / + (Number(n.data.churnScore != null) + Number(n.data.complexityScore != null)), + })) + .sort((a, b) => b.combinedScore - a.combinedScore) + .slice(0, 5); - return { fileCount, folderCount, depCount, contributorCount, highCoupling, entryPoints, degrees, nodeById, maintenanceHotspots }; + return { + fileCount, + folderCount, + depCount, + contributorCount, + highCoupling, + entryPoints, + degrees, + nodeById, + maintenanceHotspots, + }; }, [analysis]); const totalFiles = analysis.totalFiles ?? fileCount; @@ -110,7 +133,9 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab Total
- {graphDiff.added.size + graphDiff.modified.size + graphDiff.deleted.size} + + {graphDiff.added.size + graphDiff.modified.size + graphDiff.deleted.size} +
@@ -121,11 +146,11 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab Added Files
- {Array.from(graphDiff.added).map(id => { + {Array.from(graphDiff.added).map((id) => { const node = nodeById.get(id); return ( -
)} - + {graphDiff.modified.size > 0 && (

Modified Files

- {Array.from(graphDiff.modified).map(id => { + {Array.from(graphDiff.modified).map((id) => { const node = nodeById.get(id); return ( - + )}
{maintenanceHotspots.map(({ node }) => ( - @@ -390,26 +463,29 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab
{analysis.timeline.length === 0 ? ( -
- No commit history available. -
+
No commit history available.
) : (
{(() => { - const maxCount = Math.max(1, Math.max(...analysis.timeline.map(w => w.count))); + const maxCount = Math.max(1, Math.max(...analysis.timeline.map((w) => w.count))); return analysis.timeline.slice(-24).map((week, idx) => { const ratio = week.count / maxCount; const heightPercentage = Math.max(10, ratio * 100); const opacity = Math.max(0.2, ratio); return ( -
+ +
+ ); }); })()} diff --git a/src/components/viz/SettingsPopover.tsx b/src/components/viz/SettingsPopover.tsx index db9c2b7..63169eb 100644 --- a/src/components/viz/SettingsPopover.tsx +++ b/src/components/viz/SettingsPopover.tsx @@ -1,15 +1,15 @@ +import { refreshChatConfig, writeChatConfigItem } from '@/stores/chatConfigStore'; +import { DropdownMenu, DropdownMenuTrigger, DropdownMenuContent, DropdownMenuRadioGroup, DropdownMenuRadioItem } from '@/components/ui/dropdown-menu'; import { useState, useRef, useEffect, useCallback, type ReactNode, type RefObject } from 'react'; -import { Settings, X, Check, Eye, EyeOff, KeyRound, GitBranch } from 'lucide-react'; +import { Settings, X, Check, Eye, EyeOff, KeyRound, GitBranch, ChevronDown } from 'lucide-react'; import { cn } from '@/lib/utils'; -import { buttonVariants } from '@/components/ui/button'; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from '@/components/ui/tooltip'; +import { MotionSettings } from './MotionSettings'; +import { buttonVariants } from '@/components/ui/button-variants'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; const GEMINI_KEY = 'gitsdm_gemini_api_key'; const PAT_KEY = 'gitsdm_github_pat'; +const chatProviders = [['', 'Auto-detect'], ['gemini', 'Google Gemini'], ['openai', 'OpenAI-compatible'], ['anthropic', 'Anthropic']] as const; function getStoredKey(key: string): string | null { try { @@ -20,10 +20,7 @@ function getStoredKey(key: string): string | null { } function setStoredKey(key: string, val: string | null) { - try { - if (val) localStorage.setItem(key, val); - else localStorage.removeItem(key); - } catch { /* ignore */ } + writeChatConfigItem(key, val); } interface SettingsPopoverProps { @@ -47,14 +44,18 @@ export function SettingsPopover({ }: SettingsPopoverProps) { const [uncontrolledOpen, setUncontrolledOpen] = useState(false); const open = controlledOpen ?? uncontrolledOpen; - const setOpen = useCallback((next: boolean | ((open: boolean) => boolean)) => { - const resolved = typeof next === 'function' ? next(open) : next; - if (controlledOpen === undefined) { - setUncontrolledOpen(resolved); - } - onOpenChange?.(resolved); - }, [controlledOpen, onOpenChange, open]); + const setOpen = useCallback( + (next: boolean | ((open: boolean) => boolean)) => { + const resolved = typeof next === 'function' ? next(open) : next; + if (controlledOpen === undefined) { + setUncontrolledOpen(resolved); + } + onOpenChange?.(resolved); + }, + [controlledOpen, onOpenChange, open], + ); const popoverRef = useRef(null); + const providerMenuRef = useRef(null); const triggerRef = useRef(null); // True opener across every close path (X button, outside click, Escape). // openerRef tracks the actual element that opened the popover (the mobile @@ -90,9 +91,7 @@ export function SettingsPopover({ // No visible popover trigger (hideTrigger mode, or the breakpoint moved // under an open popover): fall back to the visible mobile menu trigger // rather than an invisible anchor. - const menuTrigger = document.querySelector( - 'button[aria-label="Open menu"]', - ); + const menuTrigger = document.querySelector('button[aria-label="Open menu"]'); if (menuTrigger && isVisibleFocusable(menuTrigger)) { menuTrigger.focus(); return; @@ -111,8 +110,7 @@ export function SettingsPopover({ useEffect(() => { if (!open) return; - previousFocusRef.current = - document.activeElement instanceof HTMLElement ? document.activeElement : null; + previousFocusRef.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; popoverRef.current?.querySelector('[role="dialog"] input')?.focus(); function handleEscape(event: KeyboardEvent) { if (event.key !== 'Escape') return; @@ -130,6 +128,24 @@ export function SettingsPopover({ const [geminiSaved, setGeminiSaved] = useState(false); const [showGemini, setShowGemini] = useState(false); + const [chatProvider, setChatProvider] = useState(() => getStoredKey('gitsdm_ai_provider') ?? ''); + const [chatModel, setChatModel] = useState(() => getStoredKey('gitsdm_ai_model') ?? ''); + const [chatBase, setChatBase] = useState(() => getStoredKey('gitsdm_ai_base_url') ?? ''); + const [settingsError, setSettingsError] = useState(null); + + // Desktop + mobile instances both mount: a save in one leaves the other + // stale, so a breakpoint switch would overwrite with old values. Re-read + // stored keys every time the popover opens. + useEffect(() => { + if (!open) return; + setGeminiValue(getStoredKey(GEMINI_KEY) ?? ''); + setChatProvider(getStoredKey('gitsdm_ai_provider') ?? ''); + setChatModel(getStoredKey('gitsdm_ai_model') ?? ''); + setChatBase(getStoredKey('gitsdm_ai_base_url') ?? ''); + setPatValue(getStoredKey(PAT_KEY) ?? ''); + setSettingsError(null); + }, [open]); + // GitHub State const [patValue, setPatValue] = useState(() => getStoredKey(PAT_KEY) ?? ''); const [patSaved, setPatSaved] = useState(false); @@ -141,7 +157,7 @@ export function SettingsPopover({ useEffect(() => { if (!open) return; function handler(e: MouseEvent) { - if (popoverRef.current && !popoverRef.current.contains(e.target as Node)) { + if (popoverRef.current && !popoverRef.current.contains(e.target as Node) && !providerMenuRef.current?.contains(e.target as Node)) { requestClose(); } } @@ -151,20 +167,49 @@ export function SettingsPopover({ const saveGemini = useCallback(() => { const trimmed = geminiValue.trim(); + const trimmedModel = chatModel.trim(); + if (trimmedModel && (/\s/.test(trimmedModel) || !/^[\x21-\x7e]{1,200}$/.test(trimmedModel))) { + setSettingsError('Model must contain 1–200 characters without spaces.'); + return; + } + const trimmedBase = chatBase.trim(); + if (chatProvider === 'openai' && trimmedBase) { + try { + const url = new URL(trimmedBase); + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || trimmedBase.length > 2048) { + setSettingsError('Enter a public HTTPS base URL without credentials, query, or fragment.'); + return; + } + } catch { + setSettingsError('Enter a valid HTTPS base URL.'); + return; + } + } + setSettingsError(null); setStoredKey(GEMINI_KEY, trimmed || null); + setStoredKey('gitsdm_ai_provider', chatProvider || null); + setStoredKey('gitsdm_ai_model', trimmedModel || null); + setStoredKey('gitsdm_ai_base_url', chatProvider === 'openai' ? trimmedBase || null : null); + refreshChatConfig(); setGeminiSaved(true); setTimeout(() => setGeminiSaved(false), 1200); - }, [geminiValue]); + }, [geminiValue, chatProvider, chatModel, chatBase]); const clearGemini = useCallback(() => { setGeminiValue(''); + setChatProvider(''); + setChatModel(''); + setChatBase(''); + ['gitsdm_ai_provider', 'gitsdm_ai_model', 'gitsdm_ai_base_url'].forEach((key) => setStoredKey(key, null)); setStoredKey(GEMINI_KEY, null); + refreshChatConfig(); setGeminiSaved(false); }, []); const savePat = useCallback(() => { const trimmed = patValue.trim(); setStoredKey(PAT_KEY, trimmed || null); + refreshChatConfig(); setPatSaved(true); setTimeout(() => setPatSaved(false), 1200); }, [patValue]); @@ -172,6 +217,7 @@ export function SettingsPopover({ const clearPat = useCallback(() => { setPatValue(''); setStoredKey(PAT_KEY, null); + refreshChatConfig(); setPatSaved(false); }, []); @@ -189,32 +235,37 @@ export function SettingsPopover({ triggerClassName ? cn(triggerClassName, hasAnyKey && 'text-ui-active-text-green') : cn( - buttonVariants({ variant: "outline", size: "sm" }), - "h-7 w-7 rounded-md p-0 border-border bg-secondary text-muted-foreground hover:text-foreground hover:bg-secondary hover:border-ring/50 transition-all duration-150 relative", - hasAnyKey && 'border-accent/30 text-accent bg-accent/10' + buttonVariants({ variant: 'outline', size: 'sm' }), + 'h-7 w-7 rounded-md p-0 border-border bg-secondary text-muted-foreground hover:text-foreground hover:bg-secondary hover:border-ring/50 transition-all duration-150 relative', + hasAnyKey && 'border-accent/30 text-accent bg-accent/10', ) } onClick={() => setOpen((o) => !o)} > {triggerChildren ?? ( <> - + {hasAnyKey && ( )} )} - Settings & Credentials + Settings & Credentials )} {open && ( -
{/* Header */}
- Credentials + + Credentials +
setGeminiValue(e.target.value)} - placeholder="AIzaSy..." + placeholder="API key" className="w-full rounded-md border border-border bg-background py-1.5 pl-3 pr-8 font-mono text-xs text-foreground placeholder:text-muted-foreground outline-none focus:border-accent focus:ring-1 focus:ring-accent/50 transition-all" />
@@ -133,7 +135,7 @@ export function StagedLoader({ owner, repo }: StagedLoaderProps) { initial={{ opacity: 0, y: 6 }} animate={{ opacity: 1, y: 0 }} exit={{ opacity: 0, y: -6 }} - transition={{ duration: 0.25 }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { duration: 0.25 }} className="mb-5 flex items-center gap-3 rounded-xl border border-ui-active/35 bg-ui-active/10 px-3.5 py-3" >
@@ -159,7 +161,7 @@ export function StagedLoader({ owner, repo }: StagedLoaderProps) { key={stage.id} initial={{ opacity: 0 }} animate={{ opacity: 1 }} - transition={{ delay: i * 0.07 }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { delay: i * 0.07 }} className={`flex items-center gap-2.5 rounded-lg px-2.5 py-2 transition-colors ${isActive ? 'bg-secondary' : '' }`} > diff --git a/src/components/viz/VizError.tsx b/src/components/viz/VizError.tsx index 0048997..59da88e 100644 --- a/src/components/viz/VizError.tsx +++ b/src/components/viz/VizError.tsx @@ -1,8 +1,10 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useState } from 'react'; import { motion } from 'framer-motion'; import { Link } from 'react-router-dom'; import { AlertCircle, Lock, Wifi, ArrowLeft, RefreshCw, GitBranch, Check, Eye, EyeOff } from 'lucide-react'; import { GlowButton } from '@/components/ui/GlowButton'; +import { writeChatConfigItem } from '@/stores/chatConfigStore'; const LS_KEY = 'gitsdm_github_pat'; @@ -15,10 +17,7 @@ function getStoredPat(): string | null { } function setStoredPat(token: string | null) { - try { - if (token) localStorage.setItem(LS_KEY, token); - else localStorage.removeItem(LS_KEY); - } catch { /* ignore */ } + writeChatConfigItem(LS_KEY, token); } @@ -76,6 +75,7 @@ function getErrorMeta(errStr: string, code?: string): { } export function VizError({ error, message }: VizErrorProps) { + const reducedMotion = useMotionPreference(); let errStr = message || 'An unexpected analysis error occurred'; let errCode = ''; @@ -117,7 +117,7 @@ export function VizError({ error, message }: VizErrorProps) {
@@ -145,7 +145,7 @@ export function VizError({ error, message }: VizErrorProps) {

Add a{' '} {/* Intelligence Response Card */} - + ) : {refactorData?.suggestions && refactorData.suggestions.length > 0 ? (

-

Key Refactoring Suggestions & Risks

+

Key Refactoring Suggestions & Risks

{refactorData.suggestions.map((s, idx) => ( Select a node in the graph or use the actions below to generate architectural analysis. ) )} - + } {/* Action Tool Cards sections */}
@@ -289,12 +284,8 @@ export function AiCenterTab({ analysis }: AiCenterTabProps) { onClick={() => { setAiSubTab('health'); setHealthSubMode('audit'); - if (!healthData && !health.isPending && !pendingToolRequests.has(healthKey)) { - pendingToolRequests.add(healthKey); - health.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => healthCache.set(healthKey, data), - onSettled: () => pendingToolRequests.delete(healthKey), - }); + if (!healthData && !health.isPending) { + health.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } }} /> @@ -307,12 +298,8 @@ export function AiCenterTab({ analysis }: AiCenterTabProps) { onClick={() => { setAiSubTab('health'); setHealthSubMode('risks'); - if (!refactorData && !refactor.isPending && !pendingToolRequests.has(refactorKey)) { - pendingToolRequests.add(refactorKey); - refactor.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => refactorCache.set(refactorKey, data), - onSettled: () => pendingToolRequests.delete(refactorKey), - }); + if (!refactorData && !refactor.isPending) { + refactor.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } }} /> @@ -331,12 +318,8 @@ export function AiCenterTab({ analysis }: AiCenterTabProps) { onClick={() => { setAiSubTab('playground'); setActivePlayground('roast'); - if (!roastData && !roast.isPending && !pendingToolRequests.has(roastKey)) { - pendingToolRequests.add(roastKey); - roast.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => roastCache.set(roastKey, data), - onSettled: () => pendingToolRequests.delete(roastKey), - }); + if (!roastData && !roast.isPending) { + roast.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } }} /> @@ -349,12 +332,8 @@ export function AiCenterTab({ analysis }: AiCenterTabProps) { onClick={() => { setAiSubTab('playground'); setActivePlayground('readme'); - if (!readmeEnhanceData && !readmeEnhance.isPending && !pendingToolRequests.has(readmeEnhanceKey)) { - pendingToolRequests.add(readmeEnhanceKey); - readmeEnhance.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => readmeEnhanceCache.set(readmeEnhanceKey, data), - onSettled: () => pendingToolRequests.delete(readmeEnhanceKey), - }); + if (!readmeEnhanceData && !readmeEnhance.isPending) { + readmeEnhance.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } }} /> diff --git a/src/components/viz/ai-sidebar/hooks/useAiCenterState.ts b/src/components/viz/ai-sidebar/hooks/useAiCenterState.ts index 1a3a7ef..784b1ea 100644 --- a/src/components/viz/ai-sidebar/hooks/useAiCenterState.ts +++ b/src/components/viz/ai-sidebar/hooks/useAiCenterState.ts @@ -1,25 +1,22 @@ +import { + explanationCache, + runExplanation, + healthCache, + refactorCache, + roastCache, + readmeEnhanceCache, + getToolKey, +} from '@/features/ai/tool-cache'; import { useEffect, useState, useRef, useMemo } from 'react'; +import { useChatConfigRevision } from '@/stores/chatConfigStore'; import { useVizStore } from '@/stores/vizStore'; import { useHealth, useRefactor, useRoast, useReadmeEnhance } from '@/features/ai/useAiTasks'; import { aiExplain, aiExplainLif } from '@/lib/apiClient'; -import type { - AIHealthResponse, - AIReadmeEnhanceResponse, - AIRefactorResponse, - AIRoastResponse, - RepoAnalysis, -} from '@/types'; - -// Persistent layout states & caches across tab remounts -export const explanationCache = new Map(); -export const pendingExplanationRequests = new Map>(); -export const pendingToolRequests = new Set(); -export const healthCache = new Map(); -export const refactorCache = new Map(); -export const roastCache = new Map(); -export const readmeEnhanceCache = new Map(); +import type { RepoAnalysis } from '@/types'; + +export { explanationCache }; let persistedEli5Mode = false; let persistedActivePlayground: 'roast' | 'readme' = 'roast'; @@ -27,6 +24,7 @@ let persistedAiSubTab: 'explain' | 'health' | 'playground' = 'explain'; let persistedHealthSubMode: 'audit' | 'risks' = 'audit'; export function useAiCenterState(analysis: RepoAnalysis) { + const { revision } = useChatConfigRevision(); const { sidebarTab, selectedNodeId, @@ -37,11 +35,12 @@ export function useAiCenterState(analysis: RepoAnalysis) { const [activePlayground, setActivePlaygroundState] = useState<'roast' | 'readme'>(persistedActivePlayground); const [aiSubTab, setAiSubTabState] = useState<'explain' | 'health' | 'playground'>(persistedAiSubTab); const [healthSubMode, setHealthSubModeState] = useState<'audit' | 'risks'>(persistedHealthSubMode); - const [cachedExplanation, setCachedExplanation] = useState(null); + const [cachedExplanation, setCachedExplanation] = useState<{ key: string; value: string } | null>(null); const [loadingExplanationKey, setLoadingExplanationKey] = useState(null); const [readmeCopied, setReadmeCopied] = useState(false); const lastExplanationKeyRef = useRef(null); + const setActivePlayground = (mode: 'roast' | 'readme') => { persistedActivePlayground = mode; setActivePlaygroundState(mode); @@ -57,26 +56,43 @@ export function useAiCenterState(analysis: RepoAnalysis) { setHealthSubModeState(mode); }; + const [explanationErrors, setExplanationErrors] = useState>({}); + const [explanationRetry, setExplanationRetry] = useState(0); const health = useHealth(); const refactor = useRefactor(); const roast = useRoast(); const readmeEnhance = useReadmeEnhance(); - const currentExplanationKey = `${analysis.meta.owner}/${analysis.meta.repo}/${selectedNodeId ?? 'repo'}/${eli5Mode ? 'eli5' : 'normal'}/${selectedBranch ?? 'default'}`; - const currentExplanation = explanationCache.get(currentExplanationKey) ?? cachedExplanation; + const currentExplanationKey = `${revision}/${analysis.meta.owner}/${analysis.meta.repo}/${analysis.meta.sha}/${selectedNodeId ?? 'repo'}/${eli5Mode ? 'eli5' : 'normal'}/${selectedBranch ?? 'default'}`; + const currentExplanation = explanationCache.get(currentExplanationKey) ?? (cachedExplanation?.key === currentExplanationKey ? cachedExplanation.value : null); const isExplainLoading = !currentExplanation && loadingExplanationKey === currentExplanationKey; - const { owner, repo } = analysis.meta; - const branchKey = selectedBranch ?? 'default'; - const healthKey = `health:${owner}/${repo}/${branchKey}`; - const refactorKey = `refactor:${owner}/${repo}/${branchKey}`; - const roastKey = `roast:${owner}/${repo}/${branchKey}`; - const readmeEnhanceKey = `readme-enhance:${owner}/${repo}/${branchKey}`; - - const healthData = health.data ?? healthCache.get(healthKey); - const refactorData = refactor.data ?? refactorCache.get(refactorKey); - const roastData = roast.data ?? roastCache.get(roastKey); - const readmeEnhanceData = readmeEnhance.data ?? readmeEnhanceCache.get(readmeEnhanceKey); + const { owner, repo, sha } = analysis.meta; + // Finished mutations keep their data truthy across config revisions, which + // would block a fresh request keyed for the new revision — reset on both changes. + const prevMetaRef = useRef(`${sha}/${revision}`); + useEffect(() => { + const metaKey = `${sha}/${revision}`; + if (prevMetaRef.current === metaKey) return; + prevMetaRef.current = metaKey; + health.reset(); + refactor.reset(); + roast.reset(); + readmeEnhance.reset(); + }, [sha, revision, health, refactor, roast, readmeEnhance]); + + const healthData = + (health.data?.revision === revision ? health.data : undefined) ?? + healthCache.get(getToolKey('health', owner, repo, revision, selectedBranch, sha)); + const refactorData = + (refactor.data?.revision === revision ? refactor.data : undefined) ?? + refactorCache.get(getToolKey('refactor', owner, repo, revision, selectedBranch, sha)); + const roastData = + (roast.data?.revision === revision ? roast.data : undefined) ?? + roastCache.get(getToolKey('roast', owner, repo, revision, selectedBranch, sha)); + const readmeEnhanceData = + (readmeEnhance.data?.revision === revision ? readmeEnhance.data : undefined) ?? + readmeEnhanceCache.get(getToolKey('readme-enhance', owner, repo, revision, selectedBranch, sha)); const nodeById = useMemo( () => new Map(analysis.graph.nodes.map((n) => [n.id, n])), @@ -95,7 +111,7 @@ export function useAiCenterState(analysis: RepoAnalysis) { if (lastExplanationKeyRef.current !== currentKey) { const cached = explanationCache.get(currentKey); - setCachedExplanation(cached ?? null); + setCachedExplanation(cached ? { key: currentKey, value: cached } : null); if (cached) { lastExplanationKeyRef.current = currentKey; return; @@ -106,9 +122,8 @@ export function useAiCenterState(analysis: RepoAnalysis) { const loadExplanation = async () => { try { - let request = pendingExplanationRequests.get(currentKey); - if (!request) { - request = (!selectedNodeId && eli5Mode + const explanation = await runExplanation(currentKey, () => + (!selectedNodeId && eli5Mode ? aiExplainLif(owner, repo) : aiExplain({ owner, @@ -118,82 +133,77 @@ export function useAiCenterState(analysis: RepoAnalysis) { branch: selectedBranch || undefined, eli5: eli5Mode, }) - ).then((data) => data.explanation); - pendingExplanationRequests.set(currentKey, request); - } + ).then((data) => data.explanation) + ); - const explanation = await request; - explanationCache.set(currentKey, explanation); - setCachedExplanation(explanation); + setCachedExplanation({ key: currentKey, value: explanation }); + setExplanationErrors((errors) => ({ ...errors, [currentKey]: null })); + } catch { + setExplanationErrors((errors) => ({ ...errors, [currentKey]: 'AI explanation failed. Please try again.' })); } finally { - pendingExplanationRequests.delete(currentKey); setLoadingExplanationKey((key) => key === currentKey ? null : key); } }; void loadExplanation(); } - }, [sidebarTab, aiSubTab, owner, repo, selectedNodeId, eli5Mode, selectedBranch, currentExplanationKey]); + }, [sidebarTab, aiSubTab, owner, repo, selectedNodeId, eli5Mode, selectedBranch, currentExplanationKey, explanationRetry]); // Trigger selected health module only useEffect(() => { if (sidebarTab !== 'ai' || aiSubTab !== 'health') return; - if (healthSubMode === 'audit' && !healthData && !health.isPending && !pendingToolRequests.has(healthKey)) { - pendingToolRequests.add(healthKey); - health.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => healthCache.set(healthKey, data), - onSettled: () => pendingToolRequests.delete(healthKey), - }); + if (healthSubMode === 'audit' && !healthData && !health.isPending && !health.isError) { + health.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } - if (healthSubMode === 'risks' && !refactorData && !refactor.isPending && !pendingToolRequests.has(refactorKey)) { - pendingToolRequests.add(refactorKey); - refactor.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => refactorCache.set(refactorKey, data), - onSettled: () => pendingToolRequests.delete(refactorKey), - }); + if (healthSubMode === 'risks' && !refactorData && !refactor.isPending && !refactor.isError) { + refactor.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } - }, [sidebarTab, aiSubTab, healthSubMode, owner, repo, selectedBranch, health, refactor, healthKey, refactorKey, healthData, refactorData]); + }, [sidebarTab, aiSubTab, healthSubMode, owner, repo, selectedBranch, health, refactor, healthData, refactorData, sha]); // Trigger playground modules useEffect(() => { if (sidebarTab === 'ai' && aiSubTab === 'playground') { - if (activePlayground === 'roast' && !roastData && !roast.isPending && !pendingToolRequests.has(roastKey)) { - pendingToolRequests.add(roastKey); - roast.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => roastCache.set(roastKey, data), - onSettled: () => pendingToolRequests.delete(roastKey), - }); - } else if (activePlayground === 'readme' && !readmeEnhanceData && !readmeEnhance.isPending && !pendingToolRequests.has(readmeEnhanceKey)) { - pendingToolRequests.add(readmeEnhanceKey); - readmeEnhance.mutate({ owner, repo, branch: selectedBranch || undefined }, { - onSuccess: (data) => readmeEnhanceCache.set(readmeEnhanceKey, data), - onSettled: () => pendingToolRequests.delete(readmeEnhanceKey), - }); + if (activePlayground === 'roast' && !roastData && !roast.isPending && !roast.isError) { + roast.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); + } else if (activePlayground === 'readme' && !readmeEnhanceData && !readmeEnhance.isPending && !readmeEnhance.isError) { + readmeEnhance.mutate({ owner, repo, branch: selectedBranch || undefined, sha }); } } - }, [sidebarTab, aiSubTab, activePlayground, owner, repo, selectedBranch, roast, readmeEnhance, roastKey, readmeEnhanceKey, roastData, readmeEnhanceData]); + }, [sidebarTab, aiSubTab, activePlayground, owner, repo, selectedBranch, roast, readmeEnhance, roastData, readmeEnhanceData, sha]); // Resolve headers dynamically for IntelligenceCard - const cardTitle = aiSubTab === 'health' + const cardTitle = aiSubTab === 'health' ? 'Diagnostics' : aiSubTab === 'playground' ? 'Playground' : 'Intelligence'; - const cardSubtitle = aiSubTab === 'health' - ? (healthSubMode === 'audit' ? 'System Health Audit' : 'Critical Risk Analysis') + const cardSubtitle = aiSubTab === 'health' + ? (healthSubMode === 'audit' ? 'System Health Audit' : 'Critical Risk Analysis') : aiSubTab === 'playground' ? (activePlayground === 'roast' ? 'Project Roaster' : 'Documentation Enhancer') : (selectedNode ? selectedNode.data.label : 'Architectural Overview'); - const cardLoading = - (aiSubTab === 'explain' && isExplainLoading) || - (aiSubTab === 'health' && healthSubMode === 'audit' && health.isPending && !healthData) || - (aiSubTab === 'health' && healthSubMode === 'risks' && refactor.isPending && !refactorData) || - (aiSubTab === 'playground' && activePlayground === 'roast' && roast.isPending && !roastData) || - (aiSubTab === 'playground' && activePlayground === 'readme' && readmeEnhance.isPending && !readmeEnhanceData); + const activeMutation = aiSubTab === 'health' + ? (healthSubMode === 'audit' ? health : refactor) + : aiSubTab === 'playground' ? (activePlayground === 'roast' ? roast : readmeEnhance) : null; + const activeData = aiSubTab === 'health' + ? (healthSubMode === 'audit' ? healthData : refactorData) + : aiSubTab === 'playground' ? (activePlayground === 'roast' ? roastData : readmeEnhanceData) : null; + const cardLoading = aiSubTab === 'explain' ? isExplainLoading : Boolean(activeMutation?.isPending && !activeData); + + const cardError = aiSubTab === 'explain' + ? explanationErrors[currentExplanationKey] ?? null + : activeMutation?.isError ? 'AI request failed. Check your settings and try again.' : null; + const retryCard = () => { + if (aiSubTab === 'explain') { + lastExplanationKeyRef.current = ''; + setExplanationErrors((errors) => ({ ...errors, [currentExplanationKey]: null })); + setExplanationRetry(value => value + 1); + } else activeMutation?.reset(); + }; const toggleEli5Mode = () => { const nextEli5Mode = !eli5Mode; @@ -217,6 +227,8 @@ export function useAiCenterState(analysis: RepoAnalysis) { cardTitle, cardSubtitle, cardLoading, + cardError, + retryCard, readmeCopied, setReadmeCopied, healthData, @@ -228,10 +240,5 @@ export function useAiCenterState(analysis: RepoAnalysis) { refactor, roast, readmeEnhance, - healthKey, - refactorKey, - roastKey, - readmeEnhanceKey, - pendingToolRequests, }; } diff --git a/src/components/viz/analysis-relations.ts b/src/components/viz/analysis-relations.ts new file mode 100644 index 0000000..d7952ff --- /dev/null +++ b/src/components/viz/analysis-relations.ts @@ -0,0 +1,42 @@ +import type { GraphData, GraphEdge } from "@/types"; +/** + * Pure relation logic for the Analysis tab (testable without rendering). + * + * Matches the tested `getFileContext` semantics for files: excludes + * `contains` + self-loops, dedups by opposite endpoint, and drops edges + * whose opposite endpoint is missing from the graph. For non-file nodes + * (folder/repo/package) containment edges ARE the relations, so they are + * included instead of excluded. + */ +export function getAnalysisRelations(graph: GraphData, selectedNodeId: string | null): { + outgoing: GraphEdge[]; + incoming: GraphEdge[]; +} { + if (!selectedNodeId) return { outgoing: [], incoming: [] }; + const nodeById = new Map(graph.nodes.map((n) => [n.id, n])); + const selected = nodeById.get(selectedNodeId); + if (!selected) return { outgoing: [], incoming: [] }; + const isFile = selected.type === 'file'; + const rawOutgoing = graph.edges.filter( + (edge) => + edge.source === selected.id && + edge.source !== edge.target && + (isFile ? edge.type !== 'contains' : true) && + nodeById.has(edge.target), + ); + const rawIncoming = graph.edges.filter( + (edge) => + edge.target === selected.id && + edge.source !== edge.target && + (isFile ? edge.type !== 'contains' : true) && + nodeById.has(edge.source), + ); + const outgoing = rawOutgoing.filter( + (edge, index) => rawOutgoing.findIndex((e) => e.target === edge.target) === index, + ); + const incoming = rawIncoming.filter( + (edge, index) => rawIncoming.findIndex((e) => e.source === edge.source) === index, + ); + return { outgoing, incoming }; +} + diff --git a/src/components/viz/architecture/hooks/useArchitectureState.ts b/src/components/viz/architecture/hooks/useArchitectureState.ts index 0c62230..027a6f9 100644 --- a/src/components/viz/architecture/hooks/useArchitectureState.ts +++ b/src/components/viz/architecture/hooks/useArchitectureState.ts @@ -6,6 +6,7 @@ import { ensureMermaidConfigured } from '../mermaid-config'; import { stripMermaidFences } from '../stripMermaidFences'; import { createRenderSequence, type RenderSequence } from './render-sequence'; import { useVizStore } from '@/stores/vizStore'; +import { useChatConfigRevision } from '@/stores/chatConfigStore'; export function useArchitectureState( analysis: RepoAnalysis, @@ -13,7 +14,11 @@ export function useArchitectureState( repo: string, resetView: () => void ) { - const { mutate: generate, data, isPending, isError, error } = useMermaid(); + const { revision } = useChatConfigRevision(); + const branch = useVizStore((state) => state.selectedBranch) ?? undefined; + const sha = analysis.meta.sha; + const { mutate: generate, data: response, isPending, isError, error } = useMermaid(); + const data = response?.revision === revision && response.sha === sha && response.branch === branch ? response : undefined; const [svg, setSvg] = useState(''); const [renderError, setRenderError] = useState(null); const [mode, setMode] = useState<'code' | 'ai'>('code'); @@ -28,9 +33,9 @@ export function useArchitectureState( useEffect(() => { if (mode === 'ai') { - generate({ owner, repo }); + generate({ owner, repo, branch, sha }); } - }, [mode, owner, repo, generate]); + }, [mode, owner, repo, generate, revision, branch, sha]); useEffect(() => { if (mode === 'code' && !analysis) return; @@ -88,12 +93,12 @@ export function useArchitectureState( }, [mode, data, analysis, resetView, theme, renderSequence]); return { - generate, + generate: () => generate({ owner, repo, branch, sha }), data, isPending, isError, error, - svg, + svg: mode === 'ai' && !data ? '' : svg, renderError, mode, setMode, diff --git a/src/components/viz/top-nav/BranchSwitcher.tsx b/src/components/viz/top-nav/BranchSwitcher.tsx index 648c9c0..68658c5 100644 --- a/src/components/viz/top-nav/BranchSwitcher.tsx +++ b/src/components/viz/top-nav/BranchSwitcher.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useState, useRef, useEffect } from 'react'; import { useVizStore } from '@/stores/vizStore'; import { useRepoBranches } from '@/hooks/useRepoBranches'; @@ -22,6 +23,7 @@ interface BranchSwitcherProps { } export function BranchSwitcher({ owner, repo, defaultBranch }: BranchSwitcherProps) { + const reducedMotion = useMotionPreference(); const { selectedBranch, setSelectedBranch, @@ -195,7 +197,7 @@ export function BranchSwitcher({ owner, repo, defaultBranch }: BranchSwitcherPro initial={{ opacity: 0, y: 8, scale: 0.97 }} animate={{ opacity: 1, y: 0, scale: 1 }} exit={{ opacity: 0, y: 8, scale: 0.97 }} - transition={{ duration: 0.12, ease: 'easeOut' }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { duration: 0.12, ease: 'easeOut' }} className="fixed left-2 right-2 top-14 z-[100] max-h-[calc(100dvh-4rem)] rounded-md border border-border bg-card p-1.5 shadow-2xl sm:absolute sm:left-0 sm:right-auto sm:top-auto sm:mt-2 sm:w-[min(18rem,calc(100vw-1rem))]" > { setActiveView(id as 'graph' | 'architecture' | 'contributors' | 'commits'); closeMenu(); }} onWorkspaceModeChange={(id) => { - setWorkspaceMode(id as 'focus' | 'analysis' | 'learning' | 'full'); + setWorkspaceMode(id); closeMenu(); }} onClose={closeMenu} diff --git a/src/components/viz/top-nav/workspaceModes.ts b/src/components/viz/top-nav/workspaceModes.ts index 866c5b4..f5185cc 100644 --- a/src/components/viz/top-nav/workspaceModes.ts +++ b/src/components/viz/top-nav/workspaceModes.ts @@ -1,7 +1,7 @@ -import { LayoutPanelLeft, Monitor, ActivitySquare, BookOpen } from 'lucide-react'; +import { LayoutPanelLeft, PanelLeft, PanelRight, Monitor } from 'lucide-react'; export interface WorkspaceMode { - id: 'full' | 'focus' | 'analysis' | 'learning'; + id: 'full' | 'explorer' | 'insights' | 'focus'; label: string; description: string; icon: React.ComponentType<{ className?: string }>; @@ -19,30 +19,30 @@ export const WORKSPACE_MODES: WorkspaceMode[] = [ bgColor: 'bg-secondary border-border text-foreground', }, { - id: 'focus', - label: 'Focus Mode', - description: 'Minimizes sidebars to focus purely on the canvas', - icon: Monitor, + id: 'explorer', + label: 'Explorer Only', + description: 'Show the file explorer with maximized canvas space', + icon: PanelLeft, color: 'text-foreground', bgColor: 'bg-secondary border-border text-foreground', }, { - id: 'analysis', - label: 'Analysis Mode', - description: 'Inspect file details and relationships', - icon: ActivitySquare, + id: 'insights', + label: 'Insights Only', + description: 'Show repository insights without the file tree', + icon: PanelRight, color: 'text-foreground', bgColor: 'bg-secondary border-border text-foreground', }, { - id: 'learning', - label: 'Learning Mode', - description: 'Follow a guided reading path through the codebase', - icon: BookOpen, + id: 'focus', + label: 'Focus Mode', + description: 'Minimizes sidebars to focus purely on the canvas', + icon: Monitor, color: 'text-foreground', bgColor: 'bg-secondary border-border text-foreground', }, ]; -export const LAYOUT_MODE_IDS = ['full', 'focus', 'analysis', 'learning'] as const; +export const LAYOUT_MODE_IDS = ['full', 'explorer', 'insights', 'focus'] as const; export type WorkspaceModeId = (typeof LAYOUT_MODE_IDS)[number]; diff --git a/src/features/ai/tool-cache.test.ts b/src/features/ai/tool-cache.test.ts new file mode 100644 index 0000000..6951f58 --- /dev/null +++ b/src/features/ai/tool-cache.test.ts @@ -0,0 +1,60 @@ +import { test, expect } from 'bun:test'; +import { MutationObserver, QueryClient } from '@tanstack/query-core'; +import { createCachedTask, getToolKey, mermaidCache, runMermaid } from './tool-cache'; + +test('tool completion survives unmount and a new observer joins the pending work', async () => { + const cache = new Map(); + const run = createCachedTask(cache); + let release = (_value: string) => {}; + const held = new Promise((resolve) => { release = resolve; }); + let calls = 0; + const options = { mutationFn: () => run('branch-a', () => { calls++; return held; }) }; + const client = new QueryClient(); + const first = new MutationObserver(client, options); + const unsubscribe = first.subscribe(() => {}); + const request = first.mutate(); + unsubscribe(); + const second = new MutationObserver(client, options); + const joined = second.mutate(); + release('complete'); + expect(await request).toBe('complete'); + expect(await joined).toBe('complete'); + expect(cache.get('branch-a')).toBe('complete'); + expect(calls).toBe(1); +}); + +test('failed work releases its key for retry', async () => { + const run = createCachedTask(new Map()); + await expect(run('a', async () => { throw new Error('failed'); })).rejects.toThrow('failed'); + expect(await run('a', async () => 'retried')).toBe('retried'); +}); + +test('max <= 0 does not store entries in cache', async () => { + const cache = new Map(); + const run = createCachedTask(cache, 0); + const result = await run('key-1', async () => 'data-1'); + expect(result).toBe('data-1'); + expect(cache.size).toBe(0); +}); + +test('runMermaid shares pending work and caches by key', async () => { + const key = getToolKey('mermaid', 'o', 'mermaid-dedupe', 7); + let calls = 0; + const first = runMermaid(key, async () => { calls++; return { diagram: 'graph TD', cached: false }; }); + const second = runMermaid(key, async () => { calls++; return { diagram: 'graph TD', cached: false }; }); + expect(await first).toEqual(await second); + expect(calls).toBe(1); + expect(mermaidCache.get(key)).toEqual({ diagram: 'graph TD', cached: false }); +}); +test('a branch literally named default does not collide with an omitted branch', () => { + expect(getToolKey('refactor', 'o', 'r', 1, 'default')).not.toBe(getToolKey('refactor', 'o', 'r', 1)); + expect(getToolKey('refactor', 'o', 'r', 1, undefined)).toBe(getToolKey('refactor', 'o', 'r', 1, null)); +}); + +test('Mermaid results are separate for different commits on the same branch', async () => { + const first = getToolKey('mermaid', 'o', 'snapshot-test', 1, 'main', 'sha-a'); + const second = getToolKey('mermaid', 'o', 'snapshot-test', 1, 'main', 'sha-b'); + await runMermaid(first, async () => ({ diagram: 'graph TD; A', cached: false })); + expect(await runMermaid(second, async () => ({ diagram: 'graph TD; B', cached: false }))) + .toMatchObject({ diagram: 'graph TD; B' }); +}); diff --git a/src/features/ai/tool-cache.ts b/src/features/ai/tool-cache.ts new file mode 100644 index 0000000..d2dd9d5 --- /dev/null +++ b/src/features/ai/tool-cache.ts @@ -0,0 +1,46 @@ +// ponytail: in-memory promise/result cache covers tab switching; upgrade to persistent DB cache if offline AI replay is needed. +import type { AIHealthResponse, AIMermaidResponse, AIRefactorResponse, AIRoastResponse, AIReadmeEnhanceResponse } from '@/types'; + +export const explanationCache = new Map(); +export const healthCache = new Map(); +export const mermaidCache = new Map(); +export const refactorCache = new Map(); +export const roastCache = new Map(); +export const readmeEnhanceCache = new Map(); + +export function getToolKey(kind: string, owner: string, repo: string, revision: number, branch?: string | null, sha?: string | null) { + // JSON pair encoding keeps "branch literally named default" distinct from "no branch" without a sentinel. + return `${kind}:${owner}/${repo}/${revision}/${sha || 'head'}:${JSON.stringify([branch ?? null])}`; +} + +export function createCachedTask(cache: Map, max = 100) { + const pending = new Map>(); + return (key: string, run: () => Promise): Promise => { + const cached = cache.get(key); + if (cached !== undefined) return Promise.resolve(cached); + const existing = pending.get(key); + if (existing) return existing; + const request = Promise.resolve() + .then(run) + .then((data) => { + if (max <= 0) return data; + if (cache.has(key)) cache.delete(key); + else if (cache.size >= max) { + const oldest = cache.keys().next().value; + if (oldest !== undefined) cache.delete(oldest); + } + cache.set(key, data); + return data; + }) + .finally(() => pending.delete(key)); + pending.set(key, request); + return request; + }; +} + +export const runExplanation = createCachedTask(explanationCache); +export const runHealth = createCachedTask(healthCache); +export const runMermaid = createCachedTask(mermaidCache); +export const runRefactor = createCachedTask(refactorCache); +export const runRoast = createCachedTask(roastCache); +export const runReadmeEnhance = createCachedTask(readmeEnhanceCache); diff --git a/src/features/ai/useAiTasks.ts b/src/features/ai/useAiTasks.ts index 8da157a..0370c14 100644 --- a/src/features/ai/useAiTasks.ts +++ b/src/features/ai/useAiTasks.ts @@ -1,3 +1,5 @@ +import { runHealth, runMermaid, runRefactor, runRoast, runReadmeEnhance, getToolKey } from './tool-cache'; +import { useChatConfigRevision } from '@/stores/chatConfigStore'; import { useMutation, useQuery } from '@tanstack/react-query'; import { aiRefactor, @@ -9,47 +11,68 @@ import { } from '@/lib/apiClient'; export function useRefactor() { + const { revision } = useChatConfigRevision(); return useMutation({ - mutationFn: ({ owner, repo, branch }: { owner: string; repo: string; branch?: string }) => - aiRefactor(owner, repo, branch), + mutationKey: ['refactor', revision], + mutationFn: async ({ owner, repo, branch, sha }: { owner: string; repo: string; branch?: string; sha?: string }) => ({ + ...(await runRefactor(getToolKey('refactor', owner, repo, revision, branch, sha), () => aiRefactor(owner, repo, branch))), + revision, + }), }); } export function useHealth() { + const { revision } = useChatConfigRevision(); return useMutation({ - mutationFn: ({ owner, repo, branch }: { owner: string; repo: string; branch?: string }) => - aiHealth(owner, repo, branch), + mutationKey: ['health', revision], + mutationFn: async ({ owner, repo, branch, sha }: { owner: string; repo: string; branch?: string; sha?: string }) => ({ + ...(await runHealth(getToolKey('health', owner, repo, revision, branch, sha), () => aiHealth(owner, repo, branch))), + revision, + }), }); } export function useMermaid() { + const { revision } = useChatConfigRevision(); return useMutation({ - mutationFn: ({ owner, repo }: { owner: string; repo: string }) => - aiMermaid(owner, repo), + mutationKey: ['mermaid', revision], + mutationFn: async ({ owner, repo, branch, sha }: { owner: string; repo: string; branch?: string; sha?: string }) => ({ + ...(await runMermaid(getToolKey('mermaid', owner, repo, revision, branch, sha), () => aiMermaid(owner, repo, sha || branch))), + revision, + sha, + branch, + }), }); } export function useRoast() { + const { revision } = useChatConfigRevision(); return useMutation({ - mutationFn: ({ owner, repo, branch }: { owner: string; repo: string; branch?: string }) => - aiRoast(owner, repo, branch), + mutationKey: ['roast', revision], + mutationFn: async ({ owner, repo, branch, sha }: { owner: string; repo: string; branch?: string; sha?: string }) => ({ + ...(await runRoast(getToolKey('roast', owner, repo, revision, branch, sha), () => aiRoast(owner, repo, branch))), + revision, + }), }); } export function useReadmeEnhance() { + const { revision } = useChatConfigRevision(); return useMutation({ - mutationFn: ({ owner, repo, branch }: { owner: string; repo: string; branch?: string }) => - aiReadmeEnhance(owner, repo, branch), + mutationKey: ['readme-enhance', revision], + mutationFn: async ({ owner, repo, branch, sha }: { owner: string; repo: string; branch?: string; sha?: string }) => ({ + ...(await runReadmeEnhance(getToolKey('readme-enhance', owner, repo, revision, branch, sha), () => aiReadmeEnhance(owner, repo, branch))), + revision, + }), }); } export function useLearningPath(owner: string, repo: string, branch: string | null = null, enabled = true, goal?: string) { + const { revision } = useChatConfigRevision(); return useQuery({ - queryKey: ['learningPath', owner, repo, branch, goal], + queryKey: ['learningPath', owner, repo, branch, goal, revision], queryFn: () => aiLearningPath(owner, repo, branch || undefined, goal), enabled: enabled && !!owner && !!repo, staleTime: 1000 * 60 * 30, }); } - - diff --git a/src/features/graph/canvas/ForceGraphCanvas.tsx b/src/features/graph/canvas/ForceGraphCanvas.tsx index 0aa1e75..ecd6004 100644 --- a/src/features/graph/canvas/ForceGraphCanvas.tsx +++ b/src/features/graph/canvas/ForceGraphCanvas.tsx @@ -1,4 +1,6 @@ +import { motionDuration } from '@/lib/motion-preference'; import { useCallback, useRef, useState } from "react"; +import { useMotionPreference } from '@/hooks/useMotionPreference'; import ForceGraph2D, { type ForceGraphMethods } from "react-force-graph-2d"; import { Loader2 } from "lucide-react"; import type { GraphEdge, GraphNode } from "@/types"; @@ -66,6 +68,8 @@ export function NetworkCanvas({ onVisibleCounts, }: NetworkCanvasProps) { const [tick, setTick] = useState(0); + const reducedMotion = useMotionPreference(); + const [settledLayout, setSettledLayout] = useState<{ data: unknown; layout: LayoutType; size: SizeMode } | null>(null); const theme = useVizStore((s) => s.theme); const internalForceGraphRef = useRef< @@ -126,8 +130,8 @@ export function NetworkCanvas({ ...(typeof window !== 'undefined' && window.innerWidth >= 1024 ? { aiSidebarOpen: true } : {}), }); if (typeof node.x === "number" && typeof node.y === "number") { - forceGraphRef.current?.centerAt(node.x, node.y, 300); - forceGraphRef.current?.zoom(3.2, 300); + forceGraphRef.current?.centerAt(node.x, node.y, motionDuration(300)); + forceGraphRef.current?.zoom(3.2, motionDuration(300)); } }, [prevFocusRef, forceGraphRef], @@ -152,10 +156,11 @@ export function NetworkCanvas({ ]); const handleEngineStop = useCallback(() => { + setSettledLayout({ data: forceGraphData, layout: layoutType, size: sizeMode }); handleLayoutStop(); if (showMinimap) setTick((t) => t + 1); forceInitialViewDoneRef.current = true; - }, [showMinimap, handleLayoutStop]); + }, [showMinimap, handleLayoutStop, forceGraphData, layoutType, sizeMode]); const handleEngineTick = useCallback(() => { if (!showMinimap) return; @@ -239,12 +244,14 @@ export function NetworkCanvas({ colorMode, sizeMode, theme, + metadataSource: graph, }); }, [ blastRadiusActive, colorMode, compareBranch, + graph, highlightedNodeIds, hoveredForceNode, forceGraphData.nodes.length, @@ -273,12 +280,19 @@ export function NetworkCanvas({ // --- Render --- const isLoading = !graph || !graph.nodes; const isEmpty = graph.nodes.length === 0; + const settling = settledLayout?.data !== forceGraphData || settledLayout?.layout !== layoutType || settledLayout?.size !== sizeMode; return (
+ {reducedMotion && settling && !isEmpty && ( +

+ Arranging dependency graph… +

+ )} {isLoading && (
diff --git a/src/features/graph/canvas/force/forcePainter.ts b/src/features/graph/canvas/force/forcePainter.ts index ba64b2e..19f629d 100644 --- a/src/features/graph/canvas/force/forcePainter.ts +++ b/src/features/graph/canvas/force/forcePainter.ts @@ -21,6 +21,7 @@ interface NodePaintProps { colorMode?: ColorMode; sizeMode?: SizeMode; theme?: GraphTheme; + metadataSource?: unknown; } /** Map a 0-1 score to a color from the given 5-stop gradient */ diff --git a/src/features/graph/canvas/force/useD3Physics.ts b/src/features/graph/canvas/force/useD3Physics.ts index df102fd..9cfb65d 100644 --- a/src/features/graph/canvas/force/useD3Physics.ts +++ b/src/features/graph/canvas/force/useD3Physics.ts @@ -1,3 +1,4 @@ +import { motionDuration } from '@/lib/motion-preference'; import { useCallback, useEffect, useRef } from 'react'; import { forceCollide, forceCenter, forceX, forceY } from 'd3-force'; import { stratify, tree } from 'd3-hierarchy'; @@ -24,7 +25,7 @@ export function useD3Physics({ forceGraphRef, nodes, links, layoutType, sizeMode const handleLayoutStop = useCallback(() => { if (!fitAfterSimulation.current) return; fitAfterSimulation.current = false; - forceGraphRef.current?.zoomToFit(400, 60); + forceGraphRef.current?.zoomToFit(motionDuration(400), 60); }, [forceGraphRef]); useEffect(() => { diff --git a/src/features/graph/canvas/force/useForceSync.ts b/src/features/graph/canvas/force/useForceSync.ts index e7af4f6..da2d274 100644 --- a/src/features/graph/canvas/force/useForceSync.ts +++ b/src/features/graph/canvas/force/useForceSync.ts @@ -1,3 +1,4 @@ +import { motionDuration } from '@/lib/motion-preference'; import { useEffect, useRef } from 'react'; import type { ForceGraphMethods } from 'react-force-graph-2d'; import type { ForceGraphNode, ForceGraphLink } from '../../force/forceGraphConstants'; @@ -54,8 +55,8 @@ export function useForceSync({ if (node && typeof node.x === "number" && typeof node.y === "number") { forceInitialViewDoneRef.current = true; - ref.centerAt(node.x, node.y, 300); - ref.zoom(3.2, 300); + ref.centerAt(node.x, node.y, motionDuration(300)); + ref.zoom(3.2, motionDuration(300)); prevFocusRef.current = targetId; return; } @@ -86,15 +87,15 @@ export function useForceSync({ if (action === 'zoomIn') { const currentZoom = ref.zoom(); - if (typeof currentZoom === 'number') ref.zoom(currentZoom * 1.3, 300); + if (typeof currentZoom === 'number') ref.zoom(currentZoom * 1.3, motionDuration(300)); } else if (action === 'zoomOut') { const currentZoom = ref.zoom(); - if (typeof currentZoom === 'number') ref.zoom(currentZoom * 0.7, 300); + if (typeof currentZoom === 'number') ref.zoom(currentZoom * 0.7, motionDuration(300)); } else if (action === 'fitView') { - ref.zoomToFit(400, 60); + ref.zoomToFit(motionDuration(400), 60); } else if (action === 'reset') { - ref.centerAt(0, 0, 400); - ref.zoom(1.5, 400); + ref.centerAt(0, 0, motionDuration(400)); + ref.zoom(1.5, motionDuration(400)); prevFocusRef.current = null; } else if (action === 'focusGraph') { const activeId = @@ -111,11 +112,11 @@ export function useForceSync({ : nodes.find((n) => n.nodeType === 'repo'); if (targetNode && typeof targetNode.x === "number" && typeof targetNode.y === "number") { - ref.centerAt(targetNode.x, targetNode.y, 400); - ref.zoom(activeId ? 3.2 : 1.5, 400); + ref.centerAt(targetNode.x, targetNode.y, motionDuration(400)); + ref.zoom(activeId ? 3.2 : 1.5, motionDuration(400)); } else { - ref.centerAt(0, 0, 400); - ref.zoom(1.5, 400); + ref.centerAt(0, 0, motionDuration(400)); + ref.zoom(1.5, motionDuration(400)); } } }, [ diff --git a/src/features/graph/canvas/hooks/useForceCanvasState.ts b/src/features/graph/canvas/hooks/useForceCanvasState.ts index 4b460d8..ef1272a 100644 --- a/src/features/graph/canvas/hooks/useForceCanvasState.ts +++ b/src/features/graph/canvas/hooks/useForceCanvasState.ts @@ -1,14 +1,15 @@ -import { useMemo, useEffect, useState } from "react"; -import { useParams } from "react-router-dom"; -import type { GraphNode, GraphEdge } from "@/types"; -import { useVizStore, type LayoutType, type SizeMode } from "@/stores/vizStore"; -import { buildForceGraphData } from "../../force/buildForceGraphData"; -import { computeBlastRadius } from "../../force/forceGraphUtils"; -import { useGraphExport } from "../../useGraphExport"; -import { useD3Physics } from "../force/useD3Physics"; -import { useForceSync } from "../force/useForceSync"; -import type { ForceGraphNode, ForceGraphLink } from "../../force/forceGraphConstants"; -import type { ForceGraphMethods } from "react-force-graph-2d"; +import { useMemo, useEffect, useState, useRef } from 'react'; +import { reconcileGraph } from '../../force/reconcileGraph'; +import { useParams } from 'react-router-dom'; +import type { GraphNode, GraphEdge } from '@/types'; +import { useVizStore, type LayoutType, type SizeMode } from '@/stores/vizStore'; +import { buildForceGraphData } from '../../force/buildForceGraphData'; +import { computeBlastRadius } from '../../force/forceGraphUtils'; +import { useGraphExport } from '../../useGraphExport'; +import { useD3Physics } from '../force/useD3Physics'; +import { useForceSync } from '../force/useForceSync'; +import type { ForceGraphNode, ForceGraphLink } from '../../force/forceGraphConstants'; +import type { ForceGraphMethods } from 'react-force-graph-2d'; interface UseForceCanvasStateProps { graph: { nodes: GraphNode[]; edges: GraphEdge[] }; @@ -55,15 +56,15 @@ export function useForceCanvasState({ const [forceSize, setForceSize] = useState({ width: 1024, height: 720 }); const [hoveredForceNode, setHoveredForceNode] = useState(null); - const { owner = "", repo = "" } = useParams(); + const { owner = '', repo = '' } = useParams(); const { isExporting, exportFormat, handleExport } = useGraphExport({ - mode: "force", + mode: 'force', forceGraphRef, forceHostRef, owner, repo, - filenameSuffix: "force_graph", + filenameSuffix: 'force_graph', }); const connectedNodeIdsByNodeId = useMemo(() => { @@ -77,20 +78,21 @@ export function useForceCanvasState({ return lookup; }, [graph.edges]); + const previousGraph = useRef | undefined>(undefined); const forceGraphData = useMemo( () => - buildForceGraphData(graph.nodes, graph.edges, { - nodeTypeFilters, - fileTypeFilters, - readOnly, - }), - [graph.nodes, graph.edges, nodeTypeFilters, fileTypeFilters, readOnly] + (previousGraph.current = reconcileGraph( + previousGraph.current, + buildForceGraphData(graph.nodes, graph.edges, { + nodeTypeFilters, + fileTypeFilters, + readOnly, + }), + )), + [graph.nodes, graph.edges, nodeTypeFilters, fileTypeFilters, readOnly], ); - const forceNodeById = useMemo( - () => new Map(forceGraphData.nodes.map((n) => [n.id, n])), - [forceGraphData.nodes] - ); + const forceNodeById = useMemo(() => new Map(forceGraphData.nodes.map((n) => [n.id, n])), [forceGraphData.nodes]); const blastRadiusNodeIds = useMemo(() => { if (!blastRadiusActive || !selectedNodeId) return new Set(); @@ -102,30 +104,15 @@ export function useForceCanvasState({ // Read-only canvases never adopt workspace focus/selection state. if (readOnly || !focusedFilePath) return; - const focusedNodeIds = [ - `file:${focusedFilePath}`, - `folder:${focusedFilePath}`, - focusedFilePath, - ]; + const focusedNodeIds = [`file:${focusedFilePath}`, `folder:${focusedFilePath}`, focusedFilePath]; const focusedNode = focusedNodeIds.map((id) => forceNodeById.get(id)).find(Boolean) ?? - forceGraphData.nodes.find( - (node) => - node.sourceFile === focusedFilePath || - focusedNodeIds.includes(node.id), - ); + forceGraphData.nodes.find((node) => node.sourceFile === focusedFilePath || focusedNodeIds.includes(node.id)); if (focusedNode && selectedNodeId !== focusedNode.id) { setSelectedNodeId(focusedNode.id); } - }, [ - readOnly, - focusedFilePath, - forceGraphData.nodes, - forceNodeById, - selectedNodeId, - setSelectedNodeId, - ]); + }, [readOnly, focusedFilePath, forceGraphData.nodes, forceNodeById, selectedNodeId, setSelectedNodeId]); useEffect(() => { // Read-only canvases must not rewrite shared highlight state from the @@ -138,9 +125,7 @@ export function useForceCanvasState({ if (blastRadiusActive) { setHighlightedNodeIds(blastRadiusNodeIds); } else { - setHighlightedNodeIds( - new Set(connectedNodeIdsByNodeId.get(selectedNodeId) ?? [selectedNodeId]) - ); + setHighlightedNodeIds(new Set(connectedNodeIdsByNodeId.get(selectedNodeId) ?? [selectedNodeId])); } }, [ readOnly, diff --git a/src/features/graph/canvas/widgets/LegendPanel.tsx b/src/features/graph/canvas/widgets/LegendPanel.tsx index 7570977..e746809 100644 --- a/src/features/graph/canvas/widgets/LegendPanel.tsx +++ b/src/features/graph/canvas/widgets/LegendPanel.tsx @@ -232,9 +232,9 @@ function LegendContent({
)} - {/* Focus & Selection — merged blast radius + interactive states */} + {/* Focus & Selection — merged blast radius + interactive states */}
- Focus & Selection + Focus & Selection
{blastRadiusActive ? ( <> diff --git a/src/features/graph/force/reconcileGraph.test.ts b/src/features/graph/force/reconcileGraph.test.ts new file mode 100644 index 0000000..ea17eee --- /dev/null +++ b/src/features/graph/force/reconcileGraph.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from 'bun:test'; +import { reconcileGraph } from './reconcileGraph'; +import { buildForceGraphData } from './buildForceGraphData'; +import type { GraphNode } from '@/types'; +test('metadata updates preserve simulation identity and coordinates', () => { + const node: GraphNode = { + id: 'a', + type: 'file', + position: { x: 0, y: 0 }, + data: { label: 'a' }, + }; + const options = { + nodeTypeFilters: new Set(['file']), + fileTypeFilters: new Set(), + }; + const edge = { id: 'a->b', source: 'a', target: 'b' }; + const other: GraphNode = { id: 'b', type: 'file', position: { x: 1, y: 1 }, data: { label: 'b' } }; + const first = buildForceGraphData([node, other], [edge], options); + first.nodes[0].x = 125; + first.nodes[0].y = 80; + const next = buildForceGraphData( + [{ ...node, data: { ...node.data, churnScore: 0.7 } }, other], + [edge], + options, + ); + const reconciled = reconcileGraph(first, next); + expect(reconciled).not.toBe(first); + expect(reconciled.nodes[0]).toBe(first.nodes[0]); + expect(reconciled.links[0]).toBe(first.links[0]); + expect(first.nodes[0].x).toBe(125); + expect(first.nodes[0].churnScore).toBe(0.7); + expect(reconcileGraph(first, { nodes: [], links: [] })).not.toBe(first); +}); diff --git a/src/features/graph/force/reconcileGraph.ts b/src/features/graph/force/reconcileGraph.ts new file mode 100644 index 0000000..ece5241 --- /dev/null +++ b/src/features/graph/force/reconcileGraph.ts @@ -0,0 +1,21 @@ +import type { ForceGraphNode, ForceGraphLink } from './forceGraphConstants'; + +type Data = { nodes: ForceGraphNode[]; links: ForceGraphLink[] }; +/** Keep simulation objects for metric-only updates; topology changes get fresh data. */ +export function reconcileGraph(previous: Data | undefined, next: Data): Data { + if (!previous || previous.nodes.length !== next.nodes.length || previous.links.length !== next.links.length) + return next; + const id = (value: ForceGraphLink['source']) => (typeof value === 'object' ? value.id : value); + if ( + next.nodes.some((node, i) => node.id !== previous.nodes[i].id) || + next.links.some((link, i) => { + const old = previous.links[i]; + return id(link.source) !== id(old.source) || id(link.target) !== id(old.target) || link.type !== old.type; + }) + ) + return next; + next.nodes.forEach((node, i) => Object.assign(previous.nodes[i], node)); + // New wrapper identity makes ForceGraph2D re-digest and repaint metadata, + // while the same node/link objects keep their simulation coordinates. + return { nodes: [...previous.nodes], links: [...previous.links] }; +} diff --git a/src/features/search/IndexingStatusPanel.tsx b/src/features/search/IndexingStatusPanel.tsx index d489311..bbb85f5 100644 --- a/src/features/search/IndexingStatusPanel.tsx +++ b/src/features/search/IndexingStatusPanel.tsx @@ -1,97 +1,90 @@ -import { Database, AlertCircle, CheckCircle2, RefreshCw, Loader2, Zap } from 'lucide-react'; +import { useEffect, useRef, useState } from 'react'; import { useSearchStore } from './searchStore'; +import { coverageMessage } from '@/lib/search-coverage'; +import { formatRetryDelay } from './retry-delay'; -interface IndexingStatusProps { - onRetry?: () => void; -} - -export function IndexingStatusPanel({ onRetry }: IndexingStatusProps) { - const { indexingStatus } = useSearchStore(); - - if (indexingStatus.state === 'idle') { - return ( -
-
- -
-
-

Index missing

-

Build an index to search this repository by meaning

-
- {onRetry && ( - - )} -
- ); - } - - if (indexingStatus.state === 'indexing') { - const progress = indexingStatus.progress; - const hasFileCount = indexingStatus.totalFiles > 0; - return ( -
-
- - - {hasFileCount ? 'Indexing...' : 'Scanning repository...'} - - {hasFileCount && ( - - {indexingStatus.filesProcessed}/{indexingStatus.totalFiles} files - - )} -
-
-
void; onCancel?: () => void }) { + const status = useSearchStore((state) => state.indexingStatus); + const action = useSearchStore((state) => state.indexAction); + const resumed = useRef(null); + const [now, setNow] = useState(Date.now); + useEffect(() => { + if (status.state !== 'paused' || !status.retryAt) return; + const timer = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(timer); + }, [status]); + const waiting = status.state === 'paused' && !!status.retryAt && status.retryAt > now; + useEffect(() => { + if (action || status.state !== 'paused' || !status.retryAt || status.retryAt > now) return; + const deadline = `${status.snapshotSha}:${status.retryAt}`; + if (resumed.current === deadline) return; + resumed.current = deadline; + onRetry?.(); + }, [status, now, onRetry, action]); + const working = status.state === 'indexing' || status.state === 'paused'; + const button = + 'rounded-md border border-border bg-background px-3 py-1.5 text-xs text-foreground hover:bg-secondary disabled:opacity-50 disabled:cursor-not-allowed'; + return ( +
+

+ {status.state === 'complete' + ? 'Index ready' + : status.state === 'indexing' + ? 'Indexing repository…' + : status.state === 'paused' + ? 'Indexing paused' + : status.state === 'failed' + ? 'Index failed' + : 'Build a search index'} +

+ {working && ( + <> +

+ {status.filesProcessed} files indexed · {Math.max(0, status.totalFiles - status.filesProcessed)} remaining +

+ -
-

- {hasFileCount - ? 'Building semantic embeddings...' - : 'Preparing files...'} + + )} + {status.state === 'complete' &&

{status.chunkCount} chunks indexed

} + {(status.state === 'paused' || status.state === 'failed') && ( +

{status.error}

+ )} + {status.state === 'paused' && ( +

+ {status.retryAt + ? `Retry ${new Date(status.retryAt).toLocaleString()}${waiting ? ` (in ${formatRetryDelay(status.retryAt - now)})` : ' — ready to resume'}` + : 'Resume when the service is available.'}

-
- ); - } - - if (indexingStatus.state === 'complete') { - return ( -
- - Index ready - {indexingStatus.chunkCount} chunks -
- ); - } - - if (indexingStatus.state === 'failed') { - return ( -
- -
-

Index failed

-

{indexingStatus.error}

-
- {onRetry && ( - + )} + {working && onCancel && ( + )}
- ); - } - - return null; + + ); } diff --git a/src/features/search/SearchEmptyState.tsx b/src/features/search/SearchEmptyState.tsx new file mode 100644 index 0000000..1e9f6b2 --- /dev/null +++ b/src/features/search/SearchEmptyState.tsx @@ -0,0 +1,60 @@ +import { useSearchStore } from './searchStore'; + +export function SearchEmptyState({ chunkCount, onSubmit }: { chunkCount: number; onSubmit: (query: string) => void }) { + return ( +
+
+

+ Search Examples +

+
+ {[ + 'How are API errors handled?', + 'Where is GitHub data fetched?', + 'How is the dependency graph generated?', + ].map((example) => ( + + ))} +
+
+ +
+
+

+ Recent Queries +

+
+ No recent queries yet. +
+
+
+

+ Index Details +

+
+
+ Status + + + Ready + +
+
+ Chunks Indexed + {chunkCount} +
+
+
+
+
+ ); +} diff --git a/src/features/search/SearchIndexControls.tsx b/src/features/search/SearchIndexControls.tsx new file mode 100644 index 0000000..6f4b8d6 --- /dev/null +++ b/src/features/search/SearchIndexControls.tsx @@ -0,0 +1,71 @@ +import { cancelIndexing, type IndexScope } from '@/lib/apiClient'; +import { useSearchStore } from './searchStore'; +import { IndexingStatusPanel } from './IndexingStatusPanel'; +import { useQueryClient } from '@tanstack/react-query'; +import { beginIndexOperation, finishIndexOperation } from './index-operations'; + +interface SearchIndexControlsProps { + owner: string; + repo: string; + branch?: string; + scope: IndexScope; + includeInput: string; + excludeInput: string; + onIncludeChange: (value: string) => void; + onExcludeChange: (value: string) => void; + onIndex: () => void; +} + +export function SearchIndexControls(props: SearchIndexControlsProps) { + const queryClient = useQueryClient(); + const handleCancel = async () => { + const context = beginIndexOperation('cancel', props.owner, props.repo); + const buildId = useSearchStore.getState().indexBuildId; + try { + await queryClient.cancelQueries({ queryKey: ['indexingStatus', props.owner, props.repo] }); + const status = await cancelIndexing(props.owner, props.repo, props.branch, props.scope, buildId); + finishIndexOperation(context, status); + } catch { + const previous = + context.previous.state === 'paused' ? { ...context.previous, retryAt: undefined } : context.previous; + if (finishIndexOperation(context, previous)) + useSearchStore.getState().setError('Could not cancel indexing. Please retry.'); + } + }; + return ( +
+
+ {[ + { + label: 'Include paths', + value: props.includeInput, + onChange: props.onIncludeChange, + placeholder: 'packages/next/src, apps/web', + }, + { + label: 'Exclude paths', + value: props.excludeInput, + onChange: props.onExcludeChange, + placeholder: 'test, docs, examples', + }, + ].map((field) => ( + + ))} +
+ { + void handleCancel(); + }} + /> +
+ ); +} diff --git a/src/features/search/index-operations.test.ts b/src/features/search/index-operations.test.ts new file mode 100644 index 0000000..57f78a1 --- /dev/null +++ b/src/features/search/index-operations.test.ts @@ -0,0 +1,29 @@ +import { test, expect } from 'bun:test'; +import { useSearchStore } from './searchStore'; +import { beginIndexOperation, finishIndexOperation } from './index-operations'; + +test('cancel invalidates the preceding indexing callback', () => { + useSearchStore.getState().reset(); + const indexing = beginIndexOperation('index'); + const cancelling = beginIndexOperation('cancel'); + expect(finishIndexOperation(cancelling, { state: 'idle' })).toBe(true); + expect(finishIndexOperation(indexing, { state: 'complete', chunkCount: 10, timestamp: 0 })).toBe(false); + expect(useSearchStore.getState().indexingStatus.state).toBe('idle'); + expect(useSearchStore.getState().indexOperation).toBeGreaterThan(cancelling.operation); +}); + +test('navigation invalidates an outstanding operation', () => { + const operation = beginIndexOperation('index'); + useSearchStore.getState().reset(); + expect(finishIndexOperation(operation, { state: 'complete', chunkCount: 1, timestamp: 0 })).toBe(false); + expect(useSearchStore.getState().indexAction).toBeNull(); +}); + +test('query reset preserves an outstanding index operation', () => { + useSearchStore.getState().reset(); + const operation = beginIndexOperation('index', 'o', 'r'); + useSearchStore.getState().resetQueryResults(); + expect(finishIndexOperation(operation, { state: 'complete', chunkCount: 1, timestamp: 0 })).toBe(true); + expect(useSearchStore.getState().indexOwner).toBe('o'); + expect(useSearchStore.getState().indexRepo).toBe('r'); +}); diff --git a/src/features/search/index-operations.ts b/src/features/search/index-operations.ts new file mode 100644 index 0000000..be3616b --- /dev/null +++ b/src/features/search/index-operations.ts @@ -0,0 +1,24 @@ +import type { IndexingStatus } from '@/types'; +import { useSearchStore } from './searchStore'; + +export function beginIndexOperation(action: 'index' | 'cancel', owner?: string, repo?: string) { + const state = useSearchStore.getState(); + const operation = state.indexOperation + 1; + useSearchStore.setState({ + indexOperation: operation, + indexAction: action, + indexOwner: owner ?? state.indexOwner, + indexRepo: repo ?? state.indexRepo, + }); + return { operation, revision: state.revision, previous: state.indexingStatus }; +} + +export function finishIndexOperation(context: ReturnType, status: IndexingStatus): boolean { + const state = useSearchStore.getState(); + // Index operations are keyed by indexOperation only: query resets bump + // revision (invalidating search/ask) without abandoning the index build. + // Full resets bump indexOperation, invalidating outstanding operations. + if (state.indexOperation !== context.operation) return false; + useSearchStore.setState({ indexOperation: context.operation + 1, indexAction: null, indexingStatus: status }); + return true; +} diff --git a/src/features/search/indexing-recovery.test.ts b/src/features/search/indexing-recovery.test.ts new file mode 100644 index 0000000..888ce75 --- /dev/null +++ b/src/features/search/indexing-recovery.test.ts @@ -0,0 +1,53 @@ +import { test, expect } from 'bun:test'; +import { recoverIndexingStatus, sanitizeIndexingErrorMessage } from './indexing-recovery'; +import type { IndexingStatus } from '@/types'; +import { ApiError } from '@/lib/apiClient'; + +const paused: IndexingStatus = { + state: 'paused', + progress: 80, + filesProcessed: 32, + totalFiles: 40, + snapshotSha: 'original', + retryAt: 1000, + reason: 'EMBEDDING_RATE_LIMITED', + error: 'Provider limit', + coverage: { kind: 'partial', commitSha: 'original', indexedFiles: 32, totalFiles: 40 }, +}; + +test('unknown credentials and provider response fragments never reach the banner', () => { + for (const message of ['Invalid key: AIza' + 'x'.repeat(35), 'Provider response: confidential source code']) { + expect(sanitizeIndexingErrorMessage(message)).toBe('Indexing could not finish. Please retry or check your provider settings.'); + } +}); + +test('network failure preserves snapshot, progress and coverage without retrying an expired timer', () => { + const recovered = recoverIndexingStatus(paused, new ApiError('Network unavailable', 0), 2000); + expect(recovered).toMatchObject({ + state: 'paused', + snapshotSha: 'original', + filesProcessed: 32, + coverage: paused.coverage, + }); + expect(recovered.state === 'paused' && recovered.retryAt).toBeUndefined(); +}); +test('admission failure uses the new retry time while preserving the paused build', () => { + const error = Object.assign(new Error('Busy'), { status: 429, details: { retryAfterSeconds: 60 } }); + expect(recoverIndexingStatus(paused, error, 2000)).toMatchObject({ + state: 'paused', + retryAt: 62000, + filesProcessed: 32, + }); +}); +test('terminal rejection does not advertise a discarded partial index', () => { + expect(recoverIndexingStatus(paused, Object.assign(new Error('Too large'), { status: 413 }))).toMatchObject({ + state: 'failed', + coverage: undefined, + }); +}); +test('error messages are sanitized before reaching the status banner', () => { + expect(sanitizeIndexingErrorMessage('fetch https://api.example.com/embed?key=abc failed')).not.toContain('https://'); + expect(sanitizeIndexingErrorMessage('token ghp_abcDEF1234567890 leaked')).not.toContain('ghp_abcDEF1234567890'); + const recovered = recoverIndexingStatus(paused, Object.assign(new Error('boom https://evil.example/x'), { status: 413 })); + expect(recovered.error).not.toContain('https://'); +}); diff --git a/src/features/search/indexing-recovery.ts b/src/features/search/indexing-recovery.ts new file mode 100644 index 0000000..c98b16c --- /dev/null +++ b/src/features/search/indexing-recovery.ts @@ -0,0 +1,34 @@ +import type { IndexingStatus } from '@/types'; + +/** Untrusted provider text must never become user-facing status content. */ +export function sanitizeIndexingErrorMessage(_message: string): string { + return 'Indexing could not finish. Please retry or check your provider settings.'; +} + +/** Recover a rejected request without pretending that its saved server-side build disappeared. */ +export function recoverIndexingStatus(previous: IndexingStatus, error: Error, now = Date.now()): IndexingStatus { + const message = sanitizeIndexingErrorMessage(error.message); + const status = 'status' in error && typeof error.status === 'number' ? error.status : undefined; + const recoverable = + status === undefined || status === 0 || status === 408 || status === 409 || status === 429 || status >= 500; + if (previous.state === 'paused' && recoverable) { + const details = 'details' in error ? error.details : undefined; + const seconds = + details && typeof details === 'object' && 'retryAfterSeconds' in details ? details.retryAfterSeconds : undefined; + return { + ...previous, + error: message, + retryAt: + typeof seconds === 'number' && Number.isFinite(seconds) && seconds > 0 + ? now + Math.ceil(seconds) * 1000 + : undefined, + }; + } + return { + state: 'failed', + error: message, + failedFiles: 0, + snapshotSha: previous.snapshotSha, + coverage: previous.coverage?.kind === 'previous' ? previous.coverage : undefined, + }; +} diff --git a/src/features/search/parsePaths.ts b/src/features/search/parsePaths.ts new file mode 100644 index 0000000..18b7e4c --- /dev/null +++ b/src/features/search/parsePaths.ts @@ -0,0 +1,6 @@ +export function parsePaths(value: string): string[] { + return value + .split(',') + .map((path) => path.trim()) + .filter(Boolean); +} diff --git a/src/features/search/retry-delay.test.ts b/src/features/search/retry-delay.test.ts new file mode 100644 index 0000000..7d80ecd --- /dev/null +++ b/src/features/search/retry-delay.test.ts @@ -0,0 +1,8 @@ +import { expect, test } from 'bun:test'; +import { formatRetryDelay } from './retry-delay'; + +test('retry countdown uses readable minutes and hours', () => { + expect(formatRetryDelay(63176000)).toBe('17 hr 33 min'); + expect(formatRetryDelay(60000)).toBe('1 min'); + expect(formatRetryDelay(3600000)).toBe('1 hr'); +}); diff --git a/src/features/search/retry-delay.ts b/src/features/search/retry-delay.ts new file mode 100644 index 0000000..3b47f70 --- /dev/null +++ b/src/features/search/retry-delay.ts @@ -0,0 +1,8 @@ +export function formatRetryDelay(milliseconds: number): string { + const minutes = Math.ceil(Math.max(0, milliseconds) / 60000); + if (minutes < 1) return 'less than a minute'; + if (minutes < 60) return `${minutes} min`; + const hours = Math.floor(minutes / 60); + const remaining = minutes % 60; + return `${hours} hr${remaining ? ` ${remaining} min` : ''}`; +} diff --git a/src/features/search/searchStore.ts b/src/features/search/searchStore.ts index 3c8e9a7..24c1ea8 100644 --- a/src/features/search/searchStore.ts +++ b/src/features/search/searchStore.ts @@ -1,9 +1,18 @@ import { create } from 'zustand'; -import type { SearchResultCard, QAAnswer, IndexingStatus } from '@/types'; +import type { IndexScope } from '@/lib/apiClient'; +import type { SearchResultCard, QAAnswer, IndexingStatus, SearchCoverage } from '@/types'; export type SearchMode = 'search' | 'ask'; interface SearchState { + indexBuildId?: string; + indexOperation: number; + indexAction: 'index' | 'cancel' | null; + indexScope: IndexScope | null; + indexOwner: string | null; + indexRepo: string | null; + resultCoverage: SearchCoverage | undefined; + revision: number; mode: SearchMode; query: string; results: SearchResultCard[]; @@ -12,8 +21,6 @@ interface SearchState { error: string | null; recentQueries: string[]; indexingStatus: IndexingStatus; - askCache: Map; - searchCache: Map; setMode: (mode: SearchMode) => void; setQuery: (query: string) => void; @@ -23,6 +30,7 @@ interface SearchState { setError: (error: string | null) => void; setIndexingStatus: (status: IndexingStatus) => void; addRecentQuery: (query: string) => void; + resetQueryResults: () => void; reset: () => void; } @@ -47,6 +55,13 @@ function saveRecentQueries(queries: string[]): void { } export const useSearchStore = create((set, get) => ({ + indexOperation: 0, + indexAction: null, + indexScope: null, + indexOwner: null, + indexRepo: null, + resultCoverage: undefined, + revision: 0, mode: 'search', query: '', results: [], @@ -55,8 +70,6 @@ export const useSearchStore = create((set, get) => ({ error: null, recentQueries: loadRecentQueries(), indexingStatus: { state: 'idle' }, - askCache: new Map(), - searchCache: new Map(), setMode: (mode) => set({ mode }), setQuery: (query) => set({ query }), @@ -75,15 +88,34 @@ export const useSearchStore = create((set, get) => ({ set({ recentQueries: updated }); }, + resetQueryResults: () => + set({ + results: [], + answer: null, + error: null, + resultCoverage: undefined, + isLoading: false, + // Invalidate in-flight search/ask responses; the active index operation + // is keyed by indexOperation (not revision) so polling keeps tracking it. + revision: get().revision + 1, + }), + reset: () => set({ + revision: get().revision + 1, + indexOperation: get().indexOperation + 1, + indexAction: null, + indexBuildId: undefined, + indexScope: null, + indexOwner: null, + indexRepo: null, mode: 'search', query: '', results: [], + resultCoverage: undefined, answer: null, isLoading: false, error: null, indexingStatus: { state: 'idle' }, - // Preserve caches across repo switches — they're keyed by owner/repo }), })); diff --git a/src/features/search/useIndexingStatus.ts b/src/features/search/useIndexingStatus.ts index ccd16ec..cd1468c 100644 --- a/src/features/search/useIndexingStatus.ts +++ b/src/features/search/useIndexingStatus.ts @@ -1,25 +1,46 @@ import { useQuery } from '@tanstack/react-query'; +import { useEffect } from 'react'; import { fetchIndexingStatus } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; import type { IndexingStatus } from '@/types'; +import type { IndexScope } from '@/lib/apiClient'; -export function useIndexingStatus(owner: string, repo: string, enabled = true) { - const { setIndexingStatus } = useSearchStore(); +export function useIndexingStatus(owner: string, repo: string, enabled = true, branch?: string, scope?: IndexScope) { + const { setIndexingStatus, indexingStatus, indexOperation, indexAction } = useSearchStore(); - return useQuery({ - queryKey: ['indexingStatus', owner, repo], - queryFn: async () => { - const status = await fetchIndexingStatus(owner, repo); - setIndexingStatus(status); - return status; - }, - enabled: enabled && !!owner && !!repo, - refetchInterval: () => { - // Read from Zustand store (updated by useTriggerIndexing) instead of - // query.state.data which is undefined on first call, preventing polling. - const storeState = useSearchStore.getState().indexingStatus.state; - return storeState === 'indexing' || storeState === 'idle' ? 3000 : false; - }, + const query = useQuery<{ status: IndexingStatus; operation: number; owner: string; repo: string }>({ + queryKey: ['indexingStatus', owner, repo, branch, scope, indexOperation], + queryFn: async () => ({ + status: await fetchIndexingStatus(owner, repo, branch, scope), + operation: indexOperation, + owner, + repo, + }), + enabled: + enabled && + !!owner && + !!repo && + indexAction !== 'cancel' && + (indexingStatus.state !== 'paused' || indexAction === 'index'), + refetchInterval: indexingStatus.state === 'indexing' ? 3000 : false, + // Inherits the global bounded retry (one retry, none for not-found): a + // transient status failure must not leave an existing index looking idle. staleTime: 0, }); -} \ No newline at end of file + // React Query exposes only the current repository's result; old requests cannot update the store. + useEffect(() => { + const state = useSearchStore.getState(); + if (!query.data || query.data.operation !== state.indexOperation || state.indexAction === 'cancel') return; + // The global operation number is shared across repos: only accept a status + // that targets this repo, and that matches the repo owning an active op. + if (query.data.owner !== owner || query.data.repo !== repo) return; + if (state.indexOwner !== null && (state.indexOwner !== owner || state.indexRepo !== repo)) return; + if (state.indexAction === 'index') { + // A long indexing POST owns its final result. Polls may report progress, never undo the optimistic start. + if (query.data.status.state !== 'indexing') return; + } + setIndexingStatus(query.data.status); + useSearchStore.setState({ indexOwner: owner, indexRepo: repo, indexBuildId: query.data.status.buildId }); + }, [query.data, query.dataUpdatedAt, setIndexingStatus, owner, repo]); + return query; +} diff --git a/src/features/search/useSemanticAsk.ts b/src/features/search/useSemanticAsk.ts index 3d783be..7acde8d 100644 --- a/src/features/search/useSemanticAsk.ts +++ b/src/features/search/useSemanticAsk.ts @@ -1,41 +1,68 @@ import { useMutation } from '@tanstack/react-query'; -import { semanticAsk } from '@/lib/apiClient'; +import { ApiError, semanticAsk } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; +import { scopeKey } from '@/lib/scope-key'; + +interface AskVariables { + question: string; + owner: string; + repo: string; + branch?: string; + scope: import('@/lib/apiClient').IndexScope; +} + +type AskContext = { revision: number; owner: string; repo: string; branch?: string; scope: string } | undefined; + +function isStaleResponse(variables: AskVariables, context: AskContext): boolean { + const state = useSearchStore.getState(); + if (!context || context.revision !== state.revision) return true; + return ( + context.owner !== variables.owner || + context.repo !== variables.repo || + context.branch !== variables.branch || + context.scope !== scopeKey(variables.scope) + ); +} + +function handleIndexNotFound(): void { + const state = useSearchStore.getState(); + if (state.indexAction) return; + if (state.indexingStatus.state === 'indexing' || state.indexingStatus.state === 'paused') return; + state.setIndexingStatus({ state: 'idle' }); +} export function useSemanticAsk() { const { setAnswer, setIsLoading, setError, addRecentQuery } = useSearchStore(); return useMutation({ - mutationFn: ({ - question, - owner, - repo, - branch, - }: { - question: string; - owner: string; - repo: string; - branch?: string; - }) => semanticAsk(question, owner, repo, branch), + mutationFn: ({ question, owner, repo, branch, scope }: AskVariables) => + semanticAsk(question, owner, repo, branch, scope), onMutate: (vars) => { setIsLoading(true); setError(null); + useSearchStore.setState({ resultCoverage: undefined }); addRecentQuery(vars.question); + return { + revision: useSearchStore.getState().revision, + owner: vars.owner, + repo: vars.repo, + branch: vars.branch, + scope: scopeKey(vars.scope), + }; }, - onSuccess: (data, vars) => { + onSuccess: (data, variables, context) => { + if (isStaleResponse(variables, context)) return; const qa = { answer: data.answer, citations: data.citations }; setAnswer(qa); + useSearchStore.setState({ resultCoverage: data.coverage }); setIsLoading(false); - // Cache answer for this question - const cacheKey = `${vars.owner}/${vars.repo}:${vars.question}`; - const store = useSearchStore.getState(); - const newCache = new Map(store.askCache); - newCache.set(cacheKey, qa); - useSearchStore.setState({ askCache: newCache }); }, - onError: (err: Error) => { + onError: (err: Error, variables, context) => { + if (isStaleResponse(variables, context)) return; setError(err.message); + useSearchStore.setState({ resultCoverage: undefined }); + if (err instanceof ApiError && err.code === 'INDEX_NOT_FOUND') handleIndexNotFound(); setIsLoading(false); }, }); -} \ No newline at end of file +} diff --git a/src/features/search/useSemanticSearch.ts b/src/features/search/useSemanticSearch.ts index 0f3f310..e0ea1ef 100644 --- a/src/features/search/useSemanticSearch.ts +++ b/src/features/search/useSemanticSearch.ts @@ -1,28 +1,59 @@ import { useMutation } from '@tanstack/react-query'; -import { semanticSearch } from '@/lib/apiClient'; +import { ApiError, semanticSearch } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; +import { scopeKey } from '@/lib/scope-key'; import type { SearchResultCard } from '@/types'; +interface SearchVariables { + query: string; + owner: string; + repo: string; + branch?: string; + scope: import('@/lib/apiClient').IndexScope; +} + +type SearchContext = { revision: number; owner: string; repo: string; branch?: string; scope: string } | undefined; + +function isStaleResponse(variables: SearchVariables, context: SearchContext): boolean { + const state = useSearchStore.getState(); + if (!context || context.revision !== state.revision) return true; + return ( + context.owner !== variables.owner || + context.repo !== variables.repo || + context.branch !== variables.branch || + context.scope !== scopeKey(variables.scope) + ); +} + +// A paused index owns partial coverage: INDEX_NOT_FOUND from a scope edit must +// not flip the global status to idle and hide the resumable operation. +function handleIndexNotFound(): void { + const state = useSearchStore.getState(); + if (state.indexAction) return; + if (state.indexingStatus.state === 'indexing' || state.indexingStatus.state === 'paused') return; + state.setIndexingStatus({ state: 'idle' }); +} + export function useSemanticSearch() { const { setResults, setIsLoading, setError, addRecentQuery } = useSearchStore(); return useMutation({ - mutationFn: ({ - query, - owner, - repo, - branch, - }: { - query: string; - owner: string; - repo: string; - branch?: string; - }) => semanticSearch(query, owner, repo, branch), - onMutate: () => { + mutationFn: ({ query, owner, repo, branch, scope }: SearchVariables) => + semanticSearch(query, owner, repo, branch, scope), + onMutate: (variables) => { setIsLoading(true); setError(null); + useSearchStore.setState({ resultCoverage: undefined }); + return { + revision: useSearchStore.getState().revision, + owner: variables.owner, + repo: variables.repo, + branch: variables.branch, + scope: scopeKey(variables.scope), + }; }, - onSuccess: (data, vars) => { + onSuccess: (data, variables, context) => { + if (isStaleResponse(variables, context)) return; const cards: SearchResultCard[] = data.results.map((r) => ({ filePath: r.chunk.filePath, startLine: r.chunk.startLine, @@ -32,18 +63,16 @@ export function useSemanticSearch() { score: r.score, })); setResults(cards); + useSearchStore.setState({ resultCoverage: data.coverage }); addRecentQuery(data.query); setIsLoading(false); - // Cache results for this query - const cacheKey = `${vars.owner}/${vars.repo}:${vars.query}`; - const store = useSearchStore.getState(); - const newCache = new Map(store.searchCache); - newCache.set(cacheKey, cards); - useSearchStore.setState({ searchCache: newCache }); }, - onError: (err: Error) => { + onError: (err: Error, variables, context) => { + if (isStaleResponse(variables, context)) return; setError(err.message); + useSearchStore.setState({ resultCoverage: undefined }); + if (err instanceof ApiError && err.code === 'INDEX_NOT_FOUND') handleIndexNotFound(); setIsLoading(false); }, }); -} \ No newline at end of file +} diff --git a/src/features/search/useTriggerIndexing.ts b/src/features/search/useTriggerIndexing.ts index 2946108..e702ca5 100644 --- a/src/features/search/useTriggerIndexing.ts +++ b/src/features/search/useTriggerIndexing.ts @@ -1,8 +1,11 @@ -import { useMutation } from '@tanstack/react-query'; -import { triggerIndexing } from '@/lib/apiClient'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { triggerIndexing, type IndexScope } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; +import { recoverIndexingStatus } from './indexing-recovery'; +import { beginIndexOperation, finishIndexOperation } from './index-operations'; export function useTriggerIndexing() { + const queryClient = useQueryClient(); const { setIndexingStatus } = useSearchStore(); return useMutation({ @@ -10,25 +13,43 @@ export function useTriggerIndexing() { owner, repo, branch, + scope, + buildId, }: { owner: string; repo: string; branch?: string; - }) => triggerIndexing(owner, repo, branch), - onMutate: () => { + scope: IndexScope; + buildId: string; + }) => triggerIndexing(owner, repo, branch, scope, buildId), + onMutate: async (variables) => { + const context = beginIndexOperation('index', variables.owner, variables.repo); + // Polling keeps targeting this scope until the operation settles. + useSearchStore.setState({ indexBuildId: variables.buildId, indexScope: variables.scope }); + const { previous } = context; + // Only a paused build resumes its snapshot; a fresh build (e.g. retry after + // a branch advance) must not inherit the old snapshotSha/coverage or polls + // keep targeting a stale snapshot. + const resuming = previous.state === 'paused'; setIndexingStatus({ state: 'indexing', - progress: 0, - filesProcessed: 0, - totalFiles: 0, + progress: resuming ? previous.progress : 0, + filesProcessed: resuming ? previous.filesProcessed : 0, + totalFiles: resuming ? previous.totalFiles : 0, + snapshotSha: resuming ? previous.snapshotSha : undefined, + coverage: resuming ? previous.coverage : undefined, }); + await queryClient.cancelQueries({ queryKey: ['indexingStatus', variables.owner, variables.repo] }); + return context; }, - onError: (error) => { - setIndexingStatus({ - state: 'failed', - error: error instanceof Error ? error.message : 'Failed to start indexing', - failedFiles: 0, - }); + onSuccess: (status, _variables, context) => { + if (context) finishIndexOperation(context, status); + }, + onError: async (error, variables, context) => { + const state = useSearchStore.getState(); + if (!context || context.operation !== state.indexOperation) return; + await queryClient.cancelQueries({ queryKey: ['indexingStatus', variables.owner, variables.repo] }); + finishIndexOperation(context, recoverIndexingStatus(context.previous, error)); }, }); -} \ No newline at end of file +} diff --git a/src/hooks/useAnalyzeRepo.ts b/src/hooks/useAnalyzeRepo.ts index 3eae81c..0a5824d 100644 --- a/src/hooks/useAnalyzeRepo.ts +++ b/src/hooks/useAnalyzeRepo.ts @@ -1,13 +1,41 @@ import { useQuery } from '@tanstack/react-query'; import { analyzeRepo } from '@/lib/apiClient'; +import { useRepoChurn } from './useRepoChurn'; +import { useRepoHealth } from './useRepoHealth'; +import { useMemo } from 'react'; +import { useVizStore } from '@/stores/vizStore'; import type { RepoAnalysis } from '@/types'; -export function useAnalyzeRepo(owner: string, repo: string, branch: string | null = null, enabled = true) { +export function useAnalyzeRepo(owner: string, repo: string, branch: string | null = null, enabled = true, options?: { enrichHealth?: boolean }) { const url = `https://github.com/${owner}/${repo}`; - return useQuery({ + const analysis = useQuery({ queryKey: ['analyze-repo', owner, repo, branch], queryFn: () => analyzeRepo(url, branch || undefined), enabled: enabled && !!owner && !!repo, staleTime: 1000 * 60 * 30, }); + const colorMode = useVizStore((s) => s.colorMode); + const sha = analysis.data?.meta.sha ?? ''; + const health = useRepoHealth(owner, repo, sha, enabled && !!analysis.data && (options?.enrichHealth ?? true)); + const churn = useRepoChurn(owner, repo, sha, enabled && colorMode === 'churn'); + const data = useMemo(() => { + if (!analysis.data) return undefined; + return { + ...analysis.data, + dependencyHealth: health.data, + graph: { + ...analysis.data.graph, + nodes: analysis.data.graph.nodes.map((node) => { + const extra = node.data.path ? churn.data?.files[node.data.path] : undefined; + return extra ? { ...node, data: { ...node.data, ...extra } } : node; + }), + }, + }; + }, [analysis.data, churn.data, health.data]); + return { + ...analysis, + data, + churnLoading: churn.isPending && churn.isFetching, + churnError: churn.error, + }; } diff --git a/src/hooks/useMotionPreference.ts b/src/hooks/useMotionPreference.ts new file mode 100644 index 0000000..d7e65d1 --- /dev/null +++ b/src/hooks/useMotionPreference.ts @@ -0,0 +1,17 @@ +import { useSyncExternalStore } from 'react'; +import { prefersReducedMotion, REDUCED_MOTION_QUERY } from '@/lib/motion-preference'; +import { useMotionStore } from '@/stores/motionStore'; + +function subscribe(listener: () => void) { + const query = window.matchMedia(REDUCED_MOTION_QUERY); + query.addEventListener('change', listener); + const unsubscribe = useMotionStore.subscribe(listener); + return () => { + query.removeEventListener('change', listener); + unsubscribe(); + }; +} + +export function useMotionPreference() { + return useSyncExternalStore(subscribe, prefersReducedMotion, () => false); +} diff --git a/src/hooks/useRepoChurn.ts b/src/hooks/useRepoChurn.ts new file mode 100644 index 0000000..4dbb8c8 --- /dev/null +++ b/src/hooks/useRepoChurn.ts @@ -0,0 +1,34 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { fetchChurnBatch } from '@/lib/apiClient'; +import { mergeChurn, pendingChurn, churnInterval } from '@/lib/churn-progress'; +import { useChatConfigRevision } from '@/stores/chatConfigStore'; + +import type { ChurnResponse } from '@/types/churn'; + +export function useRepoChurn(owner: string, repo: string, sha: string, enabled: boolean) { + const client = useQueryClient(); + // Credential revision scopes cached churn to the PAT that fetched it, so a + // PAT change discards prior-credential results instead of marking them done. + const { revision } = useChatConfigRevision(); + const queryKey = ['repo-churn', owner, repo, sha, revision]; + return useQuery({ + queryKey, + queryFn: async () => { + const previous = client.getQueryData(queryKey); + const pending = previous ? pendingChurn(previous) : undefined; + if (previous && !previous.complete && !pending?.length) return previous; + return mergeChurn( + previous, + await fetchChurnBatch(owner, repo, sha, { + completed: Object.keys(previous?.files ?? {}), + pending, + scope: previous?.scope, + }), + ); + }, + enabled: enabled && !!sha, + retry: false, + staleTime: 30 * 60 * 1000, + refetchInterval: (query) => (enabled ? churnInterval(query.state.data) : false), + }); +} diff --git a/src/hooks/useRepoHealth.ts b/src/hooks/useRepoHealth.ts new file mode 100644 index 0000000..32ecbea --- /dev/null +++ b/src/hooks/useRepoHealth.ts @@ -0,0 +1,17 @@ +import { useQuery } from '@tanstack/react-query'; +import { fetchRepoEnrichment } from '@/lib/apiClient'; +import { useChatConfigRevision } from '@/stores/chatConfigStore'; +import type { DependencyHealthReport } from '@/types'; + +export function useRepoHealth(owner: string, repo: string, sha: string, enabled: boolean) { + // Credential revision keeps one account's private health report from serving + // another account after a PAT change. + const { revision } = useChatConfigRevision(); + return useQuery({ + queryKey: ['repo-health', owner, repo, sha, revision], + queryFn: () => fetchRepoEnrichment(owner, repo, sha, 'health'), + enabled: enabled && !!sha, + retry: false, + staleTime: 30 * 60 * 1000, + }); +} diff --git a/src/hooks/useWorkspaceShortcuts.ts b/src/hooks/useWorkspaceShortcuts.ts index 5c777eb..79b7198 100644 --- a/src/hooks/useWorkspaceShortcuts.ts +++ b/src/hooks/useWorkspaceShortcuts.ts @@ -2,9 +2,8 @@ import { useEffect } from "react"; import { useVizStore } from "@/stores/vizStore"; export function useWorkspaceShortcuts() { - const { - setExplorerOpen, - setAiSidebarOpen + const { + closePanelsPreservingMode, } = useVizStore(); // Responsive panel management @@ -14,8 +13,7 @@ export function useWorkspaceShortcuts() { const handleResize = () => { const isMobile = window.innerWidth < 1024; if (isMobile && wasDesktop) { - setExplorerOpen(false); - setAiSidebarOpen(false); + closePanelsPreservingMode(); } else if (!isMobile && !wasDesktop) { const state = useVizStore.getState(); state.setWorkspaceMode(state.workspaceMode); @@ -25,13 +23,12 @@ export function useWorkspaceShortcuts() { // Initial check on mount if (window.innerWidth < 1024) { - setExplorerOpen(false); - setAiSidebarOpen(false); + closePanelsPreservingMode(); } window.addEventListener("resize", handleResize); return () => window.removeEventListener("resize", handleResize); - }, [setExplorerOpen, setAiSidebarOpen]); + }, [closePanelsPreservingMode]); // Global keyboard shortcuts useEffect(() => { @@ -63,8 +60,7 @@ export function useWorkspaceShortcuts() { return; } if (window.innerWidth < 1024 && (state.explorerOpen || state.aiSidebarOpen)) { - state.setExplorerOpen(false); - state.setAiSidebarOpen(false); + state.closePanelsPreservingMode(); return; } state.setSelectedNodeId(null); diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index 3ff41ed..4d5e00a 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -1,3 +1,4 @@ +import type { ChurnContinuation, ChurnResponse } from '@/types/churn'; import type { AIExplainRequest, AIExplainResponse, @@ -15,10 +16,28 @@ import type { IndexingStatus, } from '@/types'; +export interface IndexScope { + includePaths: string[]; + excludePaths: string[]; +} + function getApiKeyHeader(): Record { try { const key = localStorage.getItem('gitsdm_gemini_api_key'); - return key ? { 'X-Gemini-API-Key': key } : {}; + if (!key) return {}; + const headers: Record = { + 'X-AI-API-Key': key, + 'X-Gemini-API-Key': key, + }; + for (const [storage, header] of [ + ['gitsdm_ai_provider', 'X-AI-Provider'], + ['gitsdm_ai_model', 'X-AI-Model'], + ['gitsdm_ai_base_url', 'X-AI-Base-URL'], + ]) { + const value = localStorage.getItem(storage)?.trim(); + if (value) headers[header] = value; + } + return headers; } catch { return {}; } @@ -39,7 +58,16 @@ export class ApiError extends Error { public details?: unknown; public error?: string; - constructor(message: string, status: number, data?: { code?: string; details?: unknown; context?: unknown; error?: string }) { + constructor( + message: string, + status: number, + data?: { + code?: string; + details?: unknown; + context?: unknown; + error?: string; + }, + ) { super(message); this.name = 'ApiError'; this.status = status; @@ -69,7 +97,7 @@ async function parseBody(res: Response): Promise { async function request(path: string, options?: RequestInit): Promise { try { - const isAiRoute = path.startsWith('/api/ai'); + const isAiRoute = path.startsWith('/api/ai') || path === '/api/search/ask'; const res = await fetch(path, { ...options, headers: { @@ -85,26 +113,23 @@ async function request(path: string, options?: RequestInit): Promise { data = await parseBody(res); } catch (error) { if (!res.ok) { - throw new ApiError(error instanceof Error ? error.message : `Request failed with status ${res.status}`, res.status); + throw new ApiError( + error instanceof Error ? error.message : `Request failed with status ${res.status}`, + res.status, + ); } throw error; } if (!res.ok) { const message = - typeof data === 'object' && - data !== null && - 'message' in data && - typeof data.message === 'string' + typeof data === 'object' && data !== null && 'message' in data && typeof data.message === 'string' ? data.message - : typeof data === 'object' && - data !== null && - 'error' in data && - typeof data.error === 'string' + : typeof data === 'object' && data !== null && 'error' in data && typeof data.error === 'string' ? data.error : typeof data === 'string' - ? data - : `Request failed with status ${res.status}`; + ? data + : `Request failed with status ${res.status}`; throw new ApiError(message, res.status, typeof data === 'object' && data !== null ? data : undefined); } @@ -116,7 +141,6 @@ async function request(path: string, options?: RequestInit): Promise { } } - export async function analyzeRepo(url: string, branch?: string): Promise { return request('/api/repo/analyze', { method: 'POST', @@ -124,18 +148,23 @@ export async function analyzeRepo(url: string, branch?: string): Promise { +export function fetchRepoEnrichment(owner: string, repo: string, sha: string, kind: 'churn' | 'health'): Promise { + return request(`/api/repo/${kind}?${new URLSearchParams({ owner, repo, branch: sha })}`); +} + +export function fetchChurnBatch(owner: string, repo: string, sha: string, continuation: ChurnContinuation) { + return request(`/api/repo/churn?${new URLSearchParams({ owner, repo, branch: sha })}`, { + method: 'POST', + body: JSON.stringify(continuation), + }); +} + +export async function fetchRepoBranches(owner: string, repo: string): Promise<{ name: string; protected: boolean }[]> { const params = new URLSearchParams({ owner, repo }); return request<{ name: string; protected: boolean }[]>(`/api/repo/branches?${params}`); } -export async function fetchRepoTags( - owner: string, - repo: string, -): Promise<{ name: string; sha: string }[]> { +export async function fetchRepoTags(owner: string, repo: string): Promise<{ name: string; sha: string }[]> { const params = new URLSearchParams({ owner, repo }); return request<{ name: string; sha: string }[]>(`/api/repo/tags?${params}`); } @@ -149,9 +178,7 @@ export async function fetchRepoFile( const queryObj: Record = { owner, repo, path }; if (branch) queryObj.branch = branch; const params = new URLSearchParams(queryObj); - return request<{ path: string; content: string; sha: string }>( - `/api/repo/file?${params}`, - ); + return request<{ path: string; content: string; sha: string }>(`/api/repo/file?${params}`); } export async function fetchTrending(): Promise { @@ -170,66 +197,42 @@ export async function aiExplain(body: AIExplainRequest & { branch?: string }): P }); } -export async function aiExplainLif( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiExplainLif(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/explain-lif', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiRefactor( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiRefactor(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/refactor', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiHealth( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiHealth(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/health', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiMermaid( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiMermaid(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/mermaid', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiRoast( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiRoast(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/roast', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiReadmeEnhance( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiReadmeEnhance(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/readme-enhance', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), @@ -255,10 +258,11 @@ export async function semanticSearch( owner: string, repo: string, branch?: string, + scope?: IndexScope, ): Promise { return request('/api/search', { method: 'POST', - body: JSON.stringify({ query, owner, repo, branch }), + body: JSON.stringify({ query, owner, repo, branch, ...scope }), }); } @@ -267,10 +271,11 @@ export async function semanticAsk( owner: string, repo: string, branch?: string, + scope?: IndexScope, ): Promise { return request('/api/search/ask', { method: 'POST', - body: JSON.stringify({ question, owner, repo, branch }), + body: JSON.stringify({ question, owner, repo, branch, ...scope }), }); } @@ -278,18 +283,40 @@ export async function triggerIndexing( owner: string, repo: string, branch?: string, -): Promise<{ status: string }> { - return request<{ status: string }>('/api/search/index', { + scope?: IndexScope, + buildId?: string, +): Promise { + return request('/api/search/index', { method: 'POST', - body: JSON.stringify({ owner, repo, branch }), + body: JSON.stringify({ owner, repo, branch, ...scope, buildId }), + }); +} + +export async function cancelIndexing( + owner: string, + repo: string, + branch?: string, + scope?: IndexScope, + buildId?: string, +): Promise { + return request('/api/search/cancel', { + method: 'POST', + body: JSON.stringify({ owner, repo, branch, ...scope, buildId }), }); } export async function fetchIndexingStatus( owner: string, repo: string, + branch?: string, + scope?: IndexScope, ): Promise { - const params = new URLSearchParams({ owner, repo }); + const params = new URLSearchParams({ + owner, + repo, + ...(branch ? { branch } : {}), + ...(scope?.includePaths.length ? { includePaths: JSON.stringify(scope.includePaths) } : {}), + ...(scope?.excludePaths.length ? { excludePaths: JSON.stringify(scope.excludePaths) } : {}), + }); return request(`/api/search/status?${params}`); } - diff --git a/src/lib/churn-progress.test.ts b/src/lib/churn-progress.test.ts new file mode 100644 index 0000000..30e79c8 --- /dev/null +++ b/src/lib/churn-progress.test.ts @@ -0,0 +1,52 @@ +import { expect, test } from 'bun:test'; +import { mergeChurn, pendingChurn, churnInterval } from './churn-progress'; +import type { ChurnResponse } from '@/types/churn'; +const first: ChurnResponse = { + files: { a: { commitCount: 2, authorCount: 1, churnScore: 1 } }, + checked: 1, + total: 3, + complete: false, + remaining: ['b', 'c'], + failures: {}, +}; +test('responses from fresh instances accumulate without losing prior successes', () => { + const result = mergeChurn(first, { + ...first, + files: { b: { commitCount: 4, authorCount: 1, churnScore: 1 } }, + remaining: ['c'], + }); + expect(result.checked).toBe(2); + expect(result.files.a.churnScore).toBe(0.5); + expect(result.remaining).toEqual(['c']); +}); +test('failed files do not starve unchecked files and rate limits schedule recovery', () => { + const failed: ChurnResponse = { + ...first, + failures: { b: { issue: 'timeout-or-network', retryAt: 2000 } }, + issue: 'timeout-or-network', + retryAt: 2000, + }; + expect(pendingChurn(failed, 1000)).toEqual(['c']); + expect(pendingChurn(failed, 3000)).toEqual(['c', 'b']); + expect(churnInterval({ ...failed, issue: 'rate-limit' }, 1000)).toBe(1500); + expect(churnInterval({ ...first, complete: true })).toBe(false); +}); +test('per-file access failures do not halt polling for the remaining batch', () => { + // 'b' failed access but 'c' was never attempted: keep polling instead of halting. + expect(churnInterval({ ...first, issue: 'access', remaining: ['b', 'c'], failures: {} }, 1000)).toBe(1500); + expect( + churnInterval( + { + ...first, + issue: 'access', + remaining: ['b'], + failures: { b: { issue: 'access', retryAt: 5000 } }, + }, + 1000, + ), + ).toBe(false); +}); +test('merged batches keep the latest credential scope so continuations stay scoped', () => { + const result = mergeChurn({ ...first, scope: 'scope-a' }, { ...first, scope: 'scope-b', remaining: [] }); + expect(result.scope).toBe('scope-b'); +}); diff --git a/src/lib/churn-progress.ts b/src/lib/churn-progress.ts new file mode 100644 index 0000000..2835657 --- /dev/null +++ b/src/lib/churn-progress.ts @@ -0,0 +1,37 @@ +import type { ChurnResponse } from '@/types/churn'; + +export function mergeChurn(previous: ChurnResponse | undefined, batch: ChurnResponse): ChurnResponse { + const files = { ...previous?.files, ...batch.files }; + const max = Math.max(1, ...Object.values(files).map((file) => file.commitCount)); + for (const [path, file] of Object.entries(files)) files[path] = { ...file, churnScore: file.commitCount / max }; + const remaining = batch.remaining.filter((path) => !files[path]); + const failures = { ...previous?.failures, ...batch.failures }; + for (const path of Object.keys(failures)) if (!remaining.includes(path)) delete failures[path]; + const issue = Object.values(failures).sort((a, b) => a.retryAt - b.retryAt)[0]; + return { + ...batch, + files, + remaining, + failures, + checked: Object.keys(files).length, + complete: remaining.length === 0, + issue: issue?.issue, + retryAt: issue?.retryAt, + }; +} + +export function pendingChurn(data: ChurnResponse, now = Date.now()): string[] { + if (data.issue === 'rate-limit' && (data.retryAt ?? 0) > now) return []; + return data.remaining + .filter((path) => !data.failures[path] || data.failures[path].retryAt <= now) + .sort((a, b) => Number(!!data.failures[a]) - Number(!!data.failures[b])); +} + +export function churnInterval(data: ChurnResponse | undefined, now = Date.now()): number | false { + if (!data || data.complete) return false; + // Repo-wide 401/403 halt the whole batch; per-file 404s must not stop polling the rest. + if (data.issue === 'access' && data.remaining.every((path) => data.failures[path]?.issue === 'access')) + return false; + if (pendingChurn(data, now).length) return 1500; + return Math.max(1500, (data.retryAt ?? now) - now); +} diff --git a/src/lib/motion-preference.test.ts b/src/lib/motion-preference.test.ts new file mode 100644 index 0000000..e97ff37 --- /dev/null +++ b/src/lib/motion-preference.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from 'bun:test'; +import { motionDuration, prefersReducedMotion, REDUCED_MOTION_QUERY } from './motion-preference'; +import { useMotionStore } from '@/stores/motionStore'; + +test('graph camera movement is immediate only when reduced motion is enabled', () => { + expect(motionDuration(400, true)).toBe(0); + expect(motionDuration(400, false)).toBe(400); +}); + +test('preference defaults to system and stays full-motion without an OS signal', () => { + expect(useMotionStore.getState().preference).toBe('system'); + // Headless (no window/matchMedia): OS signal unavailable, keep full motion. + expect(prefersReducedMotion()).toBe(false); +}); + +test('explicit preference overrides the OS reduced-motion signal', () => { + useMotionStore.getState().setPreference('reduced'); + expect(prefersReducedMotion()).toBe(true); + useMotionStore.getState().setPreference('full'); + expect(prefersReducedMotion()).toBe(false); + useMotionStore.setState({ preference: 'system' }); +}); + +test("the 'system' preference consults window.matchMedia", () => { + const globalRef = globalThis as unknown as { + window?: { matchMedia: (query: string) => { matches: boolean } }; + }; + const original = globalRef.window; + useMotionStore.setState({ preference: 'system' }); + globalRef.window = { matchMedia: (query) => ({ matches: query === REDUCED_MOTION_QUERY }) }; + try { + expect(prefersReducedMotion()).toBe(true); + globalRef.window = { matchMedia: () => ({ matches: false }) }; + expect(prefersReducedMotion()).toBe(false); + } finally { + globalRef.window = original; + useMotionStore.setState({ preference: 'system' }); + } +}); diff --git a/src/lib/motion-preference.ts b/src/lib/motion-preference.ts new file mode 100644 index 0000000..8c693be --- /dev/null +++ b/src/lib/motion-preference.ts @@ -0,0 +1,12 @@ +export const REDUCED_MOTION_QUERY = '(prefers-reduced-motion: reduce)'; + +export function prefersReducedMotion(): boolean { + const preference = useMotionStore.getState().preference; + if (preference !== 'system') return preference === 'reduced'; + return typeof window !== 'undefined' && window.matchMedia(REDUCED_MOTION_QUERY).matches; +} + +export function motionDuration(duration: number, reducedMotion = prefersReducedMotion()): number { + return reducedMotion ? 0 : duration; +} +import { useMotionStore } from '@/stores/motionStore'; diff --git a/src/lib/page-metadata.test.ts b/src/lib/page-metadata.test.ts new file mode 100644 index 0000000..de18535 --- /dev/null +++ b/src/lib/page-metadata.test.ts @@ -0,0 +1,17 @@ +import { expect, test } from 'bun:test'; +import { pageMetadata } from './page-metadata'; +test('valid navigation clears 404 robots and privacy canonical metadata', () => { + expect(pageMetadata('/missing').robots).toBe('noindex'); + expect(pageMetadata('/').robots).toBe('index, follow'); + expect(pageMetadata('/privacy').canonical).toBe('https://gsdm.site/privacy'); + expect(pageMetadata('/').canonical).toBe('https://gsdm.site/'); + expect(pageMetadata('/mock/todo-app').canonical).toBeUndefined(); +}); + +test('terms metadata supports direct and trailing-slash navigation', () => { + for (const path of ['/terms', '/terms/']) { + expect(pageMetadata(path).title).toBe('Terms of use — gitSdm'); + expect(pageMetadata(path).canonical).toBe('https://gsdm.site/terms'); + expect(pageMetadata(path).robots).toBe('index, follow'); + } +}); diff --git a/src/lib/page-metadata.ts b/src/lib/page-metadata.ts new file mode 100644 index 0000000..35e86b9 --- /dev/null +++ b/src/lib/page-metadata.ts @@ -0,0 +1,34 @@ +export function pageMetadata(path: string) { + const privacy = /^\/privacy\/?$/.test(path); + const terms = /^\/terms\/?$/.test(path); + const valid = + path === '/' || + privacy || + terms || + /^\/(?!api\/|assets\/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38}\/[a-zA-Z0-9_.-]{1,100}(?:\/search)?\/?$/.test(path); + return { + title: privacy + ? 'Privacy policy — gitSdm' + : terms + ? 'Terms of use — gitSdm' + : valid + ? 'gitSdmᵝ — Git Software Dependency Map' + : 'Page not found — gitSdm', + description: privacy + ? 'How gitSdm stores settings, sends repository content to providers, and handles caches and logs.' + : terms + ? 'Rules for using gitSdm, including commercial use, automation, repository content, and service limits.' + : valid + ? 'Interactive dependency visualization for exploring repository files, modules, and architecture.' + : 'This page could not be found. Return to gitSdm to explore a repository.', + canonical: + path === '/' + ? 'https://gsdm.site/' + : privacy + ? 'https://gsdm.site/privacy' + : terms + ? 'https://gsdm.site/terms' + : undefined, + robots: valid ? 'index, follow' : 'noindex', + }; +} diff --git a/src/lib/scope-key.ts b/src/lib/scope-key.ts new file mode 100644 index 0000000..7177443 --- /dev/null +++ b/src/lib/scope-key.ts @@ -0,0 +1,11 @@ +import type { IndexScope } from '@/lib/apiClient'; + +// Canonical scope identity for guarding stale async responses: JSON with +// include/exclude paths sorted, so equal scopes produce equal keys +// regardless of input order. +export function scopeKey(scope: IndexScope): string { + return JSON.stringify({ + include: [...scope.includePaths].sort(), + exclude: [...scope.excludePaths].sort(), + }); +} diff --git a/src/lib/search-coverage.ts b/src/lib/search-coverage.ts new file mode 100644 index 0000000..565b3f5 --- /dev/null +++ b/src/lib/search-coverage.ts @@ -0,0 +1,9 @@ +import type { SearchCoverage } from '@/types/api'; + +export function coverageMessage(coverage: SearchCoverage): string { + if (coverage.kind === 'previous') + return `Showing results from the previous index (${coverage.commitSha.slice(0, 7)}).`; + if (coverage.kind === 'partial') + return `Partial results — ${coverage.indexedFiles} of ${coverage.totalFiles} files indexed.`; + return 'Showing results from the complete index.'; +} diff --git a/src/lib/utils.test.ts b/src/lib/utils.test.ts index f2146b0..21c4fa0 100644 --- a/src/lib/utils.test.ts +++ b/src/lib/utils.test.ts @@ -99,6 +99,11 @@ describe('utils', () => { expect(parseRepoFromUrl('https://github.com/facebook/react.git')).toEqual({ owner: 'facebook', repo: 'react' }); }); + it('strips .git before query/hash', () => { + expect(parseRepoFromUrl('https://github.com/facebook/react.git?tab=readme-ov-file')).toEqual({ owner: 'facebook', repo: 'react' }); + expect(parseRepoFromUrl('https://github.com/facebook/react.git#readme')).toEqual({ owner: 'facebook', repo: 'react' }); + }); + it('handles URLs with query parameters', () => { expect(parseRepoFromUrl('https://github.com/facebook/react?tab=readme-ov-file')).toEqual({ owner: 'facebook', repo: 'react' }); }); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index df88e28..62a4b3f 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -14,12 +14,15 @@ export function formatStars(n: number): string { } export function parseRepoFromUrl(url: string): { owner: string; repo: string } | null { - const trimmed = url.trim().replace(/\/$/, '').replace(/\.git$/, ''); - const sshMatch = trimmed.match(/^git@github\.com:([^/]+)\/([^/?#]+)$/i); + const trimmed = url.trim().replace(/\/$/, ''); + // Strip a trailing .git before query/hash splitting so `repo.git?x=1` resolves to `repo`. + const withoutGitSuffix = trimmed.replace(/\.git(?=[?#]|$)/i, ''); + const sshMatch = withoutGitSuffix.match(/^git@github\.com:([^/]+)\/([^/?#]+)$/i); if (sshMatch) return { owner: sshMatch[1], repo: sshMatch[2] }; - const shorthandMatch = trimmed.match(/^([A-Za-z0-9-]+)\/([A-Za-z0-9._-]+)$/); - if (shorthandMatch) return { owner: shorthandMatch[1], repo: shorthandMatch[2] }; + const shorthandMatch = withoutGitSuffix.match(/^([A-Za-z0-9-]+)\/([A-Za-z0-9._-]+)$/); + if (shorthandMatch && !withoutGitSuffix.includes('?') && !withoutGitSuffix.includes('#')) + return { owner: shorthandMatch[1], repo: shorthandMatch[2] }; try { let urlToParse = trimmed; @@ -35,7 +38,8 @@ export function parseRepoFromUrl(url: string): { owner: string; repo: string } | const parts = urlObj.pathname.split('/').filter(Boolean); if (parts.length >= 2) { - return { owner: parts[0], repo: parts[1] }; + // Same .git strip covers https URLs with query/hash (`repo.git?x=1`). + return { owner: parts[0], repo: parts[1].replace(/\.git$/i, '') }; } return null; } catch { diff --git a/src/lib/workspace-mode.test.ts b/src/lib/workspace-mode.test.ts new file mode 100644 index 0000000..a6a8629 --- /dev/null +++ b/src/lib/workspace-mode.test.ts @@ -0,0 +1,18 @@ +import { expect, test } from 'bun:test'; +import { modeFromPanels, panelsForMode } from './workspace-mode'; +test('full workspace has a useful mobile panel and restores both desktop panels', () => { + expect(panelsForMode('full', true)).toEqual({ + explorerOpen: true, + aiSidebarOpen: false, + }); + expect(panelsForMode('full', false)).toEqual({ + explorerOpen: true, + aiSidebarOpen: true, + }); +}); +test('legacy modes migrate according to actual saved panel visibility', () => { + expect(modeFromPanels(false, true)).toBe('insights'); + expect(modeFromPanels(true, false)).toBe('explorer'); + expect(modeFromPanels(true, true)).toBe('full'); + expect(modeFromPanels(false, false)).toBe('focus'); +}); diff --git a/src/lib/workspace-mode.ts b/src/lib/workspace-mode.ts new file mode 100644 index 0000000..45eb268 --- /dev/null +++ b/src/lib/workspace-mode.ts @@ -0,0 +1,10 @@ +export type SpatialMode = 'full' | 'explorer' | 'insights' | 'focus'; +export function modeFromPanels(explorer: boolean, insights: boolean): SpatialMode { + return explorer ? (insights ? 'full' : 'explorer') : insights ? 'insights' : 'focus'; +} +export function panelsForMode(mode: SpatialMode, mobile: boolean) { + return { + explorerOpen: mode === 'explorer' || mode === 'full', + aiSidebarOpen: mode === 'insights' || (mode === 'full' && !mobile), + }; +} diff --git a/src/main.tsx b/src/main.tsx index bf162d5..6dbaaef 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -4,10 +4,31 @@ import { Providers } from '@/app/providers'; import { AppRouter } from '@/app/router'; import '@/styles/globals.css'; -createRoot(document.getElementById('root')!).render( - - - - - , -); +async function start() { + const initialPath = window.location.pathname; + const initialPage = + initialPath === '/' + ? (await import('@/pages/HomePage')).HomePage + : /^\/privacy\/?$/.test(initialPath) + ? (await import('@/pages/PrivacyPage')).PrivacyPage + : /^\/terms\/?$/.test(initialPath) + ? (await import('@/pages/TermsPage')).TermsPage + : /^\/[^/]+\/[^/]+\/search\/?$/.test(initialPath) + ? (await import('@/pages/SearchPage')).SearchPage + : /^\/[^/]+\/[^/]+\/?$/.test(initialPath) + ? (await import('@/pages/VizPage')).VizPage + : (await import('@/pages/NotFoundPage')).NotFoundPage; + createRoot(document.getElementById('root')!).render( + + + + + , + ); +} +void start().catch(() => { + const message = document.createElement('p'); + message.setAttribute('role', 'alert'); + message.textContent = 'Interactive features could not load. Please refresh to try again.'; + document.body.appendChild(message); +}); diff --git a/src/pages/HomePage.tsx b/src/pages/HomePage.tsx index ff7eec3..ba0cad6 100644 --- a/src/pages/HomePage.tsx +++ b/src/pages/HomePage.tsx @@ -1,14 +1,18 @@ -import { useEffect, useState } from "react"; -import { Navbar } from "@/components/layout/Navbar"; -import { HeroSection } from "@/components/home/HeroSection"; -import { StatsStrip } from "@/components/home/StatsStrip"; -import { HowItWorks } from "@/components/home/HowItWorks"; -import { Trending } from "@/components/home/Trending"; -import { CapabilityGroups } from "@/components/home/CapabilityGroups"; -import { LAST_REPO_KEY } from "@/lib/utils"; +import { SiteFooter } from '@/components/layout/SiteFooter'; +import { prefersReducedMotion } from '@/lib/motion-preference'; +import { useEffect, useState } from 'react'; +import { Navbar } from '@/components/layout/Navbar'; +import { HeroSection } from '@/components/home/HeroSection'; +import { StatsStrip } from '@/components/home/StatsStrip'; +import { HowItWorks } from '@/components/home/HowItWorks'; +import { Trending } from '@/components/home/Trending'; +import { CapabilityGroups } from '@/components/home/CapabilityGroups'; +import { LAST_REPO_KEY } from '@/lib/utils'; export function HomePage() { - const [repoUrl] = useState(() => localStorage.getItem(LAST_REPO_KEY) ?? ""); + const [repoUrl] = useState(() => + typeof localStorage === 'undefined' ? '' : (localStorage.getItem(LAST_REPO_KEY) ?? ''), + ); useEffect(() => { const sectionId = window.location.hash.slice(1); @@ -16,8 +20,8 @@ export function HomePage() { const timer = window.setTimeout(() => { document.getElementById(sectionId)?.scrollIntoView({ - behavior: "smooth", - block: "start", + behavior: prefersReducedMotion() ? 'instant' : 'smooth', + block: 'start', }); }, 80); @@ -34,42 +38,7 @@ export function HomePage() { - +
); } diff --git a/src/pages/NotFoundPage.tsx b/src/pages/NotFoundPage.tsx index 90b9de0..99e97af 100644 --- a/src/pages/NotFoundPage.tsx +++ b/src/pages/NotFoundPage.tsx @@ -1,81 +1,25 @@ -import { useNavigate } from 'react-router-dom'; -import { Home, ArrowLeft, Terminal } from 'lucide-react'; -import { motion } from 'framer-motion'; -import { Button } from '@/components/ui/button'; +import { Link } from 'react-router-dom'; +import { ArrowRight, FileQuestion, GitBranch } from 'lucide-react'; +import { InfoPageLayout } from '@/components/layout/InfoPageLayout'; export function NotFoundPage() { - const navigate = useNavigate(); - return ( -
- {/* Subtle gradient background */} -
-
-
-
- - {/* Faint grid texture */} -
- - - {/* Brand mark */} -
- - gitSdm -
- - {/* 404 */} -
-

- 404 -

+ +
+

404 / Page not found

+

This path ends here.

+

The page may have moved, or the address may be incorrect. Return home to open a repository and keep exploring.

+
+
+
+
+

Find your next codebase

Open a GitHub link or try the sample project.

+ Go to homepage
- - {/* Heading */} -

Page not found

-

- The page you're looking for doesn't exist or has been moved. -

- - {/* Actions */} -
- - - -
- - {/* Helper text */} -

- Can't find what you need? Try checking our{' '} - - documentation - . -

- -
+
+ ); } diff --git a/src/pages/PrivacyPage.tsx b/src/pages/PrivacyPage.tsx new file mode 100644 index 0000000..9fc9a2d --- /dev/null +++ b/src/pages/PrivacyPage.tsx @@ -0,0 +1,104 @@ +import { Link } from 'react-router-dom'; +import { InfoPageLayout } from '@/components/layout/InfoPageLayout'; + +export function PrivacyPage() { + return ( + +
+
+

Your data / Privacy

+

Privacy policy

+

Updated September 12, 2026

+

+ gitSdm saves your settings in your browser and sends data to its server, GitHub, and the services used by + its AI features. This page explains what is stored and shared. If you use a self-hosted installation, its + hosting and provider settings may differ. +

+
+
+

What your browser saves

+

+ Your browser saves GitHub tokens and AI API keys in localStorage on this device. It also remembers your + workspace preferences, saved chat provider/model/base URL settings, and the last repository you opened. These stay saved after you close the browser. Use + Clear in Settings to remove saved keys, or clear this site's browser data to remove all local settings. + Avoid saving keys on a shared device. +

+
+
+

What gitSdm sends to other services

+

+ When you request a repository, your browser sends its owner and name to the gitSdm server, along with + relevant saved credentials in request headers. The server uses your GitHub credentials to fetch repository + details and files from GitHub. Saved credentials therefore also leave your device when used for these + requests. +

+

+ AI features may send your prompts, questions, and relevant repository content to the configured AI provider. + That content can include private code accessed with your GitHub token. To build a semantic search index, + gitSdm sends sections of code to the configured embedding provider. The Ask feature can send code found by + the search to an AI provider to answer your question. +

+

+ The provider may be Google Gemini, OpenAI, Anthropic, or another compatible service. Each provider + has its own rules for privacy and data retention. If you have not added your own API key, the installation + may use a key supplied by its operator. Only submit code you have permission to share with those services. +

+
+
+

Server storage and logs

+

+ The server keeps repository analysis and AI results in memory so it can reuse them. These caches have size + limits and expire after a set time. Completed semantic search indexes expire 30 minutes after they are + stored. Saved progress for resumable indexing contains code snippets, embeddings, and a list of completed + files; its 25-hour expiry resets whenever progress is saved. Expired entries are removed during later access + or cleanup. Memory limits can remove completed indexes sooner, and a server restart clears in-memory data. + Clearing your browser data does not delete server caches, search indexes, or records held by providers. +

+

+ Application logs record request routes, timing, repository identifiers, and errors. Provider error messages + may contain information you submitted. Hosting and network providers may also process connection details, + such as your IP address. How long logs are kept depends on who operates the installation; there is no single + retention period across deployments. +

+
+
+

Usage limits

+

+ The server processes your IP address to limit requests and stores a hash of that address with a request + counter. A hash is not a guarantee of anonymity. It also tracks shared usage totals to control provider + costs. Depending on the installation, these counters are held in memory or sent to Upstash Redis. Counters + expire at the end of their usage window; in-memory cleanup happens on later requests. +

+

+ Repository caches and search indexes use hashed credential identifiers to separate access scopes. Raw API + keys and tokens are not used as cache keys. Clearing browser data does not reset server-side usage limits. +

+
+
+

Other services and your choices

+

+ gitSdm loads fonts from Google Fonts and may show images hosted by GitHub or other services. Your browser + shares connection details with those services when it loads these files. The app itself does not include a + dedicated advertising or analytics tracker. +

+

+ You can try the mock demos without adding any credentials. To remove saved keys, use Settings. To revoke a + key or token, use the service that issued it. For privacy questions, email{' '} + + hello@gsdm.site + + , or contact your operator if you use a self-hosted installation. Do not post tokens, private code, or + personal information in public issues. +

+

+ See the{' '} + + Terms of use + {' '} + for the rules on commercial use, automation, and service limits. +

+
+
+ + ); +} diff --git a/src/pages/SearchPage.test.ts b/src/pages/SearchPage.test.ts new file mode 100644 index 0000000..4674efd --- /dev/null +++ b/src/pages/SearchPage.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it } from 'bun:test'; +import { parsePaths } from '@/features/search/parsePaths'; +import { useSearchStore } from '@/features/search/searchStore'; +import type { IndexingStatus } from '@/types'; +import { + isOperationActive, + isSearchEnabled, + resolveBuildId, + resolvePollingScope, + resolveRequestBranch, + resolveRunningSha, + shouldResetForRepoChange, + shouldResetSearchState, +} from './search-page-logic'; +import { scopeKey } from '@/lib/scope-key'; + +const snapshot = ( + indexAction: 'index' | 'cancel' | null, + indexingStatus: IndexingStatus, +) => ({ indexAction, indexingStatus }); + +describe('SearchPage logic & state contracts', () => { + it('parses scoped include and exclude path inputs cleanly', () => { + expect(parsePaths('')).toEqual([]); + expect(parsePaths(' ')).toEqual([]); + expect(parsePaths('src, test, ')).toEqual(['src', 'test']); + expect(parsePaths('server/api, components/ui/button.tsx')).toEqual([ + 'server/api', + 'components/ui/button.tsx', + ]); + }); + + it('skips the scope-change reset while an index operation is active', () => { + expect(shouldResetSearchState(snapshot(null, { state: 'idle' }))).toBe(true); + expect(shouldResetSearchState(snapshot(null, { state: 'complete', chunkCount: 1, timestamp: 1 }))).toBe(true); + expect(shouldResetSearchState(snapshot(null, { state: 'failed', error: 'x', failedFiles: 1 }))).toBe(true); + expect(shouldResetSearchState(snapshot('index', { state: 'idle' }))).toBe(false); + expect(shouldResetSearchState(snapshot('cancel', { state: 'idle' }))).toBe(false); + expect( + shouldResetSearchState(snapshot(null, { state: 'indexing', progress: 1, filesProcessed: 1, totalFiles: 2 })), + ).toBe(false); + expect(shouldResetSearchState(snapshot(null, { state: 'paused', error: 'x', reason: 'rate-limit' }))).toBe(false); + }); + + it('scope change on an idle index removes cached results and coverage', () => { + useSearchStore.setState({ + indexingStatus: { state: 'complete', chunkCount: 5, timestamp: 1 }, + indexAction: null, + results: [{ filePath: 'a.ts', startLine: 1, endLine: 5, snippet: 'code', language: 'ts', score: 0.9 }], + answer: { answer: 'text', citations: [] }, + resultCoverage: { indexedFiles: 10, totalFiles: 50, percent: 20 }, + }); + + expect(shouldResetSearchState(useSearchStore.getState())).toBe(true); + useSearchStore.getState().reset(); + + const updated = useSearchStore.getState(); + expect(updated.results).toEqual([]); + expect(updated.answer).toBeNull(); + expect(updated.resultCoverage).toBeUndefined(); + expect(updated.indexingStatus.state).toBe('idle'); + }); + + it('scope change during an active operation clears results but preserves the operation', () => { + useSearchStore.setState({ + indexingStatus: { state: 'indexing', progress: 20, filesProcessed: 5, totalFiles: 25, snapshotSha: 'abc1234' }, + indexAction: null, + indexOperation: 7, + indexBuildId: 'build-7', + indexScope: { includePaths: ['server'], excludePaths: [] }, + results: [{ filePath: 'a.ts', startLine: 1, endLine: 5, snippet: 'code', language: 'ts', score: 0.9 }], + resultCoverage: { indexedFiles: 3, totalFiles: 25, percent: 12 }, + }); + + expect(shouldResetSearchState(useSearchStore.getState())).toBe(false); + useSearchStore.getState().resetQueryResults(); + + const updated = useSearchStore.getState(); + expect(updated.results).toEqual([]); + expect(updated.resultCoverage).toBeUndefined(); + expect(updated.indexingStatus.state).toBe('indexing'); + expect(updated.indexOperation).toBe(7); + expect(updated.indexBuildId).toBe('build-7'); + expect(updated.indexScope).toEqual({ includePaths: ['server'], excludePaths: [] }); + }); + + it('polls the operation scope while active and follows the live scope after it settles', () => { + const live = { includePaths: ['new'], excludePaths: [] }; + const captured = { includePaths: ['old'], excludePaths: [] }; + + expect(isOperationActive('index', { state: 'idle' })).toBe(true); + expect(isOperationActive(null, { state: 'indexing', progress: 1, filesProcessed: 1, totalFiles: 2 })).toBe(true); + expect(isOperationActive(null, { state: 'paused', error: 'x', reason: 'rate-limit' })).toBe(true); + expect(isOperationActive(null, { state: 'complete', chunkCount: 1, timestamp: 1 })).toBe(false); + + expect(resolvePollingScope(true, captured, live)).toBe(captured); + expect(resolvePollingScope(true, null, live)).toBe(live); + expect(resolvePollingScope(false, captured, live)).toBe(live); + }); + + it('resolves the request branch from the running snapshot before the selected branch', () => { + expect(resolveRunningSha({ state: 'idle' })).toBeUndefined(); + expect(resolveRunningSha({ state: 'complete', chunkCount: 1, timestamp: 1 })).toBeUndefined(); + expect( + resolveRunningSha({ state: 'indexing', progress: 1, filesProcessed: 1, totalFiles: 2, snapshotSha: 'abc1234' }), + ).toBe('abc1234'); + expect(resolveRunningSha({ state: 'paused', error: 'x', reason: 'rate-limit', snapshotSha: 'def5678' })).toBe( + 'def5678', + ); + + expect(resolveRequestBranch('abc1234', 'main')).toBe('abc1234'); + expect(resolveRequestBranch(undefined, 'main')).toBe('main'); + expect(resolveRequestBranch(undefined, undefined)).toBeUndefined(); + }); + + it('reuses the paused build id and starts a fresh one otherwise', () => { + expect(resolveBuildId({ state: 'paused', error: 'x', reason: 'rate-limit' }, 'build-1')).toBe('build-1'); + expect(resolveBuildId({ state: 'paused', error: 'x', reason: 'rate-limit' }, undefined)).toMatch(/^[0-9a-f-]{36}$/); + expect(resolveBuildId({ state: 'idle' }, 'build-1')).not.toBe('build-1'); + expect(resolveBuildId({ state: 'complete', chunkCount: 1, timestamp: 1 }, 'build-1')).toMatch(/^[0-9a-f-]{36}$/); + }); + + it('keeps search enabled for partial coverage and complete indexes only', () => { + expect(isSearchEnabled({ state: 'complete', chunkCount: 1, timestamp: 1 })).toBe(true); + expect( + isSearchEnabled({ + state: 'paused', + error: 'x', + reason: 'rate-limit', + coverage: { indexedFiles: 15, totalFiles: 30, percent: 50 }, + }), + ).toBe(true); + expect(isSearchEnabled({ state: 'idle' })).toBe(false); + expect(isSearchEnabled({ state: 'paused', error: 'x', reason: 'rate-limit' })).toBe(false); + expect( + isSearchEnabled({ state: 'indexing', progress: 1, filesProcessed: 1, totalFiles: 2 }), + ).toBe(false); + }); + + it('resetQueryResults preserves active indexing state and operation ID', () => { + useSearchStore.setState({ + indexingStatus: { + state: 'indexing', + progress: { current: 10, total: 50 }, + snapshotSha: 'abc1234', + }, + indexOperation: 42, + indexBuildId: 'build-xyz', + results: [{ filePath: 'a.ts', startLine: 1, endLine: 5, snippet: 'code', language: 'ts', score: 0.9 }], + answer: { answer: 'text', citations: [] }, + error: 'some error', + resultCoverage: { indexedFiles: 10, totalFiles: 50, percent: 20 }, + }); + + useSearchStore.getState().resetQueryResults(); + + const updated = useSearchStore.getState(); + expect(updated.indexingStatus.state).toBe('indexing'); + expect(updated.indexOperation).toBe(42); + expect(updated.indexBuildId).toBe('build-xyz'); + expect(updated.results).toEqual([]); + expect(updated.answer).toBeNull(); + expect(updated.error).toBeNull(); + expect(updated.resultCoverage).toBeUndefined(); + }); + + it('full reset clears active indexing and increments revision', () => { + const prevRev = useSearchStore.getState().revision; + const prevOp = useSearchStore.getState().indexOperation; + useSearchStore.setState({ + indexingStatus: { state: 'paused', reason: 'rate-limit' }, + indexBuildId: 'build-123', + }); + + useSearchStore.getState().reset(); + + const updated = useSearchStore.getState(); + expect(updated.indexingStatus.state).toBe('idle'); + expect(updated.indexBuildId).toBeUndefined(); + expect(updated.revision).toBe(prevRev + 1); + expect(updated.indexOperation).toBe(prevOp + 1); + }); + + it('resetQueryResults invalidates in-flight queries while preserving the index operation', () => { + useSearchStore.setState({ indexOperation: 9, indexAction: 'index' }); + const prevRev = useSearchStore.getState().revision; + const prevOp = useSearchStore.getState().indexOperation; + + useSearchStore.getState().resetQueryResults(); + + const updated = useSearchStore.getState(); + expect(updated.revision).toBe(prevRev + 1); + expect(updated.indexOperation).toBe(prevOp); + expect(updated.indexAction).toBe('index'); + }); + + it('abandons an operation started for another repo on navigation', () => { + expect( + shouldResetForRepoChange('o', 'b', { indexOwner: 'o', indexRepo: 'a', indexAction: 'index' }), + ).toBe(true); + expect( + shouldResetForRepoChange('o', 'a', { indexOwner: 'o', indexRepo: 'a', indexAction: 'index' }), + ).toBe(false); + expect(shouldResetForRepoChange('o', 'a', { indexOwner: null, indexRepo: null, indexAction: 'index' })).toBe( + false, + ); + expect(shouldResetForRepoChange('o', 'b', { indexOwner: 'o', indexRepo: 'a', indexAction: null })).toBe(true); + }); + + it('scopeKey is order-insensitive and distinguishes different scopes', () => { + expect(scopeKey({ includePaths: ['b', 'a'], excludePaths: [] })).toBe( + scopeKey({ includePaths: ['a', 'b'], excludePaths: [] }), + ); + expect(scopeKey({ includePaths: ['a'], excludePaths: [] })).not.toBe( + scopeKey({ includePaths: ['a'], excludePaths: ['a'] }), + ); + }); +}); diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index f29cd83..0ecdfd8 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useParams, useNavigate } from 'react-router-dom'; import { ArrowLeft, Search } from 'lucide-react'; import { useSearchStore } from '@/features/search/searchStore'; @@ -7,61 +7,95 @@ import { SearchBar } from '@/features/search/SearchBar'; import { ModeToggle } from '@/features/search/ModeToggle'; import { SearchResults } from '@/features/search/SearchResults'; import { QAAnswerView } from '@/features/search/QAAnswerView'; -import { IndexingStatusPanel } from '@/features/search/IndexingStatusPanel'; +import { SearchIndexControls } from '@/features/search/SearchIndexControls'; import { useSemanticSearch } from '@/features/search/useSemanticSearch'; import { useSemanticAsk } from '@/features/search/useSemanticAsk'; import { useTriggerIndexing } from '@/features/search/useTriggerIndexing'; import { useIndexingStatus } from '@/features/search/useIndexingStatus'; +import { coverageMessage } from '@/lib/search-coverage'; +import { SearchEmptyState } from '@/features/search/SearchEmptyState'; + +import { parsePaths } from '@/features/search/parsePaths'; +import { + isOperationActive, + isSearchEnabled, + resolveBuildId, + resolvePollingScope, + resolveRequestBranch, + resolveRunningSha, + shouldResetForRepoChange, + shouldResetSearchState, +} from './search-page-logic'; export function SearchPage() { const { owner = '', repo = '' } = useParams(); const navigate = useNavigate(); - const { mode, error, isLoading, results, answer, indexingStatus, askCache, searchCache } = useSearchStore(); + const { mode, error, isLoading, results, answer, indexingStatus, indexAction, indexScope: operationScope } = + useSearchStore(); const searchMutation = useSemanticSearch(); const askMutation = useSemanticAsk(); const indexMutation = useTriggerIndexing(); - useIndexingStatus(owner, repo); + const branch = useVizStore((state) => state.selectedBranch) ?? undefined; + const [includeInput, setIncludeInput] = useState(''); + const [excludeInput, setExcludeInput] = useState(''); + const indexScope = useMemo( + () => ({ + includePaths: parsePaths(includeInput), + excludePaths: parsePaths(excludeInput), + }), + [includeInput, excludeInput], + ); + const runningSha = resolveRunningSha(indexingStatus); + // While an operation is active, polls keep targeting the scope it was started with; + // edited inputs only apply to requests and new builds after it settles. + const pollingScope = resolvePollingScope( + isOperationActive(indexAction, indexingStatus), + operationScope, + indexScope, + ); + useIndexingStatus(owner, repo, true, runningSha ?? branch, pollingScope); + // Changing scope invalidates cached results, but must never abandon an active + // index operation (its completion would be ignored and a duplicate build could start). + useEffect(() => { + const state = useSearchStore.getState(); + if (shouldResetSearchState(state)) state.reset(); + else state.resetQueryResults(); + }, [includeInput, excludeInput]); - // Reset search state when navigating to a different repo's search page - const prevRepoRef = useRef(`${owner}/${repo}`); + // Clear previous results on entry and when the repository snapshot changes. useEffect(() => { - const key = `${owner}/${repo}`; - if (key !== prevRepoRef.current) { - prevRepoRef.current = key; - useSearchStore.getState().reset(); - } - }, [owner, repo]); + const state = useSearchStore.getState(); + // An operation started for another repo must never survive navigation: its + // completion would overwrite the new repository's state. + if (shouldResetForRepoChange(owner, repo, state)) state.reset(); + // Preserve an active index operation across remounts and branch switches; + // only its stale results are cleared while polling keeps tracking the snapshot. + else if (shouldResetSearchState(state)) state.reset(); + else state.resetQueryResults(); + setIncludeInput(''); + setExcludeInput(''); + }, [owner, repo, branch]); const handleSubmit = useCallback( (query: string) => { - const cacheKey = `${owner}/${repo}:${query}`; - - if (mode === 'search') { - // Return cached search results if same query - const cached = searchCache.get(cacheKey); - if (cached) { - const { setResults, setIsLoading, setError } = useSearchStore.getState(); - setResults(cached); - setIsLoading(false); - setError(null); - return; - } - searchMutation.mutate({ query, owner, repo }); - } else { - // Return cached answer if same question - const cached = askCache.get(cacheKey); - if (cached) { - const { setAnswer, setIsLoading, setError, addRecentQuery } = useSearchStore.getState(); - setAnswer(cached); - setIsLoading(false); - setError(null); - addRecentQuery(query); - return; - } - askMutation.mutate({ question: query, owner, repo }); - } + if (mode === 'search') + searchMutation.mutate({ + query, + owner, + repo, + branch: resolveRequestBranch(runningSha, branch), + scope: indexScope, + }); + else + askMutation.mutate({ + question: query, + owner, + repo, + branch: resolveRequestBranch(runningSha, branch), + scope: indexScope, + }); }, - [mode, owner, repo, searchMutation, askMutation, askCache, searchCache], + [mode, owner, repo, branch, runningSha, indexScope, searchMutation, askMutation], ); const prevModeRef = useRef(mode); @@ -76,29 +110,38 @@ export function SearchPage() { }, [mode, handleSubmit]); const handleIndex = useCallback(() => { - indexMutation.mutate({ owner, repo }); - }, [owner, repo, indexMutation]); + const state = useSearchStore.getState(); + if (state.indexAction || shouldResetForRepoChange(owner, repo, state)) return; + indexMutation.mutate({ + owner, + repo, + branch: resolveRequestBranch(resolveRunningSha(state.indexingStatus), branch), + scope: state.indexingStatus.state === 'paused' ? state.indexScope ?? indexScope : indexScope, + buildId: resolveBuildId(state.indexingStatus, state.indexBuildId), + }); + }, [owner, repo, branch, indexScope, indexMutation]); const handleSelectFile = useCallback( (filePath: string, _startLine: number, action: 'open' | 'inspect' = 'open') => { const { setFocusedFilePath, setInspectorOpen, setSelectedNodeId } = useVizStore.getState(); - + setSelectedNodeId(null); setFocusedFilePath(filePath); - + if (action === 'open') { setInspectorOpen(true); } else { setInspectorOpen(false); } - + navigate(`/${owner}/${repo}`); }, [navigate, owner, repo], ); const isIndexed = indexingStatus.state === 'complete'; - const isSearchDisabled = !isIndexed; + const isSearchDisabled = !isSearchEnabled(indexingStatus); + const responseCoverage = useSearchStore((state) => state.resultCoverage); const hasResults = mode === 'search' ? results.length > 0 : answer !== null; const showEmptyHero = !hasResults && !isLoading; @@ -124,14 +167,26 @@ export function SearchPage() { {/* Content */}
-

Search & discover

+

+ Search & discover +

Find the idea behind the code.

-

Search by meaning, ask a question, and follow the answer to its source.

+

+ Search by meaning, ask a question, and follow the answer to its source. +

{/* Indexing status banner */} -
- -
+ {/* Compact controls area */}
@@ -152,88 +207,40 @@ export function SearchPage() { {mode === 'search' ? 'Find code snippets by semantic similarity.' - : 'Ask a repository question with source citations.'} + : 'Answers cover indexed files only, with source citations.'}
{/* Search bar */}
- +
{/* Empty State Content */} {showEmptyHero && isIndexed && ( -
-
-

Search Examples

-
- {[ - "How are API errors handled?", - "Where is GitHub data fetched?", - "How is the dependency graph generated?" - ].map((example) => ( - - ))} -
-
- -
-
-

Recent Queries

-
- No recent queries yet. -
-
-
-

Index Details

-
-
- Status - - - Ready - -
-
- Chunks Indexed - {indexingStatus.chunkCount} -
-
-
-
-
+ )} {/* Error */} {error && ( -
+
{error}
)} {/* No results message */} - {!isLoading && - !error && - mode === 'search' && - results.length === 0 && - searchMutation.isSuccess && ( -
- No matching code found. Try broader terms or switch to Ask mode. -
- )} + {responseCoverage &&

{coverageMessage(responseCoverage)}

} + {!isLoading && !error && mode === 'search' && results.length === 0 && searchMutation.isSuccess && ( +
+ {responseCoverage?.kind === 'partial' + ? 'No matches in indexed files yet.' + : 'No matching code found. Try broader terms or switch to Ask mode.'} +
+ )} {/* Results */} {mode === 'search' && } diff --git a/src/pages/TermsPage.tsx b/src/pages/TermsPage.tsx new file mode 100644 index 0000000..ef3c9e5 --- /dev/null +++ b/src/pages/TermsPage.tsx @@ -0,0 +1,94 @@ +import { Link } from 'react-router-dom'; +import { InfoPageLayout } from '@/components/layout/InfoPageLayout'; + +export function TermsPage() { + return ( + +
+
+

Using gitSdm / Terms

+

Terms of use

+

Updated September 12, 2026

+

+ These terms describe how you may use the gitSdm service at gsdm.site. Questions or permission requests can + be sent to{' '} + + hello@gsdm.site + + . +

+
+
+

Free access and commercial use

+

+ gitSdm is free to use for personal and commercial projects. If you supply your own API key, your provider + may charge you under its own pricing and terms. Permission to use gitSdm does not change the licenses or + permissions attached to repositories you explore. +

+

+ These rules apply to the hosted service. The gitSdm source code remains governed by its software license; + self-hosted installations may have different operating policies. +

+
+
+

Repository content and credentials

+

+ Only access repositories and submit content you have permission to use. Before using AI or semantic search + with private code, make sure you are allowed to send it to the configured providers. You retain your rights + in your content. +

+

+ Keep your keys and tokens secure and revoke them with their provider if exposed. Read the{' '} + + Privacy policy + {' '} + for details about credentials, repository content, and storage. +

+
+
+

Automation and service limits

+

+ Contact us for permission before running automated bulk requests against the hosted service, including + large-scale repository indexing or repeated API jobs. Normal use of the app, including the indexing it + performs for you, does not require separate permission. +

+

+ Do not bypass rate limits, rotate credentials or network addresses to evade restrictions, attempt + unauthorized access, or deliberately disrupt the service. Bringing your own API key does not exempt you from + these rules. +

+

+ Requests may be queued, paused, or refused when capacity or usage limits are reached. We may restrict access + to address abuse or protect availability. If you believe a restriction is a mistake, contact us. +

+
+
+

Check results before relying on them

+

+ Dependency maps, search results, and AI answers can be incomplete, outdated, or wrong. Check the source code + before making changes or decisions. Partial search results cover only the files indexed so far. +

+

+ gitSdm depends on GitHub, hosting services, and AI providers. Features may be unavailable during outages, + maintenance, or provider limits. Search indexes and cached results are temporary; keep your own copies of + anything you need. +

+
+
+

Changes and questions

+

+ We will update this page and its date when these terms change. Changes apply going forward. Nothing in these + terms removes rights that applicable law does not allow to be excluded. +

+

+ For questions about these terms or use of the service, email{' '} + + hello@gsdm.site + + . Do not include API keys, tokens, or private code. +

+
+
+ + ); +} diff --git a/src/pages/VizPage.tsx b/src/pages/VizPage.tsx index e8a66a7..55c58f3 100644 --- a/src/pages/VizPage.tsx +++ b/src/pages/VizPage.tsx @@ -1,52 +1,50 @@ -import { useEffect, useMemo, useCallback, useState, lazy, Suspense } from "react"; -import { useParams } from "react-router-dom"; -import { motion, AnimatePresence } from "framer-motion"; -import { useAnalyzeRepo } from "@/hooks/useAnalyzeRepo"; -import { useVizStore } from "@/stores/vizStore"; -import { useVizDiff } from "@/hooks/useVizDiff"; -import { useCodeInspectorState } from "@/hooks/useCodeInspectorState"; -import { useWorkspaceShortcuts } from "@/hooks/useWorkspaceShortcuts"; -import { TopNav } from "@/components/viz/top-nav/TopNav"; -import { AISidebar } from "@/components/viz/AISidebar"; +import { useMotionPreference } from '@/hooks/useMotionPreference'; +import { useEffect, useMemo, useCallback, useState, lazy, Suspense } from 'react'; +import { useParams } from 'react-router-dom'; +import { motion, AnimatePresence } from 'framer-motion'; +import { useAnalyzeRepo } from '@/hooks/useAnalyzeRepo'; +import { useVizStore } from '@/stores/vizStore'; +import { useVizDiff } from '@/hooks/useVizDiff'; +import { useCodeInspectorState } from '@/hooks/useCodeInspectorState'; +import { useWorkspaceShortcuts } from '@/hooks/useWorkspaceShortcuts'; +import { TopNav } from '@/components/viz/top-nav/TopNav'; +import { AISidebar } from '@/components/viz/AISidebar'; const GraphCanvas = lazy(() => - import("@/features/graph/canvas/GraphCanvas").then((m) => ({ default: m.GraphCanvas })), + import('@/features/graph/canvas/GraphCanvas').then((m) => ({ + default: m.GraphCanvas, + })), ); const ArchitectureView = lazy(() => - import("@/components/viz/ArchitectureView").then((m) => ({ default: m.ArchitectureView })), + import('@/components/viz/ArchitectureView').then((m) => ({ + default: m.ArchitectureView, + })), ); -import { ContributorsView } from "@/components/contributors/ContributorsView"; -import { ExplorerPanel } from "@/components/explorer/ExplorerPanel"; -import { VizError } from "@/components/viz/VizError"; -import { Info, Loader2 } from "lucide-react"; -import { StagedLoader } from "@/components/viz/StagedLoader"; -import { Card } from "@/components/ui/card"; -import type { GraphNode } from "@/types"; -import { FullCommitHistoryView } from "@/components/timeline/CommitHistoryView"; -import { BottomStatusBar } from "@/components/viz/BottomStatusBar"; -import { CodeInspectorDock } from "@/components/explorer/CodeInspectorDock"; -import { VizSidebar } from "@/components/viz/layout/VizSidebar"; -import { fitWorkspacePanels } from "@/lib/workspace-panels"; +const ContributorsView = lazy(() => import('@/components/contributors/ContributorsView').then((m) => ({ default: m.ContributorsView }))); +import { ExplorerPanel } from '@/components/explorer/ExplorerPanel'; +import { VizError } from '@/components/viz/VizError'; +import { Info, Loader2 } from 'lucide-react'; +import { StagedLoader } from '@/components/viz/StagedLoader'; +import { Card } from '@/components/ui/card'; +import type { GraphNode } from '@/types'; +import { FullCommitHistoryView } from '@/components/timeline/CommitHistoryView'; +import { BottomStatusBar } from '@/components/viz/BottomStatusBar'; +import { CodeInspectorDock } from '@/components/explorer/CodeInspectorDock'; +import { VizSidebar } from '@/components/viz/layout/VizSidebar'; +import { fitWorkspacePanels } from '@/lib/workspace-panels'; +import { ChurnStatus } from '@/components/viz/ChurnStatus'; export function VizPage() { - const { owner = "", repo = "" } = useParams(); + const reducedMotion = useMotionPreference(); + const { owner = '', repo = '' } = useParams(); const { selectedBranch, compareBranch } = useVizStore(); // Fetch selected/active branch analysis - const { data, isLoading, error } = useAnalyzeRepo( - owner, - repo, - selectedBranch, - ); + const { data, isLoading, error, churnLoading, churnError } = useAnalyzeRepo(owner, repo, selectedBranch); // Fetch comparison branch analysis - const { data: compareData } = useAnalyzeRepo( - owner, - repo, - compareBranch, - !!compareBranch, - ); + const { data: compareData } = useAnalyzeRepo(owner, repo, compareBranch, !!compareBranch, { enrichHealth: false }); const { reset, @@ -65,20 +63,25 @@ export function VizPage() { } = useVizStore(); const selectedFilePath = focusedFilePath; + const colorMode = useVizStore((s) => s.colorMode); + useEffect(() => { + if (churnError) setToastMessage('Churn history could not be loaded. Check GitHub access or try again later.'); + else if (churnLoading) setToastMessage('Loading churn history…'); + }, [churnError, churnLoading, setToastMessage]); // Code Inspector state & auto-open helpers - const { - codeInspectorState, - setCodeInspectorState, - setPreferredOpenState, - } = useCodeInspectorState(focusedFilePath); + const { codeInspectorState, setCodeInspectorState, setPreferredOpenState } = useCodeInspectorState(focusedFilePath); // Resizable columns state const [leftWidth, setLeftWidth] = useState(240); const [rightWidth, setRightWidth] = useState(360); const [viewportWidth, setViewportWidth] = useState(() => window.innerWidth); const minibarWidth = viewportWidth >= 1024 ? (Number(!explorerOpen) + Number(!aiSidebarOpen)) * 40 : 0; - const panelWidths = fitWorkspacePanels(viewportWidth - minibarWidth, explorerOpen ? leftWidth : 0, aiSidebarOpen ? rightWidth : 0); + const panelWidths = fitWorkspacePanels( + viewportWidth - minibarWidth, + explorerOpen ? leftWidth : 0, + aiSidebarOpen ? rightWidth : 0, + ); useEffect(() => { const update = () => setViewportWidth(window.innerWidth); window.addEventListener('resize', update); @@ -90,18 +93,18 @@ export function VizPage() { useWorkspaceShortcuts(); useEffect(() => { - const media = window.matchMedia("(max-width: 1023px)"); + const media = window.matchMedia('(max-width: 1023px)'); const closeMobileSidebars = () => { - if (!media.matches) return; - setExplorerOpen(false); - setAiSidebarOpen(false); + const state = useVizStore.getState(); + if (media.matches) state.closePanelsPreservingMode(); + else state.setWorkspaceMode(state.workspaceMode); }; closeMobileSidebars(); - media.addEventListener("change", closeMobileSidebars); + media.addEventListener('change', closeMobileSidebars); - return () => media.removeEventListener("change", closeMobileSidebars); + return () => media.removeEventListener('change', closeMobileSidebars); }, [setAiSidebarOpen, setExplorerOpen]); // Compute file map structures, repository graphs, and branch diff calculations @@ -116,8 +119,14 @@ export function VizPage() { const handleSelectFile = useCallback( (path: string) => { - const node = (combinedGraph || data?.graph)?.nodes.find((item) => item.type === 'file' && item.data.path === path); - useVizStore.setState({ focusedFilePath: path, selectedNodeId: node?.id ?? null, sidebarTab: 'analysis' }); + const node = (combinedGraph || data?.graph)?.nodes.find( + (item) => item.type === 'file' && item.data.path === path, + ); + useVizStore.setState({ + focusedFilePath: path, + selectedNodeId: node?.id ?? null, + sidebarTab: 'analysis', + }); if (window.innerWidth >= 1024) { setAiSidebarOpen(true); } else { @@ -133,9 +142,8 @@ export function VizPage() { if (activeRepoKey !== key) { reset(); setActiveRepoKey(key); - if (window.matchMedia("(max-width: 1023px)").matches) { - setExplorerOpen(false); - setAiSidebarOpen(false); + if (window.matchMedia('(max-width: 1023px)').matches) { + useVizStore.getState().closePanelsPreservingMode(); } } }, [owner, repo, activeRepoKey, reset, setActiveRepoKey, setExplorerOpen, setAiSidebarOpen]); @@ -151,7 +159,7 @@ export function VizPage() { }, [data, selectedNodeId, nodeById]); useEffect(() => { - if (selectedNode?.type === "file" && selectedNode.data.path) { + if (selectedNode?.type === 'file' && selectedNode.data.path) { setFocusedFilePath(selectedNode.data.path); } else if (selectedNode) { setFocusedFilePath(null); @@ -159,169 +167,159 @@ export function VizPage() { }, [selectedNode, setFocusedFilePath]); if (error) { - return ( - - ); + return ; } return (
- + -
- {isLoading ? ( - - ) : data ? ( -
- {/* Left Sidebar wrapper */} - setExplorerOpen(false)} - onOpen={() => setExplorerOpen(true)} - > - - +
+ {isLoading ? ( + + ) : data ? ( +
+ {/* Left Sidebar wrapper */} + setExplorerOpen(false)} + onOpen={() => setExplorerOpen(true)} + > + + - {/* Center Workspace */} - -
- + {/* Center Workspace */} + + {activeView === 'graph' && colorMode === 'churn' && ( + + )} +
+ Loading graph…
) : null - }> -
- - {data.treeTruncated && ( -
- Partial repository tree: some files are unavailable. -
- )} -
- + } + > +
+ + {data.treeTruncated && ( +
+ Partial repository tree: some files are unavailable. +
+ )} +
+ - {activeView === "architecture" && ( - + {activeView === 'architecture' && ( + Loading architecture…
- }> - - - )} - {activeView === "contributors" && ( - - )} - {activeView === "commits" && ( - - )} -
- - {focusedFilePath && codeInspectorState !== 'closed' && ( - { - setCodeInspectorState(s); - if (s === 'closed') { - useVizStore.setState({ focusedFilePath: null, selectedNodeId: null, highlightedNodeIds: new Set() }); - } else { - setPreferredOpenState(s); - } - }} - filePath={focusedFilePath} - graph={combinedGraph || data.graph} - onSelectFile={handleSelectFile} + } + > + + + )} + {activeView === 'contributors' && ( + Loading contributors…
}> + + + )} + {activeView === 'commits' && ( + )} - +
- {/* Right Sidebar wrapper */} - setAiSidebarOpen(false)} - onOpen={() => setAiSidebarOpen(true)} - > - { + setCodeInspectorState(s); + if (s === 'closed') { + useVizStore.setState({ + focusedFilePath: null, + selectedNodeId: null, + highlightedNodeIds: new Set(), + }); + } else { + setPreferredOpenState(s); + } + }} + filePath={focusedFilePath} + graph={combinedGraph || data.graph} + onSelectFile={handleSelectFile} + owner={owner} + repo={repo} /> - -
- ) : null} -
- - + )} + - - {toastMessage && ( - setAiSidebarOpen(false)} + onOpen={() => setAiSidebarOpen(true)} > - - -
-

- {toastMessage} -

-
- - - )} - + + +
+ ) : null}
+ + + + + {toastMessage && ( + + + +
+

{toastMessage}

+
+ + + )} + +
); } - diff --git a/src/pages/search-page-logic.ts b/src/pages/search-page-logic.ts new file mode 100644 index 0000000..d92ea0b --- /dev/null +++ b/src/pages/search-page-logic.ts @@ -0,0 +1,65 @@ +import type { IndexScope } from '@/lib/apiClient'; +import type { IndexingStatus } from '@/types'; + +export interface SearchStateSnapshot { + indexAction: 'index' | 'cancel' | null; + indexingStatus: IndexingStatus; +} + +// An index operation is active while its mutation is in flight or its status is +// mid-flight (indexing / paused-resumable). Resetting the store then would +// abandon the operation: its completion gets ignored and the controls could +// start a duplicate build while the original keeps running on the server. +export function shouldResetSearchState(state: SearchStateSnapshot): boolean { + if (state.indexAction) return false; + return state.indexingStatus.state !== 'indexing' && state.indexingStatus.state !== 'paused'; +} + +// Only mid-flight statuses carry the snapshot the index was built against; +// requests must target that snapshot, not a moved branch head. +export function resolveRunningSha(status: IndexingStatus): string | undefined { + return status.state === 'indexing' || status.state === 'paused' ? status.snapshotSha : undefined; +} + +// The running snapshot outranks the selected branch while an operation is active. +export function resolveRequestBranch(runningSha: string | undefined, branch: string | undefined): string | undefined { + return runningSha ?? branch; +} + +// A paused operation resumes its existing build; anything else starts a fresh one. +export function resolveBuildId(status: IndexingStatus, indexBuildId: string | undefined): string { + if (status.state === 'paused' && indexBuildId) return indexBuildId; + return crypto.randomUUID(); +} + +// A complete index or any reported coverage enables search (partial coverage included). +export function isSearchEnabled(status: IndexingStatus): boolean { + return status.state === 'complete' || Boolean(status.coverage); +} + +// An operation is active while its mutation is in flight or its status is mid-flight. +export function isOperationActive(indexAction: 'index' | 'cancel' | null, status: IndexingStatus): boolean { + return indexAction !== null || status.state === 'indexing' || status.state === 'paused'; +} + +// While an operation is active, polling must target the scope it was started with; +// edited inputs only apply to requests and new builds after the operation settles. +export function resolvePollingScope( + operationActive: boolean, + operationScope: IndexScope | null, + liveScope: IndexScope, +): IndexScope { + return operationActive && operationScope ? operationScope : liveScope; +} + +// An in-flight index operation belongs to the repo it was started for; navigating +// to another repo must abandon it instead of letting its completion overwrite the +// new repository's state. +export function shouldResetForRepoChange( + owner: string, + repo: string, + state: { indexOwner: string | null; indexRepo: string | null; indexAction: 'index' | 'cancel' | null }, +): boolean { + if (state.indexOwner === null && state.indexRepo === null) return false; + return state.indexOwner !== owner || state.indexRepo !== repo; +} diff --git a/src/stores/chatConfigStore.test.ts b/src/stores/chatConfigStore.test.ts new file mode 100644 index 0000000..af9c63b --- /dev/null +++ b/src/stores/chatConfigStore.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'bun:test'; + +// bun test has no Web Storage, and the store snapshots localStorage at import +// time, so the stub must be installed before the module under test is imported. +class MemoryStorage { + private map = new Map(); + getItem(key: string): string | null { + return this.map.get(key) ?? null; + } + setItem(key: string, value: string): void { + this.map.set(key, value); + } + removeItem(key: string): void { + this.map.delete(key); + } + clear(): void { + this.map.clear(); + } + key(index: number): string | null { + return [...this.map.keys()][index] ?? null; + } + get length(): number { + return this.map.size; + } +} +Object.defineProperty(globalThis, 'localStorage', { value: new MemoryStorage(), configurable: true, writable: true }); + +const { refreshChatConfig, useChatConfigRevision, writeChatConfigItem } = await import('./chatConfigStore'); + +const revision = () => useChatConfigRevision.getState().revision; + +describe('chatConfigStore revision inputs', () => { + it('bumps the revision when the GitHub PAT is saved, but not when nothing changed', () => { + const before = revision(); + + localStorage.setItem('gitsdm_github_pat', 'ghp_test_token'); + refreshChatConfig(); + expect(revision()).toBe(before + 1); + + refreshChatConfig(); + expect(revision()).toBe(before + 1); + }); + + it('bumps the revision when the GitHub PAT is cleared', () => { + const before = revision(); + + localStorage.removeItem('gitsdm_github_pat'); + refreshChatConfig(); + expect(revision()).toBe(before + 1); + }); + + it('bumps the revision when an AI setting changes', () => { + const before = revision(); + + localStorage.setItem('gitsdm_ai_model', 'test-model'); + refreshChatConfig(); + expect(revision()).toBe(before + 1); + }); + + it('bumps the revision through the tracked same-tab writer, but not when the value is unchanged', () => { + const before = revision(); + + writeChatConfigItem('gitsdm_github_pat', 'ghp_same_tab_token'); + expect(revision()).toBe(before + 1); + + writeChatConfigItem('gitsdm_github_pat', 'ghp_same_tab_token'); + expect(revision()).toBe(before + 1); + + writeChatConfigItem('gitsdm_github_pat', null); + expect(revision()).toBe(before + 2); + expect(localStorage.getItem('gitsdm_github_pat')).toBeNull(); + }); +}); diff --git a/src/stores/chatConfigStore.ts b/src/stores/chatConfigStore.ts new file mode 100644 index 0000000..070f965 --- /dev/null +++ b/src/stores/chatConfigStore.ts @@ -0,0 +1,47 @@ +import { create } from 'zustand'; + +const chatStorageKeys = [ + 'gitsdm_gemini_api_key', + 'gitsdm_github_pat', + 'gitsdm_ai_provider', + 'gitsdm_ai_model', + 'gitsdm_ai_base_url', +]; + +function readSettings(): string { + try { + return JSON.stringify(chatStorageKeys.map((key) => localStorage.getItem(key))); + } catch { + return ''; + } +} + +// Credentials stay out of observable state and cache keys. +let savedSettings = readSettings(); +export const useChatConfigRevision = create(() => ({ revision: 0 })); + +export function refreshChatConfig() { + const next = readSettings(); + if (next === savedSettings) return; + savedSettings = next; + useChatConfigRevision.setState((state) => ({ revision: state.revision + 1 })); +} + +// Tracked writer for credential/AI settings. The 'storage' event only fires in +// OTHER tabs, so same-tab writers must go through here to notify subscribers +// (credential-scoped query keys, AI task caches) without a reload. +export function writeChatConfigItem(key: string, value: string | null) { + try { + if (value) localStorage.setItem(key, value); + else localStorage.removeItem(key); + } catch { + /* ignore */ + } + refreshChatConfig(); +} + +if (typeof window !== 'undefined') { + window.addEventListener('storage', (event) => { + if (event.key === null || chatStorageKeys.includes(event.key)) refreshChatConfig(); + }); +} diff --git a/src/stores/motionStore.ts b/src/stores/motionStore.ts new file mode 100644 index 0000000..04cdaaf --- /dev/null +++ b/src/stores/motionStore.ts @@ -0,0 +1,16 @@ +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; + +export type MotionPreference = 'system' | 'reduced' | 'full'; +export const useMotionStore = create( + persist<{ + preference: MotionPreference; + setPreference: (preference: MotionPreference) => void; + }>( + (set) => ({ + preference: 'system', + setPreference: (preference) => set({ preference }), + }), + { name: 'gitsdm-motion', partialize: (state) => state }, + ), +); diff --git a/src/stores/vizStore.test.ts b/src/stores/vizStore.test.ts new file mode 100644 index 0000000..1bf7423 --- /dev/null +++ b/src/stores/vizStore.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'bun:test'; + +// bun test has no Web Storage, and the store snapshots localStorage at import +// time, so the stub must be installed before the module under test is imported. +class MemoryStorage { + private map = new Map(); + getItem(key: string): string | null { + return this.map.get(key) ?? null; + } + setItem(key: string, value: string): void { + this.map.set(key, value); + } + removeItem(key: string): void { + this.map.delete(key); + } + clear(): void { + this.map.clear(); + } + key(index: number): string | null { + return [...this.map.keys()][index] ?? null; + } + get length(): number { + return this.map.size; + } +} +Object.defineProperty(globalThis, 'localStorage', { value: new MemoryStorage(), configurable: true, writable: true }); + +const { useVizStore } = await import('./vizStore'); + +describe('closePanelsPreservingMode', () => { + it('closes both panels in one update without rewriting the saved workspaceMode', () => { + const store = useVizStore.getState(); + store.setWorkspaceMode('full'); + expect(useVizStore.getState().explorerOpen).toBe(true); + expect(useVizStore.getState().aiSidebarOpen).toBe(true); + + useVizStore.getState().closePanelsPreservingMode(); + + const state = useVizStore.getState(); + expect(state.explorerOpen).toBe(false); + expect(state.aiSidebarOpen).toBe(false); + expect(state.workspaceMode).toBe('full'); + }); + + it('restores both panels when the preserved mode is reapplied (desktop return)', () => { + useVizStore.getState().setWorkspaceMode('full'); + useVizStore.getState().closePanelsPreservingMode(); + + const saved = useVizStore.getState().workspaceMode; + useVizStore.getState().setWorkspaceMode(saved); + + const state = useVizStore.getState(); + expect(state.workspaceMode).toBe('full'); + expect(state.explorerOpen).toBe(true); + }); +}); diff --git a/src/stores/vizStore.ts b/src/stores/vizStore.ts index e90392f..260bf30 100644 --- a/src/stores/vizStore.ts +++ b/src/stores/vizStore.ts @@ -1,16 +1,18 @@ -import { create } from "zustand"; -import { persist } from "zustand/middleware"; -import type { NodeType } from "@/types"; +import { create } from 'zustand'; +import { modeFromPanels, panelsForMode } from '@/lib/workspace-mode'; +import { persist } from 'zustand/middleware'; +import type { NodeType } from '@/types'; -export type SidebarTab = "overview" | "analysis" | "dependencies" | "ai" | "learning"; -export type WorkspaceMode = "focus" | "analysis" | "learning" | "full"; -export type GraphScope = "important" | "source" | "grouped" | "full"; -export type ContentFilter = "source" | "config" | "docs" | "tests" | "github" | "examples" | "generated" | "translations"; -export type CompareRefType = "branch" | "tag" | "commit"; -export type ColorMode = "default" | "churn" | "complexity"; -export type SizeMode = "default" | "complexity"; +export type SidebarTab = 'overview' | 'analysis' | 'dependencies' | 'ai' | 'learning'; +export type WorkspaceMode = 'full' | 'explorer' | 'insights' | 'focus'; +export type GraphScope = 'important' | 'source' | 'grouped' | 'full'; +export type ContentFilter = + 'source' | 'config' | 'docs' | 'tests' | 'github' | 'examples' | 'generated' | 'translations'; +export type CompareRefType = 'branch' | 'tag' | 'commit'; +export type ColorMode = 'default' | 'churn' | 'complexity'; +export type SizeMode = 'default' | 'complexity'; -export type LayoutType = "tree" | "d3-tree-horiz" | "d3-tree-vert"; +export type LayoutType = 'tree' | 'd3-tree-horiz' | 'd3-tree-vert'; interface VizState { searchQuery: string; @@ -25,20 +27,20 @@ interface VizState { inspectorOpen: boolean; focusedFilePath: string | null; toastMessage: string | null; - theme: "dark" | "light"; + theme: 'dark' | 'light'; selectedBranch: string | null; compareBranch: string | null; compareRefType: CompareRefType; availableBranches: string[]; - activeView: "graph" | "architecture" | "contributors" | "commits"; + activeView: 'graph' | 'architecture' | 'contributors' | 'commits'; zoom: number; visibleNodeCount: number; visibleEdgeCount: number; activeRepoKey: string | null; // Interactive upgrades - activeFocusLayer: "all" | "api" | "ui" | "core" | "config"; - diffStatusFilters: Set<"added" | "modified" | "deleted">; + activeFocusLayer: 'all' | 'api' | 'ui' | 'core' | 'config'; + diffStatusFilters: Set<'added' | 'modified' | 'deleted'>; blastRadiusActive: boolean; // Churn / Complexity overlay @@ -60,9 +62,9 @@ interface VizState { }; // Shared UI synchronization states - activeDropdown: "filter" | "display" | "layout" | "export" | "legend" | null; + activeDropdown: 'filter' | 'display' | 'layout' | 'export' | 'legend' | null; graphActionTrigger: { - action: "zoomIn" | "zoomOut" | "fitView" | "reset" | "focusGraph"; + action: 'zoomIn' | 'zoomOut' | 'fitView' | 'reset' | 'focusGraph'; timestamp: number; } | null; @@ -70,7 +72,7 @@ interface VizState { toggleContentFilter: (filter: ContentFilter) => void; setContentFilters: (filters: Set) => void; - toggleDiffStatusFilter: (status: "added" | "modified" | "deleted") => void; + toggleDiffStatusFilter: (status: 'added' | 'modified' | 'deleted') => void; setSelectedBranch: (branch: string | null) => void; setCompareBranch: (branch: string | null) => void; setCompareRefType: (type: CompareRefType) => void; @@ -85,49 +87,44 @@ interface VizState { setWorkspaceMode: (mode: WorkspaceMode) => void; setExplorerOpen: (open: boolean) => void; setAiSidebarOpen: (open: boolean) => void; + closePanelsPreservingMode: () => void; setInspectorOpen: (open: boolean) => void; setFocusedFilePath: (path: string | null) => void; - setTheme: (theme: "dark" | "light") => void; + setTheme: (theme: 'dark' | 'light') => void; toggleTheme: () => void; setZoom: (zoom: number) => void; setVisibleCounts: (nodes: number, edges: number) => void; - setActiveFocusLayer: ( - layer: "all" | "api" | "ui" | "core" | "config", - ) => void; - setActiveView: ( - view: "graph" | "architecture" | "contributors" | "commits", - ) => void; + setActiveFocusLayer: (layer: 'all' | 'api' | 'ui' | 'core' | 'config') => void; + setActiveView: (view: 'graph' | 'architecture' | 'contributors' | 'commits') => void; setBlastRadiusActive: (active: boolean) => void; setColorMode: (mode: ColorMode) => void; setSizeMode: (mode: SizeMode) => void; setLayoutType: (layout: LayoutType) => void; setGraphSidebarOpen: (open: boolean) => void; - toggleGraphSidebarSection: ( - key: keyof VizState["graphSidebarSections"], - ) => void; + toggleGraphSidebarSection: (key: keyof VizState['graphSidebarSections']) => void; resetFilters: () => void; - setActiveDropdown: (dropdown: "filter" | "display" | "layout" | "export" | "legend" | null) => void; - triggerGraphAction: (action: "zoomIn" | "zoomOut" | "fitView" | "reset" | "focusGraph") => void; + setActiveDropdown: (dropdown: 'filter' | 'display' | 'layout' | 'export' | 'legend' | null) => void; + triggerGraphAction: (action: 'zoomIn' | 'zoomOut' | 'fitView' | 'reset' | 'focusGraph') => void; setActiveRepoKey: (key: string | null) => void; reset: () => void; } -const defaultFilters = new Set(["repo", "package", "folder", "file"]); -const defaultContentFilters = new Set(["source", "config"]); +const defaultFilters = new Set(['repo', 'package', 'folder', 'file']); +const defaultContentFilters = new Set(['source', 'config']); export const useVizStore = create()( persist( (set) => ({ - searchQuery: "", + searchQuery: '', nodeTypeFilters: new Set(defaultFilters), fileTypeFilters: new Set(), selectedNodeId: null, highlightedNodeIds: new Set(), - sidebarTab: "overview", - workspaceMode: "full", + sidebarTab: 'overview', + workspaceMode: 'full', explorerOpen: true, aiSidebarOpen: true, inspectorOpen: false, @@ -136,210 +133,250 @@ export const useVizStore = create()( visibleNodeCount: 0, visibleEdgeCount: 0, toastMessage: null, - theme: "dark", + theme: 'dark', selectedBranch: null, compareBranch: null, - compareRefType: "branch" as CompareRefType, + compareRefType: 'branch' as CompareRefType, availableBranches: [], - activeView: "graph", + activeView: 'graph', activeRepoKey: null, - activeFocusLayer: "all", - diffStatusFilters: new Set(), - blastRadiusActive: false, - colorMode: "default" as ColorMode, - sizeMode: "default" as SizeMode, - layoutType: "tree" as LayoutType, + activeFocusLayer: 'all', + diffStatusFilters: new Set(), + blastRadiusActive: false, + colorMode: 'default' as ColorMode, + sizeMode: 'default' as SizeMode, + layoutType: 'tree' as LayoutType, - graphScope: "source", - contentFilters: new Set(defaultContentFilters), + graphScope: 'source', + contentFilters: new Set(defaultContentFilters), - graphSidebarOpen: true, - graphSidebarSections: { - nodeInfo: true, - tools: false, - nodeTypes: true, - fileTypes: false, - compare: true, - }, + graphSidebarOpen: true, + graphSidebarSections: { + nodeInfo: true, + tools: false, + nodeTypes: true, + fileTypes: false, + compare: true, + }, - activeDropdown: null, - graphActionTrigger: null, + activeDropdown: null, + graphActionTrigger: null, - setGraphScope: (graphScope: GraphScope) => set((s: VizState) => { - let newContentFilters = new Set(s.contentFilters); - if (graphScope === 'full') { - newContentFilters = new Set(["source", "config", "docs", "tests", "github", "examples", "generated", "translations"]); - } else if (graphScope === 'source' || graphScope === 'important' || graphScope === 'grouped') { - newContentFilters = new Set(["source", "config"]); - } - return { graphScope, contentFilters: newContentFilters }; - }), - toggleContentFilter: (filter: ContentFilter) => - set((s: VizState) => { - const next = new Set(s.contentFilters); - if (next.has(filter)) next.delete(filter); - else next.add(filter); - return { contentFilters: next }; - }), - setContentFilters: (contentFilters: Set) => set({ contentFilters }), + setGraphScope: (graphScope: GraphScope) => + set((s: VizState) => { + let newContentFilters = new Set(s.contentFilters); + if (graphScope === 'full') { + newContentFilters = new Set([ + 'source', + 'config', + 'docs', + 'tests', + 'github', + 'examples', + 'generated', + 'translations', + ]); + } else if (graphScope === 'source' || graphScope === 'important' || graphScope === 'grouped') { + newContentFilters = new Set(['source', 'config']); + } + return { graphScope, contentFilters: newContentFilters }; + }), + toggleContentFilter: (filter: ContentFilter) => + set((s: VizState) => { + const next = new Set(s.contentFilters); + if (next.has(filter)) next.delete(filter); + else next.add(filter); + return { contentFilters: next }; + }), + setContentFilters: (contentFilters: Set) => set({ contentFilters }), - toggleDiffStatusFilter: (status: "added" | "modified" | "deleted") => - set((s: VizState) => { - const next = new Set(s.diffStatusFilters); - if (next.has(status)) next.delete(status); - else next.add(status); - return { diffStatusFilters: next }; - }), - setSelectedBranch: (selectedBranch: string | null) => set({ selectedBranch }), - setCompareBranch: (compareBranch: string | null) => - set((s: VizState) => ({ - compareBranch, - compareRefType: "branch", - diffStatusFilters: compareBranch ? s.diffStatusFilters : new Set(), - })), - setCompareRefType: (compareRefType: CompareRefType) => set({ compareRefType }), - setAvailableBranches: (availableBranches: string[]) => set({ availableBranches }), - setToastMessage: (toastMessage: string | null) => set({ toastMessage }), - setSearchQuery: (searchQuery: string) => set({ searchQuery }), - toggleNodeTypeFilter: (type: NodeType) => - set((s: VizState) => { - const next = new Set(s.nodeTypeFilters); - if (next.has(type)) next.delete(type); - else next.add(type); - return { nodeTypeFilters: next }; - }), - toggleFileTypeFilter: (type: string) => - set((s: VizState) => { - const next = new Set(s.fileTypeFilters); - if (next.has(type)) next.delete(type); - else next.add(type); - return { fileTypeFilters: next }; - }), - setSelectedNodeId: (selectedNodeId: string | null) => set({ selectedNodeId }), - setHighlightedNodeIds: (highlightedNodeIds: Set) => set({ highlightedNodeIds }), - setSidebarTab: (sidebarTab: SidebarTab) => set({ sidebarTab }), - setWorkspaceMode: (workspaceMode: WorkspaceMode) => { - const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; - set({ - workspaceMode, - explorerOpen: workspaceMode !== 'focus' && (!isMobile || workspaceMode === 'full'), - aiSidebarOpen: workspaceMode !== 'focus' && (!isMobile || workspaceMode === 'analysis' || workspaceMode === 'learning'), - inspectorOpen: false, - ...(workspaceMode === 'analysis' ? { sidebarTab: 'analysis' as const } : {}), - ...(workspaceMode === 'learning' ? { sidebarTab: 'learning' as const } : {}), - }); - }, - setExplorerOpen: (explorerOpen: boolean) => set({ explorerOpen }), - setAiSidebarOpen: (aiSidebarOpen: boolean) => set({ aiSidebarOpen }), - setInspectorOpen: (inspectorOpen: boolean) => set({ inspectorOpen }), - setFocusedFilePath: (focusedFilePath: string | null) => set({ focusedFilePath }), - setZoom: (zoom: number) => set({ zoom }), - setVisibleCounts: (visibleNodeCount: number, visibleEdgeCount: number) => set({ visibleNodeCount, visibleEdgeCount }), - setTheme: (theme: "dark" | "light") => { - if (typeof window !== "undefined") localStorage.setItem("theme", theme); - set({ theme }); - }, - toggleTheme: () => - set((s: VizState) => { - const nextTheme = s.theme === "dark" ? "light" : "dark"; - if (typeof window !== "undefined") - localStorage.setItem("theme", nextTheme); - return { theme: nextTheme }; - }), - - setActiveFocusLayer: (activeFocusLayer: "all" | "api" | "ui" | "core" | "config") => set({ activeFocusLayer }), - setActiveView: (activeView: "graph" | "architecture" | "contributors" | "commits") => set({ activeView }), - setBlastRadiusActive: (blastRadiusActive: boolean) => set({ blastRadiusActive }), - setColorMode: (colorMode: ColorMode) => set({ colorMode }), - setSizeMode: (sizeMode: SizeMode) => set({ sizeMode }), - setLayoutType: (layoutType: LayoutType) => set({ layoutType }), - setGraphSidebarOpen: (graphSidebarOpen: boolean) => set({ graphSidebarOpen }), - toggleGraphSidebarSection: (key: keyof VizState["graphSidebarSections"]) => - set((s: VizState) => ({ - graphSidebarSections: { - ...s.graphSidebarSections, - [key]: !s.graphSidebarSections[key], + toggleDiffStatusFilter: (status: 'added' | 'modified' | 'deleted') => + set((s: VizState) => { + const next = new Set(s.diffStatusFilters); + if (next.has(status)) next.delete(status); + else next.add(status); + return { diffStatusFilters: next }; + }), + setSelectedBranch: (selectedBranch: string | null) => set({ selectedBranch }), + setCompareBranch: (compareBranch: string | null) => + set((s: VizState) => ({ + compareBranch, + compareRefType: 'branch', + diffStatusFilters: compareBranch ? s.diffStatusFilters : new Set(), + })), + setCompareRefType: (compareRefType: CompareRefType) => set({ compareRefType }), + setAvailableBranches: (availableBranches: string[]) => set({ availableBranches }), + setToastMessage: (toastMessage: string | null) => set({ toastMessage }), + setSearchQuery: (searchQuery: string) => set({ searchQuery }), + toggleNodeTypeFilter: (type: NodeType) => + set((s: VizState) => { + const next = new Set(s.nodeTypeFilters); + if (next.has(type)) next.delete(type); + else next.add(type); + return { nodeTypeFilters: next }; + }), + toggleFileTypeFilter: (type: string) => + set((s: VizState) => { + const next = new Set(s.fileTypeFilters); + if (next.has(type)) next.delete(type); + else next.add(type); + return { fileTypeFilters: next }; + }), + setSelectedNodeId: (selectedNodeId: string | null) => set({ selectedNodeId }), + setHighlightedNodeIds: (highlightedNodeIds: Set) => set({ highlightedNodeIds }), + setSidebarTab: (sidebarTab: SidebarTab) => set({ sidebarTab }), + setWorkspaceMode: (workspaceMode: WorkspaceMode) => { + const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; + set({ + workspaceMode, + ...panelsForMode(workspaceMode, isMobile), + inspectorOpen: false, + }); + }, + setExplorerOpen: (explorerOpen: boolean) => + set((s: VizState) => { + const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; + const aiSidebarOpen = isMobile && explorerOpen ? false : s.aiSidebarOpen; + return { + explorerOpen, + aiSidebarOpen, + workspaceMode: modeFromPanels(explorerOpen, aiSidebarOpen), + }; + }), + setAiSidebarOpen: (aiSidebarOpen: boolean) => + set((s: VizState) => { + const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; + const explorerOpen = isMobile && aiSidebarOpen ? false : s.explorerOpen; + return { + explorerOpen, + aiSidebarOpen, + workspaceMode: modeFromPanels(explorerOpen, aiSidebarOpen), + }; + }), + // Responsive close (breakpoint switch, repo change, Escape on mobile) must + // not rewrite the saved workspaceMode — a sequential close of both panels + // would otherwise collapse full -> focus with no way to restore on return + // to desktop. Single state update, mode untouched. + closePanelsPreservingMode: () => set({ explorerOpen: false, aiSidebarOpen: false }), + setInspectorOpen: (inspectorOpen: boolean) => set({ inspectorOpen }), + setFocusedFilePath: (focusedFilePath: string | null) => set({ focusedFilePath }), + setZoom: (zoom: number) => set({ zoom }), + setVisibleCounts: (visibleNodeCount: number, visibleEdgeCount: number) => + set({ visibleNodeCount, visibleEdgeCount }), + setTheme: (theme: 'dark' | 'light') => { + if (typeof window !== 'undefined') localStorage.setItem('theme', theme); + set({ theme }); }, - })), + toggleTheme: () => + set((s: VizState) => { + const nextTheme = s.theme === 'dark' ? 'light' : 'dark'; + if (typeof window !== 'undefined') localStorage.setItem('theme', nextTheme); + return { theme: nextTheme }; + }), - resetFilters: () => - set({ - nodeTypeFilters: new Set(defaultFilters), - fileTypeFilters: new Set(), - diffStatusFilters: new Set(), - activeFocusLayer: "all", - searchQuery: "", - graphScope: "source", - contentFilters: new Set(defaultContentFilters), - colorMode: "default" as ColorMode, - sizeMode: "default" as SizeMode, - }), + setActiveFocusLayer: (activeFocusLayer: 'all' | 'api' | 'ui' | 'core' | 'config') => set({ activeFocusLayer }), + setActiveView: (activeView: 'graph' | 'architecture' | 'contributors' | 'commits') => set({ activeView }), + setBlastRadiusActive: (blastRadiusActive: boolean) => set({ blastRadiusActive }), + setColorMode: (colorMode: ColorMode) => set({ colorMode }), + setSizeMode: (sizeMode: SizeMode) => set({ sizeMode }), + setLayoutType: (layoutType: LayoutType) => set({ layoutType }), + setGraphSidebarOpen: (graphSidebarOpen: boolean) => set({ graphSidebarOpen }), + toggleGraphSidebarSection: (key: keyof VizState['graphSidebarSections']) => + set((s: VizState) => ({ + graphSidebarSections: { + ...s.graphSidebarSections, + [key]: !s.graphSidebarSections[key], + }, + })), - setActiveDropdown: (activeDropdown: "filter" | "display" | "layout" | "export" | "legend" | null) => set({ activeDropdown }), - triggerGraphAction: (action: "zoomIn" | "zoomOut" | "fitView" | "reset" | "focusGraph") => set({ graphActionTrigger: { action, timestamp: Date.now() } }), + resetFilters: () => + set({ + nodeTypeFilters: new Set(defaultFilters), + fileTypeFilters: new Set(), + diffStatusFilters: new Set(), + activeFocusLayer: 'all', + searchQuery: '', + graphScope: 'source', + contentFilters: new Set(defaultContentFilters), + colorMode: 'default' as ColorMode, + sizeMode: 'default' as SizeMode, + }), - setActiveRepoKey: (activeRepoKey: string | null) => set({ activeRepoKey }), + setActiveDropdown: (activeDropdown: 'filter' | 'display' | 'layout' | 'export' | 'legend' | null) => + set({ activeDropdown }), + triggerGraphAction: (action: 'zoomIn' | 'zoomOut' | 'fitView' | 'reset' | 'focusGraph') => + set({ graphActionTrigger: { action, timestamp: Date.now() } }), - reset: () => - set({ - searchQuery: "", - nodeTypeFilters: new Set(defaultFilters), - fileTypeFilters: new Set(), - selectedNodeId: null, - highlightedNodeIds: new Set(), - sidebarTab: "overview", - workspaceMode: "full", - explorerOpen: true, - aiSidebarOpen: true, - inspectorOpen: false, - focusedFilePath: null, - zoom: 1.0, - visibleNodeCount: 0, - visibleEdgeCount: 0, - toastMessage: null, - selectedBranch: null, - compareBranch: null, - compareRefType: "branch" as CompareRefType, - availableBranches: [], - activeFocusLayer: "all", - diffStatusFilters: new Set(), - activeView: "graph", - blastRadiusActive: false, - colorMode: "default" as ColorMode, - sizeMode: "default" as SizeMode, - layoutType: "tree" as LayoutType, - graphScope: "source", - contentFilters: new Set(defaultContentFilters), - activeDropdown: null, - graphActionTrigger: null, - }), - }), - { - name: 'gitsdm-viz-storage', - partialize: (state) => ({ - workspaceMode: state.workspaceMode, - explorerOpen: state.explorerOpen, - aiSidebarOpen: state.aiSidebarOpen, - inspectorOpen: state.inspectorOpen, - sidebarTab: state.sidebarTab, - theme: state.theme, - activeView: state.activeView, - graphSidebarOpen: state.graphSidebarOpen, - activeDropdown: state.activeDropdown, - graphScope: state.graphScope, - layoutType: state.layoutType, - contentFilters: Array.from(state.contentFilters), + setActiveRepoKey: (activeRepoKey: string | null) => set({ activeRepoKey }), + + reset: () => + set({ + searchQuery: '', + nodeTypeFilters: new Set(defaultFilters), + fileTypeFilters: new Set(), + selectedNodeId: null, + highlightedNodeIds: new Set(), + sidebarTab: 'overview', + workspaceMode: 'full', + explorerOpen: true, + aiSidebarOpen: true, + inspectorOpen: false, + focusedFilePath: null, + zoom: 1.0, + visibleNodeCount: 0, + visibleEdgeCount: 0, + toastMessage: null, + selectedBranch: null, + compareBranch: null, + compareRefType: 'branch' as CompareRefType, + availableBranches: [], + activeFocusLayer: 'all', + diffStatusFilters: new Set(), + activeView: 'graph', + blastRadiusActive: false, + colorMode: 'default' as ColorMode, + sizeMode: 'default' as SizeMode, + layoutType: 'tree' as LayoutType, + graphScope: 'source', + contentFilters: new Set(defaultContentFilters), + activeDropdown: null, + graphActionTrigger: null, + }), }), - merge: (persistedState: unknown, currentState: VizState): VizState => { - const p = persistedState as Partial & { contentFilters: ContentFilter[] }>; - return { - ...currentState, - ...p, - contentFilters: p?.contentFilters - ? new Set(p.contentFilters) - : currentState.contentFilters, - } as VizState; + { + name: 'gitsdm-viz-storage', + partialize: (state) => ({ + workspaceMode: state.workspaceMode, + explorerOpen: state.explorerOpen, + aiSidebarOpen: state.aiSidebarOpen, + inspectorOpen: state.inspectorOpen, + sidebarTab: state.sidebarTab, + theme: state.theme, + activeView: state.activeView, + graphSidebarOpen: state.graphSidebarOpen, + activeDropdown: state.activeDropdown, + graphScope: state.graphScope, + layoutType: state.layoutType, + contentFilters: Array.from(state.contentFilters), + }), + merge: (persistedState: unknown, currentState: VizState): VizState => { + const p = persistedState as Partial & { contentFilters: ContentFilter[] }>; + const validModes: WorkspaceMode[] = ['full', 'explorer', 'insights', 'focus']; + const workspaceMode = + p?.workspaceMode && validModes.includes(p.workspaceMode as WorkspaceMode) + ? (p.workspaceMode as WorkspaceMode) + : modeFromPanels( + p?.explorerOpen ?? currentState.explorerOpen, + p?.aiSidebarOpen ?? currentState.aiSidebarOpen, + ); + return { + ...currentState, + ...p, + workspaceMode, + contentFilters: p?.contentFilters ? new Set(p.contentFilters) : currentState.contentFilters, + } as VizState; + }, }, - } -)); + ), +); diff --git a/src/styles/interface.css b/src/styles/interface.css index c16878e..130edcd 100644 --- a/src/styles/interface.css +++ b/src/styles/interface.css @@ -195,10 +195,23 @@ padding: 24px; } } +[data-reduced-settling] canvas { + visibility: hidden; +} +html[data-reduced-motion='true'] *, +html[data-reduced-motion='true'] ::before, +html[data-reduced-motion='true'] ::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + scroll-behavior: auto !important; +} +/* Before React sets data-reduced-motion, honor the OS setting; the :not() in the + selector keeps an explicit 'false' (user picked Full) on full animation. */ @media (prefers-reduced-motion: reduce) { - *, - ::before, - ::after { + html:not([data-reduced-motion='false']) *, + html:not([data-reduced-motion='false']) ::before, + html:not([data-reduced-motion='false']) ::after { animation-duration: 0.01ms !important; animation-iteration-count: 1 !important; transition-duration: 0.01ms !important; diff --git a/src/types/api.ts b/src/types/api.ts index 13d66d2..194d659 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -87,7 +87,36 @@ export interface QAAnswer { citations: { filePath: string; startLine: number; endLine: number }[]; } +export interface SearchCoverage { + kind: 'complete' | 'previous' | 'partial'; + commitSha: string; + indexedFiles: number; + totalFiles: number; +} + +export type IndexingStatus = ( + | { state: 'idle' } + | { + state: 'indexing'; + progress: number; + filesProcessed: number; + totalFiles: number; + } + | { state: 'complete'; chunkCount: number; timestamp: number } + | { + state: 'paused'; + error: string; + reason: string; + retryAt?: number; + filesProcessed: number; + totalFiles: number; + progress: number; + } + | { state: 'failed'; error: string; failedFiles: number } +) & { coverage?: SearchCoverage; snapshotSha?: string; buildId?: string }; + export interface SearchResponse { + coverage?: SearchCoverage; results: { chunk: { filePath: string; @@ -106,13 +135,8 @@ export interface SearchResponse { } export interface QAResponse { + coverage?: SearchCoverage; answer: string; citations: { filePath: string; startLine: number; endLine: number }[]; cached: boolean; } - -export type IndexingStatus = - | { state: 'idle' } - | { state: 'indexing'; progress: number; filesProcessed: number; totalFiles: number } - | { state: 'complete'; chunkCount: number; timestamp: number } - | { state: 'failed'; error: string; failedFiles: number }; diff --git a/src/types/churn.ts b/src/types/churn.ts new file mode 100644 index 0000000..f8db67c --- /dev/null +++ b/src/types/churn.ts @@ -0,0 +1,27 @@ +export interface FileChurnData { + commitCount: number; + authorCount: number; + lastModified?: string; + churnScore: number; +} + +export interface ChurnResponse { + files: Record; + checked: number; + total: number; + complete: boolean; + issue?: 'rate-limit' | 'access' | 'timeout-or-network'; + retryAt?: number; + remaining: string[]; + failures: Record; retryAt: number }>; + /** Opaque credential-scope fingerprint echoed back in the next continuation. + * Lets the server reject stale progress after a credential change. Never a secret. */ + scope?: string; +} + +export interface ChurnContinuation { + completed?: string[]; + pending?: string[]; + /** Scope echoed from a prior ChurnResponse; omitted by older clients (trusted for back-compat). */ + scope?: string; +} diff --git a/src/types/domain.ts b/src/types/domain.ts index 06bc6b8..2ffdcb5 100644 --- a/src/types/domain.ts +++ b/src/types/domain.ts @@ -189,6 +189,7 @@ export interface RepoAnalysis { dependencies: Dependency[]; workspacePackages?: WorkspacePackage[]; dependencyHealth?: DependencyHealthReport; + scopedDependencies?: ScopedDependency[]; graph: GraphData; contributors: Contributor[]; timeline: TimelineWeek[]; diff --git a/vercel.json b/vercel.json index 56bdef0..36b67e0 100644 --- a/vercel.json +++ b/vercel.json @@ -2,9 +2,29 @@ "buildCommand": "bun run build", "bunVersion": "1.x", "outputDirectory": "dist", + "headers": [ + { + "source": "/assets/(.*)", + "headers": [ + { + "key": "Cache-Control", + "value": "public, max-age=31536000, immutable" + } + ] + } + ], "rewrites": [ { "source": "/api/(.*)", "destination": "/api/$1" }, - { "source": "/((?!api/).*)", "destination": "/index.html" } + { "source": "/privacy", "destination": "/privacy.html" }, + { "source": "/terms", "destination": "/terms.html" }, + { + "source": "/:owner((?!api/|assets/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})", + "destination": "/app.html" + }, + { + "source": "/:owner((?!api/|assets/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})/search", + "destination": "/app.html" + } ], "functions": { "api/**/*.ts": { diff --git a/vite.config.ts b/vite.config.ts index 59c0a8b..f4ad3ec 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -17,7 +17,14 @@ export default defineConfig({ }), ], build: { - chunkSizeWarningLimit: 3000, + chunkSizeWarningLimit: 1000, + rollupOptions: { + output: { + manualChunks(id) { + if (/[\\/]node_modules[\\/]highlight\.js[\\/]/.test(id)) return 'syntax-highlighting'; + }, + }, + }, }, resolve: { alias: {