From d80d41b676478724c538e9f86eeb116018661c02 Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Tue, 8 Sep 2026 19:30:31 +0700 Subject: [PATCH 01/27] chore(release): v3.0.1 - audit fixes, chunk mermaid, font FOIT, SEO, cache headers --- AGENTS.md | 2 +- ARCHITECTURE.md | 2 +- CHANGELOG.md | 11 +++ README.md | 5 +- index.html | 61 +++++++--------- package.json | 2 +- public/og-image.svg | 91 ++++++++++++++++++++++++ public/robots.txt | 5 ++ public/sitemap.xml | 18 +++++ server/api-router.ts | 7 -- server/prod-server.ts | 10 ++- server/utils/http.test.ts | 2 +- server/utils/http.ts | 2 +- src/components/home/CapabilityGroups.tsx | 6 +- vercel.json | 11 +++ vite.config.ts | 11 ++- 16 files changed, 191 insertions(+), 55 deletions(-) create mode 100644 public/og-image.svg create mode 100644 public/robots.txt create mode 100644 public/sitemap.xml diff --git a/AGENTS.md b/AGENTS.md index f6cc67f..cf917b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ ## Project Identity -**gitSdm** (Git Software Dependency Map) — v3.0.0. Graph-first repository analysis tool: visualize file dependencies, AI-powered codebase insights, semantic search, commit timelines, architecture diagrams, and dependency health. Single-page app with an embedded Express backend + Vercel serverless functions. +**gitSdm** (Git Software Dependency Map) — v3.0.1. Interactive repository dependency visualization tool: map file dependencies, AI-powered codebase insights, semantic search, commit timelines, architecture diagrams, and dependency health. Single-page app with an embedded Express backend + Vercel serverless functions. --- diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 35e70f1..a79e12a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architecture -`gitSdm` is a graph-first repository analysis application. It visualizes file dependencies, architecture diagrams, module hierarchies, commit history, and AI-driven codebase insights in a single-page app with an embedded Express backend and Vercel serverless functions. +`gitSdm` is an interactive repository dependency visualization tool. It maps file dependencies, architecture diagrams, module hierarchies, commit history, and codebase insights in a single-page app with an embedded Express backend and Vercel serverless functions. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index ed2f835..7fee0b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.0.1] - 2026-09-08 + +### Fixed + +- Eliminated unauthenticated `/api/cache/clear` endpoint. +- Code-split `mermaid` library to dedicated vendor chunk, shrinking `ArchitectureView` from 1.44 MB to 22 KB. +- Removed font-gating FOIT blocker in `index.html` and enabled `display=swap`. +- Added immutable Cache-Control headers for hashed static assets across Bun server, Vercel, and EdgeOne. +- Added search engine metadata: `robots.txt`, `sitemap.xml`, OpenGraph SVG banner, and social meta tags. +- Replaced marketing buzzword copy across documentation and landing pages. + ## [3.0.0] - 2026-09-07 ### Added diff --git a/README.md b/README.md index 801296c..d218ab9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -

gitSdm (Git Software Dependency Map v3.0.0)

+

gitSdm — Git Software Dependency Map

- Graph-first repository analysis for exploring files, dependencies, modules, architecture diagrams, and AI-powered codebase intelligence. + Interactive dependency maps for exploring files, modules, architecture diagrams, and codebase intelligence.

+ Live Demo License PRs Welcome CI diff --git a/index.html b/index.html index c50f653..52273d6 100644 --- a/index.html +++ b/index.html @@ -5,30 +5,41 @@ - + gitSdmᵝ — Git Software Dependency Map + + + + + + + + + + + + + + + + + + + + + - - gitSdmᵝ — Git Software Dependency Map - + + diff --git a/package.json b/package.json index 43a34dc..c59552d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "gitsdm", "private": true, - "version": "3.0.0", + "version": "3.0.1", "type": "module", "scripts": { "dev": "bunx --bun vite", diff --git a/public/og-image.svg b/public/og-image.svg new file mode 100644 index 0000000..9e3ba91 --- /dev/null +++ b/public/og-image.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + gitSdm + + + v3.0 + + + + Interactive Repository Intelligence + + + + Explore files, dependencies, modules, and architecture in your browser. + + + + + + Interactive Graph + + + Mermaid Diagrams + + + Semantic Search + + + AI Insights + + diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..cdcff03 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Allow: / +Disallow: /api/ + +Sitemap: https://gsdm.site/sitemap.xml diff --git a/public/sitemap.xml b/public/sitemap.xml new file mode 100644 index 0000000..ca147ef --- /dev/null +++ b/public/sitemap.xml @@ -0,0 +1,18 @@ + + + + https://gsdm.site/ + weekly + 1.0 + + + https://gsdm.site/mock/gitsdm + monthly + 0.8 + + + https://gsdm.site/mock/todo-app + monthly + 0.8 + + diff --git a/server/api-router.ts b/server/api-router.ts index 0b8087c..5d7df76 100644 --- a/server/api-router.ts +++ b/server/api-router.ts @@ -2,7 +2,6 @@ import { getOctokit } from './github/client'; import type { RequestContext } from './utils/context'; import { fetchTrending } from './services/trending'; import { logApi, logError } from './utils/logger'; -import { clearAllCaches } from './cache/lru'; import { getPublicAppConfig } from './config/app-config'; import { toErrorPayload } from './utils/errors'; import type { TrendingRepo } from '../src/types'; @@ -46,12 +45,6 @@ export async function handleApiRequest( return addSecurityHeaders(Response.json(getPublicAppConfig(), { status: 200 })); } - if (pathname === '/api/cache/clear' && method === 'POST') { - clearAllCaches(); - logApi('/api/cache/clear', { durationMs: Date.now() - start }); - return addSecurityHeaders(Response.json({ cleared: true }, { status: 200 })); - } - // ── Repository Routes ─────────────────────────────────────────── const repoResponse = await handleRepoRoutes(pathname, req, query, ctx, start); if (repoResponse) return addSecurityHeaders(repoResponse); diff --git a/server/prod-server.ts b/server/prod-server.ts index 3e7b82d..c9ada01 100644 --- a/server/prod-server.ts +++ b/server/prod-server.ts @@ -38,11 +38,17 @@ Bun.serve({ const filePath = safeJoin(distDir, decodedPath); const file = Bun.file(filePath); if (await file.exists()) { - return addSecurityHeaders(new Response(file)); + const res = new Response(file); + if (pathname.startsWith('/assets/')) { + res.headers.set('Cache-Control', 'public, max-age=31536000, immutable'); + } + return addSecurityHeaders(res); } // SPA fallback - return addSecurityHeaders(new Response(Bun.file(indexFile))); + const fallbackRes = new Response(Bun.file(indexFile)); + fallbackRes.headers.set('Cache-Control', 'no-cache'); + return addSecurityHeaders(fallbackRes); }, }); diff --git a/server/utils/http.test.ts b/server/utils/http.test.ts index 80c6df9..58fbef4 100644 --- a/server/utils/http.test.ts +++ b/server/utils/http.test.ts @@ -9,6 +9,6 @@ describe('addSecurityHeaders', () => { expect(secureResponse.headers.get('X-Content-Type-Options')).toBe('nosniff'); expect(secureResponse.headers.get('X-Frame-Options')).toBe('DENY'); expect(secureResponse.headers.get('Strict-Transport-Security')).toBe('max-age=31536000; includeSubDomains'); - expect(secureResponse.headers.get('Content-Security-Policy')).toBe("default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-mhDq8RP/TAuNwiFSk7hwZsZ3tIWH410AupSuJ9xEhZg=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); + expect(secureResponse.headers.get('Content-Security-Policy')).toBe("default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-1HwijBPaKaOtZGvlT2Hw0JpmgDixxpRo6XKpkDG9xvs=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); }); }); diff --git a/server/utils/http.ts b/server/utils/http.ts index 70d0107..78c9e5a 100644 --- a/server/utils/http.ts +++ b/server/utils/http.ts @@ -64,6 +64,6 @@ export function addSecurityHeaders(response: Response): Response { response.headers.set('X-Frame-Options', 'DENY'); // X-XSS-Protection is deprecated; use CSP instead response.headers.set('Strict-Transport-Security', 'max-age=31536000; includeSubDomains'); - response.headers.set('Content-Security-Policy', "default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-mhDq8RP/TAuNwiFSk7hwZsZ3tIWH410AupSuJ9xEhZg=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); + response.headers.set('Content-Security-Policy', "default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-1HwijBPaKaOtZGvlT2Hw0JpmgDixxpRo6XKpkDG9xvs=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); return response; } diff --git a/src/components/home/CapabilityGroups.tsx b/src/components/home/CapabilityGroups.tsx index 30c8df3..6a47159 100644 --- a/src/components/home/CapabilityGroups.tsx +++ b/src/components/home/CapabilityGroups.tsx @@ -44,9 +44,9 @@ export function CapabilityGroups() {

- From your first look at a repository -
to your next architectural - decision. + Explore codebase structure and dependencies +
to make architectural decisions + with confidence.

diff --git a/vercel.json b/vercel.json index 56bdef0..4dcb1d2 100644 --- a/vercel.json +++ b/vercel.json @@ -2,6 +2,17 @@ "buildCommand": "bun run build", "bunVersion": "1.x", "outputDirectory": "dist", + "headers": [ + { + "source": "/assets/(.*)", + "headers": [ + { + "key": "Cache-Control", + "value": "public, max-age=31536000, immutable" + } + ] + } + ], "rewrites": [ { "source": "/api/(.*)", "destination": "/api/$1" }, { "source": "/((?!api/).*)", "destination": "/index.html" } diff --git a/vite.config.ts b/vite.config.ts index 59c0a8b..ad00bac 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -17,7 +17,16 @@ export default defineConfig({ }), ], build: { - chunkSizeWarningLimit: 3000, + chunkSizeWarningLimit: 1000, + rollupOptions: { + output: { + manualChunks(id) { + if (id.includes('/mermaid/') || id.includes('\\mermaid\\') || id.endsWith('/mermaid') || id.endsWith('\\mermaid')) { + return 'mermaid'; + } + }, + }, + }, }, resolve: { alias: { From 4f30fc65b317f49209c12bbf824cadf82bdebae0 Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Tue, 8 Sep 2026 21:37:48 +0700 Subject: [PATCH 02/27] fix(workspace): correct sidebar modes and bump to v3.0.2 --- AGENTS.md | 2 +- CHANGELOG.md | 7 +++++ e2e/interface.e2e.ts | 19 ++++++++---- package.json | 4 +-- .../viz/top-nav/HeaderActionMenu.tsx | 2 +- src/components/viz/top-nav/workspaceModes.ts | 30 +++++++++---------- src/stores/vizStore.ts | 15 ++++++---- 7 files changed, 50 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cf917b0..040fe5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ ## Project Identity -**gitSdm** (Git Software Dependency Map) — v3.0.1. Interactive repository dependency visualization tool: map file dependencies, AI-powered codebase insights, semantic search, commit timelines, architecture diagrams, and dependency health. Single-page app with an embedded Express backend + Vercel serverless functions. +**gitSdm** (Git Software Dependency Map) — v3.0.2. Interactive repository dependency visualization tool: map file dependencies, AI-powered codebase insights, semantic search, commit timelines, architecture diagrams, and dependency health. Single-page app with an embedded Express backend + Vercel serverless functions. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fee0b2..7f01dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.0.2] - 2026-09-08 + +### Fixed + +- Made workspace modes mutually exclusive: Explorer Only opens only the file explorer, Insights Only opens only repository insights, and Focus Mode closes both panels. +- Added browser regression coverage for desktop and mobile workspace-mode panel visibility. + ## [3.0.1] - 2026-09-08 ### Fixed diff --git a/e2e/interface.e2e.ts b/e2e/interface.e2e.ts index 9549217..0cea692 100644 --- a/e2e/interface.e2e.ts +++ b/e2e/interface.e2e.ts @@ -23,18 +23,27 @@ test('learning Explain responds to Enter and synchronizes the inspected file', a await expect(page.getByRole('region', { name: 'File inspector' })).toContainText('src/App.tsx'); }); -test('workspace modes hide both sidebars in focus and open learning on mobile', async ({ page }) => { +test('workspace modes show only the selected sidebar', async ({ page }) => { await page.setViewportSize({ width: 1256, height: 912 }); await page.goto('/mock/todo-app'); await page.getByRole('button', { name: 'Full Workspace', exact: true }).click(); - await page.getByRole('menuitem', { name: 'Focus Mode Minimizes sidebars to focus purely on the canvas', exact: true }).click(); + await page.getByRole('menuitem', { name: /Explorer Only/ }).click(); + await expect(page.getByRole('button', { name: 'Collapse file explorer', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Open repository insights', exact: true })).toBeVisible(); + + await page.getByRole('button', { name: 'Explorer Only', exact: true }).click(); + await page.getByRole('menuitem', { name: /Insights Only/ }).click(); + await expect(page.getByRole('button', { name: 'Open file explorer', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Collapse repository insights', exact: true })).toBeVisible(); + + await page.getByRole('button', { name: 'Insights Only', exact: true }).click(); + await page.getByRole('menuitem', { name: /Focus Mode/ }).click(); await expect(page.getByRole('button', { name: 'Open file explorer', exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Open repository insights', exact: true })).toBeVisible(); await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: 'Open menu', exact: true }).click(); - await page.getByRole('button', { name: 'Learning Mode', exact: true }).click(); - await expect(page.getByRole('tab', { name: 'Learning', exact: true })).toBeVisible(); - await expect(page.getByRole('tab', { name: 'Learning', exact: true })).toHaveAttribute('aria-selected', 'true'); + await page.getByRole('button', { name: 'Insights Only', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Collapse repository insights', exact: true })).toBeVisible(); }); test('layout changes fit the graph after focusing a file', async ({ page }) => { diff --git a/package.json b/package.json index c59552d..9ecf1fc 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "gitsdm", "private": true, - "version": "3.0.1", + "version": "3.0.2", "type": "module", "scripts": { "dev": "bunx --bun vite", @@ -78,4 +78,4 @@ "esbuild", "protobufjs" ] -} \ No newline at end of file +} diff --git a/src/components/viz/top-nav/HeaderActionMenu.tsx b/src/components/viz/top-nav/HeaderActionMenu.tsx index 3952ba2..9669817 100644 --- a/src/components/viz/top-nav/HeaderActionMenu.tsx +++ b/src/components/viz/top-nav/HeaderActionMenu.tsx @@ -188,7 +188,7 @@ export function HeaderActionMenu({ owner, repo, analysis, meta: propsMeta }: Hea workspaceMode={workspaceMode} onViewChange={(id) => { setActiveView(id as 'graph' | 'architecture' | 'contributors' | 'commits'); closeMenu(); }} onWorkspaceModeChange={(id) => { - setWorkspaceMode(id as 'focus' | 'analysis' | 'learning' | 'full'); + setWorkspaceMode(id); closeMenu(); }} onClose={closeMenu} diff --git a/src/components/viz/top-nav/workspaceModes.ts b/src/components/viz/top-nav/workspaceModes.ts index 866c5b4..f5185cc 100644 --- a/src/components/viz/top-nav/workspaceModes.ts +++ b/src/components/viz/top-nav/workspaceModes.ts @@ -1,7 +1,7 @@ -import { LayoutPanelLeft, Monitor, ActivitySquare, BookOpen } from 'lucide-react'; +import { LayoutPanelLeft, PanelLeft, PanelRight, Monitor } from 'lucide-react'; export interface WorkspaceMode { - id: 'full' | 'focus' | 'analysis' | 'learning'; + id: 'full' | 'explorer' | 'insights' | 'focus'; label: string; description: string; icon: React.ComponentType<{ className?: string }>; @@ -19,30 +19,30 @@ export const WORKSPACE_MODES: WorkspaceMode[] = [ bgColor: 'bg-secondary border-border text-foreground', }, { - id: 'focus', - label: 'Focus Mode', - description: 'Minimizes sidebars to focus purely on the canvas', - icon: Monitor, + id: 'explorer', + label: 'Explorer Only', + description: 'Show the file explorer with maximized canvas space', + icon: PanelLeft, color: 'text-foreground', bgColor: 'bg-secondary border-border text-foreground', }, { - id: 'analysis', - label: 'Analysis Mode', - description: 'Inspect file details and relationships', - icon: ActivitySquare, + id: 'insights', + label: 'Insights Only', + description: 'Show repository insights without the file tree', + icon: PanelRight, color: 'text-foreground', bgColor: 'bg-secondary border-border text-foreground', }, { - id: 'learning', - label: 'Learning Mode', - description: 'Follow a guided reading path through the codebase', - icon: BookOpen, + id: 'focus', + label: 'Focus Mode', + description: 'Minimizes sidebars to focus purely on the canvas', + icon: Monitor, color: 'text-foreground', bgColor: 'bg-secondary border-border text-foreground', }, ]; -export const LAYOUT_MODE_IDS = ['full', 'focus', 'analysis', 'learning'] as const; +export const LAYOUT_MODE_IDS = ['full', 'explorer', 'insights', 'focus'] as const; export type WorkspaceModeId = (typeof LAYOUT_MODE_IDS)[number]; diff --git a/src/stores/vizStore.ts b/src/stores/vizStore.ts index e90392f..42ce5b1 100644 --- a/src/stores/vizStore.ts +++ b/src/stores/vizStore.ts @@ -3,7 +3,7 @@ import { persist } from "zustand/middleware"; import type { NodeType } from "@/types"; export type SidebarTab = "overview" | "analysis" | "dependencies" | "ai" | "learning"; -export type WorkspaceMode = "focus" | "analysis" | "learning" | "full"; +export type WorkspaceMode = "full" | "explorer" | "insights" | "focus"; export type GraphScope = "important" | "source" | "grouped" | "full"; export type ContentFilter = "source" | "config" | "docs" | "tests" | "github" | "examples" | "generated" | "translations"; export type CompareRefType = "branch" | "tag" | "commit"; @@ -221,13 +221,13 @@ export const useVizStore = create()( setSidebarTab: (sidebarTab: SidebarTab) => set({ sidebarTab }), setWorkspaceMode: (workspaceMode: WorkspaceMode) => { const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; + const showExplorer = workspaceMode === 'full' || workspaceMode === 'explorer'; + const showAiSidebar = workspaceMode === 'full' || workspaceMode === 'insights'; set({ workspaceMode, - explorerOpen: workspaceMode !== 'focus' && (!isMobile || workspaceMode === 'full'), - aiSidebarOpen: workspaceMode !== 'focus' && (!isMobile || workspaceMode === 'analysis' || workspaceMode === 'learning'), + explorerOpen: showExplorer && (!isMobile || workspaceMode === 'explorer'), + aiSidebarOpen: showAiSidebar && (!isMobile || workspaceMode === 'insights'), inspectorOpen: false, - ...(workspaceMode === 'analysis' ? { sidebarTab: 'analysis' as const } : {}), - ...(workspaceMode === 'learning' ? { sidebarTab: 'learning' as const } : {}), }); }, setExplorerOpen: (explorerOpen: boolean) => set({ explorerOpen }), @@ -333,9 +333,14 @@ export const useVizStore = create()( }), merge: (persistedState: unknown, currentState: VizState): VizState => { const p = persistedState as Partial & { contentFilters: ContentFilter[] }>; + const validModes: WorkspaceMode[] = ['full', 'explorer', 'insights', 'focus']; + const workspaceMode = p?.workspaceMode && validModes.includes(p.workspaceMode as WorkspaceMode) + ? (p.workspaceMode as WorkspaceMode) + : 'full'; return { ...currentState, ...p, + workspaceMode, contentFilters: p?.contentFilters ? new Set(p.contentFilters) : currentState.contentFilters, From f1c226ba4004f8e021436535ce9af957f0eb298e Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Tue, 8 Sep 2026 22:23:35 +0700 Subject: [PATCH 03/27] feat(privacy): add disclosures and prerender public pages Add a plain-language privacy page and credential transmission disclosures. Prerender the homepage, privacy page, and matching 404 page, and limit SPA fallbacks to repository routes. --- package.json | 2 +- public/sitemap.xml | 4 + scripts/prerender.tsx | 26 +++++++ server/prod-server.ts | 14 ++-- server/utils/page-route.test.ts | 7 ++ server/utils/page-route.ts | 4 + src/app/router.tsx | 2 + src/components/home/HomeGraphPreview.tsx | 4 +- src/components/layout/InfoPageLayout.tsx | 22 ++++++ src/components/viz/SettingsPopover.tsx | 3 +- src/pages/HomePage.tsx | 5 +- src/pages/NotFoundPage.tsx | 94 +++++------------------- src/pages/PrivacyPage.tsx | 36 +++++++++ vercel.json | 4 +- 14 files changed, 140 insertions(+), 87 deletions(-) create mode 100644 scripts/prerender.tsx create mode 100644 server/utils/page-route.test.ts create mode 100644 server/utils/page-route.ts create mode 100644 src/components/layout/InfoPageLayout.tsx create mode 100644 src/pages/PrivacyPage.tsx diff --git a/package.json b/package.json index 9ecf1fc..ee7a001 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "dev": "bunx --bun vite", - "build": "bunx --bun vite build", + "build": "bunx --bun vite build && bun scripts/prerender.tsx", "build:server": "bunx --bun vite build --ssr server/prod-server.ts --outDir dist-server", "build:docker": "bun run build && bun run build:server", "start": "bun dist-server/prod-server.js", diff --git a/public/sitemap.xml b/public/sitemap.xml index ca147ef..280243c 100644 --- a/public/sitemap.xml +++ b/public/sitemap.xml @@ -1,5 +1,9 @@ + + https://gsdm.site/privacy + monthly + https://gsdm.site/ weekly diff --git a/scripts/prerender.tsx b/scripts/prerender.tsx new file mode 100644 index 0000000..c464795 --- /dev/null +++ b/scripts/prerender.tsx @@ -0,0 +1,26 @@ +import { renderToStaticMarkup } from 'react-dom/server'; +import { StaticRouter } from 'react-router-dom'; +import { Providers } from '../src/app/providers'; +import { HomePage } from '../src/pages/HomePage'; +import { PrivacyPage } from '../src/pages/PrivacyPage'; +import { NotFoundPage } from '../src/pages/NotFoundPage'; + +const shell = await Bun.file('dist/index.html').text(); +if (!shell.includes('
')) throw new Error('Prerender root marker missing'); +// Repository routes use a blank shell, never the homepage's SEO or content. +await Bun.write('dist/app.html', shell.replace(/]*>/, '')); +for (const [route, file, page] of [ + ['/', 'dist/index.html', ], + ['/privacy', 'dist/privacy.html', ], +] as const) { + const content = renderToStaticMarkup({page}); + const html = shell.replace('
', `
${content}
`); + await Bun.write(file, route === '/' ? html : html + .replaceAll('"https://gsdm.site/"', '"https://gsdm.site/privacy"') + .replace('gitSdmᵝ — Git Software Dependency Map', 'Privacy policy — gitSdm')); +} +const notFound = renderToStaticMarkup(); +await Bun.write('dist/404.html', shell + .replace('
', `
${notFound}
`) + .replace('gitSdmᵝ — Git Software Dependency Map', 'Page not found — gitSdm') + .replace(/]*>/, '')); diff --git a/server/prod-server.ts b/server/prod-server.ts index c9ada01..e4f2331 100644 --- a/server/prod-server.ts +++ b/server/prod-server.ts @@ -3,9 +3,9 @@ import { handleApiRequest } from './api-router'; import { resetOctokit } from './github/client'; import { addSecurityHeaders } from './utils/http'; import { logInfo } from './utils/logger'; +import { isRepositoryPage } from './utils/page-route'; const distDir = path.resolve(import.meta.dir, '../dist'); -const indexFile = path.join(distDir, 'index.html'); const port = Number(process.env.PORT ?? 3000); const host = process.env.HOST ?? '0.0.0.0'; @@ -35,7 +35,9 @@ Bun.serve({ return new Response('Bad Request', { status: 400 }); } - const filePath = safeJoin(distDir, decodedPath); + const pageFile = decodedPath === '/' ? '/index.html' : decodedPath === '/privacy' || decodedPath === '/privacy/' ? '/privacy.html' : decodedPath; + const filePath = safeJoin(distDir, pageFile); + if (!filePath) return addSecurityHeaders(new Response('Not found', { status: 404 })); const file = Bun.file(filePath); if (await file.exists()) { const res = new Response(file); @@ -45,8 +47,8 @@ Bun.serve({ return addSecurityHeaders(res); } - // SPA fallback - const fallbackRes = new Response(Bun.file(indexFile)); + const validRoute = isRepositoryPage(decodedPath); + const fallbackRes = new Response(Bun.file(path.join(distDir, validRoute ? 'app.html' : '404.html')), { status: validRoute ? 200 : 404 }); fallbackRes.headers.set('Cache-Control', 'no-cache'); return addSecurityHeaders(fallbackRes); }, @@ -56,7 +58,7 @@ const hasToken = Boolean(process.env.GITHUB_TOKEN?.trim()); logInfo(`[gitSdm] listening on http://${host}:${port}`); logInfo(`[gitSdm] GitHub API: ${hasToken ? 'authenticated' : 'unauthenticated'}`); -function safeJoin(root: string, pathname: string): string { +function safeJoin(root: string, pathname: string): string | null { const normalized = pathname.replace(/^[/\\]+/, ''); const resolved = path.resolve(root, normalized); @@ -68,5 +70,5 @@ function safeJoin(root: string, pathname: string): string { return resolved; } - return indexFile; + return null; } diff --git a/server/utils/page-route.test.ts b/server/utils/page-route.test.ts new file mode 100644 index 0000000..fa985c2 --- /dev/null +++ b/server/utils/page-route.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from 'bun:test'; +import { isRepositoryPage } from './page-route'; + +test('repository routes support deep links without treating missing assets as pages', () => { + for (const path of ['/mock/todo-app', '/mbayue/gitSdm/search', '/owner/repo/']) expect(isRepositoryPage(path)).toBe(true); + for (const path of ['/missing', '/assets/missing.js', '/api/missing', '/owner/repo/unknown', '/a/b/c/d']) expect(isRepositoryPage(path)).toBe(false); +}); diff --git a/server/utils/page-route.ts b/server/utils/page-route.ts new file mode 100644 index 0000000..ecfdaed --- /dev/null +++ b/server/utils/page-route.ts @@ -0,0 +1,4 @@ +/** Valid SPA paths; repository existence is resolved by the repository API. */ +export function isRepositoryPage(path: string): boolean { + return /^\/(?!api\/|assets\/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38}\/[a-zA-Z0-9_.-]{1,100}(?:\/search)?\/?$/.test(path); +} diff --git a/src/app/router.tsx b/src/app/router.tsx index 30fac6e..2c49027 100644 --- a/src/app/router.tsx +++ b/src/app/router.tsx @@ -6,6 +6,7 @@ const HomePage = lazy(() => import('@/pages/HomePage').then((module) => ({ defau const VizPage = lazy(() => import('@/pages/VizPage').then((module) => ({ default: module.VizPage }))); const SearchPage = lazy(() => import('@/pages/SearchPage').then((module) => ({ default: module.SearchPage }))); const NotFoundPage = lazy(() => import('@/pages/NotFoundPage').then((module) => ({ default: module.NotFoundPage }))); +const PrivacyPage = lazy(() => import('@/pages/PrivacyPage').then((module) => ({ default: module.PrivacyPage }))); function RouterContent() { const location = useLocation(); @@ -15,6 +16,7 @@ function RouterContent() { }> } /> + } /> } /> } /> } /> diff --git a/src/components/home/HomeGraphPreview.tsx b/src/components/home/HomeGraphPreview.tsx index cb1256b..dd78aa5 100644 --- a/src/components/home/HomeGraphPreview.tsx +++ b/src/components/home/HomeGraphPreview.tsx @@ -53,7 +53,7 @@ export function HomeGraphPreview() {
} > - Explore files and dependencies in the interactive map when JavaScript is enabled.

: + />}
Drag to explore · Scroll to zoom diff --git a/src/components/layout/InfoPageLayout.tsx b/src/components/layout/InfoPageLayout.tsx new file mode 100644 index 0000000..30ccac9 --- /dev/null +++ b/src/components/layout/InfoPageLayout.tsx @@ -0,0 +1,22 @@ +import type { ReactNode } from 'react'; +import { Link } from 'react-router-dom'; +import { Navbar } from './Navbar'; + +export function InfoPageLayout({ children }: { children: ReactNode }) { + return ( +
+ +
{children}
+
+
+ gitSdm · Repository explorer + +
+
+
+ ); +} diff --git a/src/components/viz/SettingsPopover.tsx b/src/components/viz/SettingsPopover.tsx index db9c2b7..97668a4 100644 --- a/src/components/viz/SettingsPopover.tsx +++ b/src/components/viz/SettingsPopover.tsx @@ -357,7 +357,8 @@ export function SettingsPopover({ {/* Footer note */}
- Keys are stored locally in your browser. Do not use shared devices. +

Saved keys are stored in this browser and sent to the gitSdm backend for relevant requests. AI and search features may send repository content, including private code, to configured AI and embedding providers.

+

Do not save keys on shared devices. Privacy policy (opens in a new tab)

)} diff --git a/src/pages/HomePage.tsx b/src/pages/HomePage.tsx index ff7eec3..cc4f26e 100644 --- a/src/pages/HomePage.tsx +++ b/src/pages/HomePage.tsx @@ -8,7 +8,7 @@ import { CapabilityGroups } from "@/components/home/CapabilityGroups"; import { LAST_REPO_KEY } from "@/lib/utils"; export function HomePage() { - const [repoUrl] = useState(() => localStorage.getItem(LAST_REPO_KEY) ?? ""); + const [repoUrl] = useState(() => typeof localStorage === 'undefined' ? '' : localStorage.getItem(LAST_REPO_KEY) ?? ""); useEffect(() => { const sectionId = window.location.hash.slice(1); @@ -43,6 +43,9 @@ export function HomePage() { © 2026
+ + Privacy + - {/* Subtle gradient background */} -
-
-
-
- - {/* Faint grid texture */} -
- - - {/* Brand mark */} -
- - gitSdm -
- - {/* 404 */} -
-

- 404 -

+ +
+

404 / Page not found

+

This path ends here.

+

The page may have moved, or the address may be incorrect. Return home to open a repository and keep exploring.

+
+
+
+
+

Find your next codebase

Open a GitHub link or try the sample project.

+ Go to homepage
- - {/* Heading */} -

Page not found

-

- The page you're looking for doesn't exist or has been moved. -

- - {/* Actions */} -
- - - -
- - {/* Helper text */} -

- Can't find what you need? Try checking our{' '} - - documentation - . -

- -
+
+ ); } diff --git a/src/pages/PrivacyPage.tsx b/src/pages/PrivacyPage.tsx new file mode 100644 index 0000000..b9d8c89 --- /dev/null +++ b/src/pages/PrivacyPage.tsx @@ -0,0 +1,36 @@ +import { InfoPageLayout } from '@/components/layout/InfoPageLayout'; + +export function PrivacyPage() { + return ( + +
+
+

Your data / Privacy

+

Privacy policy

+

Updated September 8, 2026

+

gitSdm saves your settings in your browser and sends data to its server, GitHub, and the services used by its AI features. This page explains what is stored and shared. If you use a self-hosted installation, its hosting and provider settings may differ.

+
+
+

What your browser saves

+

Your browser saves GitHub tokens and AI API keys in localStorage on this device. It also remembers your workspace preferences and the last repository you opened. These stay saved after you close the browser. Use Clear in Settings to remove saved keys, or clear this site's browser data to remove all local settings. Avoid saving keys on a shared device.

+
+
+

What gitSdm sends to other services

+

When you request a repository, your browser sends its owner and name to the gitSdm server, along with relevant saved credentials in request headers. The server uses your GitHub credentials to fetch repository details and files from GitHub. Saved credentials therefore also leave your device when used for these requests.

+

AI features may send your prompts, questions, and relevant repository content to the configured AI provider. That content can include private code accessed with your GitHub token. To build a semantic search index, gitSdm sends sections of code to the configured embedding provider. The Ask feature can send code found by the search to an AI provider to answer your question.

+

The provider may be Google Gemini, OpenAI, Anthropic, EdgeOne, or another compatible service. Each provider has its own rules for privacy and data retention. If you have not added your own API key, the installation may use a key supplied by its operator. Only submit code you have permission to share with those services.

+
+
+

Server storage and logs

+

The server keeps repository analysis and AI results in memory so it can reuse them. These caches have size limits and expire after a set time. Semantic search indexes are also kept in memory, but have no fixed expiry. Clearing your browser data does not delete server caches, search indexes, or records held by providers.

+

Application logs record request routes, timing, repository identifiers, and errors. Provider error messages may contain information you submitted. Hosting and network providers may also process connection details, such as your IP address. How long logs are kept depends on who operates the installation; there is no single retention period across deployments.

+
+
+

Other services and your choices

+

gitSdm loads fonts from Google Fonts and may show images hosted by GitHub or other services. Your browser shares connection details with those services when it loads these files. The app itself does not include a dedicated advertising or analytics tracker.

+

You can try the mock demos without adding any credentials. To remove saved keys, use Settings. To revoke a key or token, use the service that issued it. For privacy questions, see the contact information on the maintainer's profile, or contact your operator if you use a self-hosted installation. Do not post tokens, private code, or personal information in public issues.

+
+
+ + ); +} diff --git a/vercel.json b/vercel.json index 4dcb1d2..dd3a68f 100644 --- a/vercel.json +++ b/vercel.json @@ -15,7 +15,9 @@ ], "rewrites": [ { "source": "/api/(.*)", "destination": "/api/$1" }, - { "source": "/((?!api/).*)", "destination": "/index.html" } + { "source": "/privacy", "destination": "/privacy.html" }, + { "source": "/:owner((?!api$|assets$)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})", "destination": "/app.html" }, + { "source": "/:owner((?!api$|assets$)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})/search", "destination": "/app.html" } ], "functions": { "api/**/*.ts": { From 7780620ab6e58aaa16bbb128803a03cef728dba5 Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Tue, 8 Sep 2026 23:45:37 +0700 Subject: [PATCH 04/27] fix: make repository enrichment resilient and resolve review regressions Defer health and churn enrichment, preserve graph state, and resume credential-scoped churn batches across instances. Add request limits and fix prerender metadata, asset routing, and workspace mode recovery with regression coverage. --- api/repo/churn.ts | 2 + api/repo/health.ts | 2 + e2e/enrichment-regressions.e2e.ts | 52 ++ scripts/prerender.tsx | 37 +- server/api-router.ts | 32 +- server/router/repo-routes.ts | 37 +- server/router/schemas.ts | 78 ++- server/services/analyze-repo.test.ts | 169 +++++-- server/services/analyze-repo.ts | 69 +-- server/services/churn-service.test.ts | 233 +++++---- server/services/churn-service.ts | 207 ++++---- server/services/repo-enrichment.ts | 27 + server/utils/http.ts | 45 +- server/utils/request-limits.test.ts | 65 +++ server/utils/request-limits.ts | 66 +++ src/app/router.tsx | 45 +- src/components/layout/PageMetadata.tsx | 32 ++ src/components/viz/ChurnStatus.tsx | 19 + src/components/viz/DependencyHealthTab.tsx | 283 ++++++----- src/components/viz/OverviewTab.tsx | 236 ++++++--- .../graph/canvas/hooks/useForceCanvasState.ts | 75 ++- .../graph/force/reconcileGraph.test.ts | 24 + src/features/graph/force/reconcileGraph.ts | 19 + src/hooks/useAnalyzeRepo.ts | 30 +- src/hooks/useRepoChurn.ts | 29 ++ src/hooks/useRepoHealth.ts | 13 + src/lib/apiClient.ts | 112 ++--- src/lib/churn-progress.test.ts | 33 ++ src/lib/churn-progress.ts | 34 ++ src/lib/page-metadata.test.ts | 9 + src/lib/page-metadata.ts | 21 + src/lib/workspace-mode.test.ts | 18 + src/lib/workspace-mode.ts | 10 + src/main.tsx | 33 +- src/pages/VizPage.tsx | 382 +++++++------- src/stores/vizStore.ts | 468 +++++++++--------- src/types/churn.ts | 22 + src/types/domain.ts | 1 + vercel.json | 10 +- 39 files changed, 1980 insertions(+), 1099 deletions(-) create mode 100644 api/repo/churn.ts create mode 100644 api/repo/health.ts create mode 100644 e2e/enrichment-regressions.e2e.ts create mode 100644 server/services/repo-enrichment.ts create mode 100644 server/utils/request-limits.test.ts create mode 100644 server/utils/request-limits.ts create mode 100644 src/components/layout/PageMetadata.tsx create mode 100644 src/components/viz/ChurnStatus.tsx create mode 100644 src/features/graph/force/reconcileGraph.test.ts create mode 100644 src/features/graph/force/reconcileGraph.ts create mode 100644 src/hooks/useRepoChurn.ts create mode 100644 src/hooks/useRepoHealth.ts create mode 100644 src/lib/churn-progress.test.ts create mode 100644 src/lib/churn-progress.ts create mode 100644 src/lib/page-metadata.test.ts create mode 100644 src/lib/page-metadata.ts create mode 100644 src/lib/workspace-mode.test.ts create mode 100644 src/lib/workspace-mode.ts create mode 100644 src/types/churn.ts diff --git a/api/repo/churn.ts b/api/repo/churn.ts new file mode 100644 index 0000000..33cbea9 --- /dev/null +++ b/api/repo/churn.ts @@ -0,0 +1,2 @@ +import { createVercelHandler } from '../../server/vercel-handler'; +export default createVercelHandler('/api/repo/churn'); diff --git a/api/repo/health.ts b/api/repo/health.ts new file mode 100644 index 0000000..01189fe --- /dev/null +++ b/api/repo/health.ts @@ -0,0 +1,2 @@ +import { createVercelHandler } from '../../server/vercel-handler'; +export default createVercelHandler('/api/repo/health'); diff --git a/e2e/enrichment-regressions.e2e.ts b/e2e/enrichment-regressions.e2e.ts new file mode 100644 index 0000000..aa4ca07 --- /dev/null +++ b/e2e/enrichment-regressions.e2e.ts @@ -0,0 +1,52 @@ +import { test, expect } from '@playwright/test'; + +test('404 and privacy metadata are reset when navigating home', async ({ page }) => { + const response = await page.goto('/missing-page'); + expect(response?.status()).toBe(404); + await expect(page.locator('meta[name="robots"]')).toHaveAttribute('content', 'noindex'); + await page.getByRole('link', { name: 'Go to homepage', exact: true }).click(); + await expect(page).toHaveTitle(/Git Software Dependency Map/); + await expect(page.locator('meta[name="robots"]')).toHaveAttribute('content', 'index, follow'); + await page.goto('/privacy'); + await expect(page).toHaveTitle('Privacy policy — gitSdm'); + await page.getByRole('link', { name: 'Home', exact: true }).click(); + await expect(page.locator('link[rel="canonical"]')).toHaveAttribute('href', 'https://gsdm.site/'); +}); + +test('prerendered homepage stays visible while its route chunk loads', async ({ page }) => { + let release = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + await page.route('**/assets/HomePage-*.js', async (route) => { + await gate; + await route.continue(); + }); + try { + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await expect(page.locator('h1')).toBeVisible(); + await expect(page.getByRole('link', { name: 'Privacy', exact: true })).toBeVisible(); + } finally { + release(); + } +}); + +test('full workspace restores both panels after mobile resize', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto('/mock/todo-app'); + await page.getByRole('button', { name: 'Open menu', exact: true }).click(); + await page.getByRole('button', { name: 'Full Workspace', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Collapse file explorer', exact: true })).toBeVisible(); + await page.setViewportSize({ width: 1256, height: 912 }); + await expect(page.getByRole('button', { name: 'Collapse file explorer', exact: true })).toBeVisible(); + await expect( + page.getByRole('button', { + name: 'Collapse repository insights', + exact: true, + }), + ).toBeVisible(); +}); + +test('missing assets return an actual 404', async ({ request }) => { + expect((await request.get('/assets/missing.js')).status()).toBe(404); +}); diff --git a/scripts/prerender.tsx b/scripts/prerender.tsx index c464795..3495f12 100644 --- a/scripts/prerender.tsx +++ b/scripts/prerender.tsx @@ -4,23 +4,38 @@ import { Providers } from '../src/app/providers'; import { HomePage } from '../src/pages/HomePage'; import { PrivacyPage } from '../src/pages/PrivacyPage'; import { NotFoundPage } from '../src/pages/NotFoundPage'; +import { pageMetadata } from '../src/lib/page-metadata'; const shell = await Bun.file('dist/index.html').text(); if (!shell.includes('
')) throw new Error('Prerender root marker missing'); -// Repository routes use a blank shell, never the homepage's SEO or content. await Bun.write('dist/app.html', shell.replace(/]*>/, '')); for (const [route, file, page] of [ ['/', 'dist/index.html', ], ['/privacy', 'dist/privacy.html', ], + ['/not-found', 'dist/404.html', ], ] as const) { - const content = renderToStaticMarkup({page}); - const html = shell.replace('
', `
${content}
`); - await Bun.write(file, route === '/' ? html : html - .replaceAll('"https://gsdm.site/"', '"https://gsdm.site/privacy"') - .replace('gitSdmᵝ — Git Software Dependency Map', 'Privacy policy — gitSdm')); + const meta = pageMetadata(route); + const content = renderToStaticMarkup( + + {page} + , + ); + let html = shell + .replace('
', `
${content}
`) + .replace(/[^<]*<\/title>/, `<title>${meta.title}`) + .replace(/]*>/, meta.canonical ? `` : '') + .replace('', ``); + for (const [attribute, name, value] of [ + ['name', 'description', meta.description], + ['property', 'og:title', meta.title], + ['property', 'og:description', meta.description], + ['property', 'og:url', `https://gsdm.site${route}`], + ['name', 'twitter:title', meta.title], + ['name', 'twitter:description', meta.description], + ]) + html = html.replace( + new RegExp(`]*>`), + ``, + ); + await Bun.write(file, html); } -const notFound = renderToStaticMarkup(); -await Bun.write('dist/404.html', shell - .replace('
', `
${notFound}
`) - .replace('gitSdmᵝ — Git Software Dependency Map', 'Page not found — gitSdm') - .replace(/]*>/, '')); diff --git a/server/api-router.ts b/server/api-router.ts index 5d7df76..118cf20 100644 --- a/server/api-router.ts +++ b/server/api-router.ts @@ -11,10 +11,11 @@ import { handleAiRoutes } from './router/ai-routes'; import { handleRepoRoutes } from './router/repo-routes'; import { handleSearchRoutes } from './router/search-routes'; import { addSecurityHeaders } from './utils/http'; +import { createAdmissionLimit, limitRequestBody } from './utils/request-limits'; -export async function handleApiRequest( - req: Request, -): Promise { +const admitRequest = createAdmissionLimit(); + +export async function handleApiRequest(req: Request): Promise { const start = Date.now(); const method = req.method; const url = new URL(req.url); @@ -31,13 +32,34 @@ export async function handleApiRequest( const ctx: RequestContext = { octokit: getOctokit(gitHubToken), + gitHubToken, }; + const expensive = + pathname.startsWith('/api/ai/') || pathname.startsWith('/api/search/') || pathname.startsWith('/api/repo/'); + const release = expensive ? admitRequest() : undefined; + if (release === null) { + return addSecurityHeaders( + Response.json( + { + error: 'Server is busy. Please try again shortly.', + code: 'RATE_LIMIT_EXCEEDED', + status: 429, + retryable: true, + }, + { status: 429, headers: { 'Retry-After': '60' } }, + ), + ); + } try { + req = await limitRequestBody(req); // ── Global System Utilities ───────────────────────────────────── if (pathname === '/api/trending' && method === 'GET') { const repos: TrendingRepo[] = await fetchTrending(); - logApi('/api/trending', { durationMs: Date.now() - start, count: repos.length }); + logApi('/api/trending', { + durationMs: Date.now() - start, + count: repos.length, + }); return addSecurityHeaders(Response.json({ repos }, { status: 200 })); } @@ -62,5 +84,7 @@ export async function handleApiRequest( const payload = toErrorPayload(error); logError(pathname, error, { durationMs: Date.now() - start }); return addSecurityHeaders(Response.json(payload, { status: payload.status })); + } finally { + release?.(); } } diff --git a/server/router/repo-routes.ts b/server/router/repo-routes.ts index dca1eb3..cfe6522 100644 --- a/server/router/repo-routes.ts +++ b/server/router/repo-routes.ts @@ -7,6 +7,13 @@ import { getRepoFileContent } from '../services/get-file'; import { parseRepoParams } from '../github/parse-url'; import { logApi } from '../utils/logger'; import type { RepoAnalysis } from '../../src/types'; +import { enrichRepository } from '../services/repo-enrichment'; +import { z } from 'zod'; + +const continuationSchema = z.object({ + completed: z.array(z.string().max(1000)).max(200).optional(), + pending: z.array(z.string().max(1000)).max(200).optional(), +}); export async function handleRepoRoutes( pathname: string, @@ -15,6 +22,15 @@ export async function handleRepoRoutes( ctx: RequestContext, start: number, ): Promise { + if (pathname === '/api/repo/churn' || pathname === '/api/repo/health') { + const parsed = repoQuerySchema.safeParse(query); + if (!parsed.success) throw new AppError(400, 'Invalid repository', 'INVALID_PARAMS'); + const continuation = continuationSchema.safeParse(req.method === 'POST' ? await req.json().catch(() => null) : {}); + if (!continuation.success) throw new AppError(400, 'Invalid churn continuation', 'INVALID_PARAMS'); + return Response.json( + await enrichRepository(parsed.data, pathname.endsWith('/churn') ? 'churn' : 'health', ctx, continuation.data), + ); + } if (pathname === '/api/repo/analyze') { const body = await req.json().catch(() => ({})); const parsed = analyzeBodySchema.safeParse(body); @@ -71,11 +87,14 @@ export async function handleRepoRoutes( throw new AppError(400, 'Invalid owner/repo', 'INVALID_PARAMS'); } const analysis: RepoAnalysis = await analyzeRepository(q.data, ctx); - return Response.json({ - tree: analysis.tree, - truncated: analysis.treeTruncated, - importantFiles: analysis.importantFiles, - }, { status: 200 }); + return Response.json( + { + tree: analysis.tree, + truncated: analysis.treeTruncated, + importantFiles: analysis.importantFiles, + }, + { status: 200 }, + ); } if (pathname === '/api/repo/file') { @@ -83,7 +102,13 @@ export async function handleRepoRoutes( if (!q.success) { throw new AppError(400, 'Invalid owner, repo, or path', 'INVALID_PARAMS'); } - const file: { content: string } = await getRepoFileContent(q.data.owner, q.data.repo, q.data.path, q.data.branch, ctx); + const file: { content: string } = await getRepoFileContent( + q.data.owner, + q.data.repo, + q.data.path, + q.data.branch, + ctx, + ); return Response.json(file, { status: 200 }); } diff --git a/server/router/schemas.ts b/server/router/schemas.ts index 8d28f38..d114a3c 100644 --- a/server/router/schemas.ts +++ b/server/router/schemas.ts @@ -2,21 +2,39 @@ import { z } from 'zod'; export const analyzeBodySchema = z.object({ url: z.string().optional(), - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/).optional(), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/).optional(), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/) + .optional(), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/) + .optional(), branch: z.string().optional(), }); export const aiExplainSchema = z .object({ - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), sha: z.string().optional(), scope: z.enum(['repo', 'node', 'file']).default('repo'), nodeId: z.string().optional(), filePath: z.string().optional(), - fileSnippet: z.string().optional(), + fileSnippet: z.string().max(100000).optional(), context: z.string().max(10000).optional(), eli5: z.boolean().optional(), }) @@ -28,12 +46,20 @@ export const aiExplainSchema = z }, { message: 'nodeId is required when scope is "node", filePath is required when scope is "file"', - } + }, ); export const repoQuerySchema = z.object({ - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), goal: z.string().min(1).max(500).optional(), }); @@ -44,20 +70,44 @@ export const fileQuerySchema = repoQuerySchema.extend({ export const searchBodySchema = z.object({ query: z.string().min(3).max(500), - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), }); export const askBodySchema = z.object({ question: z.string().min(3).max(500), - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), }); export const indexBodySchema = z.object({ - owner: z.string().min(1).max(39).regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), - repo: z.string().min(1).max(100).regex(/^[a-zA-Z0-9-._]+$/), + owner: z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$/), + repo: z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9-._]+$/), branch: z.string().optional(), }); diff --git a/server/services/analyze-repo.test.ts b/server/services/analyze-repo.test.ts index d47a3c3..5f1f411 100644 --- a/server/services/analyze-repo.test.ts +++ b/server/services/analyze-repo.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, mock, beforeEach } from 'bun:test'; import { analyzeRepository } from './analyze-repo'; +import { enrichRepository } from './repo-enrichment'; import { clearAllCaches } from '../cache/lru'; import type { GraphBuildInput } from '../graph/graph-builder'; import type { RequestContext } from '../utils/context'; @@ -20,19 +21,33 @@ const mockRepoInfo = { }; const workspaceFileContents = { - 'package.json': JSON.stringify({ name: '@repo/root', workspaces: ['packages/*'] }), - 'packages/a/package.json': JSON.stringify({ name: '@repo/a', dependencies: { '@repo/b': 'workspace:*' } }), + 'package.json': JSON.stringify({ + name: '@repo/root', + workspaces: ['packages/*'], + }), + 'packages/a/package.json': JSON.stringify({ + name: '@repo/a', + dependencies: { '@repo/b': 'workspace:*' }, + }), 'packages/b/package.json': JSON.stringify({ name: '@repo/b' }), 'src/main.ts': 'export const value = 1', }; const plainFileContents = { - 'package.json': JSON.stringify({ name: 'plain-app', description: 'no workspaces here', dependencies: { react: '^19' } }), + 'package.json': JSON.stringify({ + name: 'plain-app', + description: 'no workspaces here', + dependencies: { react: '^19' }, + }), 'src/main.ts': 'export const value = 1', }; const excludedWorkspaceFileContents = { - 'package.json': JSON.stringify({ name: '@repo/root', packageManager: 'yarn@4.0.0', workspaces: ['packages/*', '!packages/excluded'] }), + 'package.json': JSON.stringify({ + name: '@repo/root', + packageManager: 'yarn@4.0.0', + workspaces: ['packages/*', '!packages/excluded'], + }), 'packages/a/package.json': JSON.stringify({ name: '@repo/a' }), 'packages/excluded/package.json': JSON.stringify({ name: '@repo/excluded' }), 'src/main.ts': 'export const value = 1', @@ -40,20 +55,22 @@ const excludedWorkspaceFileContents = { let activeFileContents: Record = workspaceFileContents; const buildGraphMock = mock((input: GraphBuildInput) => ({ - nodes: input.workspacePackages?.map((pkg) => ({ - id: `package:${pkg.rootPath}`, - type: 'package' as const, - position: { x: 0, y: 0 }, - data: { label: pkg.name ?? pkg.rootPath }, - })) ?? [], - edges: input.scopedDependencies - ?.filter((dep) => dep.name === '@repo/b') - .map((dep) => ({ - id: `e:${dep.manifestPath}->${dep.name}`, - source: 'package:packages/a', - target: 'package:packages/b', - type: 'depends_on' as const, + nodes: + input.workspacePackages?.map((pkg) => ({ + id: `package:${pkg.rootPath}`, + type: 'package' as const, + position: { x: 0, y: 0 }, + data: { label: pkg.name ?? pkg.rootPath }, })) ?? [], + edges: + input.scopedDependencies + ?.filter((dep) => dep.name === '@repo/b') + .map((dep) => ({ + id: `e:${dep.manifestPath}->${dep.name}`, + source: 'package:packages/a', + target: 'package:packages/b', + type: 'depends_on' as const, + })) ?? [], layout: 'dagre' as const, })); @@ -61,7 +78,13 @@ const originalFetch = globalThis.fetch; const stubOctokit = { repos: { listCommits: async () => ({ - data: [{ commit: { author: { name: 'test-user', date: '2026-01-01T00:00:00Z' } } }], + data: [ + { + commit: { + author: { name: 'test-user', date: '2026-01-01T00:00:00Z' }, + }, + }, + ], headers: {}, }), }, @@ -71,17 +94,23 @@ const fetchMock = mock(async (input: RequestInfo | URL) => { const packageName = decodeURIComponent(new URL(String(input)).pathname.slice(1)); if (packageName === '@repo/b') { - return new Response(JSON.stringify({ - 'dist-tags': { latest: 'workspace:*' }, - license: 'MIT', - }), { status: 200 }); + return new Response( + JSON.stringify({ + 'dist-tags': { latest: 'workspace:*' }, + license: 'MIT', + }), + { status: 200 }, + ); } if (packageName === 'react') { - return new Response(JSON.stringify({ - 'dist-tags': { latest: '19.1.0' }, - license: 'MIT', - }), { status: 200 }); + return new Response( + JSON.stringify({ + 'dist-tags': { latest: '19.1.0' }, + license: 'MIT', + }), + { status: 200 }, + ); } throw new TypeError(`unexpected fetch for ${packageName}`); @@ -92,7 +121,10 @@ describe('services/analyze-repo', () => { mock.module('../github/fetch-tree', () => ({ fetchRepoInfo: async () => mockRepoInfo, fetchFlatTree: async () => ({ - items: Object.keys(activeFileContents).map((path) => ({ path, type: 'file' })), + items: Object.keys(activeFileContents).map((path) => ({ + path, + type: 'file', + })), truncated: false, }), fetchContributors: async () => [], @@ -142,22 +174,41 @@ describe('services/analyze-repo', () => { expect(analysis.treeTruncated).toBe(false); expect(analysis.workspacePackages?.map((pkg) => pkg.rootPath)).toEqual(['', 'packages/a', 'packages/b']); expect(analysis.dependencies).toEqual([ - { name: '@repo/b', version: 'workspace:*', type: 'prod', ecosystem: 'npm' }, + { + name: '@repo/b', + version: 'workspace:*', + type: 'prod', + ecosystem: 'npm', + }, ]); expect(analysis.graph.nodes.some((node) => node.id === 'package:packages/a')).toBe(true); - expect(analysis.graph.edges.some((edge) => edge.source === 'package:packages/a' && edge.target === 'package:packages/b')).toBe(true); - expect(analysis.dependencyHealth).toEqual(expect.objectContaining({ - summary: expect.objectContaining({ total: 1, current: 1, unsupported: 0 }), - })); - expect(buildGraphMock).toHaveBeenCalledWith(expect.objectContaining({ - workspacePackages: expect.arrayContaining([ - expect.objectContaining({ rootPath: 'packages/a', name: '@repo/a' }), - expect.objectContaining({ rootPath: 'packages/b', name: '@repo/b' }), - ]), - scopedDependencies: expect.arrayContaining([ - expect.objectContaining({ manifestPath: 'packages/a/package.json', name: '@repo/b' }), - ]), - })); + expect( + analysis.graph.edges.some((edge) => edge.source === 'package:packages/a' && edge.target === 'package:packages/b'), + ).toBe(true); + expect(analysis.dependencyHealth).toBeUndefined(); + expect(await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx)).toEqual( + expect.objectContaining({ + summary: expect.objectContaining({ + total: 1, + current: 1, + unsupported: 0, + }), + }), + ); + expect(buildGraphMock).toHaveBeenCalledWith( + expect.objectContaining({ + workspacePackages: expect.arrayContaining([ + expect.objectContaining({ rootPath: 'packages/a', name: '@repo/a' }), + expect.objectContaining({ rootPath: 'packages/b', name: '@repo/b' }), + ]), + scopedDependencies: expect.arrayContaining([ + expect.objectContaining({ + manifestPath: 'packages/a/package.json', + name: '@repo/b', + }), + ]), + }), + ); expect(fetchMock).toHaveBeenCalledTimes(0); // Call again, should return cached @@ -171,13 +222,15 @@ describe('services/analyze-repo', () => { const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); - expect(analysis.dependencies).toEqual([ - { name: 'react', version: '^19', type: 'prod', ecosystem: 'npm' }, - ]); + expect(analysis.dependencies).toEqual([{ name: 'react', version: '^19', type: 'prod', ecosystem: 'npm' }]); + expect(fetchMock).toHaveBeenCalledTimes(0); + await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx); expect(fetchMock).toHaveBeenCalledTimes(1); expect(String(fetchMock.mock.calls[0]?.[0])).toContain('/react'); expect(analysis.workspacePackages).toEqual([]); - expect(analysis.graph.edges.some((edge) => edge.source.startsWith('package:') && edge.target.startsWith('package:'))).toBe(false); + expect( + analysis.graph.edges.some((edge) => edge.source.startsWith('package:') && edge.target.startsWith('package:')), + ).toBe(false); }); it('skips registry metadata for non-npm dependencies', async () => { @@ -189,12 +242,26 @@ describe('services/analyze-repo', () => { const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); expect(fetchMock).toHaveBeenCalledTimes(0); - expect(analysis.dependencyHealth).toEqual(expect.objectContaining({ - summary: expect.objectContaining({ total: 1, unsupported: 1, current: 0, outdated: 0, unknown: 0, errors: 0 }), - items: expect.arrayContaining([ - expect.objectContaining({ ecosystem: 'go', name: 'github.com/gin-gonic/gin', state: 'unsupported' }), - ]), - })); + expect(analysis.dependencyHealth).toBeUndefined(); + expect(await enrichRepository({ owner: 'test-owner', repo: 'test-repo' }, 'health', mockCtx)).toEqual( + expect.objectContaining({ + summary: expect.objectContaining({ + total: 1, + unsupported: 1, + current: 0, + outdated: 0, + unknown: 0, + errors: 0, + }), + items: expect.arrayContaining([ + expect.objectContaining({ + ecosystem: 'go', + name: 'github.com/gin-gonic/gin', + state: 'unsupported', + }), + ]), + }), + ); }); it('honors negated workspace globs and explicit yarn packageManager', async () => { diff --git a/server/services/analyze-repo.ts b/server/services/analyze-repo.ts index b6a8b05..55070b3 100644 --- a/server/services/analyze-repo.ts +++ b/server/services/analyze-repo.ts @@ -11,12 +11,13 @@ import { } from '../github/fetch-tree'; import { parseGitHubUrl } from '../github/parse-url'; import { buildGraph } from '../graph/graph-builder'; -import { analyzeDependencies, analyzeManifestDependencies, analyzeWorkspacePackages } from '../parser/dependency-analyzer'; +import { + analyzeDependencies, + analyzeManifestDependencies, + analyzeWorkspacePackages, +} from '../parser/dependency-analyzer'; import { annotateTree, findImportantFiles } from '../parser/file-classifier'; import { analyzeAllFileComplexity } from '../parser/complexity-analyzer'; -import { fetchRepoChurn } from './churn-service'; -import { buildDependencyHealthReport } from './dependency-health'; -import { fetchNpmDependencyMetadataBatch } from './npm-registry'; import type { RepoAnalysis } from '../../src/types'; import type { RequestContext } from '../utils/context'; @@ -24,10 +25,7 @@ export async function analyzeRepository( input: string | { owner: string; repo: string; branch?: string }, tokenOrCtx?: string | RequestContext, ): Promise { - const parsed = - typeof input === 'string' - ? parseGitHubUrl(input) - : input; + const parsed = typeof input === 'string' ? parseGitHubUrl(input) : input; if (!parsed) { throw new Error('Invalid GitHub repository URL'); @@ -51,18 +49,15 @@ export async function analyzeRepository( const tree = annotateTree(buildTreeFromPaths(items)); const manifestPaths = findManifestPaths(items); const importantFiles = findImportantFiles(items.map((i) => i.path)); - + // Extract up to 20 important code files to parse internal imports const sourceExtensions = ['.ts', '.tsx', '.js', '.jsx', '.py', '.go']; - const importantSourceFiles = importantFiles.filter((path) => - sourceExtensions.some((ext) => path.endsWith(ext)) - ); + const importantSourceFiles = importantFiles.filter((path) => sourceExtensions.some((ext) => path.endsWith(ext))); - // All source files from the tree — used for churn (API endpoint: listCommits) - // and to expand content fetching for complexity coverage + // Expand content fetching for complexity coverage. const allSourceFilePaths = items - .filter(i => i.type === 'blob' && sourceExtensions.some(ext => i.path.endsWith(ext))) - .map(i => i.path); + .filter((i) => i.type === 'blob' && sourceExtensions.some((ext) => i.path.endsWith(ext))) + .map((i) => i.path); const pathsToFetch = Array.from(new Set([...manifestPaths, ...importantSourceFiles])); @@ -77,34 +72,17 @@ export async function analyzeRepository( // Compute complexity from file content (synchronous, no API calls) const complexityData = analyzeAllFileComplexity(fileContents); - // For churn: use all source file paths from the tree (not just important ones) - // so the overlay covers all rendered graph file nodes. Cap at 200 to - // avoid excessive API calls on large repos. - const churnFetchPaths = allSourceFilePaths.length > 0 - ? allSourceFilePaths.slice(0, 200) - : pathsToFetch; - - // Fetch churn (async API calls) and npm metadata in parallel - const [npmDependencyMetadata, churnData] = await Promise.all([ - fetchNpmDependencyMetadataBatch( - dependencies.filter((dependency) => dependency.ecosystem === 'npm'), - ), - fetchRepoChurn(owner, repo, churnFetchPaths, branch, tokenOrCtx), - ]); - - const graph = await buildGraph({ - owner, - repo, - tree, - dependencies, - contributors, - fileContents, - workspacePackages, - scopedDependencies, - churnData, - complexityData, - }); - const dependencyHealth = buildDependencyHealthReport(dependencies, scopedDependencies, npmDependencyMetadata); + const graph = await buildGraph({ + owner, + repo, + tree, + dependencies, + contributors, + fileContents, + workspacePackages, + scopedDependencies, + complexityData, + }); const analysis: RepoAnalysis = { meta: { @@ -127,15 +105,16 @@ export async function analyzeRepository( treeTruncated: truncated, dependencies, workspacePackages, + scopedDependencies, graph, contributors, timeline, importantFiles, totalFiles, totalCommits, - dependencyHealth, }; cache.set(cacheKey, analysis); + cache.set(analyzeCacheKey(owner, repo, info.sha, info.sha), analysis); return analysis; } diff --git a/server/services/churn-service.test.ts b/server/services/churn-service.test.ts index d416d9a..c3c918b 100644 --- a/server/services/churn-service.test.ts +++ b/server/services/churn-service.test.ts @@ -1,107 +1,156 @@ -import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { Octokit } from '@octokit/rest'; import { fetchRepoChurn } from './churn-service'; import { clearAllCaches } from '../cache/lru'; -describe('churn-service', () => { - beforeEach(() => { - clearAllCaches(); +test('successful empty histories count as checked and batches resume without repeating files', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return Response.json([]); + }, + }, }); - - afterEach(() => { - mock.restore(); + const paths = Array.from({ length: 10 }, (_, i) => `${i}.ts`); + const first = await fetchRepoChurn('owner', 'repo', paths, 'sha', { + octokit, }); - - it('handles mock repos via fetchMockChurn', async () => { - const res = await fetchRepoChurn('mock', 'mock-repo', ['src/index.ts', 'src/app.ts']); - expect(res['src/index.ts']).toBeDefined(); - expect(res['src/index.ts'].commitCount).toBeGreaterThanOrEqual(1); - expect(res['src/app.ts'].churnScore).toBeGreaterThanOrEqual(0); + expect(first.checked).toBe(7); + expect(first.complete).toBe(false); + expect(first.files['0.ts'].commitCount).toBe(0); + expect(first.files['0.ts'].lastModified).toBeUndefined(); + const second = await fetchRepoChurn('owner', 'repo', paths, 'sha', { + octokit, }); + expect(second.checked).toBe(10); + expect(second.complete).toBe(true); + expect(calls).toBe(10); + await fetchRepoChurn('owner', 'repo', paths, 'sha', { octokit }); + expect(calls).toBe(10); +}); - it('fetches churn data from octokit, caches result, parses link header and author info', async () => { - const listCommitsMock = mock(async ({ path }: { path: string }) => { - if (path === 'err.ts') { - throw new Error('Github API rate limited'); - } - if (path === 'empty.ts') { - return { data: [], headers: {} }; - } - return { - data: [ - { - commit: { - author: { name: 'Alice', date: '2025-01-01T00:00:00Z' }, - }, - author: { login: 'Alice' }, - }, - { - commit: { - author: { name: 'Bob', date: '2024-12-01T00:00:00Z' }, - }, - author: { login: 'Bob' }, - }, - ], - headers: { - link: '; rel="last"', - }, - }; - }); - - const octokitMock = { - repos: { - listCommits: listCommitsMock, +test('partial results survive failure and cooldown prevents immediate retry', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return calls === 2 + ? Response.json({ message: 'limited' }, { status: 429, headers: { 'retry-after': '120' } }) + : Response.json([]); }, - }; - - const consoleErrorSpy = spyOn(console, 'error').mockImplementation(() => {}); - - const res = await fetchRepoChurn( - 'real-owner', - 'real-repo', - ['file1.ts', 'empty.ts', 'err.ts'], - 'main', - { octokit: octokitMock } as any, - ); - - expect(res['file1.ts']).toBeDefined(); - expect(res['file1.ts'].commitCount).toBe(300); // 3 * 100 from Link header - expect(res['file1.ts'].authorCount).toBe(2); - expect(res['file1.ts'].churnScore).toBe(1); // max commits + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const paths = ['a.ts', 'b.ts', 'c.ts']; + const result = await fetchRepoChurn('owner', 'repo', paths, 'sha', { + octokit, + }); + expect(result.checked).toBe(2); + expect(result.issue).toBe('rate-limit'); + expect(result.files['b.ts']).toBeUndefined(); + expect(result.retryAt).toBeGreaterThan(Date.now() + 110000); + await fetchRepoChurn('owner', 'repo', paths, 'sha', { octokit }); + expect(calls).toBe(3); + const other = await fetchRepoChurn('owner', 'repo', paths, 'other-sha', { + octokit, + }); + expect(other.complete).toBe(true); + expect(calls).toBe(6); +}); - expect(res['empty.ts']).toBeUndefined(); - expect(res['err.ts']).toBeUndefined(); +test('concurrent cold requests share in-flight work', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + await new Promise((resolve) => setTimeout(resolve, 5)); + return Response.json([]); + }, + }, + }); + const run = () => fetchRepoChurn('owner', 'concurrent', ['a.ts', 'b.ts'], 'sha', { octokit }); + const results = await Promise.all([run(), run()]); + expect(calls).toBe(2); + expect(results.map((result) => result.checked)).toEqual([2, 2]); +}); - // Verify cache hit - const cachedRes = await fetchRepoChurn( - 'real-owner', - 'real-repo', - ['file1.ts'], - 'main', - { octokit: octokitMock } as any, - ); - expect(cachedRes).toEqual(res); +test('credential-specific failures do not block another caller', async () => { + clearAllCaches(); + const failed = new Octokit({ + request: { + fetch: async () => Response.json({ message: 'Forbidden' }, { status: 403 }), + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const good = new Octokit({ + request: { fetch: async () => Response.json([]) }, + }); + expect( + ( + await fetchRepoChurn('owner', 'scope', ['a.ts'], 'sha', { + octokit: failed, + gitHubToken: 'test-limited', + }) + ).issue, + ).toBe('access'); + expect( + ( + await fetchRepoChurn('owner', 'scope', ['a.ts'], 'sha', { + octokit: good, + gitHubToken: 'test-valid', + }) + ).complete, + ).toBe(true); +}); - // Test 403/429 silent error handling (console.error still suppressed so the - // whole-batch-failure warning does not leak into the test output) - const rateLimitMock = { - repos: { - listCommits: mock(async () => { - const error: any = new Error('Rate limit'); - error.status = 403; - throw error; - }), +test('explicit continuation resumes after losing server cache', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return Response.json([]); }, - }; - const rateLimitRes = await fetchRepoChurn( - 'rate-limit-owner', - 'rate-limit-repo', - ['blocked.ts'], - 'main', - { octokit: rateLimitMock } as any, - ); - expect(rateLimitRes).toEqual({}); - - consoleErrorSpy.mockRestore(); + }, }); + const paths = Array.from({ length: 10 }, (_, i) => `${i}.ts`); + const first = await fetchRepoChurn('owner', 'cold', paths, 'sha', { + octokit, + }); + clearAllCaches(); + const second = await fetchRepoChurn('owner', 'cold', paths, 'sha', { octokit }, 90, { + completed: Object.keys(first.files), + pending: first.remaining, + }); + expect(second.complete).toBe(true); + expect(calls).toBe(10); }); +test('failed first batch does not prevent checking later files', async () => { + clearAllCaches(); + let calls = 0; + const octokit = new Octokit({ + request: { + fetch: async () => { + calls++; + return calls <= 7 ? Response.json({ message: 'Unavailable' }, { status: 500 }) : Response.json([]); + }, + }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + }); + const paths = Array.from({ length: 10 }, (_, i) => `${i}.ts`); + await fetchRepoChurn('owner', 'later', paths, 'sha', { octokit }); + const second = await fetchRepoChurn('owner', 'later', paths, 'sha', { + octokit, + }); + expect(second.checked).toBe(3); + expect(second.files['7.ts']).toBeDefined(); +}); diff --git a/server/services/churn-service.ts b/server/services/churn-service.ts index ff20db4..49f15ea 100644 --- a/server/services/churn-service.ts +++ b/server/services/churn-service.ts @@ -1,122 +1,131 @@ +import { createHash } from 'node:crypto'; import { cache, churnCacheKey } from '../cache/lru'; import { getOctokit } from '../github/client'; import { isMockRepo, fetchMockChurn } from '../github/mock-data'; import type { RequestContext } from '../utils/context'; +import type { ChurnResponse, ChurnContinuation, FileChurnData } from '../../src/types/churn'; +export type { FileChurnData } from '../../src/types/churn'; -export interface FileChurnData { - commitCount: number; - authorCount: number; - lastModified: string; - churnScore: number; -} +type Failure = ChurnResponse['failures'][string]; +type Stored = { + files: Record; + failures: Record; +}; +const inFlight = new Map>(); -/** - * Fetch per-file churn data: commit count, distinct authors, last-modified date. - * For each file path, calls listCommits with the path filter (last N days). - * Processes paths in concurrent batches. - * - * Note on commitCount precision: - * When the response is paginated (Link header contains a rel="last" page), - * commitCount is approximated as `lastPage * 100`. This overestimates when - * the last page has fewer than 100 commits. For example, 101 commits → 200, - * 201 commits → 300. The error is non-uniform across files and compounds - * in the churnScore normalization step. This is an intentional trade-off - * to avoid O(repos) extra API calls. If precise counts are needed, callers - * should paginate fully. - */ +/** Credential-scoped cached results plus explicit continuation work across server instances. */ export async function fetchRepoChurn( owner: string, repo: string, - paths: string[], + inputPaths: string[], branch?: string, tokenOrCtx?: string | RequestContext, days = 90, -): Promise> { - if (isMockRepo(owner)) { - return fetchMockChurn(paths); - } - - const cacheKey = churnCacheKey(owner, repo, branch, days); - const cached = cache.get>(cacheKey); - if (cached) return cached; - - const octokit = (tokenOrCtx && typeof tokenOrCtx === 'object' && 'octokit' in tokenOrCtx) - ? tokenOrCtx.octokit - : getOctokit(tokenOrCtx as string | undefined); - + continuation: ChurnContinuation = {}, +): Promise { + const paths = [...new Set(inputPaths)].slice(0, 200); + if (isMockRepo(owner)) + return { + files: await fetchMockChurn(paths), + checked: paths.length, + total: paths.length, + complete: true, + remaining: [], + failures: {}, + }; + const token = typeof tokenOrCtx === 'string' ? tokenOrCtx : tokenOrCtx?.gitHubToken; + const scope = createHash('sha256') + .update(token || process.env.GITHUB_TOKEN || 'anonymous') + .digest('hex'); const since = new Date(); - since.setDate(since.getDate() - days); - - const result: Record = {}; - const CONCURRENCY = 7; - - for (let i = 0; i < paths.length; i += CONCURRENCY) { - const chunk = paths.slice(i, i + CONCURRENCY); - const entries = await Promise.all( - chunk.map(async (path) => { + since.setUTCHours(0, 0, 0, 0); + since.setUTCDate(since.getUTCDate() - days); + const key = `${churnCacheKey(owner, repo, branch, days)}:incremental:${since.toISOString()}:${scope}`; + const state = cache.get(key) ?? { files: {}, failures: {} }; + cache.set(key, state); + const completed = new Set((continuation.completed ?? []).filter((path) => paths.includes(path))); + const order = [...new Set([...(continuation.pending ?? []).filter((path) => paths.includes(path)), ...paths])]; + const pending = order + .filter( + (path) => + (!continuation.pending || continuation.pending.includes(path)) && + !completed.has(path) && + !state.files[path] && + !(state.failures[path]?.retryAt > Date.now()), + ) + .slice(0, 7); + const octokit = tokenOrCtx && typeof tokenOrCtx === 'object' ? tokenOrCtx.octokit : getOctokit(tokenOrCtx); + await Promise.all( + pending.map((path) => { + const flightKey = `${key}:${path}`; + const existing = inFlight.get(flightKey); + if (existing) return existing; + const task = (async () => { try { const { data, headers } = await octokit.repos.listCommits({ owner, repo, - sha: branch || undefined, + sha: branch, path, since: since.toISOString(), per_page: 100, + request: { signal: AbortSignal.timeout(5000) }, }); - - if (!data || data.length === 0) return null; - - // Count from Link header if paginated, else data length - const link = headers?.link ?? ''; - const lastPage = /[?&]page=(\d+)>;\s*rel="last"/.exec(link)?.[1]; - const commitCount = lastPage ? Number(lastPage) * 100 : data.length; - - // Distinct authors - const authors = new Set(); - for (const c of data) { - if (c.commit?.author?.name) authors.add(c.commit.author.name); - if (c.author?.login) authors.add(c.author.login); - } - - const lastCommit = data[0]; - const lastModified = - lastCommit?.commit?.author?.date ?? - lastCommit?.commit?.committer?.date ?? - new Date().toISOString(); - - return { path, commitCount, authorCount: authors.size, lastModified }; - } catch (err) { - const status = err && typeof err === 'object' && 'status' in err ? (err as { status: number }).status : undefined; - if (status !== 403 && status !== 429) { - console.error(`[churn-service] Failed to fetch churn for ${path}:`, err instanceof Error ? err.message : String(err)); - } - return null; + const lastPage = /[?&]page=(\d+)>;\s*rel="last"/.exec(headers.link ?? '')?.[1]; + const authors = new Set(data.map((c) => c.author?.login ?? c.commit.author?.name).filter(Boolean)); + state.files[path] = { + commitCount: lastPage ? Number(lastPage) * 100 : data.length, + authorCount: authors.size, + lastModified: data[0]?.commit.author?.date ?? data[0]?.commit.committer?.date ?? undefined, + churnScore: 0, + }; + delete state.failures[path]; + } catch (error) { + const err = error as { + status?: number; + response?: { headers?: Record }; + }; + const headers = err.response?.headers ?? {}; + const limited = + err.status === 429 || + (err.status === 403 && (headers['x-ratelimit-remaining'] === '0' || !!headers['retry-after'])); + const issue = limited + ? 'rate-limit' + : err.status === 401 || err.status === 403 || err.status === 404 + ? 'access' + : 'timeout-or-network'; + const seconds = Number(headers['retry-after']); + const reset = Number(headers['x-ratelimit-reset']) * 1000; + const retryAt = limited + ? Math.max( + Date.now() + 60_000, + Number.isFinite(seconds) ? Date.now() + seconds * 1000 : 0, + Number.isFinite(reset) ? reset : 0, + ) + : Date.now() + 10_000; + state.failures[path] = { issue, retryAt }; } - }), - ); - - for (const entry of entries) { - if (!entry) continue; - result[entry.path] = { - commitCount: entry.commitCount, - authorCount: entry.authorCount, - lastModified: entry.lastModified, - churnScore: 0, // normalized later - }; - } - - // Warn if an entire batch failed — possible auth/rate-limit issue - if (chunk.length > 0 && entries.every((e) => e === null)) { - console.error(`[churn-service] All ${chunk.length} paths in batch failed for ${owner}/${repo} — possible auth/rate-limit issue`); - } - } - - // Normalize churnScore: find max commit count - const maxCommits = Math.max(1, ...Object.values(result).map((d) => d.commitCount)); - for (const data of Object.values(result)) { - data.churnScore = +(data.commitCount / maxCommits).toFixed(4); - } - - cache.set(cacheKey, result); - return result; + })().finally(() => inFlight.delete(flightKey)); + inFlight.set(flightKey, task); + return task; + }), + ); + const entries = paths.flatMap((path) => (state.files[path] ? [[path, state.files[path]] as const] : [])); + entries.forEach(([path]) => completed.add(path)); + const remaining = order.filter((path) => !completed.has(path)); + const failures = Object.fromEntries( + remaining.flatMap((path) => (state.failures[path] ? [[path, state.failures[path]]] : [])), + ); + const stopped = Object.values(failures).sort((a, b) => b.retryAt - a.retryAt)[0]; + const max = Math.max(1, ...entries.map(([, file]) => file.commitCount)); + return { + files: Object.fromEntries(entries.map(([path, file]) => [path, { ...file, churnScore: file.commitCount / max }])), + checked: completed.size, + total: paths.length, + complete: remaining.length === 0, + remaining, + failures, + issue: stopped?.issue, + retryAt: stopped?.retryAt, + }; } diff --git a/server/services/repo-enrichment.ts b/server/services/repo-enrichment.ts new file mode 100644 index 0000000..9e80be7 --- /dev/null +++ b/server/services/repo-enrichment.ts @@ -0,0 +1,27 @@ +import { analyzeRepository } from './analyze-repo'; +import { fetchRepoChurn } from './churn-service'; +import { fetchNpmDependencyMetadataBatch } from './npm-registry'; +import { buildDependencyHealthReport } from './dependency-health'; +import type { RequestContext } from '../utils/context'; +import type { ChurnContinuation } from '../../src/types/churn'; + +export async function enrichRepository( + input: { owner: string; repo: string; branch?: string }, + kind: 'churn' | 'health', + ctx: RequestContext, + continuation: ChurnContinuation = {}, +) { + const analysis = await analyzeRepository(input, ctx); + if (kind === 'churn') { + const paths = analysis.graph.nodes + .flatMap((node) => + node.type === 'file' && node.data.path && /\.(tsx?|jsx?|py|go)$/.test(node.data.path) ? [node.data.path] : [], + ) + .slice(0, 200); + return fetchRepoChurn(input.owner, input.repo, paths, analysis.meta.sha, ctx, 90, continuation); + } + const metadata = await fetchNpmDependencyMetadataBatch( + analysis.dependencies.filter((dep) => dep.ecosystem === 'npm'), + ); + return buildDependencyHealthReport(analysis.dependencies, analysis.scopedDependencies, metadata); +} diff --git a/server/utils/http.ts b/server/utils/http.ts index 78c9e5a..26e6146 100644 --- a/server/utils/http.ts +++ b/server/utils/http.ts @@ -1,15 +1,14 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import type { TLSSocket } from 'node:tls'; import { handleApiRequest } from '../api-router'; +import { checkBodySize } from './request-limits'; +import { toErrorPayload } from './errors'; /** * Adapter to handle Node-compatible requests (Vite dev server, Vercel) * and route them through the Web-standard handleApiRequest. */ -export async function handleNodeRequest( - nodeReq: IncomingMessage, - nodeRes: ServerResponse, -): Promise { +export async function handleNodeRequest(nodeReq: IncomingMessage, nodeRes: ServerResponse): Promise { const headers = new Headers(); for (const [key, value] of Object.entries(nodeReq.headers)) { if (value) { @@ -26,16 +25,31 @@ export async function handleNodeRequest( const url = `${protocol}://${host}${nodeReq.url}`; let body: BodyInit | undefined; - const requestWithBody = nodeReq as IncomingMessage & { body?: unknown }; - if (requestWithBody.body !== undefined) { - const parsedBody = requestWithBody.body; - body = typeof parsedBody === 'string' ? parsedBody : JSON.stringify(parsedBody); - } else if (nodeReq.method !== 'GET' && nodeReq.method !== 'HEAD') { - const chunks: Buffer[] = []; - for await (const chunk of nodeReq) { - chunks.push(chunk as Buffer); + try { + checkBodySize(Number(nodeReq.headers['content-length'] ?? 0)); + const requestWithBody = nodeReq as IncomingMessage & { body?: unknown }; + if (requestWithBody.body !== undefined) { + const parsedBody = requestWithBody.body; + body = typeof parsedBody === 'string' ? parsedBody : JSON.stringify(parsedBody); + checkBodySize(Buffer.byteLength(body)); + } else if (nodeReq.method !== 'GET' && nodeReq.method !== 'HEAD') { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of nodeReq.iterator({ destroyOnReturn: false })) { + size += Buffer.byteLength(chunk); + checkBodySize(size); + chunks.push(chunk as Buffer); + } + body = Buffer.concat(chunks); } - body = Buffer.concat(chunks); + } catch (error) { + nodeReq.resume(); + const payload = toErrorPayload(error); + const response = addSecurityHeaders(Response.json(payload, { status: payload.status })); + nodeRes.statusCode = response.status; + response.headers.forEach((value, key) => nodeRes.setHeader(key, value)); + nodeRes.end(await response.text()); + return true; } const webReq = new Request(url, { @@ -64,6 +78,9 @@ export function addSecurityHeaders(response: Response): Response { response.headers.set('X-Frame-Options', 'DENY'); // X-XSS-Protection is deprecated; use CSP instead response.headers.set('Strict-Transport-Security', 'max-age=31536000; includeSubDomains'); - response.headers.set('Content-Security-Policy', "default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-1HwijBPaKaOtZGvlT2Hw0JpmgDixxpRo6XKpkDG9xvs=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;"); + response.headers.set( + 'Content-Security-Policy', + "default-src 'self'; base-uri 'self'; script-src 'self' 'sha256-1HwijBPaKaOtZGvlT2Hw0JpmgDixxpRo6XKpkDG9xvs=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https:; connect-src 'self' https: wss:;", + ); return response; } diff --git a/server/utils/request-limits.test.ts b/server/utils/request-limits.test.ts new file mode 100644 index 0000000..616d7e1 --- /dev/null +++ b/server/utils/request-limits.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from 'bun:test'; +import { createAdmissionLimit, limitRequestBody, MAX_BODY_BYTES } from './request-limits'; + +test('admission rejects overload, releases once, and resets the time window', () => { + let time = 0; + const admit = createAdmissionLimit(1, 2, () => time); + const release = admit(); + expect(release).not.toBeNull(); + expect(admit()).toBeNull(); + release?.(); + release?.(); + const second = admit(); + expect(second).not.toBeNull(); + expect(admit()).toBeNull(); + second?.(); + expect(admit()).toBeNull(); + time = 60_000; + expect(admit()).not.toBeNull(); +}); + +test('body limit preserves JSON at the byte boundary', async () => { + const body = 'a'.repeat(MAX_BODY_BYTES); + const request = await limitRequestBody(new Request('http://localhost/api', { method: 'POST', body })); + expect(await request.text()).toBe(body); +}); + +test('body limit rejects streamed excess even with a false content length', async () => { + let cancelled = false; + const stream = new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(MAX_BODY_BYTES)); + }, + cancel() { + cancelled = true; + }, + }); + const request = new Request('http://localhost/api', { + method: 'POST', + body: stream, + headers: { 'content-length': '1' }, + }); + await expect(limitRequestBody(request)).rejects.toMatchObject({ + status: 413, + }); + expect(cancelled).toBe(true); +}); + +test('body limit counts UTF-8 bytes and rejects oversized declared lengths', async () => { + await expect( + limitRequestBody( + new Request('http://localhost/api', { + method: 'POST', + body: 'é'.repeat(MAX_BODY_BYTES), + }), + ), + ).rejects.toMatchObject({ status: 413 }); + await expect( + limitRequestBody( + new Request('http://localhost/api', { + method: 'POST', + headers: { 'content-length': String(MAX_BODY_BYTES + 1) }, + }), + ), + ).rejects.toMatchObject({ status: 413 }); +}); diff --git a/server/utils/request-limits.ts b/server/utils/request-limits.ts new file mode 100644 index 0000000..f3cc5bc --- /dev/null +++ b/server/utils/request-limits.ts @@ -0,0 +1,66 @@ +import { AppError } from './errors'; + +export const MAX_BODY_BYTES = 256 * 1024; + +export function checkBodySize(bytes: number): void { + if (bytes > MAX_BODY_BYTES) throw new AppError(413, 'Request body is too large', 'PAYLOAD_TOO_LARGE'); +} + +/** Bound the stream before JSON parsing, including requests without Content-Length. */ +export async function limitRequestBody(req: Request): Promise { + checkBodySize(Number(req.headers.get('content-length') ?? 0)); + if (!req.body) return req; + const reader = req.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + checkBodySize(size); + chunks.push(value); + } + } catch (error) { + await reader.cancel().catch(() => undefined); + throw error; + } finally { + reader.releaseLock(); + } + const body = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return new Request(req.url, { + method: req.method, + headers: req.headers, + body, + signal: req.signal, + }); +} + +/** Process-wide ceiling; no growing client map and no waiting queue. */ +export function createAdmissionLimit(maxActive = 8, maxPerMinute = 120, now = Date.now) { + let active = 0; + let started = now(); + let count = 0; + return () => { + const time = now(); + if (time - started >= 60_000) { + started = time; + count = 0; + } + if (active >= maxActive || count >= maxPerMinute) return null; + active++; + count++; + let released = false; + return () => { + if (!released) { + active--; + released = true; + } + }; + }; +} diff --git a/src/app/router.tsx b/src/app/router.tsx index 2c49027..aea3056 100644 --- a/src/app/router.tsx +++ b/src/app/router.tsx @@ -1,35 +1,56 @@ -import { lazy, Suspense } from 'react'; +import { lazy, Suspense, type ComponentType } from 'react'; +import { PageMetadata } from '@/components/layout/PageMetadata'; import { BrowserRouter, Route, Routes, useLocation } from 'react-router-dom'; import { ErrorBoundary } from '@/components/ErrorBoundary'; const HomePage = lazy(() => import('@/pages/HomePage').then((module) => ({ default: module.HomePage }))); const VizPage = lazy(() => import('@/pages/VizPage').then((module) => ({ default: module.VizPage }))); -const SearchPage = lazy(() => import('@/pages/SearchPage').then((module) => ({ default: module.SearchPage }))); -const NotFoundPage = lazy(() => import('@/pages/NotFoundPage').then((module) => ({ default: module.NotFoundPage }))); -const PrivacyPage = lazy(() => import('@/pages/PrivacyPage').then((module) => ({ default: module.PrivacyPage }))); +const SearchPage = lazy(() => + import('@/pages/SearchPage').then((module) => ({ + default: module.SearchPage, + })), +); +const NotFoundPage = lazy(() => + import('@/pages/NotFoundPage').then((module) => ({ + default: module.NotFoundPage, + })), +); +const PrivacyPage = lazy(() => + import('@/pages/PrivacyPage').then((module) => ({ + default: module.PrivacyPage, + })), +); -function RouterContent() { +function RouterContent({ initialPage, initialPath }: { initialPage: ComponentType; initialPath: string }) { const location = useLocation(); + const InitialPage = initialPage; return ( + }> - } /> - } /> - } /> - } /> - } /> + : } /> + : } /> + : } + /> + : } + /> + : } /> ); } -export function AppRouter() { +export function AppRouter(props: { initialPage: ComponentType; initialPath: string }) { return ( - + ); } diff --git a/src/components/layout/PageMetadata.tsx b/src/components/layout/PageMetadata.tsx new file mode 100644 index 0000000..09629d4 --- /dev/null +++ b/src/components/layout/PageMetadata.tsx @@ -0,0 +1,32 @@ +import { useEffect } from 'react'; +import { pageMetadata } from '@/lib/page-metadata'; + +export function PageMetadata({ path }: { path: string }) { + useEffect(() => { + const meta = pageMetadata(path); + document.title = meta.title; + for (const [attribute, name, value] of [ + ['name', 'description', meta.description], + ['name', 'robots', meta.robots], + ['property', 'og:title', meta.title], + ['property', 'og:description', meta.description], + ['property', 'og:url', new URL(path, 'https://gsdm.site').href], + ['name', 'twitter:title', meta.title], + ['name', 'twitter:description', meta.description], + ]) { + const element = + document.head.querySelector(`meta[${attribute}="${name}"]`) ?? + document.head.appendChild(document.createElement('meta')); + element.setAttribute(attribute, name); + element.setAttribute('content', value); + } + const canonical = document.head.querySelector('link[rel="canonical"]'); + if (!meta.canonical) canonical?.remove(); + else { + const link = canonical ?? document.head.appendChild(document.createElement('link')); + link.setAttribute('rel', 'canonical'); + link.setAttribute('href', meta.canonical); + } + }, [path]); + return null; +} diff --git a/src/components/viz/ChurnStatus.tsx b/src/components/viz/ChurnStatus.tsx new file mode 100644 index 0000000..e17960d --- /dev/null +++ b/src/components/viz/ChurnStatus.tsx @@ -0,0 +1,19 @@ +import { useRepoChurn } from '@/hooks/useRepoChurn'; + +export function ChurnStatus({ owner, repo, sha }: { owner: string; repo: string; sha: string }) { + const query = useRepoChurn(owner, repo, sha, true); + const data = query.data; + return ( +
+ {data ? `Churn: ${data.checked} of ${data.total} sampled files checked.` : 'Loading churn history…'} + {data?.issue && + ` ${data.issue === 'access' ? 'Check GitHub access.' : data.issue === 'rate-limit' ? 'Waiting for GitHub rate limit reset.' : 'Some files timed out; remaining files continue loading.'}`} + {query.isError && ' Churn request failed.'} + {(query.isError || data?.issue) && ( + + )} +
+ ); +} diff --git a/src/components/viz/DependencyHealthTab.tsx b/src/components/viz/DependencyHealthTab.tsx index bb9f5bf..2102af1 100644 --- a/src/components/viz/DependencyHealthTab.tsx +++ b/src/components/viz/DependencyHealthTab.tsx @@ -1,8 +1,20 @@ import { TooltipHint } from '@/components/ui/tooltip'; import { useState } from 'react'; -import { AlertCircle, CheckCircle2, Clock, HelpCircle, PackageX, ChevronDown, ChevronRight, RefreshCw, XCircle, Info } from 'lucide-react'; +import { + AlertCircle, + CheckCircle2, + Clock, + HelpCircle, + PackageX, + ChevronDown, + ChevronRight, + RefreshCw, + XCircle, + Info, +} from 'lucide-react'; import { cn } from '@/lib/utils'; import type { RepoAnalysis, DependencyHealthItem } from '@/types'; +import { useRepoHealth } from '@/hooks/useRepoHealth'; interface DependencyHealthTabProps { analysis: RepoAnalysis; @@ -11,7 +23,8 @@ interface DependencyHealthTabProps { type FilterType = 'all' | 'current' | 'outdated' | 'unknown' | 'unsupported'; export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { - const health = analysis.dependencyHealth; + const query = useRepoHealth(analysis.meta.owner, analysis.meta.repo, analysis.meta.sha, true); + const health = query.data; const [filter, setFilter] = useState('all'); const [expandedRows, setExpandedRows] = useState>(new Set()); @@ -19,10 +32,17 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { return (
-
- -
-

Health data not found or still analyzing...

+
+ +
+

+ {query.isError ? 'Dependency health could not be loaded.' : 'Loading dependency health…'} +

+ {query.isError && ( + + )}
); @@ -33,17 +53,17 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { if (items.length === 0) { return (
-
-
- -
-

No dependencies found in supported manifests.

+
+
+ +
+

No dependencies found in supported manifests.

); } - const filteredItems = items.filter(item => { + const filteredItems = items.filter((item) => { if (filter === 'all') return true; if (filter === 'current') return item.state === 'current'; if (filter === 'outdated') return item.state === 'outdated'; @@ -56,7 +76,7 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) { }); const toggleRow = (id: string) => { - setExpandedRows(prev => { + setExpandedRows((prev) => { const next = new Set(prev); if (next.has(id)) next.delete(id); else next.add(id); @@ -70,8 +90,8 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('all')} > @@ -80,8 +100,8 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('current')} > @@ -90,8 +110,8 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('outdated')} > @@ -103,22 +123,26 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {
setFilter('unknown')} > -
{summary.unknown + summary.errors}
+
+ {summary.unknown + summary.errors} +
Unknown
setFilter('unsupported')} > -
{summary.unsupported}
+
+ {summary.unsupported} +
Unsupported
@@ -127,117 +151,130 @@ export function DependencyHealthTab({ analysis }: DependencyHealthTabProps) {

- Freshness checks currently support npm only. Other ecosystems are inventoried but versions remain unchecked. + Freshness checks currently support npm only. Other ecosystems are inventoried but versions + remain unchecked.

-
-

- Dependencies -

- - {filteredItems.length} {filter !== 'all' && `(${filter})`} - -
- -
- {filteredItems.map(item => { - const id = `${item.ecosystem}-${item.name}-${item.type}`; - const isExpanded = expandedRows.has(id); - return ( -
-
toggleRow(id)} - > -
- {isExpanded ? : } - -
-
{item.name}
-
- {item.ecosystem} - - {item.currentVersion || 'unknown'} - {item.latestVersion && item.latestVersion !== item.currentVersion && ( - <> - - → {item.latestVersion} - - )} -
-
-
-
- - {item.type} - - {item.manifestPaths.length > 1 && ( - - x{item.manifestPaths.length} - - )} -
-
- - {isExpanded && ( -
-
-
Manifests
-
- {item.manifestPaths.map((p, i) => ( -
{p}
 - ))} -
-
- {item.packageNames.length > 0 && item.packageNames.some(n => n !== item.name) && ( -
-
Imported By
-
- {item.packageNames.filter(n => n !== item.name).map((name, i) => ( -
{name}
 +
+

Dependencies

+ + {filteredItems.length} {filter !== 'all' && `(${filter})`} + +
+ +
+ {filteredItems.map((item) => { + const id = `${item.ecosystem}-${item.name}-${item.type}`; + const isExpanded = expandedRows.has(id); + return ( +
+
toggleRow(id)} + > +
+ {isExpanded ? ( + + ) : ( + + )} + +
+
{item.name}
+
+ {item.ecosystem} + + {item.currentVersion || 'unknown'} + {item.latestVersion && item.latestVersion !== item.currentVersion && ( + <> + + → {item.latestVersion} + + )} +
+
+
+
+ + {item.type} + + {item.manifestPaths.length > 1 && ( + + x{item.manifestPaths.length} + + )} +
+
+ + {isExpanded && ( +
+
+
Manifests
+
+ {item.manifestPaths.map((p, i) => ( + +
{p}
+ + ))} +
+
+ {item.packageNames.length > 0 && item.packageNames.some((n) => n !== item.name) && ( +
+
Imported By
+
+ {item.packageNames + .filter((n) => n !== item.name) + .map((name, i) => ( + +
{name}
+ ))} -
- )} - {item.license && ( -
-
License
-
{item.license}
-
- )} - {item.checkedAt && ( -
-
Checked
-
{new Date(item.checkedAt).toLocaleString()}
-
- )} - {item.error && ( -
-
Error
-
{item.error}
-
- )} -
- )} -
- ); - })} -
+
+ )} + {item.license && ( +
+
License
+
{item.license}
+
+ )} + {item.checkedAt && ( +
+
Checked
+
{new Date(item.checkedAt).toLocaleString()}
+
+ )} + {item.error && ( +
+
Error
+
{item.error}
+
+ )} +
+ )} +
+ ); + })} +
-
); } function StateIcon({ state }: { state: DependencyHealthItem['state'] }) { switch (state) { - case 'current': return ; - case 'outdated': return ; - case 'error': return ; - case 'unknown': return ; - default: return ; + case 'current': + return ; + case 'outdated': + return ; + case 'error': + return ; + case 'unknown': + return ; + default: + return ; } } diff --git a/src/components/viz/OverviewTab.tsx b/src/components/viz/OverviewTab.tsx index 9c8222d..7545805 100644 --- a/src/components/viz/OverviewTab.tsx +++ b/src/components/viz/OverviewTab.tsx @@ -2,9 +2,20 @@ import { TooltipHint } from '@/components/ui/tooltip'; import { useCallback, useMemo } from 'react'; import { useVizStore } from '@/stores/vizStore'; import type { RepoAnalysis } from '@/types'; +import { useRepoChurn } from '@/hooks/useRepoChurn'; import { - Plus, Activity, AlertTriangle, GitBranch, - FileCode, Folder, Package, Users, Code2, ShieldAlert, Info, Flame + Plus, + Activity, + AlertTriangle, + GitBranch, + FileCode, + Folder, + Package, + Users, + Code2, + ShieldAlert, + Info, + Flame, } from 'lucide-react'; const GRAPH_FILE_NODE_CAP = 1200; @@ -21,6 +32,8 @@ interface OverviewTabProps { } export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTabProps) { + const churn = useRepoChurn(analysis.meta.owner, analysis.meta.repo, analysis.meta.sha, true); + const hasChurn = analysis.graph.nodes.some((node) => node.data.churnScore != null); // Direct graph centering helper — works for both file and folder nodes const focusOnNode = useCallback((nodeId: string, filePath?: string | null) => { useVizStore.getState().setSelectedNodeId(nodeId); @@ -40,40 +53,50 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab nodeById, maintenanceHotspots, } = useMemo(() => { - const fileCount = analysis.graph.nodes.filter(n => n.type === 'file').length; - const folderCount = analysis.graph.nodes.filter(n => n.type === 'folder').length; + const fileCount = analysis.graph.nodes.filter((n) => n.type === 'file').length; + const folderCount = analysis.graph.nodes.filter((n) => n.type === 'folder').length; const depCount = analysis.dependencies.length; const contributorCount = analysis.contributors.length; // Calculate High Coupling const degrees: Record = {}; - analysis.graph.edges.forEach(e => { + analysis.graph.edges.forEach((e) => { degrees[e.source] = (degrees[e.source] || 0) + 1; degrees[e.target] = (degrees[e.target] || 0) + 1; }); const highCoupling = analysis.graph.nodes - .filter(n => n.type === 'file' || n.type === 'folder') + .filter((n) => n.type === 'file' || n.type === 'folder') .sort((a, b) => (degrees[b.id] || 0) - (degrees[a.id] || 0)) .slice(0, 5); - const entryPoints = analysis.graph.nodes - .filter(n => n.data.fileClass === 'entry') - .slice(0, 5); + const entryPoints = analysis.graph.nodes.filter((n) => n.data.fileClass === 'entry').slice(0, 5); - const nodeById = new Map(analysis.graph.nodes.map(n => [n.id, n])); + const nodeById = new Map(analysis.graph.nodes.map((n) => [n.id, n])); - // Compute Maintenance Hotspots: files with churnScore or complexityScore, - // ranked by combined score - const maintenanceHotspots = analysis.graph.nodes - .filter(n => n.type === 'file' && (n.data.churnScore != null || n.data.complexityScore != null)) - .map(n => ({ - node: n, - combinedScore: (n.data.churnScore ?? 0) * 0.5 + (n.data.complexityScore ?? 0) * 0.5, - })) - .sort((a, b) => b.combinedScore - a.combinedScore) - .slice(0, 5); + // Compute Maintenance Hotspots: files with churnScore or complexityScore, + // ranked by combined score + const maintenanceHotspots = analysis.graph.nodes + .filter((n) => n.type === 'file' && (n.data.churnScore != null || n.data.complexityScore != null)) + .map((n) => ({ + node: n, + combinedScore: + ((n.data.churnScore ?? 0) + (n.data.complexityScore ?? 0)) / + (Number(n.data.churnScore != null) + Number(n.data.complexityScore != null)), + })) + .sort((a, b) => b.combinedScore - a.combinedScore) + .slice(0, 5); - return { fileCount, folderCount, depCount, contributorCount, highCoupling, entryPoints, degrees, nodeById, maintenanceHotspots }; + return { + fileCount, + folderCount, + depCount, + contributorCount, + highCoupling, + entryPoints, + degrees, + nodeById, + maintenanceHotspots, + }; }, [analysis]); const totalFiles = analysis.totalFiles ?? fileCount; @@ -110,7 +133,9 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab Total
- {graphDiff.added.size + graphDiff.modified.size + graphDiff.deleted.size} + + {graphDiff.added.size + graphDiff.modified.size + graphDiff.deleted.size} +
@@ -121,11 +146,11 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab Added Files
- {Array.from(graphDiff.added).map(id => { + {Array.from(graphDiff.added).map((id) => { const node = nodeById.get(id); return ( -
)} - + {graphDiff.modified.size > 0 && (

Modified Files

- {Array.from(graphDiff.modified).map(id => { + {Array.from(graphDiff.modified).map((id) => { const node = nodeById.get(id); return ( - + )}
{maintenanceHotspots.map(({ node }) => ( - @@ -390,26 +463,29 @@ export function OverviewTab({ analysis, selectedBranch, graphDiff }: OverviewTab
{analysis.timeline.length === 0 ? ( -
- No commit history available. -
+
No commit history available.
) : (
{(() => { - const maxCount = Math.max(1, Math.max(...analysis.timeline.map(w => w.count))); + const maxCount = Math.max(1, Math.max(...analysis.timeline.map((w) => w.count))); return analysis.timeline.slice(-24).map((week, idx) => { const ratio = week.count / maxCount; const heightPercentage = Math.max(10, ratio * 100); const opacity = Math.max(0.2, ratio); return ( -
+ +
+ ); }); })()} diff --git a/src/features/graph/canvas/hooks/useForceCanvasState.ts b/src/features/graph/canvas/hooks/useForceCanvasState.ts index 4b460d8..ef1272a 100644 --- a/src/features/graph/canvas/hooks/useForceCanvasState.ts +++ b/src/features/graph/canvas/hooks/useForceCanvasState.ts @@ -1,14 +1,15 @@ -import { useMemo, useEffect, useState } from "react"; -import { useParams } from "react-router-dom"; -import type { GraphNode, GraphEdge } from "@/types"; -import { useVizStore, type LayoutType, type SizeMode } from "@/stores/vizStore"; -import { buildForceGraphData } from "../../force/buildForceGraphData"; -import { computeBlastRadius } from "../../force/forceGraphUtils"; -import { useGraphExport } from "../../useGraphExport"; -import { useD3Physics } from "../force/useD3Physics"; -import { useForceSync } from "../force/useForceSync"; -import type { ForceGraphNode, ForceGraphLink } from "../../force/forceGraphConstants"; -import type { ForceGraphMethods } from "react-force-graph-2d"; +import { useMemo, useEffect, useState, useRef } from 'react'; +import { reconcileGraph } from '../../force/reconcileGraph'; +import { useParams } from 'react-router-dom'; +import type { GraphNode, GraphEdge } from '@/types'; +import { useVizStore, type LayoutType, type SizeMode } from '@/stores/vizStore'; +import { buildForceGraphData } from '../../force/buildForceGraphData'; +import { computeBlastRadius } from '../../force/forceGraphUtils'; +import { useGraphExport } from '../../useGraphExport'; +import { useD3Physics } from '../force/useD3Physics'; +import { useForceSync } from '../force/useForceSync'; +import type { ForceGraphNode, ForceGraphLink } from '../../force/forceGraphConstants'; +import type { ForceGraphMethods } from 'react-force-graph-2d'; interface UseForceCanvasStateProps { graph: { nodes: GraphNode[]; edges: GraphEdge[] }; @@ -55,15 +56,15 @@ export function useForceCanvasState({ const [forceSize, setForceSize] = useState({ width: 1024, height: 720 }); const [hoveredForceNode, setHoveredForceNode] = useState(null); - const { owner = "", repo = "" } = useParams(); + const { owner = '', repo = '' } = useParams(); const { isExporting, exportFormat, handleExport } = useGraphExport({ - mode: "force", + mode: 'force', forceGraphRef, forceHostRef, owner, repo, - filenameSuffix: "force_graph", + filenameSuffix: 'force_graph', }); const connectedNodeIdsByNodeId = useMemo(() => { @@ -77,20 +78,21 @@ export function useForceCanvasState({ return lookup; }, [graph.edges]); + const previousGraph = useRef | undefined>(undefined); const forceGraphData = useMemo( () => - buildForceGraphData(graph.nodes, graph.edges, { - nodeTypeFilters, - fileTypeFilters, - readOnly, - }), - [graph.nodes, graph.edges, nodeTypeFilters, fileTypeFilters, readOnly] + (previousGraph.current = reconcileGraph( + previousGraph.current, + buildForceGraphData(graph.nodes, graph.edges, { + nodeTypeFilters, + fileTypeFilters, + readOnly, + }), + )), + [graph.nodes, graph.edges, nodeTypeFilters, fileTypeFilters, readOnly], ); - const forceNodeById = useMemo( - () => new Map(forceGraphData.nodes.map((n) => [n.id, n])), - [forceGraphData.nodes] - ); + const forceNodeById = useMemo(() => new Map(forceGraphData.nodes.map((n) => [n.id, n])), [forceGraphData.nodes]); const blastRadiusNodeIds = useMemo(() => { if (!blastRadiusActive || !selectedNodeId) return new Set(); @@ -102,30 +104,15 @@ export function useForceCanvasState({ // Read-only canvases never adopt workspace focus/selection state. if (readOnly || !focusedFilePath) return; - const focusedNodeIds = [ - `file:${focusedFilePath}`, - `folder:${focusedFilePath}`, - focusedFilePath, - ]; + const focusedNodeIds = [`file:${focusedFilePath}`, `folder:${focusedFilePath}`, focusedFilePath]; const focusedNode = focusedNodeIds.map((id) => forceNodeById.get(id)).find(Boolean) ?? - forceGraphData.nodes.find( - (node) => - node.sourceFile === focusedFilePath || - focusedNodeIds.includes(node.id), - ); + forceGraphData.nodes.find((node) => node.sourceFile === focusedFilePath || focusedNodeIds.includes(node.id)); if (focusedNode && selectedNodeId !== focusedNode.id) { setSelectedNodeId(focusedNode.id); } - }, [ - readOnly, - focusedFilePath, - forceGraphData.nodes, - forceNodeById, - selectedNodeId, - setSelectedNodeId, - ]); + }, [readOnly, focusedFilePath, forceGraphData.nodes, forceNodeById, selectedNodeId, setSelectedNodeId]); useEffect(() => { // Read-only canvases must not rewrite shared highlight state from the @@ -138,9 +125,7 @@ export function useForceCanvasState({ if (blastRadiusActive) { setHighlightedNodeIds(blastRadiusNodeIds); } else { - setHighlightedNodeIds( - new Set(connectedNodeIdsByNodeId.get(selectedNodeId) ?? [selectedNodeId]) - ); + setHighlightedNodeIds(new Set(connectedNodeIdsByNodeId.get(selectedNodeId) ?? [selectedNodeId])); } }, [ readOnly, diff --git a/src/features/graph/force/reconcileGraph.test.ts b/src/features/graph/force/reconcileGraph.test.ts new file mode 100644 index 0000000..40c71dd --- /dev/null +++ b/src/features/graph/force/reconcileGraph.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from 'bun:test'; +import { reconcileGraph } from './reconcileGraph'; +import { buildForceGraphData } from './buildForceGraphData'; +import type { GraphNode } from '@/types'; +test('metadata updates preserve simulation identity and coordinates', () => { + const node: GraphNode = { + id: 'a', + type: 'file', + position: { x: 0, y: 0 }, + data: { label: 'a' }, + }; + const options = { + nodeTypeFilters: new Set(['file']), + fileTypeFilters: new Set(), + }; + const first = buildForceGraphData([node], [], options); + first.nodes[0].x = 125; + first.nodes[0].y = 80; + const next = buildForceGraphData([{ ...node, data: { ...node.data, churnScore: 0.7 } }], [], options); + expect(reconcileGraph(first, next)).toBe(first); + expect(first.nodes[0].x).toBe(125); + expect(first.nodes[0].churnScore).toBe(0.7); + expect(reconcileGraph(first, { nodes: [], links: [] })).not.toBe(first); +}); diff --git a/src/features/graph/force/reconcileGraph.ts b/src/features/graph/force/reconcileGraph.ts new file mode 100644 index 0000000..0d66934 --- /dev/null +++ b/src/features/graph/force/reconcileGraph.ts @@ -0,0 +1,19 @@ +import type { ForceGraphNode, ForceGraphLink } from './forceGraphConstants'; + +type Data = { nodes: ForceGraphNode[]; links: ForceGraphLink[] }; +/** Keep simulation objects for metric-only updates; topology changes get fresh data. */ +export function reconcileGraph(previous: Data | undefined, next: Data): Data { + if (!previous || previous.nodes.length !== next.nodes.length || previous.links.length !== next.links.length) + return next; + const id = (value: ForceGraphLink['source']) => (typeof value === 'object' ? value.id : value); + if ( + next.nodes.some((node, i) => node.id !== previous.nodes[i].id) || + next.links.some((link, i) => { + const old = previous.links[i]; + return id(link.source) !== id(old.source) || id(link.target) !== id(old.target) || link.type !== old.type; + }) + ) + return next; + next.nodes.forEach((node, i) => Object.assign(previous.nodes[i], node)); + return previous; +} diff --git a/src/hooks/useAnalyzeRepo.ts b/src/hooks/useAnalyzeRepo.ts index 3eae81c..2c57406 100644 --- a/src/hooks/useAnalyzeRepo.ts +++ b/src/hooks/useAnalyzeRepo.ts @@ -1,13 +1,41 @@ import { useQuery } from '@tanstack/react-query'; import { analyzeRepo } from '@/lib/apiClient'; +import { useRepoChurn } from './useRepoChurn'; +import { useRepoHealth } from './useRepoHealth'; +import { useMemo } from 'react'; +import { useVizStore } from '@/stores/vizStore'; import type { RepoAnalysis } from '@/types'; export function useAnalyzeRepo(owner: string, repo: string, branch: string | null = null, enabled = true) { const url = `https://github.com/${owner}/${repo}`; - return useQuery({ + const analysis = useQuery({ queryKey: ['analyze-repo', owner, repo, branch], queryFn: () => analyzeRepo(url, branch || undefined), enabled: enabled && !!owner && !!repo, staleTime: 1000 * 60 * 30, }); + const colorMode = useVizStore((s) => s.colorMode); + const sha = analysis.data?.meta.sha ?? ''; + const health = useRepoHealth(owner, repo, sha, enabled && !!analysis.data); + const churn = useRepoChurn(owner, repo, sha, enabled && colorMode === 'churn'); + const data = useMemo(() => { + if (!analysis.data) return undefined; + return { + ...analysis.data, + dependencyHealth: health.data, + graph: { + ...analysis.data.graph, + nodes: analysis.data.graph.nodes.map((node) => { + const extra = node.data.path ? churn.data?.files[node.data.path] : undefined; + return extra ? { ...node, data: { ...node.data, ...extra } } : node; + }), + }, + }; + }, [analysis.data, churn.data, health.data]); + return { + ...analysis, + data, + churnLoading: churn.isPending && churn.isFetching, + churnError: churn.error, + }; } diff --git a/src/hooks/useRepoChurn.ts b/src/hooks/useRepoChurn.ts new file mode 100644 index 0000000..2c0deb0 --- /dev/null +++ b/src/hooks/useRepoChurn.ts @@ -0,0 +1,29 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { fetchChurnBatch } from '@/lib/apiClient'; +import { mergeChurn, pendingChurn, churnInterval } from '@/lib/churn-progress'; + +import type { ChurnResponse } from '@/types/churn'; + +export function useRepoChurn(owner: string, repo: string, sha: string, enabled: boolean) { + const client = useQueryClient(); + const queryKey = ['repo-churn', owner, repo, sha]; + return useQuery({ + queryKey, + queryFn: async () => { + const previous = client.getQueryData(queryKey); + const pending = previous ? pendingChurn(previous) : undefined; + if (previous && !previous.complete && !pending?.length) return previous; + return mergeChurn( + previous, + await fetchChurnBatch(owner, repo, sha, { + completed: Object.keys(previous?.files ?? {}), + pending, + }), + ); + }, + enabled: enabled && !!sha, + retry: false, + staleTime: 30 * 60 * 1000, + refetchInterval: (query) => (enabled ? churnInterval(query.state.data) : false), + }); +} diff --git a/src/hooks/useRepoHealth.ts b/src/hooks/useRepoHealth.ts new file mode 100644 index 0000000..4e0944a --- /dev/null +++ b/src/hooks/useRepoHealth.ts @@ -0,0 +1,13 @@ +import { useQuery } from '@tanstack/react-query'; +import { fetchRepoEnrichment } from '@/lib/apiClient'; +import type { DependencyHealthReport } from '@/types'; + +export function useRepoHealth(owner: string, repo: string, sha: string, enabled: boolean) { + return useQuery({ + queryKey: ['repo-health', owner, repo, sha], + queryFn: () => fetchRepoEnrichment(owner, repo, sha, 'health'), + enabled: enabled && !!sha, + retry: false, + staleTime: 30 * 60 * 1000, + }); +} diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index 3ff41ed..f9c9444 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -39,7 +39,16 @@ export class ApiError extends Error { public details?: unknown; public error?: string; - constructor(message: string, status: number, data?: { code?: string; details?: unknown; context?: unknown; error?: string }) { + constructor( + message: string, + status: number, + data?: { + code?: string; + details?: unknown; + context?: unknown; + error?: string; + }, + ) { super(message); this.name = 'ApiError'; this.status = status; @@ -85,26 +94,23 @@ async function request(path: string, options?: RequestInit): Promise { data = await parseBody(res); } catch (error) { if (!res.ok) { - throw new ApiError(error instanceof Error ? error.message : `Request failed with status ${res.status}`, res.status); + throw new ApiError( + error instanceof Error ? error.message : `Request failed with status ${res.status}`, + res.status, + ); } throw error; } if (!res.ok) { const message = - typeof data === 'object' && - data !== null && - 'message' in data && - typeof data.message === 'string' + typeof data === 'object' && data !== null && 'message' in data && typeof data.message === 'string' ? data.message - : typeof data === 'object' && - data !== null && - 'error' in data && - typeof data.error === 'string' + : typeof data === 'object' && data !== null && 'error' in data && typeof data.error === 'string' ? data.error : typeof data === 'string' - ? data - : `Request failed with status ${res.status}`; + ? data + : `Request failed with status ${res.status}`; throw new ApiError(message, res.status, typeof data === 'object' && data !== null ? data : undefined); } @@ -116,7 +122,6 @@ async function request(path: string, options?: RequestInit): Promise { } } - export async function analyzeRepo(url: string, branch?: string): Promise { return request('/api/repo/analyze', { method: 'POST', @@ -124,18 +129,28 @@ export async function analyzeRepo(url: string, branch?: string): Promise(owner: string, repo: string, sha: string, kind: 'churn' | 'health'): Promise { + return request(`/api/repo/${kind}?${new URLSearchParams({ owner, repo, branch: sha })}`); +} + +export function fetchChurnBatch( owner: string, repo: string, -): Promise<{ name: string; protected: boolean }[]> { + sha: string, + continuation: import('@/types/churn').ChurnContinuation, +) { + return request( + `/api/repo/churn?${new URLSearchParams({ owner, repo, branch: sha })}`, + { method: 'POST', body: JSON.stringify(continuation) }, + ); +} + +export async function fetchRepoBranches(owner: string, repo: string): Promise<{ name: string; protected: boolean }[]> { const params = new URLSearchParams({ owner, repo }); return request<{ name: string; protected: boolean }[]>(`/api/repo/branches?${params}`); } -export async function fetchRepoTags( - owner: string, - repo: string, -): Promise<{ name: string; sha: string }[]> { +export async function fetchRepoTags(owner: string, repo: string): Promise<{ name: string; sha: string }[]> { const params = new URLSearchParams({ owner, repo }); return request<{ name: string; sha: string }[]>(`/api/repo/tags?${params}`); } @@ -149,9 +164,7 @@ export async function fetchRepoFile( const queryObj: Record = { owner, repo, path }; if (branch) queryObj.branch = branch; const params = new URLSearchParams(queryObj); - return request<{ path: string; content: string; sha: string }>( - `/api/repo/file?${params}`, - ); + return request<{ path: string; content: string; sha: string }>(`/api/repo/file?${params}`); } export async function fetchTrending(): Promise { @@ -170,66 +183,42 @@ export async function aiExplain(body: AIExplainRequest & { branch?: string }): P }); } -export async function aiExplainLif( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiExplainLif(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/explain-lif', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiRefactor( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiRefactor(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/refactor', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiHealth( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiHealth(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/health', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiMermaid( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiMermaid(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/mermaid', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiRoast( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiRoast(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/roast', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function aiReadmeEnhance( - owner: string, - repo: string, - branch?: string, -): Promise { +export async function aiReadmeEnhance(owner: string, repo: string, branch?: string): Promise { return request('/api/ai/readme-enhance', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), @@ -262,34 +251,21 @@ export async function semanticSearch( }); } -export async function semanticAsk( - question: string, - owner: string, - repo: string, - branch?: string, -): Promise { +export async function semanticAsk(question: string, owner: string, repo: string, branch?: string): Promise { return request('/api/search/ask', { method: 'POST', body: JSON.stringify({ question, owner, repo, branch }), }); } -export async function triggerIndexing( - owner: string, - repo: string, - branch?: string, -): Promise<{ status: string }> { +export async function triggerIndexing(owner: string, repo: string, branch?: string): Promise<{ status: string }> { return request<{ status: string }>('/api/search/index', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function fetchIndexingStatus( - owner: string, - repo: string, -): Promise { +export async function fetchIndexingStatus(owner: string, repo: string): Promise { const params = new URLSearchParams({ owner, repo }); return request(`/api/search/status?${params}`); } - diff --git a/src/lib/churn-progress.test.ts b/src/lib/churn-progress.test.ts new file mode 100644 index 0000000..5286c6e --- /dev/null +++ b/src/lib/churn-progress.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from 'bun:test'; +import { mergeChurn, pendingChurn, churnInterval } from './churn-progress'; +import type { ChurnResponse } from '@/types/churn'; +const first: ChurnResponse = { + files: { a: { commitCount: 2, authorCount: 1, churnScore: 1 } }, + checked: 1, + total: 3, + complete: false, + remaining: ['b', 'c'], + failures: {}, +}; +test('responses from fresh instances accumulate without losing prior successes', () => { + const result = mergeChurn(first, { + ...first, + files: { b: { commitCount: 4, authorCount: 1, churnScore: 1 } }, + remaining: ['c'], + }); + expect(result.checked).toBe(2); + expect(result.files.a.churnScore).toBe(0.5); + expect(result.remaining).toEqual(['c']); +}); +test('failed files do not starve unchecked files and rate limits schedule recovery', () => { + const failed: ChurnResponse = { + ...first, + failures: { b: { issue: 'timeout-or-network', retryAt: 2000 } }, + issue: 'timeout-or-network', + retryAt: 2000, + }; + expect(pendingChurn(failed, 1000)).toEqual(['c']); + expect(pendingChurn(failed, 3000)).toEqual(['c', 'b']); + expect(churnInterval({ ...failed, issue: 'rate-limit' }, 1000)).toBe(1500); + expect(churnInterval({ ...first, complete: true })).toBe(false); +}); diff --git a/src/lib/churn-progress.ts b/src/lib/churn-progress.ts new file mode 100644 index 0000000..474de89 --- /dev/null +++ b/src/lib/churn-progress.ts @@ -0,0 +1,34 @@ +import type { ChurnResponse } from '@/types/churn'; + +export function mergeChurn(previous: ChurnResponse | undefined, batch: ChurnResponse): ChurnResponse { + const files = { ...previous?.files, ...batch.files }; + const max = Math.max(1, ...Object.values(files).map((file) => file.commitCount)); + for (const [path, file] of Object.entries(files)) files[path] = { ...file, churnScore: file.commitCount / max }; + const remaining = batch.remaining.filter((path) => !files[path]); + const failures = { ...previous?.failures, ...batch.failures }; + for (const path of Object.keys(failures)) if (!remaining.includes(path)) delete failures[path]; + const issue = Object.values(failures).sort((a, b) => b.retryAt - a.retryAt)[0]; + return { + ...batch, + files, + remaining, + failures, + checked: Object.keys(files).length, + complete: remaining.length === 0, + issue: issue?.issue, + retryAt: issue?.retryAt, + }; +} + +export function pendingChurn(data: ChurnResponse, now = Date.now()): string[] { + if (data.issue === 'rate-limit' && (data.retryAt ?? 0) > now) return []; + return data.remaining + .filter((path) => !data.failures[path] || data.failures[path].retryAt <= now) + .sort((a, b) => Number(!!data.failures[a]) - Number(!!data.failures[b])); +} + +export function churnInterval(data: ChurnResponse | undefined, now = Date.now()): number | false { + if (!data || data.complete || data.issue === 'access') return false; + if (pendingChurn(data, now).length) return 1500; + return Math.max(1500, (data.retryAt ?? now) - now); +} diff --git a/src/lib/page-metadata.test.ts b/src/lib/page-metadata.test.ts new file mode 100644 index 0000000..188d565 --- /dev/null +++ b/src/lib/page-metadata.test.ts @@ -0,0 +1,9 @@ +import { expect, test } from 'bun:test'; +import { pageMetadata } from './page-metadata'; +test('valid navigation clears 404 robots and privacy canonical metadata', () => { + expect(pageMetadata('/missing').robots).toBe('noindex'); + expect(pageMetadata('/').robots).toBe('index, follow'); + expect(pageMetadata('/privacy').canonical).toBe('https://gsdm.site/privacy'); + expect(pageMetadata('/').canonical).toBe('https://gsdm.site/'); + expect(pageMetadata('/mock/todo-app').canonical).toBeUndefined(); +}); diff --git a/src/lib/page-metadata.ts b/src/lib/page-metadata.ts new file mode 100644 index 0000000..b3bd8d9 --- /dev/null +++ b/src/lib/page-metadata.ts @@ -0,0 +1,21 @@ +export function pageMetadata(path: string) { + const privacy = /^\/privacy\/?$/.test(path); + const valid = + path === '/' || + privacy || + /^\/(?!api\/|assets\/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38}\/[a-zA-Z0-9_.-]{1,100}(?:\/search)?\/?$/.test(path); + return { + title: privacy + ? 'Privacy policy — gitSdm' + : valid + ? 'gitSdmᵝ — Git Software Dependency Map' + : 'Page not found — gitSdm', + description: privacy + ? 'How gitSdm stores settings, sends repository content to providers, and handles caches and logs.' + : valid + ? 'Interactive dependency visualization for exploring repository files, modules, and architecture.' + : 'This page could not be found. Return to gitSdm to explore a repository.', + canonical: path === '/' ? 'https://gsdm.site/' : privacy ? 'https://gsdm.site/privacy' : undefined, + robots: valid ? 'index, follow' : 'noindex', + }; +} diff --git a/src/lib/workspace-mode.test.ts b/src/lib/workspace-mode.test.ts new file mode 100644 index 0000000..a6a8629 --- /dev/null +++ b/src/lib/workspace-mode.test.ts @@ -0,0 +1,18 @@ +import { expect, test } from 'bun:test'; +import { modeFromPanels, panelsForMode } from './workspace-mode'; +test('full workspace has a useful mobile panel and restores both desktop panels', () => { + expect(panelsForMode('full', true)).toEqual({ + explorerOpen: true, + aiSidebarOpen: false, + }); + expect(panelsForMode('full', false)).toEqual({ + explorerOpen: true, + aiSidebarOpen: true, + }); +}); +test('legacy modes migrate according to actual saved panel visibility', () => { + expect(modeFromPanels(false, true)).toBe('insights'); + expect(modeFromPanels(true, false)).toBe('explorer'); + expect(modeFromPanels(true, true)).toBe('full'); + expect(modeFromPanels(false, false)).toBe('focus'); +}); diff --git a/src/lib/workspace-mode.ts b/src/lib/workspace-mode.ts new file mode 100644 index 0000000..45eb268 --- /dev/null +++ b/src/lib/workspace-mode.ts @@ -0,0 +1,10 @@ +export type SpatialMode = 'full' | 'explorer' | 'insights' | 'focus'; +export function modeFromPanels(explorer: boolean, insights: boolean): SpatialMode { + return explorer ? (insights ? 'full' : 'explorer') : insights ? 'insights' : 'focus'; +} +export function panelsForMode(mode: SpatialMode, mobile: boolean) { + return { + explorerOpen: mode === 'explorer' || mode === 'full', + aiSidebarOpen: mode === 'insights' || (mode === 'full' && !mobile), + }; +} diff --git a/src/main.tsx b/src/main.tsx index bf162d5..61186ec 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -4,10 +4,29 @@ import { Providers } from '@/app/providers'; import { AppRouter } from '@/app/router'; import '@/styles/globals.css'; -createRoot(document.getElementById('root')!).render( - - - - - , -); +async function start() { + const initialPath = window.location.pathname; + const initialPage = + initialPath === '/' + ? (await import('@/pages/HomePage')).HomePage + : /^\/privacy\/?$/.test(initialPath) + ? (await import('@/pages/PrivacyPage')).PrivacyPage + : /^\/[^/]+\/[^/]+\/search\/?$/.test(initialPath) + ? (await import('@/pages/SearchPage')).SearchPage + : /^\/[^/]+\/[^/]+\/?$/.test(initialPath) + ? (await import('@/pages/VizPage')).VizPage + : (await import('@/pages/NotFoundPage')).NotFoundPage; + createRoot(document.getElementById('root')!).render( + + + + + , + ); +} +void start().catch(() => { + const message = document.createElement('p'); + message.setAttribute('role', 'alert'); + message.textContent = 'Interactive features could not load. Please refresh to try again.'; + document.body.appendChild(message); +}); diff --git a/src/pages/VizPage.tsx b/src/pages/VizPage.tsx index e8a66a7..8d61bdf 100644 --- a/src/pages/VizPage.tsx +++ b/src/pages/VizPage.tsx @@ -1,52 +1,48 @@ -import { useEffect, useMemo, useCallback, useState, lazy, Suspense } from "react"; -import { useParams } from "react-router-dom"; -import { motion, AnimatePresence } from "framer-motion"; -import { useAnalyzeRepo } from "@/hooks/useAnalyzeRepo"; -import { useVizStore } from "@/stores/vizStore"; -import { useVizDiff } from "@/hooks/useVizDiff"; -import { useCodeInspectorState } from "@/hooks/useCodeInspectorState"; -import { useWorkspaceShortcuts } from "@/hooks/useWorkspaceShortcuts"; -import { TopNav } from "@/components/viz/top-nav/TopNav"; -import { AISidebar } from "@/components/viz/AISidebar"; +import { useEffect, useMemo, useCallback, useState, lazy, Suspense } from 'react'; +import { useParams } from 'react-router-dom'; +import { motion, AnimatePresence } from 'framer-motion'; +import { useAnalyzeRepo } from '@/hooks/useAnalyzeRepo'; +import { useVizStore } from '@/stores/vizStore'; +import { useVizDiff } from '@/hooks/useVizDiff'; +import { useCodeInspectorState } from '@/hooks/useCodeInspectorState'; +import { useWorkspaceShortcuts } from '@/hooks/useWorkspaceShortcuts'; +import { TopNav } from '@/components/viz/top-nav/TopNav'; +import { AISidebar } from '@/components/viz/AISidebar'; const GraphCanvas = lazy(() => - import("@/features/graph/canvas/GraphCanvas").then((m) => ({ default: m.GraphCanvas })), + import('@/features/graph/canvas/GraphCanvas').then((m) => ({ + default: m.GraphCanvas, + })), ); const ArchitectureView = lazy(() => - import("@/components/viz/ArchitectureView").then((m) => ({ default: m.ArchitectureView })), + import('@/components/viz/ArchitectureView').then((m) => ({ + default: m.ArchitectureView, + })), ); -import { ContributorsView } from "@/components/contributors/ContributorsView"; -import { ExplorerPanel } from "@/components/explorer/ExplorerPanel"; -import { VizError } from "@/components/viz/VizError"; -import { Info, Loader2 } from "lucide-react"; -import { StagedLoader } from "@/components/viz/StagedLoader"; -import { Card } from "@/components/ui/card"; -import type { GraphNode } from "@/types"; -import { FullCommitHistoryView } from "@/components/timeline/CommitHistoryView"; -import { BottomStatusBar } from "@/components/viz/BottomStatusBar"; -import { CodeInspectorDock } from "@/components/explorer/CodeInspectorDock"; -import { VizSidebar } from "@/components/viz/layout/VizSidebar"; -import { fitWorkspacePanels } from "@/lib/workspace-panels"; +import { ContributorsView } from '@/components/contributors/ContributorsView'; +import { ExplorerPanel } from '@/components/explorer/ExplorerPanel'; +import { VizError } from '@/components/viz/VizError'; +import { Info, Loader2 } from 'lucide-react'; +import { StagedLoader } from '@/components/viz/StagedLoader'; +import { Card } from '@/components/ui/card'; +import type { GraphNode } from '@/types'; +import { FullCommitHistoryView } from '@/components/timeline/CommitHistoryView'; +import { BottomStatusBar } from '@/components/viz/BottomStatusBar'; +import { CodeInspectorDock } from '@/components/explorer/CodeInspectorDock'; +import { VizSidebar } from '@/components/viz/layout/VizSidebar'; +import { fitWorkspacePanels } from '@/lib/workspace-panels'; +import { ChurnStatus } from '@/components/viz/ChurnStatus'; export function VizPage() { - const { owner = "", repo = "" } = useParams(); + const { owner = '', repo = '' } = useParams(); const { selectedBranch, compareBranch } = useVizStore(); // Fetch selected/active branch analysis - const { data, isLoading, error } = useAnalyzeRepo( - owner, - repo, - selectedBranch, - ); + const { data, isLoading, error, churnLoading, churnError } = useAnalyzeRepo(owner, repo, selectedBranch); // Fetch comparison branch analysis - const { data: compareData } = useAnalyzeRepo( - owner, - repo, - compareBranch, - !!compareBranch, - ); + const { data: compareData } = useAnalyzeRepo(owner, repo, compareBranch, !!compareBranch); const { reset, @@ -65,20 +61,25 @@ export function VizPage() { } = useVizStore(); const selectedFilePath = focusedFilePath; + const colorMode = useVizStore((s) => s.colorMode); + useEffect(() => { + if (churnError) setToastMessage('Churn history could not be loaded. Check GitHub access or try again later.'); + else if (churnLoading) setToastMessage('Loading churn history…'); + }, [churnError, churnLoading, setToastMessage]); // Code Inspector state & auto-open helpers - const { - codeInspectorState, - setCodeInspectorState, - setPreferredOpenState, - } = useCodeInspectorState(focusedFilePath); + const { codeInspectorState, setCodeInspectorState, setPreferredOpenState } = useCodeInspectorState(focusedFilePath); // Resizable columns state const [leftWidth, setLeftWidth] = useState(240); const [rightWidth, setRightWidth] = useState(360); const [viewportWidth, setViewportWidth] = useState(() => window.innerWidth); const minibarWidth = viewportWidth >= 1024 ? (Number(!explorerOpen) + Number(!aiSidebarOpen)) * 40 : 0; - const panelWidths = fitWorkspacePanels(viewportWidth - minibarWidth, explorerOpen ? leftWidth : 0, aiSidebarOpen ? rightWidth : 0); + const panelWidths = fitWorkspacePanels( + viewportWidth - minibarWidth, + explorerOpen ? leftWidth : 0, + aiSidebarOpen ? rightWidth : 0, + ); useEffect(() => { const update = () => setViewportWidth(window.innerWidth); window.addEventListener('resize', update); @@ -90,18 +91,17 @@ export function VizPage() { useWorkspaceShortcuts(); useEffect(() => { - const media = window.matchMedia("(max-width: 1023px)"); + const media = window.matchMedia('(max-width: 1023px)'); const closeMobileSidebars = () => { - if (!media.matches) return; - setExplorerOpen(false); - setAiSidebarOpen(false); + const state = useVizStore.getState(); + state.setWorkspaceMode(state.workspaceMode); }; closeMobileSidebars(); - media.addEventListener("change", closeMobileSidebars); + media.addEventListener('change', closeMobileSidebars); - return () => media.removeEventListener("change", closeMobileSidebars); + return () => media.removeEventListener('change', closeMobileSidebars); }, [setAiSidebarOpen, setExplorerOpen]); // Compute file map structures, repository graphs, and branch diff calculations @@ -116,8 +116,14 @@ export function VizPage() { const handleSelectFile = useCallback( (path: string) => { - const node = (combinedGraph || data?.graph)?.nodes.find((item) => item.type === 'file' && item.data.path === path); - useVizStore.setState({ focusedFilePath: path, selectedNodeId: node?.id ?? null, sidebarTab: 'analysis' }); + const node = (combinedGraph || data?.graph)?.nodes.find( + (item) => item.type === 'file' && item.data.path === path, + ); + useVizStore.setState({ + focusedFilePath: path, + selectedNodeId: node?.id ?? null, + sidebarTab: 'analysis', + }); if (window.innerWidth >= 1024) { setAiSidebarOpen(true); } else { @@ -133,7 +139,7 @@ export function VizPage() { if (activeRepoKey !== key) { reset(); setActiveRepoKey(key); - if (window.matchMedia("(max-width: 1023px)").matches) { + if (window.matchMedia('(max-width: 1023px)').matches) { setExplorerOpen(false); setAiSidebarOpen(false); } @@ -151,7 +157,7 @@ export function VizPage() { }, [data, selectedNodeId, nodeById]); useEffect(() => { - if (selectedNode?.type === "file" && selectedNode.data.path) { + if (selectedNode?.type === 'file' && selectedNode.data.path) { setFocusedFilePath(selectedNode.data.path); } else if (selectedNode) { setFocusedFilePath(null); @@ -159,169 +165,155 @@ export function VizPage() { }, [selectedNode, setFocusedFilePath]); if (error) { - return ( - - ); + return ; } return (
- + -
- {isLoading ? ( - - ) : data ? ( -
- {/* Left Sidebar wrapper */} - setExplorerOpen(false)} - onOpen={() => setExplorerOpen(true)} - > - - +
+ {isLoading ? ( + + ) : data ? ( +
+ {/* Left Sidebar wrapper */} + setExplorerOpen(false)} + onOpen={() => setExplorerOpen(true)} + > + + - {/* Center Workspace */} - -
- + {/* Center Workspace */} + + {activeView === 'graph' && colorMode === 'churn' && ( + + )} +
+ Loading graph…
) : null - }> -
- - {data.treeTruncated && ( -
- Partial repository tree: some files are unavailable. -
- )} -
- + } + > +
+ + {data.treeTruncated && ( +
+ Partial repository tree: some files are unavailable. +
+ )} +
+ - {activeView === "architecture" && ( - + {activeView === 'architecture' && ( + Loading architecture…
- }> - - - )} - {activeView === "contributors" && ( - - )} - {activeView === "commits" && ( - - )} -
- - {focusedFilePath && codeInspectorState !== 'closed' && ( - { - setCodeInspectorState(s); - if (s === 'closed') { - useVizStore.setState({ focusedFilePath: null, selectedNodeId: null, highlightedNodeIds: new Set() }); - } else { - setPreferredOpenState(s); - } - }} - filePath={focusedFilePath} - graph={combinedGraph || data.graph} - onSelectFile={handleSelectFile} + } + > + + + )} + {activeView === 'contributors' && } + {activeView === 'commits' && ( + )} - +
- {/* Right Sidebar wrapper */} - setAiSidebarOpen(false)} - onOpen={() => setAiSidebarOpen(true)} - > - { + setCodeInspectorState(s); + if (s === 'closed') { + useVizStore.setState({ + focusedFilePath: null, + selectedNodeId: null, + highlightedNodeIds: new Set(), + }); + } else { + setPreferredOpenState(s); + } + }} + filePath={focusedFilePath} + graph={combinedGraph || data.graph} + onSelectFile={handleSelectFile} + owner={owner} + repo={repo} /> - -
- ) : null} -
- - + )} + - - {toastMessage && ( - setAiSidebarOpen(false)} + onOpen={() => setAiSidebarOpen(true)} > - - -
-

- {toastMessage} -

-
- - - )} - + + +
+ ) : null}
+ + + + + {toastMessage && ( + + + +
+

{toastMessage}

+
+ + + )} + +
); } - diff --git a/src/stores/vizStore.ts b/src/stores/vizStore.ts index 42ce5b1..2cf5e46 100644 --- a/src/stores/vizStore.ts +++ b/src/stores/vizStore.ts @@ -1,16 +1,18 @@ -import { create } from "zustand"; -import { persist } from "zustand/middleware"; -import type { NodeType } from "@/types"; +import { create } from 'zustand'; +import { modeFromPanels, panelsForMode } from '@/lib/workspace-mode'; +import { persist } from 'zustand/middleware'; +import type { NodeType } from '@/types'; -export type SidebarTab = "overview" | "analysis" | "dependencies" | "ai" | "learning"; -export type WorkspaceMode = "full" | "explorer" | "insights" | "focus"; -export type GraphScope = "important" | "source" | "grouped" | "full"; -export type ContentFilter = "source" | "config" | "docs" | "tests" | "github" | "examples" | "generated" | "translations"; -export type CompareRefType = "branch" | "tag" | "commit"; -export type ColorMode = "default" | "churn" | "complexity"; -export type SizeMode = "default" | "complexity"; +export type SidebarTab = 'overview' | 'analysis' | 'dependencies' | 'ai' | 'learning'; +export type WorkspaceMode = 'full' | 'explorer' | 'insights' | 'focus'; +export type GraphScope = 'important' | 'source' | 'grouped' | 'full'; +export type ContentFilter = + 'source' | 'config' | 'docs' | 'tests' | 'github' | 'examples' | 'generated' | 'translations'; +export type CompareRefType = 'branch' | 'tag' | 'commit'; +export type ColorMode = 'default' | 'churn' | 'complexity'; +export type SizeMode = 'default' | 'complexity'; -export type LayoutType = "tree" | "d3-tree-horiz" | "d3-tree-vert"; +export type LayoutType = 'tree' | 'd3-tree-horiz' | 'd3-tree-vert'; interface VizState { searchQuery: string; @@ -25,20 +27,20 @@ interface VizState { inspectorOpen: boolean; focusedFilePath: string | null; toastMessage: string | null; - theme: "dark" | "light"; + theme: 'dark' | 'light'; selectedBranch: string | null; compareBranch: string | null; compareRefType: CompareRefType; availableBranches: string[]; - activeView: "graph" | "architecture" | "contributors" | "commits"; + activeView: 'graph' | 'architecture' | 'contributors' | 'commits'; zoom: number; visibleNodeCount: number; visibleEdgeCount: number; activeRepoKey: string | null; // Interactive upgrades - activeFocusLayer: "all" | "api" | "ui" | "core" | "config"; - diffStatusFilters: Set<"added" | "modified" | "deleted">; + activeFocusLayer: 'all' | 'api' | 'ui' | 'core' | 'config'; + diffStatusFilters: Set<'added' | 'modified' | 'deleted'>; blastRadiusActive: boolean; // Churn / Complexity overlay @@ -60,9 +62,9 @@ interface VizState { }; // Shared UI synchronization states - activeDropdown: "filter" | "display" | "layout" | "export" | "legend" | null; + activeDropdown: 'filter' | 'display' | 'layout' | 'export' | 'legend' | null; graphActionTrigger: { - action: "zoomIn" | "zoomOut" | "fitView" | "reset" | "focusGraph"; + action: 'zoomIn' | 'zoomOut' | 'fitView' | 'reset' | 'focusGraph'; timestamp: number; } | null; @@ -70,7 +72,7 @@ interface VizState { toggleContentFilter: (filter: ContentFilter) => void; setContentFilters: (filters: Set) => void; - toggleDiffStatusFilter: (status: "added" | "modified" | "deleted") => void; + toggleDiffStatusFilter: (status: 'added' | 'modified' | 'deleted') => void; setSelectedBranch: (branch: string | null) => void; setCompareBranch: (branch: string | null) => void; setCompareRefType: (type: CompareRefType) => void; @@ -87,47 +89,41 @@ interface VizState { setAiSidebarOpen: (open: boolean) => void; setInspectorOpen: (open: boolean) => void; setFocusedFilePath: (path: string | null) => void; - setTheme: (theme: "dark" | "light") => void; + setTheme: (theme: 'dark' | 'light') => void; toggleTheme: () => void; setZoom: (zoom: number) => void; setVisibleCounts: (nodes: number, edges: number) => void; - setActiveFocusLayer: ( - layer: "all" | "api" | "ui" | "core" | "config", - ) => void; - setActiveView: ( - view: "graph" | "architecture" | "contributors" | "commits", - ) => void; + setActiveFocusLayer: (layer: 'all' | 'api' | 'ui' | 'core' | 'config') => void; + setActiveView: (view: 'graph' | 'architecture' | 'contributors' | 'commits') => void; setBlastRadiusActive: (active: boolean) => void; setColorMode: (mode: ColorMode) => void; setSizeMode: (mode: SizeMode) => void; setLayoutType: (layout: LayoutType) => void; setGraphSidebarOpen: (open: boolean) => void; - toggleGraphSidebarSection: ( - key: keyof VizState["graphSidebarSections"], - ) => void; + toggleGraphSidebarSection: (key: keyof VizState['graphSidebarSections']) => void; resetFilters: () => void; - setActiveDropdown: (dropdown: "filter" | "display" | "layout" | "export" | "legend" | null) => void; - triggerGraphAction: (action: "zoomIn" | "zoomOut" | "fitView" | "reset" | "focusGraph") => void; + setActiveDropdown: (dropdown: 'filter' | 'display' | 'layout' | 'export' | 'legend' | null) => void; + triggerGraphAction: (action: 'zoomIn' | 'zoomOut' | 'fitView' | 'reset' | 'focusGraph') => void; setActiveRepoKey: (key: string | null) => void; reset: () => void; } -const defaultFilters = new Set(["repo", "package", "folder", "file"]); -const defaultContentFilters = new Set(["source", "config"]); +const defaultFilters = new Set(['repo', 'package', 'folder', 'file']); +const defaultContentFilters = new Set(['source', 'config']); export const useVizStore = create()( persist( (set) => ({ - searchQuery: "", + searchQuery: '', nodeTypeFilters: new Set(defaultFilters), fileTypeFilters: new Set(), selectedNodeId: null, highlightedNodeIds: new Set(), - sidebarTab: "overview", - workspaceMode: "full", + sidebarTab: 'overview', + workspaceMode: 'full', explorerOpen: true, aiSidebarOpen: true, inspectorOpen: false, @@ -136,215 +132,227 @@ export const useVizStore = create()( visibleNodeCount: 0, visibleEdgeCount: 0, toastMessage: null, - theme: "dark", + theme: 'dark', selectedBranch: null, compareBranch: null, - compareRefType: "branch" as CompareRefType, + compareRefType: 'branch' as CompareRefType, availableBranches: [], - activeView: "graph", + activeView: 'graph', activeRepoKey: null, - activeFocusLayer: "all", - diffStatusFilters: new Set(), - blastRadiusActive: false, - colorMode: "default" as ColorMode, - sizeMode: "default" as SizeMode, - layoutType: "tree" as LayoutType, + activeFocusLayer: 'all', + diffStatusFilters: new Set(), + blastRadiusActive: false, + colorMode: 'default' as ColorMode, + sizeMode: 'default' as SizeMode, + layoutType: 'tree' as LayoutType, - graphScope: "source", - contentFilters: new Set(defaultContentFilters), + graphScope: 'source', + contentFilters: new Set(defaultContentFilters), - graphSidebarOpen: true, - graphSidebarSections: { - nodeInfo: true, - tools: false, - nodeTypes: true, - fileTypes: false, - compare: true, - }, + graphSidebarOpen: true, + graphSidebarSections: { + nodeInfo: true, + tools: false, + nodeTypes: true, + fileTypes: false, + compare: true, + }, - activeDropdown: null, - graphActionTrigger: null, + activeDropdown: null, + graphActionTrigger: null, - setGraphScope: (graphScope: GraphScope) => set((s: VizState) => { - let newContentFilters = new Set(s.contentFilters); - if (graphScope === 'full') { - newContentFilters = new Set(["source", "config", "docs", "tests", "github", "examples", "generated", "translations"]); - } else if (graphScope === 'source' || graphScope === 'important' || graphScope === 'grouped') { - newContentFilters = new Set(["source", "config"]); - } - return { graphScope, contentFilters: newContentFilters }; - }), - toggleContentFilter: (filter: ContentFilter) => - set((s: VizState) => { - const next = new Set(s.contentFilters); - if (next.has(filter)) next.delete(filter); - else next.add(filter); - return { contentFilters: next }; - }), - setContentFilters: (contentFilters: Set) => set({ contentFilters }), + setGraphScope: (graphScope: GraphScope) => + set((s: VizState) => { + let newContentFilters = new Set(s.contentFilters); + if (graphScope === 'full') { + newContentFilters = new Set([ + 'source', + 'config', + 'docs', + 'tests', + 'github', + 'examples', + 'generated', + 'translations', + ]); + } else if (graphScope === 'source' || graphScope === 'important' || graphScope === 'grouped') { + newContentFilters = new Set(['source', 'config']); + } + return { graphScope, contentFilters: newContentFilters }; + }), + toggleContentFilter: (filter: ContentFilter) => + set((s: VizState) => { + const next = new Set(s.contentFilters); + if (next.has(filter)) next.delete(filter); + else next.add(filter); + return { contentFilters: next }; + }), + setContentFilters: (contentFilters: Set) => set({ contentFilters }), - toggleDiffStatusFilter: (status: "added" | "modified" | "deleted") => - set((s: VizState) => { - const next = new Set(s.diffStatusFilters); - if (next.has(status)) next.delete(status); - else next.add(status); - return { diffStatusFilters: next }; - }), - setSelectedBranch: (selectedBranch: string | null) => set({ selectedBranch }), - setCompareBranch: (compareBranch: string | null) => - set((s: VizState) => ({ - compareBranch, - compareRefType: "branch", - diffStatusFilters: compareBranch ? s.diffStatusFilters : new Set(), - })), - setCompareRefType: (compareRefType: CompareRefType) => set({ compareRefType }), - setAvailableBranches: (availableBranches: string[]) => set({ availableBranches }), - setToastMessage: (toastMessage: string | null) => set({ toastMessage }), - setSearchQuery: (searchQuery: string) => set({ searchQuery }), - toggleNodeTypeFilter: (type: NodeType) => - set((s: VizState) => { - const next = new Set(s.nodeTypeFilters); - if (next.has(type)) next.delete(type); - else next.add(type); - return { nodeTypeFilters: next }; - }), - toggleFileTypeFilter: (type: string) => - set((s: VizState) => { - const next = new Set(s.fileTypeFilters); - if (next.has(type)) next.delete(type); - else next.add(type); - return { fileTypeFilters: next }; - }), - setSelectedNodeId: (selectedNodeId: string | null) => set({ selectedNodeId }), - setHighlightedNodeIds: (highlightedNodeIds: Set) => set({ highlightedNodeIds }), - setSidebarTab: (sidebarTab: SidebarTab) => set({ sidebarTab }), - setWorkspaceMode: (workspaceMode: WorkspaceMode) => { - const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; - const showExplorer = workspaceMode === 'full' || workspaceMode === 'explorer'; - const showAiSidebar = workspaceMode === 'full' || workspaceMode === 'insights'; - set({ - workspaceMode, - explorerOpen: showExplorer && (!isMobile || workspaceMode === 'explorer'), - aiSidebarOpen: showAiSidebar && (!isMobile || workspaceMode === 'insights'), - inspectorOpen: false, - }); - }, - setExplorerOpen: (explorerOpen: boolean) => set({ explorerOpen }), - setAiSidebarOpen: (aiSidebarOpen: boolean) => set({ aiSidebarOpen }), - setInspectorOpen: (inspectorOpen: boolean) => set({ inspectorOpen }), - setFocusedFilePath: (focusedFilePath: string | null) => set({ focusedFilePath }), - setZoom: (zoom: number) => set({ zoom }), - setVisibleCounts: (visibleNodeCount: number, visibleEdgeCount: number) => set({ visibleNodeCount, visibleEdgeCount }), - setTheme: (theme: "dark" | "light") => { - if (typeof window !== "undefined") localStorage.setItem("theme", theme); - set({ theme }); - }, - toggleTheme: () => - set((s: VizState) => { - const nextTheme = s.theme === "dark" ? "light" : "dark"; - if (typeof window !== "undefined") - localStorage.setItem("theme", nextTheme); - return { theme: nextTheme }; - }), - - setActiveFocusLayer: (activeFocusLayer: "all" | "api" | "ui" | "core" | "config") => set({ activeFocusLayer }), - setActiveView: (activeView: "graph" | "architecture" | "contributors" | "commits") => set({ activeView }), - setBlastRadiusActive: (blastRadiusActive: boolean) => set({ blastRadiusActive }), - setColorMode: (colorMode: ColorMode) => set({ colorMode }), - setSizeMode: (sizeMode: SizeMode) => set({ sizeMode }), - setLayoutType: (layoutType: LayoutType) => set({ layoutType }), - setGraphSidebarOpen: (graphSidebarOpen: boolean) => set({ graphSidebarOpen }), - toggleGraphSidebarSection: (key: keyof VizState["graphSidebarSections"]) => - set((s: VizState) => ({ - graphSidebarSections: { - ...s.graphSidebarSections, - [key]: !s.graphSidebarSections[key], + toggleDiffStatusFilter: (status: 'added' | 'modified' | 'deleted') => + set((s: VizState) => { + const next = new Set(s.diffStatusFilters); + if (next.has(status)) next.delete(status); + else next.add(status); + return { diffStatusFilters: next }; + }), + setSelectedBranch: (selectedBranch: string | null) => set({ selectedBranch }), + setCompareBranch: (compareBranch: string | null) => + set((s: VizState) => ({ + compareBranch, + compareRefType: 'branch', + diffStatusFilters: compareBranch ? s.diffStatusFilters : new Set(), + })), + setCompareRefType: (compareRefType: CompareRefType) => set({ compareRefType }), + setAvailableBranches: (availableBranches: string[]) => set({ availableBranches }), + setToastMessage: (toastMessage: string | null) => set({ toastMessage }), + setSearchQuery: (searchQuery: string) => set({ searchQuery }), + toggleNodeTypeFilter: (type: NodeType) => + set((s: VizState) => { + const next = new Set(s.nodeTypeFilters); + if (next.has(type)) next.delete(type); + else next.add(type); + return { nodeTypeFilters: next }; + }), + toggleFileTypeFilter: (type: string) => + set((s: VizState) => { + const next = new Set(s.fileTypeFilters); + if (next.has(type)) next.delete(type); + else next.add(type); + return { fileTypeFilters: next }; + }), + setSelectedNodeId: (selectedNodeId: string | null) => set({ selectedNodeId }), + setHighlightedNodeIds: (highlightedNodeIds: Set) => set({ highlightedNodeIds }), + setSidebarTab: (sidebarTab: SidebarTab) => set({ sidebarTab }), + setWorkspaceMode: (workspaceMode: WorkspaceMode) => { + const isMobile = typeof window !== 'undefined' && window.innerWidth < 1024; + set({ + workspaceMode, + ...panelsForMode(workspaceMode, isMobile), + inspectorOpen: false, + }); + }, + setExplorerOpen: (explorerOpen: boolean) => set({ explorerOpen }), + setAiSidebarOpen: (aiSidebarOpen: boolean) => set({ aiSidebarOpen }), + setInspectorOpen: (inspectorOpen: boolean) => set({ inspectorOpen }), + setFocusedFilePath: (focusedFilePath: string | null) => set({ focusedFilePath }), + setZoom: (zoom: number) => set({ zoom }), + setVisibleCounts: (visibleNodeCount: number, visibleEdgeCount: number) => + set({ visibleNodeCount, visibleEdgeCount }), + setTheme: (theme: 'dark' | 'light') => { + if (typeof window !== 'undefined') localStorage.setItem('theme', theme); + set({ theme }); }, - })), + toggleTheme: () => + set((s: VizState) => { + const nextTheme = s.theme === 'dark' ? 'light' : 'dark'; + if (typeof window !== 'undefined') localStorage.setItem('theme', nextTheme); + return { theme: nextTheme }; + }), - resetFilters: () => - set({ - nodeTypeFilters: new Set(defaultFilters), - fileTypeFilters: new Set(), - diffStatusFilters: new Set(), - activeFocusLayer: "all", - searchQuery: "", - graphScope: "source", - contentFilters: new Set(defaultContentFilters), - colorMode: "default" as ColorMode, - sizeMode: "default" as SizeMode, - }), + setActiveFocusLayer: (activeFocusLayer: 'all' | 'api' | 'ui' | 'core' | 'config') => set({ activeFocusLayer }), + setActiveView: (activeView: 'graph' | 'architecture' | 'contributors' | 'commits') => set({ activeView }), + setBlastRadiusActive: (blastRadiusActive: boolean) => set({ blastRadiusActive }), + setColorMode: (colorMode: ColorMode) => set({ colorMode }), + setSizeMode: (sizeMode: SizeMode) => set({ sizeMode }), + setLayoutType: (layoutType: LayoutType) => set({ layoutType }), + setGraphSidebarOpen: (graphSidebarOpen: boolean) => set({ graphSidebarOpen }), + toggleGraphSidebarSection: (key: keyof VizState['graphSidebarSections']) => + set((s: VizState) => ({ + graphSidebarSections: { + ...s.graphSidebarSections, + [key]: !s.graphSidebarSections[key], + }, + })), - setActiveDropdown: (activeDropdown: "filter" | "display" | "layout" | "export" | "legend" | null) => set({ activeDropdown }), - triggerGraphAction: (action: "zoomIn" | "zoomOut" | "fitView" | "reset" | "focusGraph") => set({ graphActionTrigger: { action, timestamp: Date.now() } }), + resetFilters: () => + set({ + nodeTypeFilters: new Set(defaultFilters), + fileTypeFilters: new Set(), + diffStatusFilters: new Set(), + activeFocusLayer: 'all', + searchQuery: '', + graphScope: 'source', + contentFilters: new Set(defaultContentFilters), + colorMode: 'default' as ColorMode, + sizeMode: 'default' as SizeMode, + }), - setActiveRepoKey: (activeRepoKey: string | null) => set({ activeRepoKey }), + setActiveDropdown: (activeDropdown: 'filter' | 'display' | 'layout' | 'export' | 'legend' | null) => + set({ activeDropdown }), + triggerGraphAction: (action: 'zoomIn' | 'zoomOut' | 'fitView' | 'reset' | 'focusGraph') => + set({ graphActionTrigger: { action, timestamp: Date.now() } }), - reset: () => - set({ - searchQuery: "", - nodeTypeFilters: new Set(defaultFilters), - fileTypeFilters: new Set(), - selectedNodeId: null, - highlightedNodeIds: new Set(), - sidebarTab: "overview", - workspaceMode: "full", - explorerOpen: true, - aiSidebarOpen: true, - inspectorOpen: false, - focusedFilePath: null, - zoom: 1.0, - visibleNodeCount: 0, - visibleEdgeCount: 0, - toastMessage: null, - selectedBranch: null, - compareBranch: null, - compareRefType: "branch" as CompareRefType, - availableBranches: [], - activeFocusLayer: "all", - diffStatusFilters: new Set(), - activeView: "graph", - blastRadiusActive: false, - colorMode: "default" as ColorMode, - sizeMode: "default" as SizeMode, - layoutType: "tree" as LayoutType, - graphScope: "source", - contentFilters: new Set(defaultContentFilters), - activeDropdown: null, - graphActionTrigger: null, - }), - }), - { - name: 'gitsdm-viz-storage', - partialize: (state) => ({ - workspaceMode: state.workspaceMode, - explorerOpen: state.explorerOpen, - aiSidebarOpen: state.aiSidebarOpen, - inspectorOpen: state.inspectorOpen, - sidebarTab: state.sidebarTab, - theme: state.theme, - activeView: state.activeView, - graphSidebarOpen: state.graphSidebarOpen, - activeDropdown: state.activeDropdown, - graphScope: state.graphScope, - layoutType: state.layoutType, - contentFilters: Array.from(state.contentFilters), + setActiveRepoKey: (activeRepoKey: string | null) => set({ activeRepoKey }), + + reset: () => + set({ + searchQuery: '', + nodeTypeFilters: new Set(defaultFilters), + fileTypeFilters: new Set(), + selectedNodeId: null, + highlightedNodeIds: new Set(), + sidebarTab: 'overview', + workspaceMode: 'full', + explorerOpen: true, + aiSidebarOpen: true, + inspectorOpen: false, + focusedFilePath: null, + zoom: 1.0, + visibleNodeCount: 0, + visibleEdgeCount: 0, + toastMessage: null, + selectedBranch: null, + compareBranch: null, + compareRefType: 'branch' as CompareRefType, + availableBranches: [], + activeFocusLayer: 'all', + diffStatusFilters: new Set(), + activeView: 'graph', + blastRadiusActive: false, + colorMode: 'default' as ColorMode, + sizeMode: 'default' as SizeMode, + layoutType: 'tree' as LayoutType, + graphScope: 'source', + contentFilters: new Set(defaultContentFilters), + activeDropdown: null, + graphActionTrigger: null, + }), }), - merge: (persistedState: unknown, currentState: VizState): VizState => { - const p = persistedState as Partial & { contentFilters: ContentFilter[] }>; - const validModes: WorkspaceMode[] = ['full', 'explorer', 'insights', 'focus']; - const workspaceMode = p?.workspaceMode && validModes.includes(p.workspaceMode as WorkspaceMode) - ? (p.workspaceMode as WorkspaceMode) - : 'full'; - return { - ...currentState, - ...p, - workspaceMode, - contentFilters: p?.contentFilters - ? new Set(p.contentFilters) - : currentState.contentFilters, - } as VizState; + { + name: 'gitsdm-viz-storage', + partialize: (state) => ({ + workspaceMode: state.workspaceMode, + explorerOpen: state.explorerOpen, + aiSidebarOpen: state.aiSidebarOpen, + inspectorOpen: state.inspectorOpen, + sidebarTab: state.sidebarTab, + theme: state.theme, + activeView: state.activeView, + graphSidebarOpen: state.graphSidebarOpen, + activeDropdown: state.activeDropdown, + graphScope: state.graphScope, + layoutType: state.layoutType, + contentFilters: Array.from(state.contentFilters), + }), + merge: (persistedState: unknown, currentState: VizState): VizState => { + const p = persistedState as Partial & { contentFilters: ContentFilter[] }>; + const validModes: WorkspaceMode[] = ['full', 'explorer', 'insights', 'focus']; + const workspaceMode = + p?.workspaceMode && validModes.includes(p.workspaceMode as WorkspaceMode) + ? (p.workspaceMode as WorkspaceMode) + : modeFromPanels( + p?.explorerOpen ?? currentState.explorerOpen, + p?.aiSidebarOpen ?? currentState.aiSidebarOpen, + ); + return { + ...currentState, + ...p, + workspaceMode, + contentFilters: p?.contentFilters ? new Set(p.contentFilters) : currentState.contentFilters, + } as VizState; + }, }, - } -)); + ), +); diff --git a/src/types/churn.ts b/src/types/churn.ts new file mode 100644 index 0000000..ea2fe7d --- /dev/null +++ b/src/types/churn.ts @@ -0,0 +1,22 @@ +export interface FileChurnData { + commitCount: number; + authorCount: number; + lastModified?: string; + churnScore: number; +} + +export interface ChurnResponse { + files: Record; + checked: number; + total: number; + complete: boolean; + issue?: 'rate-limit' | 'access' | 'timeout-or-network'; + retryAt?: number; + remaining: string[]; + failures: Record; retryAt: number }>; +} + +export interface ChurnContinuation { + completed?: string[]; + pending?: string[]; +} diff --git a/src/types/domain.ts b/src/types/domain.ts index 06bc6b8..2ffdcb5 100644 --- a/src/types/domain.ts +++ b/src/types/domain.ts @@ -189,6 +189,7 @@ export interface RepoAnalysis { dependencies: Dependency[]; workspacePackages?: WorkspacePackage[]; dependencyHealth?: DependencyHealthReport; + scopedDependencies?: ScopedDependency[]; graph: GraphData; contributors: Contributor[]; timeline: TimelineWeek[]; diff --git a/vercel.json b/vercel.json index dd3a68f..c222f3f 100644 --- a/vercel.json +++ b/vercel.json @@ -16,8 +16,14 @@ "rewrites": [ { "source": "/api/(.*)", "destination": "/api/$1" }, { "source": "/privacy", "destination": "/privacy.html" }, - { "source": "/:owner((?!api$|assets$)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})", "destination": "/app.html" }, - { "source": "/:owner((?!api$|assets$)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})/search", "destination": "/app.html" } + { + "source": "/:owner((?!api/|assets/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})", + "destination": "/app.html" + }, + { + "source": "/:owner((?!api/|assets/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38})/:repo([a-zA-Z0-9_.-]{1,100})/search", + "destination": "/app.html" + } ], "functions": { "api/**/*.ts": { From 85404c414eff15f8ab859f3e89e0d3741330a53f Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Wed, 9 Sep 2026 00:19:40 +0700 Subject: [PATCH 05/27] fix(search): isolate snapshots and bound indexing resources Publish indexes atomically, enforce chunk and memory budgets, stop idle polling, and recover from eviction without caching empty results. --- README.md | 2 + e2e/search.e2e.ts | 54 ++- server/github/fetch-tree.ts | 34 +- server/router/search-routes.ts | 33 +- server/search/chunker.test.ts | 26 +- server/search/chunker.ts | 69 ++-- server/search/embedding-provider.ts | 27 +- server/search/index-identity.test.ts | 18 + server/search/index-identity.ts | 30 ++ server/search/indexing-pipeline.test.ts | 417 +++++++--------------- server/search/indexing-pipeline.ts | 353 ++++++++---------- server/search/limits.ts | 11 + server/search/qa-engine.test.ts | 20 +- server/search/qa-engine.ts | 27 +- server/search/search-engine.test.ts | 34 +- server/search/search-engine.ts | 35 +- server/search/search-eviction.test.ts | 44 +++ server/search/types.ts | 21 +- server/search/vector-store.test.ts | 44 ++- server/search/vector-store.ts | 131 +++---- src/features/search/searchStore.ts | 8 +- src/features/search/useIndexingStatus.ts | 30 +- src/features/search/useSemanticAsk.ts | 20 +- src/features/search/useSemanticSearch.ts | 33 +- src/features/search/useTriggerIndexing.ts | 20 +- src/lib/apiClient.ts | 8 +- src/pages/SearchPage.tsx | 106 +++--- 27 files changed, 801 insertions(+), 854 deletions(-) create mode 100644 server/search/index-identity.test.ts create mode 100644 server/search/index-identity.ts create mode 100644 server/search/limits.ts create mode 100644 server/search/search-eviction.test.ts diff --git a/README.md b/README.md index d218ab9..e46d1ec 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ - 🌡️ **Code Churn & Complexity Overlays** — Heatmap modes visualizing 90-day commit frequency, author ownership risk, and cyclomatic/line complexity. - 🧠 **AI Codebase Intelligence** — Multi-provider AI orchestration for architecture overviews, dependency health audits, refactoring risk analysis, and interactive learning paths. - 🔍 **Semantic Search & QA Engine** — Vector-embedded code chunking with natural language codebase question-answering. + +Search indexes are process-local and scoped to GitHub credentials, commit SHA, and embedding configuration. Use a persistent backend for search; indexes are not shared between serverless instances and are lost on restart. Per process, indexing allows two active jobs with no waiting queue. Each repository is limited to 2,000 supported files, 256 KiB per file, 4,000 chunks, and 32 MiB of estimated index data. Published indexes have a 128 MiB total budget, a 16-index cap, and a 30-minute lifetime, with least-recently-used eviction. Active builds retain at most 32 MiB each in addition to published indexes. Oversized or incomplete builds fail without replacing a published snapshot. - 📐 **Architecture Diagrams** — Automated Mermaid diagram generation with interactive zoom/pan and PNG/PDF/SVG export. - 🔀 **Branch, Tag & Commit Comparison** — Visual diffing of dependencies and module trees across Git branches, tags, and specific commit SHAs. - 👥 **Contributor Analytics & Timeline** — Commit history activity charts, timeline weeks, and author metrics. diff --git a/e2e/search.e2e.ts b/e2e/search.e2e.ts index d8df0d2..f7b4558 100644 --- a/e2e/search.e2e.ts +++ b/e2e/search.e2e.ts @@ -1,6 +1,59 @@ import { test, expect } from '@playwright/test'; +test('index completion, scoped search, and status agree', async ({ request }) => { + const headers = { 'X-GitHub-Token': 'offline-search-test' }; + const data = { owner: 'mock', repo: 'todo-app' }; + const indexed = await request.post('/api/search/index', { data, headers }); + expect(indexed.status()).toBe(200); + expect(await indexed.json()).toMatchObject({ state: 'complete' }); + const status = await request.get('/api/search/status?owner=mock&repo=todo-app', { headers }); + expect(await status.json()).toMatchObject({ state: 'complete' }); + const found = await request.post('/api/search', { data: { ...data, query: 'todo state' }, headers }); + expect(found.status()).toBe(200); + const isolated = await request.post('/api/search', { + data: { ...data, query: 'todo state' }, + headers: { 'X-GitHub-Token': 'different-user' }, + }); + expect(isolated.status()).toBe(404); +}); + test.describe('Search', () => { + test('idle status does not poll and an existing index is restored on entry', async ({ page }) => { + let calls = 0; + await page.route('**/api/search/status?**', (route) => { + calls++; + return route.fulfill({ json: { state: 'idle' } }); + }); + await page.goto('/mock/todo-app/search'); + await expect.poll(() => calls).toBe(1); + // Observe beyond the old three-second polling interval. + await page.waitForTimeout(3500); + expect(calls).toBe(1); + await page.unroute('**/api/search/status?**'); + await page.route('**/api/search/status?**', (route) => + route.fulfill({ + json: { state: 'complete', chunkCount: 12, timestamp: Date.now() }, + }), + ); + await page.reload(); + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); + await expect(page.getByLabel('Search codebase or ask a question')).toBeEnabled(); + }); + test('build completion enables search and a missing snapshot offers rebuilding', async ({ page }) => { + await page.goto('/mock/todo-app/search'); + await page.getByRole('button', { name: 'Build Index', exact: true }).click(); + await expect(page.getByText('Index ready', { exact: true })).toBeVisible(); + await page.route('**/api/search', (route) => + route.fulfill({ + status: 404, + contentType: 'application/json', + body: JSON.stringify({ error: 'Index expired. Please rebuild.', code: 'INDEX_NOT_FOUND' }), + }), + ); + await page.getByLabel('Search codebase or ask a question').fill('todo state'); + await page.getByRole('button', { name: 'Go', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Build Index', exact: true })).toBeVisible(); + }); test('search page loads and has search input', async ({ page }) => { // Navigate to a repo's search page await page.goto('/mock/gitsdm/search', { timeout: 15000 }); @@ -9,5 +62,4 @@ test.describe('Search', () => { const searchInput = page.locator('input[type="search"], input[placeholder*="Search"i], input[placeholder*="Ask"i]'); await expect(searchInput.first()).toBeVisible({ timeout: 15000 }); }); - }); diff --git a/server/github/fetch-tree.ts b/server/github/fetch-tree.ts index 710b4d1..26f2455 100644 --- a/server/github/fetch-tree.ts +++ b/server/github/fetch-tree.ts @@ -210,7 +210,16 @@ export async function fetchFlatTree( const isNoise = (p: string) => { const lower = p.toLowerCase(); // Match explicit noise folders - if (lower.includes('docs/') || lower.includes('test/') || lower.includes('tests/') || lower.includes('spec/') || lower.includes('dist/') || lower.includes('build/') || lower.includes('example/')) return true; + if ( + lower.includes('docs/') || + lower.includes('test/') || + lower.includes('tests/') || + lower.includes('spec/') || + lower.includes('dist/') || + lower.includes('build/') || + lower.includes('example/') + ) + return true; // Match any hidden folder starting with a dot (e.g., .github, .vscode, .husky) if (lower.includes('/.') || lower.startsWith('.')) return true; return false; @@ -218,8 +227,9 @@ export async function fetchFlatTree( const mapped = allBlobs.map((item) => { let priority = 0; - if (isManifest(item.path)) priority = -1; // Highest priority: never truncate manifests - else if (isNoise(item.path)) priority = 1; // Lowest priority: noise + if (isManifest(item.path)) + priority = -1; // Highest priority: never truncate manifests + else if (isNoise(item.path)) priority = 1; // Lowest priority: noise return { item, priority }; }); @@ -331,6 +341,7 @@ export async function fetchFileContents( paths: string[], ref: string, tokenOrCtx?: string | RequestContext, + maxBytes?: number, ): Promise> { if (isMockRepo(owner)) { return fetchMockFileContents(owner, repo, paths); @@ -350,10 +361,20 @@ export async function fetchFileContents( path, ref, }); - if (!Array.isArray(data) && data.type === 'file' && 'content' in data && data.content) { - result[path] = Buffer.from(data.content, 'base64').toString('utf-8').slice(0, 50000); + if (!Array.isArray(data) && data.type === 'file' && 'content' in data && typeof data.content === 'string') { + if ( + maxBytes !== undefined && + (data.size > maxBytes || data.content.length > Math.ceil(maxBytes * 1.4) + 1024) + ) + throw new Error('File exceeds the indexing byte limit'); + const decoded = Buffer.from(data.content, 'base64'); + if (maxBytes !== undefined && decoded.byteLength > maxBytes) + throw new Error('File exceeds the indexing byte limit'); + result[path] = + maxBytes === undefined ? decoded.toString('utf-8').slice(0, 50000) : decoded.toString('utf-8'); } - } catch { + } catch (error) { + if (maxBytes !== undefined) throw error; // skip unreadable files } }), @@ -479,4 +500,3 @@ export async function fetchTimeline( return []; } } - diff --git a/server/router/search-routes.ts b/server/router/search-routes.ts index 9683c6d..e301f9e 100644 --- a/server/router/search-routes.ts +++ b/server/router/search-routes.ts @@ -1,11 +1,12 @@ import type { RequestContext } from '../utils/context'; import { AppError } from '../utils/errors'; -import { searchBodySchema, askBodySchema, indexBodySchema } from './schemas'; +import { searchBodySchema, askBodySchema, indexBodySchema, repoQuerySchema } from './schemas'; import { fetchRepoInfo } from '../github/fetch-tree'; import { getSearchEngine } from '../search/search-engine'; import { getQAEngine } from '../search/qa-engine'; import { getIndexingPipeline } from '../search/indexing-pipeline'; import { logApi } from '../utils/logger'; +import { searchIndexKey } from '../search/index-identity'; export async function handleSearchRoutes( pathname: string, @@ -28,6 +29,7 @@ export async function handleSearchRoutes( owner: parsed.data.owner, repo: parsed.data.repo, commitSha: info.sha, + gitHubToken: ctx.gitHubToken, }); logApi('/api/search', { durationMs: Date.now() - start, results: result.results.length }); return Response.json(result, { status: 200 }); @@ -47,6 +49,7 @@ export async function handleSearchRoutes( repo: parsed.data.repo, commitSha: info.sha, apiKey: userKey, + gitHubToken: ctx.gitHubToken, }); logApi('/api/search/ask', { durationMs: Date.now() - start, citations: result.citations.length }); return Response.json(result, { status: 200 }); @@ -62,12 +65,15 @@ export async function handleSearchRoutes( const pipeline = getIndexingPipeline(); try { - await pipeline.startIndexing({ - owner: parsed.data.owner, - repo: parsed.data.repo, - branch: parsed.data.branch, - commitSha: info.sha, - }, ctx); + await pipeline.startIndexing( + { + owner: parsed.data.owner, + repo: parsed.data.repo, + branch: parsed.data.branch, + commitSha: info.sha, + }, + ctx, + ); } catch (err) { if (err instanceof AppError && err.status === 409) { return Response.json({ status: 'rejected', error: 'Indexing already in progress' }, { status: 409 }); @@ -76,17 +82,20 @@ export async function handleSearchRoutes( } logApi('/api/search/index', { durationMs: Date.now() - start, repo: `${parsed.data.owner}/${parsed.data.repo}` }); - return Response.json({ status: 'started' }, { status: 200 }); + return Response.json(pipeline.getStatus(searchIndexKey(parsed.data.owner, parsed.data.repo, info.sha, ctx)), { + status: 200, + }); } if (pathname === '/api/search/status') { - const owner = query.owner; - const repo = query.repo; - if (!owner || !repo) { + const parsed = repoQuerySchema.safeParse(query); + if (!parsed.success) { throw new AppError(400, 'owner and repo query params required', 'INVALID_PARAMS'); } + const { owner, repo, branch } = parsed.data; const pipeline = getIndexingPipeline(); - const status = pipeline.getStatus(`${owner}/${repo}`); + const info = await fetchRepoInfo(owner, repo, branch, ctx); + const status = pipeline.getStatus(searchIndexKey(owner, repo, info.sha, ctx)); return Response.json(status, { status: 200 }); } diff --git a/server/search/chunker.test.ts b/server/search/chunker.test.ts index e288309..d6e6b60 100644 --- a/server/search/chunker.test.ts +++ b/server/search/chunker.test.ts @@ -7,6 +7,24 @@ const MAX_CHUNK_CHARS = MAX_CHUNK_TOKENS * 4; describe('createChunker', () => { const chunker = createChunker(); + it('does not duplicate nested blocks', () => { + const content = 'function f(){\n'.repeat(1000) + '}\n'.repeat(1000); + const chunks = chunker.chunkFile(content, 'nested.ts', 'typescript'); + expect(chunks.length).toBeLessThan(20); + expect(chunks.reduce((sum, chunk) => sum + chunk.content.length, 0)).toBeLessThanOrEqual(content.length); + }); + + it('stops emitting when count or byte budgets are exhausted', () => { + const content = Array.from({ length: 1000 }, (_, i) => `function f${i}() { return ${i}; }`).join('\n'); + for (const budget of [ + { maxChunks: 2, maxBytes: 100000, bytesPerChunk: 100 }, + { maxChunks: 1000, maxBytes: 100, bytesPerChunk: 100 }, + ]) { + expect(() => chunker.chunkFile(content, 'many.ts', 'typescript', budget)).toThrow('search chunk or memory limit'); + expect(() => chunker.chunkFile(content, 'many.txt', 'text', budget)).toThrow('search chunk or memory limit'); + } + }); + describe('chunkFile', () => { it('returns empty array for empty content', () => { expect(chunker.chunkFile('', 'src/a.ts', 'typescript')).toEqual([]); @@ -98,7 +116,9 @@ describe('createChunker', () => { }); it('sub-splits oversized AST blocks', () => { - const content = `function huge() {\n${Array(4).fill(' ' + 'x'.repeat(MAX_CHUNK_CHARS)).join('\n')}\n}`; + const content = `function huge() {\n${Array(4) + .fill(' ' + 'x'.repeat(MAX_CHUNK_CHARS)) + .join('\n')}\n}`; const chunks = chunker.chunkFile(content, 'huge.ts', 'typescript'); expect(chunks.length).toBeGreaterThan(1); }); @@ -141,7 +161,9 @@ describe('createChunker', () => { it('splits large content across multiple chunks', () => { const line = 'x'.repeat(100); - const content = Array(Math.ceil((MAX_CHUNK_CHARS / 100) * 3)).fill(line).join('\n'); + const content = Array(Math.ceil((MAX_CHUNK_CHARS / 100) * 3)) + .fill(line) + .join('\n'); const chunks = chunker.chunkFile(content, 'big.go', 'go'); expect(chunks.length).toBeGreaterThanOrEqual(2); }); diff --git a/server/search/chunker.ts b/server/search/chunker.ts index 3884bea..7367520 100644 --- a/server/search/chunker.ts +++ b/server/search/chunker.ts @@ -1,26 +1,26 @@ import type { Chunk, Chunker } from './types'; +import { AppError } from '../utils/errors'; import { MAX_CHUNK_TOKENS, AST_SUPPORTED_LANGUAGES } from './constants'; const MAX_CHUNK_CHARS = MAX_CHUNK_TOKENS * 4; // ~4 chars per token export function createChunker(): Chunker { return { - chunkFile(content: string, filePath: string, language: string): Chunk[] { + chunkFile(content, filePath, language, budget): Chunk[] { if (!content || content.trim().length === 0) return []; - - const useAst = AST_SUPPORTED_LANGUAGES.has(language); - const rawChunks = useAst - ? astAwareChunk(content, language) - : slidingWindowChunk(content); - - return rawChunks.map((c, i) => ({ - content: c.content, - startLine: c.startLine, - endLine: c.endLine, - chunkIndex: i, - language, - filePath, - })); + const chunks: Chunk[] = []; + let bytes = 0; + const emit = (chunk: RawChunk) => { + const nextBytes = bytes + chunk.content.length * 2 + (budget?.bytesPerChunk ?? 512); + if (budget && (chunks.length >= budget.maxChunks || nextBytes > budget.maxBytes)) { + throw new AppError(413, 'Repository exceeds the search chunk or memory limit.', 'INDEX_TOO_LARGE'); + } + bytes = nextBytes; + chunks.push({ ...chunk, chunkIndex: chunks.length, language, filePath }); + }; + if (AST_SUPPORTED_LANGUAGES.has(language)) astAwareChunk(content, language, emit); + else slidingWindowChunk(content, emit); + return chunks; }, }; } @@ -33,9 +33,8 @@ interface RawChunk { endLine: number; } -function slidingWindowChunk(content: string): RawChunk[] { +function slidingWindowChunk(content: string, emit: (chunk: RawChunk) => void): void { const lines = content.split('\n'); - const chunks: RawChunk[] = []; let current = ''; let startLine = 1; @@ -50,7 +49,7 @@ function slidingWindowChunk(content: string): RawChunk[] { const before = current.slice(0, breakAt); const after = current.slice(breakAt + 1); const beforeLines = before.split('\n'); - chunks.push({ + emit({ content: before, startLine: startLine, endLine: startLine + beforeLines.length - 1, @@ -60,7 +59,7 @@ function slidingWindowChunk(content: string): RawChunk[] { } else { // Hard split const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine: startLine, endLine: startLine + currentLines.length - 1, @@ -75,14 +74,12 @@ function slidingWindowChunk(content: string): RawChunk[] { if (current.length > 0) { const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine, endLine: startLine + currentLines.length - 1, }); } - - return chunks.length > 0 ? chunks : [{ content, startLine: 1, endLine: lines.length }]; } function findLastBlankLineBreak(text: string): number { @@ -106,15 +103,15 @@ function findLastBlankLineBreak(text: string): number { * Lightweight regex-based chunking that splits at function/class boundaries * for TypeScript, JavaScript, and Python files. */ -function astAwareChunk(content: string, language: string): RawChunk[] { +function astAwareChunk(content: string, language: string, emit: (chunk: RawChunk) => void): void { const lines = content.split('\n'); const boundaries = findBoundaries(lines, language); if (boundaries.length === 0) { - return slidingWindowChunk(content); + slidingWindowChunk(content, emit); + return; } - const chunks: RawChunk[] = []; let prevEnd = 0; for (const boundary of boundaries) { @@ -122,21 +119,21 @@ function astAwareChunk(content: string, language: string): RawChunk[] { if (boundary.startLine > prevEnd) { const before = lines.slice(prevEnd, boundary.startLine).join('\n'); if (before.trim().length > 0) { - addSizedChunks(chunks, before, prevEnd + 1); + addSizedChunks(emit, before, prevEnd + 1); } } // Add the boundary block (function/class) const block = lines.slice(boundary.startLine, boundary.endLine + 1).join('\n'); if (block.length <= MAX_CHUNK_CHARS) { - chunks.push({ + emit({ content: block, startLine: boundary.startLine + 1, endLine: boundary.endLine + 1, }); } else { // Sub-split large blocks - addSizedChunks(chunks, block, boundary.startLine + 1); + addSizedChunks(emit, block, boundary.startLine + 1); } prevEnd = boundary.endLine + 1; @@ -146,11 +143,9 @@ function astAwareChunk(content: string, language: string): RawChunk[] { if (prevEnd < lines.length) { const remaining = lines.slice(prevEnd).join('\n'); if (remaining.trim().length > 0) { - addSizedChunks(chunks, remaining, prevEnd + 1); + addSizedChunks(emit, remaining, prevEnd + 1); } } - - return chunks.length > 0 ? chunks : slidingWindowChunk(content); } interface Boundary { @@ -168,6 +163,7 @@ function findBoundaries(lines: string[], language: string): Boundary[] { if (pattern.test(trimmed)) { const end = findBlockEnd(lines, i, language); boundaries.push({ startLine: i, endLine: end }); + i = end; // Nested blocks are already included in the outer block. break; } } @@ -187,10 +183,7 @@ const JS_PATTERNS: RegExp[] = [ /^(export\s+)?abstract\s+class\s+/, ]; -const PY_PATTERNS: RegExp[] = [ - /^(async\s+)?def\s+/, - /^class\s+/, -]; +const PY_PATTERNS: RegExp[] = [/^(async\s+)?def\s+/, /^class\s+/]; const PATTERN_MAP: Record = { typescript: JS_PATTERNS, @@ -248,7 +241,7 @@ function findPythonBlockEnd(lines: string[], startLine: number): number { return end; } -function addSizedChunks(chunks: RawChunk[], content: string, baseLine: number): void { +function addSizedChunks(emit: (chunk: RawChunk) => void, content: string, baseLine: number): void { const lines = content.split('\n'); let current = ''; let startLine = baseLine; @@ -259,7 +252,7 @@ function addSizedChunks(chunks: RawChunk[], content: string, baseLine: number): if (test.length > MAX_CHUNK_CHARS && current.length > 0) { const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine, endLine: startLine + currentLines.length - 1, @@ -273,7 +266,7 @@ function addSizedChunks(chunks: RawChunk[], content: string, baseLine: number): if (current.length > 0) { const currentLines = current.split('\n'); - chunks.push({ + emit({ content: current, startLine, endLine: startLine + currentLines.length - 1, diff --git a/server/search/embedding-provider.ts b/server/search/embedding-provider.ts index 928b2e3..b770527 100644 --- a/server/search/embedding-provider.ts +++ b/server/search/embedding-provider.ts @@ -1,6 +1,8 @@ import type { EmbeddingProvider, EmbeddingResult } from './types'; import { EMBEDDING_DIMENSIONS } from './constants'; import { AppError } from '../utils/errors'; +import { hashToken } from '../cache/lru'; +import { embeddingIdentity } from './index-identity'; // ── Factory ──────────────────────────────────────────────────────────── @@ -9,7 +11,16 @@ let cachedProviderKey: string | null = null; export async function createEmbeddingProvider(): Promise { const envProvider = (process.env.AI_PROVIDER ?? 'mock').toLowerCase(); - const cacheKey = `${envProvider}:${process.env.GEMINI_API_KEY ? 'g' : ''}${process.env.OPENAI_API_KEY ? 'o' : ''}${process.env.EDGEONE_API_KEY || process.env.MAKERS_MODELS_KEY ? 'e' : ''}`; + const cacheKey = + embeddingIdentity() + + hashToken( + JSON.stringify([ + process.env.OPENAI_API_KEY, + process.env.GEMINI_API_KEY, + process.env.EDGEONE_API_KEY, + process.env.MAKERS_MODELS_KEY, + ]), + ); if (cachedProvider && cachedProviderKey === cacheKey) { return cachedProvider; @@ -150,7 +161,10 @@ function createEdgeOneEmbeddingProvider(): EmbeddingProvider { providerName: 'edgeone', apiKey, baseURL: process.env.EDGEONE_API_BASE ?? 'https://ai-gateway.edgeone.link/v1', - model: process.env.EDGEONE_EMBEDDING_MODEL ?? process.env.OPENAI_EMBEDDING_MODEL ?? 'openrouter/openai/text-embedding-3-large', + model: + process.env.EDGEONE_EMBEDDING_MODEL ?? + process.env.OPENAI_EMBEDDING_MODEL ?? + 'openrouter/openai/text-embedding-3-large', }); } @@ -161,6 +175,7 @@ async function openAIEmbed(apiKey: string, model: string, text: string, baseURL? const response = await withRetry(() => client.embeddings.create({ model, + dimensions: EMBEDDING_DIMENSIONS, input: text, }), ); @@ -168,7 +183,12 @@ async function openAIEmbed(apiKey: string, model: string, text: string, baseURL? return normalizeVector(new Float32Array(response.data[0].embedding)); } -async function openAIEmbedBatch(apiKey: string, model: string, texts: string[], baseURL?: string): Promise { +async function openAIEmbedBatch( + apiKey: string, + model: string, + texts: string[], + baseURL?: string, +): Promise { if (texts.length === 0) return []; const { default: OpenAI } = await import('openai'); const client = new OpenAI({ apiKey, baseURL: baseURL ?? process.env.OPENAI_API_BASE }); @@ -180,6 +200,7 @@ async function openAIEmbedBatch(apiKey: string, model: string, texts: string[], const response = await withRetry(() => client.embeddings.create({ model, + dimensions: EMBEDDING_DIMENSIONS, input: batch, }), ); diff --git a/server/search/index-identity.test.ts b/server/search/index-identity.test.ts new file mode 100644 index 0000000..2c92093 --- /dev/null +++ b/server/search/index-identity.test.ts @@ -0,0 +1,18 @@ +import { test, expect } from 'bun:test'; +import { searchIndexKey } from './index-identity'; + +test('index identity isolates credentials, snapshots and embedding models without exposing tokens', () => { + const key = searchIndexKey('Org', 'Repo', 'sha', { gitHubToken: 'private-token' }); + expect(key).not.toContain('private-token'); + expect(key).toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'private-token' })); + expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha2', { gitHubToken: 'private-token' })); + expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'other' })); + const original = process.env.OPENAI_EMBEDDING_MODEL; + try { + process.env.OPENAI_EMBEDDING_MODEL = 'different-model'; + expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'private-token' })); + } finally { + if (original === undefined) delete process.env.OPENAI_EMBEDDING_MODEL; + else process.env.OPENAI_EMBEDDING_MODEL = original; + } +}); diff --git a/server/search/index-identity.ts b/server/search/index-identity.ts new file mode 100644 index 0000000..f9ba43a --- /dev/null +++ b/server/search/index-identity.ts @@ -0,0 +1,30 @@ +import { hashToken, hashContext } from '../cache/lru'; +import type { RequestContext } from '../utils/context'; + +/** Include model and endpoint configuration so incompatible embeddings never mix. */ +export function embeddingIdentity(): string { + return hashContext( + JSON.stringify([ + process.env.AI_PROVIDER ?? 'mock', + process.env.EMBEDDING_DIMENSIONS ?? '3072', + process.env.OPENAI_EMBEDDING_MODEL, + process.env.GEMINI_EMBEDDING_MODEL, + process.env.EDGEONE_EMBEDDING_MODEL, + process.env.OPENAI_API_BASE, + process.env.EDGEONE_API_BASE, + !!process.env.GEMINI_API_KEY, + !!process.env.OPENAI_API_KEY, + !!(process.env.EDGEONE_API_KEY || process.env.MAKERS_MODELS_KEY), + ]), + ); +} + +export function searchIndexKey( + owner: string, + repo: string, + sha: string, + ctx?: Pick, +): string { + const scope = hashToken(ctx?.gitHubToken || process.env.GITHUB_TOKEN || 'anonymous'); + return `${owner.toLowerCase()}/${repo.toLowerCase()}@${sha}:${scope}:${embeddingIdentity()}`; +} diff --git a/server/search/indexing-pipeline.test.ts b/server/search/indexing-pipeline.test.ts index cea67aa..bb0fe5e 100644 --- a/server/search/indexing-pipeline.test.ts +++ b/server/search/indexing-pipeline.test.ts @@ -1,295 +1,142 @@ -import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; -import * as lruModule from '../cache/lru'; -import * as loggerModule from '../utils/logger'; - -const realLoggerExports = { ...loggerModule }; - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - if (a.length !== b.length) return 0; - let dot = 0, magA = 0, magB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - magA += a[i] * a[i]; - magB += b[i] * b[i]; - } - const denom = Math.sqrt(magA) * Math.sqrt(magB); - if (denom === 0) return 0; - return Math.max(0, Math.min(1, dot / denom)); -} - -const vectorStore = { - chunks: [] as any[], - hasIndexValue: false, - addChunks: mock((chunks: any[]) => vectorStore.chunks.push(...chunks)), - removeByRepo: mock(() => { - vectorStore.chunks = []; - }), - removeByFile: mock((repoKey: string, filePath: string) => { - vectorStore.chunks = vectorStore.chunks.filter((chunk) => chunk.metadata.repoKey !== repoKey || chunk.metadata.filePath !== filePath); - }), - getChunkCount: mock((repoKey: string) => vectorStore.chunks.filter((chunk) => chunk.metadata.repoKey === repoKey).length), - hasIndex: mock(() => vectorStore.hasIndexValue || vectorStore.chunks.length > 0), - search: mock((queryVector: Float32Array, repoKey: string, topK: number, minScore: number) => { - return vectorStore.chunks - .filter((c) => c.metadata.repoKey === repoKey) - .map((c) => ({ chunk: c.metadata, score: cosineSimilarity(queryVector, c.vector) })) - .filter((r) => r.score >= minScore) - .sort((a, b) => b.score - a.score) - .slice(0, topK); - }), -}; - -let repoInfoSha = 'sha-current'; -let flatTrees: Record = {}; -let fileContents: Record = {}; -let fetchFileContentsError = false; -let embedBatchError = false; -let chunkFileResult: any[] | null = null; - -const fetchRepoInfo = mock(async () => ({ sha: repoInfoSha })); -const fetchFlatTree = mock(async (_owner: string, _repo: string, sha: string) => ({ items: flatTrees[sha] ?? [] })); -const fetchFileContents = mock(async (_owner: string, _repo: string, paths: string[]) => { - if (fetchFileContentsError) throw new Error('fetch failed'); - return Object.fromEntries(paths.map((path) => [path, fileContents[path]])); -}); -const embedBatch = mock(async (texts: string[]) => { - if (embedBatchError) throw new Error('embed failed'); - return texts.map(() => ({ vector: new Float32Array([1, 0]), tokenCount: 1 })); -}); -const chunkFile = mock((content: string, filePath: string, language: string) => chunkFileResult ?? [{ - content, - filePath, - language, - startLine: 1, - endLine: 1, - chunkIndex: 0, -}]); -const invalidateSearchCache = mock(() => undefined); -const logError = mock(() => undefined); - -const { createIndexingPipeline, getIndexingPipeline } = await import('./indexing-pipeline'); -const { AppError } = await import('../utils/errors'); - -const options = { owner: 'o', repo: 'r', commitSha: 'sha-current' }; -const key = 'o/r'; - -function setupModuleMocks() { - mock.module('./vector-store', () => ({ getVectorStore: () => vectorStore })); - mock.module('../github/fetch-tree', () => ({ fetchRepoInfo, fetchFlatTree, fetchFileContents })); - mock.module('./embedding-provider', () => ({ - createEmbeddingProvider: async () => ({ - embed: async (_text: string) => ({ vector: new Float32Array([1, 0]), tokenCount: 1 }), - embedBatch, - }), - })); - mock.module('./chunker', () => ({ createChunker: () => ({ chunkFile }) })); - mock.module('../cache/lru', () => ({ ...lruModule, invalidateSearchCache })); - mock.module('../utils/logger', () => ({ ...loggerModule, logError })); -} - -beforeEach(() => { - mock.restore(); - setupModuleMocks(); - for (const fn of [ - vectorStore.addChunks, - vectorStore.removeByRepo, - vectorStore.removeByFile, - vectorStore.getChunkCount, - vectorStore.hasIndex, - vectorStore.search, - fetchRepoInfo, - fetchFlatTree, - fetchFileContents, +import { test, expect, mock, spyOn } from 'bun:test'; +import { Octokit } from '@octokit/rest'; +import { createIndexingPipeline } from './indexing-pipeline'; +import { createVectorStore } from './vector-store'; +import { createChunker } from './chunker'; +import { searchIndexKey } from './index-identity'; +import { SEARCH_LIMITS } from './limits'; +import type { EmbeddingProvider } from './types'; +const ctx = { octokit: new Octokit(), gitHubToken: 'test-token' }; +const options = { owner: 'owner', repo: 'repo', commitSha: 'first' }; +function setup(overrides: Partial = {}) { + const limits = { ...SEARCH_LIMITS, ...overrides }; + const store = createVectorStore(limits); + const embedBatch = mock(async (texts: string[]) => + texts.map(() => ({ vector: new Float32Array([1, 0]), tokenCount: 1 })), + ); + const provider: EmbeddingProvider = { + dimensions: 2, + maxTokens: 512, + providerName: 'test', embedBatch, - chunkFile, - invalidateSearchCache, - logError, - ]) { - fn.mockClear(); - } - vectorStore.chunks = []; - vectorStore.hasIndexValue = false; - repoInfoSha = 'sha-current'; - flatTrees = { - 'sha-current': [ - { path: 'src/a.ts', type: 'blob', sha: 'a1' }, - { path: 'README.md', type: 'blob', sha: 'readme' }, - { path: 'image.png', type: 'blob', sha: 'png' }, - ], - 'sha-prev': [ - { path: 'src/a.ts', type: 'blob', sha: 'old-a' }, - { path: 'src/deleted.ts', type: 'blob', sha: 'deleted' }, - ], + embed: async () => ({ vector: new Float32Array([1, 0]), tokenCount: 1 }), }; - fileContents = { - 'src/a.ts': 'export const a = 1;', - 'README.md': '# readme', - }; - fetchFileContentsError = false; - embedBatchError = false; - chunkFileResult = null; -}); - -afterEach(() => { - mock.restore(); - mock.module('../utils/logger', () => realLoggerExports); + const fetchFlatTree = mock(async () => ({ + items: [{ path: 'a.ts', type: 'blob' as const, sha: 'blob', size: 20 }], + truncated: false, + totalFiles: 1, + })); + const fetchFileContents = mock(async () => ({ 'a.ts': 'export const value = 1;' })); + const pipeline = createIndexingPipeline( + { + fetchFlatTree, + fetchFileContents, + createChunker, + getVectorStore: () => store, + createEmbeddingProvider: async () => provider, + }, + limits, + ); + return { store, pipeline, embedBatch, fetchFlatTree, fetchFileContents }; +} +test('same snapshot is reused; changed SHA and credential build separate indices', async () => { + const s = setup(); + await s.pipeline.startIndexing(options, ctx); + await s.pipeline.startIndexing(options, ctx); + expect(s.embedBatch).toHaveBeenCalledTimes(1); + await s.pipeline.startIndexing({ ...options, commitSha: 'second' }, ctx); + await s.pipeline.startIndexing(options, { ...ctx, gitHubToken: 'other' }); + expect(s.embedBatch).toHaveBeenCalledTimes(3); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'first', ctx))).toBe(true); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'second', ctx))).toBe(true); }); - -describe('indexing pipeline', () => { - it('returns idle status and can cancel to idle', () => { - const pipeline = createIndexingPipeline(); - - expect(pipeline.getStatus(key)).toEqual({ state: 'idle' }); - pipeline.cancelIndexing(key); - expect(pipeline.getStatus(key)).toEqual({ state: 'idle' }); - }); - - it('indexes supported files and stores chunks', async () => { - const pipeline = createIndexingPipeline(); - - await pipeline.startIndexing(options, {} as any); - - expect(fetchRepoInfo).toHaveBeenCalledWith('o', 'r', undefined, {}); - expect(fetchFlatTree).toHaveBeenCalledWith('o', 'r', 'sha-current', {}); - expect(vectorStore.removeByRepo).toHaveBeenCalledWith(key); - expect(fetchFileContents).toHaveBeenCalledWith('o', 'r', ['src/a.ts', 'README.md'], 'sha-current', {}); - expect(vectorStore.addChunks).toHaveBeenCalledTimes(1); - expect(vectorStore.chunks.map((chunk) => chunk.metadata.filePath)).toEqual(['src/a.ts', 'README.md']); - expect(invalidateSearchCache).toHaveBeenCalledWith('o', 'r'); - expect(pipeline.getStatus(key).state).toBe('complete'); - }); - - it('uses repo info sha when commitSha is empty', async () => { - const pipeline = createIndexingPipeline(); - repoInfoSha = 'resolved-sha'; - flatTrees['resolved-sha'] = flatTrees['sha-current']; - - await pipeline.startIndexing({ ...options, commitSha: '' }, {} as any); - - expect(fetchFlatTree).toHaveBeenCalledWith('o', 'r', 'resolved-sha', {}); - }); - - it('skips indexing when repo already has chunks and no previousSha', async () => { - const pipeline = createIndexingPipeline(); - vectorStore.hasIndexValue = true; - vectorStore.chunks = [{ metadata: { repoKey: key } }]; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchRepoInfo).not.toHaveBeenCalled(); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'complete', chunkCount: 1 }); - }); - - it('still reindexes when index exists but chunk count is zero', async () => { - const pipeline = createIndexingPipeline(); - vectorStore.hasIndexValue = true; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchRepoInfo).toHaveBeenCalled(); - }); - - it('does incremental indexing and removes deleted files', async () => { - const pipeline = createIndexingPipeline(); - vectorStore.hasIndexValue = true; - vectorStore.chunks = [{ metadata: { repoKey: key, filePath: 'src/deleted.ts' } }]; - - await pipeline.startIndexing({ ...options, previousSha: 'sha-prev' }, {} as any); - - expect(fetchFlatTree).toHaveBeenCalledWith('o', 'r', 'sha-prev', {}); - expect(vectorStore.removeByFile).toHaveBeenCalledWith(key, 'src/deleted.ts'); - expect(fetchFileContents).toHaveBeenCalledWith('o', 'r', ['src/a.ts', 'README.md'], 'sha-current', {}); - }); - - it('handles no supported files', async () => { - const pipeline = createIndexingPipeline(); - flatTrees['sha-current'] = [{ path: 'asset.png', type: 'blob', sha: 'png' }]; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchFileContents).not.toHaveBeenCalled(); - expect(vectorStore.addChunks).toHaveBeenCalledWith([]); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'complete', chunkCount: 0 }); - }); - - it('fetches file contents in batches of at most five paths', async () => { - const pipeline = createIndexingPipeline(); - flatTrees['sha-current'] = Array.from({ length: 12 }, (_, i) => ({ path: `src/file-${i}.ts`, type: 'blob', sha: `sha-${i}` })); - fileContents = Object.fromEntries(flatTrees['sha-current'].map((item) => [item.path, `export const n = ${item.sha};`])) as Record; - - await pipeline.startIndexing(options, {} as any); - - expect(fetchFileContents.mock.calls.map((call) => call[2].length)).toEqual([5, 5, 2]); +test('failed replacement throws and preserves the published old snapshot', async () => { + const s = setup(); + await s.pipeline.startIndexing(options, ctx); + s.embedBatch.mockImplementation(async () => { + throw new Error('secret provider detail'); }); - - it('counts missing content as failed and aborts over threshold', async () => { - const pipeline = createIndexingPipeline(); - fileContents = {}; - - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', failedFiles: 2 }); - expect(vectorStore.addChunks).not.toHaveBeenCalled(); - }); - - it('skips files with no chunks', async () => { - const pipeline = createIndexingPipeline(); - chunkFileResult = []; - - await pipeline.startIndexing(options, {} as any); - - expect(embedBatch).not.toHaveBeenCalled(); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'complete', chunkCount: 0 }); - }); - - it('counts fetch content failures and aborts over threshold', async () => { - const pipeline = createIndexingPipeline(); - fetchFileContentsError = true; - - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', failedFiles: 2 }); - }); - - it('counts embedding failures and aborts over threshold', async () => { - const pipeline = createIndexingPipeline(); - embedBatchError = true; - - await pipeline.startIndexing(options, {} as any); - - expect(logError).toHaveBeenCalledWith('indexing:embed', expect.any(Error), { file: expect.any(String) }); - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', failedFiles: 2 }); - }); - - it('records failed status when setup throws', async () => { - const pipeline = createIndexingPipeline(); - fetchRepoInfo.mockImplementationOnce(async () => { - throw new Error('boom'); + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + await expect(s.pipeline.startIndexing({ ...options, commitSha: 'new' }, ctx)).rejects.toMatchObject({ + status: 502, + }); + expect(s.store.hasIndex(searchIndexKey('owner', 'repo', 'first', ctx))).toBe(true); + const key = searchIndexKey('owner', 'repo', 'new', ctx); + expect(s.store.hasIndex(key)).toBe(false); + expect(s.pipeline.getStatus(key)).toMatchObject({ + state: 'failed', + error: 'Repository indexing failed. Please retry.', }); - - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key)).toMatchObject({ state: 'failed', error: 'boom', failedFiles: 0 }); - expect(logError).toHaveBeenCalledWith('/api/search/index', expect.any(Error), { repo: key }); + } finally { + log.mockRestore(); + } +}); +test('global job cap rejects other repositories and duplicate jobs without queuing', async () => { + const s = setup({ concurrentJobs: 1 }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); - - it('rejects concurrent indexing and then clears active flag', async () => { - const pipeline = createIndexingPipeline(); - let release!: () => void; - fetchRepoInfo.mockImplementationOnce(async () => new Promise((resolve) => { - release = () => resolve({ sha: 'sha-current' }); - })); - - const first = pipeline.startIndexing(options, {} as any); - await expect(pipeline.startIndexing(options, {} as any)).rejects.toBeInstanceOf(AppError); + s.fetchFileContents.mockImplementation(async () => { + await gate; + return { 'a.ts': 'const a = 1;' }; + }); + const first = s.pipeline.startIndexing(options, ctx); + try { + await expect(s.pipeline.startIndexing(options, ctx)).rejects.toMatchObject({ status: 409 }); + await expect(s.pipeline.startIndexing({ ...options, repo: 'other' }, ctx)).rejects.toMatchObject({ status: 429 }); + } finally { release(); await first; - await pipeline.startIndexing(options, {} as any); - - expect(pipeline.getStatus(key).state).toBe('complete'); + } +}); +test('file and chunk limits reject before embedding', async () => { + for (const limits of [{ fileBytes: 10 }, { chunks: 0 }, { indexBytes: 1 }]) { + const s = setup(limits); + const log = spyOn(console, 'error').mockImplementation(() => {}); + try { + await expect(s.pipeline.startIndexing(options, ctx)).rejects.toMatchObject({ status: 413 }); + expect(s.embedBatch).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + } +}); +test('empty repository is an actual completed index', async () => { + const s = setup(); + s.fetchFlatTree.mockResolvedValue({ items: [], truncated: false, totalFiles: 0 }); + await s.pipeline.startIndexing(options, ctx); + expect(s.pipeline.getStatus(searchIndexKey('owner', 'repo', 'first', ctx))).toMatchObject({ + state: 'complete', + chunkCount: 0, }); - - it('returns singleton pipeline', () => { - expect(getIndexingPipeline()).toBe(getIndexingPipeline()); +}); +test('status becomes idle when its index is evicted', async () => { + const s = setup({ indices: 1 }); + await s.pipeline.startIndexing(options, ctx); + await s.pipeline.startIndexing({ ...options, commitSha: 'new' }, ctx); + expect(s.pipeline.getStatus(searchIndexKey('owner', 'repo', 'first', ctx))).toEqual({ state: 'idle' }); +}); +test('cancellation keeps the concurrency slot until the pending operation settles', async () => { + const s = setup({ concurrentJobs: 1 }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); -}); \ No newline at end of file + s.fetchFileContents.mockImplementation(async () => { + await gate; + return { 'a.ts': 'const a = 1;' }; + }); + const log = spyOn(console, 'error').mockImplementation(() => {}); + const work = s.pipeline.startIndexing(options, ctx); + const rejection = work.catch((error) => error); + s.pipeline.cancelIndexing(searchIndexKey('owner', 'repo', 'first', ctx)); + try { + await expect(s.pipeline.startIndexing({ ...options, repo: 'other' }, ctx)).rejects.toMatchObject({ status: 429 }); + } finally { + release(); + expect(await rejection).toMatchObject({ code: 'INDEXING_CANCELLED' }); + log.mockRestore(); + } +}); diff --git a/server/search/indexing-pipeline.ts b/server/search/indexing-pipeline.ts index 1eadc41..94a9cb2 100644 --- a/server/search/indexing-pipeline.ts +++ b/server/search/indexing-pipeline.ts @@ -1,233 +1,158 @@ -import type { IndexingOptions, IndexingStatus, IndexingPipeline, RequestContext, IndexedChunk } from './types'; +import type { IndexingStatus, IndexingPipeline, IndexedChunk } from './types'; import { SUPPORTED_EXTENSIONS, extToLanguage } from './constants'; import { createEmbeddingProvider } from './embedding-provider'; import { createChunker } from './chunker'; -import { getVectorStore } from './vector-store'; -import { fetchFlatTree, fetchFileContents, fetchRepoInfo } from '../github/fetch-tree'; -import { invalidateSearchCache } from '../cache/lru'; +import { getVectorStore, chunkBytes } from './vector-store'; +import { fetchFlatTree, fetchFileContents } from '../github/fetch-tree'; import { AppError } from '../utils/errors'; import { logError } from '../utils/logger'; - -function repoKey(owner: string, repo: string): string { - return `${owner}/${repo}`; -} - -/** Tracks per-repo indexing status. */ -const statusMap = new Map(); -const activeIndexing = new Set(); // prevents concurrent indexing - -export function createIndexingPipeline(): IndexingPipeline { +import { searchIndexKey } from './index-identity'; +import { SEARCH_LIMITS } from './limits'; + +const defaults = { fetchFlatTree, fetchFileContents, createEmbeddingProvider, createChunker, getVectorStore }; +export function createIndexingPipeline(deps = defaults, limits = SEARCH_LIMITS, now = Date.now): IndexingPipeline { + const statuses = new Map(); + const active = new Map(); + const save = (key: string, value: IndexingStatus) => { + for (const [id, entry] of statuses) if (entry.expires <= now() && !active.has(id)) statuses.delete(id); + statuses.delete(key); + while (statuses.size >= limits.statuses) { + const oldest = [...statuses.keys()].find((id) => !active.has(id)); + if (!oldest) break; + statuses.delete(oldest); + } + statuses.set(key, { value, expires: now() + limits.ttlMs }); + }; return { - async startIndexing(options: IndexingOptions, ctx: RequestContext): Promise { - const key = repoKey(options.owner, options.repo); - - // Prevent concurrent indexing - if (activeIndexing.has(key)) { - throw new AppError(409, 'Indexing already in progress for this repository.', 'INDEXING_IN_PROGRESS'); - } - - activeIndexing.add(key); - statusMap.set(key, { state: 'indexing', progress: 0, filesProcessed: 0, totalFiles: 0 }); - + async startIndexing(options, ctx) { + const { owner, repo, commitSha } = options; + const key = searchIndexKey(owner, repo, commitSha, ctx); + if (active.has(key)) throw new AppError(409, 'Indexing is already in progress.', 'INDEXING_IN_PROGRESS'); + if (active.size >= limits.concurrentJobs) + throw new AppError(429, 'Search indexing is busy. Please retry later.', 'INDEXING_BUSY', true); + const job = { cancelled: false }; + active.set(key, job); + const check = () => { + if (job.cancelled) throw new AppError(409, 'Indexing was cancelled.', 'INDEXING_CANCELLED'); + }; + save(key, { state: 'indexing', progress: 0, filesProcessed: 0, totalFiles: 0 }); try { - await runIndexing(options, ctx, key); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - statusMap.set(key, { state: 'failed', error: msg, failedFiles: 0 }); - logError('/api/search/index', err, { repo: key }); + const store = deps.getVectorStore(); + if (store.hasIndex(key)) { + save(key, { state: 'complete', chunkCount: store.getChunkCount(key), timestamp: now() }); + return; + } + const tree = await deps.fetchFlatTree(owner, repo, commitSha, ctx); + check(); + const files = tree.items.filter((item) => SUPPORTED_EXTENSIONS.has(extension(item.path))); + if (tree.truncated || files.length > limits.files || files.some((file) => (file.size ?? 0) > limits.fileBytes)) + throw new AppError(413, 'Repository exceeds search indexing file limits.', 'INDEX_TOO_LARGE'); + const provider = await deps.createEmbeddingProvider(); + const chunker = deps.createChunker(); + const indexed: IndexedChunk[] = []; + let bytes = 0; + let processed = 0; + // Fetch one bounded file at a time; never accumulate the repository's source text. + for (const file of files) { + check(); + const contents = await deps.fetchFileContents(owner, repo, [file.path], commitSha, ctx, limits.fileBytes); + check(); + const content = contents[file.path]; + if (content === undefined) + throw new AppError(502, 'A repository file could not be read.', 'INDEXING_FAILED', true); + if (Buffer.byteLength(content, 'utf8') > limits.fileBytes) + throw new AppError(413, 'A file exceeds the search indexing size limit.', 'INDEX_TOO_LARGE'); + const chunks = chunker.chunkFile(content, file.path, extToLanguage(extension(file.path)), { + maxChunks: limits.chunks - indexed.length, + maxBytes: limits.indexBytes - bytes, + bytesPerChunk: provider.dimensions * 4 + 4 * (key.length + file.path.length) + 1024, + }); + if (indexed.length + chunks.length > limits.chunks) + throw new AppError(413, 'Repository exceeds the search chunk limit.', 'INDEX_TOO_LARGE'); + // Reserve conservatively before paying for embeddings or retaining their vectors. + const estimated = chunks.reduce( + (sum, chunk) => + sum + provider.dimensions * 4 + 2 * (chunk.content.length + key.length * 2 + file.path.length * 2) + 1024, + 0, + ); + if (bytes + estimated > limits.indexBytes) + throw new AppError(413, 'Repository exceeds the search index memory limit.', 'INDEX_TOO_LARGE'); + for (let offset = 0; offset < chunks.length; offset += 32) { + check(); + const batch = chunks.slice(offset, offset + 32); + const embeddings = await provider.embedBatch(batch.map((chunk) => chunk.content)); + check(); + if (embeddings.length !== batch.length) + throw new AppError(502, 'Embedding provider returned an incomplete batch.', 'INDEXING_FAILED', true); + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + const vector = embeddings[i].vector; + if (vector.length !== provider.dimensions || !vector.every(Number.isFinite)) + throw new AppError(502, 'Embedding provider returned an invalid vector.', 'INDEXING_FAILED', true); + const entry: IndexedChunk = { + id: key + ':' + file.path + ':' + chunk.chunkIndex, + vector, + metadata: { + filePath: file.path, + startLine: chunk.startLine, + endLine: chunk.endLine, + chunkIndex: chunk.chunkIndex, + language: chunk.language, + content: chunk.content.slice(0, 2000), + repoKey: key, + commitSha, + }, + }; + bytes += chunkBytes(entry); + if (bytes > limits.indexBytes) + throw new AppError(413, 'Search index memory limit exceeded.', 'INDEX_TOO_LARGE'); + indexed.push(entry); + } + } + processed++; + save(key, { + state: 'indexing', + progress: Math.round((processed / files.length) * 100), + filesProcessed: processed, + totalFiles: files.length, + }); + } + check(); + store.replaceIndex(key, indexed); + save(key, { state: 'complete', chunkCount: indexed.length, timestamp: now() }); + } catch (error) { + const failure = + error instanceof AppError + ? error + : new AppError(502, 'Repository indexing failed. Please retry.', 'INDEXING_FAILED', true); + save(key, { state: 'failed', error: failure.message, failedFiles: 0 }); + logError('/api/search/index', error, { repo: owner + '/' + repo }); + throw failure; } finally { - activeIndexing.delete(key); + active.delete(key); } }, - - getStatus(key: string): IndexingStatus { - return statusMap.get(key) ?? { state: 'idle' }; + getStatus(key) { + const entry = statuses.get(key); + if (entry && (active.has(key) || entry.expires > now())) { + if (entry.value.state !== 'complete' || deps.getVectorStore().hasIndex(key)) return entry.value; + } + statuses.delete(key); + if (deps.getVectorStore().hasIndex(key)) + return { state: 'complete', chunkCount: deps.getVectorStore().getChunkCount(key), timestamp: now() }; + return { state: 'idle' }; }, - - cancelIndexing(key: string): void { - activeIndexing.delete(key); - statusMap.set(key, { state: 'idle' }); + cancelIndexing(key) { + const job = active.get(key); + if (job) job.cancelled = true; // Keep the slot until the awaited operation settles. }, }; } - -async function runIndexing(options: IndexingOptions, ctx: RequestContext, key: string): Promise { - const { owner, repo, commitSha, previousSha } = options; - const vectorStore = getVectorStore(); - - // Idempotent: if same SHA already indexed, skip - if (vectorStore.hasIndex(key) && !previousSha) { - // Check if existing index matches this SHA – we can't easily verify, so re-index only if not present - const existingCount = vectorStore.getChunkCount(key); - if (existingCount > 0) { - statusMap.set(key, { state: 'complete', chunkCount: existingCount, timestamp: Date.now() }); - return; - } - } - - // 1. Fetch the file tree first so we can report totalFiles early - const info = await fetchRepoInfo(owner, repo, options.branch, ctx); - const targetSha = commitSha || info.sha; - const { items } = await fetchFlatTree(owner, repo, targetSha, ctx); - - // 2. Filter to supported extensions - const sourceFiles = items.filter((item) => { - const ext = getExtension(item.path); - return SUPPORTED_EXTENSIONS.has(ext); - }); - - const totalFiles = sourceFiles.length; - statusMap.set(key, { state: 'indexing', progress: 0, filesProcessed: 0, totalFiles }); - - // 3. Create embedding provider (may fail on proxy issues) - const provider = await createEmbeddingProvider(); - const chunker = createChunker(); - - // 4. Determine files to process (incremental: only changed files) - let filesToProcess = sourceFiles; - - if (previousSha && vectorStore.hasIndex(key)) { - const { items: prevItems } = await fetchFlatTree(owner, repo, previousSha, ctx); - const prevFiles = new Set( - prevItems.filter((i) => SUPPORTED_EXTENSIONS.has(getExtension(i.path))).map((i) => i.path), - ); - const currFiles = new Set(sourceFiles.map((i) => i.path)); - - // Files in current but not prev, or SHA changed → treat as modified - // Files in prev but not current → delete - const filesToDelete = [...prevFiles].filter((f) => !currFiles.has(f)); - const prevItemsMap = new Map(prevItems.map((p) => [p.path, p])); - const modifiedOrAdded = sourceFiles.filter((item) => { - const prevItem = prevItemsMap.get(item.path); - return !prevItem || prevItem.sha !== item.sha; // SHA changed or new file - }); - filesToProcess = modifiedOrAdded; - - // Remove deleted files from store - for (const filePath of filesToDelete) { - vectorStore.removeByFile(key, filePath); - } - } else { - // Full re-index: clear existing - vectorStore.removeByRepo(key); - } - - const BATCH_SIZE = 5; - let filesProcessed = 0; - let failedChunks = 0; - let totalChunks = 0; - const allIndexedChunks: IndexedChunk[] = []; - - const batches: (typeof filesToProcess)[] = []; - for (let i = 0; i < filesToProcess.length; i += BATCH_SIZE) { - batches.push(filesToProcess.slice(i, i + BATCH_SIZE)); - } - - for (const batch of batches) { - const paths = batch.map((b) => b.path); - - let contents: Record; - try { - contents = await fetchFileContents(owner, repo, paths, targetSha, ctx); - } catch { - failedChunks += batch.length; - filesProcessed += batch.length; - continue; - } - - for (const item of batch) { - const content = contents[item.path]; - if (!content) { - failedChunks++; - filesProcessed++; - continue; - } - - const ext = getExtension(item.path); - const language = extToLanguage(ext); - const chunks = chunker.chunkFile(content, item.path, language); - - if (chunks.length === 0) { - filesProcessed++; - continue; - } - - totalChunks += chunks.length; - - try { - const chunkContents = new Array(chunks.length); - for (let j = 0; j < chunks.length; j++) { - chunkContents[j] = chunks[j].content; - } - const embeddings = await provider.embedBatch(chunkContents); - - for (let j = 0; j < chunks.length; j++) { - const chunk = chunks[j]; - const embedding = embeddings[j]; - allIndexedChunks.push({ - id: `${key}:${item.path}:${chunk.chunkIndex}`, - vector: embedding.vector, - metadata: { - filePath: item.path, - startLine: chunk.startLine, - endLine: chunk.endLine, - chunkIndex: chunk.chunkIndex, - language: chunk.language, - content: chunk.content.slice(0, 2000), - repoKey: key, - commitSha: targetSha, - }, - }); - } - } catch (err) { - failedChunks++; - logError('indexing:embed', err, { file: item.path }); - } - - filesProcessed++; - } - - const progress = totalFiles > 0 ? Math.round((filesProcessed / totalFiles) * 100) : 100; - statusMap.set(key, { state: 'indexing', progress, filesProcessed, totalFiles }); - } - - // 6. Check failure threshold - const totalAttempted = totalChunks + failedChunks; - if (totalAttempted > 0 && failedChunks / totalAttempted > 0.1) { - statusMap.set(key, { - state: 'failed', - error: `Indexing aborted: ${failedChunks} of ${totalAttempted} chunks failed (>10% threshold)`, - failedFiles: failedChunks, - }); - return; - } - - // 7. Store all chunks - vectorStore.addChunks(allIndexedChunks); - - // 8. Invalidate search cache for this repo (SHA changed) - invalidateSearchCache(owner, repo); - - statusMap.set(key, { - state: 'complete', - chunkCount: vectorStore.getChunkCount(key), - timestamp: Date.now(), - }); -} - -function getExtension(path: string): string { +function extension(path: string): string { const dot = path.lastIndexOf('.'); - if (dot < 0) return ''; - return path.slice(dot).toLowerCase(); + return dot < 0 ? '' : path.slice(dot).toLowerCase(); } - -// ── Singleton ────────────────────────────────────────────────────────── - let globalPipeline: IndexingPipeline | null = null; - export function getIndexingPipeline(): IndexingPipeline { - if (!globalPipeline) { - globalPipeline = createIndexingPipeline(); - } - return globalPipeline; + return (globalPipeline ??= createIndexingPipeline()); } diff --git a/server/search/limits.ts b/server/search/limits.ts new file mode 100644 index 0000000..478814b --- /dev/null +++ b/server/search/limits.ts @@ -0,0 +1,11 @@ +export const SEARCH_LIMITS = { + concurrentJobs: 2, + files: 2000, + fileBytes: 256 * 1024, + chunks: 4000, + indexBytes: 32 * 1024 * 1024, + totalBytes: 128 * 1024 * 1024, + indices: 16, + statuses: 64, + ttlMs: 30 * 60 * 1000, +}; diff --git a/server/search/qa-engine.test.ts b/server/search/qa-engine.test.ts index 232a3b2..9152255 100644 --- a/server/search/qa-engine.test.ts +++ b/server/search/qa-engine.test.ts @@ -1,3 +1,4 @@ +import { searchIndexKey } from './index-identity'; import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test'; import { createQAEngine, getQAEngine } from './qa-engine'; import { clearAllCaches } from '../cache/lru'; @@ -14,7 +15,7 @@ mock.module('./embedding-provider', () => ({ }), })); -const repoKey = 'owner/repo'; +const repoKey = () => searchIndexKey('owner', 'repo', 'sha'); function indexedChunk(id: string, scoreVector: Float32Array): IndexedChunk { return { @@ -27,7 +28,7 @@ function indexedChunk(id: string, scoreVector: Float32Array): IndexedChunk { chunkIndex: 0, language: 'typescript', content: `export const ${id} = true;`, - repoKey, + repoKey: repoKey(), commitSha: 'sha', }, }; @@ -36,7 +37,7 @@ function indexedChunk(id: string, scoreVector: Float32Array): IndexedChunk { describe('createQAEngine', () => { beforeEach(() => { clearAllCaches(); - getVectorStore().removeByRepo(repoKey); + getVectorStore().removeByRepo(repoKey()); process.env.AI_PROVIDER = 'mock'; }); @@ -47,16 +48,9 @@ describe('createQAEngine', () => { const qa = createQAEngine(); it('returns not-available answer when no index exists', async () => { - const response = await qa.ask({ - question: 'Where is the auth middleware?', - owner: 'nobody', - repo: 'ghost', - commitSha: 'abc', - }); - - expect(response.answer).toContain('could not find'); - expect(response.citations).toEqual([]); - expect(typeof response.cached).toBe('boolean'); + await expect( + qa.ask({ question: 'Where is the auth middleware?', owner: 'nobody', repo: 'ghost', commitSha: 'abc' }), + ).rejects.toMatchObject({ status: 404 }); }); it('returns empty citations when scores are below threshold', async () => { diff --git a/server/search/qa-engine.ts b/server/search/qa-engine.ts index a6632a1..2bb9b5b 100644 --- a/server/search/qa-engine.ts +++ b/server/search/qa-engine.ts @@ -15,24 +15,17 @@ export function createQAEngine(): QAEngine { const { question, owner, repo, commitSha, apiKey } = options; // Retrieve top 5 chunks via semantic search - let searchResponse; - try { - searchResponse = await searchEngine.search({ - query: question, - owner, - repo, - commitSha, - topK: QA_TOP_K, - minScore: DEFAULT_MIN_SCORE, - }); - } catch { - // If search fails (no index, etc.) return not-available - return { answer: NOT_AVAILABLE_MESSAGE, citations: [], cached: false }; - } + const searchResponse = await searchEngine.search({ + query: question, + owner, + repo, + commitSha, + gitHubToken: options.gitHubToken, + topK: QA_TOP_K, + minScore: DEFAULT_MIN_SCORE, + }); - const relevantResults = searchResponse.results.filter( - (r) => r.score >= DEFAULT_MIN_SCORE, - ); + const relevantResults = searchResponse.results.filter((r) => r.score >= DEFAULT_MIN_SCORE); if (relevantResults.length < 1) { return { answer: NOT_AVAILABLE_MESSAGE, citations: [], cached: searchResponse.cached }; diff --git a/server/search/search-engine.test.ts b/server/search/search-engine.test.ts index aeed866..ac5aa3b 100644 --- a/server/search/search-engine.test.ts +++ b/server/search/search-engine.test.ts @@ -1,10 +1,11 @@ +import { searchIndexKey } from './index-identity'; import { beforeEach, describe, expect, it, mock } from 'bun:test'; import { cache, clearAllCaches, hashContext } from '../cache/lru'; import { createSearchEngine, getSearchEngine } from './search-engine'; import { getVectorStore } from './vector-store'; import type { IndexedChunk, SearchEngine, SearchResponse } from './types'; -const repoKey = 'owner/repo'; +const repoKey = () => searchIndexKey('owner', 'repo', 'sha'); function chunk(id: string, vector: Float32Array): IndexedChunk { return { @@ -17,7 +18,7 @@ function chunk(id: string, vector: Float32Array): IndexedChunk { chunkIndex: 0, language: 'typescript', content: `content ${id}`, - repoKey, + repoKey: repoKey(), commitSha: 'sha', }, }; @@ -27,16 +28,16 @@ describe('createSearchEngine — input validation', () => { const engine: SearchEngine = createSearchEngine(); it('rejects query shorter than 3 chars', async () => { - await expect( - engine.search({ query: 'ab', owner: 'o', repo: 'r', commitSha: 'sha' }), - ).rejects.toMatchObject({ status: 400 }); + await expect(engine.search({ query: 'ab', owner: 'o', repo: 'r', commitSha: 'sha' })).rejects.toMatchObject({ + status: 400, + }); }); it('rejects query longer than 500 chars', async () => { const longQuery = 'x'.repeat(501); - await expect( - engine.search({ query: longQuery, owner: 'o', repo: 'r', commitSha: 'sha' }), - ).rejects.toMatchObject({ status: 400 }); + await expect(engine.search({ query: longQuery, owner: 'o', repo: 'r', commitSha: 'sha' })).rejects.toMatchObject({ + status: 400, + }); }); it('rejects when no index exists for repo', async () => { @@ -54,17 +55,14 @@ describe('createSearchEngine — input validation', () => { describe('createSearchEngine — success/cache paths', () => { beforeEach(() => { clearAllCaches(); - getVectorStore().removeByRepo(repoKey); + getVectorStore().removeByRepo(repoKey()); process.env.AI_PROVIDER = 'mock'; delete process.env.OPENAI_API_KEY; delete process.env.GEMINI_API_KEY; }); it('searches indexed chunks and caches response', async () => { - getVectorStore().addChunks([ - chunk('a', new Float32Array([1, 0, 0])), - chunk('b', new Float32Array([0, 1, 0])), - ]); + getVectorStore().addChunks([chunk('a', new Float32Array([1, 0, 0])), chunk('b', new Float32Array([0, 1, 0]))]); const engine = createSearchEngine(); const first = await engine.search({ @@ -105,7 +103,7 @@ describe('createSearchEngine — success/cache paths', () => { query: 'cached query', cached: false, }; - cache.set(`search:owner/repo@sha:${hashContext('cached query')}`, cached); + cache.set(`search:${repoKey()}:${hashContext('cached query')}:10:0.3`, cached); const result = await createSearchEngine().search({ query: 'cached query', @@ -124,7 +122,9 @@ describe('createSearchEngine — success/cache paths', () => { // Mock createEmbeddingProvider to return a provider that throws on embed mock.module('./embedding-provider', () => ({ createEmbeddingProvider: async () => ({ - embed: async () => { throw new Error('embedding service down'); }, + embed: async () => { + throw new Error('embedding service down'); + }, embedBatch: async () => [], tokenCount: (text: string) => text.split(/\s+/).length, }), @@ -139,10 +139,10 @@ describe('createSearchEngine — success/cache paths', () => { query: 'some failing query', owner: 'owner', repo: 'repo', - commitSha: 'sha2', + commitSha: 'sha', }), ).rejects.toMatchObject({ status: 503 }); mock.restore(); }); -}); \ No newline at end of file +}); diff --git a/server/search/search-engine.ts b/server/search/search-engine.ts index 546bacd..e2b1ef1 100644 --- a/server/search/search-engine.ts +++ b/server/search/search-engine.ts @@ -1,61 +1,52 @@ import type { SearchEngine, SearchOptions, SearchResponse } from './types'; -import { DEFAULT_TOP_K, DEFAULT_MIN_SCORE, searchCacheKey } from './constants'; +import { DEFAULT_TOP_K, DEFAULT_MIN_SCORE } from './constants'; +import { searchIndexKey } from './index-identity'; import { createEmbeddingProvider } from './embedding-provider'; import { getVectorStore } from './vector-store'; import { cache } from '../cache/lru'; import { hashContext } from '../cache/lru'; import { AppError } from '../utils/errors'; -export function createSearchEngine(): SearchEngine { +export function createSearchEngine(deps = { createEmbeddingProvider, getVectorStore }): SearchEngine { return { async search(options: SearchOptions): Promise { const { query, owner, repo, commitSha } = options; const topK = options.topK ?? DEFAULT_TOP_K; const minScore = options.minScore ?? DEFAULT_MIN_SCORE; - const repoKey = `${owner}/${repo}`; + const repoKey = searchIndexKey(owner, repo, commitSha, options); // Validate query length if (query.length < 3 || query.length > 500) { - throw new AppError( - 400, - 'Query must be between 3 and 500 characters.', - 'INVALID_QUERY_LENGTH', - ); + throw new AppError(400, 'Query must be between 3 and 500 characters.', 'INVALID_QUERY_LENGTH'); } // Check if index exists - const vectorStore = getVectorStore(); + const vectorStore = deps.getVectorStore(); if (!vectorStore.hasIndex(repoKey)) { - throw new AppError( - 404, - 'No index found. Please index the repository first.', - 'INDEX_NOT_FOUND', - ); + throw new AppError(404, 'No index found. Please index the repository first.', 'INDEX_NOT_FOUND'); } // Check cache const queryHash = hashContext(query); - const cacheKey = searchCacheKey(owner, repo, commitSha, queryHash); + const cacheKey = `search:${repoKey}:${queryHash}:${topK}:${minScore}`; const cached = cache.get(cacheKey); if (cached) { return { ...cached, cached: true }; } // Generate embedding for query - const provider = await createEmbeddingProvider(); + const provider = await deps.createEmbeddingProvider(); let embeddingResult; try { embeddingResult = await provider.embed(query); } catch { - throw new AppError( - 503, - 'Search temporarily unavailable. Please retry.', - 'EMBEDDING_FAILURE', - true, - ); + throw new AppError(503, 'Search temporarily unavailable. Please retry.', 'EMBEDDING_FAILURE', true); } // Search vector store + if (!vectorStore.hasIndex(repoKey)) { + throw new AppError(404, 'Search index expired. Please rebuild it.', 'INDEX_NOT_FOUND'); + } const results = vectorStore.search(embeddingResult.vector, repoKey, topK, minScore); const response: SearchResponse = { diff --git a/server/search/search-eviction.test.ts b/server/search/search-eviction.test.ts new file mode 100644 index 0000000..895d168 --- /dev/null +++ b/server/search/search-eviction.test.ts @@ -0,0 +1,44 @@ +import { test, expect } from 'bun:test'; +import { createSearchEngine } from './search-engine'; +import { createVectorStore } from './vector-store'; +import { searchIndexKey } from './index-identity'; +import type { EmbeddingProvider, IndexedChunk } from './types'; + +test('eviction during embedding rejects without caching an empty result', async () => { + const store = createVectorStore(); + const key = searchIndexKey('eviction', 'test', 'sha'); + const entry: IndexedChunk = { + id: 'a', + vector: new Float32Array([1]), + metadata: { + repoKey: key, + commitSha: 'sha', + filePath: 'a.ts', + language: 'typescript', + content: 'hello', + startLine: 1, + endLine: 1, + chunkIndex: 0, + }, + }; + store.replaceIndex(key, [entry]); + let calls = 0; + const provider: EmbeddingProvider = { + dimensions: 1, + maxTokens: 512, + providerName: 'test', + embedBatch: async () => [], + embed: async () => { + if (++calls === 1) store.removeByRepo(key); + return { vector: new Float32Array([1]), tokenCount: 1 }; + }, + }; + const engine = createSearchEngine({ getVectorStore: () => store, createEmbeddingProvider: async () => provider }); + const options = { owner: 'eviction', repo: 'test', commitSha: 'sha', query: 'hello eviction' }; + await expect(engine.search(options)).rejects.toMatchObject({ code: 'INDEX_NOT_FOUND' }); + store.replaceIndex(key, [entry]); + const result = await engine.search(options); + expect(result.cached).toBe(false); + expect(result.results).toHaveLength(1); + expect(calls).toBe(2); +}); diff --git a/server/search/types.ts b/server/search/types.ts index 7fc7c02..5ae5b04 100644 --- a/server/search/types.ts +++ b/server/search/types.ts @@ -40,15 +40,11 @@ export interface SearchResult { } export interface VectorStore { + replaceIndex(repoKey: string, chunks: IndexedChunk[]): void; addChunks(chunks: IndexedChunk[]): void; removeByRepo(repoKey: string): void; removeByFile(repoKey: string, filePath: string): void; - search( - queryVector: Float32Array, - repoKey: string, - topK: number, - minScore: number, - ): SearchResult[]; + search(queryVector: Float32Array, repoKey: string, topK: number, minScore: number): SearchResult[]; getChunkCount(repoKey: string): number; hasIndex(repoKey: string): boolean; } @@ -83,6 +79,7 @@ export interface IndexingPipeline { // ── Search Engine ────────────────────────────────────────────────────── export interface SearchOptions { + gitHubToken?: string; query: string; owner: string; repo: string; @@ -104,6 +101,7 @@ export interface SearchEngine { // ── QA Engine ────────────────────────────────────────────────────────── export interface QAOptions { + gitHubToken?: string; question: string; owner: string; repo: string; @@ -139,7 +137,16 @@ export interface Chunk { } export interface Chunker { - chunkFile(content: string, filePath: string, language: string): Chunk[]; + chunkFile( + content: string, + filePath: string, + language: string, + budget?: { + maxChunks: number; + maxBytes: number; + bytesPerChunk: number; + }, + ): Chunk[]; } export type { RequestContext }; diff --git a/server/search/vector-store.test.ts b/server/search/vector-store.test.ts index 8c8ea67..57e5598 100644 --- a/server/search/vector-store.test.ts +++ b/server/search/vector-store.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach } from 'bun:test'; import { createVectorStore } from './vector-store'; import type { IndexedChunk } from './types'; +import { SEARCH_LIMITS } from './limits'; function makeVector(size: number, val: number): Float32Array { const v = new Float32Array(size).fill(0); @@ -9,12 +10,7 @@ function makeVector(size: number, val: number): Float32Array { return v.map((x) => x / mag); } -function makeChunk( - id: string, - repoKey: string, - filePath: string, - vector: Float32Array, -): IndexedChunk { +function makeChunk(id: string, repoKey: string, filePath: string, vector: Float32Array): IndexedChunk { return { id, vector, @@ -32,6 +28,33 @@ function makeChunk( } describe('createVectorStore', () => { + it('evicts least recently used indices and expires snapshots', () => { + let time = 0; + const bounded = createVectorStore({ ...SEARCH_LIMITS, indices: 2, ttlMs: 10 }, () => time); + const add = (key: string) => bounded.addChunks([makeChunk(key, key, 'a.ts', new Float32Array([1]))]); + add('one'); + add('two'); + bounded.hasIndex('one'); + add('three'); + expect(bounded.hasIndex('two')).toBe(false); + expect(bounded.hasIndex('one')).toBe(true); + time = 11; + expect(bounded.hasIndex('one')).toBe(false); + }); + it('rejects oversized replacements without destroying the old index', () => { + const bounded = createVectorStore({ ...SEARCH_LIMITS, chunks: 1 }); + const one = makeChunk('one', 'repo', 'a.ts', new Float32Array([1])); + bounded.replaceIndex('repo', [one]); + expect(() => bounded.replaceIndex('repo', [one, one])).toThrow(); + expect(bounded.getChunkCount('repo')).toBe(1); + }); + it('enforces the global byte budget', () => { + const bounded = createVectorStore({ ...SEARCH_LIMITS, totalBytes: 1000 }); + bounded.addChunks([makeChunk('one', 'one', 'a.ts', new Float32Array([1]))]); + bounded.addChunks([makeChunk('two', 'two', 'a.ts', new Float32Array([1]))]); + expect(bounded.hasIndex('one')).toBe(false); + expect(bounded.hasIndex('two')).toBe(true); + }); let store: ReturnType; beforeEach(() => { @@ -83,10 +106,7 @@ describe('createVectorStore', () => { describe('removeByFile', () => { it('removes only chunks for the specified file', () => { const vec = makeVector(4, 1); - store.addChunks([ - makeChunk('c1', 'owner/repo', 'a.ts', vec), - makeChunk('c2', 'owner/repo', 'b.ts', vec), - ]); + store.addChunks([makeChunk('c1', 'owner/repo', 'a.ts', vec), makeChunk('c2', 'owner/repo', 'b.ts', vec)]); store.removeByFile('owner/repo', 'a.ts'); expect(store.getChunkCount('owner/repo')).toBe(1); }); @@ -147,9 +167,7 @@ describe('createVectorStore', () => { const dim = 4; const vec = new Float32Array(dim); vec[0] = 1; - const chunks = Array.from({ length: 10 }, (_, i) => - makeChunk(`c${i}`, 'owner/repo', `${i}.ts`, vec), - ); + const chunks = Array.from({ length: 10 }, (_, i) => makeChunk(`c${i}`, 'owner/repo', `${i}.ts`, vec)); store.addChunks(chunks); const results = store.search(vec, 'owner/repo', 3, 0); expect(results).toHaveLength(3); diff --git a/server/search/vector-store.ts b/server/search/vector-store.ts index f8eff5a..220e272 100644 --- a/server/search/vector-store.ts +++ b/server/search/vector-store.ts @@ -1,97 +1,84 @@ import type { IndexedChunk, SearchResult, VectorStore } from './types'; +import { SEARCH_LIMITS } from './limits'; +import { AppError } from '../utils/errors'; -interface RepoIndex { - chunks: IndexedChunk[]; - commitSha: string; - createdAt: number; +export function chunkBytes(chunk: IndexedChunk): number { + return chunk.vector.byteLength + 2 * (chunk.id.length + JSON.stringify(chunk.metadata).length) + 256; } -export function createVectorStore(): VectorStore { - const indices = new Map(); - +export function createVectorStore(limits = SEARCH_LIMITS, now = Date.now): VectorStore { + const indices = new Map(); + const prune = () => { + for (const [key, index] of indices) if (index.expires <= now()) indices.delete(key); + }; + const get = (key: string) => { + prune(); + const index = indices.get(key); + if (index) { + indices.delete(key); + indices.set(key, index); + } + return index; + }; + const replaceIndex = (key: string, chunks: IndexedChunk[]) => { + const bytes = chunks.reduce((sum, chunk) => sum + chunkBytes(chunk), 0); + if (chunks.length > limits.chunks || bytes > limits.indexBytes || bytes > limits.totalBytes) + throw new AppError(413, 'Search index exceeds the repository size limit.', 'INDEX_TOO_LARGE'); + if (chunks.some((chunk) => chunk.metadata.repoKey !== key)) + throw new AppError(400, 'Index identity mismatch.', 'INVALID_INDEX'); + prune(); + // Validate before touching the published index. + indices.delete(key); + let total = [...indices.values()].reduce((sum, index) => sum + index.bytes, 0); + while (indices.size && (total + bytes > limits.totalBytes || indices.size >= limits.indices)) { + const oldest = indices.keys().next().value!; + total -= indices.get(oldest)!.bytes; + indices.delete(oldest); + } + indices.set(key, { chunks: [...chunks], bytes, expires: now() + limits.ttlMs }); + }; return { - addChunks(chunks: IndexedChunk[]): void { - if (chunks.length === 0) return; - const repoKey = chunks[0].metadata.repoKey; - const existing = indices.get(repoKey); - - if (existing) { - existing.chunks.push(...chunks); - } else { - indices.set(repoKey, { - chunks: [...chunks], - commitSha: chunks[0].metadata.commitSha, - createdAt: Date.now(), - }); - } + replaceIndex, + addChunks(chunks) { + if (!chunks.length) return; + const key = chunks[0].metadata.repoKey; + replaceIndex(key, [...(get(key)?.chunks ?? []), ...chunks]); }, - - removeByRepo(repoKey: string): void { - indices.delete(repoKey); + removeByRepo(key) { + indices.delete(key); }, - - removeByFile(repoKey: string, filePath: string): void { - const index = indices.get(repoKey); + removeByFile(key, path) { + const index = get(key); if (!index) return; - index.chunks = index.chunks.filter((c) => c.metadata.filePath !== filePath); - if (index.chunks.length === 0) { - indices.delete(repoKey); - } + const chunks = index.chunks.filter((chunk) => chunk.metadata.filePath !== path); + if (!chunks.length) indices.delete(key); + else replaceIndex(key, chunks); }, - - search( - queryVector: Float32Array, - repoKey: string, - topK: number, - minScore: number, - ): SearchResult[] { - const index = indices.get(repoKey); - if (!index || index.chunks.length === 0) return []; - + search(vector, key, topK, minScore): SearchResult[] { + const index = get(key); + if (!index) return []; const scored: SearchResult[] = []; - for (const chunk of index.chunks) { - const score = cosineSimilarity(queryVector, chunk.vector); - if (score >= minScore) { - scored.push({ chunk: chunk.metadata, score }); - } + const score = cosineSimilarity(vector, chunk.vector); + if (score >= minScore) scored.push({ chunk: chunk.metadata, score }); } - - // Sort descending by score - scored.sort((a, b) => b.score - a.score); - - return scored.slice(0, topK); + return scored.sort((a, b) => b.score - a.score).slice(0, topK); }, - - getChunkCount(repoKey: string): number { - return indices.get(repoKey)?.chunks.length ?? 0; + getChunkCount(key) { + return get(key)?.chunks.length ?? 0; }, - - hasIndex(repoKey: string): boolean { - return indices.has(repoKey); + hasIndex(key) { + return !!get(key); }, }; } - -/** Compute cosine similarity between two unit vectors. */ function cosineSimilarity(a: Float32Array, b: Float32Array): number { if (a.length !== b.length) return 0; let dot = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - } - // Vectors are pre-normalized, so dot product = cosine similarity - // Clamp to [0, 1] range (negative similarity → 0) + for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; return Math.max(0, Math.min(1, dot)); } - -// ── Singleton ────────────────────────────────────────────────────────── - let globalStore: VectorStore | null = null; - export function getVectorStore(): VectorStore { - if (!globalStore) { - globalStore = createVectorStore(); - } - return globalStore; + return (globalStore ??= createVectorStore()); } diff --git a/src/features/search/searchStore.ts b/src/features/search/searchStore.ts index 3c8e9a7..eed99c0 100644 --- a/src/features/search/searchStore.ts +++ b/src/features/search/searchStore.ts @@ -4,6 +4,7 @@ import type { SearchResultCard, QAAnswer, IndexingStatus } from '@/types'; export type SearchMode = 'search' | 'ask'; interface SearchState { + revision: number; mode: SearchMode; query: string; results: SearchResultCard[]; @@ -12,8 +13,6 @@ interface SearchState { error: string | null; recentQueries: string[]; indexingStatus: IndexingStatus; - askCache: Map; - searchCache: Map; setMode: (mode: SearchMode) => void; setQuery: (query: string) => void; @@ -47,6 +46,7 @@ function saveRecentQueries(queries: string[]): void { } export const useSearchStore = create((set, get) => ({ + revision: 0, mode: 'search', query: '', results: [], @@ -55,8 +55,6 @@ export const useSearchStore = create((set, get) => ({ error: null, recentQueries: loadRecentQueries(), indexingStatus: { state: 'idle' }, - askCache: new Map(), - searchCache: new Map(), setMode: (mode) => set({ mode }), setQuery: (query) => set({ query }), @@ -77,6 +75,7 @@ export const useSearchStore = create((set, get) => ({ reset: () => set({ + revision: get().revision + 1, mode: 'search', query: '', results: [], @@ -84,6 +83,5 @@ export const useSearchStore = create((set, get) => ({ isLoading: false, error: null, indexingStatus: { state: 'idle' }, - // Preserve caches across repo switches — they're keyed by owner/repo }), })); diff --git a/src/features/search/useIndexingStatus.ts b/src/features/search/useIndexingStatus.ts index ccd16ec..aa2b036 100644 --- a/src/features/search/useIndexingStatus.ts +++ b/src/features/search/useIndexingStatus.ts @@ -1,25 +1,23 @@ import { useQuery } from '@tanstack/react-query'; +import { useEffect } from 'react'; import { fetchIndexingStatus } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; import type { IndexingStatus } from '@/types'; -export function useIndexingStatus(owner: string, repo: string, enabled = true) { - const { setIndexingStatus } = useSearchStore(); +export function useIndexingStatus(owner: string, repo: string, enabled = true, branch?: string) { + const { setIndexingStatus, indexingStatus } = useSearchStore(); - return useQuery({ - queryKey: ['indexingStatus', owner, repo], - queryFn: async () => { - const status = await fetchIndexingStatus(owner, repo); - setIndexingStatus(status); - return status; - }, + const query = useQuery({ + queryKey: ['indexingStatus', owner, repo, branch], + queryFn: () => fetchIndexingStatus(owner, repo, branch), enabled: enabled && !!owner && !!repo, - refetchInterval: () => { - // Read from Zustand store (updated by useTriggerIndexing) instead of - // query.state.data which is undefined on first call, preventing polling. - const storeState = useSearchStore.getState().indexingStatus.state; - return storeState === 'indexing' || storeState === 'idle' ? 3000 : false; - }, + refetchInterval: indexingStatus.state === 'indexing' ? 3000 : false, + retry: false, staleTime: 0, }); -} \ No newline at end of file + // React Query exposes only the current repository's result; old requests cannot update the store. + useEffect(() => { + if (query.data) setIndexingStatus(query.data); + }, [query.data, query.dataUpdatedAt, setIndexingStatus]); + return query; +} diff --git a/src/features/search/useSemanticAsk.ts b/src/features/search/useSemanticAsk.ts index 3d783be..d41a2b1 100644 --- a/src/features/search/useSemanticAsk.ts +++ b/src/features/search/useSemanticAsk.ts @@ -1,5 +1,5 @@ import { useMutation } from '@tanstack/react-query'; -import { semanticAsk } from '@/lib/apiClient'; +import { ApiError, semanticAsk } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; export function useSemanticAsk() { @@ -21,21 +21,21 @@ export function useSemanticAsk() { setIsLoading(true); setError(null); addRecentQuery(vars.question); + return { revision: useSearchStore.getState().revision }; }, - onSuccess: (data, vars) => { + onSuccess: (data, _variables, context) => { + if (context?.revision !== useSearchStore.getState().revision) return; const qa = { answer: data.answer, citations: data.citations }; setAnswer(qa); setIsLoading(false); - // Cache answer for this question - const cacheKey = `${vars.owner}/${vars.repo}:${vars.question}`; - const store = useSearchStore.getState(); - const newCache = new Map(store.askCache); - newCache.set(cacheKey, qa); - useSearchStore.setState({ askCache: newCache }); }, - onError: (err: Error) => { + onError: (err: Error, _variables, context) => { + if (context?.revision !== useSearchStore.getState().revision) return; setError(err.message); + if (err instanceof ApiError && err.code === 'INDEX_NOT_FOUND') { + useSearchStore.getState().setIndexingStatus({ state: 'idle' }); + } setIsLoading(false); }, }); -} \ No newline at end of file +} diff --git a/src/features/search/useSemanticSearch.ts b/src/features/search/useSemanticSearch.ts index 0f3f310..737c1bc 100644 --- a/src/features/search/useSemanticSearch.ts +++ b/src/features/search/useSemanticSearch.ts @@ -1,5 +1,5 @@ import { useMutation } from '@tanstack/react-query'; -import { semanticSearch } from '@/lib/apiClient'; +import { ApiError, semanticSearch } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; import type { SearchResultCard } from '@/types'; @@ -7,22 +7,15 @@ export function useSemanticSearch() { const { setResults, setIsLoading, setError, addRecentQuery } = useSearchStore(); return useMutation({ - mutationFn: ({ - query, - owner, - repo, - branch, - }: { - query: string; - owner: string; - repo: string; - branch?: string; - }) => semanticSearch(query, owner, repo, branch), + mutationFn: ({ query, owner, repo, branch }: { query: string; owner: string; repo: string; branch?: string }) => + semanticSearch(query, owner, repo, branch), onMutate: () => { setIsLoading(true); setError(null); + return { revision: useSearchStore.getState().revision }; }, - onSuccess: (data, vars) => { + onSuccess: (data, _variables, context) => { + if (context?.revision !== useSearchStore.getState().revision) return; const cards: SearchResultCard[] = data.results.map((r) => ({ filePath: r.chunk.filePath, startLine: r.chunk.startLine, @@ -34,16 +27,14 @@ export function useSemanticSearch() { setResults(cards); addRecentQuery(data.query); setIsLoading(false); - // Cache results for this query - const cacheKey = `${vars.owner}/${vars.repo}:${vars.query}`; - const store = useSearchStore.getState(); - const newCache = new Map(store.searchCache); - newCache.set(cacheKey, cards); - useSearchStore.setState({ searchCache: newCache }); }, - onError: (err: Error) => { + onError: (err: Error, _variables, context) => { + if (context?.revision !== useSearchStore.getState().revision) return; setError(err.message); + if (err instanceof ApiError && err.code === 'INDEX_NOT_FOUND') { + useSearchStore.getState().setIndexingStatus({ state: 'idle' }); + } setIsLoading(false); }, }); -} \ No newline at end of file +} diff --git a/src/features/search/useTriggerIndexing.ts b/src/features/search/useTriggerIndexing.ts index 2946108..ac66b62 100644 --- a/src/features/search/useTriggerIndexing.ts +++ b/src/features/search/useTriggerIndexing.ts @@ -6,15 +6,8 @@ export function useTriggerIndexing() { const { setIndexingStatus } = useSearchStore(); return useMutation({ - mutationFn: ({ - owner, - repo, - branch, - }: { - owner: string; - repo: string; - branch?: string; - }) => triggerIndexing(owner, repo, branch), + mutationFn: ({ owner, repo, branch }: { owner: string; repo: string; branch?: string }) => + triggerIndexing(owner, repo, branch), onMutate: () => { setIndexingStatus({ state: 'indexing', @@ -22,8 +15,13 @@ export function useTriggerIndexing() { filesProcessed: 0, totalFiles: 0, }); + return { revision: useSearchStore.getState().revision }; }, - onError: (error) => { + onSuccess: (status, _variables, context) => { + if (context?.revision === useSearchStore.getState().revision) setIndexingStatus(status); + }, + onError: (error, _variables, context) => { + if (context?.revision !== useSearchStore.getState().revision) return; setIndexingStatus({ state: 'failed', error: error instanceof Error ? error.message : 'Failed to start indexing', @@ -31,4 +29,4 @@ export function useTriggerIndexing() { }); }, }); -} \ No newline at end of file +} diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index f9c9444..36895f4 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -258,14 +258,14 @@ export async function semanticAsk(question: string, owner: string, repo: string, }); } -export async function triggerIndexing(owner: string, repo: string, branch?: string): Promise<{ status: string }> { - return request<{ status: string }>('/api/search/index', { +export async function triggerIndexing(owner: string, repo: string, branch?: string): Promise { + return request('/api/search/index', { method: 'POST', body: JSON.stringify({ owner, repo, branch }), }); } -export async function fetchIndexingStatus(owner: string, repo: string): Promise { - const params = new URLSearchParams({ owner, repo }); +export async function fetchIndexingStatus(owner: string, repo: string, branch?: string): Promise { + const params = new URLSearchParams({ owner, repo, ...(branch ? { branch } : {}) }); return request(`/api/search/status?${params}`); } diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index f29cd83..f61fde3 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -16,52 +16,24 @@ import { useIndexingStatus } from '@/features/search/useIndexingStatus'; export function SearchPage() { const { owner = '', repo = '' } = useParams(); const navigate = useNavigate(); - const { mode, error, isLoading, results, answer, indexingStatus, askCache, searchCache } = useSearchStore(); + const { mode, error, isLoading, results, answer, indexingStatus } = useSearchStore(); const searchMutation = useSemanticSearch(); const askMutation = useSemanticAsk(); const indexMutation = useTriggerIndexing(); - useIndexingStatus(owner, repo); + const branch = useVizStore((state) => state.selectedBranch) ?? undefined; + useIndexingStatus(owner, repo, true, branch); - // Reset search state when navigating to a different repo's search page - const prevRepoRef = useRef(`${owner}/${repo}`); + // Clear previous results on entry and when the repository snapshot changes. useEffect(() => { - const key = `${owner}/${repo}`; - if (key !== prevRepoRef.current) { - prevRepoRef.current = key; - useSearchStore.getState().reset(); - } - }, [owner, repo]); + useSearchStore.getState().reset(); + }, [owner, repo, branch]); const handleSubmit = useCallback( (query: string) => { - const cacheKey = `${owner}/${repo}:${query}`; - - if (mode === 'search') { - // Return cached search results if same query - const cached = searchCache.get(cacheKey); - if (cached) { - const { setResults, setIsLoading, setError } = useSearchStore.getState(); - setResults(cached); - setIsLoading(false); - setError(null); - return; - } - searchMutation.mutate({ query, owner, repo }); - } else { - // Return cached answer if same question - const cached = askCache.get(cacheKey); - if (cached) { - const { setAnswer, setIsLoading, setError, addRecentQuery } = useSearchStore.getState(); - setAnswer(cached); - setIsLoading(false); - setError(null); - addRecentQuery(query); - return; - } - askMutation.mutate({ question: query, owner, repo }); - } + if (mode === 'search') searchMutation.mutate({ query, owner, repo, branch }); + else askMutation.mutate({ question: query, owner, repo, branch }); }, - [mode, owner, repo, searchMutation, askMutation, askCache, searchCache], + [mode, owner, repo, branch, searchMutation, askMutation], ); const prevModeRef = useRef(mode); @@ -76,22 +48,22 @@ export function SearchPage() { }, [mode, handleSubmit]); const handleIndex = useCallback(() => { - indexMutation.mutate({ owner, repo }); - }, [owner, repo, indexMutation]); + indexMutation.mutate({ owner, repo, branch }); + }, [owner, repo, branch, indexMutation]); const handleSelectFile = useCallback( (filePath: string, _startLine: number, action: 'open' | 'inspect' = 'open') => { const { setFocusedFilePath, setInspectorOpen, setSelectedNodeId } = useVizStore.getState(); - + setSelectedNodeId(null); setFocusedFilePath(filePath); - + if (action === 'open') { setInspectorOpen(true); } else { setInspectorOpen(false); } - + navigate(`/${owner}/${repo}`); }, [navigate, owner, repo], @@ -124,9 +96,13 @@ export function SearchPage() { {/* Content */}
-

Search & discover

+

+ Search & discover +

Find the idea behind the code.

-

Search by meaning, ask a question, and follow the answer to its source.

+

+ Search by meaning, ask a question, and follow the answer to its source. +

{/* Indexing status banner */}
@@ -158,10 +134,7 @@ export function SearchPage() { {/* Search bar */}
- +
@@ -169,12 +142,14 @@ export function SearchPage() { {showEmptyHero && isIndexed && (
-

Search Examples

+

+ Search Examples +

{[ - "How are API errors handled?", - "Where is GitHub data fetched?", - "How is the dependency graph generated?" + 'How are API errors handled?', + 'Where is GitHub data fetched?', + 'How is the dependency graph generated?', ].map((example) => ( - )} -
- ); - } - - if (indexingStatus.state === 'indexing') { - const progress = indexingStatus.progress; - const hasFileCount = indexingStatus.totalFiles > 0; - return ( -
-
- - - {hasFileCount ? 'Indexing...' : 'Scanning repository...'} - - {hasFileCount && ( - - {indexingStatus.filesProcessed}/{indexingStatus.totalFiles} files - - )} -
-
-
void; onCancel?: () => void }) { + const status = useSearchStore((state) => state.indexingStatus); + const action = useSearchStore((state) => state.indexAction); + const resumed = useRef(null); + const [now, setNow] = useState(Date.now); + useEffect(() => { + if (status.state !== 'paused' || !status.retryAt) return; + const timer = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(timer); + }, [status]); + const waiting = status.state === 'paused' && !!status.retryAt && status.retryAt > now; + useEffect(() => { + if (action || status.state !== 'paused' || !status.retryAt || status.retryAt > now) return; + const deadline = `${status.snapshotSha}:${status.retryAt}`; + if (resumed.current === deadline) return; + resumed.current = deadline; + onRetry?.(); + }, [status, now, onRetry, action]); + const working = status.state === 'indexing' || status.state === 'paused'; + const button = + 'rounded-md border border-border bg-background px-3 py-1.5 text-xs text-foreground hover:bg-secondary disabled:opacity-50 disabled:cursor-not-allowed'; + return ( +
+

+ {status.state === 'complete' + ? 'Index ready' + : status.state === 'indexing' + ? 'Indexing repository…' + : status.state === 'paused' + ? 'Indexing paused' + : status.state === 'failed' + ? 'Index failed' + : 'Build a search index'} +

+ {working && ( + <> +

+ {status.filesProcessed} files indexed · {Math.max(0, status.totalFiles - status.filesProcessed)} remaining +

+ -
-

- {hasFileCount - ? 'Building semantic embeddings...' - : 'Preparing files...'} + + )} + {status.state === 'complete' &&

{status.chunkCount} chunks indexed

} + {(status.state === 'paused' || status.state === 'failed') && ( +

{status.error}

+ )} + {status.state === 'paused' && ( +

+ {status.retryAt + ? `Retry ${new Date(status.retryAt).toLocaleString()}${waiting ? ` (in ${formatRetryDelay(status.retryAt - now)})` : ' — ready to resume'}` + : 'Resume when the service is available.'}

-
- ); - } - - if (indexingStatus.state === 'complete') { - return ( -
- - Index ready - {indexingStatus.chunkCount} chunks -
- ); - } - - if (indexingStatus.state === 'failed') { - return ( -
- -
-

Index failed

-

{indexingStatus.error}

-
- {onRetry && ( - + )} + {working && onCancel && ( + )}
- ); - } - - return null; + + ); } diff --git a/src/features/search/SearchEmptyState.tsx b/src/features/search/SearchEmptyState.tsx new file mode 100644 index 0000000..1e9f6b2 --- /dev/null +++ b/src/features/search/SearchEmptyState.tsx @@ -0,0 +1,60 @@ +import { useSearchStore } from './searchStore'; + +export function SearchEmptyState({ chunkCount, onSubmit }: { chunkCount: number; onSubmit: (query: string) => void }) { + return ( +
+
+

+ Search Examples +

+
+ {[ + 'How are API errors handled?', + 'Where is GitHub data fetched?', + 'How is the dependency graph generated?', + ].map((example) => ( + + ))} +
+
+ +
+
+

+ Recent Queries +

+
+ No recent queries yet. +
+
+
+

+ Index Details +

+
+
+ Status + + + Ready + +
+
+ Chunks Indexed + {chunkCount} +
+
+
+
+
+ ); +} diff --git a/src/features/search/SearchIndexControls.tsx b/src/features/search/SearchIndexControls.tsx new file mode 100644 index 0000000..423c096 --- /dev/null +++ b/src/features/search/SearchIndexControls.tsx @@ -0,0 +1,71 @@ +import { cancelIndexing, type IndexScope } from '@/lib/apiClient'; +import { useSearchStore } from './searchStore'; +import { IndexingStatusPanel } from './IndexingStatusPanel'; +import { useQueryClient } from '@tanstack/react-query'; +import { beginIndexOperation, finishIndexOperation } from './index-operations'; + +interface SearchIndexControlsProps { + owner: string; + repo: string; + branch?: string; + scope: IndexScope; + includeInput: string; + excludeInput: string; + onIncludeChange: (value: string) => void; + onExcludeChange: (value: string) => void; + onIndex: () => void; +} + +export function SearchIndexControls(props: SearchIndexControlsProps) { + const queryClient = useQueryClient(); + const handleCancel = async () => { + const context = beginIndexOperation('cancel'); + const buildId = useSearchStore.getState().indexBuildId; + try { + await queryClient.cancelQueries({ queryKey: ['indexingStatus', props.owner, props.repo] }); + const status = await cancelIndexing(props.owner, props.repo, props.branch, props.scope, buildId); + finishIndexOperation(context, status); + } catch { + const previous = + context.previous.state === 'paused' ? { ...context.previous, retryAt: undefined } : context.previous; + if (finishIndexOperation(context, previous)) + useSearchStore.getState().setError('Could not cancel indexing. Please retry.'); + } + }; + return ( +
+
+ {[ + { + label: 'Include paths', + value: props.includeInput, + onChange: props.onIncludeChange, + placeholder: 'packages/next/src, apps/web', + }, + { + label: 'Exclude paths', + value: props.excludeInput, + onChange: props.onExcludeChange, + placeholder: 'test, docs, examples', + }, + ].map((field) => ( + + ))} +
+ { + void handleCancel(); + }} + /> +
+ ); +} diff --git a/src/features/search/index-operations.test.ts b/src/features/search/index-operations.test.ts new file mode 100644 index 0000000..8087caf --- /dev/null +++ b/src/features/search/index-operations.test.ts @@ -0,0 +1,20 @@ +import { test, expect } from 'bun:test'; +import { useSearchStore } from './searchStore'; +import { beginIndexOperation, finishIndexOperation } from './index-operations'; + +test('cancel invalidates the preceding indexing callback', () => { + useSearchStore.getState().reset(); + const indexing = beginIndexOperation('index'); + const cancelling = beginIndexOperation('cancel'); + expect(finishIndexOperation(cancelling, { state: 'idle' })).toBe(true); + expect(finishIndexOperation(indexing, { state: 'complete', chunkCount: 10, timestamp: 0 })).toBe(false); + expect(useSearchStore.getState().indexingStatus.state).toBe('idle'); + expect(useSearchStore.getState().indexOperation).toBeGreaterThan(cancelling.operation); +}); + +test('navigation invalidates an outstanding operation', () => { + const operation = beginIndexOperation('index'); + useSearchStore.getState().reset(); + expect(finishIndexOperation(operation, { state: 'complete', chunkCount: 1, timestamp: 0 })).toBe(false); + expect(useSearchStore.getState().indexAction).toBeNull(); +}); diff --git a/src/features/search/index-operations.ts b/src/features/search/index-operations.ts new file mode 100644 index 0000000..9c3c969 --- /dev/null +++ b/src/features/search/index-operations.ts @@ -0,0 +1,16 @@ +import type { IndexingStatus } from '@/types'; +import { useSearchStore } from './searchStore'; + +export function beginIndexOperation(action: 'index' | 'cancel') { + const state = useSearchStore.getState(); + const operation = state.indexOperation + 1; + useSearchStore.setState({ indexOperation: operation, indexAction: action }); + return { operation, revision: state.revision, previous: state.indexingStatus }; +} + +export function finishIndexOperation(context: ReturnType, status: IndexingStatus): boolean { + const state = useSearchStore.getState(); + if (state.indexOperation !== context.operation || state.revision !== context.revision) return false; + useSearchStore.setState({ indexOperation: context.operation + 1, indexAction: null, indexingStatus: status }); + return true; +} diff --git a/src/features/search/indexing-recovery.test.ts b/src/features/search/indexing-recovery.test.ts new file mode 100644 index 0000000..f862e5a --- /dev/null +++ b/src/features/search/indexing-recovery.test.ts @@ -0,0 +1,41 @@ +import { test, expect } from 'bun:test'; +import { recoverIndexingStatus } from './indexing-recovery'; +import type { IndexingStatus } from '@/types'; +import { ApiError } from '@/lib/apiClient'; + +const paused: IndexingStatus = { + state: 'paused', + progress: 80, + filesProcessed: 32, + totalFiles: 40, + snapshotSha: 'original', + retryAt: 1000, + reason: 'EMBEDDING_RATE_LIMITED', + error: 'Provider limit', + coverage: { kind: 'partial', commitSha: 'original', indexedFiles: 32, totalFiles: 40 }, +}; + +test('network failure preserves snapshot, progress and coverage without retrying an expired timer', () => { + const recovered = recoverIndexingStatus(paused, new ApiError('Network unavailable', 0), 2000); + expect(recovered).toMatchObject({ + state: 'paused', + snapshotSha: 'original', + filesProcessed: 32, + coverage: paused.coverage, + }); + expect(recovered.state === 'paused' && recovered.retryAt).toBeUndefined(); +}); +test('admission failure uses the new retry time while preserving the paused build', () => { + const error = Object.assign(new Error('Busy'), { status: 429, details: { retryAfterSeconds: 60 } }); + expect(recoverIndexingStatus(paused, error, 2000)).toMatchObject({ + state: 'paused', + retryAt: 62000, + filesProcessed: 32, + }); +}); +test('terminal rejection does not advertise a discarded partial index', () => { + expect(recoverIndexingStatus(paused, Object.assign(new Error('Too large'), { status: 413 }))).toMatchObject({ + state: 'failed', + coverage: undefined, + }); +}); diff --git a/src/features/search/indexing-recovery.ts b/src/features/search/indexing-recovery.ts new file mode 100644 index 0000000..92007b1 --- /dev/null +++ b/src/features/search/indexing-recovery.ts @@ -0,0 +1,28 @@ +import type { IndexingStatus } from '@/types'; + +/** Recover a rejected request without pretending that its saved server-side build disappeared. */ +export function recoverIndexingStatus(previous: IndexingStatus, error: Error, now = Date.now()): IndexingStatus { + const status = 'status' in error && typeof error.status === 'number' ? error.status : undefined; + const recoverable = + status === undefined || status === 0 || status === 408 || status === 409 || status === 429 || status >= 500; + if (previous.state === 'paused' && recoverable) { + const details = 'details' in error ? error.details : undefined; + const seconds = + details && typeof details === 'object' && 'retryAfterSeconds' in details ? details.retryAfterSeconds : undefined; + return { + ...previous, + error: error.message, + retryAt: + typeof seconds === 'number' && Number.isFinite(seconds) && seconds > 0 + ? now + Math.ceil(seconds) * 1000 + : undefined, + }; + } + return { + state: 'failed', + error: error.message, + failedFiles: 0, + snapshotSha: previous.snapshotSha, + coverage: previous.coverage?.kind === 'previous' ? previous.coverage : undefined, + }; +} diff --git a/src/features/search/retry-delay.test.ts b/src/features/search/retry-delay.test.ts new file mode 100644 index 0000000..7d80ecd --- /dev/null +++ b/src/features/search/retry-delay.test.ts @@ -0,0 +1,8 @@ +import { expect, test } from 'bun:test'; +import { formatRetryDelay } from './retry-delay'; + +test('retry countdown uses readable minutes and hours', () => { + expect(formatRetryDelay(63176000)).toBe('17 hr 33 min'); + expect(formatRetryDelay(60000)).toBe('1 min'); + expect(formatRetryDelay(3600000)).toBe('1 hr'); +}); diff --git a/src/features/search/retry-delay.ts b/src/features/search/retry-delay.ts new file mode 100644 index 0000000..3b47f70 --- /dev/null +++ b/src/features/search/retry-delay.ts @@ -0,0 +1,8 @@ +export function formatRetryDelay(milliseconds: number): string { + const minutes = Math.ceil(Math.max(0, milliseconds) / 60000); + if (minutes < 1) return 'less than a minute'; + if (minutes < 60) return `${minutes} min`; + const hours = Math.floor(minutes / 60); + const remaining = minutes % 60; + return `${hours} hr${remaining ? ` ${remaining} min` : ''}`; +} diff --git a/src/features/search/searchStore.ts b/src/features/search/searchStore.ts index eed99c0..d5c3006 100644 --- a/src/features/search/searchStore.ts +++ b/src/features/search/searchStore.ts @@ -4,6 +4,10 @@ import type { SearchResultCard, QAAnswer, IndexingStatus } from '@/types'; export type SearchMode = 'search' | 'ask'; interface SearchState { + indexBuildId?: string; + indexOperation: number; + indexAction: 'index' | 'cancel' | null; + resultCoverage: import('../../../server/search/coverage').SearchCoverage | undefined; revision: number; mode: SearchMode; query: string; @@ -46,6 +50,9 @@ function saveRecentQueries(queries: string[]): void { } export const useSearchStore = create((set, get) => ({ + indexOperation: 0, + indexAction: null, + resultCoverage: undefined, revision: 0, mode: 'search', query: '', @@ -76,9 +83,12 @@ export const useSearchStore = create((set, get) => ({ reset: () => set({ revision: get().revision + 1, + indexAction: null, + indexBuildId: undefined, mode: 'search', query: '', results: [], + resultCoverage: undefined, answer: null, isLoading: false, error: null, diff --git a/src/features/search/useIndexingStatus.ts b/src/features/search/useIndexingStatus.ts index aa2b036..facb9e7 100644 --- a/src/features/search/useIndexingStatus.ts +++ b/src/features/search/useIndexingStatus.ts @@ -3,21 +3,32 @@ import { useEffect } from 'react'; import { fetchIndexingStatus } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; import type { IndexingStatus } from '@/types'; +import type { IndexScope } from '@/lib/apiClient'; -export function useIndexingStatus(owner: string, repo: string, enabled = true, branch?: string) { - const { setIndexingStatus, indexingStatus } = useSearchStore(); +export function useIndexingStatus(owner: string, repo: string, enabled = true, branch?: string, scope?: IndexScope) { + const { setIndexingStatus, indexingStatus, indexOperation, indexAction } = useSearchStore(); - const query = useQuery({ - queryKey: ['indexingStatus', owner, repo, branch], - queryFn: () => fetchIndexingStatus(owner, repo, branch), - enabled: enabled && !!owner && !!repo, + const query = useQuery<{ status: IndexingStatus; operation: number }>({ + queryKey: ['indexingStatus', owner, repo, branch, scope, indexOperation], + queryFn: async () => ({ status: await fetchIndexingStatus(owner, repo, branch, scope), operation: indexOperation }), + enabled: + enabled && + !!owner && + !!repo && + indexAction !== 'cancel' && + (indexingStatus.state !== 'paused' || indexAction === 'index'), refetchInterval: indexingStatus.state === 'indexing' ? 3000 : false, retry: false, staleTime: 0, }); // React Query exposes only the current repository's result; old requests cannot update the store. useEffect(() => { - if (query.data) setIndexingStatus(query.data); + const state = useSearchStore.getState(); + if (!query.data || query.data.operation !== state.indexOperation || state.indexAction === 'cancel') return; + // A long indexing POST owns its final result. Polls may report progress, never undo the optimistic start. + if (state.indexAction === 'index' && query.data.status.state !== 'indexing') return; + setIndexingStatus(query.data.status); + if (query.data.status.buildId) useSearchStore.setState({ indexBuildId: query.data.status.buildId }); }, [query.data, query.dataUpdatedAt, setIndexingStatus]); return query; } diff --git a/src/features/search/useSemanticAsk.ts b/src/features/search/useSemanticAsk.ts index d41a2b1..f61bd10 100644 --- a/src/features/search/useSemanticAsk.ts +++ b/src/features/search/useSemanticAsk.ts @@ -11,12 +11,14 @@ export function useSemanticAsk() { owner, repo, branch, + scope, }: { question: string; owner: string; repo: string; branch?: string; - }) => semanticAsk(question, owner, repo, branch), + scope: import('@/lib/apiClient').IndexScope; + }) => semanticAsk(question, owner, repo, branch, scope), onMutate: (vars) => { setIsLoading(true); setError(null); @@ -27,6 +29,7 @@ export function useSemanticAsk() { if (context?.revision !== useSearchStore.getState().revision) return; const qa = { answer: data.answer, citations: data.citations }; setAnswer(qa); + useSearchStore.setState({ resultCoverage: data.coverage }); setIsLoading(false); }, onError: (err: Error, _variables, context) => { diff --git a/src/features/search/useSemanticSearch.ts b/src/features/search/useSemanticSearch.ts index 737c1bc..388141d 100644 --- a/src/features/search/useSemanticSearch.ts +++ b/src/features/search/useSemanticSearch.ts @@ -7,8 +7,19 @@ export function useSemanticSearch() { const { setResults, setIsLoading, setError, addRecentQuery } = useSearchStore(); return useMutation({ - mutationFn: ({ query, owner, repo, branch }: { query: string; owner: string; repo: string; branch?: string }) => - semanticSearch(query, owner, repo, branch), + mutationFn: ({ + query, + owner, + repo, + branch, + scope, + }: { + query: string; + owner: string; + repo: string; + branch?: string; + scope: import('@/lib/apiClient').IndexScope; + }) => semanticSearch(query, owner, repo, branch, scope), onMutate: () => { setIsLoading(true); setError(null); @@ -25,6 +36,7 @@ export function useSemanticSearch() { score: r.score, })); setResults(cards); + useSearchStore.setState({ resultCoverage: data.coverage }); addRecentQuery(data.query); setIsLoading(false); }, diff --git a/src/features/search/useTriggerIndexing.ts b/src/features/search/useTriggerIndexing.ts index ac66b62..15ee76d 100644 --- a/src/features/search/useTriggerIndexing.ts +++ b/src/features/search/useTriggerIndexing.ts @@ -1,32 +1,50 @@ -import { useMutation } from '@tanstack/react-query'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; import { triggerIndexing } from '@/lib/apiClient'; import { useSearchStore } from './searchStore'; +import { recoverIndexingStatus } from './indexing-recovery'; +import { beginIndexOperation, finishIndexOperation } from './index-operations'; export function useTriggerIndexing() { + const queryClient = useQueryClient(); const { setIndexingStatus } = useSearchStore(); return useMutation({ - mutationFn: ({ owner, repo, branch }: { owner: string; repo: string; branch?: string }) => - triggerIndexing(owner, repo, branch), - onMutate: () => { + mutationFn: ({ + owner, + repo, + branch, + scope, + buildId, + }: { + owner: string; + repo: string; + branch?: string; + scope: import('@/lib/apiClient').IndexScope; + buildId: string; + }) => triggerIndexing(owner, repo, branch, scope, buildId), + onMutate: async (variables) => { + const context = beginIndexOperation('index'); + useSearchStore.setState({ indexBuildId: variables.buildId }); + const { previous } = context; setIndexingStatus({ state: 'indexing', - progress: 0, - filesProcessed: 0, - totalFiles: 0, + progress: previous.state === 'paused' ? previous.progress : 0, + filesProcessed: previous.state === 'paused' ? previous.filesProcessed : 0, + totalFiles: previous.state === 'paused' ? previous.totalFiles : 0, + snapshotSha: previous.snapshotSha, + coverage: previous.coverage, }); - return { revision: useSearchStore.getState().revision }; + await queryClient.cancelQueries({ queryKey: ['indexingStatus', variables.owner, variables.repo] }); + return context; }, onSuccess: (status, _variables, context) => { - if (context?.revision === useSearchStore.getState().revision) setIndexingStatus(status); + if (context) finishIndexOperation(context, status); }, - onError: (error, _variables, context) => { - if (context?.revision !== useSearchStore.getState().revision) return; - setIndexingStatus({ - state: 'failed', - error: error instanceof Error ? error.message : 'Failed to start indexing', - failedFiles: 0, - }); + onError: async (error, variables, context) => { + const state = useSearchStore.getState(); + if (!context || context.operation !== state.indexOperation || context.revision !== state.revision) return; + await queryClient.cancelQueries({ queryKey: ['indexingStatus', variables.owner, variables.repo] }); + finishIndexOperation(context, recoverIndexingStatus(context.previous, error)); }, }); } diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index 36895f4..7a6fcf2 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -15,6 +15,11 @@ import type { IndexingStatus, } from '@/types'; +export interface IndexScope { + includePaths: string[]; + excludePaths: string[]; +} + function getApiKeyHeader(): Record { try { const key = localStorage.getItem('gitsdm_gemini_api_key'); @@ -78,7 +83,7 @@ async function parseBody(res: Response): Promise { async function request(path: string, options?: RequestInit): Promise { try { - const isAiRoute = path.startsWith('/api/ai'); + const isAiRoute = path.startsWith('/api/ai') || path === '/api/search/ask'; const res = await fetch(path, { ...options, headers: { @@ -244,28 +249,65 @@ export async function semanticSearch( owner: string, repo: string, branch?: string, + scope?: IndexScope, ): Promise { return request('/api/search', { method: 'POST', - body: JSON.stringify({ query, owner, repo, branch }), + body: JSON.stringify({ query, owner, repo, branch, ...scope }), }); } -export async function semanticAsk(question: string, owner: string, repo: string, branch?: string): Promise { +export async function semanticAsk( + question: string, + owner: string, + repo: string, + branch?: string, + scope?: IndexScope, +): Promise { return request('/api/search/ask', { method: 'POST', - body: JSON.stringify({ question, owner, repo, branch }), + body: JSON.stringify({ question, owner, repo, branch, ...scope }), }); } -export async function triggerIndexing(owner: string, repo: string, branch?: string): Promise { +export async function triggerIndexing( + owner: string, + repo: string, + branch?: string, + scope?: IndexScope, + buildId?: string, +): Promise { return request('/api/search/index', { method: 'POST', - body: JSON.stringify({ owner, repo, branch }), + body: JSON.stringify({ owner, repo, branch, ...scope, buildId }), + }); +} + +export async function cancelIndexing( + owner: string, + repo: string, + branch?: string, + scope?: IndexScope, + buildId?: string, +): Promise { + return request('/api/search/cancel', { + method: 'POST', + body: JSON.stringify({ owner, repo, branch, ...scope, buildId }), }); } -export async function fetchIndexingStatus(owner: string, repo: string, branch?: string): Promise { - const params = new URLSearchParams({ owner, repo, ...(branch ? { branch } : {}) }); +export async function fetchIndexingStatus( + owner: string, + repo: string, + branch?: string, + scope?: IndexScope, +): Promise { + const params = new URLSearchParams({ + owner, + repo, + ...(branch ? { branch } : {}), + ...(scope?.includePaths.length ? { includePaths: scope.includePaths.join(',') } : {}), + ...(scope?.excludePaths.length ? { excludePaths: scope.excludePaths.join(',') } : {}), + }); return request(`/api/search/status?${params}`); } diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index f61fde3..8ea801b 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useParams, useNavigate } from 'react-router-dom'; import { ArrowLeft, Search } from 'lucide-react'; import { useSearchStore } from '@/features/search/searchStore'; @@ -7,11 +7,20 @@ import { SearchBar } from '@/features/search/SearchBar'; import { ModeToggle } from '@/features/search/ModeToggle'; import { SearchResults } from '@/features/search/SearchResults'; import { QAAnswerView } from '@/features/search/QAAnswerView'; -import { IndexingStatusPanel } from '@/features/search/IndexingStatusPanel'; +import { SearchIndexControls } from '@/features/search/SearchIndexControls'; import { useSemanticSearch } from '@/features/search/useSemanticSearch'; import { useSemanticAsk } from '@/features/search/useSemanticAsk'; import { useTriggerIndexing } from '@/features/search/useTriggerIndexing'; import { useIndexingStatus } from '@/features/search/useIndexingStatus'; +import { coverageMessage } from '../../server/search/coverage'; +import { SearchEmptyState } from '@/features/search/SearchEmptyState'; + +function parsePaths(value: string): string[] { + return value + .split(',') + .map((path) => path.trim()) + .filter(Boolean); +} export function SearchPage() { const { owner = '', repo = '' } = useParams(); @@ -21,19 +30,49 @@ export function SearchPage() { const askMutation = useSemanticAsk(); const indexMutation = useTriggerIndexing(); const branch = useVizStore((state) => state.selectedBranch) ?? undefined; - useIndexingStatus(owner, repo, true, branch); + const [includeInput, setIncludeInput] = useState(''); + const [excludeInput, setExcludeInput] = useState(''); + const indexScope = useMemo( + () => ({ + includePaths: parsePaths(includeInput), + excludePaths: parsePaths(excludeInput), + }), + [includeInput, excludeInput], + ); + const runningSha = + indexingStatus.state === 'paused' || indexingStatus.state === 'indexing' ? indexingStatus.snapshotSha : undefined; + useIndexingStatus(owner, repo, true, runningSha ?? branch, indexScope); + useEffect(() => { + useSearchStore.getState().reset(); + }, [includeInput, excludeInput]); // Clear previous results on entry and when the repository snapshot changes. useEffect(() => { useSearchStore.getState().reset(); + setIncludeInput(''); + setExcludeInput(''); }, [owner, repo, branch]); const handleSubmit = useCallback( (query: string) => { - if (mode === 'search') searchMutation.mutate({ query, owner, repo, branch }); - else askMutation.mutate({ question: query, owner, repo, branch }); + if (mode === 'search') + searchMutation.mutate({ + query, + owner, + repo, + branch: runningSha ?? branch, + scope: indexScope, + }); + else + askMutation.mutate({ + question: query, + owner, + repo, + branch: runningSha ?? branch, + scope: indexScope, + }); }, - [mode, owner, repo, branch, searchMutation, askMutation], + [mode, owner, repo, branch, runningSha, indexScope, searchMutation, askMutation], ); const prevModeRef = useRef(mode); @@ -48,8 +87,12 @@ export function SearchPage() { }, [mode, handleSubmit]); const handleIndex = useCallback(() => { - indexMutation.mutate({ owner, repo, branch }); - }, [owner, repo, branch, indexMutation]); + if (useSearchStore.getState().indexAction) return; + const state = useSearchStore.getState(); + const buildId = + state.indexingStatus.state === 'paused' ? (state.indexBuildId ?? crypto.randomUUID()) : crypto.randomUUID(); + indexMutation.mutate({ owner, repo, branch: runningSha ?? branch, scope: indexScope, buildId }); + }, [owner, repo, branch, runningSha, indexScope, indexMutation]); const handleSelectFile = useCallback( (filePath: string, _startLine: number, action: 'open' | 'inspect' = 'open') => { @@ -70,7 +113,8 @@ export function SearchPage() { ); const isIndexed = indexingStatus.state === 'complete'; - const isSearchDisabled = !isIndexed; + const isSearchDisabled = !isIndexed && !indexingStatus.coverage; + const responseCoverage = useSearchStore((state) => state.resultCoverage); const hasResults = mode === 'search' ? results.length > 0 : answer !== null; const showEmptyHero = !hasResults && !isLoading; @@ -105,9 +149,17 @@ export function SearchPage() {

{/* Indexing status banner */} -
- -
+ {/* Compact controls area */}
@@ -128,7 +180,7 @@ export function SearchPage() { {mode === 'search' ? 'Find code snippets by semantic similarity.' - : 'Ask a repository question with source citations.'} + : 'Answers cover indexed files only, with source citations.'}
@@ -140,60 +192,7 @@ export function SearchPage() { {/* Empty State Content */} {showEmptyHero && isIndexed && ( -
-
-

- Search Examples -

-
- {[ - 'How are API errors handled?', - 'Where is GitHub data fetched?', - 'How is the dependency graph generated?', - ].map((example) => ( - - ))} -
-
- -
-
-

- Recent Queries -

-
- No recent queries yet. -
-
-
-

- Index Details -

-
-
- Status - - - Ready - -
-
- Chunks Indexed - {indexingStatus.chunkCount} -
-
-
-
-
+ )} {/* Error */} @@ -207,9 +206,12 @@ export function SearchPage() { )} {/* No results message */} + {responseCoverage &&

{coverageMessage(responseCoverage)}

} {!isLoading && !error && mode === 'search' && results.length === 0 && searchMutation.isSuccess && (
- No matching code found. Try broader terms or switch to Ask mode. + {responseCoverage?.kind === 'partial' + ? 'No matches in indexed files yet.' + : 'No matching code found. Try broader terms or switch to Ask mode.'}
)} diff --git a/src/types/api.ts b/src/types/api.ts index 13d66d2..2db6e4a 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -88,6 +88,7 @@ export interface QAAnswer { } export interface SearchResponse { + coverage?: import('../../server/search/coverage').SearchCoverage; results: { chunk: { filePath: string; @@ -106,13 +107,10 @@ export interface SearchResponse { } export interface QAResponse { + coverage?: import('../../server/search/coverage').SearchCoverage; answer: string; citations: { filePath: string; startLine: number; endLine: number }[]; cached: boolean; } -export type IndexingStatus = - | { state: 'idle' } - | { state: 'indexing'; progress: number; filesProcessed: number; totalFiles: number } - | { state: 'complete'; chunkCount: number; timestamp: number } - | { state: 'failed'; error: string; failedFiles: number }; +export type IndexingStatus = import('../../server/search/types').IndexingStatus; From 59f76590dfe820aa0f84d71699eba54219f4146c Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Sat, 12 Sep 2026 14:58:38 +0700 Subject: [PATCH 07/27] fix(cache): isolate repository analysis by credentials Scope branch and SHA cache entries to effective GitHub credentials. Replace the fixed hash secret with a random process secret locally and require configuration in shared mode. --- .env.example | 2 ++ README.md | 4 ++++ server/cache/hash-secret.test.ts | 29 ++++++++++++++++++++++++++++ server/cache/hash-secret.ts | 13 +++++++++++++ server/cache/lru.test.ts | 14 +++++--------- server/cache/lru.ts | 21 +++++++------------- server/services/analyze-repo.test.ts | 20 ++++++++++++++++++- server/services/analyze-repo.ts | 5 +++-- 8 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 server/cache/hash-secret.test.ts create mode 100644 server/cache/hash-secret.ts diff --git a/.env.example b/.env.example index 601ae19..50a0766 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,8 @@ GEMINI_EMBEDDING_MODEL=gemini-embedding-001 EMBEDDING_DIMENSIONS=3072 # Cache key hashing secret (optional, set a stable random value in production) +# Required in shared mode (including when Redis is configured). Use a random secret. +# Local mode generates a process-only secret when this is blank. TOKEN_CACHE_HASH_SECRET= # Production server binding (optional) diff --git a/README.md b/README.md index fe0a16a..830d478 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ Embedding operations start at least `EMBEDDING_REQUEST_INTERVAL_MS` apart (defau Provider calls use separate AI and embedding queues, each allowing two active calls and eight waiting calls. Waiting expires after 10 seconds; active calls have a 30-second abort deadline. Operations that ignore cancellation retain their slot until they settle. GitHub requests time out after 15 seconds. AI prompts are capped at 64,000 characters with at most 4,096 output tokens. Mock providers do not spend the hosted-provider budget. +Repository analysis caches are scoped to the effective GitHub credential, including entries addressed by commit SHA. Repository access is checked before returning cached analysis. Raw credentials are never included in cache keys. + +Set `TOKEN_CACHE_HASH_SECRET` to a securely generated random secret in shared mode, including when Redis configuration enables shared mode automatically. Use the same secret across instances. Local mode uses a random per-process secret when the setting is blank; restarting changes cache identities. Rotating a configured secret also changes those identities. + Usage controls combine per-IP counters with deployment-wide budgets. Each IP gets 120 expensive API requests per minute by default, including requests using user-provided credentials. Users on the same network may share this allowance. Configure these environment variables on the backend: | Variable | Default | Meaning | diff --git a/server/cache/hash-secret.test.ts b/server/cache/hash-secret.test.ts new file mode 100644 index 0000000..eb4b158 --- /dev/null +++ b/server/cache/hash-secret.test.ts @@ -0,0 +1,29 @@ +import { expect, test } from 'bun:test'; +import { cacheHashSecret } from './hash-secret'; +import { analyzeCacheKey } from './lru'; + +test('local fallback is random process-stable material and empty configuration uses it', () => { + const secret = cacheHashSecret({}); + expect(secret).toMatch(/^[a-f0-9]{64}$/); + expect(cacheHashSecret({ TOKEN_CACHE_HASH_SECRET: ' ' })).toBe(secret); +}); + +test('shared mode requires a configured secret, including implicit Redis mode', () => { + for (const env of [ + { RATE_LIMIT_MODE: 'shared' }, + { UPSTASH_REDIS_REST_URL: 'https://example.com' }, + { UPSTASH_REDIS_REST_TOKEN: 'test' }, + ]) { + expect(() => cacheHashSecret(env)).toThrow('TOKEN_CACHE_HASH_SECRET is required'); + expect(cacheHashSecret({ ...env, TOKEN_CACHE_HASH_SECRET: 'configured-test-secret' })).toBe( + 'configured-test-secret', + ); + } +}); + +test('analysis keys isolate credentials without exposing tokens', () => { + const first = analyzeCacheKey('owner', 'repo', 'sha', 'main', 'credential-a'); + expect(first).not.toContain('credential-a'); + expect(first).not.toBe(analyzeCacheKey('owner', 'repo', 'sha', 'main', 'credential-b')); + expect(first).toBe(analyzeCacheKey('owner', 'repo', 'sha', 'main', 'credential-a')); +}); diff --git a/server/cache/hash-secret.ts b/server/cache/hash-secret.ts new file mode 100644 index 0000000..9d7654d --- /dev/null +++ b/server/cache/hash-secret.ts @@ -0,0 +1,13 @@ +import { randomBytes } from 'node:crypto'; +import { AppError } from '../utils/errors'; + +const processSecret = randomBytes(32).toString('hex'); + +/** Read lazily: server environment loading can happen after module imports. */ +export function cacheHashSecret(env: Record = process.env): string { + const configured = env.TOKEN_CACHE_HASH_SECRET?.trim(); + if (configured) return configured; + const shared = env.RATE_LIMIT_MODE === 'shared' || !!env.UPSTASH_REDIS_REST_URL || !!env.UPSTASH_REDIS_REST_TOKEN; + if (shared) throw new AppError(503, 'TOKEN_CACHE_HASH_SECRET is required in shared mode.', 'CACHE_SECRET_REQUIRED'); + return processSecret; +} diff --git a/server/cache/lru.test.ts b/server/cache/lru.test.ts index 8d8f80b..03c139d 100644 --- a/server/cache/lru.test.ts +++ b/server/cache/lru.test.ts @@ -85,8 +85,8 @@ describe('cache/lru', () => { }); it('builds cache keys deterministically', () => { - expect(analyzeCacheKey('o', 'r', 's')).toBe('analyze:o/r@s'); - expect(analyzeCacheKey('o', 'r', 's', 'main')).toBe('analyze:o/r@s:main'); + expect(analyzeCacheKey('o', 'r', 's')).toBe(analyzeCacheKey('o', 'r', 's')); + expect(analyzeCacheKey('o', 'r', 's', 'main')).not.toBe(analyzeCacheKey('o', 'r', 's')); expect(aiCacheKey('summary', 'o', 'r', 's', 'ctx')).toBe('ai:summary:o/r@s:ctx'); }); }); @@ -94,9 +94,7 @@ describe('cache/lru', () => { describe('hashContext', () => { it('produces the correct SHA-256 hex string for a known input', () => { // "hello world" -> SHA256 -> b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 - expect(hashContext('hello world')).toBe( - 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9', - ); + expect(hashContext('hello world')).toBe('b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9'); }); it('produces different hashes for different strings', () => { @@ -108,9 +106,7 @@ describe('hashContext', () => { it('produces the correct hash for an empty string', () => { // "" -> SHA256 -> e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 - expect(hashContext('')).toBe( - 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', - ); + expect(hashContext('')).toBe('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'); }); }); @@ -125,4 +121,4 @@ describe('hashToken', () => { expect(hash1).not.toBe(hash2); expect(hash1).toBe(hashToken(key1)); // Deterministic }); -}); \ No newline at end of file +}); diff --git a/server/cache/lru.ts b/server/cache/lru.ts index 80cba32..3eb5b90 100644 --- a/server/cache/lru.ts +++ b/server/cache/lru.ts @@ -1,5 +1,6 @@ import { LRUCache } from 'lru-cache'; import crypto from 'crypto'; +import { cacheHashSecret } from './hash-secret'; export interface CacheStore { get(key: string): T | undefined; @@ -85,10 +86,9 @@ export function invalidateSearchCache(owner: string, repo: string): void { } } -export function analyzeCacheKey(owner: string, repo: string, sha: string, branch?: string): string { - return branch - ? `analyze:${owner}/${repo}@${sha}:${branch}` - : `analyze:${owner}/${repo}@${sha}`; +export function analyzeCacheKey(owner: string, repo: string, sha: string, branch?: string, token?: string): string { + const scope = hashToken(token || process.env.GITHUB_TOKEN?.trim() || 'anonymous'); + return `analyze:${owner}/${repo}@${sha}:${scope}:${JSON.stringify(branch ?? null)}`; } export function aiCacheKey( @@ -105,22 +105,15 @@ export function aiCacheKey( } export function churnCacheKey(owner: string, repo: string, branch?: string, days = 90): string { - return branch - ? `churn:${owner}/${repo}@${branch}:${days}d` - : `churn:${owner}/${repo}@default:${days}d`; + return branch ? `churn:${owner}/${repo}@${branch}:${days}d` : `churn:${owner}/${repo}@default:${days}d`; } export function hashContext(input: string): string { return crypto.createHash('sha256').update(input).digest('hex'); } -const TOKEN_CACHE_HASH_SECRET = process.env.TOKEN_CACHE_HASH_SECRET ?? 'token-cache-v1'; - // This is cache-key derivation, not password storage. -// HMAC avoids exposing raw tokens without blocking request handling. +// HMAC keeps raw tokens out of cache identities. export function hashToken(token: string): string { - return crypto - .createHmac('sha256', TOKEN_CACHE_HASH_SECRET) - .update(token) - .digest('hex'); + return crypto.createHmac('sha256', cacheHashSecret()).update(token).digest('hex'); } diff --git a/server/services/analyze-repo.test.ts b/server/services/analyze-repo.test.ts index 5f1f411..72b0e15 100644 --- a/server/services/analyze-repo.test.ts +++ b/server/services/analyze-repo.test.ts @@ -54,6 +54,7 @@ const excludedWorkspaceFileContents = { }; let activeFileContents: Record = workspaceFileContents; +let denyAccess = false; const buildGraphMock = mock((input: GraphBuildInput) => ({ nodes: input.workspacePackages?.map((pkg) => ({ @@ -119,7 +120,10 @@ const fetchMock = mock(async (input: RequestInfo | URL) => { describe('services/analyze-repo', () => { beforeEach(() => { mock.module('../github/fetch-tree', () => ({ - fetchRepoInfo: async () => mockRepoInfo, + fetchRepoInfo: async () => { + if (denyAccess) throw new Error('Repository access denied'); + return mockRepoInfo; + }, fetchFlatTree: async () => ({ items: Object.keys(activeFileContents).map((path) => ({ path, @@ -153,6 +157,7 @@ describe('services/analyze-repo', () => { clearAllCaches(); activeFileContents = workspaceFileContents; + denyAccess = false; buildGraphMock.mockClear(); fetchMock.mockClear(); globalThis.fetch = fetchMock as typeof fetch; @@ -167,6 +172,19 @@ describe('services/analyze-repo', () => { expect(analyzeRepository('invalid-url')).rejects.toThrow('Invalid GitHub repository URL'); }); + it('isolates analysis and SHA aliases by credentials and rechecks access on cache hits', async () => { + const input = { owner: 'test-owner', repo: 'test-repo', branch: 'main' }; + const first = { ...mockCtx, gitHubToken: 'first-credential' }; + const second = { ...mockCtx, gitHubToken: 'second-credential' }; + const original = await analyzeRepository(input, first); + expect(await analyzeRepository({ ...input, branch: 'test-sha' }, first)).toBe(original); + expect(buildGraphMock).toHaveBeenCalledTimes(1); + await analyzeRepository({ ...input, branch: 'test-sha' }, second); + expect(buildGraphMock).toHaveBeenCalledTimes(2); + denyAccess = true; + await expect(analyzeRepository(input, first)).rejects.toThrow('Repository access denied'); + }); + it('runs the full repository analysis pipeline and caches the result', async () => { const analysis = await analyzeRepository('https://github.com/test-owner/test-repo', mockCtx); expect(analysis.meta.fullName).toBe('test-owner/test-repo'); diff --git a/server/services/analyze-repo.ts b/server/services/analyze-repo.ts index 55070b3..84bd995 100644 --- a/server/services/analyze-repo.ts +++ b/server/services/analyze-repo.ts @@ -34,7 +34,8 @@ export async function analyzeRepository( const { owner, repo } = parsed; const branch = typeof parsed === 'object' && 'branch' in parsed ? (parsed as { branch?: string }).branch : undefined; const info = await fetchRepoInfo(owner, repo, branch, tokenOrCtx); - const cacheKey = analyzeCacheKey(owner, repo, info.sha, branch); + const token = typeof tokenOrCtx === 'string' ? tokenOrCtx : tokenOrCtx?.gitHubToken; + const cacheKey = analyzeCacheKey(owner, repo, info.sha, branch, token); const cached = cache.get(cacheKey); if (cached) return cached; @@ -115,6 +116,6 @@ export async function analyzeRepository( }; cache.set(cacheKey, analysis); - cache.set(analyzeCacheKey(owner, repo, info.sha, info.sha), analysis); + cache.set(analyzeCacheKey(owner, repo, info.sha, info.sha, token), analysis); return analysis; } From 2237b69566d7aec24a0029ccf2a9a08edc4fcf55 Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Sat, 12 Sep 2026 15:32:11 +0700 Subject: [PATCH 08/27] feat(a11y): add persistent motion preferences --- e2e/reduced-motion.e2e.ts | 69 +++++++++++++++++++ src/app/providers.tsx | 20 ++++-- src/components/layout/Navbar.tsx | 3 +- src/components/ui/GlassCard.tsx | 8 ++- src/components/ui/GlowButton.tsx | 8 ++- src/components/viz/LearningPathTab.tsx | 4 +- src/components/viz/MotionSettings.tsx | 28 ++++++++ src/components/viz/SettingsPopover.tsx | 2 + src/components/viz/StagedLoader.tsx | 10 +-- src/components/viz/VizError.tsx | 4 +- src/components/viz/top-nav/BranchSwitcher.tsx | 4 +- .../graph/canvas/ForceGraphCanvas.tsx | 18 ++++- .../graph/canvas/force/useD3Physics.ts | 3 +- .../graph/canvas/force/useForceSync.ts | 23 ++++--- src/hooks/useMotionPreference.ts | 17 +++++ src/lib/motion-preference.test.ts | 7 ++ src/lib/motion-preference.ts | 12 ++++ src/pages/HomePage.tsx | 3 +- src/pages/VizPage.tsx | 6 +- src/stores/motionStore.ts | 16 +++++ src/styles/interface.css | 11 +-- 21 files changed, 236 insertions(+), 40 deletions(-) create mode 100644 e2e/reduced-motion.e2e.ts create mode 100644 src/components/viz/MotionSettings.tsx create mode 100644 src/hooks/useMotionPreference.ts create mode 100644 src/lib/motion-preference.test.ts create mode 100644 src/lib/motion-preference.ts create mode 100644 src/stores/motionStore.ts diff --git a/e2e/reduced-motion.e2e.ts b/e2e/reduced-motion.e2e.ts new file mode 100644 index 0000000..23748a6 --- /dev/null +++ b/e2e/reduced-motion.e2e.ts @@ -0,0 +1,69 @@ +import { test, expect } from '@playwright/test'; + +test('mounted buttons respond to motion changes without a reload', async ({ page }) => { + await page.addInitScript(() => { + localStorage.setItem('gitsdm-motion', JSON.stringify({ state: { preference: 'system' }, version: 0 })); + }); + await page.emulateMedia({ reducedMotion: 'no-preference' }); + await page.route('**/api/repo/analyze**', (route) => route.fulfill({ + status: 400, contentType: 'application/json', body: JSON.stringify({ error: 'Analysis failed', code: 'VALIDATION_ERROR' }), + })); + await page.goto('/mock/todo-app'); + const button = page.getByRole('button', { name: 'Back to Home', exact: true }); + await expect(button).toBeVisible(); + await button.hover(); + const scale = () => button.evaluate((element) => new DOMMatrix(getComputedStyle(element).transform).a); + await expect.poll(scale).toBeGreaterThan(1.01); + const original = await button.elementHandle(); + await page.emulateMedia({ reducedMotion: 'reduce' }); + await expect.poll(scale).toBe(1); + expect(await original?.evaluate((element) => element.isConnected)).toBe(true); + await page.emulateMedia({ reducedMotion: 'no-preference' }); + await page.mouse.move(0, 0); + await button.hover(); + await expect.poll(scale).toBeGreaterThan(1.01); +}); + +for (const reducedMotion of ['reduce', 'no-preference'] as const) { + test(`workspace settles and controls work with motion ${reducedMotion}`, async ({ page }) => { + await page.emulateMedia({ reducedMotion }); + await page.addInitScript(() => { + localStorage.setItem('gitsdm-motion', JSON.stringify({ state: { preference: 'system' }, version: 0 })); + }); + const errors: string[] = []; + page.on('pageerror', (error) => errors.push(error.message)); + await page.goto('/'); + await expect(page.getByRole('heading', { name: 'Understand the whole codebase.' })).toBeVisible(); + await page.goto('/mock/todo-app'); + const graph = page.locator('.graph-canvas-host').last(); + await expect(graph.locator('canvas').first()).toBeVisible({ timeout: 15000 }); + await expect(graph).not.toHaveAttribute('data-reduced-settling', '', { timeout: 15000 }); + await page.getByRole('button', { name: 'Zoom in', exact: true }).click(); + for (const layout of ['Horizontal tree', 'Force layout']) { + await page.getByRole('button', { name: 'Layout', exact: true }).click(); + await page.getByRole('button', { name: layout, exact: true }).click(); + await expect(graph).not.toHaveAttribute('data-reduced-settling', '', { timeout: 15000 }); + await expect(graph.locator('canvas').first()).toBeVisible(); + } + await page.emulateMedia({ reducedMotion: reducedMotion === 'reduce' ? 'no-preference' : 'reduce' }); + await expect(graph.locator('canvas').first()).toBeVisible(); + expect(errors).toEqual([]); + }); +} + +test('app motion choice overrides the system setting and persists on reload', async ({ page }) => { + await page.emulateMedia({ reducedMotion: 'no-preference' }); + await page.goto('/mock/todo-app'); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'Reduced', exact: true }).click(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'true'); + await page.reload(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'true'); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await expect(page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'Reduced', exact: true })).toHaveAttribute('aria-pressed', 'true'); + await page.emulateMedia({ reducedMotion: 'reduce' }); + await page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'Full', exact: true }).click(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'false'); + await page.getByRole('group', { name: 'Motion', exact: true }).getByRole('button', { name: 'System', exact: true }).click(); + await expect(page.locator('html')).toHaveAttribute('data-reduced-motion', 'true'); +}); diff --git a/src/app/providers.tsx b/src/app/providers.tsx index d2028d2..3c8fcb3 100644 --- a/src/app/providers.tsx +++ b/src/app/providers.tsx @@ -1,14 +1,19 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; -import type { ReactNode } from 'react'; +import { useEffect, type ReactNode } from 'react'; +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { ThemeSync } from '@/components/theme/ThemeSync'; import { TooltipProvider } from '@/components/ui/tooltip'; +import { MotionConfig } from 'framer-motion'; const queryClient = new QueryClient({ defaultOptions: { queries: { staleTime: 1000 * 60 * 5, retry: (failureCount, error) => { - if (error instanceof Error && (error.message.includes('404') || error.message.toLowerCase().includes('not found'))) { + if ( + error instanceof Error && + (error.message.includes('404') || error.message.toLowerCase().includes('not found')) + ) { return false; } return failureCount < 1; @@ -18,12 +23,17 @@ const queryClient = new QueryClient({ }); export function Providers({ children }: { children: ReactNode }) { + const reducedMotion = useMotionPreference(); + useEffect(() => { + document.documentElement.dataset.reducedMotion = String(reducedMotion); + }, [reducedMotion]); return ( - - {children} - + {/* Components apply the reactive preference; Motion's built-in flag is mount-only. */} + + {children} + ); } diff --git a/src/components/layout/Navbar.tsx b/src/components/layout/Navbar.tsx index 51e09bf..f85bcdb 100644 --- a/src/components/layout/Navbar.tsx +++ b/src/components/layout/Navbar.tsx @@ -1,3 +1,4 @@ +import { prefersReducedMotion } from '@/lib/motion-preference'; import { TooltipHint } from '@/components/ui/tooltip'; import { Link, useLocation, useNavigate } from "react-router-dom"; import { ArrowUpRight, GitBranch, Moon, Sun } from "lucide-react"; @@ -15,7 +16,7 @@ export function Navbar() { const handleSectionClick = (id: string) => (event: React.MouseEvent) => { event.preventDefault(); const scrollNow = () => { - document.getElementById(id)?.scrollIntoView({ behavior: "smooth", block: "start" }); + document.getElementById(id)?.scrollIntoView({ behavior: prefersReducedMotion() ? "instant" : "smooth", block: "start" }); }; if (location.pathname !== "/") { void navigate("/"); diff --git a/src/components/ui/GlassCard.tsx b/src/components/ui/GlassCard.tsx index 170b773..0fbbc55 100644 --- a/src/components/ui/GlassCard.tsx +++ b/src/components/ui/GlassCard.tsx @@ -1,6 +1,7 @@ import { motion } from 'framer-motion'; import type { ReactNode } from 'react'; import { cn } from '@/lib/utils'; +import { useMotionPreference } from '@/hooks/useMotionPreference'; interface GlassCardProps { children: ReactNode; @@ -10,6 +11,7 @@ interface GlassCardProps { } export function GlassCard({ children, className, hover, onClick }: GlassCardProps) { + const reducedMotion = useMotionPreference(); const classes = cn( 'glass rounded-xl p-5 shadow-glow', onClick && 'cursor-pointer', @@ -20,8 +22,10 @@ export function GlassCard({ children, className, hover, onClick }: GlassCardProp return ( {children} diff --git a/src/components/ui/GlowButton.tsx b/src/components/ui/GlowButton.tsx index a820b7f..a46fbcf 100644 --- a/src/components/ui/GlowButton.tsx +++ b/src/components/ui/GlowButton.tsx @@ -1,6 +1,7 @@ import { motion, HTMLMotionProps } from 'framer-motion'; import type { ReactNode } from 'react'; import { cn } from '@/lib/utils'; +import { useMotionPreference } from '@/hooks/useMotionPreference'; interface GlowButtonProps extends HTMLMotionProps<"button"> { children: ReactNode; @@ -16,10 +17,9 @@ export function GlowButton({ disabled, ...props }: GlowButtonProps) { + const reducedMotion = useMotionPreference(); return ( {loading && ( diff --git a/src/components/viz/LearningPathTab.tsx b/src/components/viz/LearningPathTab.tsx index c243e34..faf2dd7 100644 --- a/src/components/viz/LearningPathTab.tsx +++ b/src/components/viz/LearningPathTab.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useState, useMemo } from 'react'; import { useLearningPath } from '@/features/ai/useAiTasks'; import { useVizStore } from '@/stores/vizStore'; @@ -12,6 +13,7 @@ import { AIErrorCard } from './AIErrorCard'; // Decoupled subcomponents export function LearningPathTab({ analysis }: { analysis: RepoAnalysis }) { + const reducedMotion = useMotionPreference(); const { owner, repo } = analysis.meta; const selectedBranch = useVizStore((s) => s.selectedBranch); const activeFocusLayer = useVizStore((s) => s.activeFocusLayer); @@ -170,7 +172,7 @@ export function LearningPathTab({ analysis }: { analysis: RepoAnalysis }) { key={item.path} initial={{ opacity: 0, y: 8 }} animate={{ opacity: 1, y: 0 }} - transition={{ delay: idx * 0.05 }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { delay: idx * 0.05 }} role="group" aria-label={`Learning step ${idx + 1}: ${item.path}`} className={`group flex gap-3 items-start rounded-md border p-3 transition-all duration-200 ${ diff --git a/src/components/viz/MotionSettings.tsx b/src/components/viz/MotionSettings.tsx new file mode 100644 index 0000000..4dc2e05 --- /dev/null +++ b/src/components/viz/MotionSettings.tsx @@ -0,0 +1,28 @@ +import { useMotionStore, type MotionPreference } from '@/stores/motionStore'; + +const options: { value: MotionPreference; label: string }[] = [ + { value: 'system', label: 'System' }, + { value: 'full', label: 'Full' }, + { value: 'reduced', label: 'Reduced' }, +]; + +export function MotionSettings() { + const { preference, setPreference } = useMotionStore(); + return ( +
+ Motion +
+ {options.map(({ value, label }) => ( + + ))} +
+ + Reduce interface and graph animations. Changes are saved automatically. + +
+ ); +} diff --git a/src/components/viz/SettingsPopover.tsx b/src/components/viz/SettingsPopover.tsx index 97668a4..787af6c 100644 --- a/src/components/viz/SettingsPopover.tsx +++ b/src/components/viz/SettingsPopover.tsx @@ -1,6 +1,7 @@ import { useState, useRef, useEffect, useCallback, type ReactNode, type RefObject } from 'react'; import { Settings, X, Check, Eye, EyeOff, KeyRound, GitBranch } from 'lucide-react'; import { cn } from '@/lib/utils'; +import { MotionSettings } from './MotionSettings'; import { buttonVariants } from '@/components/ui/button'; import { Tooltip, @@ -356,6 +357,7 @@ export function SettingsPopover({
{/* Footer note */} +

Saved keys are stored in this browser and sent to the gitSdm backend for relevant requests. AI and search features may send repository content, including private code, to configured AI and embedding providers.

Do not save keys on shared devices. Privacy policy (opens in a new tab)

diff --git a/src/components/viz/StagedLoader.tsx b/src/components/viz/StagedLoader.tsx index afd84a7..3ec666d 100644 --- a/src/components/viz/StagedLoader.tsx +++ b/src/components/viz/StagedLoader.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useEffect, useState } from 'react'; import { motion, AnimatePresence } from 'framer-motion'; import { CheckCircle2, Loader2, GitBranch, Brain, Layers, Zap, Map } from 'lucide-react'; @@ -54,6 +55,7 @@ interface StagedLoaderProps { } export function StagedLoader({ owner, repo }: StagedLoaderProps) { + const reducedMotion = useMotionPreference(); const [currentStageIndex, setCurrentStageIndex] = useState(0); const [progress, setProgress] = useState(0); @@ -94,7 +96,7 @@ export function StagedLoader({ owner, repo }: StagedLoaderProps) { {/* Glow border */} @@ -121,7 +123,7 @@ export function StagedLoader({ owner, repo }: StagedLoaderProps) { className="h-full rounded-full bg-gradient-to-r from-ui-active-text-green via-accent to-accent" initial={{ width: 0 }} animate={{ width: `${progress}%` }} - transition={{ ease: 'easeOut' }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { ease: 'easeOut' }} />
@@ -133,7 +135,7 @@ export function StagedLoader({ owner, repo }: StagedLoaderProps) { initial={{ opacity: 0, y: 6 }} animate={{ opacity: 1, y: 0 }} exit={{ opacity: 0, y: -6 }} - transition={{ duration: 0.25 }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { duration: 0.25 }} className="mb-5 flex items-center gap-3 rounded-xl border border-ui-active/35 bg-ui-active/10 px-3.5 py-3" >
@@ -159,7 +161,7 @@ export function StagedLoader({ owner, repo }: StagedLoaderProps) { key={stage.id} initial={{ opacity: 0 }} animate={{ opacity: 1 }} - transition={{ delay: i * 0.07 }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { delay: i * 0.07 }} className={`flex items-center gap-2.5 rounded-lg px-2.5 py-2 transition-colors ${isActive ? 'bg-secondary' : '' }`} > diff --git a/src/components/viz/VizError.tsx b/src/components/viz/VizError.tsx index 0048997..d3b19b3 100644 --- a/src/components/viz/VizError.tsx +++ b/src/components/viz/VizError.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useState } from 'react'; import { motion } from 'framer-motion'; import { Link } from 'react-router-dom'; @@ -76,6 +77,7 @@ function getErrorMeta(errStr: string, code?: string): { } export function VizError({ error, message }: VizErrorProps) { + const reducedMotion = useMotionPreference(); let errStr = message || 'An unexpected analysis error occurred'; let errCode = ''; @@ -117,7 +119,7 @@ export function VizError({ error, message }: VizErrorProps) {
diff --git a/src/components/viz/top-nav/BranchSwitcher.tsx b/src/components/viz/top-nav/BranchSwitcher.tsx index 648c9c0..68658c5 100644 --- a/src/components/viz/top-nav/BranchSwitcher.tsx +++ b/src/components/viz/top-nav/BranchSwitcher.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useState, useRef, useEffect } from 'react'; import { useVizStore } from '@/stores/vizStore'; import { useRepoBranches } from '@/hooks/useRepoBranches'; @@ -22,6 +23,7 @@ interface BranchSwitcherProps { } export function BranchSwitcher({ owner, repo, defaultBranch }: BranchSwitcherProps) { + const reducedMotion = useMotionPreference(); const { selectedBranch, setSelectedBranch, @@ -195,7 +197,7 @@ export function BranchSwitcher({ owner, repo, defaultBranch }: BranchSwitcherPro initial={{ opacity: 0, y: 8, scale: 0.97 }} animate={{ opacity: 1, y: 0, scale: 1 }} exit={{ opacity: 0, y: 8, scale: 0.97 }} - transition={{ duration: 0.12, ease: 'easeOut' }} + transition={reducedMotion ? { duration: 0, delay: 0 } : { duration: 0.12, ease: 'easeOut' }} className="fixed left-2 right-2 top-14 z-[100] max-h-[calc(100dvh-4rem)] rounded-md border border-border bg-card p-1.5 shadow-2xl sm:absolute sm:left-0 sm:right-auto sm:top-auto sm:mt-2 sm:w-[min(18rem,calc(100vw-1rem))]" > (null); const theme = useVizStore((s) => s.theme); const internalForceGraphRef = useRef< @@ -126,8 +130,8 @@ export function NetworkCanvas({ ...(typeof window !== 'undefined' && window.innerWidth >= 1024 ? { aiSidebarOpen: true } : {}), }); if (typeof node.x === "number" && typeof node.y === "number") { - forceGraphRef.current?.centerAt(node.x, node.y, 300); - forceGraphRef.current?.zoom(3.2, 300); + forceGraphRef.current?.centerAt(node.x, node.y, motionDuration(300)); + forceGraphRef.current?.zoom(3.2, motionDuration(300)); } }, [prevFocusRef, forceGraphRef], @@ -152,10 +156,11 @@ export function NetworkCanvas({ ]); const handleEngineStop = useCallback(() => { + setSettledLayout({ data: forceGraphData, layout: layoutType, size: sizeMode }); handleLayoutStop(); if (showMinimap) setTick((t) => t + 1); forceInitialViewDoneRef.current = true; - }, [showMinimap, handleLayoutStop]); + }, [showMinimap, handleLayoutStop, forceGraphData, layoutType, sizeMode]); const handleEngineTick = useCallback(() => { if (!showMinimap) return; @@ -273,12 +278,19 @@ export function NetworkCanvas({ // --- Render --- const isLoading = !graph || !graph.nodes; const isEmpty = graph.nodes.length === 0; + const settling = settledLayout?.data !== forceGraphData || settledLayout?.layout !== layoutType || settledLayout?.size !== sizeMode; return (
+ {reducedMotion && settling && !isEmpty && ( +

+ Arranging dependency graph… +

+ )} {isLoading && (
diff --git a/src/features/graph/canvas/force/useD3Physics.ts b/src/features/graph/canvas/force/useD3Physics.ts index df102fd..9cfb65d 100644 --- a/src/features/graph/canvas/force/useD3Physics.ts +++ b/src/features/graph/canvas/force/useD3Physics.ts @@ -1,3 +1,4 @@ +import { motionDuration } from '@/lib/motion-preference'; import { useCallback, useEffect, useRef } from 'react'; import { forceCollide, forceCenter, forceX, forceY } from 'd3-force'; import { stratify, tree } from 'd3-hierarchy'; @@ -24,7 +25,7 @@ export function useD3Physics({ forceGraphRef, nodes, links, layoutType, sizeMode const handleLayoutStop = useCallback(() => { if (!fitAfterSimulation.current) return; fitAfterSimulation.current = false; - forceGraphRef.current?.zoomToFit(400, 60); + forceGraphRef.current?.zoomToFit(motionDuration(400), 60); }, [forceGraphRef]); useEffect(() => { diff --git a/src/features/graph/canvas/force/useForceSync.ts b/src/features/graph/canvas/force/useForceSync.ts index e7af4f6..da2d274 100644 --- a/src/features/graph/canvas/force/useForceSync.ts +++ b/src/features/graph/canvas/force/useForceSync.ts @@ -1,3 +1,4 @@ +import { motionDuration } from '@/lib/motion-preference'; import { useEffect, useRef } from 'react'; import type { ForceGraphMethods } from 'react-force-graph-2d'; import type { ForceGraphNode, ForceGraphLink } from '../../force/forceGraphConstants'; @@ -54,8 +55,8 @@ export function useForceSync({ if (node && typeof node.x === "number" && typeof node.y === "number") { forceInitialViewDoneRef.current = true; - ref.centerAt(node.x, node.y, 300); - ref.zoom(3.2, 300); + ref.centerAt(node.x, node.y, motionDuration(300)); + ref.zoom(3.2, motionDuration(300)); prevFocusRef.current = targetId; return; } @@ -86,15 +87,15 @@ export function useForceSync({ if (action === 'zoomIn') { const currentZoom = ref.zoom(); - if (typeof currentZoom === 'number') ref.zoom(currentZoom * 1.3, 300); + if (typeof currentZoom === 'number') ref.zoom(currentZoom * 1.3, motionDuration(300)); } else if (action === 'zoomOut') { const currentZoom = ref.zoom(); - if (typeof currentZoom === 'number') ref.zoom(currentZoom * 0.7, 300); + if (typeof currentZoom === 'number') ref.zoom(currentZoom * 0.7, motionDuration(300)); } else if (action === 'fitView') { - ref.zoomToFit(400, 60); + ref.zoomToFit(motionDuration(400), 60); } else if (action === 'reset') { - ref.centerAt(0, 0, 400); - ref.zoom(1.5, 400); + ref.centerAt(0, 0, motionDuration(400)); + ref.zoom(1.5, motionDuration(400)); prevFocusRef.current = null; } else if (action === 'focusGraph') { const activeId = @@ -111,11 +112,11 @@ export function useForceSync({ : nodes.find((n) => n.nodeType === 'repo'); if (targetNode && typeof targetNode.x === "number" && typeof targetNode.y === "number") { - ref.centerAt(targetNode.x, targetNode.y, 400); - ref.zoom(activeId ? 3.2 : 1.5, 400); + ref.centerAt(targetNode.x, targetNode.y, motionDuration(400)); + ref.zoom(activeId ? 3.2 : 1.5, motionDuration(400)); } else { - ref.centerAt(0, 0, 400); - ref.zoom(1.5, 400); + ref.centerAt(0, 0, motionDuration(400)); + ref.zoom(1.5, motionDuration(400)); } } }, [ diff --git a/src/hooks/useMotionPreference.ts b/src/hooks/useMotionPreference.ts new file mode 100644 index 0000000..d7e65d1 --- /dev/null +++ b/src/hooks/useMotionPreference.ts @@ -0,0 +1,17 @@ +import { useSyncExternalStore } from 'react'; +import { prefersReducedMotion, REDUCED_MOTION_QUERY } from '@/lib/motion-preference'; +import { useMotionStore } from '@/stores/motionStore'; + +function subscribe(listener: () => void) { + const query = window.matchMedia(REDUCED_MOTION_QUERY); + query.addEventListener('change', listener); + const unsubscribe = useMotionStore.subscribe(listener); + return () => { + query.removeEventListener('change', listener); + unsubscribe(); + }; +} + +export function useMotionPreference() { + return useSyncExternalStore(subscribe, prefersReducedMotion, () => false); +} diff --git a/src/lib/motion-preference.test.ts b/src/lib/motion-preference.test.ts new file mode 100644 index 0000000..8c5506e --- /dev/null +++ b/src/lib/motion-preference.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from 'bun:test'; +import { motionDuration } from './motion-preference'; + +test('graph camera movement is immediate only when reduced motion is enabled', () => { + expect(motionDuration(400, true)).toBe(0); + expect(motionDuration(400, false)).toBe(400); +}); diff --git a/src/lib/motion-preference.ts b/src/lib/motion-preference.ts new file mode 100644 index 0000000..8c693be --- /dev/null +++ b/src/lib/motion-preference.ts @@ -0,0 +1,12 @@ +export const REDUCED_MOTION_QUERY = '(prefers-reduced-motion: reduce)'; + +export function prefersReducedMotion(): boolean { + const preference = useMotionStore.getState().preference; + if (preference !== 'system') return preference === 'reduced'; + return typeof window !== 'undefined' && window.matchMedia(REDUCED_MOTION_QUERY).matches; +} + +export function motionDuration(duration: number, reducedMotion = prefersReducedMotion()): number { + return reducedMotion ? 0 : duration; +} +import { useMotionStore } from '@/stores/motionStore'; diff --git a/src/pages/HomePage.tsx b/src/pages/HomePage.tsx index cc4f26e..8874d69 100644 --- a/src/pages/HomePage.tsx +++ b/src/pages/HomePage.tsx @@ -1,3 +1,4 @@ +import { prefersReducedMotion } from '@/lib/motion-preference'; import { useEffect, useState } from "react"; import { Navbar } from "@/components/layout/Navbar"; import { HeroSection } from "@/components/home/HeroSection"; @@ -16,7 +17,7 @@ export function HomePage() { const timer = window.setTimeout(() => { document.getElementById(sectionId)?.scrollIntoView({ - behavior: "smooth", + behavior: prefersReducedMotion() ? "instant" : "smooth", block: "start", }); }, 80); diff --git a/src/pages/VizPage.tsx b/src/pages/VizPage.tsx index 8d61bdf..beddd32 100644 --- a/src/pages/VizPage.tsx +++ b/src/pages/VizPage.tsx @@ -1,3 +1,4 @@ +import { useMotionPreference } from '@/hooks/useMotionPreference'; import { useEffect, useMemo, useCallback, useState, lazy, Suspense } from 'react'; import { useParams } from 'react-router-dom'; import { motion, AnimatePresence } from 'framer-motion'; @@ -35,6 +36,7 @@ import { fitWorkspacePanels } from '@/lib/workspace-panels'; import { ChurnStatus } from '@/components/viz/ChurnStatus'; export function VizPage() { + const reducedMotion = useMotionPreference(); const { owner = '', repo = '' } = useParams(); const { selectedBranch, compareBranch } = useVizStore(); @@ -192,7 +194,7 @@ export function VizPage() { {/* Center Workspace */} - diff --git a/src/stores/motionStore.ts b/src/stores/motionStore.ts new file mode 100644 index 0000000..5d1874d --- /dev/null +++ b/src/stores/motionStore.ts @@ -0,0 +1,16 @@ +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; + +export type MotionPreference = 'system' | 'reduced' | 'full'; +export const useMotionStore = create( + persist<{ + preference: MotionPreference; + setPreference: (preference: MotionPreference) => void; + }>( + (set) => ({ + preference: 'full', + setPreference: (preference) => set({ preference }), + }), + { name: 'gitsdm-motion', partialize: (state) => state }, + ), +); diff --git a/src/styles/interface.css b/src/styles/interface.css index c16878e..5e7d841 100644 --- a/src/styles/interface.css +++ b/src/styles/interface.css @@ -195,16 +195,17 @@ padding: 24px; } } -@media (prefers-reduced-motion: reduce) { - *, - ::before, - ::after { + [data-reduced-settling] canvas { + visibility: hidden; + } + html[data-reduced-motion='true'] *, + html[data-reduced-motion='true'] ::before, + html[data-reduced-motion='true'] ::after { animation-duration: 0.01ms !important; animation-iteration-count: 1 !important; transition-duration: 0.01ms !important; scroll-behavior: auto !important; } -} /* Classic VS Code status bar, with readable text in both themes. */ .workspace-status { From 46113124822ca3b8559934e4ec0ee1393a09c841 Mon Sep 17 00:00:00 2001 From: Bayu Erich Date: Sat, 12 Sep 2026 16:00:06 +0700 Subject: [PATCH 09/27] feat(site): add policies and consistent navigation Add prerendered Terms, update privacy disclosures, share footer navigation, and reset page scroll. Resolve React Refresh lint warnings and Graphify parser compatibility. Restore Mermaid splitting and defer contributor charts to reduce bundle sizes. --- public/sitemap.xml | 4 + scripts/prerender.tsx | 2 + server/parser/file-classifier.ts | 3 +- server/prod-server.ts | 11 ++- src/app/router.tsx | 13 ++- .../contributors/ContributorsView.test.ts | 2 +- .../contributors/ContributorsView.tsx | 23 +---- .../contributors/timeline-layout.ts | 20 ++++ src/components/home/RepoInput.test.ts | 2 +- src/components/home/RepoInput.tsx | 66 +------------ src/components/home/repo-navigation.ts | 64 +++++++++++++ src/components/layout/InfoPageLayout.tsx | 13 +-- src/components/layout/SiteFooter.tsx | 43 +++++++++ src/components/ui/badge.tsx | 2 +- src/components/ui/button-variants.ts | 38 ++++++++ src/components/ui/button.tsx | 42 +-------- src/components/ui/sidebar.tsx | 1 - src/components/ui/tabs.tsx | 2 +- src/components/viz/AnalysisTab.test.ts | 2 +- src/components/viz/AnalysisTab.tsx | 44 +-------- src/components/viz/SettingsPopover.tsx | 6 +- src/components/viz/VizError.tsx | 2 +- src/components/viz/ai-sidebar/AiCenterTab.tsx | 2 +- src/components/viz/analysis-relations.ts | 42 +++++++++ .../graph/canvas/widgets/LegendPanel.tsx | 4 +- src/lib/apiClient.ts | 5 +- src/lib/page-metadata.test.ts | 8 ++ src/lib/page-metadata.ts | 27 ++++-- src/main.tsx | 12 ++- src/pages/HomePage.tsx | 65 +++---------- src/pages/PrivacyPage.tsx | 88 +++++++++++++++-- src/pages/SearchPage.tsx | 2 +- src/pages/TermsPage.tsx | 94 +++++++++++++++++++ src/pages/VizPage.tsx | 8 +- vercel.json | 1 + vite.config.ts | 4 +- 36 files changed, 493 insertions(+), 274 deletions(-) create mode 100644 src/components/contributors/timeline-layout.ts create mode 100644 src/components/home/repo-navigation.ts create mode 100644 src/components/layout/SiteFooter.tsx create mode 100644 src/components/ui/button-variants.ts create mode 100644 src/components/viz/analysis-relations.ts create mode 100644 src/pages/TermsPage.tsx diff --git a/public/sitemap.xml b/public/sitemap.xml index 280243c..b9b639f 100644 --- a/public/sitemap.xml +++ b/public/sitemap.xml @@ -1,5 +1,9 @@ + + https://gsdm.site/terms + monthly + https://gsdm.site/privacy monthly diff --git a/scripts/prerender.tsx b/scripts/prerender.tsx index 3495f12..fc8538b 100644 --- a/scripts/prerender.tsx +++ b/scripts/prerender.tsx @@ -3,6 +3,7 @@ import { StaticRouter } from 'react-router-dom'; import { Providers } from '../src/app/providers'; import { HomePage } from '../src/pages/HomePage'; import { PrivacyPage } from '../src/pages/PrivacyPage'; +import { TermsPage } from '../src/pages/TermsPage'; import { NotFoundPage } from '../src/pages/NotFoundPage'; import { pageMetadata } from '../src/lib/page-metadata'; @@ -12,6 +13,7 @@ await Bun.write('dist/app.html', shell.replace(/]*>/, '' for (const [route, file, page] of [ ['/', 'dist/index.html', ], ['/privacy', 'dist/privacy.html', ], + ['/terms', 'dist/terms.html', ], ['/not-found', 'dist/404.html', ], ] as const) { const meta = pageMetadata(route); diff --git a/server/parser/file-classifier.ts b/server/parser/file-classifier.ts index 8c439fb..4daeb9d 100644 --- a/server/parser/file-classifier.ts +++ b/server/parser/file-classifier.ts @@ -1,3 +1,4 @@ +import type { TreeNode } from '../../src/types'; import type { FileClass } from '../../src/types'; const CONFIG_PATTERNS = [ @@ -56,7 +57,7 @@ export function classifyFile(path: string): FileClass { return 'other'; } -export function annotateTree(nodes: import('../../src/types').TreeNode[]): import('../../src/types').TreeNode[] { +export function annotateTree(nodes: TreeNode[]): TreeNode[] { return nodes.map((node) => ({ ...node, fileClass: node.type === 'file' ? classifyFile(node.path) : undefined, diff --git a/server/prod-server.ts b/server/prod-server.ts index a5b38bd..14e0c79 100644 --- a/server/prod-server.ts +++ b/server/prod-server.ts @@ -35,7 +35,12 @@ Bun.serve({ return new Response('Bad Request', { status: 400 }); } - const pageFile = decodedPath === '/' ? '/index.html' : decodedPath === '/privacy' || decodedPath === '/privacy/' ? '/privacy.html' : decodedPath; + const pageFile = + decodedPath === '/' + ? '/index.html' + : /^\/(privacy|terms)\/?$/.test(decodedPath) + ? `/${decodedPath.split('/')[1]}.html` + : decodedPath; const filePath = safeJoin(distDir, pageFile); if (!filePath) return addSecurityHeaders(new Response('Not found', { status: 404 })); const file = Bun.file(filePath); @@ -48,7 +53,9 @@ Bun.serve({ } const validRoute = isRepositoryPage(decodedPath); - const fallbackRes = new Response(Bun.file(path.join(distDir, validRoute ? 'app.html' : '404.html')), { status: validRoute ? 200 : 404 }); + const fallbackRes = new Response(Bun.file(path.join(distDir, validRoute ? 'app.html' : '404.html')), { + status: validRoute ? 200 : 404, + }); fallbackRes.headers.set('Cache-Control', 'no-cache'); return addSecurityHeaders(fallbackRes); }, diff --git a/src/app/router.tsx b/src/app/router.tsx index aea3056..22e8693 100644 --- a/src/app/router.tsx +++ b/src/app/router.tsx @@ -1,9 +1,10 @@ -import { lazy, Suspense, type ComponentType } from 'react'; +import { lazy, Suspense, useLayoutEffect, type ComponentType } from 'react'; import { PageMetadata } from '@/components/layout/PageMetadata'; import { BrowserRouter, Route, Routes, useLocation } from 'react-router-dom'; import { ErrorBoundary } from '@/components/ErrorBoundary'; const HomePage = lazy(() => import('@/pages/HomePage').then((module) => ({ default: module.HomePage }))); +const TermsPage = lazy(() => import('@/pages/TermsPage').then((module) => ({ default: module.TermsPage }))); const VizPage = lazy(() => import('@/pages/VizPage').then((module) => ({ default: module.VizPage }))); const SearchPage = lazy(() => import('@/pages/SearchPage').then((module) => ({ @@ -21,6 +22,14 @@ const PrivacyPage = lazy(() => })), ); +function PageScrollReset() { + const { pathname } = useLocation(); + useLayoutEffect(() => { + if (!window.location.hash) window.scrollTo({ top: 0, left: 0, behavior: 'instant' }); + }, [pathname]); + return null; +} + function RouterContent({ initialPage, initialPath }: { initialPage: ComponentType; initialPath: string }) { const location = useLocation(); const InitialPage = initialPage; @@ -29,9 +38,11 @@ function RouterContent({ initialPage, initialPath }: { initialPage: ComponentTyp }> + : } /> : } /> + : } /> : } diff --git a/src/components/contributors/ContributorsView.test.ts b/src/components/contributors/ContributorsView.test.ts index 1227bdc..684c53c 100644 --- a/src/components/contributors/ContributorsView.test.ts +++ b/src/components/contributors/ContributorsView.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { CONTRIBUTOR_TIMELINE_HEIGHT, getContributorTimelineWrapperStyle, -} from './ContributorsView'; +} from './timeline-layout'; // Responsive-height contract: the commit activity timeline renders inside a // sized wrapper and recharts' ResponsiveContainer needs a sized parent (it diff --git a/src/components/contributors/ContributorsView.tsx b/src/components/contributors/ContributorsView.tsx index a584571..a0ad4e0 100644 --- a/src/components/contributors/ContributorsView.tsx +++ b/src/components/contributors/ContributorsView.tsx @@ -1,3 +1,4 @@ +import { CONTRIBUTOR_TIMELINE_HEIGHT, getContributorTimelineWrapperStyle } from "./timeline-layout"; import { TooltipHint } from '@/components/ui/tooltip'; import { useMemo } from 'react'; import { Users, Calendar, Flame, TrendingUp, GitCommit, ExternalLink, Trophy } from 'lucide-react'; @@ -13,26 +14,6 @@ interface ContributorsViewProps { const statLabelClass = "block text-xs leading-5 text-muted-foreground font-medium"; const statValueClass = "text-2xl leading-8 font-bold text-foreground mt-0.5 truncate"; -/** - * Fixed chart height for the commit activity timeline. - * Responsive-height contract: the wrapper's min-height (see - * {@link getContributorTimelineWrapperStyle}) must match this value so the - * recharts `ResponsiveContainer` always has a sized parent (it renders - * nothing when the parent height is 0). Width stays fluid. - */ -export const CONTRIBUTOR_TIMELINE_HEIGHT = 280; - -/** - * Wrapper style for the commit activity timeline. Derived from - * {@link CONTRIBUTOR_TIMELINE_HEIGHT} (inline style — Tailwind cannot - * interpolate a TS constant into an arbitrary-value class, so a hardcoded class would - * silently drift from the chart height). Changing the constant changes - * rendering. - */ -export function getContributorTimelineWrapperStyle(): { minHeight: number } { - return { minHeight: CONTRIBUTOR_TIMELINE_HEIGHT }; -} - export function ContributorsView({ analysis, owner, repo }: ContributorsViewProps) { const { contributors, timeline } = analysis; @@ -140,7 +121,7 @@ export function ContributorsView({ analysis, owner, repo }: ContributorsViewProp

- Leaderboard & Commit Share + Leaderboard & Commit Share

{sortedContributors.map((c, idx) => { diff --git a/src/components/contributors/timeline-layout.ts b/src/components/contributors/timeline-layout.ts new file mode 100644 index 0000000..f6e37e1 --- /dev/null +++ b/src/components/contributors/timeline-layout.ts @@ -0,0 +1,20 @@ +/** + * Fixed chart height for the commit activity timeline. + * Responsive-height contract: the wrapper's min-height (see + * {@link getContributorTimelineWrapperStyle}) must match this value so the + * recharts `ResponsiveContainer` always has a sized parent (it renders + * nothing when the parent height is 0). Width stays fluid. + */ +export const CONTRIBUTOR_TIMELINE_HEIGHT = 280; + +/** + * Wrapper style for the commit activity timeline. Derived from + * {@link CONTRIBUTOR_TIMELINE_HEIGHT} (inline style — Tailwind cannot + * interpolate a TS constant into an arbitrary-value class, so a hardcoded class would + * silently drift from the chart height). Changing the constant changes + * rendering. + */ +export function getContributorTimelineWrapperStyle(): { minHeight: number } { + return { minHeight: CONTRIBUTOR_TIMELINE_HEIGHT }; +} + diff --git a/src/components/home/RepoInput.test.ts b/src/components/home/RepoInput.test.ts index efd28eb..5f3f172 100644 --- a/src/components/home/RepoInput.test.ts +++ b/src/components/home/RepoInput.test.ts @@ -4,7 +4,7 @@ import { resolveOpenRepository, resolvePresetNavigation, resolveRepoNavigation, -} from './RepoInput'; +} from './repo-navigation'; // Locks the intended preset-click behavior: presets navigate immediately to // `/owner/repo` (they don't just fill the input). These pure helpers drive diff --git a/src/components/home/RepoInput.tsx b/src/components/home/RepoInput.tsx index a9859b8..9286b31 100644 --- a/src/components/home/RepoInput.tsx +++ b/src/components/home/RepoInput.tsx @@ -1,3 +1,4 @@ +import { resolveOpenRepository, resolvePresetNavigation, type RepoNavigation } from "./repo-navigation"; import { useState, useMemo, useRef } from "react"; import { useNavigate } from "react-router-dom"; import { useQuery } from "@tanstack/react-query"; @@ -6,75 +7,12 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/Input"; import { getVisibleRepoPresets } from "@/components/home/repoPresets"; import { fetchAppConfig } from "@/lib/apiClient"; -import { parseRepoFromUrl, LAST_REPO_KEY } from "@/lib/utils"; +import { LAST_REPO_KEY } from "@/lib/utils"; interface RepoInputProps { initialUrl?: string; } -export { REPO_PRESETS as PRESETS } from "@/components/home/repoPresets"; - -export interface RepoNavigation { - owner: string; - repo: string; - route: string; - pendingUrl: string; -} - -/** Build the canonical GitHub URL for an `owner/repo` preset slug. */ -export function getPresetUrl(repo: string): string { - return `https://github.com/${repo}`; -} - -/** - * Resolve a raw input (URL or `owner/repo`) to its navigation target. - * Returns `null` when the input is not a valid GitHub repo reference. - * Pure helper so the preset-click → navigate contract is unit-testable. - */ -export function resolveRepoNavigation(value: string): RepoNavigation | null { - const trimmed = value.trim(); - const parsed = parseRepoFromUrl(trimmed); - if (!parsed) return null; - return { - owner: parsed.owner, - repo: parsed.repo, - route: `/${parsed.owner}/${parsed.repo}`, - pendingUrl: trimmed, - }; -} - -export type OpenRepositoryResult = - | { ok: true; nav: RepoNavigation } - | { ok: false; error: string }; - -/** - * Pure decision logic for opening a repository from raw input. - * Returns the navigation target or the error message to display — - * no side effects, so both the submit flow and the preset-click flow - * (`handlePreset` → `openRepository`) are unit-testable. - */ -export function resolveOpenRepository(value: string): OpenRepositoryResult { - const nav = resolveRepoNavigation(value); - if (!nav) { - return { - ok: false, - error: 'Enter a valid GitHub URL or owner/repo (e.g. facebook/react)', - }; - } - return { ok: true, nav }; -} - -/** - * Resolve a preset slug (e.g. `facebook/react`) to the navigation that a - * preset click performs. Preset clicks navigate immediately — they don't - * just fill the input — so this composes `getPresetUrl` with the same - * resolution `openRepository` uses, and is covered by interaction-contract - * tests in `RepoInput.test.ts`. - */ -export function resolvePresetNavigation(repo: string): OpenRepositoryResult { - return resolveOpenRepository(getPresetUrl(repo)); -} - export function RepoInput({ initialUrl = "" }: RepoInputProps) { const [url, setUrl] = useState(initialUrl); const [error, setError] = useState(""); diff --git a/src/components/home/repo-navigation.ts b/src/components/home/repo-navigation.ts new file mode 100644 index 0000000..5b05d93 --- /dev/null +++ b/src/components/home/repo-navigation.ts @@ -0,0 +1,64 @@ +import { parseRepoFromUrl } from "@/lib/utils"; +export { REPO_PRESETS as PRESETS } from "@/components/home/repoPresets"; + +export interface RepoNavigation { + owner: string; + repo: string; + route: string; + pendingUrl: string; +} + +/** Build the canonical GitHub URL for an `owner/repo` preset slug. */ +export function getPresetUrl(repo: string): string { + return `https://github.com/${repo}`; +} + +/** + * Resolve a raw input (URL or `owner/repo`) to its navigation target. + * Returns `null` when the input is not a valid GitHub repo reference. + * Pure helper so the preset-click → navigate contract is unit-testable. + */ +export function resolveRepoNavigation(value: string): RepoNavigation | null { + const trimmed = value.trim(); + const parsed = parseRepoFromUrl(trimmed); + if (!parsed) return null; + return { + owner: parsed.owner, + repo: parsed.repo, + route: `/${parsed.owner}/${parsed.repo}`, + pendingUrl: trimmed, + }; +} + +export type OpenRepositoryResult = + | { ok: true; nav: RepoNavigation } + | { ok: false; error: string }; + +/** + * Pure decision logic for opening a repository from raw input. + * Returns the navigation target or the error message to display — + * no side effects, so both the submit flow and the preset-click flow + * (`handlePreset` → `openRepository`) are unit-testable. + */ +export function resolveOpenRepository(value: string): OpenRepositoryResult { + const nav = resolveRepoNavigation(value); + if (!nav) { + return { + ok: false, + error: 'Enter a valid GitHub URL or owner/repo (e.g. facebook/react)', + }; + } + return { ok: true, nav }; +} + +/** + * Resolve a preset slug (e.g. `facebook/react`) to the navigation that a + * preset click performs. Preset clicks navigate immediately — they don't + * just fill the input — so this composes `getPresetUrl` with the same + * resolution `openRepository` uses, and is covered by interaction-contract + * tests in `RepoInput.test.ts`. + */ +export function resolvePresetNavigation(repo: string): OpenRepositoryResult { + return resolveOpenRepository(getPresetUrl(repo)); +} + diff --git a/src/components/layout/InfoPageLayout.tsx b/src/components/layout/InfoPageLayout.tsx index 30ccac9..746c2ee 100644 --- a/src/components/layout/InfoPageLayout.tsx +++ b/src/components/layout/InfoPageLayout.tsx @@ -1,5 +1,5 @@ +import { SiteFooter } from '@/components/layout/SiteFooter'; import type { ReactNode } from 'react'; -import { Link } from 'react-router-dom'; import { Navbar } from './Navbar'; export function InfoPageLayout({ children }: { children: ReactNode }) { @@ -7,16 +7,7 @@ export function InfoPageLayout({ children }: { children: ReactNode }) {
{children}
-
-
- gitSdm · Repository explorer - -
-
+
); } diff --git a/src/components/layout/SiteFooter.tsx b/src/components/layout/SiteFooter.tsx new file mode 100644 index 0000000..dc0e16c --- /dev/null +++ b/src/components/layout/SiteFooter.tsx @@ -0,0 +1,43 @@ +import { Link, useLocation } from 'react-router-dom'; +import { prefersReducedMotion } from '@/lib/motion-preference'; + +const links = [ + ['Home', '/'], + ['Privacy', '/privacy'], + ['Terms', '/terms'], +] as const; + +export function SiteFooter() { + const { pathname } = useLocation(); + return ( +
+
+
+ gitSdm + © 2026 +
+ +
+
+ ); +} diff --git a/src/components/ui/badge.tsx b/src/components/ui/badge.tsx index b20959d..10d4bbe 100644 --- a/src/components/ui/badge.tsx +++ b/src/components/ui/badge.tsx @@ -49,4 +49,4 @@ function Badge({ }) } -export { Badge, badgeVariants } +export { Badge } diff --git a/src/components/ui/button-variants.ts b/src/components/ui/button-variants.ts new file mode 100644 index 0000000..88edf34 --- /dev/null +++ b/src/components/ui/button-variants.ts @@ -0,0 +1,38 @@ +import { cva } from "class-variance-authority"; +export const buttonVariants = cva( + "group/button inline-flex shrink-0 items-center justify-center rounded-lg border border-transparent bg-clip-padding text-sm font-medium whitespace-nowrap transition-all outline-none select-none focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 active:not-aria-[haspopup]:translate-y-px disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground hover:bg-primary/80", + outline: + "border-border bg-background hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:border-input dark:bg-input/30 dark:hover:bg-input/50", + secondary: + "bg-secondary text-secondary-foreground hover:bg-[color-mix(in_oklch,var(--secondary),var(--foreground)_5%)] aria-expanded:bg-secondary aria-expanded:text-secondary-foreground", + ghost: + "hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:hover:bg-muted/50", + destructive: + "bg-destructive/10 text-destructive hover:bg-destructive/20 focus-visible:border-destructive/40 focus-visible:ring-destructive/20 dark:bg-destructive/20 dark:hover:bg-destructive/30 dark:focus-visible:ring-destructive/40", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: + "h-8 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", + xs: "h-6 gap-1 rounded-[min(var(--radius-md),10px)] px-2 text-xs in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3", + sm: "h-7 gap-1 rounded-[min(var(--radius-md),12px)] px-2.5 text-[0.8rem] in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3.5", + lg: "h-9 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", + icon: "size-8", + "icon-xs": + "size-6 rounded-[min(var(--radius-md),10px)] in-data-[slot=button-group]:rounded-lg [&_svg:not([class*='size-'])]:size-3", + "icon-sm": + "size-7 rounded-[min(var(--radius-md),12px)] in-data-[slot=button-group]:rounded-lg", + "icon-lg": "size-9", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + diff --git a/src/components/ui/button.tsx b/src/components/ui/button.tsx index b033601..df64257 100644 --- a/src/components/ui/button.tsx +++ b/src/components/ui/button.tsx @@ -1,45 +1,9 @@ +import { buttonVariants } from "./button-variants"; import { Button as ButtonPrimitive } from "@base-ui/react/button" -import { cva, type VariantProps } from "class-variance-authority" +import { type VariantProps } from "class-variance-authority" import { cn } from "@/lib/utils" -const buttonVariants = cva( - "group/button inline-flex shrink-0 items-center justify-center rounded-lg border border-transparent bg-clip-padding text-sm font-medium whitespace-nowrap transition-all outline-none select-none focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 active:not-aria-[haspopup]:translate-y-px disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", - { - variants: { - variant: { - default: "bg-primary text-primary-foreground hover:bg-primary/80", - outline: - "border-border bg-background hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:border-input dark:bg-input/30 dark:hover:bg-input/50", - secondary: - "bg-secondary text-secondary-foreground hover:bg-[color-mix(in_oklch,var(--secondary),var(--foreground)_5%)] aria-expanded:bg-secondary aria-expanded:text-secondary-foreground", - ghost: - "hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:hover:bg-muted/50", - destructive: - "bg-destructive/10 text-destructive hover:bg-destructive/20 focus-visible:border-destructive/40 focus-visible:ring-destructive/20 dark:bg-destructive/20 dark:hover:bg-destructive/30 dark:focus-visible:ring-destructive/40", - link: "text-primary underline-offset-4 hover:underline", - }, - size: { - default: - "h-8 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", - xs: "h-6 gap-1 rounded-[min(var(--radius-md),10px)] px-2 text-xs in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3", - sm: "h-7 gap-1 rounded-[min(var(--radius-md),12px)] px-2.5 text-[0.8rem] in-data-[slot=button-group]:rounded-lg has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 [&_svg:not([class*='size-'])]:size-3.5", - lg: "h-9 gap-1.5 px-2.5 has-data-[icon=inline-end]:pr-2 has-data-[icon=inline-start]:pl-2", - icon: "size-8", - "icon-xs": - "size-6 rounded-[min(var(--radius-md),10px)] in-data-[slot=button-group]:rounded-lg [&_svg:not([class*='size-'])]:size-3", - "icon-sm": - "size-7 rounded-[min(var(--radius-md),12px)] in-data-[slot=button-group]:rounded-lg", - "icon-lg": "size-9", - }, - }, - defaultVariants: { - variant: "default", - size: "default", - }, - } -) - function Button({ className, variant = "default", @@ -55,4 +19,4 @@ function Button({ ) } -export { Button, buttonVariants } +export { Button } diff --git a/src/components/ui/sidebar.tsx b/src/components/ui/sidebar.tsx index 284b317..fdef713 100644 --- a/src/components/ui/sidebar.tsx +++ b/src/components/ui/sidebar.tsx @@ -717,5 +717,4 @@ export { SidebarRail, SidebarSeparator, SidebarTrigger, - useSidebar, } diff --git a/src/components/ui/tabs.tsx b/src/components/ui/tabs.tsx index 2adaeb6..8200e66 100644 --- a/src/components/ui/tabs.tsx +++ b/src/components/ui/tabs.tsx @@ -77,4 +77,4 @@ function TabsContent({ className, ...props }: TabsPrimitive.Panel.Props) { ) } -export { Tabs, TabsList, TabsTrigger, TabsContent, tabsListVariants } +export { Tabs, TabsList, TabsTrigger, TabsContent } diff --git a/src/components/viz/AnalysisTab.test.ts b/src/components/viz/AnalysisTab.test.ts index 8becb4d..4496fe7 100644 --- a/src/components/viz/AnalysisTab.test.ts +++ b/src/components/viz/AnalysisTab.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'bun:test'; -import { getAnalysisRelations } from './AnalysisTab'; +import { getAnalysisRelations } from './analysis-relations'; import type { GraphData } from '@/types'; const graph: GraphData = { diff --git a/src/components/viz/AnalysisTab.tsx b/src/components/viz/AnalysisTab.tsx index ff6cfad..694e83c 100644 --- a/src/components/viz/AnalysisTab.tsx +++ b/src/components/viz/AnalysisTab.tsx @@ -1,53 +1,13 @@ +import { getAnalysisRelations } from "./analysis-relations"; import { TooltipHint } from '@/components/ui/tooltip'; import { useMemo } from 'react'; import { cn } from '@/lib/utils'; import { useVizStore } from '@/stores/vizStore'; -import type { GraphData, GraphEdge, GraphNode, RepoAnalysis } from '@/types'; +import type { GraphNode, RepoAnalysis } from '@/types'; import { getFileContext } from '@/lib/file-context'; import { copyToClipboard } from '@/lib/clipboard'; import { Network } from 'lucide-react'; -/** - * Pure relation logic for the Analysis tab (testable without rendering). - * - * Matches the tested `getFileContext` semantics for files: excludes - * `contains` + self-loops, dedups by opposite endpoint, and drops edges - * whose opposite endpoint is missing from the graph. For non-file nodes - * (folder/repo/package) containment edges ARE the relations, so they are - * included instead of excluded. - */ -export function getAnalysisRelations(graph: GraphData, selectedNodeId: string | null): { - outgoing: GraphEdge[]; - incoming: GraphEdge[]; -} { - if (!selectedNodeId) return { outgoing: [], incoming: [] }; - const nodeById = new Map(graph.nodes.map((n) => [n.id, n])); - const selected = nodeById.get(selectedNodeId); - if (!selected) return { outgoing: [], incoming: [] }; - const isFile = selected.type === 'file'; - const rawOutgoing = graph.edges.filter( - (edge) => - edge.source === selected.id && - edge.source !== edge.target && - (isFile ? edge.type !== 'contains' : true) && - nodeById.has(edge.target), - ); - const rawIncoming = graph.edges.filter( - (edge) => - edge.target === selected.id && - edge.source !== edge.target && - (isFile ? edge.type !== 'contains' : true) && - nodeById.has(edge.source), - ); - const outgoing = rawOutgoing.filter( - (edge, index) => rawOutgoing.findIndex((e) => e.target === edge.target) === index, - ); - const incoming = rawIncoming.filter( - (edge, index) => rawIncoming.findIndex((e) => e.source === edge.source) === index, - ); - return { outgoing, incoming }; -} - interface AnalysisTabProps { analysis: RepoAnalysis; selectedNodeId: string | null; diff --git a/src/components/viz/SettingsPopover.tsx b/src/components/viz/SettingsPopover.tsx index 787af6c..fb1266e 100644 --- a/src/components/viz/SettingsPopover.tsx +++ b/src/components/viz/SettingsPopover.tsx @@ -2,7 +2,7 @@ import { useState, useRef, useEffect, useCallback, type ReactNode, type RefObjec import { Settings, X, Check, Eye, EyeOff, KeyRound, GitBranch } from 'lucide-react'; import { cn } from '@/lib/utils'; import { MotionSettings } from './MotionSettings'; -import { buttonVariants } from '@/components/ui/button'; +import { buttonVariants } from '@/components/ui/button-variants'; import { Tooltip, TooltipContent, @@ -206,7 +206,7 @@ export function SettingsPopover({ )} - Settings & Credentials + Settings & Credentials )} @@ -300,7 +300,7 @@ export function SettingsPopover({ GitHub PAT
Add a{' '} {refactorData?.suggestions && refactorData.suggestions.length > 0 ? (
-

Key Refactoring Suggestions & Risks

+

Key Refactoring Suggestions & Risks

{refactorData.suggestions.map((s, idx) => ( [n.id, n])); + const selected = nodeById.get(selectedNodeId); + if (!selected) return { outgoing: [], incoming: [] }; + const isFile = selected.type === 'file'; + const rawOutgoing = graph.edges.filter( + (edge) => + edge.source === selected.id && + edge.source !== edge.target && + (isFile ? edge.type !== 'contains' : true) && + nodeById.has(edge.target), + ); + const rawIncoming = graph.edges.filter( + (edge) => + edge.target === selected.id && + edge.source !== edge.target && + (isFile ? edge.type !== 'contains' : true) && + nodeById.has(edge.source), + ); + const outgoing = rawOutgoing.filter( + (edge, index) => rawOutgoing.findIndex((e) => e.target === edge.target) === index, + ); + const incoming = rawIncoming.filter( + (edge, index) => rawIncoming.findIndex((e) => e.source === edge.source) === index, + ); + return { outgoing, incoming }; +} + diff --git a/src/features/graph/canvas/widgets/LegendPanel.tsx b/src/features/graph/canvas/widgets/LegendPanel.tsx index 7570977..e746809 100644 --- a/src/features/graph/canvas/widgets/LegendPanel.tsx +++ b/src/features/graph/canvas/widgets/LegendPanel.tsx @@ -232,9 +232,9 @@ function LegendContent({
)} - {/* Focus & Selection — merged blast radius + interactive states */} + {/* Focus & Selection — merged blast radius + interactive states */}
- Focus & Selection + Focus & Selection
{blastRadiusActive ? ( <> diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index 7a6fcf2..f995fc9 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -1,3 +1,4 @@ +import type { ChurnContinuation, ChurnResponse } from '@/types/churn'; import type { AIExplainRequest, AIExplainResponse, @@ -142,9 +143,9 @@ export function fetchChurnBatch( owner: string, repo: string, sha: string, - continuation: import('@/types/churn').ChurnContinuation, + continuation: ChurnContinuation, ) { - return request( + return request( `/api/repo/churn?${new URLSearchParams({ owner, repo, branch: sha })}`, { method: 'POST', body: JSON.stringify(continuation) }, ); diff --git a/src/lib/page-metadata.test.ts b/src/lib/page-metadata.test.ts index 188d565..de18535 100644 --- a/src/lib/page-metadata.test.ts +++ b/src/lib/page-metadata.test.ts @@ -7,3 +7,11 @@ test('valid navigation clears 404 robots and privacy canonical metadata', () => expect(pageMetadata('/').canonical).toBe('https://gsdm.site/'); expect(pageMetadata('/mock/todo-app').canonical).toBeUndefined(); }); + +test('terms metadata supports direct and trailing-slash navigation', () => { + for (const path of ['/terms', '/terms/']) { + expect(pageMetadata(path).title).toBe('Terms of use — gitSdm'); + expect(pageMetadata(path).canonical).toBe('https://gsdm.site/terms'); + expect(pageMetadata(path).robots).toBe('index, follow'); + } +}); diff --git a/src/lib/page-metadata.ts b/src/lib/page-metadata.ts index b3bd8d9..35e86b9 100644 --- a/src/lib/page-metadata.ts +++ b/src/lib/page-metadata.ts @@ -1,21 +1,34 @@ export function pageMetadata(path: string) { const privacy = /^\/privacy\/?$/.test(path); + const terms = /^\/terms\/?$/.test(path); const valid = path === '/' || privacy || + terms || /^\/(?!api\/|assets\/)[a-zA-Z0-9][a-zA-Z0-9-]{0,38}\/[a-zA-Z0-9_.-]{1,100}(?:\/search)?\/?$/.test(path); return { title: privacy ? 'Privacy policy — gitSdm' - : valid - ? 'gitSdmᵝ — Git Software Dependency Map' - : 'Page not found — gitSdm', + : terms + ? 'Terms of use — gitSdm' + : valid + ? 'gitSdmᵝ — Git Software Dependency Map' + : 'Page not found — gitSdm', description: privacy ? 'How gitSdm stores settings, sends repository content to providers, and handles caches and logs.' - : valid - ? 'Interactive dependency visualization for exploring repository files, modules, and architecture.' - : 'This page could not be found. Return to gitSdm to explore a repository.', - canonical: path === '/' ? 'https://gsdm.site/' : privacy ? 'https://gsdm.site/privacy' : undefined, + : terms + ? 'Rules for using gitSdm, including commercial use, automation, repository content, and service limits.' + : valid + ? 'Interactive dependency visualization for exploring repository files, modules, and architecture.' + : 'This page could not be found. Return to gitSdm to explore a repository.', + canonical: + path === '/' + ? 'https://gsdm.site/' + : privacy + ? 'https://gsdm.site/privacy' + : terms + ? 'https://gsdm.site/terms' + : undefined, robots: valid ? 'index, follow' : 'noindex', }; } diff --git a/src/main.tsx b/src/main.tsx index 61186ec..6dbaaef 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -11,11 +11,13 @@ async function start() { ? (await import('@/pages/HomePage')).HomePage : /^\/privacy\/?$/.test(initialPath) ? (await import('@/pages/PrivacyPage')).PrivacyPage - : /^\/[^/]+\/[^/]+\/search\/?$/.test(initialPath) - ? (await import('@/pages/SearchPage')).SearchPage - : /^\/[^/]+\/[^/]+\/?$/.test(initialPath) - ? (await import('@/pages/VizPage')).VizPage - : (await import('@/pages/NotFoundPage')).NotFoundPage; + : /^\/terms\/?$/.test(initialPath) + ? (await import('@/pages/TermsPage')).TermsPage + : /^\/[^/]+\/[^/]+\/search\/?$/.test(initialPath) + ? (await import('@/pages/SearchPage')).SearchPage + : /^\/[^/]+\/[^/]+\/?$/.test(initialPath) + ? (await import('@/pages/VizPage')).VizPage + : (await import('@/pages/NotFoundPage')).NotFoundPage; createRoot(document.getElementById('root')!).render( diff --git a/src/pages/HomePage.tsx b/src/pages/HomePage.tsx index 8874d69..ba0cad6 100644 --- a/src/pages/HomePage.tsx +++ b/src/pages/HomePage.tsx @@ -1,15 +1,18 @@ +import { SiteFooter } from '@/components/layout/SiteFooter'; import { prefersReducedMotion } from '@/lib/motion-preference'; -import { useEffect, useState } from "react"; -import { Navbar } from "@/components/layout/Navbar"; -import { HeroSection } from "@/components/home/HeroSection"; -import { StatsStrip } from "@/components/home/StatsStrip"; -import { HowItWorks } from "@/components/home/HowItWorks"; -import { Trending } from "@/components/home/Trending"; -import { CapabilityGroups } from "@/components/home/CapabilityGroups"; -import { LAST_REPO_KEY } from "@/lib/utils"; +import { useEffect, useState } from 'react'; +import { Navbar } from '@/components/layout/Navbar'; +import { HeroSection } from '@/components/home/HeroSection'; +import { StatsStrip } from '@/components/home/StatsStrip'; +import { HowItWorks } from '@/components/home/HowItWorks'; +import { Trending } from '@/components/home/Trending'; +import { CapabilityGroups } from '@/components/home/CapabilityGroups'; +import { LAST_REPO_KEY } from '@/lib/utils'; export function HomePage() { - const [repoUrl] = useState(() => typeof localStorage === 'undefined' ? '' : localStorage.getItem(LAST_REPO_KEY) ?? ""); + const [repoUrl] = useState(() => + typeof localStorage === 'undefined' ? '' : (localStorage.getItem(LAST_REPO_KEY) ?? ''), + ); useEffect(() => { const sectionId = window.location.hash.slice(1); @@ -17,8 +20,8 @@ export function HomePage() { const timer = window.setTimeout(() => { document.getElementById(sectionId)?.scrollIntoView({ - behavior: prefersReducedMotion() ? "instant" : "smooth", - block: "start", + behavior: prefersReducedMotion() ? 'instant' : 'smooth', + block: 'start', }); }, 80); @@ -35,45 +38,7 @@ export function HomePage() {
- +
); } diff --git a/src/pages/PrivacyPage.tsx b/src/pages/PrivacyPage.tsx index b9d8c89..7910205 100644 --- a/src/pages/PrivacyPage.tsx +++ b/src/pages/PrivacyPage.tsx @@ -1,3 +1,4 @@ +import { Link } from 'react-router-dom'; import { InfoPageLayout } from '@/components/layout/InfoPageLayout'; export function PrivacyPage() { @@ -7,28 +8,95 @@ export function PrivacyPage() {

Your data / Privacy

Privacy policy

-

Updated September 8, 2026

-

gitSdm saves your settings in your browser and sends data to its server, GitHub, and the services used by its AI features. This page explains what is stored and shared. If you use a self-hosted installation, its hosting and provider settings may differ.

+

Updated September 12, 2026

+

+ gitSdm saves your settings in your browser and sends data to its server, GitHub, and the services used by + its AI features. This page explains what is stored and shared. If you use a self-hosted installation, its + hosting and provider settings may differ. +

What your browser saves

-

Your browser saves GitHub tokens and AI API keys in localStorage on this device. It also remembers your workspace preferences and the last repository you opened. These stay saved after you close the browser. Use Clear in Settings to remove saved keys, or clear this site's browser data to remove all local settings. Avoid saving keys on a shared device.

+

+ Your browser saves GitHub tokens and AI API keys in localStorage on this device. It also remembers your + workspace preferences and the last repository you opened. These stay saved after you close the browser. Use + Clear in Settings to remove saved keys, or clear this site's browser data to remove all local settings. + Avoid saving keys on a shared device. +

What gitSdm sends to other services

-

When you request a repository, your browser sends its owner and name to the gitSdm server, along with relevant saved credentials in request headers. The server uses your GitHub credentials to fetch repository details and files from GitHub. Saved credentials therefore also leave your device when used for these requests.

-

AI features may send your prompts, questions, and relevant repository content to the configured AI provider. That content can include private code accessed with your GitHub token. To build a semantic search index, gitSdm sends sections of code to the configured embedding provider. The Ask feature can send code found by the search to an AI provider to answer your question.

-

The provider may be Google Gemini, OpenAI, Anthropic, EdgeOne, or another compatible service. Each provider has its own rules for privacy and data retention. If you have not added your own API key, the installation may use a key supplied by its operator. Only submit code you have permission to share with those services.

+

+ When you request a repository, your browser sends its owner and name to the gitSdm server, along with + relevant saved credentials in request headers. The server uses your GitHub credentials to fetch repository + details and files from GitHub. Saved credentials therefore also leave your device when used for these + requests. +

+

+ AI features may send your prompts, questions, and relevant repository content to the configured AI provider. + That content can include private code accessed with your GitHub token. To build a semantic search index, + gitSdm sends sections of code to the configured embedding provider. The Ask feature can send code found by + the search to an AI provider to answer your question. +

+

+ The provider may be Google Gemini, OpenAI, Anthropic, EdgeOne, or another compatible service. Each provider + has its own rules for privacy and data retention. If you have not added your own API key, the installation + may use a key supplied by its operator. Only submit code you have permission to share with those services. +

Server storage and logs

-

The server keeps repository analysis and AI results in memory so it can reuse them. These caches have size limits and expire after a set time. Semantic search indexes are also kept in memory, but have no fixed expiry. Clearing your browser data does not delete server caches, search indexes, or records held by providers.

-

Application logs record request routes, timing, repository identifiers, and errors. Provider error messages may contain information you submitted. Hosting and network providers may also process connection details, such as your IP address. How long logs are kept depends on who operates the installation; there is no single retention period across deployments.

+

+ The server keeps repository analysis and AI results in memory so it can reuse them. These caches have size + limits and expire after a set time. Completed semantic search indexes expire 30 minutes after they are + stored. Saved progress for resumable indexing contains code snippets, embeddings, and a list of completed + files; its 25-hour expiry resets whenever progress is saved. Expired entries are removed during later access + or cleanup. Memory limits can remove completed indexes sooner, and a server restart clears in-memory data. + Clearing your browser data does not delete server caches, search indexes, or records held by providers. +

+

+ Application logs record request routes, timing, repository identifiers, and errors. Provider error messages + may contain information you submitted. Hosting and network providers may also process connection details, + such as your IP address. How long logs are kept depends on who operates the installation; there is no single + retention period across deployments. +

+
+
+

Usage limits

+

+ The server processes your IP address to limit requests and stores a hash of that address with a request + counter. A hash is not a guarantee of anonymity. It also tracks shared usage totals to control provider + costs. Depending on the installation, these counters are held in memory or sent to Upstash Redis. Counters + expire at the end of their usage window; in-memory cleanup happens on later requests. +

+

+ Repository caches and search indexes use hashed credential identifiers to separate access scopes. Raw API + keys and tokens are not used as cache keys. Clearing browser data does not reset server-side usage limits. +

Other services and your choices

-

gitSdm loads fonts from Google Fonts and may show images hosted by GitHub or other services. Your browser shares connection details with those services when it loads these files. The app itself does not include a dedicated advertising or analytics tracker.

-

You can try the mock demos without adding any credentials. To remove saved keys, use Settings. To revoke a key or token, use the service that issued it. For privacy questions, see the contact information on the maintainer's profile, or contact your operator if you use a self-hosted installation. Do not post tokens, private code, or personal information in public issues.

+

+ gitSdm loads fonts from Google Fonts and may show images hosted by GitHub or other services. Your browser + shares connection details with those services when it loads these files. The app itself does not include a + dedicated advertising or analytics tracker. +

+

+ You can try the mock demos without adding any credentials. To remove saved keys, use Settings. To revoke a + key or token, use the service that issued it. For privacy questions, email{' '} + + hello@gsdm.site + + , or contact your operator if you use a self-hosted installation. Do not post tokens, private code, or + personal information in public issues. +

+

+ See the{' '} + + Terms of use + {' '} + for the rules on commercial use, automation, and service limits. +

diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 8ea801b..ed4df46 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -141,7 +141,7 @@ export function SearchPage() {

- Search & discover + Search & discover

Find the idea behind the code.

diff --git a/src/pages/TermsPage.tsx b/src/pages/TermsPage.tsx new file mode 100644 index 0000000..ef3c9e5 --- /dev/null +++ b/src/pages/TermsPage.tsx @@ -0,0 +1,94 @@ +import { Link } from 'react-router-dom'; +import { InfoPageLayout } from '@/components/layout/InfoPageLayout'; + +export function TermsPage() { + return ( + +

+
+

Using gitSdm / Terms

+

Terms of use

+

Updated September 12, 2026

+

+ These terms describe how you may use the gitSdm service at gsdm.site. Questions or permission requests can + be sent to{' '} + + hello@gsdm.site + + . +

+
+
+

Free access and commercial use

+

+ gitSdm is free to use for personal and commercial projects. If you supply your own API key, your provider + may charge you under its own pricing and terms. Permission to use gitSdm does not change the licenses or + permissions attached to repositories you explore. +

+

+ These rules apply to the hosted service. The gitSdm source code remains governed by its software license; + self-hosted installations may have different operating policies. +

+
+
+

Repository content and credentials

+

+ Only access repositories and submit content you have permission to use. Before using AI or semantic search + with private code, make sure you are allowed to send it to the configured providers. You retain your rights + in your content. +

+

+ Keep your keys and tokens secure and revoke them with their provider if exposed. Read the{' '} + + Privacy policy + {' '} + for details about credentials, repository content, and storage. +

+
+
+

Automation and service limits

+

+ Contact us for permission before running automated bulk requests against the hosted service, including + large-scale repository indexing or repeated API jobs. Normal use of the app, including the indexing it + performs for you, does not require separate permission. +

+

+ Do not bypass rate limits, rotate credentials or network addresses to evade restrictions, attempt + unauthorized access, or deliberately disrupt the service. Bringing your own API key does not exempt you from + these rules. +

+

+ Requests may be queued, paused, or refused when capacity or usage limits are reached. We may restrict access + to address abuse or protect availability. If you believe a restriction is a mistake, contact us. +

+
+
+

Check results before relying on them

+

+ Dependency maps, search results, and AI answers can be incomplete, outdated, or wrong. Check the source code + before making changes or decisions. Partial search results cover only the files indexed so far. +

+

+ gitSdm depends on GitHub, hosting services, and AI providers. Features may be unavailable during outages, + maintenance, or provider limits. Search indexes and cached results are temporary; keep your own copies of + anything you need. +

+
+
+

Changes and questions

+

+ We will update this page and its date when these terms change. Changes apply going forward. Nothing in these + terms removes rights that applicable law does not allow to be excluded. +

+

+ For questions about these terms or use of the service, email{' '} + + hello@gsdm.site + + . Do not include API keys, tokens, or private code. +

+
+
+ + ); +} diff --git a/src/pages/VizPage.tsx b/src/pages/VizPage.tsx index beddd32..11773ee 100644 --- a/src/pages/VizPage.tsx +++ b/src/pages/VizPage.tsx @@ -21,7 +21,7 @@ const ArchitectureView = lazy(() => default: m.ArchitectureView, })), ); -import { ContributorsView } from '@/components/contributors/ContributorsView'; +const ContributorsView = lazy(() => import('@/components/contributors/ContributorsView').then((m) => ({ default: m.ContributorsView }))); import { ExplorerPanel } from '@/components/explorer/ExplorerPanel'; import { VizError } from '@/components/viz/VizError'; import { Info, Loader2 } from 'lucide-react'; @@ -243,7 +243,11 @@ export function VizPage() { )} - {activeView === 'contributors' && } + {activeView === 'contributors' && ( + Loading contributors…
}> + + + )} {activeView === 'commits' && ( Date: Sat, 12 Sep 2026 17:38:15 +0700 Subject: [PATCH 10/27] feat(ai): unify providers, add custom endpoint settings, and harden tool lifecycle --- .env.example | 10 +- AGENTS.md | 14 +- ARCHITECTURE.md | 1 - CONTRIBUTING.md | 2 +- README.md | 28 +++- ROADMAP.md | 2 +- SECURITY.md | 2 +- e2e/ai-settings-regressions.e2e.ts | 87 ++++++++++ e2e/interface.e2e.ts | 18 +- server/ai/chat-config.test.ts | 37 +++++ server/ai/chat-config.ts | 75 +++++++++ server/ai/provider.test.ts | 13 -- server/ai/provider.ts | 148 ++++------------- server/ai/public-chat-fetch.test.ts | 12 ++ server/ai/public-chat-fetch.ts | 73 +++++++++ server/ai/service.ts | 42 +---- server/api-router.ts | 26 ++- server/search/embedding-config.test.ts | 45 +++++ server/search/embedding-config.ts | 23 +++ server/search/embedding-provider.test.ts | 59 +++++-- server/search/embedding-provider.ts | 58 ++----- server/search/index-identity.test.ts | 18 +- server/search/index-identity.ts | 13 +- src/components/viz/AIErrorCard.tsx | 12 +- src/components/viz/AISidebar.tsx | 4 +- src/components/viz/LearningPathTab.tsx | 2 - src/components/viz/SettingsPopover.tsx | 155 +++++++++++++----- src/components/viz/ai-sidebar/AiCenterTab.tsx | 54 ++---- .../viz/ai-sidebar/hooks/useAiCenterState.ts | 141 +++++++--------- .../hooks/useArchitectureState.ts | 9 +- src/features/ai/tool-cache.test.ts | 30 ++++ src/features/ai/tool-cache.ts | 34 ++++ src/features/ai/useAiTasks.ts | 25 ++- src/lib/apiClient.ts | 30 ++-- src/pages/PrivacyPage.tsx | 4 +- src/stores/chatConfigStore.ts | 28 ++++ 36 files changed, 865 insertions(+), 469 deletions(-) create mode 100644 e2e/ai-settings-regressions.e2e.ts create mode 100644 server/ai/chat-config.test.ts create mode 100644 server/ai/chat-config.ts create mode 100644 server/ai/public-chat-fetch.test.ts create mode 100644 server/ai/public-chat-fetch.ts create mode 100644 server/search/embedding-config.test.ts create mode 100644 server/search/embedding-config.ts create mode 100644 src/features/ai/tool-cache.test.ts create mode 100644 src/features/ai/tool-cache.ts create mode 100644 src/stores/chatConfigStore.ts diff --git a/.env.example b/.env.example index 50a0766..919b255 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,15 @@ # GitHub API (optional, increases rate limits for public repos) GITHUB_TOKEN= -# AI provider: gemini | openai | anthropic | edgeone | mock +# AI provider: gemini | openai | anthropic | mock AI_PROVIDER=mock # Optional separate provider for semantic search: openai, gemini, or mock. # EMBEDDING_PROVIDER=openai +# Independent embedding credentials and endpoint (recommended). +# Legacy OPENAI_/GEMINI_ settings are used when these are absent. +# EMBEDDING_API_KEY= +# EMBEDDING_API_BASE=https://api.openai.com/v1 +# EMBEDDING_MODEL=text-embedding-3-small # OpenAI-compatible example: OPENAI_API_BASE=https://api.featherless.ai/v1 # OPENAI_EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B, EMBEDDING_DIMENSIONS=4096 @@ -12,19 +17,16 @@ AI_PROVIDER=mock GEMINI_API_KEY= OPENAI_API_KEY= ANTHROPIC_API_KEY= -EDGEONE_API_KEY= # Model overrides (optional) GEMINI_MODEL=gemini-2.5-flash GEMINI_API_VERSION=v1alpha OPENAI_MODEL=gpt-4o-mini ANTHROPIC_MODEL=claude-3-5-haiku-latest -EDGEONE_MODEL=@makers/deepseek-v4-flash # Provider API base URL overrides (optional) OPENAI_API_BASE= ANTHROPIC_API_BASE= -EDGEONE_API_BASE=https://ai-gateway.edgeone.link/v1 # Embedding overrides (optional) OPENAI_EMBEDDING_MODEL=openrouter/openai/text-embedding-3-large diff --git a/AGENTS.md b/AGENTS.md index 040fe5b..5c5a4f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ | Graph Viz | react-force-graph-2d, d3-force, d3-hierarchy | | Diagrams | Mermaid 11, html-to-image, jsPDF | | Backend | Express (dev + prod server), Zod validation | -| AI Providers | Gemini, OpenAI, Anthropic, EdgeOne Makers, Mock — unified via `createProvider()` | +| AI Providers | Gemini, OpenAI, Anthropic, Mock — unified via `createProvider()` | | GitHub API | Octokit REST v21 | | Search | In-memory vector store with chunking + embedding + QA engine | | Security | dompurify (sanitization) | @@ -40,7 +40,7 @@ gitSdm/ ├── server/ # Backend services & router │ ├── ai/ # AI provider abstraction + task handlers │ │ ├── prompts.ts # Shared AI prompt templates -│ │ ├── provider.ts/test.ts # AI provider (Gemini/OpenAI/Anthropic/EdgeOne/Mock) +│ │ ├── provider.ts/test.ts # AI provider (Gemini/OpenAI/Anthropic/Mock) │ │ ├── service.ts/test.ts # AI service orchestration │ │ └── tasks/ # Individual AI tasks (diagram, explain, onboarding, playground, refactor) │ ├── cache/ # LRU caching layer (lru.ts/test.ts) @@ -108,7 +108,7 @@ gitSdm/ All AI interactions go through `server/ai/provider.ts`. Do NOT call SDKs directly in task code. ```text -Task handler → summarizer / task → createProvider(type) → { Gemini | OpenAI | Anthropic | EdgeOne | Mock } +Task handler → summarizer / task → createProvider(type) → { Gemini | OpenAI | Anthropic | Mock } ``` - Uses `createProvider(overrideKey?)` → returns `AIProvider` with `.complete(messages, options?)` @@ -133,7 +133,7 @@ Manual pathname-matching pattern in `server/router/`. Route files export `handle ### API Client (Frontend) -`src/lib/apiClient.ts` — Typed fetch wrapper via `apiFetch(url, options?)`. Authentication tokens stored in `localStorage` (`gitsdm_gemini_api_key`, `gitsdm_github_pat`) and sent as `X-Gemini-API-Key` / `X-GitHub-Token` headers. +`src/lib/apiClient.ts` — Typed fetch wrapper via `apiFetch(url, options?)`. Authentication tokens stored in `localStorage` (`gitsdm_gemini_api_key`, `gitsdm_github_pat`) and sent as `X-AI-API-Key` (with `X-Gemini-API-Key` fallback) / `X-GitHub-Token` headers. ### Toast Notification Pattern @@ -225,22 +225,18 @@ bun run lint # ESLint check | Variable | Default | Description | | ------------------------ | ------------------------------------------ | -------------------------------------- | | `GITHUB_TOKEN` | — | GitHub PAT for API rate limits | -| `AI_PROVIDER` | `mock` | Provider: gemini, openai, anthropic, edgeone, mock | +| `AI_PROVIDER` | `mock` | Provider: gemini, openai, anthropic, mock | | `GEMINI_API_KEY` | — | Gemini API key | | `OPENAI_API_KEY` | — | OpenAI API key | | `ANTHROPIC_API_KEY` | — | Anthropic API key | -| `EDGEONE_API_KEY` | — | EdgeOne Makers Models API key (alias: `MAKERS_MODELS_KEY`) | | `OPENAI_API_BASE` | OpenAI default | Custom API base URL | | `ANTHROPIC_API_BASE` | Anthropic default | Custom API base URL | -| `EDGEONE_API_BASE` | `https://ai-gateway.edgeone.link/v1` | EdgeOne API base URL | | `OPENAI_EMBEDDING_MODEL` | `openrouter/openai/text-embedding-3-large` | Embedding model | -| `EDGEONE_EMBEDDING_MODEL`| `openrouter/openai/text-embedding-3-large` | Embedding model | | `EMBEDDING_DIMENSIONS` | `3072` | Vector dimension count | | `GEMINI_MODEL` | `gemini-2.5-flash` | Gemini model override | | `GEMINI_API_VERSION` | `v1alpha` | Gemini API version | | `OPENAI_MODEL` | `gpt-4o-mini` | OpenAI model override | | `ANTHROPIC_MODEL` | `claude-3-5-haiku-latest` | Anthropic model override | -| `EDGEONE_MODEL` | `@makers/deepseek-v4-flash` | EdgeOne model override | | `GEMINI_EMBEDDING_MODEL` | `gemini-embedding-001` | Gemini embedding model override | | `TOKEN_CACHE_HASH_SECRET`| — | Cache key hashing secret (production) | | `HOST` | `0.0.0.0` | Production server bind | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a79e12a..badef3b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -67,7 +67,6 @@ - Google Gemini (`gemini-2.5-flash`) - OpenAI (`gpt-4o-mini`) - Anthropic (`claude-3-5-haiku-latest`) - - EdgeOne Makers (`@makers/deepseek-v4-flash`) - Offline Mock Provider (fixtures for testing and development without API keys) - **AI Tasks (`server/ai/tasks/`):** Dedicated pipelines for architectural summaries, Mermaid flowchart generation, onboarding tours, refactoring risk detection, and dependency health audits. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b22d8e1..a1e4a8d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ bun install cp .env.example .env ``` -Make sure to add your `GITHUB_TOKEN` and any AI provider API keys (`GEMINI_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `EDGEONE_API_KEY`) to the `.env` file for full feature support during development. Offline development works out-of-the-box with `AI_PROVIDER=mock`. +Make sure to add your `GITHUB_TOKEN` and any AI provider API keys (`GEMINI_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) to the `.env` file for full feature support during development. Offline development works out-of-the-box with `AI_PROVIDER=mock`. --- diff --git a/README.md b/README.md index 830d478..c71033f 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ Usage controls combine per-IP counters with deployment-wide budgets. Each IP get Direct connections use the socket IP. Without trusted-proxy configuration, proxied users share the proxy's allowance; missing socket identities share an `unknown` bucket. Configure only proxies you control that append or replace the forwarded chain. Local limiter storage caps at 10,000 live counters and rejects new counters at capacity rather than evicting active quotas. IPv6 addresses are normalized but counted individually. -EdgeOne serves completions only. Semantic search with `AI_PROVIDER=edgeone` requires `GEMINI_API_KEY` or `OPENAI_API_KEY` for embeddings. Shared mode fails closed when its backend is missing or unavailable. Local counters reset on restart and cannot impose a deployment-wide budget across multiple instances. Configure shared mode before scaling out. These are request/input limits, not a dollar spending cap; provider pricing varies. Reservations are not refunded after failures because upstream work may have been billed. The shared implementation uses the [Redis REST API](https://upstash.com/docs/redis/features/restapi) with atomic Lua reservations. @@ -82,7 +81,6 @@ Open [http://localhost:5173](http://localhost:5173) in your browser. | **Google Gemini** | `gemini` | Set `GEMINI_API_KEY` (model: `gemini-2.5-flash`). | | **OpenAI** | `openai` | Set `OPENAI_API_KEY` (model: `gpt-4o-mini`). | | **Anthropic** | `anthropic` | Set `ANTHROPIC_API_KEY` (model: `claude-3-5-haiku-latest`). | -| **EdgeOne Makers** | `edgeone` | Set `EDGEONE_API_KEY` (model: `@makers/deepseek-v4-flash`). | --- @@ -124,3 +122,29 @@ bun run test:e2e # Run Playwright end-to-end tests (mock mode) ## License MIT. See [LICENSE](LICENSE). + +### Independent embedding configuration + +Set `EMBEDDING_PROVIDER=openai`, `gemini`, or `mock` explicitly to keep semantic +search independent of `AI_PROVIDER`. Set `EMBEDDING_API_KEY`, `EMBEDDING_MODEL`, +and (for OpenAI-compatible services) `EMBEDDING_API_BASE` independently of chat. +Gemini uses its SDK endpoint. Keep `EMBEDDING_DIMENSIONS` matched to the model. + +Legacy OpenAI/Gemini key and embedding-model settings remain fallbacks. Without +`EMBEDDING_PROVIDER`, legacy selection still follows `AI_PROVIDER`. Configure all +embedding settings explicitly before changing chat credentials or endpoints. +Changing embedding model or endpoint invalidates existing search index identities. + +### Browser chat settings + +Settings accepts an AI API key, an explicit chat provider, an optional model, +and an optional OpenAI-compatible base URL. Save applies the fields together; +Clear removes the saved key and chat overrides. These settings apply to AI tasks +and Ask AI, not semantic indexing or embedding generation. Existing saved keys +remain supported. + +Custom URLs require the user's own key and a public HTTPS endpoint. Local and private +network addresses are blocked, DNS addresses are checked and pinned for the connection, +and redirects are not followed. No server allowlist configuration is needed. +Requests use per-request configuration and cache identities include provider, model, +endpoint, and hashed credentials. diff --git a/ROADMAP.md b/ROADMAP.md index de466a7..121f8af 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -12,7 +12,7 @@ - **Dependency Health Report** — Core health panel, version freshness checker (npm), license compliance audits, and real-time visual highlight alerts (amber borders/warning badges) on the force-directed canvas. - **Code Churn & Hotspot Overlays** — Visual overlay for 90-day commit frequency, author ownership risk (3+ distinct authors), and LRU-cached churn service. - **Complexity Score per Module** — Graph-derived complexity metrics (LOC, import count, export count) with Color/Size by Complexity modes. -- **Unified Multi-Provider AI** — Support for Gemini, OpenAI, Anthropic, EdgeOne Makers, and an offline Mock provider. +- **Unified Multi-Provider AI** — Support for Gemini, OpenAI, Anthropic, and an offline Mock provider. - **Personalized Onboarding Paths** — AI-generated custom reading tours. User pastes a repo URL + describes their goal, and the system returns a guided walkthrough ("start with these 5 files, in this order") with graph node highlights. Implemented as `LearningPathTab`. - **Commit & Tag Snapshot Diffing** — Extended Compare Branch to accept commit SHAs and tags as comparison targets. Includes Branches/Tags/SHA sub-tabs in the picker UI, `/api/repo/tags` endpoint, and full graph overlay for any ref type. diff --git a/SECURITY.md b/SECURITY.md index 4948a4a..bef8988 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -80,6 +80,6 @@ Research conducted in good faith under this policy is authorized: ## Security Best Practices for Users -* **Environment Variables:** Never check in your `GITHUB_TOKEN`, `GEMINI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or `EDGEONE_API_KEY` into git repositories. Use the local `.env` file (which is gitignored). +* **Environment Variables:** Never check in your `GITHUB_TOKEN`, `GEMINI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY` into git repositories. Use the local `.env` file (which is gitignored). * **Private Repositories:** Ensure access tokens passed to `gitSdm` use the minimum necessary scopes (ideally read-only permissions for repository metadata/trees). * **Docker Deployments:** When running via Docker, bind the server to `127.0.0.1` unless external access is explicitly required, and configure a reverse proxy with TLS. diff --git a/e2e/ai-settings-regressions.e2e.ts b/e2e/ai-settings-regressions.e2e.ts new file mode 100644 index 0000000..870ae16 --- /dev/null +++ b/e2e/ai-settings-regressions.e2e.ts @@ -0,0 +1,87 @@ +import { test, expect } from '@playwright/test'; + +test('health completion is delivered after leaving and returning to the panel', async ({ page }) => { + let release = () => {}; + const held = new Promise((resolve) => { release = resolve; }); + let calls = 0; + await page.route('**/api/ai/health', async (route) => { + calls++; + await held; + await route.fulfill({ json: { + scores: { maintainability: 80, modularity: 80, readability: 80, architecture: 80, complexity: 80 }, + summary: 'Completed after remount', cached: false, + } }); + }); + await page.route('**/api/ai/explain', (route) => route.fulfill({ json: { explanation: 'Example' } })); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await page.getByRole('button', { name: /Health Audit/ }).click(); + await expect.poll(() => calls).toBe(1); + await page.getByRole('tab', { name: 'Overview', exact: true }).click(); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + release(); + await expect(page.getByText('Completed after remount', { exact: true })).toBeVisible(); + expect(calls).toBe(1); +}); + +test('saving chat settings refreshes the active AI architecture diagram', async ({ page }) => { + let calls = 0; + await page.route('**/api/ai/mermaid', (route) => { + calls++; + return route.fulfill({ json: { diagram: `graph TD; A[Model${calls}] --> B[Done]`, cached: false } }); + }); + await page.goto('/mock/todo-app'); + await page.getByRole('button', { name: 'Architecture', exact: true }).click(); + await page.getByRole('button', { name: 'AI Enhanced', exact: true }).click(); + await expect(page.getByText('Model1', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await page.getByRole('textbox', { name: 'AI API key', exact: true }).fill('test-key'); + await page.getByRole('textbox', { name: 'Chat model', exact: true }).fill('new-model'); + await page.getByRole('button', { name: 'Save AI API key', exact: true }).click(); + await expect(page.getByText('Model2', { exact: true })).toBeVisible(); + await expect(page.getByText('Model1', { exact: true })).toHaveCount(0); + expect(calls).toBe(2); +}); + +test('an older explanation cannot hide the current failure', async ({ page }) => { + let release = () => {}; + const held = new Promise((resolve) => { release = resolve; }); + let started = false; + await page.route('**/api/ai/explain', async (route) => { + started = true; + await held; + await route.fulfill({ json: { explanation: 'Old explanation' } }); + }); + await page.route('**/api/ai/explain-lif', (route) => route.fulfill({ status: 500, json: { error: 'Failed' } })); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await expect.poll(() => started).toBe(true); + await page.getByRole('button', { name: 'ELI5 MODE', exact: true }).click(); + await expect(page.getByText('AI explanation failed. Please try again.', { exact: true })).toBeVisible(); + const response = page.waitForResponse('**/api/ai/explain'); + release(); + await response; + await expect(page.getByRole('button', { name: 'Retry Request', exact: true })).toBeVisible(); + await expect(page.getByText('Old explanation', { exact: true })).toHaveCount(0); +}); + +test('saving changed chat settings refreshes cached explanations', async ({ page }) => { + let calls = 0; + await page.route('**/api/ai/explain', (route) => { + calls++; + return route.fulfill({ json: { explanation: `Answer ${calls}` } }); + }); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await expect(page.getByText('Answer 1', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Settings and credentials', exact: true }).click(); + await page.getByRole('textbox', { name: 'AI API key', exact: true }).fill('test-key'); + await page.getByRole('textbox', { name: 'Chat model', exact: true }).fill('test-model'); + await page.getByRole('button', { name: 'Save AI API key', exact: true }).click(); + await expect(page.getByText('Answer 2', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Close settings', exact: true }).click(); + await page.getByRole('tab', { name: 'Overview', exact: true }).click(); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await expect(page.getByText('Answer 2', { exact: true })).toBeVisible(); + expect(calls).toBe(2); +}); diff --git a/e2e/interface.e2e.ts b/e2e/interface.e2e.ts index 0cea692..e2b0706 100644 --- a/e2e/interface.e2e.ts +++ b/e2e/interface.e2e.ts @@ -6,7 +6,7 @@ test('settings closes with Escape and restores its accessible trigger', async ({ const trigger = page.getByRole('button', { name: 'Settings and credentials', exact: true }); await trigger.click(); const dialog = page.getByRole('dialog', { name: 'Settings and credentials' }); - const key = dialog.getByLabel('Google AI key', { exact: true }); + const key = dialog.getByLabel('AI API key', { exact: true }); await expect(key).toBeFocused(); await key.press('Escape'); await expect(dialog).toHaveCount(0); @@ -184,3 +184,19 @@ test('panel resizing preserves the graph and inspector context follows related f await expect(inspector).toContainText('src/context/TodoContext.tsx'); await expect(page.getByRole('tabpanel', { name: 'Details' }).getByRole('heading', { name: 'TodoContext.tsx', exact: true })).toBeVisible(); }); + + test('health audit stops after an API error and retries only on request', async ({ page }) => { + let calls = 0; + await page.route('**/api/ai/**', route => { + if (route.request().url().includes('/health')) calls++; + return route.fulfill({ status: 500, json: { error: 'Internal Server Error' } }); + }); + await page.goto('/mock/todo-app'); + await page.getByRole('tab', { name: 'AI tools', exact: true }).click(); + await page.getByRole('button', { name: /Health Audit/ }).click(); + await expect(page.getByText('AI request failed. Check your settings and try again.', { exact: true })).toBeVisible(); + await page.waitForTimeout(1000); + expect(calls).toBe(1); + await page.getByRole('button', { name: 'Retry Request', exact: true }).click(); + await expect.poll(() => calls).toBe(2); + }); diff --git a/server/ai/chat-config.test.ts b/server/ai/chat-config.test.ts new file mode 100644 index 0000000..1201689 --- /dev/null +++ b/server/ai/chat-config.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from 'bun:test'; +import { chatIdentity, chatOverrides, readChatOverrides, resolveChatConfig } from './chat-config'; + +test('custom URLs require a user key and a valid HTTPS endpoint', () => { + const headers = new Headers({ + 'x-ai-provider': 'openai', + 'x-ai-base-url': 'https://api.openai.com/v1', + 'x-ai-model': 'custom-model', + }); + expect(() => readChatOverrides(headers)).toThrow('own API key'); + expect(readChatOverrides(headers, 'user-key').model).toBe('custom-model'); + for (const url of ['http://127.0.0.1/v1', 'https://api.openai.com/v1?redirect=1']) { + headers.set('x-ai-base-url', url); + expect(() => readChatOverrides(headers, 'user-key')).toThrow(); + } +}); + +test('concurrent request overrides stay isolated and distinguish cache identities', async () => { + const run = (model: string) => + chatOverrides.run({ provider: 'openai', model, baseURL: 'https://api.openai.com/v1' }, async () => { + await Promise.resolve(); + return { config: resolveChatConfig('custom-key'), identity: chatIdentity('custom-key') }; + }); + const [a, b] = await Promise.all([run('model-a'), run('model-b')]); + expect(a.config.model).toBe('model-a'); + expect(b.config.model).toBe('model-b'); + expect(a.identity).not.toBe(b.identity); + expect(a.identity).not.toContain('custom-key'); +}); + +test('endpoint and credential changes invalidate chat identity', () => { + const a = chatOverrides.run({ provider: 'openai', baseURL: 'https://one.example/v1' }, () => chatIdentity('key-one')); + const b = chatOverrides.run({ provider: 'openai', baseURL: 'https://two.example/v1' }, () => chatIdentity('key-one')); + const c = chatOverrides.run({ provider: 'openai', baseURL: 'https://one.example/v1' }, () => chatIdentity('key-two')); + expect(a).not.toBe(b); + expect(a).not.toBe(c); +}); diff --git a/server/ai/chat-config.ts b/server/ai/chat-config.ts new file mode 100644 index 0000000..289ee24 --- /dev/null +++ b/server/ai/chat-config.ts @@ -0,0 +1,75 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { hashToken } from '../cache/lru'; +import { AppError } from '../utils/errors'; + +export interface ChatOverrides { + provider?: string; + model?: string; + baseURL?: string; +} +export const chatOverrides = new AsyncLocalStorage(); +const clean = (value?: string | null) => value?.trim() || undefined; + +export function readChatOverrides(headers: Headers, apiKey?: string): ChatOverrides { + const provider = clean(headers.get('x-ai-provider')); + const model = clean(headers.get('x-ai-model')); + const baseURL = clean(headers.get('x-ai-base-url')); + if (!provider && !model && !baseURL) return {}; + if (!clean(apiKey)) throw new AppError(400, 'Custom AI settings require your own API key.', 'INVALID_AI_CONFIG'); + if (provider && !['openai', 'gemini', 'anthropic'].includes(provider)) + throw new AppError(400, 'Choose a supported AI provider.', 'INVALID_AI_CONFIG'); + if (model && !/^[\x21-\x7e]{1,200}$/.test(model)) + throw new AppError(400, 'Model must contain 1–200 printable characters without spaces.', 'INVALID_AI_CONFIG'); + if (baseURL) { + if (provider !== 'openai') + throw new AppError(400, 'Custom base URLs require OpenAI-compatible chat.', 'INVALID_AI_CONFIG'); + let url: URL; + try { + url = new URL(baseURL); + } catch { + throw new AppError(400, 'Enter a valid HTTPS base URL.', 'INVALID_AI_CONFIG'); + } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || baseURL.length > 2048) + throw new AppError(400, 'Enter a public HTTPS base URL without credentials, query, or fragment.', 'INVALID_AI_CONFIG'); + return { provider, model, baseURL: url.href.replace(/\/+$/, '') }; + } + return { provider, model }; +} + +export function resolveChatConfig(overrideKey?: string) { + const custom = clean(overrideKey) ? chatOverrides.getStore() : undefined; + const provider = + custom?.provider ?? + (clean(overrideKey) + ? overrideKey!.trim().startsWith('sk-ant-') + ? 'anthropic' + : overrideKey!.trim().startsWith('sk-') + ? 'openai' + : 'gemini' + : (clean(process.env.AI_PROVIDER)?.toLowerCase() ?? + (clean(process.env.GEMINI_API_KEY) + ? 'gemini' + : clean(process.env.OPENAI_API_KEY) + ? 'openai' + : clean(process.env.ANTHROPIC_API_KEY) + ? 'anthropic' + : 'mock'))); + const prefix = provider.toUpperCase(); + const defaults: Record = { + openai: 'gpt-4o-mini', + gemini: 'gemini-2.5-flash', + anthropic: 'claude-3-5-haiku-latest', + mock: 'mock', + }; + return { + provider, + apiKey: clean(overrideKey) ?? clean(process.env[`${prefix}_API_KEY`]), + model: custom?.model ?? clean(process.env[`${prefix}_MODEL`]) ?? defaults[provider], + baseURL: custom?.baseURL ?? clean(process.env[`${prefix}_API_BASE`]), + apiVersion: clean(process.env.GEMINI_API_VERSION) ?? 'v1alpha', + }; +} + +export function chatIdentity(key?: string): string { + return hashToken(JSON.stringify(resolveChatConfig(key))); +} diff --git a/server/ai/provider.test.ts b/server/ai/provider.test.ts index 91c8abe..6f04dc4 100644 --- a/server/ai/provider.test.ts +++ b/server/ai/provider.test.ts @@ -45,12 +45,9 @@ describe('ai provider', () => { delete process.env.GEMINI_API_KEY; delete process.env.OPENAI_API_KEY; delete process.env.ANTHROPIC_API_KEY; - delete process.env.EDGEONE_API_KEY; - delete process.env.MAKERS_MODELS_KEY; delete process.env.GEMINI_MODEL; delete process.env.OPENAI_MODEL; delete process.env.ANTHROPIC_MODEL; - delete process.env.EDGEONE_MODEL; }); afterEach(() => { @@ -101,17 +98,9 @@ describe('ai provider', () => { { role: 'user', content: 'now' }, ])).toBe('anthropic-ok'); - delete process.env.ANTHROPIC_API_KEY; - process.env.EDGEONE_API_KEY = 'sk-eo-env'; - process.env.EDGEONE_MODEL = 'deepseek-test'; - provider = await createProvider(); - expect(await provider.complete([{ role: 'user', content: 'now' }], { json: true })).toContain('openai'); }); it('uses override key detection and validates missing env keys', async () => { - process.env.AI_PROVIDER = 'edgeone'; - expect(await (await createProvider('sk-eo-key')).complete([{ role: 'user', content: 'x' }])).toContain('openai'); - delete process.env.AI_PROVIDER; expect(await (await createProvider('sk-openai')).complete([{ role: 'user', content: 'x' }])).toContain('openai'); expect(await (await createProvider('sk-ant-anthropic')).complete([{ role: 'user', content: 'x' }])).toBe('anthropic-ok'); @@ -123,8 +112,6 @@ describe('ai provider', () => { process.env.AI_PROVIDER = 'openai'; await expect(createProvider()).rejects.toThrow('OPENAI_API_KEY'); - process.env.AI_PROVIDER = 'edgeone'; - await expect(createProvider()).rejects.toThrow('EDGEONE_API_KEY'); process.env.AI_PROVIDER = 'anthropic'; await expect(createProvider()).rejects.toThrow('ANTHROPIC_API_KEY'); diff --git a/server/ai/provider.ts b/server/ai/provider.ts index 850096e..ff37277 100644 --- a/server/ai/provider.ts +++ b/server/ai/provider.ts @@ -1,3 +1,6 @@ +import { fetchPublicChat } from './public-chat-fetch'; +import { chatOverrides } from './chat-config'; +import { resolveChatConfig, chatIdentity } from './chat-config'; import { protectAI } from './provider-controls'; import { createMockProvider } from './mock-provider'; @@ -10,82 +13,25 @@ export interface AIProvider { complete(messages: Message[], options?: { json?: boolean; signal?: AbortSignal }): Promise; } -function detectProviderType(key: string): 'gemini' | 'openai' | 'anthropic' | 'edgeone' { - const trimmed = key.trim(); - if (trimmed.startsWith('sk-ant-')) { - return 'anthropic'; - } - if (trimmed.startsWith('sk-')) { - if (process.env.AI_PROVIDER?.toLowerCase() === 'edgeone') { - return 'edgeone'; - } - return 'openai'; - } - return 'gemini'; -} - export async function createProvider(overrideKey?: string): Promise { - if (overrideKey && overrideKey.trim()) { - const type = detectProviderType(overrideKey); - switch (type) { - case 'edgeone': - return createEdgeOneProvider(overrideKey); - case 'openai': - return createOpenAIProvider(overrideKey); - case 'anthropic': - return createAnthropicProvider(overrideKey); - case 'gemini': - default: - return createGeminiProvider(overrideKey); - } - } - - // Auto-detect provider based on available environment API keys - let providerType: 'gemini' | 'openai' | 'anthropic' | 'edgeone' | 'mock' = 'mock'; - - // AI_PROVIDER takes explicit precedence over key-based auto-detection - if (process.env.AI_PROVIDER) { - const envProvider = process.env.AI_PROVIDER.toLowerCase(); - if ( - envProvider === 'gemini' || - envProvider === 'openai' || - envProvider === 'anthropic' || - envProvider === 'edgeone' || - envProvider === 'mock' - ) { - providerType = envProvider as 'gemini' | 'openai' | 'anthropic' | 'edgeone' | 'mock'; - } - } else if (process.env.EDGEONE_API_KEY && process.env.EDGEONE_API_KEY.trim()) { - providerType = 'edgeone'; - } else if (process.env.MAKERS_MODELS_KEY && process.env.MAKERS_MODELS_KEY.trim()) { - providerType = 'edgeone'; - } else if (process.env.GEMINI_API_KEY && process.env.GEMINI_API_KEY.trim()) { - providerType = 'gemini'; - } else if (process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY.trim()) { - providerType = 'openai'; - } else if (process.env.ANTHROPIC_API_KEY && process.env.ANTHROPIC_API_KEY.trim()) { - providerType = 'anthropic'; - } - - switch (providerType) { - case 'edgeone': - return createEdgeOneProvider(); + const { provider } = resolveChatConfig(overrideKey); + switch (provider) { case 'openai': - return createOpenAIProvider(); - case 'anthropic': - return createAnthropicProvider(); + return createOpenAIProvider(overrideKey); case 'gemini': - return createGeminiProvider(); - default: + return createGeminiProvider(overrideKey); + case 'anthropic': + return createAnthropicProvider(overrideKey); + case 'mock': return createMockProvider(); + default: + throw new Error('Unsupported AI_PROVIDER. Use gemini, openai, anthropic, or mock.'); } } async function createGeminiProvider(overrideKey?: string): Promise { const { GoogleGenAI } = await import('@google/genai'); - const apiKey = overrideKey ?? process.env.GEMINI_API_KEY; - const model = process.env.GEMINI_MODEL ?? 'gemini-2.5-flash'; - const apiVersion = process.env.GEMINI_API_VERSION ?? 'v1alpha'; + const { apiKey, model, apiVersion } = resolveChatConfig(overrideKey); if (!apiKey) { throw new Error('GEMINI_API_KEY is required when using Gemini provider'); @@ -127,46 +73,21 @@ async function createGeminiProvider(overrideKey?: string): Promise { async function createOpenAIProvider(overrideKey?: string): Promise { const { default: OpenAI } = await import('openai'); - const apiKey = overrideKey ?? process.env.OPENAI_API_KEY; + const { apiKey, model, baseURL } = resolveChatConfig(overrideKey); if (!apiKey) { throw new Error('OPENAI_API_KEY is required when using OpenAI provider'); } - const client = new OpenAI({ apiKey, timeout: 30000, maxRetries: 0 }); - if (process.env.OPENAI_API_BASE) { - client.baseURL = process.env.OPENAI_API_BASE; - } - const model = process.env.OPENAI_MODEL ?? 'gpt-4o-mini'; - - return protectAI( - { - async complete(messages, options) { - const response = await client.chat.completions.create( - { - model, - messages: messages.map((m) => ({ role: m.role, content: m.content })), - max_tokens: 4096, - temperature: 0.2, - response_format: options?.json ? { type: 'json_object' } : undefined, - }, - { signal: options?.signal }, - ); - return response.choices[0]?.message?.content ?? ''; - }, - }, - !overrideKey, - ); -} - -async function createEdgeOneProvider(overrideKey?: string): Promise { - // ponytail: EdgeOne Makers Models exposes OpenAI-compatible endpoint. Reuse openai SDK client. - const { default: OpenAI } = await import('openai'); - const apiKey = overrideKey ?? (process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim()); - if (!apiKey) { - throw new Error('EDGEONE_API_KEY or MAKERS_MODELS_KEY is required when using EdgeOne provider'); + const client = new OpenAI({ + apiKey, + timeout: 30000, + maxRetries: 0, + fetch: overrideKey && chatOverrides.getStore()?.baseURL + ? fetchPublicChat + : (input, init) => fetch(input, { ...init, redirect: 'error' }), + }); + if (baseURL) { + client.baseURL = baseURL; } - const baseURL = process.env.EDGEONE_API_BASE ?? 'https://ai-gateway.edgeone.link/v1'; - const model = process.env.EDGEONE_MODEL ?? '@makers/deepseek-v4-flash'; - const client = new OpenAI({ apiKey, baseURL, timeout: 30000, maxRetries: 0 }); return protectAI( { @@ -190,15 +111,14 @@ async function createEdgeOneProvider(overrideKey?: string): Promise async function createAnthropicProvider(overrideKey?: string): Promise { const Anthropic = (await import('@anthropic-ai/sdk')).default; - const apiKey = overrideKey ?? process.env.ANTHROPIC_API_KEY; + const { apiKey, model, baseURL } = resolveChatConfig(overrideKey); if (!apiKey) { throw new Error('ANTHROPIC_API_KEY is required when using Anthropic provider'); } const client = new Anthropic({ apiKey, timeout: 30000, maxRetries: 0 }); - if (process.env.ANTHROPIC_API_BASE) { - client.baseURL = process.env.ANTHROPIC_API_BASE; + if (baseURL) { + client.baseURL = baseURL; } - const model = process.env.ANTHROPIC_MODEL ?? 'claude-3-5-haiku-latest'; return protectAI( { @@ -235,19 +155,7 @@ export async function getAIProvider(overrideKey?: string): Promise { if (overrideKey) { return createProvider(overrideKey); } - // Cache by the resolved provider type so changing AI_PROVIDER invalidates the cache. - // Mirror createProvider()'s auto-detection order (edgeone keys take precedence). - const currentKey = - process.env.AI_PROVIDER || - (process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim() - ? 'edgeone' - : process.env.GEMINI_API_KEY?.trim() - ? 'gemini' - : process.env.OPENAI_API_KEY?.trim() - ? 'openai' - : process.env.ANTHROPIC_API_KEY?.trim() - ? 'anthropic' - : 'mock'); + const currentKey = chatIdentity(); if (!providerInstance || providerInstanceKey !== currentKey) { providerInstance = await createProvider(); providerInstanceKey = currentKey; diff --git a/server/ai/public-chat-fetch.test.ts b/server/ai/public-chat-fetch.test.ts new file mode 100644 index 0000000..57621bf --- /dev/null +++ b/server/ai/public-chat-fetch.test.ts @@ -0,0 +1,12 @@ +import { expect, test } from 'bun:test'; +import { isPublicAddress, fetchPublicChat } from './public-chat-fetch'; + +test('blocks local, private, reserved and mapped network addresses', () => { + for (const address of ['127.0.0.1', '10.0.0.1', '169.254.169.254', '172.16.0.1', '192.168.1.1', '100.64.0.1', '0.0.0.0', '224.0.0.1', '::1', '::ffff:127.0.0.1', 'fc00::1', 'fe80::1', '2002:7f00:1::', '2001:db8::1']) expect(isPublicAddress(address)).toBe(false); + expect(isPublicAddress('8.8.8.8')).toBe(true); + expect(isPublicAddress('2606:4700:4700::1111')).toBe(true); +}); + +test('rejects a private endpoint before sending a request', async () => { + await expect(fetchPublicChat('https://127.0.0.1/v1')).rejects.toMatchObject({ code: 'INVALID_AI_CONFIG' }); +}); diff --git a/server/ai/public-chat-fetch.ts b/server/ai/public-chat-fetch.ts new file mode 100644 index 0000000..5c3cfe2 --- /dev/null +++ b/server/ai/public-chat-fetch.ts @@ -0,0 +1,73 @@ +import { lookup } from 'node:dns/promises'; +import { request as httpsRequest } from 'node:https'; +import { BlockList, isIP } from 'node:net'; +import { AppError } from '../utils/errors'; + +const blocked = new BlockList(); +for (const [network, bits] of [ + ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], + ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24], + ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24], ['224.0.0.0', 3], +] as const) blocked.addSubnet(network, bits, 'ipv4'); + +export function isPublicAddress(address: string): boolean { + if (isIP(address) === 4) return !blocked.check(address, 'ipv4'); + if (isIP(address) !== 6) return false; + const global = new BlockList(); + global.addSubnet('2000::', 3, 'ipv6'); + const special = new BlockList(); + special.addSubnet('2001::', 23, 'ipv6'); + special.addSubnet('2001:db8::', 32, 'ipv6'); + special.addSubnet('2002::', 16, 'ipv6'); + special.addSubnet('3fff::', 20, 'ipv6'); + return global.check(address, 'ipv6') && !special.check(address, 'ipv6'); +} + +/** Resolve once and connect to the checked address, keeping the original TLS identity. */ +export async function fetchPublicChat(input: string | URL | Request, init?: RequestInit): Promise { + const source = new Request(input, init); + const url = new URL(source.url); + if (url.protocol !== 'https:' || url.username || url.password) + throw new AppError(400, 'Use a public HTTPS AI endpoint.', 'INVALID_AI_CONFIG'); + const hostname = url.hostname.replace(/^\[|\]$/g, ''); + let timer: ReturnType | undefined; + const addresses = await Promise.race([ + lookup(hostname, { all: true, verbatim: true }), + new Promise((_, reject) => { timer = setTimeout(() => reject(new Error('AI endpoint DNS lookup timed out')), 5000); }), + ]).finally(() => clearTimeout(timer)); + if (!addresses.length || addresses.some(({ address }) => !isPublicAddress(address))) + throw new AppError(400, 'Local and private-network AI endpoints are not supported.', 'INVALID_AI_CONFIG'); + const body = Buffer.from(await source.arrayBuffer()); + const headers = Object.fromEntries(source.headers); + headers.host = url.host; + headers['accept-encoding'] = 'identity'; + return new Promise((resolve, reject) => { + const req = httpsRequest(url, { + hostname: addresses[0].address, + servername: isIP(hostname) ? undefined : hostname, + method: source.method, + headers, + signal: AbortSignal.any([source.signal, AbortSignal.timeout(30000)]), + }, res => { + const chunks: Buffer[] = []; + let bytes = 0; + res.on('data', (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 10 * 1024 * 1024) { res.destroy(new Error('AI response exceeds size limit')); return; } + chunks.push(chunk); + }); + res.on('error', reject); + res.on('end', () => { + const status = res.statusCode ?? 502; + if (status >= 300 && status < 400) { reject(new Error('AI endpoint redirects are not supported')); return; } + const responseHeaders = new Headers(); + for (const [name, value] of Object.entries(res.headers)) { + if (value !== undefined) responseHeaders.set(name, Array.isArray(value) ? value.join(', ') : value); + } + resolve(new Response([204, 205, 304].includes(status) ? null : Buffer.concat(chunks), { status, headers: responseHeaders })); + }); + }); + req.on('error', reject); + req.end(body); + }); +} diff --git a/server/ai/service.ts b/server/ai/service.ts index 73f745e..d514088 100644 --- a/server/ai/service.ts +++ b/server/ai/service.ts @@ -1,44 +1,8 @@ +import { chatIdentity } from './chat-config'; import { getAIProvider } from './provider'; -import { aiCacheKey, cache, hashToken } from '../cache/lru'; +import { aiCacheKey, cache } from '../cache/lru'; import { logApi } from '../utils/logger'; -function getAiCacheDiscriminator(apiKey?: string): string { - const envProvider = process.env.AI_PROVIDER?.trim().toLowerCase(); - const provider = apiKey?.trim() - ? apiKey.trim().startsWith('sk-ant-') - ? 'anthropic' - : apiKey.trim().startsWith('sk-') - ? envProvider === 'edgeone' - ? 'edgeone' - : 'openai' - : 'gemini' - : envProvider && envProvider.length > 0 - ? envProvider - : process.env.EDGEONE_API_KEY?.trim() || process.env.MAKERS_MODELS_KEY?.trim() - ? 'edgeone' - : process.env.GEMINI_API_KEY?.trim() - ? 'gemini' - : process.env.OPENAI_API_KEY?.trim() - ? 'openai' - : process.env.ANTHROPIC_API_KEY?.trim() - ? 'anthropic' - : 'mock'; - - const model = - provider === 'edgeone' - ? (process.env.EDGEONE_MODEL ?? '@makers/deepseek-v4-flash') - : provider === 'gemini' - ? (process.env.GEMINI_MODEL ?? 'gemini-2.5-flash') - : provider === 'openai' - ? (process.env.OPENAI_MODEL ?? 'gpt-4o-mini') - : provider === 'anthropic' - ? (process.env.ANTHROPIC_MODEL ?? 'claude-3-5-haiku-latest') - : 'mock'; - - const keyScope = apiKey?.trim() ? `user-key:${hashToken(apiKey)}` : 'env-key'; - return `${provider}:${model}:${keyScope}`; -} - function safeParseJSON(raw: string): T { let cleaned = raw.trim(); @@ -105,7 +69,7 @@ export async function executeAiTask>( params.repo, params.sha, params.paramHash ?? 'v1', - getAiCacheDiscriminator(params.apiKey), + chatIdentity(params.apiKey), ); if (cache.has(key)) { return { data: cache.get(key)!, cached: true }; diff --git a/server/api-router.ts b/server/api-router.ts index 15ca560..b231501 100644 --- a/server/api-router.ts +++ b/server/api-router.ts @@ -1,3 +1,4 @@ +import { chatOverrides, readChatOverrides } from './ai/chat-config'; import { getOctokit } from './github/client'; import type { RequestContext } from './utils/context'; import { fetchTrending } from './services/trending'; @@ -30,7 +31,7 @@ export async function handleApiRequest(req: Request, remoteAddress?: string): Pr query[k] = v; }); - const userKey = req.headers.get('x-gemini-api-key') || undefined; + const userKey = req.headers.get('x-ai-api-key') || req.headers.get('x-gemini-api-key') || undefined; const gitHubToken = req.headers.get('x-github-token') || undefined; const ctx: RequestContext = { @@ -52,7 +53,8 @@ export async function handleApiRequest(req: Request, remoteAddress?: string): Pr throw new AppError(429, 'Server is busy. Please try again shortly.', 'RATE_LIMIT_EXCEEDED', true); if (expensive) await limitClient(req.headers, remoteAddress); release = expensive ? admitRequest() : undefined; - if (release === null) throw new AppError(429, 'Server is busy. Please try again shortly.', 'RATE_LIMIT_EXCEEDED', true); + if (release === null) + throw new AppError(429, 'Server is busy. Please try again shortly.', 'RATE_LIMIT_EXCEEDED', true); if (expensive) await reserveUsage('api-minute', 1, configuredLimit('API_REQUESTS_PER_MINUTE', 600), 60000); req = await limitRequestBody(req); // ── Global System Utilities ───────────────────────────────────── @@ -74,21 +76,29 @@ export async function handleApiRequest(req: Request, remoteAddress?: string): Pr if (repoResponse) return addSecurityHeaders(repoResponse); // ── AI Summary & Analysis Routes ──────────────────────────────── - const aiResponse = await handleAiRoutes(pathname, req, userKey, gitHubToken, ctx); + const aiResponse = await chatOverrides.run(readChatOverrides(req.headers, userKey), () => + handleAiRoutes(pathname, req, userKey, gitHubToken, ctx), + ); if (aiResponse) return addSecurityHeaders(aiResponse); // ── Semantic Search & Ingest Routes ───────────────────────────── - const searchResponse = await handleSearchRoutes(pathname, req, query, userKey, ctx, start); + const searchResponse = await chatOverrides.run( + pathname === '/api/search/ask' ? readChatOverrides(req.headers, userKey) : {}, + () => handleSearchRoutes(pathname, req, query, userKey, ctx, start), + ); if (searchResponse) return addSecurityHeaders(searchResponse); return null; } catch (error) { const payload = toErrorPayload(error); logError(pathname, error, { durationMs: Date.now() - start }); - return addSecurityHeaders(Response.json(payload, { - status: payload.status, - headers: payload.status === 429 ? { 'Retry-After': String(payload.context?.retryAfterSeconds ?? 60) } : undefined, - })); + return addSecurityHeaders( + Response.json(payload, { + status: payload.status, + headers: + payload.status === 429 ? { 'Retry-After': String(payload.context?.retryAfterSeconds ?? 60) } : undefined, + }), + ); } finally { release?.(); releaseConnection?.(); diff --git a/server/search/embedding-config.test.ts b/server/search/embedding-config.test.ts new file mode 100644 index 0000000..ee7f8bd --- /dev/null +++ b/server/search/embedding-config.test.ts @@ -0,0 +1,45 @@ +import { expect, test } from 'bun:test'; +import { embeddingConfig } from './embedding-config'; + +test('blank dedicated settings use nonempty legacy settings', () => { + expect( + embeddingConfig({ + EMBEDDING_PROVIDER: ' ', + AI_PROVIDER: 'openai', + EMBEDDING_API_KEY: '', + EMBEDDING_API_BASE: ' ', + EMBEDDING_MODEL: '', + OPENAI_API_KEY: ' key ', + OPENAI_API_BASE: 'https://legacy.example/v1', + OPENAI_EMBEDDING_MODEL: 'model', + }), + ).toMatchObject({ provider: 'openai', apiKey: 'key', baseURL: 'https://legacy.example/v1', model: 'model' }); +}); + +test('explicit embedding configuration is independent of chat settings', () => { + const env = { + EMBEDDING_PROVIDER: 'openai', + EMBEDDING_API_KEY: 'embedding-key', + EMBEDDING_API_BASE: 'https://embedding.example/v1', + EMBEDDING_MODEL: 'embed-model', + }; + expect( + embeddingConfig({ + ...env, + AI_PROVIDER: 'anthropic', + OPENAI_API_KEY: 'chat-key', + OPENAI_API_BASE: 'https://chat.example/v1', + }), + ).toEqual(embeddingConfig({ ...env, AI_PROVIDER: 'gemini' })); +}); + +test('existing OpenAI settings remain supported', () => { + expect( + embeddingConfig({ + AI_PROVIDER: 'openai', + OPENAI_API_KEY: 'key', + OPENAI_API_BASE: 'https://legacy.example/v1', + OPENAI_EMBEDDING_MODEL: 'legacy-model', + }), + ).toEqual({ provider: 'openai', apiKey: 'key', baseURL: 'https://legacy.example/v1', model: 'legacy-model' }); +}); diff --git a/server/search/embedding-config.ts b/server/search/embedding-config.ts new file mode 100644 index 0000000..abb34b6 --- /dev/null +++ b/server/search/embedding-config.ts @@ -0,0 +1,23 @@ +/** Explicit embedding settings take precedence; legacy provider settings remain compatible. */ +export function embeddingConfig(env: Record = process.env) { + const value = (name: string) => env[name]?.trim() || undefined; + const requested = (value('EMBEDDING_PROVIDER') ?? value('AI_PROVIDER') ?? 'mock').toLowerCase(); + const provider = + requested === 'anthropic' + ? value('GEMINI_API_KEY') + ? 'gemini' + : value('OPENAI_API_KEY') + ? 'openai' + : requested + : requested; + return { + provider, + apiKey: value('EMBEDDING_API_KEY') ?? (provider === 'gemini' ? value('GEMINI_API_KEY') : value('OPENAI_API_KEY')), + baseURL: value('EMBEDDING_API_BASE') ?? value('OPENAI_API_BASE'), + model: + value('EMBEDDING_MODEL') ?? + (provider === 'gemini' + ? (value('GEMINI_EMBEDDING_MODEL') ?? 'gemini-embedding-001') + : (value('OPENAI_EMBEDDING_MODEL') ?? 'openrouter/openai/text-embedding-3-large')), + }; +} diff --git a/server/search/embedding-provider.test.ts b/server/search/embedding-provider.test.ts index dfcfd4b..5801262 100644 --- a/server/search/embedding-provider.test.ts +++ b/server/search/embedding-provider.test.ts @@ -3,6 +3,8 @@ import { createEmbeddingProvider } from './embedding-provider'; const realSetTimeout = globalThis.setTimeout; const originalEmbeddingProvider = process.env.EMBEDDING_PROVIDER; +const dedicatedNames = ['EMBEDDING_API_KEY', 'EMBEDDING_API_BASE', 'EMBEDDING_MODEL'] as const; +const dedicatedValues = dedicatedNames.map(name => process.env[name]); const originalEmbeddingInterval = process.env.EMBEDDING_REQUEST_INTERVAL_MS; let openAIError: Error | null = null; @@ -11,9 +13,26 @@ let geminiRequests = 0; let openAIConfig: { apiKey: string; baseURL?: string } | undefined; describe('createEmbeddingProvider', () => { + it('uses the normalized base URL for single and batch embeddings', async () => { + const originalBase = process.env.OPENAI_API_BASE; + try { + process.env.AI_PROVIDER = 'openai'; + process.env.OPENAI_API_KEY = 'test-key'; + process.env.OPENAI_API_BASE = ' '; + const provider = await createEmbeddingProvider(); + await provider.embed('one'); + expect(openAIConfig?.baseURL).toBeUndefined(); + await provider.embedBatch(['two']); + expect(openAIConfig?.baseURL).toBeUndefined(); + } finally { + if (originalBase === undefined) delete process.env.OPENAI_API_BASE; + else process.env.OPENAI_API_BASE = originalBase; + } + }); beforeEach(() => { process.env.EMBEDDING_REQUEST_INTERVAL_MS = '0'; delete process.env.EMBEDDING_PROVIDER; + dedicatedNames.forEach(name => { delete process.env[name]; }); mock.module('openai', () => { class OpenAI { baseURL = ''; @@ -58,8 +77,6 @@ describe('createEmbeddingProvider', () => { delete process.env.GEMINI_API_KEY; delete process.env.OPENAI_API_KEY; delete process.env.ANTHROPIC_API_KEY; - delete process.env.EDGEONE_API_KEY; - delete process.env.MAKERS_MODELS_KEY; delete process.env.OPENAI_EMBEDDING_MODEL; delete process.env.GEMINI_EMBEDDING_MODEL; openAIError = null; @@ -68,6 +85,7 @@ describe('createEmbeddingProvider', () => { }); afterEach(() => { + dedicatedNames.forEach((name, i) => { if (dedicatedValues[i] === undefined) delete process.env[name]; else process.env[name] = dedicatedValues[i]; }); if (originalEmbeddingInterval === undefined) delete process.env.EMBEDDING_REQUEST_INTERVAL_MS; else process.env.EMBEDDING_REQUEST_INTERVAL_MS = originalEmbeddingInterval; if (originalEmbeddingProvider === undefined) delete process.env.EMBEDDING_PROVIDER; @@ -75,6 +93,26 @@ describe('createEmbeddingProvider', () => { mock.restore(); }); + it('uses dedicated embedding credentials and endpoint instead of chat settings', async () => { + const names = ['EMBEDDING_API_KEY', 'EMBEDDING_API_BASE', 'EMBEDDING_MODEL', 'OPENAI_API_BASE'] as const; + const saved = names.map((name) => process.env[name]); + try { + process.env.EMBEDDING_PROVIDER = 'openai'; + process.env.EMBEDDING_API_KEY = 'embedding-only'; + process.env.EMBEDDING_API_BASE = 'https://embedding.example/v1'; + process.env.EMBEDDING_MODEL = 'embed-test'; + process.env.OPENAI_API_BASE = 'https://chat.example/v1'; + const provider = await createEmbeddingProvider(); + await provider.embed('example'); + expect(openAIConfig).toMatchObject({ apiKey: 'embedding-only', baseURL: 'https://embedding.example/v1' }); + } finally { + names.forEach((name, index) => { + if (saved[index] === undefined) delete process.env[name]; + else process.env[name] = saved[index]; + }); + } + }); + it('sends a Gemini chunk batch in one provider request', async () => { process.env.AI_PROVIDER = 'gemini'; process.env.GEMINI_API_KEY = 'gemini-batch-regression'; @@ -88,7 +126,7 @@ describe('createEmbeddingProvider', () => { const previous = process.env.OPENAI_API_BASE; try { process.env.EMBEDDING_PROVIDER = 'openai'; - process.env.AI_PROVIDER = 'edgeone'; + process.env.AI_PROVIDER = 'anthropic'; process.env.OPENAI_API_KEY = 'unit-featherless-key'; process.env.OPENAI_API_BASE = 'https://api.featherless.ai/v1'; const provider = await createEmbeddingProvider(); @@ -169,21 +207,6 @@ describe('createEmbeddingProvider', () => { delete process.env.OPENAI_API_BASE; }); - it('requires a Gemini or OpenAI fallback for EdgeOne embeddings', async () => { - process.env.AI_PROVIDER = 'edgeone'; - process.env.EDGEONE_API_KEY = 'sk-eo-test'; - - await expect(createEmbeddingProvider()).rejects.toMatchObject({ - status: 400, - code: 'EMBEDDING_PROVIDER_UNAVAILABLE', - }); - - process.env.OPENAI_API_KEY = 'sk-edgeone-fallback'; - const provider = await createEmbeddingProvider(); - expect(provider.providerName).toBe('openai'); - expect((await provider.embed('hello')).tokenCount).toBe(2); - }); - it('uses gemini embeddings and batch path', async () => { process.env.AI_PROVIDER = 'gemini'; process.env.GEMINI_API_KEY = 'gemini-key'; diff --git a/server/search/embedding-provider.ts b/server/search/embedding-provider.ts index 4785c30..0419431 100644 --- a/server/search/embedding-provider.ts +++ b/server/search/embedding-provider.ts @@ -1,3 +1,4 @@ +import { embeddingConfig } from './embedding-config'; import { protectEmbeddings } from './embedding-controls'; import type { EmbeddingProvider, EmbeddingResult } from './types'; import { EMBEDDING_DIMENSIONS } from './constants'; @@ -11,17 +12,10 @@ let cachedProvider: EmbeddingProvider | null = null; let cachedProviderKey: string | null = null; export async function createEmbeddingProvider(): Promise { - const envProvider = (process.env.EMBEDDING_PROVIDER ?? process.env.AI_PROVIDER ?? 'mock').toLowerCase(); + const envProvider = embeddingConfig().provider; const cacheKey = embeddingIdentity() + - hashToken( - JSON.stringify([ - process.env.OPENAI_API_KEY, - process.env.GEMINI_API_KEY, - process.env.EDGEONE_API_KEY, - process.env.MAKERS_MODELS_KEY, - ]), - ); + hashToken(JSON.stringify([process.env.EMBEDDING_API_KEY, process.env.OPENAI_API_KEY, process.env.GEMINI_API_KEY])); if (cachedProvider && cachedProviderKey === cacheKey) { return cachedProvider; @@ -31,25 +25,16 @@ export async function createEmbeddingProvider(): Promise { if (envProvider === 'openai') { provider = createOpenAIEmbeddingProvider(); - } else if (envProvider === 'edgeone') { - provider = await createEdgeOneEmbeddingFallback(); } else if (envProvider === 'gemini') { provider = await createGeminiEmbeddingProvider(); - } else if (envProvider === 'anthropic') { - // Anthropic has no native embedding API – prefer Gemini, then try OpenAI - if (process.env.GEMINI_API_KEY) { - provider = await createGeminiEmbeddingProvider(); - } else if (process.env.OPENAI_API_KEY) { - provider = createOpenAIEmbeddingProvider(); - } else { - throw new AppError( - 400, - 'Anthropic does not support embeddings. Configure OPENAI_API_KEY or GEMINI_API_KEY as fallback.', - 'EMBEDDING_PROVIDER_UNAVAILABLE', - ); - } - } else { + } else if (envProvider === 'mock') { provider = createMockEmbeddingProvider(); + } else { + throw new AppError( + 400, + 'Unsupported embedding provider. Configure openai, gemini, or mock.', + 'EMBEDDING_PROVIDER_UNAVAILABLE', + ); } provider = provider.providerName === 'mock' ? provider : protectEmbeddings(provider); @@ -137,7 +122,7 @@ function createOpenAICompatibleEmbeddingProvider(config: OpenAICompatibleConfig) } function createOpenAIEmbeddingProvider(): EmbeddingProvider { - const apiKey = process.env.OPENAI_API_KEY; + const apiKey = embeddingConfig().apiKey; if (!apiKey) { throw new AppError(401, 'OPENAI_API_KEY is required for OpenAI embeddings.', 'MISSING_API_KEY'); } @@ -145,20 +130,11 @@ function createOpenAIEmbeddingProvider(): EmbeddingProvider { return createOpenAICompatibleEmbeddingProvider({ providerName: 'openai', apiKey, - model: process.env.OPENAI_EMBEDDING_MODEL ?? 'openrouter/openai/text-embedding-3-large', + model: embeddingConfig().model, + baseURL: embeddingConfig().baseURL, }); } -async function createEdgeOneEmbeddingFallback(): Promise { - if (process.env.GEMINI_API_KEY) return createGeminiEmbeddingProvider(); - if (process.env.OPENAI_API_KEY) return createOpenAIEmbeddingProvider(); - throw new AppError( - 400, - 'EdgeOne does not provide embeddings. Configure GEMINI_API_KEY or OPENAI_API_KEY for semantic search.', - 'EMBEDDING_PROVIDER_UNAVAILABLE', - ); -} - async function openAIEmbed( apiKey: string, model: string, @@ -169,7 +145,7 @@ async function openAIEmbed( const { default: OpenAI } = await import('openai'); const client = new OpenAI({ apiKey, - baseURL: baseURL ?? process.env.OPENAI_API_BASE, + baseURL, timeout: 30000, maxRetries: 0, }); @@ -198,7 +174,7 @@ async function openAIEmbedBatch( const { default: OpenAI } = await import('openai'); const client = new OpenAI({ apiKey, - baseURL: baseURL ?? process.env.OPENAI_API_BASE, + baseURL, timeout: 30000, maxRetries: 0, }); @@ -226,7 +202,7 @@ async function openAIEmbedBatch( // ── Gemini Provider ──────────────────────────────────────────────────── async function createGeminiEmbeddingProvider(): Promise { - const apiKey = process.env.GEMINI_API_KEY; + const apiKey = embeddingConfig().apiKey; if (!apiKey) { throw new AppError(401, 'GEMINI_API_KEY is required for Gemini embeddings.', 'MISSING_API_KEY'); } @@ -236,7 +212,7 @@ async function createGeminiEmbeddingProvider(): Promise { apiKey, httpOptions: { timeout: 30000, retryOptions: { attempts: 1 } }, }); - const model = process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001'; + const model = embeddingConfig().model; const maxTokens = 2048; return { diff --git a/server/search/index-identity.test.ts b/server/search/index-identity.test.ts index 0fea0cd..a06b540 100644 --- a/server/search/index-identity.test.ts +++ b/server/search/index-identity.test.ts @@ -17,25 +17,25 @@ test('index identity isolates credentials, snapshots and embedding models withou expect(key).toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'private-token' })); expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha2', { gitHubToken: 'private-token' })); expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'other' })); - const original = process.env.OPENAI_EMBEDDING_MODEL; + const original = process.env.EMBEDDING_MODEL; try { - process.env.OPENAI_EMBEDDING_MODEL = 'different-model'; + process.env.EMBEDDING_MODEL = 'different-model'; expect(key).not.toBe(searchIndexKey('org', 'repo', 'sha', { gitHubToken: 'private-token' })); } finally { - if (original === undefined) delete process.env.OPENAI_EMBEDDING_MODEL; - else process.env.OPENAI_EMBEDDING_MODEL = original; + if (original === undefined) delete process.env.EMBEDDING_MODEL; + else process.env.EMBEDDING_MODEL = original; } }); test('Featherless model changes invalidate existing search indexes', () => { - const previous = process.env.OPENAI_EMBEDDING_MODEL; + const previous = process.env.EMBEDDING_MODEL; try { - process.env.OPENAI_EMBEDDING_MODEL = 'Qwen/Qwen3-Embedding-8B'; + process.env.EMBEDDING_MODEL = 'Qwen/Qwen3-Embedding-8B'; const key = searchIndexKey('org', 'repo', 'sha'); - process.env.OPENAI_EMBEDDING_MODEL = 'different-model'; + process.env.EMBEDDING_MODEL = 'different-model'; expect(searchIndexKey('org', 'repo', 'sha')).not.toBe(key); } finally { - if (previous === undefined) delete process.env.OPENAI_EMBEDDING_MODEL; - else process.env.OPENAI_EMBEDDING_MODEL = previous; + if (previous === undefined) delete process.env.EMBEDDING_MODEL; + else process.env.EMBEDDING_MODEL = previous; } }); diff --git a/server/search/index-identity.ts b/server/search/index-identity.ts index 122362e..351dc5a 100644 --- a/server/search/index-identity.ts +++ b/server/search/index-identity.ts @@ -1,3 +1,4 @@ +import { embeddingConfig } from './embedding-config'; import { hashToken, hashContext } from '../cache/lru'; import type { RequestContext } from '../utils/context'; @@ -22,16 +23,10 @@ export function indexRequestKey( export function embeddingIdentity(): string { return hashContext( JSON.stringify([ - process.env.AI_PROVIDER ?? 'mock', - process.env.EMBEDDING_PROVIDER, + embeddingConfig().provider, + embeddingConfig().model, + embeddingConfig().provider === 'openai' ? embeddingConfig().baseURL : undefined, process.env.EMBEDDING_DIMENSIONS ?? '3072', - process.env.OPENAI_EMBEDDING_MODEL, - process.env.GEMINI_EMBEDDING_MODEL, - process.env.OPENAI_API_BASE, - process.env.EDGEONE_API_BASE, - !!process.env.GEMINI_API_KEY, - !!process.env.OPENAI_API_KEY, - !!(process.env.EDGEONE_API_KEY || process.env.MAKERS_MODELS_KEY), ]), ); } diff --git a/src/components/viz/AIErrorCard.tsx b/src/components/viz/AIErrorCard.tsx index 19190fb..973c5ce 100644 --- a/src/components/viz/AIErrorCard.tsx +++ b/src/components/viz/AIErrorCard.tsx @@ -21,12 +21,11 @@ export function AIErrorCard({ error, message, onRetry, title = "AI Request Faile } return ( -
-
+
- +
-
+
{title}
{errorCode && ( @@ -34,13 +33,14 @@ export function AIErrorCard({ error, message, onRetry, title = "AI Request Faile )}
-

+

{displayMessage}

{onRetry && (
setGeminiValue(e.target.value)} - placeholder="AIzaSy..." + placeholder="API key" className="w-full rounded-md border border-border bg-background py-1.5 pl-3 pr-8 font-mono text-xs text-foreground placeholder:text-muted-foreground outline-none focus:border-accent focus:ring-1 focus:ring-accent/50 transition-all" />