diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea89874..01e31b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: with: python-version: "3.12" - name: Install (with dev and type-check extras) - run: uv pip install -e ".[dev,typecheck]" + run: uv pip install --prerelease=allow -e ".[dev,typecheck,async]" - name: Type check run: uv run --no-sync mypy @@ -32,7 +32,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install (with dev extras) - run: uv pip install -e ".[dev,ingest,langchain,dspy,llamaindex,haystack]" + run: uv pip install --prerelease=allow -e ".[dev,ingest,langchain,dspy,llamaindex,haystack,async]" - name: Lint run: uv run --no-sync ruff check src benchmarks - name: Test diff --git a/dashboard/.gitignore b/dashboard/.gitignore deleted file mode 100644 index 9407060..0000000 --- a/dashboard/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -node_modules/ -.next/ -out/ -next-env.d.ts -*.tsbuildinfo -.DS_Store diff --git a/dashboard/README.md b/dashboard/README.md deleted file mode 100644 index d8653d0..0000000 --- a/dashboard/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# dynavec observability dashboard - -A Langfuse-style observability UI for dynavec, in the project's brand theme. It -shows **real** query telemetry — latency percentiles, cache hit-rate, query -volume, and a filterable traces table with per-trace drill-down. - -![dynavec observability dashboard](https://raw.githubusercontent.com/codeforstartups/dynavec/development/docs/assets/dashboard.png) - ---- - -## How it fits together - -Two pieces, split by language boundary: - -``` -your app ──▶ Dynavec(..., telemetry=recorder) # records real events (Python) - │ - ▼ - dynavec.dashboard.serve(recorder) # JSON API (Python, stdlib) - GET /api/metrics · /api/traces · /api/trace/{id} - │ (fetch) - ▼ - dashboard/ (Next.js + Tailwind + Recharts) # this app (TypeScript) -``` - -- **Data source:** `src/dynavec/telemetry.py` captures a `TelemetryEvent` for - every real `search`; `src/dynavec/dashboard.py::serve()` exposes it as JSON. -- **This app** reads that API via `NEXT_PUBLIC_DYNAVEC_API` and renders it. With - no API configured it falls back to sample data (so `npm run dev` works alone). - -## Run it - -```bash -# 1) produce real telemetry + serve the API (no AWS needed) -python examples/dashboard_demo.py # API on http://127.0.0.1:8779 - -# 2) run the dashboard against it -cd dashboard -npm install -NEXT_PUBLIC_DYNAVEC_API=http://127.0.0.1:8779 npm run dev # http://localhost:3000 - -# build a static export (deployable to any static host / GitHub Pages) -npm run build # -> dashboard/out/ -``` - -## File map - -| Path | Role | -|------|------| -| `app/page.tsx` | top-level state (window, filters, polling) + layout | -| `components/TopBar.tsx` | brand, time-range, auto-refresh, live/sample badge | -| `components/Sidebar.tsx` | navigation / information architecture | -| `components/Kpis.tsx` | KPI cards (QPM, p95, cache-hit, avg results, errors) | -| `components/VolumeChart.tsx` | Recharts query-volume histogram | -| `components/LatencyChart.tsx` | p50/p95/p99 percentile bars | -| `components/TracesTable.tsx` | filterable traces table | -| `components/TraceDrawer.tsx` | per-trace detail drawer | -| `lib/types.ts` | `Metrics` / `TraceEvent` types (mirror the Python API) | -| `lib/api.ts` | data fetching + sample fallback | -| `lib/mock.ts` | sample-data generator for standalone dev | -| `tailwind.config.ts` | brand tokens (coral `#e8623b`, warm ink, fonts) | - ---- - -## What's done ✅ - -- [x] Telemetry capture on the real `search` path (opt-in `telemetry=recorder`) -- [x] JSON API: `/api/metrics`, `/api/traces`, `/api/trace/{id}` -- [x] Next.js app: KPIs, volume histogram, latency percentiles, traces table + drawer -- [x] Brand theme, time-range selector, auto-refresh, sample-data fallback - -## Remaining work — pick a task 🙌 - -Each item is a self-contained contribution. Comment on the linked issue to claim -it. Parent: **[dashboard epic #122](https://github.com/codeforstartups/dynavec/issues/122)**. - -### 1. Instrument `upsert` and `graph_search` (backend) -Only `search` records telemetry today. -- [ ] Record `TelemetryEvent(op="upsert", ...)` in `Dynavec.upsert` (count, latency) -- [ ] Record `op="graph_search"` in `Dynavec.graph_search` -- [ ] Tests in `tests/test_telemetry.py` / `tests/test_client_inmemory.py` - -### 2. Evaluation panel — recall@k / faithfulness ([#134](https://github.com/codeforstartups/dynavec/issues/134), [#135](https://github.com/codeforstartups/dynavec/issues/135)) -- [ ] Backend: an eval runner that scores a labeled set (recall@k, MRR, nDCG) and stores results; a pluggable LLM-judge for faithfulness/answer-relevance -- [ ] API: `GET /api/eval/summary`, `GET /api/eval/runs` -- [ ] UI: new `components/EvalPanel.tsx` + a "Scores" route/section (wire the sidebar item, currently `soon`); trend charts with Recharts - -### 3. Resources panel — buckets / indexes / namespaces ([#128](https://github.com/codeforstartups/dynavec/issues/128), [#129](https://github.com/codeforstartups/dynavec/issues/129)) -- [ ] Backend: `GET /api/resources` (describe S3 vector bucket + index dim/metric, DynamoDB table status) and `GET /api/namespaces` (per-namespace vector/doc counts) -- [ ] UI: `components/ResourcesPanel.tsx`, `components/NamespaceExplorer.tsx`; wire the two `soon` sidebar items - -### 4. Cost panel ([#130](https://github.com/codeforstartups/dynavec/issues/130)) -- [ ] Backend: `GET /api/cost` reusing `benchmarks/cost_model` with live resource sizes + query volume -- [ ] UI: `components/CostPanel.tsx` (breakdown + Recharts) - -### 5. Per-query stage waterfall -The drawer shows total latency only. -- [ ] Backend: capture per-stage timings (embed → query → hydrate → rerank) on the event -- [ ] UI: a small waterfall in `TraceDrawer.tsx` - -### 6. Deploy to GitHub Pages -- [ ] `.github/workflows/dashboard-pages.yml` that builds `dashboard/` with - `NEXT_PUBLIC_BASE_PATH=/dynavec/dashboard` and publishes `out/` - *(note: touches a workflow file — needs a maintainer with `workflow` scope to merge)* - -### 7. Polish -- [ ] Dark mode toggle (theme parity with the landing page) -- [ ] Sessions / Users grouping views -- [ ] Column config + saved views -- [ ] Component tests (Vitest / Testing Library) - -## Conventions - -- Keep the brand tokens in `tailwind.config.ts` in sync with the site's `styles.css`. -- New API fields go in **both** `src/dynavec/telemetry.py` and `lib/types.ts`. -- Charts use **Recharts**; keep them theme-colored (coral `#e8623b`, green `#2f7d5b`). -- Run `npm run build` before opening a PR; keep `node_modules/`, `.next/`, `out/` out of git. diff --git a/dashboard/app/globals.css b/dashboard/app/globals.css deleted file mode 100644 index 9c599b8..0000000 --- a/dashboard/app/globals.css +++ /dev/null @@ -1,79 +0,0 @@ -@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;700&display=swap'); -@tailwind base; -@tailwind components; -@tailwind utilities; - -/* --- Light: the landing-site palette (styles.css) --- */ -:root { - color-scheme: light; - - --bg: #fbfaf8; /* warm paper */ - --surface: #ffffff; - --ink: #14110f; /* warm near-black */ - --muted: #6f6862; - --faint: #a99f97; - --line: #ece6df; - --accent: #e8623b; /* coral */ - --accent-ink: #b8472a; /* darker coral for text/hover on light */ - --accent-soft: #fdeee8; /* light coral wash */ - --ok: #2f7d5b; - --err: #b8472a; - - --track: #f0ece6; /* meter/track fill */ - --thead: #faf6f1; /* table header wash */ - --shadow: rgba(20, 17, 15, .07); - --scroll-thumb: #e0d9d1; - --tip-bg: #ffffff; /* recharts tooltip */ - --tip-line: #ece6df; - - /* op badges */ - --op-search-bg: #eef3ff; --op-search-fg: #3b5bdb; --op-search-line: #dbe3ff; - --op-graph-bg: #f3eeff; --op-graph-fg: #7048e8; --op-graph-line: #e5dbff; - --op-upsert-bg: #eafaf1; --op-upsert-fg: #2f7d5b; --op-upsert-line: #d3f0e0; -} - -/* --- Dark: the site's warm inverted tones (--inv-bg #14110f / --inv-fg #fbfaf8) --- */ -.dark { - color-scheme: dark; - - --bg: #14110f; /* warm near-black paper */ - --surface: #1d1916; /* lifted warm panel */ - --ink: #f4efe8; /* warm off-white */ - --muted: #a79d94; - --faint: #766c64; - --line: #2d2723; - --accent: #f0714a; /* brighter coral for dark backgrounds */ - --accent-ink: #f59a7c; /* lighter coral for text/hover on dark */ - --accent-soft: #2a1712; /* dark coral wash */ - --ok: #57b98d; - --err: #ef8a63; - - --track: #2d2723; - --thead: #1a1613; - --shadow: rgba(0, 0, 0, .4); - --scroll-thumb: #3a332e; - --tip-bg: #1d1916; - --tip-line: #2d2723; - - --op-search-bg: #1a2233; --op-search-fg: #9db6ff; --op-search-line: #2b3a5c; - --op-graph-bg: #221a33; --op-graph-fg: #b9a4f5; --op-graph-line: #3a2e5c; - --op-upsert-bg: #142720; --op-upsert-fg: #57b98d; --op-upsert-line: #24463a; -} - -html, body { padding: 0; margin: 0; } -body { - background: var(--bg); - color: var(--ink); - font-family: theme('fontFamily.sans'); - border-top: 3px solid var(--accent); - -webkit-font-smoothing: antialiased; - transition: background-color .18s ease, color .18s ease; -} -::-webkit-scrollbar { width: 10px; height: 10px; } -::-webkit-scrollbar-thumb { background: var(--scroll-thumb); border-radius: 6px; } - -/* Trace op badges — themed via CSS variables so they flip with .dark */ -.op-badge { background: var(--op-search-bg); color: var(--op-search-fg); border-color: var(--op-search-line); } -.op-search { background: var(--op-search-bg); color: var(--op-search-fg); border-color: var(--op-search-line); } -.op-graph_search { background: var(--op-graph-bg); color: var(--op-graph-fg); border-color: var(--op-graph-line); } -.op-upsert { background: var(--op-upsert-bg); color: var(--op-upsert-fg); border-color: var(--op-upsert-line); } diff --git a/dashboard/app/layout.tsx b/dashboard/app/layout.tsx deleted file mode 100644 index 735422c..0000000 --- a/dashboard/app/layout.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import type { Metadata } from "next"; -import "./globals.css"; - -export const metadata: Metadata = { - title: "dynavec · Observability", - description: "Real-time retrieval observability for dynavec — latency, cache, traces.", -}; - -// Set the theme class before first paint so there is no light→dark flash. -// Precedence: ?theme= override (deep-linkable, used for parity screenshots) > -// saved choice > OS preference; defaults to light (matching the site). -const themeInit = `(function(){try{var q=new URLSearchParams(location.search).get('theme');var s=localStorage.getItem('dynavec-theme');var d=q?q==='dark':(s?s==='dark':matchMedia('(prefers-color-scheme: dark)').matches);document.documentElement.classList.toggle('dark',d);if(q){try{localStorage.setItem('dynavec-theme',d?'dark':'light');}catch(e){}}}catch(e){}})();`; - -export default function RootLayout({ children }: { children: React.ReactNode }) { - return ( - - -