diff --git a/README.md b/README.md index 9f735fe..beceb1d 100644 --- a/README.md +++ b/README.md @@ -76,18 +76,22 @@ Set your OpenRouter API key: export OPENROUTER_API_KEY="your-key" ``` -## 60-second demo +## Visual Playground -The demo routes one query twice: first from built-in priors, then after adding -eight illustrative user observations. +Launch the dependency-free local interface: ```bash -python examples/adaptation_demo.py +python demo/app.py ``` -It prints the selected model, distribution, confidence, routing latency, and -the Jev charge for each decision. It makes two Decisions API calls but does not -invoke either selected model. +Open `http://127.0.0.1:7860`. Without an API key, the Playground replays a +recorded adaptation where user evidence changes the selected model. With +`OPENROUTER_API_KEY` set, it can run live zero-shot, few-shot, or side-by-side +decisions. The key stays in the local Python process, and candidate models are +never invoked. + +Prefer the terminal? `python examples/adaptation_demo.py` runs the same +two-decision story and prints its probabilities, latency, and Jev charge. ## Quick start diff --git a/demo/app.js b/demo/app.js new file mode 100644 index 0000000..5e3c8d0 --- /dev/null +++ b/demo/app.js @@ -0,0 +1,324 @@ +const state = { + config: null, + mode: "zero-shot", + busy: false, +}; + +const elements = { + body: document.body, + liveStatus: document.querySelector("#liveStatus"), + query: document.querySelector("#queryInput"), + charCount: document.querySelector("#charCount"), + resetQuery: document.querySelector("#resetQuery"), + routeButton: document.querySelector("#routeButton"), + compareButton: document.querySelector("#compareButton"), + replayButton: document.querySelector("#replayButton"), + modeButtons: [...document.querySelectorAll(".mode-button")], + resultGrid: document.querySelector("#resultGrid"), + emptyState: document.querySelector("#emptyState"), + resultTemplate: document.querySelector("#resultTemplate"), + runKind: document.querySelector("#runKind"), + notice: document.querySelector("#notice"), + candidateGrid: document.querySelector("#candidateGrid"), + candidateCount: document.querySelector("#candidateCount"), + evidenceList: document.querySelector("#evidenceList"), + evidenceLabel: document.querySelector("#evidenceLabel"), + benchmarkBars: document.querySelector("#benchmarkBars"), + benchmarkCount: document.querySelector("#benchmarkCount"), + benchmarkCost: document.querySelector("#benchmarkCost"), +}; + +async function requestJSON(path, options = {}) { + const response = await fetch(path, options); + const payload = await response.json().catch(() => ({ error: "Invalid server response" })); + if (!response.ok) { + throw new Error(payload.error || `Request failed with status ${response.status}`); + } + return payload; +} + +function shortLabel(model) { + const configured = state.config?.models.find((candidate) => candidate.id === model); + return configured?.label || model.split("/").pop(); +} + +function formatCost(value) { + return typeof value === "number" ? `$${value.toFixed(6)}` : "n/a"; +} + +function updateCharacterCount() { + elements.charCount.textContent = `${elements.query.value.length.toLocaleString()} characters`; +} + +function setNotice(message = "", kind = "info") { + elements.notice.textContent = message; + elements.notice.classList.toggle("hidden", !message); + elements.notice.classList.toggle("error", kind === "error"); +} + +function setBusy(busy, label = "Routing…") { + state.busy = busy; + elements.body.classList.toggle("busy", busy); + elements.routeButton.querySelector("span").textContent = busy ? label : "Route live"; +} + +function setMode(mode) { + state.mode = mode; + for (const button of elements.modeButtons) { + button.classList.toggle("active", button.dataset.mode === mode); + } +} + +function renderCandidates(models, selectedModels = []) { + elements.candidateGrid.replaceChildren(); + elements.candidateCount.textContent = `${models.length} MODELS`; + for (const model of models) { + const card = document.createElement("article"); + card.className = "candidate-card"; + card.dataset.model = model.id; + card.classList.toggle("selected", selectedModels.includes(model.id)); + + const provider = document.createElement("span"); + provider.className = "candidate-provider"; + provider.textContent = model.id.split("/")[0].slice(0, 2).toUpperCase(); + + const title = document.createElement("h3"); + title.textContent = model.label; + + const description = document.createElement("p"); + description.textContent = model.description; + + const tags = document.createElement("div"); + tags.className = "candidate-tags"; + for (const tag of model.strengths.slice(0, 2)) { + const chip = document.createElement("span"); + chip.textContent = tag; + tags.append(chip); + } + + card.append(provider, title, description, tags); + elements.candidateGrid.append(card); + } +} + +function renderResult(result) { + const node = elements.resultTemplate.content.firstElementChild.cloneNode(true); + node.dataset.mode = result.mode; + node.querySelector(".result-mode").textContent = + result.mode === "few-shot" ? "Few-shot · priors + history" : "Zero-shot · built-in priors"; + node.querySelector(".selected-model").textContent = result.selected_label; + + const confidence = Math.max(0, Math.min(1, Number(result.confidence) || 0)); + node.querySelector(".confidence-value").textContent = `${Math.round(confidence * 100)}%`; + node.querySelector(".confidence-ring").style.setProperty( + "--confidence-angle", + `${confidence * 360}deg`, + ); + + const probabilityList = node.querySelector(".probability-list"); + const probabilities = Object.entries(result.probabilities).sort((a, b) => b[1] - a[1]); + for (const [model, probability] of probabilities) { + const row = document.createElement("div"); + row.className = "probability-row"; + row.classList.toggle("selected", model === result.selected_model); + + const label = document.createElement("span"); + label.className = "probability-label"; + label.textContent = shortLabel(model); + + const track = document.createElement("div"); + track.className = "probability-track"; + const fill = document.createElement("div"); + fill.className = "probability-fill"; + fill.style.setProperty("--probability-width", `${Math.max(0, probability) * 100}%`); + track.append(fill); + + const number = document.createElement("span"); + number.className = "probability-number"; + number.textContent = `${Math.round(probability * 100)}%`; + row.append(label, track, number); + probabilityList.append(row); + } + + node.querySelector(".latency-value").textContent = `${Math.round(result.latency_ms)} ms`; + node.querySelector(".cost-value").textContent = formatCost(result.route_cost_usd); + node.querySelector(".evidence-value").textContent = `${result.evidence.length} rows`; + return node; +} + +function renderResults(results, recorded = false) { + elements.emptyState.classList.add("hidden"); + elements.resultGrid.replaceChildren(...results.map(renderResult)); + elements.resultGrid.classList.toggle("comparison", results.length > 1); + elements.runKind.textContent = recorded ? "RECORDED" : "LIVE"; + const selected = [...new Set(results.map((result) => result.selected_model))]; + renderCandidates(state.config.models, selected); + + const fewShot = results.find((result) => result.mode === "few-shot"); + renderEvidence(fewShot?.evidence || []); +} + +function renderEvidence(evidence) { + elements.evidenceList.replaceChildren(); + elements.evidenceLabel.textContent = evidence.length ? `${evidence.length} RETRIEVED` : "FEW-SHOT ONLY"; + if (!evidence.length) { + const message = document.createElement("p"); + message.className = "muted-copy"; + message.textContent = "Zero-shot decisions use model priors only."; + elements.evidenceList.append(message); + return; + } + + for (const item of evidence.slice(0, 6)) { + const row = document.createElement("div"); + row.className = "evidence-item"; + + const model = document.createElement("span"); + model.className = "evidence-model"; + model.textContent = item.model_label; + + const query = document.createElement("span"); + query.className = "evidence-query"; + query.textContent = item.query; + query.title = item.query; + + const score = document.createElement("div"); + score.className = "evidence-score"; + const performance = document.createElement("strong"); + performance.textContent = Number(item.performance).toFixed(2); + const similarity = document.createElement("span"); + similarity.textContent = `${Math.round(item.similarity * 100)}% similar`; + score.append(performance, similarity); + + row.append(model, query, score); + elements.evidenceList.append(row); + } +} + +function renderBenchmark(payload) { + elements.benchmarkBars.replaceChildren(); + elements.benchmarkCount.textContent = `${payload.target_count} QUERIES`; + const modes = [ + ["Zero-shot", payload.zero_shot, "zero-shot"], + ["Few-shot K=4", payload.few_shot, "few-shot"], + ]; + for (const [label, metrics, className] of modes) { + const row = document.createElement("div"); + row.className = `benchmark-row ${className}`; + const top = document.createElement("div"); + top.className = "benchmark-row-top"; + const name = document.createElement("span"); + name.textContent = label; + const value = document.createElement("strong"); + value.textContent = Number(metrics.quality_mean).toFixed(4); + top.append(name, value); + const track = document.createElement("div"); + track.className = "benchmark-track"; + const fill = document.createElement("div"); + fill.className = "benchmark-fill"; + fill.style.setProperty("--benchmark-width", `${metrics.quality_mean * 100}%`); + track.append(fill); + row.append(top, track); + elements.benchmarkBars.append(row); + } + elements.benchmarkCost.textContent = + `$${payload.zero_shot.router_cost_per_1000_usd.toFixed(3)} → ` + + `$${payload.few_shot.router_cost_per_1000_usd.toFixed(3)}`; +} + +async function routeLive(mode) { + return requestJSON("/api/route", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query: elements.query.value, mode }), + }); +} + +async function runSingle() { + if (state.busy || !state.config.live_available) return; + setNotice(); + setBusy(true); + try { + const result = await routeLive(state.mode); + renderResults([result]); + } catch (error) { + setNotice(error.message, "error"); + } finally { + setBusy(false); + } +} + +async function compareModes() { + if (state.busy || !state.config.live_available) return; + setNotice("Running zero-shot, then few-shot. This makes two Jev decisions."); + setBusy(true, "Comparing…"); + try { + const zeroShot = await routeLive("zero-shot"); + const fewShot = await routeLive("few-shot"); + renderResults([zeroShot, fewShot]); + setNotice("Same query, same candidates—only the historical evidence changed."); + } catch (error) { + setNotice(error.message, "error"); + } finally { + setBusy(false); + } +} + +async function replayDemo() { + if (state.busy) return; + setNotice(); + setBusy(true, "Loading…"); + try { + const replay = await requestJSON("/api/replay"); + elements.query.value = replay.query; + updateCharacterCount(); + renderResults(replay.results, true); + setNotice("Recorded replay: user evidence changes the selected route."); + } catch (error) { + setNotice(error.message, "error"); + } finally { + setBusy(false); + } +} + +async function initialize() { + try { + const [config, benchmark] = await Promise.all([ + requestJSON("/api/config"), + requestJSON("/api/benchmark"), + ]); + state.config = config; + elements.query.value = config.default_query; + updateCharacterCount(); + renderCandidates(config.models); + renderBenchmark(benchmark); + + elements.liveStatus.classList.add(config.live_available ? "live" : "replay"); + elements.liveStatus.querySelector("span:last-child").textContent = config.live_available + ? "Live routing ready" + : "Replay mode"; + elements.routeButton.disabled = !config.live_available; + elements.compareButton.disabled = !config.live_available; + if (!config.live_available) { + elements.routeButton.title = "Set OPENROUTER_API_KEY on the server to enable live routing"; + elements.compareButton.title = elements.routeButton.title; + } + await replayDemo(); + } catch (error) { + setNotice(`Playground failed to initialize: ${error.message}`, "error"); + } +} + +for (const button of elements.modeButtons) { + button.addEventListener("click", () => setMode(button.dataset.mode)); +} +elements.query.addEventListener("input", updateCharacterCount); +elements.resetQuery.addEventListener("click", () => { + elements.query.value = state.config.default_query; + updateCharacterCount(); +}); +elements.routeButton.addEventListener("click", runSingle); +elements.compareButton.addEventListener("click", compareModes); +elements.replayButton.addEventListener("click", replayDemo); + +initialize(); diff --git a/demo/app.py b/demo/app.py new file mode 100644 index 0000000..3c50814 --- /dev/null +++ b/demo/app.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Serve the dependency-free ReflexRoute Playground. + +Run from the repository root: + + python demo/app.py + +Set OPENROUTER_API_KEY to enable live decisions. Without a key, the interface +still works in replay mode using a previously recorded two-decision demo. +""" + +from __future__ import annotations + +import argparse +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import sys +import time +from typing import Any, Mapping +from urllib.parse import urlsplit + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEMO_ROOT = Path(__file__).resolve().parent +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from examples.adaptation_demo import HISTORY, MODELS, QUERY +from reflexroute import Router +from reflexroute.retrieval import retrieve_balanced + + +MODEL_LABELS = { + "meta-llama/llama-3.1-8b-instruct": "Llama 3.1 8B", + "deepseek/deepseek-r1-0528": "DeepSeek R1", + "google/gemini-2.5-flash": "Gemini 2.5 Flash", + "google/gemini-2.5-pro": "Gemini 2.5 Pro", +} +STATIC_FILES = { + "/": ("index.html", "text/html; charset=utf-8"), + "/index.html": ("index.html", "text/html; charset=utf-8"), + "/styles.css": ("styles.css", "text/css; charset=utf-8"), + "/app.js": ("app.js", "text/javascript; charset=utf-8"), +} +MAX_BODY_BYTES = 64 * 1024 + + +def _number(value: Any) -> float | None: + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return None + + +def evidence_for(query: str, top_k: int = 2) -> list[dict[str, Any]]: + retrieved = retrieve_balanced(query, MODELS, HISTORY, top_k) + evidence = [ + { + "model": model, + "model_label": MODEL_LABELS[model], + "query": item.record.query, + "performance": item.record.performance, + "cost": item.record.cost, + "similarity": item.similarity, + } + for model, records in retrieved.items() + for item in records + ] + return sorted(evidence, key=lambda item: item["similarity"], reverse=True) + + +def config_payload() -> dict[str, Any]: + router = Router(models=MODELS, history_top_k=2) + models = [] + for model in MODELS: + profile = router.profiles[model] + models.append( + { + "id": model, + "label": MODEL_LABELS[model], + "description": profile.description, + "strengths": list(profile.strengths[:3]), + "cost_level": profile.cost_level, + "latency_level": profile.latency_level, + } + ) + return { + "live_available": bool(os.environ.get("OPENROUTER_API_KEY", "").strip()), + "default_query": QUERY, + "history_count": len(HISTORY), + "history_top_k": 2, + "models": models, + } + + +def benchmark_payload() -> dict[str, Any]: + summary_path = REPOSITORY_ROOT / "benchmarks/results/mini_v1/summary.json" + try: + payload = json.loads(summary_path.read_text(encoding="utf-8")) + methods = payload["methods"] + return { + "target_count": payload["target_count"], + "zero_shot": methods["reflexroute_zero_shot"], + "few_shot": methods["reflexroute_few_shot_k4"], + } + except (OSError, KeyError, TypeError, json.JSONDecodeError): + return { + "target_count": 191, + "zero_shot": { + "quality_mean": 0.5497, + "router_latency_p50_ms": 721.8, + "router_cost_per_1000_usd": 0.1076, + }, + "few_shot": { + "quality_mean": 0.6832, + "router_latency_p50_ms": 1333.2, + "router_cost_per_1000_usd": 0.4032, + }, + } + + +def replay_payload() -> dict[str, Any]: + return { + "query": QUERY, + "recorded": True, + "results": [ + { + "mode": "zero-shot", + "selected_model": "deepseek/deepseek-r1-0528", + "selected_label": "DeepSeek R1", + "confidence": 0.71, + "probabilities": { + "meta-llama/llama-3.1-8b-instruct": 0.10, + "deepseek/deepseek-r1-0528": 0.78, + "google/gemini-2.5-flash": 0.12, + "google/gemini-2.5-pro": 0.0, + }, + "latency_ms": 640.0, + "route_cost_usd": 0.000090, + "input_tokens": None, + "output_tokens": None, + "evidence": [], + }, + { + "mode": "few-shot", + "selected_model": "google/gemini-2.5-flash", + "selected_label": "Gemini 2.5 Flash", + "confidence": 0.99, + "probabilities": { + "meta-llama/llama-3.1-8b-instruct": 0.0, + "deepseek/deepseek-r1-0528": 0.0, + "google/gemini-2.5-flash": 1.0, + "google/gemini-2.5-pro": 0.0, + }, + "latency_ms": 666.0, + "route_cost_usd": 0.000115, + "input_tokens": None, + "output_tokens": None, + "evidence": evidence_for(QUERY), + }, + ], + } + + +def route_payload(query: str, mode: str) -> dict[str, Any]: + if mode not in {"zero-shot", "few-shot"}: + raise ValueError("mode must be zero-shot or few-shot") + normalized = query.strip() + if not normalized: + raise ValueError("query must not be empty") + if len(normalized) > 12_000: + raise ValueError("query must be at most 12,000 characters") + if not os.environ.get("OPENROUTER_API_KEY", "").strip(): + raise PermissionError("Set OPENROUTER_API_KEY on the server to enable live routing.") + + router = Router(models=MODELS, history_top_k=2, timeout=30.0) + if mode == "few-shot": + router.set_history(HISTORY) + started = time.perf_counter() + result = router.route(normalized) + elapsed_ms = (time.perf_counter() - started) * 1000 + return { + "mode": mode, + "selected_model": result.model, + "selected_label": MODEL_LABELS[result.model], + "confidence": result.confidence, + "probabilities": result.probabilities, + "latency_ms": elapsed_ms, + "route_cost_usd": _number(result.usage.get("cost")), + "input_tokens": _number(result.usage.get("input_tokens")), + "output_tokens": _number(result.usage.get("output_tokens")), + "evidence": evidence_for(normalized) if mode == "few-shot" else [], + } + + +class PlaygroundHandler(BaseHTTPRequestHandler): + server_version = "ReflexRoutePlayground/0.1" + + def _headers(self, content_type: str, length: int, status: int = 200) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("X-Frame-Options", "DENY") + self.send_header("Referrer-Policy", "no-referrer") + self.send_header( + "Content-Security-Policy", + "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; " + "img-src 'self' data:; connect-src 'self'", + ) + self.end_headers() + + def _json(self, payload: Mapping[str, Any], status: int = 200) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self._headers("application/json; charset=utf-8", len(body), status) + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + path = urlsplit(self.path).path + if path == "/api/config": + self._json(config_payload()) + return + if path == "/api/benchmark": + self._json(benchmark_payload()) + return + if path == "/api/replay": + self._json(replay_payload()) + return + if path not in STATIC_FILES: + self._json({"error": "Not found"}, HTTPStatus.NOT_FOUND) + return + name, content_type = STATIC_FILES[path] + try: + body = (DEMO_ROOT / name).read_bytes() + except OSError: + self._json({"error": "Static asset unavailable"}, HTTPStatus.NOT_FOUND) + return + self._headers(content_type, len(body)) + self.wfile.write(body) + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + path = urlsplit(self.path).path + if path != "/api/route": + self._json({"error": "Not found"}, HTTPStatus.NOT_FOUND) + return + if self.headers.get_content_type() != "application/json": + self._json( + {"error": "Content-Type must be application/json"}, + HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + ) + return + origin = self.headers.get("Origin") + expected_origin = f"http://{self.headers.get('Host', '')}" + if origin and origin != expected_origin: + self._json({"error": "Cross-origin requests are not allowed"}, HTTPStatus.FORBIDDEN) + return + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + length = 0 + if length <= 0 or length > MAX_BODY_BYTES: + self._json({"error": "Invalid request size"}, HTTPStatus.BAD_REQUEST) + return + try: + value = json.loads(self.rfile.read(length).decode("utf-8")) + if not isinstance(value, Mapping): + raise ValueError("request body must be an object") + query = value.get("query") + mode = value.get("mode") + if not isinstance(query, str) or not isinstance(mode, str): + raise ValueError("query and mode must be strings") + payload = route_payload(query, mode) + except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as exc: + self._json({"error": str(exc)}, HTTPStatus.BAD_REQUEST) + return + except PermissionError as exc: + self._json({"error": str(exc)}, HTTPStatus.SERVICE_UNAVAILABLE) + return + except Exception as exc: + self._json( + {"error": f"Routing failed: {str(exc)[:500]}"}, + HTTPStatus.BAD_GATEWAY, + ) + return + self._json(payload) + + def log_message(self, format: str, *args: Any) -> None: + print(f"[{self.log_date_time_string()}] {format % args}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=7860) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if not 1 <= args.port <= 65535: + raise SystemExit("port must be between 1 and 65535") + server = ThreadingHTTPServer((args.host, args.port), PlaygroundHandler) + print(f"ReflexRoute Playground: http://{args.host}:{args.port}") + print( + "Live routing enabled." + if os.environ.get("OPENROUTER_API_KEY", "").strip() + else "Replay mode ready. Set OPENROUTER_API_KEY to enable live routing." + ) + if args.host not in {"127.0.0.1", "localhost", "::1"}: + print("Warning: this demo has no user authentication; prefer the default localhost bind.") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopping Playground.") + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/demo/index.html b/demo/index.html new file mode 100644 index 0000000..5f41486 --- /dev/null +++ b/demo/index.html @@ -0,0 +1,183 @@ + + + + + + + ReflexRoute Playground + + + +
+
+ +
+
+ + + + ReflexRoute + Playground + + +
+ + Checking server +
+
+ +
+
+

ROUTE ON PRIORS · ADAPT WITH EXAMPLES

+

Watch the route change.

+

+ Compare a cold-start decision with the same router after it sees a + handful of your outcomes. No training, embeddings, or model call. +

+
+
+ 130 + built-in model priors +
+
+ +
+ + +
+
+
+ 02 +

Routing decision

+
+ RECORDED +
+ + +
+ +
+
R
+

Ready to route

+

Run a live decision or replay the recorded adaptation.

+
+
+
+ +
+
+
+ 03 +

Candidate pool

+
+ 4 MODELS +
+
+
+ +
+
+
+
+ 04 +

Retrieved evidence

+
+ FEW-SHOT ONLY +
+
+

Zero-shot decisions use model priors only.

+
+
+ +
+
+
+ 05 +

Small result

+
+ 191 QUERIES +
+

Mean quality on a complete-case LLMRouterBench slice.

+
+
+ Router cost / 1k + — +
+
+
+ + +
+ + + + + + diff --git a/demo/styles.css b/demo/styles.css new file mode 100644 index 0000000..c742e02 --- /dev/null +++ b/demo/styles.css @@ -0,0 +1,905 @@ +:root { + color-scheme: dark; + --bg: #07100f; + --panel: rgba(13, 26, 24, 0.86); + --panel-strong: #102320; + --line: rgba(178, 255, 224, 0.13); + --line-strong: rgba(178, 255, 224, 0.24); + --text: #eefbf5; + --muted: #8eaaa0; + --mint: #62f6bb; + --lime: #c4ff61; + --cyan: #65ddd1; + --warning: #ffc96b; + --error: #ff8585; + --radius: 22px; + --shadow: 0 24px 70px rgba(0, 0, 0, 0.36); + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, + "Segoe UI", sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-height: 100vh; + color: var(--text); + background: + linear-gradient(rgba(98, 246, 187, 0.025) 1px, transparent 1px), + linear-gradient(90deg, rgba(98, 246, 187, 0.025) 1px, transparent 1px), + radial-gradient(circle at 50% -10%, #173b32 0, var(--bg) 42%); + background-size: 38px 38px, 38px 38px, auto; +} + +button, +textarea { + font: inherit; +} + +button { + color: inherit; +} + +.ambient { + position: fixed; + z-index: -1; + width: 360px; + height: 360px; + border-radius: 50%; + filter: blur(120px); + opacity: 0.12; + pointer-events: none; +} + +.ambient-one { + top: 18%; + left: -160px; + background: var(--mint); +} + +.ambient-two { + right: -180px; + bottom: 8%; + background: var(--cyan); +} + +.shell { + width: min(1220px, calc(100% - 40px)); + margin: 0 auto; + padding: 26px 0 48px; +} + +.topbar, +.brand, +.status-pill, +.panel-heading, +.panel-heading > div, +.query-meta, +.result-card-top, +.benchmark-footnote, +footer { + display: flex; + align-items: center; +} + +.topbar { + justify-content: space-between; +} + +.brand { + gap: 12px; + color: var(--text); + text-decoration: none; +} + +.brand-mark { + display: grid; + width: 38px; + height: 38px; + place-items: center; + color: #07100f; + font-weight: 900; + background: linear-gradient(135deg, var(--lime), var(--mint)); + border-radius: 11px; + box-shadow: 0 0 28px rgba(98, 246, 187, 0.22); +} + +.brand strong, +.brand small { + display: block; +} + +.brand strong { + letter-spacing: -0.02em; +} + +.brand small { + margin-top: 1px; + color: var(--muted); + font-size: 11px; + letter-spacing: 0.14em; + text-transform: uppercase; +} + +.status-pill { + gap: 8px; + padding: 8px 12px; + color: var(--muted); + font-size: 12px; + background: rgba(8, 20, 18, 0.72); + border: 1px solid var(--line); + border-radius: 999px; +} + +.status-dot { + width: 7px; + height: 7px; + background: var(--warning); + border-radius: 50%; + box-shadow: 0 0 12px currentColor; +} + +.status-pill.live .status-dot { + background: var(--mint); +} + +.status-pill.replay .status-dot { + background: var(--cyan); +} + +.hero { + display: grid; + grid-template-columns: 1fr auto; + gap: 40px; + align-items: end; + padding: 76px 6px 42px; +} + +.eyebrow, +.micro-label, +.run-kind, +.step, +.result-mode { + font-size: 10px; + font-weight: 750; + letter-spacing: 0.16em; + text-transform: uppercase; +} + +.eyebrow { + margin: 0 0 14px; + color: var(--mint); +} + +h1 { + max-width: 700px; + margin: 0; + font-size: clamp(44px, 7vw, 82px); + line-height: 0.96; + letter-spacing: -0.065em; +} + +h1 span { + color: transparent; + background: linear-gradient(100deg, var(--lime), var(--mint)); + background-clip: text; + -webkit-background-clip: text; +} + +.hero-copy { + max-width: 620px; + margin: 22px 0 0; + color: var(--muted); + font-size: 16px; + line-height: 1.7; +} + +.hero-stat { + min-width: 150px; + padding: 20px 22px; + background: rgba(8, 20, 18, 0.55); + border: 1px solid var(--line); + border-radius: 18px; +} + +.hero-stat-value, +.hero-stat-label { + display: block; +} + +.hero-stat-value { + color: var(--lime); + font-size: 40px; + font-weight: 760; + letter-spacing: -0.05em; +} + +.hero-stat-label { + margin-top: 2px; + color: var(--muted); + font-size: 12px; +} + +.panel { + background: linear-gradient(145deg, rgba(16, 35, 32, 0.92), var(--panel)); + border: 1px solid var(--line); + border-radius: var(--radius); + box-shadow: var(--shadow); + backdrop-filter: blur(16px); +} + +.workspace { + display: grid; + grid-template-columns: minmax(310px, 0.8fr) minmax(480px, 1.45fr); + gap: 16px; +} + +.controls-panel, +.decision-panel, +.candidates-section, +.evidence-panel, +.benchmark-panel { + padding: 24px; +} + +.panel-heading { + justify-content: space-between; + margin-bottom: 22px; +} + +.panel-heading > div { + gap: 10px; +} + +.panel-heading h2 { + margin: 0; + font-size: 15px; + letter-spacing: -0.01em; +} + +.step { + color: var(--mint); +} + +.micro-label, +.run-kind { + color: var(--muted); +} + +label, +.mode-label { + display: block; + margin: 0 0 9px; + color: var(--muted); + font-size: 12px; + font-weight: 650; +} + +textarea { + width: 100%; + min-height: 142px; + resize: vertical; + padding: 15px; + color: var(--text); + line-height: 1.55; + background: #091815; + border: 1px solid var(--line); + border-radius: 14px; + outline: none; + transition: border-color 160ms ease, box-shadow 160ms ease; +} + +textarea:focus { + border-color: rgba(98, 246, 187, 0.48); + box-shadow: 0 0 0 3px rgba(98, 246, 187, 0.08); +} + +.query-meta { + justify-content: space-between; + margin: 8px 2px 23px; + color: var(--muted); + font-size: 10px; +} + +.text-button { + padding: 0; + color: var(--mint); + background: none; + border: 0; + cursor: pointer; +} + +.mode-switch { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 7px; + padding: 5px; + margin-bottom: 18px; + background: #091815; + border: 1px solid var(--line); + border-radius: 14px; +} + +.mode-button { + padding: 11px; + text-align: left; + background: transparent; + border: 1px solid transparent; + border-radius: 10px; + cursor: pointer; +} + +.mode-button span, +.mode-button small { + display: block; +} + +.mode-button span { + font-size: 12px; + font-weight: 700; +} + +.mode-button small { + margin-top: 2px; + color: var(--muted); + font-size: 9px; +} + +.mode-button.active { + background: rgba(98, 246, 187, 0.1); + border-color: rgba(98, 246, 187, 0.28); +} + +.mode-button.active span { + color: var(--mint); +} + +.action-stack { + display: grid; + gap: 8px; +} + +.primary-button, +.secondary-button, +.ghost-button { + min-height: 45px; + padding: 0 15px; + border-radius: 12px; + cursor: pointer; + transition: transform 150ms ease, border-color 150ms ease, opacity 150ms ease; +} + +.primary-button:hover, +.secondary-button:hover, +.ghost-button:hover { + transform: translateY(-1px); +} + +.primary-button { + display: flex; + justify-content: space-between; + align-items: center; + color: #06110d; + font-weight: 800; + background: linear-gradient(100deg, var(--lime), var(--mint)); + border: 0; +} + +.secondary-button { + color: var(--text); + font-weight: 680; + background: rgba(98, 246, 187, 0.08); + border: 1px solid rgba(98, 246, 187, 0.24); +} + +.ghost-button { + color: var(--muted); + background: transparent; + border: 1px solid var(--line); +} + +button:disabled { + cursor: not-allowed; + opacity: 0.38; + transform: none !important; +} + +.privacy-note { + display: flex; + gap: 9px; + align-items: flex-start; + margin-top: 18px; + padding-top: 16px; + color: var(--muted); + font-size: 10px; + line-height: 1.5; + border-top: 1px solid var(--line); +} + +.privacy-note span { + color: var(--mint); + font-size: 8px; +} + +.privacy-note p { + margin: 0; +} + +.decision-panel { + min-height: 570px; +} + +.result-grid { + display: grid; + gap: 12px; +} + +.result-grid.comparison { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.result-card { + padding: 20px; + background: rgba(7, 18, 16, 0.72); + border: 1px solid var(--line); + border-radius: 17px; + animation: result-in 320ms ease both; +} + +@keyframes result-in { + from { + opacity: 0; + transform: translateY(7px); + } +} + +.result-card-top { + justify-content: space-between; + gap: 18px; + margin-bottom: 20px; +} + +.result-mode { + display: block; + margin-bottom: 7px; + color: var(--mint); +} + +.selected-model { + margin: 0; + font-size: clamp(17px, 2.1vw, 24px); + letter-spacing: -0.035em; +} + +.confidence-ring { + --confidence-angle: 0deg; + display: grid; + flex: 0 0 auto; + width: 72px; + height: 72px; + place-content: center; + text-align: center; + background: + radial-gradient(circle at center, #0c1d1a 59%, transparent 61%), + conic-gradient(var(--mint) var(--confidence-angle), rgba(98, 246, 187, 0.1) 0); + border-radius: 50%; +} + +.confidence-value, +.confidence-ring small { + display: block; +} + +.confidence-value { + font-size: 17px; + font-weight: 780; +} + +.confidence-ring small { + margin-top: -2px; + color: var(--muted); + font-size: 7px; + text-transform: uppercase; +} + +.probability-list { + display: grid; + gap: 11px; +} + +.probability-row { + display: grid; + grid-template-columns: minmax(90px, 1fr) 2fr 42px; + gap: 10px; + align-items: center; + font-size: 10px; +} + +.probability-label { + overflow: hidden; + color: var(--muted); + text-overflow: ellipsis; + white-space: nowrap; +} + +.probability-track { + height: 7px; + overflow: hidden; + background: rgba(255, 255, 255, 0.05); + border-radius: 999px; +} + +.probability-fill { + width: var(--probability-width); + height: 100%; + background: linear-gradient(90deg, var(--cyan), var(--mint)); + border-radius: inherit; + transition: width 500ms ease; +} + +.probability-number { + text-align: right; + font-variant-numeric: tabular-nums; +} + +.probability-row.selected .probability-label, +.probability-row.selected .probability-number { + color: var(--lime); + font-weight: 750; +} + +.metric-row { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 8px; + margin-top: 20px; + padding-top: 15px; + border-top: 1px solid var(--line); +} + +.metric-row span, +.metric-row strong { + display: block; +} + +.metric-row span { + margin-bottom: 4px; + color: var(--muted); + font-size: 8px; + text-transform: uppercase; +} + +.metric-row strong { + font-size: 11px; + font-variant-numeric: tabular-nums; +} + +.empty-state { + display: grid; + min-height: 430px; + place-content: center; + text-align: center; +} + +.empty-state.hidden, +.hidden { + display: none !important; +} + +.empty-orbit { + display: grid; + width: 92px; + height: 92px; + margin: 0 auto 18px; + place-items: center; + border: 1px solid rgba(98, 246, 187, 0.28); + border-radius: 50%; + box-shadow: inset 0 0 28px rgba(98, 246, 187, 0.05); +} + +.empty-orbit span { + display: grid; + width: 48px; + height: 48px; + place-items: center; + color: var(--bg); + font-weight: 900; + background: var(--mint); + border-radius: 50%; +} + +.empty-state h3 { + margin: 0 0 7px; + font-size: 17px; +} + +.empty-state p, +.muted-copy, +.benchmark-intro { + color: var(--muted); + font-size: 11px; +} + +.notice { + margin-bottom: 14px; + padding: 11px 13px; + color: var(--warning); + font-size: 11px; + background: rgba(255, 201, 107, 0.08); + border: 1px solid rgba(255, 201, 107, 0.2); + border-radius: 10px; +} + +.notice.error { + color: var(--error); + background: rgba(255, 133, 133, 0.08); + border-color: rgba(255, 133, 133, 0.22); +} + +.candidates-section { + margin-top: 16px; +} + +.candidate-grid { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 10px; +} + +.candidate-card { + min-height: 168px; + padding: 15px; + background: rgba(7, 18, 16, 0.54); + border: 1px solid var(--line); + border-radius: 14px; + transition: border-color 220ms ease, transform 220ms ease, background 220ms ease; +} + +.candidate-card.selected { + background: rgba(98, 246, 187, 0.09); + border-color: rgba(98, 246, 187, 0.48); + transform: translateY(-3px); +} + +.candidate-provider { + display: grid; + width: 30px; + height: 30px; + margin-bottom: 13px; + place-items: center; + color: var(--mint); + font-size: 10px; + font-weight: 800; + background: rgba(98, 246, 187, 0.08); + border: 1px solid var(--line); + border-radius: 9px; +} + +.candidate-card h3 { + margin: 0 0 7px; + font-size: 13px; +} + +.candidate-card p { + display: -webkit-box; + overflow: hidden; + margin: 0; + color: var(--muted); + font-size: 9px; + line-height: 1.55; + -webkit-box-orient: vertical; + -webkit-line-clamp: 3; +} + +.candidate-tags { + display: flex; + flex-wrap: wrap; + gap: 5px; + margin-top: 10px; +} + +.candidate-tags span { + padding: 3px 6px; + color: var(--muted); + font-size: 7px; + background: rgba(255, 255, 255, 0.035); + border-radius: 999px; +} + +.lower-grid { + display: grid; + grid-template-columns: 1.25fr 0.75fr; + gap: 16px; + margin-top: 16px; +} + +.evidence-list { + display: grid; + gap: 8px; +} + +.evidence-item { + display: grid; + grid-template-columns: 90px 1fr auto; + gap: 12px; + align-items: center; + padding: 11px 0; + border-top: 1px solid var(--line); +} + +.evidence-item:first-child { + border-top: 0; +} + +.evidence-model { + color: var(--mint); + font-size: 9px; + font-weight: 700; +} + +.evidence-query { + overflow: hidden; + color: var(--muted); + font-size: 10px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.evidence-score { + min-width: 54px; + text-align: right; + font-size: 9px; +} + +.evidence-score strong, +.evidence-score span { + display: block; +} + +.evidence-score span { + margin-top: 2px; + color: var(--muted); + font-size: 7px; +} + +.benchmark-intro { + margin: -5px 0 22px; +} + +.benchmark-bars { + display: grid; + gap: 18px; +} + +.benchmark-row-top { + display: flex; + justify-content: space-between; + margin-bottom: 7px; + font-size: 10px; +} + +.benchmark-row-top strong { + color: var(--lime); + font-variant-numeric: tabular-nums; +} + +.benchmark-track { + height: 11px; + overflow: hidden; + background: rgba(255, 255, 255, 0.05); + border-radius: 999px; +} + +.benchmark-fill { + width: var(--benchmark-width); + height: 100%; + background: var(--cyan); + border-radius: inherit; +} + +.benchmark-row.few-shot .benchmark-fill { + background: linear-gradient(90deg, var(--mint), var(--lime)); +} + +.benchmark-footnote { + justify-content: space-between; + margin-top: 22px; + padding-top: 14px; + color: var(--muted); + font-size: 9px; + border-top: 1px solid var(--line); +} + +.benchmark-footnote strong { + color: var(--text); + font-size: 10px; +} + +footer { + justify-content: space-between; + padding: 26px 5px 0; + color: var(--muted); + font-size: 9px; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.busy .primary-button, +.busy .secondary-button, +.busy .ghost-button { + pointer-events: none; + opacity: 0.45; +} + +@media (max-width: 920px) { + .workspace, + .lower-grid { + grid-template-columns: 1fr; + } + + .candidate-grid { + grid-template-columns: repeat(2, 1fr); + } + + .decision-panel { + min-height: auto; + } +} + +@media (max-width: 620px) { + .shell { + width: min(100% - 22px, 1220px); + padding-top: 16px; + } + + .hero { + grid-template-columns: 1fr; + padding: 56px 2px 28px; + } + + .hero-stat { + display: none; + } + + .controls-panel, + .decision-panel, + .candidates-section, + .evidence-panel, + .benchmark-panel { + padding: 18px; + } + + .result-grid.comparison, + .candidate-grid { + grid-template-columns: 1fr; + } + + .probability-row { + grid-template-columns: 90px 1fr 38px; + } + + .evidence-item { + grid-template-columns: 78px 1fr; + } + + .evidence-score { + display: none; + } + + footer { + gap: 12px; + align-items: flex-start; + } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + animation-duration: 0.01ms !important; + transition-duration: 0.01ms !important; + } +} diff --git a/tests/test_demo.py b/tests/test_demo.py new file mode 100644 index 0000000..c3b0973 --- /dev/null +++ b/tests/test_demo.py @@ -0,0 +1,44 @@ +from pathlib import Path +import sys + +import pytest + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from demo.app import benchmark_payload, config_payload, replay_payload, route_payload + + +def test_demo_config_and_replay_are_complete(monkeypatch): + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + config = config_payload() + replay = replay_payload() + + assert config["live_available"] is False + assert len(config["models"]) == 4 + assert config["history_count"] == 8 + assert [result["mode"] for result in replay["results"]] == [ + "zero-shot", + "few-shot", + ] + assert replay["results"][0]["selected_model"] != replay["results"][1][ + "selected_model" + ] + assert len(replay["results"][1]["evidence"]) == 8 + + +def test_demo_benchmark_reads_the_committed_summary(): + benchmark = benchmark_payload() + + assert benchmark["target_count"] == 191 + assert benchmark["few_shot"]["quality_mean"] > benchmark["zero_shot"][ + "quality_mean" + ] + + +def test_live_route_requires_server_side_key(monkeypatch): + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + + with pytest.raises(PermissionError, match="OPENROUTER_API_KEY"): + route_payload("Route this query", "zero-shot")