From b67139c3fe4a16fbfea501da669c00a3e2813714 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Tue, 9 Jun 2026 13:50:26 -0700 Subject: [PATCH 01/16] add provider choice (or concepts of it at least) --- CLAUDE.md | 10 +- README.md | 23 +- backend/CLAUDE.md | 6 +- backend/package-lock.json | 52 +++ backend/package.json | 4 + backend/src/config/llm.ts | 313 ++++++++++++++++++ backend/src/config/models.ts | 186 ++++++++++- backend/src/env.ts | 3 +- backend/src/index.ts | 92 ++++- backend/src/local-credential-types.ts | 8 +- backend/src/local-credentials.ts | 308 +++++++++++++++-- backend/src/mastra/agents/investigate.ts | 10 +- backend/src/mastra/agents/populate.ts | 12 +- backend/src/mastra/agents/refresh.ts | 10 +- backend/src/mastra/tools/investigate-tool.ts | 5 +- backend/src/mastra/workflows/populate.ts | 16 +- backend/src/mastra/workflows/update.ts | 6 +- backend/src/pipeline/schema-inference.ts | 14 +- frontend/Dockerfile.dev | 5 +- .../app/dashboard/settings/models/page.tsx | 135 ++++++-- frontend/app/setup/page.tsx | 142 ++++---- .../settings/LocalCredentialsPanel.tsx | 158 +++++---- .../components/settings/ModelSideSheet.tsx | 72 +++- .../components/settings/llm-providers.tsx | 156 +++++++++ frontend/convex/localCredentials.ts | 39 ++- frontend/convex/modelConfig.ts | 122 ++++--- frontend/convex/schema.ts | 35 +- frontend/lib/backend.ts | 50 ++- frontend/public/logos/providers/anthropic.svg | 1 + frontend/public/logos/providers/openai.svg | 10 + .../logos/providers/openrouter-wordmark.svg | 10 + .../public/logos/providers/openrouter.svg | 6 + makefiles/Makefile | 6 +- 33 files changed, 1684 insertions(+), 341 deletions(-) create mode 100644 backend/src/config/llm.ts create mode 100644 frontend/components/settings/llm-providers.tsx create mode 100644 frontend/public/logos/providers/anthropic.svg create mode 100644 frontend/public/logos/providers/openai.svg create mode 100644 frontend/public/logos/providers/openrouter-wordmark.svg create mode 100644 frontend/public/logos/providers/openrouter.svg diff --git a/CLAUDE.md b/CLAUDE.md index b9b15bd..43bcbd3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,9 +6,9 @@ Frontend on :3500, backend on :3501, Mastra Studio on :4111, Convex dashboard on ## Setup -1. Copy `.env.example` to `.env` and fill in your keys: +1. Local mode creates `.env` automatically and collects TinyFish + LLM provider credentials in the setup UI. Production still uses env keys: - `TINYFISH_API_KEY` — from https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2 - - `OPENROUTER_API_KEY` — from https://openrouter.ai/settings/keys + - `OPENROUTER_API_KEY` — production default LLM provider key - `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` — from Clerk API Keys - `CLERK_SECRET_KEY` — from Clerk API Keys - `CLERK_JWT_ISSUER_DOMAIN` — your Frontend API URL (e.g. `https://your-app.clerk.accounts.dev`) @@ -26,9 +26,9 @@ Frontend uses Convex React hooks (`useQuery`, `useMutation`) with `ConvexProvide Backend is Fastify + Mastra. Fastify serves the HTTP API (Clerk JWT auth on protected routes via `backend/src/clerk-auth.ts`). Mastra (`backend/src/mastra/`) is the workflow orchestration layer — it wraps pipelines into inspectable workflows with a Studio UI. Both run as separate Docker services sharing the same source code. -The schema inference pipeline: frontend calls `POST /infer-schema` → Fastify verifies the Clerk JWT → calls `inferSchema()` in `backend/src/pipeline/schema-inference.ts` → Claude Sonnet 4.6 via OpenRouter → returns a Zod-validated `DatasetSchema` → frontend maps it to editable columns in the wizard. +The schema inference pipeline: frontend calls `POST /infer-schema` → Fastify verifies the Clerk JWT → calls `inferSchema()` in `backend/src/pipeline/schema-inference.ts` → selected LLM provider/model → returns a Zod-validated `DatasetSchema` → frontend maps it to editable columns in the wizard. -The populate pipeline: frontend calls `POST /populate` with `{ datasetId, datasetName, description, columns }` → Fastify verifies the Clerk JWT → triggers `populateWorkflow` which: (1) clears existing rows, (2) builds a prompt from the schema, (3) runs the populate agent (Claude Sonnet 4.6) which searches the web via TinyFish APIs, then inserts rows into Convex one by one. Rows appear in realtime on the frontend via Convex reactive queries. +The populate pipeline: frontend calls `POST /populate` with `{ datasetId, datasetName, description, columns }` → Fastify verifies the Clerk JWT → triggers `populateWorkflow` which: (1) clears existing rows, (2) builds a prompt from the schema, (3) runs the populate agent using the selected LLM provider/model. The agent searches the web via TinyFish APIs, then inserts rows into Convex one by one. Rows appear in realtime on the frontend via Convex reactive queries. Convex functions use `ctx.auth.getUserIdentity()` to get the authenticated user. The `ownerId` field on datasets stores `identity.subject` (Clerk user ID). Do not pass `ownerId` from the client. @@ -36,7 +36,7 @@ Convex functions use `ctx.auth.getUserIdentity()` to get the authenticated user. Root `.env` is the only local env file. Docker Compose, package scripts, and Convex CLI helper targets all read it. Key variables: - `TINYFISH_API_KEY` — used by the populate agent for web search and fetch (get one at https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2) -- `OPENROUTER_API_KEY` — used by backend and Mastra for AI model calls +- `OPENROUTER_API_KEY` — production default LLM provider key; local mode can use OpenRouter, OpenAI, Anthropic, or custom OpenAI-compatible via setup UI - `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY`, `CLERK_SECRET_KEY` — shared by frontend and backend - `CONVEX_SELF_HOSTED_ADMIN_KEY` — used by backend for system-level Convex writes diff --git a/README.md b/README.md index 7995e51..365a579 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ On first launch, BigSet sends you to setup. You'll connect two services: | Service | What it's for | Get your key | |---------|--------------|-------------| | **TinyFish** | Web search + page fetching | [tinyfish.ai/api-keys](https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2) | -| **OpenRouter** | LLM calls (schema inference + agents) | [openrouter.ai/settings/keys](https://openrouter.ai/settings/keys) | +| **LLM provider** | Schema inference + agents | OpenRouter, OpenAI, Anthropic, or custom OpenAI-compatible | Local API keys are stored in your OS keychain. @@ -150,23 +150,22 @@ Once everything is ready, you'll see: | **Mastra Studio** (workflow inspector) | [localhost:4111](http://localhost:4111) | Open [localhost:3500](http://localhost:3500). The setup screen will ask for -TinyFish and OpenRouter credentials and save them to your OS keychain for this -workspace. +TinyFish credentials plus an LLM provider (OpenRouter, OpenAI, Anthropic, or a +custom OpenAI-compatible endpoint) and save local keys to your OS keychain for +this workspace. -### Step 3: Connect TinyFish and OpenRouter +### Step 3: Connect TinyFish and an LLM provider -TinyFish powers web search and page fetching. OpenRouter routes LLM calls to -the models BigSet uses for schema inference and agents. +TinyFish powers web search and page fetching. Your LLM provider powers schema +inference and dataset-building agents. 1. Create a TinyFish key at [agent.tinyfish.ai/api-keys](https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2) -2. Create an OpenRouter key at [openrouter.ai/settings/keys](https://openrouter.ai/settings/keys) -3. Paste both into BigSet's setup screen - -OpenRouter is pay-as-you-go; $5-10 is plenty to start. +2. Choose an LLM provider: OpenRouter, OpenAI, Anthropic, or custom OpenAI-compatible +3. Paste the provider key and model name into BigSet's setup screen > **Note:** root `.env` is the only local env file. If you edit Convex functions in `frontend/convex/`, run `make convex-push` to deploy the changes. -> **Free tier:** cloud signed-in accounts get **2,500 row operations per calendar month** (resets on the 1st, UTC). Local mode bypasses the cloud quota and uses your TinyFish/OpenRouter accounts directly. +> **Free tier:** cloud signed-in accounts get **2,500 row operations per calendar month** (resets on the 1st, UTC). Local mode bypasses the cloud quota and uses your TinyFish + LLM provider accounts directly. ### Step 4 (optional): Load curated datasets @@ -247,7 +246,7 @@ If you want a completely fresh start: `make clean` then `make dev`. | Auth | Local auth (dev); [Clerk](https://clerk.com) (cloud) | | Database | [Convex](https://convex.dev) (self-hosted) | | Data Collection | [TinyFish](https://www.tinyfish.ai?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2) APIs (Search, Fetch, Browser) | -| AI orchestration | [Mastra](https://mastra.ai) workflows + [Vercel AI SDK](https://sdk.vercel.ai) + [OpenRouter](https://openrouter.ai) → Claude Sonnet (schema inference + populate agent) | +| AI orchestration | [Mastra](https://mastra.ai) workflows + [Vercel AI SDK](https://sdk.vercel.ai) + local LLM provider (OpenRouter, OpenAI, Anthropic, or custom OpenAI-compatible) | | Table view | [TanStack Table](https://tanstack.com/table) + [react-window](https://github.com/bvaughn/react-window) virtualization | | Exports | CSV (built-in) + XLSX ([SheetJS](https://sheetjs.com), dynamic-imported) | | Analytics | [PostHog](https://posthog.com) — events, session replay, error tracking (optional) | diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 53218a0..741b444 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -15,7 +15,7 @@ To add a new protected route, register it inside the scoped plugin in `src/index ## Schema Inference Pipeline -`src/pipeline/schema-inference.ts` — takes a natural language prompt and returns a structured `DatasetSchema` (Zod-validated, defined in `src/pipeline/types.ts`). Uses Claude Sonnet 4.6 via OpenRouter (`@openrouter/ai-sdk-provider` + Vercel AI SDK). Auto-retries once on validation failure by feeding the error back to the model. +`src/pipeline/schema-inference.ts` — takes a natural language prompt and returns a structured `DatasetSchema` (Zod-validated, defined in `src/pipeline/types.ts`). Uses the configured LLM provider/model via `src/config/llm.ts` + Vercel AI SDK. Auto-retries once on validation failure by feeding the error back to the model. The pipeline is a pure function (`inferSchema(prompt) → DatasetSchema`). It is called by both Fastify (for the HTTP API) and Mastra (for workflow orchestration). @@ -26,7 +26,7 @@ The pipeline is a pure function (`inferSchema(prompt) → DatasetSchema`). It is - `src/mastra/index.ts` — registers workflows with the `Mastra` instance (the populate agent is built per-run, not registered as a singleton) - `src/mastra/workflows/infer-schema.ts` — `inferSchemaWorkflow`, a single-step workflow wrapping `inferSchema()` - `src/mastra/workflows/populate.ts` — `populateWorkflow`, 3-step workflow: clear rows → build prompt → run populate agent -- `src/mastra/agents/populate.ts` — `buildPopulateAgent(authorizedDatasetId, authContext, columns)`, builds the orchestrator agent (Claude Sonnet 4.6) with 3 tools: `search_web`, `fetch_page`, `investigate_row`. No write access — all inserts go through investigate subagents. +- `src/mastra/agents/populate.ts` — `buildPopulateAgent(authorizedDatasetId, authContext, columns)`, builds the orchestrator agent with the configured LLM provider/model and 3 tools: `search_web`, `fetch_page`, `investigate_row`. No write access — all inserts go through investigate subagents. - `src/mastra/agents/investigate.ts` — `buildInvestigateAgent(authorizedDatasetId, authContext, columns)`, builds a per-entity subagent with `insert_row`, `list_rows`, `search_web`, `fetch_page`. Researches one entity, inserts at most one row, returns structured feedback (`INSERTED/SUMMARY/CLUES/REASON`). - `src/mastra/tools/investigate-tool.ts` — `buildInvestigateTool(authorizedDatasetId, authContext, columns)` creates the `investigate_row` tool. The orchestrator calls it to hand off a lead; it spawns a fresh investigate agent, runs it (maxSteps: 25), parses the structured output, and returns it to the orchestrator. Errors are caught and returned as structured failures so the orchestrator can self-correct. - `src/mastra/tools/dataset-tools.ts` — `buildPopulateTools(authorizedDatasetId, authContext)` factory returning 5 Convex-backed tools: `insert_row`, `list_rows`, `get_row`, `update_row`, `delete_row`. The dataset id is captured by closure so the LLM cannot redirect writes to other datasets; `authContext` (Clerk userId + workflow run id) is captured for caller-attribution in security logs and the `CAPABILITY_VIOLATION` PostHog event. See the security note at the top of the file. @@ -48,7 +48,7 @@ Required env vars (see `.env.example`): - `CONVEX_URL` — Convex instance URL - `CONVEX_SELF_HOSTED_ADMIN_KEY` — for system-level Convex writes (internal mutations) - `CLERK_SECRET_KEY`, `CLERK_PUBLISHABLE_KEY` — for JWT verification -- `OPENROUTER_API_KEY` — for AI model calls +- `OPENROUTER_API_KEY` — production default LLM provider key; local mode can use OpenRouter, OpenAI, Anthropic, or custom OpenAI-compatible via setup UI - `TINYFISH_API_KEY` — for web search and fetch (populate agent). Get one at https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2 In Docker, these are interpolated from the root `.env` file via `docker-compose.dev.yml`. diff --git a/backend/package-lock.json b/backend/package-lock.json index e231b48..c9a41ed 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,6 +8,10 @@ "name": "bigset-backend", "version": "0.1.0", "dependencies": { + "@ai-sdk/anthropic": "^3.0.81", + "@ai-sdk/openai": "^3.0.68", + "@ai-sdk/openai-compatible": "^2.0.48", + "@ai-sdk/provider": "^3.0.10", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "@mastra/core": "^1.36.0", @@ -57,6 +61,22 @@ } } }, + "node_modules/@ai-sdk/anthropic": { + "version": "3.0.81", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.81.tgz", + "integrity": "sha512-B1JDd9Ugq9R5AgIaW3674lhGCMMYJcPUxnrZh8fzbGojgg4QvHFRv6eZahGQAUsmGHbcf74G9bdSBDLWQGY2GA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/gateway": { "version": "3.0.116", "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.116.tgz", @@ -74,6 +94,38 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/openai": { + "version": "3.0.68", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.68.tgz", + "integrity": "sha512-FCs/DPr4M95UyZ/ABHJmTmCEYRCka/4J0Bna0nsd78QCdGIS0X/zhn+fVzB7mZJo7464uOWYUjROx9PGNGOb0w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai-compatible": { + "version": "2.0.48", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-2.0.48.tgz", + "integrity": "sha512-z9MC6M4Oh/yUY/F/eszOtO8wc2nMz99XmZQKd2gWTtyIfe716xTfrKe3aYZKg20NZDtyjqPPKPSR+wqz7q1T7Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/provider": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.10.tgz", diff --git a/backend/package.json b/backend/package.json index 11df78b..1bd11aa 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,6 +10,10 @@ "mastra:dev": "node ../scripts/with-root-env.mjs mastra dev" }, "dependencies": { + "@ai-sdk/anthropic": "^3.0.81", + "@ai-sdk/openai": "^3.0.68", + "@ai-sdk/openai-compatible": "^2.0.48", + "@ai-sdk/provider": "^3.0.10", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "@mastra/core": "^1.36.0", diff --git a/backend/src/config/llm.ts b/backend/src/config/llm.ts new file mode 100644 index 0000000..0c50523 --- /dev/null +++ b/backend/src/config/llm.ts @@ -0,0 +1,313 @@ +import type { LanguageModelV3 } from "@ai-sdk/provider"; +import { createAnthropic } from "@ai-sdk/anthropic"; +import { createOpenAI } from "@ai-sdk/openai"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; + +import { env } from "../env.js"; +import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; + +export const LLM_PROVIDER_TYPES = [ + "openrouter", + "openai", + "anthropic", + "custom", +] as const; + +export type LlmProviderType = (typeof LLM_PROVIDER_TYPES)[number]; + +export type ModelRoleKey = + | "schemaInference" + | "populateOrchestrator" + | "investigateSubagent"; + +export interface LlmProviderConfig { + provider: LlmProviderType; + apiKey: string; + defaultModel: string; + baseUrl?: string; + source: "local" | "env"; +} + +export interface LlmProviderInput { + provider: LlmProviderType; + apiKey: string; + defaultModel?: string; + baseUrl?: string; +} + +export const LLM_PROVIDER_LABELS: Record = { + openrouter: "OpenRouter", + openai: "OpenAI", + anthropic: "Anthropic", + custom: "Custom OpenAI-compatible", +}; + +export const LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE: Record< + LlmProviderType, + Record +> = { + openrouter: { + schemaInference: env.SCHEMA_INFERENCE_MODEL, + populateOrchestrator: env.POPULATE_ORCHESTRATOR_MODEL, + investigateSubagent: env.INVESTIGATE_SUBAGENT_MODEL, + }, + openai: { + schemaInference: "gpt-5.4-mini", + populateOrchestrator: "gpt-5.4-mini", + investigateSubagent: "gpt-5.4-mini", + }, + anthropic: { + schemaInference: "claude-sonnet-4-6", + populateOrchestrator: "claude-haiku-4-5-20251001", + investigateSubagent: "claude-haiku-4-5-20251001", + }, + custom: { + schemaInference: "", + populateOrchestrator: "", + investigateSubagent: "", + }, +}; + +export const LLM_PROVIDER_DEFAULT_MODELS: Record = { + openrouter: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.openrouter.schemaInference, + openai: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.openai.schemaInference, + anthropic: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.anthropic.schemaInference, + custom: "", +}; + +export function isLlmProviderType(value: unknown): value is LlmProviderType { + return ( + typeof value === "string" && + (LLM_PROVIDER_TYPES as readonly string[]).includes(value) + ); +} + +export function llmProviderLabel(provider: LlmProviderType): string { + return LLM_PROVIDER_LABELS[provider]; +} + +export function defaultModelForLlmProvider(provider: LlmProviderType): string { + return LLM_PROVIDER_DEFAULT_MODELS[provider]; +} + +export function defaultModelForLlmProviderRole( + provider: LlmProviderType, + role: ModelRoleKey, +): string { + return LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE[provider][role]; +} + +export function defaultBaseUrlForLlmProvider( + provider: LlmProviderType, +): string | undefined { + if (provider === "openrouter") { + return process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1"; + } + return undefined; +} + +function isLoopbackHost(hostname: string): boolean { + return ["localhost", "127.0.0.1", "0.0.0.0", "::1", "[::1]"].includes( + hostname, + ); +} + +function normalizeLocalLoopbackForBackend(parsed: URL): void { + if (env.IS_LOCAL_MODE && isLoopbackHost(parsed.hostname)) { + // In local dev the backend runs inside Docker. From the container, + // localhost points at the container, not the host machine where LM Studio + // and other local OpenAI-compatible servers usually listen. + parsed.hostname = "host.docker.internal"; + } +} + +export function normalizeBaseUrl(baseUrl?: string): string | undefined { + const trimmed = baseUrl?.trim(); + if (!trimmed) return undefined; + const parsed = new URL(trimmed); + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { + throw new Error("Base URL must start with http:// or https://"); + } + normalizeLocalLoopbackForBackend(parsed); + return parsed.toString().replace(/\/+$/, ""); +} + +export function normalizeCustomBaseUrl(baseUrl?: string): string | undefined { + const normalized = normalizeBaseUrl(baseUrl); + if (!normalized) return undefined; + + const parsed = new URL(normalized); + if (parsed.pathname === "" || parsed.pathname === "/") { + parsed.pathname = "/v1"; + } + return parsed.toString().replace(/\/+$/, ""); +} + +export function normalizeLlmProviderInput( + input: LlmProviderInput, + source: "local" | "env", +): LlmProviderConfig { + const provider = input.provider; + const apiKey = input.apiKey.trim(); + if (!apiKey && provider !== "custom") { + throw new Error(`${llmProviderLabel(provider)} API key is required`); + } + + const baseUrl = + provider === "custom" + ? normalizeCustomBaseUrl(input.baseUrl) + : normalizeBaseUrl(input.baseUrl) ?? defaultBaseUrlForLlmProvider(provider); + + if (provider === "custom" && !baseUrl) { + throw new Error("Custom providers require a base URL"); + } + + const defaultModel = + input.defaultModel?.trim() || defaultModelForLlmProvider(provider); + + return { + provider, + apiKey, + defaultModel, + baseUrl, + source, + }; +} + +export function createLanguageModel( + config: LlmProviderConfig, + modelId?: string, +): LanguageModelV3 { + const resolvedModelId = (modelId?.trim() || config.defaultModel).trim(); + if (!resolvedModelId) { + throw new Error("Model name is required"); + } + + switch (config.provider) { + case "openrouter": { + const provider = createOpenRouter({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "openai": { + const provider = createOpenAI({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "anthropic": { + const provider = createAnthropic({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "custom": { + if (!config.baseUrl) { + throw new Error("Custom providers require a base URL"); + } + const provider = createOpenAICompatible({ + name: "custom", + apiKey: config.apiKey || undefined, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + } +} + +function providerVerificationRequest(config: LlmProviderConfig): { + url: string; + headers: Record; +} { + switch (config.provider) { + case "openrouter": { + const baseUrl = (config.baseUrl || "https://openrouter.ai/api/v1").replace( + /\/+$/, + "", + ); + return { + url: `${baseUrl}/key`, + headers: { Authorization: `Bearer ${config.apiKey}` }, + }; + } + case "openai": { + const baseUrl = (config.baseUrl || "https://api.openai.com/v1").replace( + /\/+$/, + "", + ); + return { + url: `${baseUrl}/models`, + headers: { Authorization: `Bearer ${config.apiKey}` }, + }; + } + case "anthropic": { + const baseUrl = (config.baseUrl || "https://api.anthropic.com/v1").replace( + /\/+$/, + "", + ); + return { + url: `${baseUrl}/models?limit=1`, + headers: { + "x-api-key": config.apiKey, + "anthropic-version": "2023-06-01", + }, + }; + } + case "custom": { + if (!config.baseUrl) { + throw new Error("Custom providers require a base URL"); + } + const baseUrl = config.baseUrl.replace(/\/+$/, ""); + return { + url: `${baseUrl}/models`, + headers: config.apiKey + ? { Authorization: `Bearer ${config.apiKey}` } + : {}, + }; + } + } +} + +export async function verifyLlmProviderConfig( + config: LlmProviderConfig, +): Promise { + const { url, headers } = providerVerificationRequest(config); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + + try { + const response = await fetch(url, { + headers, + signal: controller.signal, + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new Error(`${llmProviderLabel(config.provider)} rejected that API key.`); + } + throw new Error( + `${llmProviderLabel(config.provider)} verification failed with HTTP ${response.status}.`, + ); + } + } catch (err) { + if (err instanceof Error && err.name === "AbortError") { + throw new Error( + `${llmProviderLabel(config.provider)} verification timed out after ${FETCH_TIMEOUT_MS / 1000} seconds.`, + ); + } + if (err instanceof Error && err.message === "fetch failed") { + const displayUrl = url.replace("host.docker.internal", "localhost"); + throw new Error( + `${llmProviderLabel(config.provider)} verification failed: could not reach ${displayUrl}. If this is LM Studio, start the local server and use http://localhost:1234 or http://localhost:1234/v1.`, + ); + } + throw err; + } finally { + clearTimeout(timeout); + } +} diff --git a/backend/src/config/models.ts b/backend/src/config/models.ts index 1d28cbf..0b65699 100644 --- a/backend/src/config/models.ts +++ b/backend/src/config/models.ts @@ -1,12 +1,14 @@ /** * Backend configuration for AI models. * - * Defines the typed interfaces and constants for OpenRouter model management. + * Defines the typed interfaces and constants for model management. */ import { api, internal, convex } from "../convex.js"; import { env } from "../env.js"; -import { requireOpenRouterApiKey } from "../local-credentials.js"; +import { getLlmProviderConfig, requireOpenRouterApiKey } from "../local-credentials.js"; +import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; +import { defaultModelForLlmProviderRole, type ModelRoleKey } from "./llm.js"; export interface OpenRouterModel { modelName: string; @@ -17,10 +19,10 @@ export interface OpenRouterModel { } /** - * Default model slugs for each agent role. - * Read from environment variables so operators can change defaults - * without touching code. Falls back to typed literals when env vars - * are unset (useful for local dev without a .env file). + * Default model identifiers for each agent role. + * Read from environment variables so operators can change production defaults + * without touching code. Local mode falls back to the selected LLM provider's + * default model first. */ export const DEFAULT_MODEL_IDS = { SCHEMA_INFERENCE: env.SCHEMA_INFERENCE_MODEL, @@ -28,6 +30,87 @@ export const DEFAULT_MODEL_IDS = { INVESTIGATE_SUBAGENT: env.INVESTIGATE_SUBAGENT_MODEL, } as const; +const OPENAI_MODEL_EXCLUDE_PATTERNS = [ + "audio", + "babbage", + "dall-e", + "davinci", + "embedding", + "image", + "instruct", + "moderation", + "realtime", + "sora", + "transcribe", + "tts", + "whisper", +]; + +function isOpenAITextModelId(id: string): boolean { + const lower = id.toLowerCase(); + if (OPENAI_MODEL_EXCLUDE_PATTERNS.some((pattern) => lower.includes(pattern))) { + return false; + } + return ( + lower.startsWith("gpt-") || + lower.startsWith("o1") || + lower.startsWith("o3") || + lower.startsWith("o4") || + lower.startsWith("chatgpt-") + ); +} + +function sortModels(models: OpenRouterModel[]): OpenRouterModel[] { + return models.sort((a, b) => a.modelName.localeCompare(b.modelName)); +} + +function isModelCompatibleWithProvider( + modelId: string | undefined, + provider: Awaited>, +): modelId is string { + if (!modelId) return false; + if (!provider) return true; + switch (provider.provider) { + case "openrouter": + return modelId.includes("/"); + case "openai": + return isOpenAITextModelId(modelId) && !modelId.includes("/"); + case "anthropic": + return modelId.startsWith("claude-") && !modelId.includes("/"); + case "custom": + return true; + } +} + +function modelForProvider( + savedModel: string | undefined, + role: ModelRoleKey, + envDefault: string, + provider: Awaited>, +): string { + if (isModelCompatibleWithProvider(savedModel, provider)) return savedModel; + if (provider?.provider) return defaultModelForLlmProviderRole(provider.provider, role); + return envDefault; +} + +async function fetchJsonWithTimeout( + url: string, + headers: Record, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + + try { + const response = await fetch(url, { headers, signal: controller.signal }); + if (!response.ok) { + throw new Error(`Model list request failed with HTTP ${response.status}.`); + } + return (await response.json()) as T; + } finally { + clearTimeout(timeout); + } +} + /** * Model roles for the settings UI. */ @@ -58,6 +141,66 @@ export async function getCachedModels(): Promise { return fetched; } +export async function fetchModelsForCurrentLlmProvider(): Promise { + const config = await getLlmProviderConfig(); + if (!config) { + throw new Error("LLM provider is not configured."); + } + + if (config.provider === "openrouter") { + return await getCachedModels(); + } + + if (config.provider === "anthropic") { + const baseUrl = (config.baseUrl || "https://api.anthropic.com/v1").replace(/\/+$/, ""); + const json = await fetchJsonWithTimeout<{ + data?: Array<{ + id: string; + display_name?: string; + max_input_tokens?: number; + }>; + }>(`${baseUrl}/models?limit=100`, { + "x-api-key": config.apiKey, + "anthropic-version": "2023-06-01", + }); + + return sortModels( + (json.data ?? []).map((model) => ({ + modelName: model.display_name ?? model.id, + canonicalSlug: model.id, + contextLength: model.max_input_tokens ?? 0, + completionCost: 0, + promptCost: 0, + })), + ); + } + + const baseUrl = (config.baseUrl || "https://api.openai.com/v1").replace(/\/+$/, ""); + const headers: Record = + config.provider === "custom" && !config.apiKey + ? {} + : { Authorization: `Bearer ${config.apiKey}` }; + const json = await fetchJsonWithTimeout<{ + data?: Array<{ id: string }>; + }>(`${baseUrl}/models`, headers); + + const models = (json.data ?? []) + .filter((model) => + config.provider === "openai" + ? isOpenAITextModelId(model.id) + : true, + ) + .map((model) => ({ + modelName: model.id, + canonicalSlug: model.id, + contextLength: 0, + completionCost: 0, + promptCost: 0, + })); + + return sortModels(models); +} + /** * Validate that a model slug exists in the cached model list. * Throws with a clear message if the slug is not found. @@ -98,8 +241,10 @@ export async function upsertModelConfig( investigateSubagent?: string; } ): Promise { + const llmConfig = await getLlmProviderConfig(); await convex.mutation(internal.modelConfig.upsertInternal, { userId, + provider: llmConfig?.provider ?? "openrouter", schemaInference: config.schemaInference ?? undefined, populateOrchestrator: config.populateOrchestrator ?? undefined, investigateSubagent: config.investigateSubagent ?? undefined, @@ -108,7 +253,7 @@ export async function upsertModelConfig( /** * Fetch the model configuration for a specific user from Convex. - * If the user has no saved config, returns the system defaults from env. + * If the user has no saved config, returns the selected provider default or env defaults. * Callers always get a complete config — never null. */ export async function getModelConfig( @@ -118,11 +263,30 @@ export async function getModelConfig( populateOrchestrator: string; investigateSubagent: string; }> { - const config = await convex.query(internal.modelConfig.getInternal, { userId }); + const llmConfig = await getLlmProviderConfig(); + const config = await convex.query(internal.modelConfig.getInternal, { + userId, + provider: llmConfig?.provider ?? "openrouter", + }); return { - schemaInference: config?.schemaInference ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE, - populateOrchestrator: config?.populateOrchestrator ?? DEFAULT_MODEL_IDS.POPULATE_ORCHESTRATOR, - investigateSubagent: config?.investigateSubagent ?? DEFAULT_MODEL_IDS.INVESTIGATE_SUBAGENT, + schemaInference: modelForProvider( + config?.schemaInference, + "schemaInference", + DEFAULT_MODEL_IDS.SCHEMA_INFERENCE, + llmConfig, + ), + populateOrchestrator: modelForProvider( + config?.populateOrchestrator, + "populateOrchestrator", + DEFAULT_MODEL_IDS.POPULATE_ORCHESTRATOR, + llmConfig, + ), + investigateSubagent: modelForProvider( + config?.investigateSubagent, + "investigateSubagent", + DEFAULT_MODEL_IDS.INVESTIGATE_SUBAGENT, + llmConfig, + ), }; } diff --git a/backend/src/env.ts b/backend/src/env.ts index 97c410f..a43fa0b 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -45,7 +45,8 @@ export const env = { LOCAL_KEYCHAIN_TIMEOUT_MS: numberFromEnv("LOCAL_KEYCHAIN_TIMEOUT_MS", 5_000), // Default models — used when a user has not saved a preference. - // Each must be a valid OpenRouter model slug. + // In production these are still interpreted as OpenRouter model slugs; in + // local mode the selected LLM provider's default model is used first. SCHEMA_INFERENCE_MODEL: process.env.SCHEMA_INFERENCE_MODEL ?? "anthropic/claude-sonnet-4.6", POPULATE_ORCHESTRATOR_MODEL: diff --git a/backend/src/index.ts b/backend/src/index.ts index cb57cd1..834a702 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -20,9 +20,17 @@ import { getLocalSetupStatus, requireLocalSetupComplete, saveLocalCredential, + saveLocalLlmProviderConfig, + setActiveLocalLlmProvider, verifyOpenRouterApiKey, verifyTinyFishApiKey, } from "./local-credentials.js"; +import { + isLlmProviderType, + normalizeLlmProviderInput, + verifyLlmProviderConfig, + type LlmProviderInput, +} from "./config/llm.js"; /** Domain part of an email, for analytics (we never log full addresses). */ function emailDomain(email: string): string { @@ -159,7 +167,7 @@ async function ensureLocalSetupReady(reply: FastifyReply): Promise { return true; } catch { await reply.code(428).send({ - error: "Local setup is incomplete. Connect TinyFish and OpenRouter first.", + error: "Local setup is incomplete. Connect TinyFish and an LLM provider first.", }); return false; } @@ -711,6 +719,51 @@ fastify.post("/local-setup/tinyfish", async (req, reply) => { } }); +fastify.post("/local-setup/llm-provider", async (req, reply) => { + if (!env.IS_LOCAL_MODE) { + return reply.code(404).send({ error: "Not found" }); + } + + const body = (req.body ?? {}) as Partial; + const provider = body.provider; + if (!isLlmProviderType(provider)) { + return reply.code(400).send({ error: "Choose a supported LLM provider" }); + } + + try { + const apiKey = body.apiKey?.trim() ?? ""; + const isNewCustomWithoutKey = provider === "custom" && !!body.baseUrl?.trim(); + + if (!apiKey && !isNewCustomWithoutKey) { + const status = await getLocalSetupStatus(); + const savedProvider = status.services.llmProviders?.[provider]; + if (!savedProvider?.configured) { + return reply.code(400).send({ error: `${savedProvider?.providerLabel ?? provider} API key is required` }); + } + await setActiveLocalLlmProvider(provider); + return await getLocalSetupStatus(); + } + + const config = normalizeLlmProviderInput( + { + provider, + apiKey, + baseUrl: body.baseUrl, + defaultModel: body.defaultModel, + }, + "local", + ); + await verifyLlmProviderConfig(config); + await saveLocalLlmProviderConfig(config, "api_key"); + return await getLocalSetupStatus(); + } catch (err) { + const message = err instanceof Error ? err.message : "LLM provider verification failed"; + req.log.warn({ err }, "LLM provider local setup verification failed"); + return reply.code(400).send({ error: message }); + } +}); + +// Backward-compatible endpoint for older setup UI builds. fastify.post("/local-setup/openrouter-key", async (req, reply) => { if (!env.IS_LOCAL_MODE) { return reply.code(404).send({ error: "Not found" }); @@ -782,6 +835,18 @@ fastify.get("/openrouter/models", async (req, reply) => { } }); +fastify.get("/llm-provider/models", async (req, reply) => { + const { fetchModelsForCurrentLlmProvider } = await import("./config/models.js"); + try { + const models = await fetchModelsForCurrentLlmProvider(); + return { models }; + } catch (err) { + const message = err instanceof Error ? err.message : "Failed to load models"; + req.log.error(err, "Failed to load current LLM provider models"); + return reply.code(500).send({ error: message }); + } +}); + // ──────────────────────────────────────────────────────────────────────── // Protected routes — gated by Clerk JWT verification // ──────────────────────────────────────────────────────────────────────── @@ -796,27 +861,30 @@ await fastify.register(async (instance) => { }); instance.post("/settings/models", async (req, reply) => { - const { upsertModelConfig, validateModelSlug, getCachedModels } = await import("./config/models.js"); + const { upsertModelConfig, fetchModelsForCurrentLlmProvider } = await import("./config/models.js"); const body = req.body as { schemaInference?: string | null; populateOrchestrator?: string | null; investigateSubagent?: string | null; }; + const config = { + schemaInference: typeof body.schemaInference === "string" ? body.schemaInference.trim() || undefined : undefined, + populateOrchestrator: typeof body.populateOrchestrator === "string" ? body.populateOrchestrator.trim() || undefined : undefined, + investigateSubagent: typeof body.investigateSubagent === "string" ? body.investigateSubagent.trim() || undefined : undefined, + }; const toValidate: Array<{ role: "schemaInference" | "populateOrchestrator" | "investigateSubagent"; slug: string }> = []; - if (body.schemaInference) toValidate.push({ role: "schemaInference", slug: body.schemaInference }); - if (body.populateOrchestrator) toValidate.push({ role: "populateOrchestrator", slug: body.populateOrchestrator }); - if (body.investigateSubagent) toValidate.push({ role: "investigateSubagent", slug: body.investigateSubagent }); + if (config.schemaInference) toValidate.push({ role: "schemaInference", slug: config.schemaInference }); + if (config.populateOrchestrator) toValidate.push({ role: "populateOrchestrator", slug: config.populateOrchestrator }); + if (config.investigateSubagent) toValidate.push({ role: "investigateSubagent", slug: config.investigateSubagent }); if (toValidate.length > 0) { try { - const models = await getCachedModels(); + const models = await fetchModelsForCurrentLlmProvider(); for (const { role, slug } of toValidate) { const found = models.some((m) => m.canonicalSlug === slug); if (!found) { - return reply.code(400).send({ - error: `Invalid model slug "${slug}" for ${role}. Refresh the model list first or choose a different model.`, - }); + req.log.warn({ role, slug }, "Saving model slug that was not returned by the current LLM provider"); } } } catch (err) { @@ -826,9 +894,9 @@ await fastify.register(async (instance) => { try { await upsertModelConfig(req.auth!.userId, { - schemaInference: body.schemaInference ?? undefined, - populateOrchestrator: body.populateOrchestrator ?? undefined, - investigateSubagent: body.investigateSubagent ?? undefined, + schemaInference: config.schemaInference, + populateOrchestrator: config.populateOrchestrator, + investigateSubagent: config.investigateSubagent, }); return { success: true }; } catch (err) { diff --git a/backend/src/local-credential-types.ts b/backend/src/local-credential-types.ts index bcb4235..9d5fc95 100644 --- a/backend/src/local-credential-types.ts +++ b/backend/src/local-credential-types.ts @@ -1,4 +1,10 @@ -export const LOCAL_CREDENTIAL_SERVICES = ["tinyfish", "openrouter"] as const; +export const LOCAL_CREDENTIAL_SERVICES = [ + "tinyfish", + "openrouter", + "openai", + "anthropic", + "custom", +] as const; export type LocalCredentialService = (typeof LOCAL_CREDENTIAL_SERVICES)[number]; export type ConnectionMethod = "api_key" | "oauth"; diff --git a/backend/src/local-credentials.ts b/backend/src/local-credentials.ts index a6a5b4b..5b7727d 100644 --- a/backend/src/local-credentials.ts +++ b/backend/src/local-credentials.ts @@ -5,6 +5,16 @@ import { getKeychainCredential, setKeychainCredential, } from "./local-keychain-client.js"; +import { + defaultBaseUrlForLlmProvider, + defaultModelForLlmProvider, + isLlmProviderType, + llmProviderLabel, + normalizeLlmProviderInput, + type LlmProviderConfig, + type LlmProviderInput, + type LlmProviderType, +} from "./config/llm.js"; import type { ConnectionMethod, LocalCredentialService, @@ -17,13 +27,23 @@ export interface ServiceSetupStatus { source: "local" | "env" | null; connectionMethod: ConnectionMethod | null; verifiedAt: number | null; + provider?: LlmProviderType; + providerLabel?: string; + baseUrl?: string; + defaultModel?: string; } export interface LocalSetupStatus { mode: "local" | "production"; required: boolean; complete: boolean; - services: Record; + services: { + tinyfish: ServiceSetupStatus; + llm: ServiceSetupStatus; + llmProviders?: Record; + /** Deprecated compatibility alias for older UI code. */ + openrouter?: ServiceSetupStatus; + }; } function isPlaceholder(value: string, service: LocalCredentialService): boolean { @@ -35,30 +55,84 @@ function isPlaceholder(value: string, service: LocalCredentialService): boolean function envCredential(service: LocalCredentialService): string | undefined { const value = - service === "tinyfish" ? process.env.TINYFISH_API_KEY : env.OPENROUTER_API_KEY; + service === "tinyfish" + ? process.env.TINYFISH_API_KEY + : service === "openrouter" + ? env.OPENROUTER_API_KEY + : undefined; if (!value || isPlaceholder(value, service)) return undefined; return value; } +function llmProviderService(provider: LlmProviderType): LocalCredentialService { + return provider; +} + async function localCredential(service: LocalCredentialService): Promise<{ apiKey: string; connectionMethod: ConnectionMethod; verifiedAt: number | null; keychainAccount: string; + llmProvider?: LlmProviderType; + llmBaseUrl?: string; + llmDefaultModel?: string; } | null> { if (!env.IS_LOCAL_MODE) return null; - const keychain = await getKeychainCredential(service); - if (!keychain?.apiKey) return null; const row = await convex.query(internal.localCredentials.getInternal, { service, }); + const rowData = row as + | { + keychainAccount?: string; + connectionMethod?: ConnectionMethod; + verifiedAt?: number; + llmProvider?: unknown; + llmBaseUrl?: unknown; + llmDefaultModel?: unknown; + } + | null; + + const keychain = await getKeychainCredential(service); + if (!keychain?.apiKey) { + // LM Studio and many local OpenAI-compatible servers do not require an + // API key. A custom-provider row with a base URL is therefore a valid + // local credential even when there is no keychain secret. + if ( + service === "custom" && + rowData?.llmProvider === "custom" && + typeof rowData.llmBaseUrl === "string" + ) { + return { + apiKey: "", + connectionMethod: rowData.connectionMethod ?? "api_key", + verifiedAt: rowData.verifiedAt ?? null, + keychainAccount: rowData.keychainAccount ?? "", + llmProvider: "custom", + llmBaseUrl: rowData.llmBaseUrl, + llmDefaultModel: + typeof rowData.llmDefaultModel === "string" + ? rowData.llmDefaultModel + : undefined, + }; + } + return null; + } return { apiKey: keychain.apiKey, - connectionMethod: row?.connectionMethod ?? "api_key", - verifiedAt: row?.verifiedAt ?? null, + connectionMethod: rowData?.connectionMethod ?? "api_key", + verifiedAt: rowData?.verifiedAt ?? null, keychainAccount: keychain.keychainAccount, + llmProvider: isLlmProviderType(rowData?.llmProvider) + ? rowData.llmProvider + : undefined, + llmBaseUrl: + typeof rowData?.llmBaseUrl === "string" ? rowData.llmBaseUrl : undefined, + llmDefaultModel: + typeof rowData?.llmDefaultModel === "string" + ? rowData.llmDefaultModel + : undefined, }; } @@ -72,6 +146,57 @@ async function localCredentialForStatus( } } +async function activeLlmProviderForStatus(): Promise { + if (!env.IS_LOCAL_MODE) return "openrouter"; + + try { + const active = await convex.query(internal.localCredentials.getInternal, { + service: "llm", + }); + const activeProvider = (active as { llmProvider?: unknown } | null) + ?.llmProvider; + if (isLlmProviderType(activeProvider)) return activeProvider; + } catch { + // Convex functions may be one push behind during local development. Fall + // back instead of making every backend status/settings route return 500. + } + + const legacy = await localCredentialForStatus("openrouter"); + if (legacy?.llmProvider) return legacy.llmProvider; + return "openrouter"; +} + +async function localCredentialForLlmProvider( + provider: LlmProviderType, +): Promise>> { + const direct = await localCredentialForStatus(llmProviderService(provider)); + if (direct && (!direct.llmProvider || direct.llmProvider === provider)) { + return direct; + } + + if (provider !== "openrouter") { + const legacy = await localCredentialForStatus("openrouter"); + if (legacy?.llmProvider === provider) return legacy; + } + + return null; +} + +export async function setActiveLocalLlmProvider( + provider: LlmProviderType, +): Promise { + if (!env.IS_LOCAL_MODE) { + throw new Error("Local credential storage is disabled when PROD=1."); + } + + await convex.mutation(internal.localCredentials.upsertInternal, { + service: "llm", + connectionMethod: "api_key", + verifiedAt: Date.now(), + llmProvider: provider, + }); +} + export async function resolveCredential( service: LocalCredentialService, ): Promise<{ apiKey: string; source: "local" | "env" } | null> { @@ -86,14 +211,55 @@ export async function resolveCredential( return null; } +export async function getLlmProviderConfig(): Promise { + if (env.IS_LOCAL_MODE) { + const provider = await activeLlmProviderForStatus(); + const local = await localCredentialForLlmProvider(provider); + if (!local) return null; + + return normalizeLlmProviderInput( + { + provider, + apiKey: local.apiKey, + baseUrl: + local.llmBaseUrl ?? defaultBaseUrlForLlmProvider(provider), + defaultModel: + local.llmDefaultModel || defaultModelForLlmProvider(provider), + }, + "local", + ); + } + + const apiKey = envCredential("openrouter"); + if (!apiKey) return null; + return normalizeLlmProviderInput( + { + provider: "openrouter", + apiKey, + baseUrl: process.env.OPENROUTER_BASE_URL, + defaultModel: env.SCHEMA_INFERENCE_MODEL, + }, + "env", + ); +} + +export async function requireLlmProviderConfig(): Promise { + const config = await getLlmProviderConfig(); + if (!config) { + throw new Error("LLM provider is not configured. Complete local setup first."); + } + return config; +} + export async function getOpenRouterApiKey(): Promise { - return (await resolveCredential("openrouter"))?.apiKey; + const config = await getLlmProviderConfig(); + return config?.provider === "openrouter" ? config.apiKey : undefined; } export async function requireOpenRouterApiKey(): Promise { const apiKey = await getOpenRouterApiKey(); if (!apiKey) { - throw new Error("OpenRouter is not configured. Complete local setup first."); + throw new Error("OpenRouter is not configured as the current LLM provider."); } return apiKey; } @@ -140,7 +306,24 @@ export async function requireLocalSetupComplete(): Promise { export async function getLocalSetupStatus(): Promise { if (!env.IS_LOCAL_MODE) { const tinyfish = envCredential("tinyfish"); - const openrouter = envCredential("openrouter"); + const llmConfig = await getLlmProviderConfig(); + const llm: ServiceSetupStatus = llmConfig + ? { + configured: true, + source: llmConfig.source, + connectionMethod: "api_key", + verifiedAt: null, + provider: llmConfig.provider, + providerLabel: llmProviderLabel(llmConfig.provider), + baseUrl: llmConfig.baseUrl, + defaultModel: llmConfig.defaultModel, + } + : { + configured: false, + source: null, + connectionMethod: null, + verifiedAt: null, + }; return { mode: "production", required: false, @@ -152,18 +335,13 @@ export async function getLocalSetupStatus(): Promise { connectionMethod: tinyfish ? "api_key" : null, verifiedAt: null, }, - openrouter: { - configured: !!openrouter, - source: openrouter ? "env" : null, - connectionMethod: openrouter ? "api_key" : null, - verifiedAt: null, - }, + llm, + openrouter: llm, }, }; } const tinyfishLocal = await localCredentialForStatus("tinyfish"); - const openrouterLocal = await localCredentialForStatus("openrouter"); const tinyfish: ServiceSetupStatus = tinyfishLocal ? { @@ -179,25 +357,52 @@ export async function getLocalSetupStatus(): Promise { verifiedAt: null, }; - const openrouter: ServiceSetupStatus = openrouterLocal - ? { - configured: true, - source: "local", - connectionMethod: openrouterLocal.connectionMethod, - verifiedAt: openrouterLocal.verifiedAt, - } - : { - configured: false, - source: null, - connectionMethod: null, - verifiedAt: null, - }; + const providerStatuses = {} as Record; + for (const provider of [ + "openrouter", + "openai", + "anthropic", + "custom", + ] as const) { + const credential = await localCredentialForLlmProvider(provider); + providerStatuses[provider] = credential + ? { + configured: true, + source: "local", + connectionMethod: credential.connectionMethod, + verifiedAt: credential.verifiedAt, + provider, + providerLabel: llmProviderLabel(provider), + baseUrl: + credential.llmBaseUrl ?? defaultBaseUrlForLlmProvider(provider), + defaultModel: + credential.llmDefaultModel || defaultModelForLlmProvider(provider), + } + : { + configured: false, + source: null, + connectionMethod: null, + verifiedAt: null, + provider, + providerLabel: llmProviderLabel(provider), + baseUrl: defaultBaseUrlForLlmProvider(provider), + defaultModel: defaultModelForLlmProvider(provider), + }; + } + + const llmProvider = await activeLlmProviderForStatus(); + const llm = providerStatuses[llmProvider]; return { mode: "local", required: true, - complete: tinyfish.configured && openrouter.configured, - services: { tinyfish, openrouter }, + complete: tinyfish.configured && llm.configured, + services: { + tinyfish, + llm, + llmProviders: providerStatuses, + openrouter: providerStatuses.openrouter, + }, }; } @@ -215,7 +420,48 @@ export async function saveLocalCredential( keychainAccount, connectionMethod, verifiedAt: Date.now(), + ...(service === "openrouter" + ? { + llmProvider: "openrouter" as const, + llmBaseUrl: defaultBaseUrlForLlmProvider("openrouter"), + llmDefaultModel: defaultModelForLlmProvider("openrouter"), + } + : {}), + }); + + if (service === "openrouter") { + await setActiveLocalLlmProvider("openrouter"); + } +} + +export async function saveLocalLlmProviderConfig( + input: LlmProviderInput, + connectionMethod: ConnectionMethod = "api_key", +): Promise { + if (!env.IS_LOCAL_MODE) { + throw new Error("Local credential storage is disabled when PROD=1."); + } + + const config = normalizeLlmProviderInput(input, "local"); + const keychainAccount = config.apiKey + ? ( + await setKeychainCredential( + llmProviderService(config.provider), + config.apiKey, + ) + ).keychainAccount + : undefined; + await convex.mutation(internal.localCredentials.upsertInternal, { + service: llmProviderService(config.provider), + ...(keychainAccount ? { keychainAccount } : {}), + connectionMethod, + verifiedAt: Date.now(), + llmProvider: config.provider, + llmBaseUrl: config.baseUrl, + llmDefaultModel: config.defaultModel, }); + await setActiveLocalLlmProvider(config.provider); + return config; } export async function clearLegacyPlaintextLocalCredentials(): Promise { diff --git a/backend/src/mastra/agents/investigate.ts b/backend/src/mastra/agents/investigate.ts index 63a7b8c..c930f5e 100644 --- a/backend/src/mastra/agents/investigate.ts +++ b/backend/src/mastra/agents/investigate.ts @@ -1,5 +1,5 @@ import { Agent } from "@mastra/core/agent"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { createLanguageModel, type LlmProviderConfig } from "../../config/llm.js"; import { buildPopulateTools } from "../tools/dataset-tools.js"; import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; @@ -55,13 +55,9 @@ export function buildInvestigateAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], - openRouterApiKey: string, + llmConfig: LlmProviderConfig, ): Agent { const modelSlug = authContext.modelConfig!.investigateSubagent; - const openrouter = createOpenRouter({ - apiKey: openRouterApiKey, - baseURL: process.env.OPENROUTER_BASE_URL, - }); const { insert_row } = buildPopulateTools( authorizedDatasetId, @@ -71,7 +67,7 @@ export function buildInvestigateAgent( id: "investigate-agent", name: "Dataset Investigate Agent", instructions: buildInvestigateInstructions(columns), - model: openrouter(modelSlug), + model: createLanguageModel(llmConfig, modelSlug), tools: { insert_row, diff --git a/backend/src/mastra/agents/populate.ts b/backend/src/mastra/agents/populate.ts index eb9b26f..f5d9869 100644 --- a/backend/src/mastra/agents/populate.ts +++ b/backend/src/mastra/agents/populate.ts @@ -1,5 +1,5 @@ import { Agent } from "@mastra/core/agent"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { createLanguageModel, type LlmProviderConfig } from "../../config/llm.js"; import { buildSubagentTool } from "../tools/investigate-tool.js"; import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; @@ -40,21 +40,17 @@ export function buildPopulateAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], - openRouterApiKey: string, + llmConfig: LlmProviderConfig, maxRowCount: number, metrics?: RunMetrics, ): Agent { const modelSlug = authContext.modelConfig!.populateOrchestrator; - const openrouter = createOpenRouter({ - apiKey: openRouterApiKey, - baseURL: process.env.OPENROUTER_BASE_URL, - }); return new Agent({ id: "populate-agent", name: "Dataset Populate Orchestrator", instructions: buildInstructions(maxRowCount), - model: openrouter(modelSlug), + model: createLanguageModel(llmConfig, modelSlug), tools: { search_web: searchWebTool, fetch_page: fetchPageTool, @@ -62,7 +58,7 @@ export function buildPopulateAgent( authorizedDatasetId, authContext, columns, - openRouterApiKey, + llmConfig, maxRowCount, metrics, ), diff --git a/backend/src/mastra/agents/refresh.ts b/backend/src/mastra/agents/refresh.ts index 144065a..6593f96 100644 --- a/backend/src/mastra/agents/refresh.ts +++ b/backend/src/mastra/agents/refresh.ts @@ -1,5 +1,5 @@ import { Agent } from "@mastra/core/agent"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { createLanguageModel, type LlmProviderConfig } from "../../config/llm.js"; import { buildPopulateTools } from "../tools/dataset-tools.js"; import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; @@ -51,13 +51,9 @@ export function buildRefreshAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], - openRouterApiKey: string, + llmConfig: LlmProviderConfig, ): Agent { const modelSlug = authContext.modelConfig!.investigateSubagent; - const openrouter = createOpenRouter({ - apiKey: openRouterApiKey, - baseURL: process.env.OPENROUTER_BASE_URL, - }); const { update_row } = buildPopulateTools( authorizedDatasetId, authContext, @@ -66,7 +62,7 @@ export function buildRefreshAgent( id: "refresh-agent", name: "Dataset Refresh Agent", instructions: buildRefreshInstructions(columns), - model: openrouter(modelSlug), + model: createLanguageModel(llmConfig, modelSlug), tools: { update_row, search_web: searchWebTool, diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 0139aa4..1b23764 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -6,6 +6,7 @@ import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; import { getSignal } from "../../abort-registry.js"; +import type { LlmProviderConfig } from "../../config/llm.js"; const investigateInputSchema = z.object({ entity_hint: z @@ -75,7 +76,7 @@ export function buildSubagentTool( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], - openRouterApiKey: string, + llmConfig: LlmProviderConfig, maxRowCount: number, metrics?: RunMetrics, ) { @@ -108,7 +109,7 @@ export function buildSubagentTool( authorizedDatasetId, authContext, columns, - openRouterApiKey, + llmConfig, ); const pkBlock = Object.entries(primary_keys) diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index e07ed8e..3673047 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -1,11 +1,11 @@ import { createStep, createWorkflow } from "@mastra/core/workflows"; import { z } from "zod"; import { generateText } from "ai"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { datasetContextSchema, populateColumnSchema } from "../../pipeline/populate.js"; import { convex, internal } from "../../convex.js"; import { DEFAULT_MODEL_IDS } from "../../config/models.js"; -import { requireOpenRouterApiKey } from "../../local-credentials.js"; +import { createLanguageModel } from "../../config/llm.js"; +import { requireLlmProviderConfig } from "../../local-credentials.js"; import { buildPopulateAgent } from "../agents/populate.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; @@ -109,15 +109,11 @@ Respond with EXACTLY one word: scraper or search`; let classification: "scraper" | "search" = "search"; try { - const apiKey = await requireOpenRouterApiKey(); - const openrouter = createOpenRouter({ - apiKey, - baseURL: process.env.OPENROUTER_BASE_URL, - }); + const llmConfig = await requireLlmProviderConfig(); const modelSlug = - inputData.authContext?.modelConfig?.schemaInference ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE; + inputData.authContext?.modelConfig?.schemaInference ?? llmConfig.defaultModel ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE; const result = await generateText({ - model: openrouter(modelSlug), + model: createLanguageModel(llmConfig, modelSlug), prompt: classificationPrompt, maxOutputTokens: 10, abortSignal: getSignal(inputData.datasetId), @@ -251,7 +247,7 @@ const agentStep = createStep({ inputData.authorizedDatasetId, inputData.authContext, inputData.columns, - await requireOpenRouterApiKey(), + await requireLlmProviderConfig(), inputData.maxRowCount, metrics, ); diff --git a/backend/src/mastra/workflows/update.ts b/backend/src/mastra/workflows/update.ts index 28aadee..4bb8ada 100644 --- a/backend/src/mastra/workflows/update.ts +++ b/backend/src/mastra/workflows/update.ts @@ -4,7 +4,7 @@ import { datasetContextSchema, populateColumnSchema } from "../../pipeline/popul import { convex, internal } from "../../convex.js"; import { buildRefreshAgent } from "../agents/refresh.js"; import { authContextSchema } from "./populate.js"; -import { requireOpenRouterApiKey } from "../../local-credentials.js"; +import { requireLlmProviderConfig } from "../../local-credentials.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; import { getSignal } from "../../abort-registry.js"; @@ -100,7 +100,7 @@ const refreshRowsStep = createStep({ const metrics = new RunMetrics(); const startedAt = Date.now(); - const openRouterApiKey = await requireOpenRouterApiKey(); + const llmConfig = await requireLlmProviderConfig(); const pkColumns = columns.filter((c) => c.isPrimaryKey); @@ -110,7 +110,7 @@ const refreshRowsStep = createStep({ datasetId, authContext, columns, - openRouterApiKey, + llmConfig, ); const pkBlock = diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 467a393..d1ab510 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -1,8 +1,8 @@ import { generateText, Output, NoObjectGeneratedError } from "ai"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { DEFAULT_MODEL_IDS } from "../config/models.js"; -import { requireOpenRouterApiKey } from "../local-credentials.js"; +import { createLanguageModel } from "../config/llm.js"; +import { requireLlmProviderConfig } from "../local-credentials.js"; import { datasetSchemaSchema, type DatasetSchema } from "./types.js"; const SYSTEM_PROMPT = `You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you produce a precise schema definition. @@ -28,13 +28,9 @@ Rules: - When a column is a scalar numeric rating (e.g. average score like 4.3/5 for restaurants, cafes, hotels, products, apps): name it generically (e.g. "rating" not "yelp_rating") and write a retrieval_hint explaining that review sites (Yelp, TripAdvisor, Google Maps) block direct page fetches, so the agent must extract ratings from **search result snippets**. The hint should say: "Search for \\" rating reviews\\" and include location terms only when location is part of the entity identity. Look for ratings in snippets from TripAdvisor (\\"rated X.X of 5\\"), Yelp search listings (\\"X.X (N reviews)\\"), or aggregator sites (Birdeye, joe.coffee, giftly, Uber Eats, menufyy). Do NOT try to fetch yelp.com or tripadvisor.com directly — they block automated access. Accept ratings from any reputable source." If including a rating column, also add a "rating_source" text column so the agent records where the rating came from. Do not rename review-count or review-text fields to "rating" — keep those as distinct columns (e.g. "review_count") when the user explicitly asks for them.`; async function getModel(modelSlug?: string) { - const apiKey = await requireOpenRouterApiKey(); - const openrouter = createOpenRouter({ - apiKey, - baseURL: process.env.OPENROUTER_BASE_URL, - }); - const resolvedSlug = modelSlug ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE; - return openrouter(resolvedSlug); + const config = await requireLlmProviderConfig(); + const resolvedSlug = modelSlug ?? config.defaultModel ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE; + return createLanguageModel(config, resolvedSlug); } export async function inferSchema(prompt: string, modelSlug?: string): Promise { diff --git a/frontend/Dockerfile.dev b/frontend/Dockerfile.dev index daa1db8..3a07d32 100644 --- a/frontend/Dockerfile.dev +++ b/frontend/Dockerfile.dev @@ -10,4 +10,7 @@ RUN bun install COPY --chown=node:node . . -CMD ["bun", "dev", "--hostname", "0.0.0.0", "--port", "3500"] +# Run Next directly instead of through Bun's script runner, and force webpack +# for local Docker dev. Next 16 defaults to Turbopack, which can get OOM-killed +# in constrained Docker Desktop setups during the first page compile. +CMD ["node", "../scripts/with-root-env.mjs", "./node_modules/.bin/next", "dev", "--webpack", "--hostname", "0.0.0.0", "--port", "3500"] diff --git a/frontend/app/dashboard/settings/models/page.tsx b/frontend/app/dashboard/settings/models/page.tsx index 6a3acb4..f376c89 100644 --- a/frontend/app/dashboard/settings/models/page.tsx +++ b/frontend/app/dashboard/settings/models/page.tsx @@ -1,9 +1,9 @@ "use client"; -import { useState, useEffect } from "react"; +import { useState, useEffect, useCallback, useRef } from "react"; import { useQuery } from "convex/react"; import { api } from "@/convex/_generated/api"; -import { getModelConfig, saveModelConfig, getOpenRouterModels, refreshOpenRouterModels, type EffectiveModelConfig, type OpenRouterModel } from "@/lib/backend"; +import { getLocalSetupStatus, getLlmProviderModels, getModelConfig, saveModelConfig, refreshOpenRouterModels, type EffectiveModelConfig, type LlmProviderType, type LocalSetupStatus, type OpenRouterModel } from "@/lib/backend"; import { SettingsPageLayout } from "@/components/settings/SettingsPageLayout"; import { SettingsHeader } from "@/components/settings/SettingsHeader"; import { SettingsTile } from "@/components/settings/SettingsTile"; @@ -12,6 +12,17 @@ import { ModelSideSheet } from "@/components/settings/ModelSideSheet"; import { MODEL_ROLES, type ModelRole } from "@/components/settings/types"; import { SkeletonList } from "@/components/settings/Skeleton"; import { useAppAuth } from "@/lib/app-auth"; +import { isLocalMode } from "@/lib/app-mode"; + +function modelListCacheKey(status: LocalSetupStatus): string { + const llm = status.services.llm; + return [ + llm.provider ?? "openrouter", + llm.baseUrl ?? "", + llm.defaultModel ?? "", + llm.verifiedAt ?? "", + ].join("|"); +} export default function ModelSettingsPage() { const { getToken } = useAppAuth(); @@ -21,21 +32,70 @@ export default function ModelSettingsPage() { const [isLoadingConfig, setIsLoadingConfig] = useState(true); const [refreshing, setRefreshing] = useState(false); const [sheetModels, setSheetModels] = useState([]); + const [sheetModelsCacheKey, setSheetModelsCacheKey] = useState(null); const [activeSheet, setActiveSheet] = useState<{ role: ModelRole } | null>(null); + const [llmProvider, setLlmProvider] = useState( + isLocalMode ? null : "openrouter", + ); + const [activeModelListCacheKey, setActiveModelListCacheKey] = useState( + isLocalMode ? "" : "openrouter|||", + ); + const activeModelListCacheKeyRef = useRef(activeModelListCacheKey); const [isSavingModel, setIsSavingModel] = useState(false); + const [modelConfigReloadKey, setModelConfigReloadKey] = useState(0); + + const needsOpenRouterCache = !isLocalMode || llmProvider === "openrouter"; + const isLoading = + (needsOpenRouterCache && convexModels === undefined) || + isLoadingConfig || + (isLocalMode && llmProvider === null); - const isLoading = convexModels === undefined || isLoadingConfig; + const syncLlmProvider = useCallback((status: LocalSetupStatus) => { + const nextCacheKey = modelListCacheKey(status); + activeModelListCacheKeyRef.current = nextCacheKey; + setLlmProvider(status.services.llm.provider ?? "openrouter"); + setActiveModelListCacheKey(nextCacheKey); + }, []); + + const handleLocalCredentialsStatus = useCallback( + (status: LocalSetupStatus) => { + syncLlmProvider(status); + setSheetModels([]); + setSheetModelsCacheKey(null); + setModelConfigReloadKey((key) => key + 1); + }, + [syncLlmProvider], + ); useEffect(() => { + let active = true; + getToken() .then((token) => { if (!token) throw new Error("Not authenticated"); return getModelConfig(token); }) - .then((config) => setEffectiveConfig(config)) - .catch(() => setEffectiveConfig(null)) - .finally(() => setIsLoadingConfig(false)); - }, [getToken]); + .then((config) => { + if (active) setEffectiveConfig(config); + }) + .catch(() => { + if (active) setEffectiveConfig(null); + }) + .finally(() => { + if (active) setIsLoadingConfig(false); + }); + + return () => { + active = false; + }; + }, [getToken, modelConfigReloadKey]); + + useEffect(() => { + if (!isLocalMode) return; + getLocalSetupStatus() + .then(syncLlmProvider) + .catch(() => setLlmProvider("openrouter")); + }, [syncLlmProvider]); const models: OpenRouterModel[] = convexModels ? convexModels.map((m) => ({ @@ -46,19 +106,28 @@ export default function ModelSettingsPage() { promptCost: m.promptCost, })) : []; + const sideSheetModels = + sheetModels.length > 0 + ? sheetModels + : llmProvider === "openrouter" + ? models + : []; function getSelectedModel(role: ModelRole): string { return effectiveConfig?.[role.key as keyof typeof effectiveConfig] ?? ""; } - async function handleModelSelect(role: ModelRole, model: OpenRouterModel) { + async function saveModelForRole(role: ModelRole, modelId: string) { + const nextModelId = modelId.trim(); + if (!nextModelId) return; + setIsSavingModel(true); try { const token = await getToken(); if (!token) throw new Error("Not authenticated"); - await saveModelConfig({ [role.key]: model.canonicalSlug }, token); + await saveModelConfig({ [role.key]: nextModelId }, token); setEffectiveConfig((prev: EffectiveModelConfig | null) => - prev ? { ...prev, [role.key]: model.canonicalSlug } : null + prev ? { ...prev, [role.key]: nextModelId } : null ); setActiveSheet(null); } catch { @@ -69,9 +138,16 @@ export default function ModelSettingsPage() { } function openSideSheet(role: ModelRole) { - if (sheetModels.length === 0) { - getOpenRouterModels() - .then((models) => setSheetModels(models)) + const cacheKey = activeModelListCacheKeyRef.current || activeModelListCacheKey; + if (sheetModels.length === 0 || sheetModelsCacheKey !== cacheKey) { + setSheetModels([]); + setSheetModelsCacheKey(cacheKey); + getLlmProviderModels() + .then((models) => { + if (activeModelListCacheKeyRef.current !== cacheKey) return; + setSheetModels(models); + setSheetModelsCacheKey(cacheKey); + }) .catch(() => { // we will add toast later }); @@ -117,11 +193,15 @@ export default function ModelSettingsPage() { return (
- +
@@ -147,18 +227,23 @@ export default function ModelSettingsPage() { onClose={() => !isSavingModel && setActiveSheet(null)} title={`Select ${activeSheet.role.label} Model`} selectedModel={getSelectedModel(activeSheet.role)} - models={sheetModels.length > 0 ? sheetModels : models} - onSelect={(slug) => { - const sourceModels = sheetModels.length > 0 ? sheetModels : models; - const model = sourceModels.find((m) => m.canonicalSlug === slug); - if (model) handleModelSelect(activeSheet.role, model); - }} + models={sideSheetModels} + onSelect={(slug) => saveModelForRole(activeSheet.role, slug)} onRefresh={async () => { setRefreshing(true); try { - const token = await getToken(); - if (!token) throw new Error("Not authenticated"); - const models = await refreshOpenRouterModels(token); + const provider = llmProvider ?? "openrouter"; + const cacheKey = activeModelListCacheKeyRef.current || activeModelListCacheKey; + let models: OpenRouterModel[]; + if (provider === "openrouter") { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + models = await refreshOpenRouterModels(token); + } else { + models = await getLlmProviderModels(); + } + if (activeModelListCacheKeyRef.current !== cacheKey) return; + setSheetModelsCacheKey(cacheKey); setSheetModels(models); } catch { // we will add toast later @@ -170,6 +255,8 @@ export default function ModelSettingsPage() { isSaving={isSavingModel} /> )} + + ); } diff --git a/frontend/app/setup/page.tsx b/frontend/app/setup/page.tsx index 9e6bd71..2814ad3 100644 --- a/frontend/app/setup/page.tsx +++ b/frontend/app/setup/page.tsx @@ -11,18 +11,24 @@ import { } from "lucide-react"; import { getLocalSetupStatus, - saveOpenRouterApiKey, + saveLlmProviderConfig, saveTinyFishApiKey, + type LlmProviderType, type LocalSetupStatus, type ServiceSetupStatus, } from "@/lib/backend"; import { isLocalMode } from "@/lib/app-mode"; +import { + LlmProviderBrand, + LlmProviderSelector, + llmProviderOption, +} from "@/components/settings/llm-providers"; export default function SetupPage() { const router = useRouter(); const [status, setStatus] = useState(null); const [loading, setLoading] = useState(true); - const [modal, setModal] = useState<"tinyfish" | "openrouter" | null>(null); + const [modal, setModal] = useState<"tinyfish" | "llm" | null>(null); useEffect(() => { if (!isLocalMode) { @@ -59,8 +65,8 @@ export default function SetupPage() { Connect your services

- Add TinyFish and OpenRouter access to start building live - datasets. + Add TinyFish and your preferred LLM provider to start building + live datasets.

@@ -92,18 +98,18 @@ export default function SetupPage() { /> } - description="BigSet uses OpenRouter's API to power BigSet with AI model access." - status={status?.services.openrouter} + brand={} + description="BigSet uses your LLM provider for schema generation and dataset-building agents." + status={status?.services.llm} primaryLabel={ - status?.services.openrouter.configured - ? "Update key" - : "Add API key" + status?.services.llm.configured + ? "Update provider" + : "Choose provider" } - onPrimary={() => setModal("openrouter")} - helperHref="https://openrouter.ai/settings/keys" - helperLabel="Need an OpenRouter key?" - helperDescription="Open the OpenRouter keys page" + onPrimary={() => setModal("llm")} + helperHref="https://platform.openai.com/api-keys" + helperLabel="Bring your own model" + helperDescription="OpenAI, Anthropic, OpenRouter, or custom" />
@@ -166,10 +172,11 @@ function ServiceCard({ const connected = status?.configured ?? false; const detail = useMemo(() => { if (!connected) return "Not connected"; + if (status?.providerLabel) return status.providerLabel; if (status?.connectionMethod === "oauth") return "Connected through OAuth"; if (status?.source === "env") return "Connected through .env"; return "Connected through API key"; - }, [connected, status?.connectionMethod, status?.source]); + }, [connected, status?.connectionMethod, status?.providerLabel, status?.source]); return (
@@ -224,56 +231,48 @@ function ServiceCard({ ); } -function OpenRouterBrand() { - return ( -
- - OpenRouter -
- ); -} - function ApiKeyModal({ service, onClose, onSaved, }: { - service: "tinyfish" | "openrouter"; + service: "tinyfish" | "llm"; onClose: () => void; onSaved: (status: LocalSetupStatus) => void; }) { const [apiKey, setApiKey] = useState(""); + const [provider, setProvider] = useState("openrouter"); + const [baseUrl, setBaseUrl] = useState(""); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); const isTinyFish = service === "tinyfish"; + const providerCopy = llmProviderOption(provider); + + function handleProviderChange(next: LlmProviderType) { + setProvider(next); + setBaseUrl(""); + } async function handleSubmit() { - if (!apiKey.trim() || saving) return; + if (saving) return; + if (isTinyFish && !apiKey.trim()) return; + if (!isTinyFish && provider !== "custom" && !apiKey.trim()) return; + if (!isTinyFish && provider === "custom" && !baseUrl.trim()) { + setError("Custom providers require a base URL"); + return; + } + setSaving(true); setError(null); try { const next = isTinyFish ? await saveTinyFishApiKey(apiKey.trim()) - : await saveOpenRouterApiKey(apiKey.trim()); + : await saveLlmProviderConfig({ + provider, + apiKey: apiKey.trim(), + defaultModel: llmProviderOption(provider).defaultModel, + baseUrl: provider === "custom" ? baseUrl.trim() : undefined, + }); onSaved(next); } catch (err) { setError(err instanceof Error ? err.message : "Verification failed"); @@ -282,6 +281,18 @@ function ApiKeyModal({ } } + const helperHref = isTinyFish + ? "https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2" + : providerCopy.helperHref; + const helperLabel = !isTinyFish && provider === "custom" ? "Provider docs" : "Get a key"; + const canSubmit = + !saving && + (isTinyFish + ? !!apiKey.trim() + : provider === "custom" + ? !!baseUrl.trim() + : !!apiKey.trim()); + return (
@@ -114,10 +127,10 @@ export function LocalCredentialsPanel() { onApiKey={() => setModal("tinyfish")} /> setModal("openrouter")} + onApiKey={() => setModal("llm")} /> )} @@ -128,6 +141,7 @@ export function LocalCredentialsPanel() { onClose={() => setModal(null)} onSaved={(next) => { setStatus(next); + onStatusChange?.(next); setModal(null); }} /> @@ -156,7 +170,7 @@ function CredentialCard({
- +

{detail}

@@ -174,7 +188,7 @@ function CredentialCard({ className="inline-flex w-fit items-center gap-2 rounded-lg border border-accent bg-accent px-4 py-2.5 text-sm font-semibold text-accent-text transition-opacity hover:opacity-90" > - {connected ? "Update key" : "Add API key"} + {service === "llm" ? (connected ? "Update provider" : "Choose provider") : connected ? "Update key" : "Add API key"} @@ -208,35 +228,7 @@ function ServiceBrand({ service }: { service: ServiceName }) { ); } - return ; -} - -function OpenRouterBrand() { - return ( -
- - OpenRouter -
- ); + return ; } function StatusLabel({ @@ -272,10 +264,11 @@ function useCredentialDetail( return useMemo(() => { if (loading) return "Checking connection..."; if (!status?.configured) return "Not connected"; + if (status.providerLabel) return status.providerLabel; if (status.connectionMethod === "oauth") return "Connected through OAuth"; if (status.source === "env") return "Connected through .env"; return "Connected through API key"; - }, [loading, status?.configured, status?.connectionMethod, status?.source]); + }, [loading, status?.configured, status?.connectionMethod, status?.providerLabel, status?.source]); } function ApiKeyModal({ @@ -288,19 +281,39 @@ function ApiKeyModal({ onSaved: (status: LocalSetupStatus) => void; }) { const [apiKey, setApiKey] = useState(""); + const [provider, setProvider] = useState("openrouter"); + const [baseUrl, setBaseUrl] = useState(""); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); const copy = SERVICE_COPY[service]; const isTinyFish = service === "tinyfish"; + const providerCopy = llmProviderOption(provider); + + function handleProviderChange(next: LlmProviderType) { + setProvider(next); + setBaseUrl(""); + } async function handleSubmit() { - if (!apiKey.trim() || saving) return; + if (saving) return; + if (isTinyFish && !apiKey.trim()) return; + if (!isTinyFish && provider !== "custom" && !apiKey.trim()) return; + if (!isTinyFish && provider === "custom" && !baseUrl.trim()) { + setError("Custom providers require a base URL"); + return; + } + setSaving(true); setError(null); try { const next = isTinyFish ? await saveTinyFishApiKey(apiKey.trim()) - : await saveOpenRouterApiKey(apiKey.trim()); + : await saveLlmProviderConfig({ + provider, + apiKey: apiKey.trim(), + defaultModel: llmProviderOption(provider).defaultModel, + baseUrl: provider === "custom" ? baseUrl.trim() : undefined, + }); onSaved(next); } catch (err) { setError(err instanceof Error ? err.message : "Verification failed"); @@ -309,6 +322,16 @@ function ApiKeyModal({ } } + const helperHref = isTinyFish ? copy.helperHref : providerCopy.helperHref; + const helperLabel = !isTinyFish && provider === "custom" ? "Provider docs" : "Get a key"; + const canSubmit = + !saving && + (isTinyFish + ? !!apiKey.trim() + : provider === "custom" + ? !!baseUrl.trim() + : !!apiKey.trim()); + return (
+
@@ -158,7 +195,6 @@ export function ModelSideSheet({ ) : providers.length === 0 ? (

No models found

-

Try a different search term

) : (
@@ -218,19 +254,21 @@ export function ModelSideSheet({

- {model.contextLength >= 1000 - ? `${(model.contextLength / 1000).toLocaleString()}K` - : model.contextLength.toLocaleString()} + {model.contextLength > 0 + ? model.contextLength >= 1000 + ? `${(model.contextLength / 1000).toLocaleString()}K` + : model.contextLength.toLocaleString() + : "—"}

- ${model.promptCost.toFixed(2)}/1M + {model.promptCost > 0 ? `$${model.promptCost.toFixed(2)}/1M` : "—"}

- ${model.completionCost.toFixed(2)}/1M + {model.completionCost > 0 ? `$${model.completionCost.toFixed(2)}/1M` : "—"}

diff --git a/frontend/components/settings/llm-providers.tsx b/frontend/components/settings/llm-providers.tsx new file mode 100644 index 0000000..64e87b6 --- /dev/null +++ b/frontend/components/settings/llm-providers.tsx @@ -0,0 +1,156 @@ +"use client"; + +import type { LlmProviderType } from "@/lib/backend"; + +export type LlmProviderOption = { + value: LlmProviderType; + label: string; + description: string; + defaultModel: string; + apiKeyPlaceholder: string; + helperHref: string; + iconSrc?: string; + wordmarkSrc?: string; +}; + +export const LLM_PROVIDER_OPTIONS: LlmProviderOption[] = [ + { + value: "openrouter", + label: "OpenRouter", + description: "Use OpenRouter model slugs.", + defaultModel: "anthropic/claude-sonnet-4.6", + apiKeyPlaceholder: "sk-or-...", + helperHref: "https://openrouter.ai/settings/keys", + iconSrc: "/logos/providers/openrouter.svg", + wordmarkSrc: "/logos/providers/openrouter-wordmark.svg", + }, + { + value: "openai", + label: "OpenAI", + description: "Use an OpenAI API key directly.", + defaultModel: "gpt-5.4-mini", + apiKeyPlaceholder: "sk-...", + helperHref: "https://platform.openai.com/api-keys", + iconSrc: "/logos/providers/openai.svg", + wordmarkSrc: "/logos/providers/openai.svg", + }, + { + value: "anthropic", + label: "Anthropic", + description: "Use a Claude API key directly.", + defaultModel: "claude-sonnet-4-6", + apiKeyPlaceholder: "sk-ant-...", + helperHref: "https://console.anthropic.com/settings/keys", + iconSrc: "/logos/providers/anthropic.svg", + wordmarkSrc: "/logos/providers/anthropic.svg", + }, + { + value: "custom", + label: "Custom", + description: "Use LM Studio or any OpenAI-compatible base URL.", + defaultModel: "", + apiKeyPlaceholder: "Optional — leave blank for LM Studio", + helperHref: "https://lmstudio.ai/docs/app/api/endpoints/openai", + }, +]; + +export function llmProviderOption(value: LlmProviderType) { + return ( + LLM_PROVIDER_OPTIONS.find((option) => option.value === value) ?? + LLM_PROVIDER_OPTIONS[0] + ); +} + +export function LlmProviderLogo({ + provider, + variant = "wordmark", + className = "", +}: { + provider: LlmProviderType; + variant?: "icon" | "wordmark"; + className?: string; +}) { + const option = llmProviderOption(provider); + + if (provider === "custom") { + return ( + + Custom + + ); + } + + const src = variant === "icon" ? option.iconSrc : option.wordmarkSrc; + + return ( + {option.label} + ); +} + +export function LlmProviderBrand({ provider }: { provider?: LlmProviderType }) { + if (provider) { + return ( +
+ +
+ ); + } + + return ( +
+ + AI + + LLM Provider +
+ ); +} + +export function LlmProviderSelector({ + value, + onChange, +}: { + value: LlmProviderType; + onChange: (provider: LlmProviderType) => void; +}) { + return ( +
+ {LLM_PROVIDER_OPTIONS.map((option) => { + const selected = option.value === value; + + return ( + + ); + })} +
+ ); +} diff --git a/frontend/convex/localCredentials.ts b/frontend/convex/localCredentials.ts index d8e8c97..eeafd3f 100644 --- a/frontend/convex/localCredentials.ts +++ b/frontend/convex/localCredentials.ts @@ -3,7 +3,11 @@ import { v } from "convex/values"; const serviceValidator = v.union( v.literal("tinyfish"), + v.literal("llm"), v.literal("openrouter"), + v.literal("openai"), + v.literal("anthropic"), + v.literal("custom"), ); const connectionMethodValidator = v.union( @@ -11,6 +15,13 @@ const connectionMethodValidator = v.union( v.literal("oauth"), ); +const llmProviderValidator = v.union( + v.literal("openrouter"), + v.literal("openai"), + v.literal("anthropic"), + v.literal("custom"), +); + export const getInternal = internalQuery({ args: { service: serviceValidator }, handler: async (ctx, args) => { @@ -24,9 +35,12 @@ export const getInternal = internalQuery({ export const upsertInternal = internalMutation({ args: { service: serviceValidator, - keychainAccount: v.string(), + keychainAccount: v.optional(v.string()), connectionMethod: connectionMethodValidator, verifiedAt: v.number(), + llmProvider: v.optional(llmProviderValidator), + llmBaseUrl: v.optional(v.string()), + llmDefaultModel: v.optional(v.string()), }, handler: async (ctx, args) => { const existing = await ctx.db @@ -35,20 +49,39 @@ export const upsertInternal = internalMutation({ .unique(); const update = { - keychainAccount: args.keychainAccount, + ...(args.keychainAccount !== undefined + ? { keychainAccount: args.keychainAccount } + : {}), connectionMethod: args.connectionMethod, verifiedAt: args.verifiedAt, updatedAt: Date.now(), }; + const llmPatch = args.llmProvider !== undefined + ? { + llmProvider: args.llmProvider, + // Explicit undefined clears stale custom-provider values when the + // user switches back to OpenAI/Anthropic/OpenRouter. + llmBaseUrl: args.llmBaseUrl, + llmDefaultModel: args.llmDefaultModel, + } + : {}; + const llmInsert = args.llmProvider !== undefined + ? { + llmProvider: args.llmProvider, + ...(args.llmBaseUrl !== undefined ? { llmBaseUrl: args.llmBaseUrl } : {}), + ...(args.llmDefaultModel !== undefined ? { llmDefaultModel: args.llmDefaultModel } : {}), + } + : {}; if (existing) { - await ctx.db.patch(existing._id, { ...update, apiKey: undefined }); + await ctx.db.patch(existing._id, { ...update, ...llmPatch, apiKey: undefined }); return existing._id; } return await ctx.db.insert("localCredentials", { service: args.service, ...update, + ...llmInsert, }); }, }); diff --git a/frontend/convex/modelConfig.ts b/frontend/convex/modelConfig.ts index e10fd3d..dc47991 100644 --- a/frontend/convex/modelConfig.ts +++ b/frontend/convex/modelConfig.ts @@ -1,32 +1,65 @@ import { query, mutation, internalQuery, internalMutation } from "./_generated/server.js"; +import type { MutationCtx, QueryCtx } from "./_generated/server.js"; import { v } from "convex/values"; import { getIdentity } from "./lib/authz.js"; +type LlmProvider = "openrouter" | "openai" | "anthropic" | "custom"; + +const providerValidator = v.union( + v.literal("openrouter"), + v.literal("openai"), + v.literal("anthropic"), + v.literal("custom"), +); + +async function findProviderConfig( + ctx: QueryCtx | MutationCtx, + userId: string, + provider: LlmProvider, +) { + const providerRow = await ctx.db + .query("modelConfig") + .withIndex("by_user_provider", (q) => + q.eq("userId", userId).eq("provider", provider), + ) + .first(); + + if (providerRow) return providerRow; + + if (provider === "openrouter") { + return await ctx.db + .query("modelConfig") + .withIndex("by_user", (q) => q.eq("userId", userId)) + .filter((q) => q.eq(q.field("provider"), undefined)) + .first(); + } + + return null; +} + export const get = query({ - args: {}, - handler: async (ctx) => { + args: { provider: v.optional(providerValidator) }, + handler: async (ctx, args) => { const identity = await getIdentity(ctx); if (!identity) return null; - const existing = await ctx.db - .query("modelConfig") - .withIndex("by_user", (q) => q.eq("userId", identity.subject)) - .first(); - return existing ?? null; + return await findProviderConfig( + ctx, + identity.subject, + args.provider ?? "openrouter", + ); }, }); /** - * Upsert one or more model preferences for the authenticated user. + * Upsert one or more model preferences for the authenticated user and provider. * * Only fields that are explicitly provided (not undefined) are updated. * Unset fields retain their existing database values. - * - * Example: sending only { schemaInference: "x" } will update schemaInference - * while leaving populateOrchestrator and investigateSubagent untouched. */ export const upsert = mutation({ args: { + provider: v.optional(providerValidator), schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), @@ -35,67 +68,64 @@ export const upsert = mutation({ const identity = await getIdentity(ctx); if (!identity) throw new Error("Not authenticated"); - const existing = await ctx.db - .query("modelConfig") - .withIndex("by_user", (q) => q.eq("userId", identity.subject)) - .first(); + const provider = args.provider ?? "openrouter"; + const existing = await findProviderConfig(ctx, identity.subject, provider); + + const patch: { + provider?: LlmProvider; + schemaInference?: string; + populateOrchestrator?: string; + investigateSubagent?: string; + } = { provider }; + if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; + if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; + if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; if (existing) { - // Partial update — only touch fields that were explicitly provided. - // Omitting a field preserves its current database value. - const patch: Record = {}; - if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; - if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; - if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; await ctx.db.patch(existing._id, patch); } else { - // First-time save — build insert object from provided fields only. - // userId is always required and comes from the authenticated identity. - const insert: { - userId: string; - schemaInference?: string; - populateOrchestrator?: string; - investigateSubagent?: string; - } = { userId: identity.subject }; - if (args.schemaInference !== undefined) insert.schemaInference = args.schemaInference; - if (args.populateOrchestrator !== undefined) insert.populateOrchestrator = args.populateOrchestrator; - if (args.investigateSubagent !== undefined) insert.investigateSubagent = args.investigateSubagent; - await ctx.db.insert("modelConfig", insert); + await ctx.db.insert("modelConfig", { + userId: identity.subject, + ...patch, + }); } }, }); export const getInternal = internalQuery({ - args: { userId: v.string() }, + args: { userId: v.string(), provider: v.optional(providerValidator) }, handler: async (ctx, args) => { - const existing = await ctx.db - .query("modelConfig") - .withIndex("by_user", (q) => q.eq("userId", args.userId)) - .first(); - return existing ?? null; + return await findProviderConfig( + ctx, + args.userId, + args.provider ?? "openrouter", + ); }, }); /** - * Upsert model preferences for a specific user (internal, backend-only). + * Upsert model preferences for a specific user/provider (internal, backend-only). * * Only fields that are explicitly provided (not undefined) are updated. - * Unset fields are omitted from the insert, leaving the database unchanged. */ export const upsertInternal = internalMutation({ args: { userId: v.string(), + provider: v.optional(providerValidator), schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), }, handler: async (ctx, args) => { - const existing = await ctx.db - .query("modelConfig") - .withIndex("by_user", (q) => q.eq("userId", args.userId)) - .first(); + const provider = args.provider ?? "openrouter"; + const existing = await findProviderConfig(ctx, args.userId, provider); - const patch: Record = {}; + const patch: { + provider: LlmProvider; + schemaInference?: string; + populateOrchestrator?: string; + investigateSubagent?: string; + } = { provider }; if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index 83ea007..458dcc5 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -135,17 +135,48 @@ export default defineSchema({ modelConfig: defineTable({ userId: v.string(), + provider: v.optional( + v.union( + v.literal("openrouter"), + v.literal("openai"), + v.literal("anthropic"), + v.literal("custom") + ) + ), schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), - }).index("by_user", ["userId"]), + }) + .index("by_user", ["userId"]) + .index("by_user_provider", ["userId", "provider"]), localCredentials: defineTable({ - service: v.union(v.literal("tinyfish"), v.literal("openrouter")), + service: v.union( + v.literal("tinyfish"), + v.literal("llm"), + v.literal("openrouter"), + v.literal("openai"), + v.literal("anthropic"), + v.literal("custom") + ), keychainAccount: v.optional(v.string()), connectionMethod: v.union(v.literal("api_key"), v.literal("oauth")), verifiedAt: v.number(), updatedAt: v.number(), + // For service:"llm" this stores the active local LLM provider. + // Provider-specific rows store each provider's keychain account and + // optional custom base URL so users can switch providers without + // re-entering keys. + llmProvider: v.optional( + v.union( + v.literal("openrouter"), + v.literal("openai"), + v.literal("anthropic"), + v.literal("custom") + ) + ), + llmBaseUrl: v.optional(v.string()), + llmDefaultModel: v.optional(v.string()), // Legacy only: accepted so the migration can deploy, then cleared by the // backend startup purge. New code never writes this field. apiKey: v.optional(v.string()), diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index f5ea3e5..09f7951 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -46,7 +46,7 @@ export interface EffectiveModelConfig { } /** - * User's saved model preferences — stores the canonical slug (e.g. "anthropic/claude-sonnet-4.6") + * User's saved model preferences — stores the provider model id (e.g. "openai/gpt-5.4-mini" or "gpt-5.4-mini") * for each agent role. Null means no preference saved — backend will use the env default. */ export interface SavedModelConfig { @@ -63,11 +63,17 @@ export interface OpenRouterModel { promptCost: number; } +export type LlmProviderType = "openrouter" | "openai" | "anthropic" | "custom"; + export interface ServiceSetupStatus { configured: boolean; source: "local" | "env" | null; connectionMethod: "api_key" | "oauth" | null; verifiedAt: number | null; + provider?: LlmProviderType; + providerLabel?: string; + baseUrl?: string; + defaultModel?: string; } export interface LocalSetupStatus { @@ -76,7 +82,9 @@ export interface LocalSetupStatus { complete: boolean; services: { tinyfish: ServiceSetupStatus; - openrouter: ServiceSetupStatus; + llm: ServiceSetupStatus; + llmProviders?: Record; + openrouter?: ServiceSetupStatus; }; } @@ -116,13 +124,16 @@ export async function saveTinyFishApiKey( return res.json(); } -export async function saveOpenRouterApiKey( - apiKey: string, -): Promise { - const res = await fetch(`${BACKEND_URL}/local-setup/openrouter-key`, { +export async function saveLlmProviderConfig(config: { + provider: LlmProviderType; + apiKey?: string; + defaultModel: string; + baseUrl?: string; +}): Promise { + const res = await fetch(`${BACKEND_URL}/local-setup/llm-provider`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ apiKey }), + body: JSON.stringify(config), }); if (!res.ok) { @@ -132,6 +143,16 @@ export async function saveOpenRouterApiKey( return res.json(); } +export async function saveOpenRouterApiKey( + apiKey: string, +): Promise { + return saveLlmProviderConfig({ + provider: "openrouter", + apiKey, + defaultModel: "openai/gpt-5.4-mini", + }); +} + export async function exchangeOpenRouterOAuth( code: string, codeVerifier: string, @@ -238,6 +259,21 @@ export async function getOpenRouterModels(): Promise { return data.models ?? []; } +export async function getLlmProviderModels(): Promise { + const res = await fetch(`${BACKEND_URL}/llm-provider/models`, { + method: "GET", + }); + + if (!res.ok) { + const body = await res.json().catch(() => null); + const message = body?.error || `Backend error (${res.status})`; + throw new Error(message); + } + + const data = await res.json(); + return data.models ?? []; +} + /** * Refresh the OpenRouter model cache by fetching the latest list from the * OpenRouter API and storing it in Convex. diff --git a/frontend/public/logos/providers/anthropic.svg b/frontend/public/logos/providers/anthropic.svg new file mode 100644 index 0000000..cbcc39e --- /dev/null +++ b/frontend/public/logos/providers/anthropic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/logos/providers/openai.svg b/frontend/public/logos/providers/openai.svg new file mode 100644 index 0000000..367828b --- /dev/null +++ b/frontend/public/logos/providers/openai.svg @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/frontend/public/logos/providers/openrouter-wordmark.svg b/frontend/public/logos/providers/openrouter-wordmark.svg new file mode 100644 index 0000000..f4bfcbb --- /dev/null +++ b/frontend/public/logos/providers/openrouter-wordmark.svg @@ -0,0 +1,10 @@ + + OpenRouter + + + + + + + OpenRouter + diff --git a/frontend/public/logos/providers/openrouter.svg b/frontend/public/logos/providers/openrouter.svg new file mode 100644 index 0000000..d71ed64 --- /dev/null +++ b/frontend/public/logos/providers/openrouter.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/makefiles/Makefile b/makefiles/Makefile index cd7ef5e..2eccc1a 100644 --- a/makefiles/Makefile +++ b/makefiles/Makefile @@ -62,7 +62,7 @@ validate-dev-env: fi @prod="$$(grep '^PROD=' .env | cut -d= -f2-)"; \ if [[ "$$prod" != "1" ]]; then \ - echo "Local mode: Clerk, TinyFish, and OpenRouter env validation skipped."; \ + echo "Local mode: Clerk, TinyFish, and LLM provider env validation skipped."; \ fi @prod="$$(grep '^PROD=' .env | cut -d= -f2-)"; \ if [[ "$$prod" != "1" ]]; then exit 0; fi; \ @@ -198,7 +198,9 @@ convex-env: fi; \ if [[ "$$prod" != "1" ]]; then \ $(CONVEX_CLI_RUN) -e CONVEX_SELF_HOSTED_ADMIN_KEY="$$admin_key" frontend sh -lc \ - 'npx convex env set BIGSET_LOCAL_MODE 1 --url $(CONVEX_CLI_URL) --admin-key "$$CONVEX_SELF_HOSTED_ADMIN_KEY"'; \ + 'npx convex env set BIGSET_LOCAL_MODE 1 --url $(CONVEX_CLI_URL) --admin-key "$$CONVEX_SELF_HOSTED_ADMIN_KEY"' || exit 1; \ + $(CONVEX_CLI_RUN) -e CONVEX_SELF_HOSTED_ADMIN_KEY="$$admin_key" frontend sh -lc \ + 'npx convex env set CLERK_JWT_ISSUER_DOMAIN "https://bigset.local.invalid" --url $(CONVEX_CLI_URL) --admin-key "$$CONVEX_SELF_HOSTED_ADMIN_KEY"' || exit 1; \ exit 0; \ fi; \ issuer="$$(grep '^CLERK_JWT_ISSUER_DOMAIN=' .env | cut -d= -f2-)"; \ From 8cf6beaafd9b7392b07ff5389d8d0901091bf640 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Tue, 9 Jun 2026 15:57:22 -0700 Subject: [PATCH 02/16] tested a little bit and it should work i think? idk --- backend/package-lock.json | 176 ++++++ backend/package.json | 10 + backend/prompts/schema-inference.txt | 2 +- backend/src/config/llm.ts | 404 +++++++++++- backend/src/config/models.ts | 257 +++++++- backend/src/index.ts | 7 +- backend/src/local-credential-types.ts | 12 + backend/src/local-credentials.ts | 54 +- backend/src/mastra/agents/investigate.ts | 5 +- backend/src/mastra/agents/refresh.ts | 7 +- backend/src/mastra/tools/dataset-tools.ts | 43 +- backend/src/mastra/tools/investigate-tool.ts | 17 +- backend/src/mastra/workflows/populate.ts | 3 +- backend/src/pipeline/schema-inference.ts | 2 +- backend/src/pipeline/types.ts | 6 +- frontend/app/setup/page.tsx | 592 +++++++++++++++--- .../settings/LocalCredentialsPanel.tsx | 387 +++++++++--- .../components/settings/llm-providers.tsx | 560 +++++++++++++++-- frontend/convex/localCredentials.ts | 24 + frontend/convex/modelConfig.ts | 30 +- frontend/convex/schema.ts | 36 ++ frontend/lib/backend.ts | 24 +- frontend/lib/openrouter-oauth.ts | 90 +++ .../public/logos/providers/anthropic-icon.svg | 1 + frontend/public/logos/providers/deepinfra.svg | 29 + frontend/public/logos/providers/deepseek.svg | 1 + .../public/logos/providers/fireworks-ai.svg | 1 + frontend/public/logos/providers/google-g.svg | 1 + frontend/public/logos/providers/groq.svg | 1 + .../public/logos/providers/huggingface.svg | 1 + frontend/public/logos/providers/lmstudio.svg | 1 + .../public/logos/providers/mistral-ai.svg | 1 + frontend/public/logos/providers/ollama.svg | 1 + .../public/logos/providers/openai-icon.svg | 1 + frontend/public/logos/providers/qwen.svg | 1 + .../public/logos/providers/together-ai.svg | 1 + frontend/public/logos/providers/xai.svg | 1 + 37 files changed, 2419 insertions(+), 371 deletions(-) create mode 100644 frontend/public/logos/providers/anthropic-icon.svg create mode 100644 frontend/public/logos/providers/deepinfra.svg create mode 100644 frontend/public/logos/providers/deepseek.svg create mode 100644 frontend/public/logos/providers/fireworks-ai.svg create mode 100644 frontend/public/logos/providers/google-g.svg create mode 100644 frontend/public/logos/providers/groq.svg create mode 100644 frontend/public/logos/providers/huggingface.svg create mode 100644 frontend/public/logos/providers/lmstudio.svg create mode 100644 frontend/public/logos/providers/mistral-ai.svg create mode 100644 frontend/public/logos/providers/ollama.svg create mode 100644 frontend/public/logos/providers/openai-icon.svg create mode 100644 frontend/public/logos/providers/qwen.svg create mode 100644 frontend/public/logos/providers/together-ai.svg create mode 100644 frontend/public/logos/providers/xai.svg diff --git a/backend/package-lock.json b/backend/package-lock.json index c9a41ed..7ceb1f9 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,10 +8,20 @@ "name": "bigset-backend", "version": "0.1.0", "dependencies": { + "@ai-sdk/alibaba": "^1.0.26", "@ai-sdk/anthropic": "^3.0.81", + "@ai-sdk/deepinfra": "^2.0.52", + "@ai-sdk/deepseek": "^2.0.35", + "@ai-sdk/fireworks": "^2.0.53", + "@ai-sdk/google": "^3.0.80", + "@ai-sdk/groq": "^3.0.39", + "@ai-sdk/huggingface": "^1.0.50", + "@ai-sdk/mistral": "^3.0.37", "@ai-sdk/openai": "^3.0.68", "@ai-sdk/openai-compatible": "^2.0.48", "@ai-sdk/provider": "^3.0.10", + "@ai-sdk/togetherai": "^2.0.53", + "@ai-sdk/xai": "^3.0.93", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "@mastra/core": "^1.36.0", @@ -61,6 +71,23 @@ } } }, + "node_modules/@ai-sdk/alibaba": { + "version": "1.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/alibaba/-/alibaba-1.0.26.tgz", + "integrity": "sha512-8j4QPWKDTraUlXh3+6AotaLA4CewOdLMBCtk8SATWpjwSSCfrUS8ExGEuwNSXx1eLIDiGVjwIb+hbtZL1e6fLw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "2.0.48", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/anthropic": { "version": "3.0.81", "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.81.tgz", @@ -77,6 +104,56 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/deepinfra": { + "version": "2.0.52", + "resolved": "https://registry.npmjs.org/@ai-sdk/deepinfra/-/deepinfra-2.0.52.tgz", + "integrity": "sha512-/S4WchqBHlZr0wZmpWfDafM6omNz5i1tIo7WeQ6Hd5y/Jz1le1uWW0AEb1J0ehw+uYMEHc/bLA9RMBf1zehIOg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "2.0.48", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/deepseek": { + "version": "2.0.35", + "resolved": "https://registry.npmjs.org/@ai-sdk/deepseek/-/deepseek-2.0.35.tgz", + "integrity": "sha512-9DhYurbAvcurOEGN6u2myYDybrrzGfcrkG8hwmFjwTrePW6KCMggm0YxP7e8RkLYcQKqCEMgFlyEB4BM6EmiKg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/fireworks": { + "version": "2.0.53", + "resolved": "https://registry.npmjs.org/@ai-sdk/fireworks/-/fireworks-2.0.53.tgz", + "integrity": "sha512-HjeiGsdxSzrCkOf2l2V+K+opzlqxBtduBq6BCiohAdgQk2KdZmI/67SMkBM6Kdze/BjUXiZlv0d7zNICPhxVDA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "2.0.48", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/gateway": { "version": "3.0.116", "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.116.tgz", @@ -94,6 +171,71 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/google": { + "version": "3.0.80", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.80.tgz", + "integrity": "sha512-5ORbm/yFUPO0MEvZsxBMN0cdKw2+lwU/wVn5KN3KF8Dmk1LughuDuUohMh/7iU/XFTiyB0OvmTW/tdV/J7O9zg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/groq": { + "version": "3.0.39", + "resolved": "https://registry.npmjs.org/@ai-sdk/groq/-/groq-3.0.39.tgz", + "integrity": "sha512-BZAr6DjCbzWQ0Qn1/TSsHo/bmCt4JaAMb4A7HCSUZBQCAcOjne/03D0sVjHnQhUC3TpwcmYiv7tHAviK7BluRw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/huggingface": { + "version": "1.0.50", + "resolved": "https://registry.npmjs.org/@ai-sdk/huggingface/-/huggingface-1.0.50.tgz", + "integrity": "sha512-2qn7UAP4q2YrQstKtyRSJro8ujicwgEgPChKMiTbTnAH7SDagI2TfciU+hC9adOP1dM9HDqFP+lVnl4+VBv6Aw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "2.0.48", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4" + } + }, + "node_modules/@ai-sdk/mistral": { + "version": "3.0.37", + "resolved": "https://registry.npmjs.org/@ai-sdk/mistral/-/mistral-3.0.37.tgz", + "integrity": "sha512-KkdaMjs4C2y+vrZWJE990E3ZxBFiOTHQ94ZlquuIttpphcqJTMxNoIpnKT/4UzMVWXL0BUEE2vs+1UEVXkN8Kg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/openai": { "version": "3.0.68", "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.68.tgz", @@ -229,6 +371,40 @@ "node": ">=18" } }, + "node_modules/@ai-sdk/togetherai": { + "version": "2.0.53", + "resolved": "https://registry.npmjs.org/@ai-sdk/togetherai/-/togetherai-2.0.53.tgz", + "integrity": "sha512-M/qsqM1HMlFpWHxHTEorENCLFmBefwxhGTB+XVsjUh77gfyt1io8eg96c/CyWZTxCPa5GxLfl9mhPxp8wjWATg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "2.0.48", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/xai": { + "version": "3.0.93", + "resolved": "https://registry.npmjs.org/@ai-sdk/xai/-/xai-3.0.93.tgz", + "integrity": "sha512-HxazLIcSTgI0UQoq6ua0rcSR8+eXuNy0Qh4jkCY9EAWedYn6CQ0XD/j34U4/JYtO738xJdY+tE95okdqWqXkHA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "2.0.48", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", diff --git a/backend/package.json b/backend/package.json index 1bd11aa..9b1cb19 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,10 +10,20 @@ "mastra:dev": "node ../scripts/with-root-env.mjs mastra dev" }, "dependencies": { + "@ai-sdk/alibaba": "^1.0.26", "@ai-sdk/anthropic": "^3.0.81", + "@ai-sdk/deepinfra": "^2.0.52", + "@ai-sdk/deepseek": "^2.0.35", + "@ai-sdk/fireworks": "^2.0.53", + "@ai-sdk/google": "^3.0.80", + "@ai-sdk/groq": "^3.0.39", + "@ai-sdk/huggingface": "^1.0.50", + "@ai-sdk/mistral": "^3.0.37", "@ai-sdk/openai": "^3.0.68", "@ai-sdk/openai-compatible": "^2.0.48", "@ai-sdk/provider": "^3.0.10", + "@ai-sdk/togetherai": "^2.0.53", + "@ai-sdk/xai": "^3.0.93", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "@mastra/core": "^1.36.0", diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt index 9752429..8f5a720 100644 --- a/backend/prompts/schema-inference.txt +++ b/backend/prompts/schema-inference.txt @@ -3,7 +3,7 @@ You are a data engineering assistant that converts natural-language prompts into Your job is to: 1. Identify the universe of entities the user wants to collect. Each entity becomes one row in the dataset. -2. Pick a clear primary key — the column whose values uniquely identify each row. This is usually a name, ID, or canonical URL. Exactly one column must have `is_primary_key: true`, and its `name` must equal `primary_key`. The primary key column must have `nullable: false` and `is_enumerable: true`. +2. Pick a clear primary key — the column whose values uniquely identify each row. This is usually a name, ID, or canonical URL. Exactly one column must have `is_primary_key: true`, and `primary_key` must be a one-item array containing that column name. The primary key column must have `nullable: false` and `is_enumerable: true`. 3. Choose useful columns. Each column captures one fact about the entity. Use snake_case names. Mark `is_enumerable: true` only on columns whose values can be used to list all rows (typically just the primary key, and occasionally one or two others when a source page lists them alongside the primary key). 4. Set `retrieval_strategy`: - `search_fetch` — the data lives on a static page or sitemap that can be fetched as HTML. diff --git a/backend/src/config/llm.ts b/backend/src/config/llm.ts index 0c50523..cb603dc 100644 --- a/backend/src/config/llm.ts +++ b/backend/src/config/llm.ts @@ -1,7 +1,17 @@ import type { LanguageModelV3 } from "@ai-sdk/provider"; +import { createAlibaba } from "@ai-sdk/alibaba"; import { createAnthropic } from "@ai-sdk/anthropic"; +import { createDeepInfra } from "@ai-sdk/deepinfra"; +import { createDeepSeek } from "@ai-sdk/deepseek"; +import { createFireworks } from "@ai-sdk/fireworks"; +import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { createGroq } from "@ai-sdk/groq"; +import { createHuggingFace } from "@ai-sdk/huggingface"; +import { createMistral } from "@ai-sdk/mistral"; import { createOpenAI } from "@ai-sdk/openai"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; +import { createTogetherAI } from "@ai-sdk/togetherai"; +import { createXai } from "@ai-sdk/xai"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { env } from "../env.js"; @@ -11,6 +21,18 @@ export const LLM_PROVIDER_TYPES = [ "openrouter", "openai", "anthropic", + "google", + "xai", + "deepseek", + "qwen", + "mistral", + "groq", + "togetherai", + "deepinfra", + "fireworks", + "huggingface", + "ollama", + "lmstudio", "custom", ] as const; @@ -40,6 +62,18 @@ export const LLM_PROVIDER_LABELS: Record = { openrouter: "OpenRouter", openai: "OpenAI", anthropic: "Anthropic", + google: "Google Gemini", + xai: "xAI", + deepseek: "DeepSeek", + qwen: "Qwen", + mistral: "Mistral AI", + groq: "Groq", + togetherai: "Together.ai", + deepinfra: "DeepInfra", + fireworks: "Fireworks AI", + huggingface: "Hugging Face", + ollama: "Ollama", + lmstudio: "LM Studio", custom: "Custom OpenAI-compatible", }; @@ -62,6 +96,66 @@ export const LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE: Record< populateOrchestrator: "claude-haiku-4-5-20251001", investigateSubagent: "claude-haiku-4-5-20251001", }, + google: { + schemaInference: "gemini-3.5-flash", + populateOrchestrator: "gemini-3.5-flash", + investigateSubagent: "gemini-3.5-flash", + }, + xai: { + schemaInference: "grok-4.3", + populateOrchestrator: "grok-4.3", + investigateSubagent: "grok-4.3", + }, + deepseek: { + schemaInference: "deepseek-chat", + populateOrchestrator: "deepseek-chat", + investigateSubagent: "deepseek-chat", + }, + qwen: { + schemaInference: "qwen-plus", + populateOrchestrator: "qwen-plus", + investigateSubagent: "qwen-plus", + }, + mistral: { + schemaInference: "mistral-large-latest", + populateOrchestrator: "mistral-large-latest", + investigateSubagent: "mistral-large-latest", + }, + groq: { + schemaInference: "openai/gpt-oss-120b", + populateOrchestrator: "openai/gpt-oss-120b", + investigateSubagent: "openai/gpt-oss-120b", + }, + togetherai: { + schemaInference: "Qwen/Qwen3.5-397B-A17B", + populateOrchestrator: "Qwen/Qwen3.5-397B-A17B", + investigateSubagent: "Qwen/Qwen3.5-397B-A17B", + }, + deepinfra: { + schemaInference: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + populateOrchestrator: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + investigateSubagent: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + }, + fireworks: { + schemaInference: "accounts/fireworks/models/kimi-k2p5", + populateOrchestrator: "accounts/fireworks/models/kimi-k2p5", + investigateSubagent: "accounts/fireworks/models/kimi-k2p5", + }, + huggingface: { + schemaInference: "deepseek-ai/DeepSeek-V3-0324", + populateOrchestrator: "deepseek-ai/DeepSeek-V3-0324", + investigateSubagent: "deepseek-ai/DeepSeek-V3-0324", + }, + ollama: { + schemaInference: "", + populateOrchestrator: "", + investigateSubagent: "", + }, + lmstudio: { + schemaInference: "", + populateOrchestrator: "", + investigateSubagent: "", + }, custom: { schemaInference: "", populateOrchestrator: "", @@ -73,6 +167,18 @@ export const LLM_PROVIDER_DEFAULT_MODELS: Record = { openrouter: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.openrouter.schemaInference, openai: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.openai.schemaInference, anthropic: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.anthropic.schemaInference, + google: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.google.schemaInference, + xai: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.xai.schemaInference, + deepseek: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.deepseek.schemaInference, + qwen: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.qwen.schemaInference, + mistral: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.mistral.schemaInference, + groq: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.groq.schemaInference, + togetherai: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.togetherai.schemaInference, + deepinfra: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.deepinfra.schemaInference, + fireworks: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.fireworks.schemaInference, + huggingface: LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE.huggingface.schemaInference, + ollama: "", + lmstudio: "", custom: "", }; @@ -104,9 +210,62 @@ export function defaultBaseUrlForLlmProvider( if (provider === "openrouter") { return process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1"; } + if (provider === "google") { + return ( + process.env.GOOGLE_GENERATIVE_AI_BASE_URL || + "https://generativelanguage.googleapis.com/v1beta" + ); + } + if (provider === "xai") { + return process.env.XAI_BASE_URL || "https://api.x.ai/v1"; + } + if (provider === "deepseek") { + return process.env.DEEPSEEK_BASE_URL || "https://api.deepseek.com"; + } + if (provider === "qwen") { + return ( + process.env.QWEN_BASE_URL || + "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + ); + } + if (provider === "mistral") { + return process.env.MISTRAL_BASE_URL || "https://api.mistral.ai/v1"; + } + if (provider === "groq") { + return process.env.GROQ_BASE_URL || "https://api.groq.com/openai/v1"; + } + if (provider === "togetherai") { + return process.env.TOGETHER_BASE_URL || "https://api.together.xyz/v1"; + } + if (provider === "deepinfra") { + return process.env.DEEPINFRA_BASE_URL || "https://api.deepinfra.com/v1"; + } + if (provider === "fireworks") { + return ( + process.env.FIREWORKS_BASE_URL || + "https://api.fireworks.ai/inference/v1" + ); + } + if (provider === "huggingface") { + return process.env.HUGGINGFACE_BASE_URL || "https://router.huggingface.co/v1"; + } + if (provider === "ollama") { + return process.env.OLLAMA_BASE_URL || "http://localhost:11434/v1"; + } + if (provider === "lmstudio") { + return process.env.LM_STUDIO_BASE_URL || "http://localhost:1234/v1"; + } return undefined; } +function isOpenAiCompatibleProvider(provider: LlmProviderType): boolean { + return provider === "custom" || provider === "ollama" || provider === "lmstudio"; +} + +function providerAllowsMissingApiKey(provider: LlmProviderType): boolean { + return isOpenAiCompatibleProvider(provider); +} + function isLoopbackHost(hostname: string): boolean { return ["localhost", "127.0.0.1", "0.0.0.0", "::1", "[::1]"].includes( hostname, @@ -150,17 +309,19 @@ export function normalizeLlmProviderInput( ): LlmProviderConfig { const provider = input.provider; const apiKey = input.apiKey.trim(); - if (!apiKey && provider !== "custom") { + if (!apiKey && !providerAllowsMissingApiKey(provider)) { throw new Error(`${llmProviderLabel(provider)} API key is required`); } const baseUrl = - provider === "custom" - ? normalizeCustomBaseUrl(input.baseUrl) + isOpenAiCompatibleProvider(provider) + ? normalizeCustomBaseUrl( + input.baseUrl ?? defaultBaseUrlForLlmProvider(provider), + ) : normalizeBaseUrl(input.baseUrl) ?? defaultBaseUrlForLlmProvider(provider); - if (provider === "custom" && !baseUrl) { - throw new Error("Custom providers require a base URL"); + if (isOpenAiCompatibleProvider(provider) && !baseUrl) { + throw new Error(`${llmProviderLabel(provider)} requires a base URL`); } const defaultModel = @@ -206,12 +367,84 @@ export function createLanguageModel( }); return provider(resolvedModelId); } + case "google": { + const provider = createGoogleGenerativeAI({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "xai": { + const provider = createXai({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "deepseek": { + const provider = createDeepSeek({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "qwen": { + const provider = createAlibaba({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "mistral": { + const provider = createMistral({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "groq": { + const provider = createGroq({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "togetherai": { + const provider = createTogetherAI({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "deepinfra": { + const provider = createDeepInfra({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "fireworks": { + const provider = createFireworks({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "huggingface": { + const provider = createHuggingFace({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + return provider(resolvedModelId); + } + case "ollama": + case "lmstudio": case "custom": { if (!config.baseUrl) { - throw new Error("Custom providers require a base URL"); + throw new Error(`${llmProviderLabel(config.provider)} requires a base URL`); } const provider = createOpenAICompatible({ - name: "custom", + name: config.provider, apiKey: config.apiKey || undefined, baseURL: config.baseUrl, }); @@ -220,55 +453,144 @@ export function createLanguageModel( } } -function providerVerificationRequest(config: LlmProviderConfig): { +export function modelsUrlForLlmProvider( + provider: LlmProviderType, + baseUrl?: string, +): string { + const resolvedBaseUrl = ( + baseUrl || + defaultBaseUrlForLlmProvider(provider) || + "https://api.openai.com/v1" + ).replace(/\/+$/, ""); + + if (provider === "deepinfra") { + return resolvedBaseUrl.endsWith("/openai") + ? `${resolvedBaseUrl}/models` + : `${resolvedBaseUrl}/openai/models`; + } + + return `${resolvedBaseUrl}/models`; +} + +type ProviderVerificationRequest = { url: string; headers: Record; -} { + method?: "GET" | "POST"; + body?: string; + fallbackStatuses?: number[]; +}; + +function openAiStyleModelsVerificationRequest( + config: LlmProviderConfig, +): ProviderVerificationRequest { + return { + url: modelsUrlForLlmProvider(config.provider, config.baseUrl), + headers: { Authorization: `Bearer ${config.apiKey}` }, + }; +} + +function qwenChatVerificationRequest( + config: LlmProviderConfig, +): ProviderVerificationRequest { + const baseUrl = ( + config.baseUrl || + defaultBaseUrlForLlmProvider("qwen") || + "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + ).replace(/\/+$/, ""); + + return { + url: `${baseUrl}/chat/completions`, + method: "POST", + headers: { + Authorization: `Bearer ${config.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: config.defaultModel || defaultModelForLlmProvider("qwen"), + messages: [{ role: "user", content: "ping" }], + max_tokens: 1, + }), + }; +} + +function providerVerificationRequests( + config: LlmProviderConfig, +): ProviderVerificationRequest[] { switch (config.provider) { case "openrouter": { const baseUrl = (config.baseUrl || "https://openrouter.ai/api/v1").replace( /\/+$/, "", ); - return { + return [{ url: `${baseUrl}/key`, headers: { Authorization: `Bearer ${config.apiKey}` }, - }; + }]; } case "openai": { const baseUrl = (config.baseUrl || "https://api.openai.com/v1").replace( /\/+$/, "", ); - return { + return [{ url: `${baseUrl}/models`, headers: { Authorization: `Bearer ${config.apiKey}` }, - }; + }]; } case "anthropic": { const baseUrl = (config.baseUrl || "https://api.anthropic.com/v1").replace( /\/+$/, "", ); - return { + return [{ url: `${baseUrl}/models?limit=1`, headers: { "x-api-key": config.apiKey, "anthropic-version": "2023-06-01", }, - }; + }]; + } + case "google": { + const baseUrl = ( + config.baseUrl || "https://generativelanguage.googleapis.com/v1beta" + ).replace(/\/+$/, ""); + return [{ + url: `${baseUrl}/models`, + headers: { "x-goog-api-key": config.apiKey }, + }]; + } + case "qwen": { + return [ + { + ...openAiStyleModelsVerificationRequest(config), + fallbackStatuses: [404, 405], + }, + qwenChatVerificationRequest(config), + ]; + } + case "xai": + case "deepseek": + case "mistral": + case "groq": + case "togetherai": + case "deepinfra": + case "fireworks": + case "huggingface": { + return [openAiStyleModelsVerificationRequest(config)]; } + case "ollama": + case "lmstudio": case "custom": { if (!config.baseUrl) { - throw new Error("Custom providers require a base URL"); + throw new Error(`${llmProviderLabel(config.provider)} requires a base URL`); } const baseUrl = config.baseUrl.replace(/\/+$/, ""); - return { + return [{ url: `${baseUrl}/models`, headers: config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}, - }; + }]; } } } @@ -276,24 +598,41 @@ function providerVerificationRequest(config: LlmProviderConfig): { export async function verifyLlmProviderConfig( config: LlmProviderConfig, ): Promise { - const { url, headers } = providerVerificationRequest(config); + const requests = providerVerificationRequests(config); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + let lastResponseStatus: number | undefined; + let lastUrl: string | undefined; try { - const response = await fetch(url, { - headers, - signal: controller.signal, - }); + for (const request of requests) { + lastUrl = request.url; + const response = await fetch(request.url, { + method: request.method ?? "GET", + headers: request.headers, + body: request.body, + signal: controller.signal, + }); + + if (response.ok) return; - if (!response.ok) { + lastResponseStatus = response.status; + if (request.fallbackStatuses?.includes(response.status)) { + continue; + } if (response.status === 401 || response.status === 403) { - throw new Error(`${llmProviderLabel(config.provider)} rejected that API key.`); + throw new Error( + `${llmProviderLabel(config.provider)} rejected that API key.`, + ); } throw new Error( `${llmProviderLabel(config.provider)} verification failed with HTTP ${response.status}.`, ); } + + throw new Error( + `${llmProviderLabel(config.provider)} verification failed with HTTP ${lastResponseStatus ?? "unknown"}.`, + ); } catch (err) { if (err instanceof Error && err.name === "AbortError") { throw new Error( @@ -301,9 +640,20 @@ export async function verifyLlmProviderConfig( ); } if (err instanceof Error && err.message === "fetch failed") { - const displayUrl = url.replace("host.docker.internal", "localhost"); + const displayUrl = (lastUrl ?? requests[0]?.url ?? "").replace( + "host.docker.internal", + "localhost", + ); + const localHint = + config.provider === "ollama" + ? " Start Ollama and confirm the OpenAI-compatible endpoint is enabled." + : config.provider === "lmstudio" + ? " Start the LM Studio local server and confirm the port." + : config.provider === "custom" + ? " Check that the endpoint is running and reachable." + : ""; throw new Error( - `${llmProviderLabel(config.provider)} verification failed: could not reach ${displayUrl}. If this is LM Studio, start the local server and use http://localhost:1234 or http://localhost:1234/v1.`, + `${llmProviderLabel(config.provider)} verification failed: could not reach ${displayUrl}.${localHint}`, ); } throw err; diff --git a/backend/src/config/models.ts b/backend/src/config/models.ts index 0b65699..3da1baa 100644 --- a/backend/src/config/models.ts +++ b/backend/src/config/models.ts @@ -8,7 +8,12 @@ import { api, internal, convex } from "../convex.js"; import { env } from "../env.js"; import { getLlmProviderConfig, requireOpenRouterApiKey } from "../local-credentials.js"; import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; -import { defaultModelForLlmProviderRole, type ModelRoleKey } from "./llm.js"; +import { + defaultBaseUrlForLlmProvider, + defaultModelForLlmProviderRole, + modelsUrlForLlmProvider, + type ModelRoleKey, +} from "./llm.js"; export interface OpenRouterModel { modelName: string; @@ -46,6 +51,98 @@ const OPENAI_MODEL_EXCLUDE_PATTERNS = [ "whisper", ]; +const GOOGLE_MODEL_EXCLUDE_PATTERNS = [ + "audio", + "embedding", + "imagen", + "image", + "live", + "lyria", + "nano-banana", + "robotics", + "tts", + "veo", +]; + +const TEXT_MODEL_EXCLUDE_PATTERNS = [ + "audio", + "babbage", + "dall-e", + "embedding", + "image", + "moderation", + "rerank", + "safeguard", + "sdxl", + "speech", + "stable-diffusion", + "transcribe", + "tts", + "video", + "voice", + "wan", + "whisper", +]; + +const QWEN_MODELS: OpenRouterModel[] = [ + { + modelName: "qwen-plus", + canonicalSlug: "qwen-plus", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen3.5-plus", + canonicalSlug: "qwen3.5-plus", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen3-max", + canonicalSlug: "qwen3-max", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen-max", + canonicalSlug: "qwen-max", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen-flash", + canonicalSlug: "qwen-flash", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen3-235b-a22b-instruct-2507", + canonicalSlug: "qwen3-235b-a22b-instruct-2507", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen3-235b-a22b-thinking-2507", + canonicalSlug: "qwen3-235b-a22b-thinking-2507", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, + { + modelName: "qwen3-coder-plus", + canonicalSlug: "qwen3-coder-plus", + contextLength: 0, + completionCost: 0, + promptCost: 0, + }, +]; + function isOpenAITextModelId(id: string): boolean { const lower = id.toLowerCase(); if (OPENAI_MODEL_EXCLUDE_PATTERNS.some((pattern) => lower.includes(pattern))) { @@ -60,28 +157,89 @@ function isOpenAITextModelId(id: string): boolean { ); } -function sortModels(models: OpenRouterModel[]): OpenRouterModel[] { - return models.sort((a, b) => a.modelName.localeCompare(b.modelName)); +function isGenericTextModelId(id: string): boolean { + const lower = id.toLowerCase(); + return !TEXT_MODEL_EXCLUDE_PATTERNS.some((pattern) => + lower.includes(pattern), + ); } -function isModelCompatibleWithProvider( - modelId: string | undefined, +function isGoogleTextModelId(id: string): boolean { + const lower = id.toLowerCase(); + if (GOOGLE_MODEL_EXCLUDE_PATTERNS.some((pattern) => lower.includes(pattern))) { + return false; + } + return ( + lower.startsWith("gemini-") || + lower.startsWith("gemma-") || + lower.startsWith("deep-research-") + ); +} + +function isMistralTextModelId(id: string): boolean { + const lower = id.toLowerCase(); + return ( + isGenericTextModelId(id) && + (lower.startsWith("mistral-") || + lower.startsWith("magistral-") || + lower.startsWith("ministral-") || + lower.startsWith("codestral-") || + lower.startsWith("devstral-") || + lower.startsWith("pixtral-")) + ); +} + +function isProviderTextModelId( + id: string, provider: Awaited>, -): modelId is string { - if (!modelId) return false; +): boolean { if (!provider) return true; switch (provider.provider) { case "openrouter": - return modelId.includes("/"); + return id.includes("/"); case "openai": - return isOpenAITextModelId(modelId) && !modelId.includes("/"); + return isOpenAITextModelId(id) && !id.includes("/"); case "anthropic": - return modelId.startsWith("claude-") && !modelId.includes("/"); + return id.startsWith("claude-") && !id.includes("/"); + case "google": + return isGoogleTextModelId(id) && !id.includes("/"); + case "xai": + return id.startsWith("grok-") && !id.includes("imagine"); + case "deepseek": + return id.startsWith("deepseek-"); + case "qwen": + return id.startsWith("qwen") || id.startsWith("qwq-"); + case "mistral": + return isMistralTextModelId(id); + case "groq": + case "togetherai": + case "deepinfra": + case "fireworks": + case "huggingface": + return isGenericTextModelId(id); + case "ollama": + case "lmstudio": case "custom": return true; } } +function googleModelIdFromName(name: string): string { + return name.replace(/^models\//, ""); +} + +function sortModels(models: OpenRouterModel[]): OpenRouterModel[] { + return models.sort((a, b) => a.modelName.localeCompare(b.modelName)); +} + +function isModelCompatibleWithProvider( + modelId: string | undefined, + provider: Awaited>, +): modelId is string { + if (!modelId) return false; + return isProviderTextModelId(modelId, provider); +} + function modelForProvider( savedModel: string | undefined, role: ModelRoleKey, @@ -175,25 +333,82 @@ export async function fetchModelsForCurrentLlmProvider(): Promise; + }>(`${baseUrl}/models`, { + "x-goog-api-key": config.apiKey, + }); + + return sortModels( + (json.models ?? []) + .map((model) => { + const modelId = model.baseModelId || googleModelIdFromName(model.name); + return { + model, + modelId, + actions: + model.supportedActions ?? model.supportedGenerationMethods ?? [], + }; + }) + .filter(({ modelId, actions }) => { + return ( + isGoogleTextModelId(modelId) && + (actions.length === 0 || actions.includes("generateContent")) + ); + }) + .map(({ model, modelId }) => ({ + modelName: model.displayName ?? modelId, + canonicalSlug: modelId, + contextLength: model.inputTokenLimit ?? 0, + completionCost: 0, + promptCost: 0, + })), + ); + } + + if (config.provider === "qwen") { + return sortModels([...QWEN_MODELS]); + } + + const baseUrl = ( + config.baseUrl || + defaultBaseUrlForLlmProvider(config.provider) || + "https://api.openai.com/v1" + ).replace(/\/+$/, ""); const headers: Record = - config.provider === "custom" && !config.apiKey + ["custom", "ollama", "lmstudio"].includes(config.provider) && !config.apiKey ? {} : { Authorization: `Bearer ${config.apiKey}` }; const json = await fetchJsonWithTimeout<{ - data?: Array<{ id: string }>; - }>(`${baseUrl}/models`, headers); + data?: Array<{ + id: string; + display_name?: string; + name?: string; + context_length?: number; + contextLength?: number; + }>; + }>(modelsUrlForLlmProvider(config.provider, baseUrl), headers); const models = (json.data ?? []) - .filter((model) => - config.provider === "openai" - ? isOpenAITextModelId(model.id) - : true, - ) + .filter((model) => isProviderTextModelId(model.id, config)) .map((model) => ({ - modelName: model.id, + modelName: model.display_name ?? model.name ?? model.id, canonicalSlug: model.id, - contextLength: 0, + contextLength: model.context_length ?? model.contextLength ?? 0, completionCost: 0, promptCost: 0, })); diff --git a/backend/src/index.ts b/backend/src/index.ts index 834a702..beb6291 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -732,9 +732,12 @@ fastify.post("/local-setup/llm-provider", async (req, reply) => { try { const apiKey = body.apiKey?.trim() ?? ""; - const isNewCustomWithoutKey = provider === "custom" && !!body.baseUrl?.trim(); + const isKeylessProvider = + provider === "custom" || provider === "ollama" || provider === "lmstudio"; + const isNewKeylessProvider = + isKeylessProvider && (provider !== "custom" || !!body.baseUrl?.trim()); - if (!apiKey && !isNewCustomWithoutKey) { + if (!apiKey && !isNewKeylessProvider) { const status = await getLocalSetupStatus(); const savedProvider = status.services.llmProviders?.[provider]; if (!savedProvider?.configured) { diff --git a/backend/src/local-credential-types.ts b/backend/src/local-credential-types.ts index 9d5fc95..f899fc1 100644 --- a/backend/src/local-credential-types.ts +++ b/backend/src/local-credential-types.ts @@ -3,6 +3,18 @@ export const LOCAL_CREDENTIAL_SERVICES = [ "openrouter", "openai", "anthropic", + "google", + "xai", + "deepseek", + "qwen", + "mistral", + "groq", + "togetherai", + "deepinfra", + "fireworks", + "huggingface", + "ollama", + "lmstudio", "custom", ] as const; diff --git a/backend/src/local-credentials.ts b/backend/src/local-credentials.ts index 5b7727d..16456ec 100644 --- a/backend/src/local-credentials.ts +++ b/backend/src/local-credentials.ts @@ -6,6 +6,7 @@ import { setKeychainCredential, } from "./local-keychain-client.js"; import { + LLM_PROVIDER_TYPES, defaultBaseUrlForLlmProvider, defaultModelForLlmProvider, isLlmProviderType, @@ -93,29 +94,33 @@ async function localCredential(service: LocalCredentialService): Promise<{ } | null; + const rowProvider = isLlmProviderType(rowData?.llmProvider) + ? rowData.llmProvider + : undefined; + const rowBaseUrl = + typeof rowData?.llmBaseUrl === "string" ? rowData.llmBaseUrl : undefined; + if ( + rowProvider && + service === rowProvider && + ["custom", "ollama", "lmstudio"].includes(rowProvider) && + rowBaseUrl + ) { + return { + apiKey: "", + connectionMethod: rowData?.connectionMethod ?? "api_key", + verifiedAt: rowData?.verifiedAt ?? null, + keychainAccount: rowData?.keychainAccount ?? "", + llmProvider: rowProvider, + llmBaseUrl: rowBaseUrl, + llmDefaultModel: + typeof rowData?.llmDefaultModel === "string" + ? rowData.llmDefaultModel + : undefined, + }; + } + const keychain = await getKeychainCredential(service); if (!keychain?.apiKey) { - // LM Studio and many local OpenAI-compatible servers do not require an - // API key. A custom-provider row with a base URL is therefore a valid - // local credential even when there is no keychain secret. - if ( - service === "custom" && - rowData?.llmProvider === "custom" && - typeof rowData.llmBaseUrl === "string" - ) { - return { - apiKey: "", - connectionMethod: rowData.connectionMethod ?? "api_key", - verifiedAt: rowData.verifiedAt ?? null, - keychainAccount: rowData.keychainAccount ?? "", - llmProvider: "custom", - llmBaseUrl: rowData.llmBaseUrl, - llmDefaultModel: - typeof rowData.llmDefaultModel === "string" - ? rowData.llmDefaultModel - : undefined, - }; - } return null; } @@ -358,12 +363,7 @@ export async function getLocalSetupStatus(): Promise { }; const providerStatuses = {} as Record; - for (const provider of [ - "openrouter", - "openai", - "anthropic", - "custom", - ] as const) { + for (const provider of LLM_PROVIDER_TYPES) { const credential = await localCredentialForLlmProvider(provider); providerStatuses[provider] = credential ? { diff --git a/backend/src/mastra/agents/investigate.ts b/backend/src/mastra/agents/investigate.ts index c930f5e..ac80b73 100644 --- a/backend/src/mastra/agents/investigate.ts +++ b/backend/src/mastra/agents/investigate.ts @@ -7,6 +7,9 @@ import type { PopulateColumn } from "../../pipeline/populate.js"; function buildInvestigateInstructions(columns: PopulateColumn[]): string { const columnNames = columns.map((c) => c.name); + const dataExample = columnNames + .map((n) => `{"column": "${n}", "value": "value"}`) + .join(", "); const columnsDesc = columns .map( (c) => @@ -29,7 +32,7 @@ RULES: TOOL CALL FORMAT — every tool call argument must be a JSON object wrapped in curly braces: search_web: {"query": "your search terms"} fetch_page: {"url": "https://example.com"} - insert_row: {"data": {${columnNames.map((n) => `"${n}": "value"`).join(", ")}}, "sources": ["https://url-you-fetched.com"], "row_summary": "one line about this entity", "how_found": "step by step guide on how to extract the data so an agent in the future can do it too"} + insert_row: {"data": [${dataExample}], "sources": ["https://url-you-fetched.com"], "row_summary": "one line about this entity", "how_found": "step by step guide on how to extract the data so an agent in the future can do it too"} WORKFLOW: 1. Fetch 1-2 of the provided URLs to get real data (if URLs were given). diff --git a/backend/src/mastra/agents/refresh.ts b/backend/src/mastra/agents/refresh.ts index 6593f96..bc38eb4 100644 --- a/backend/src/mastra/agents/refresh.ts +++ b/backend/src/mastra/agents/refresh.ts @@ -7,6 +7,9 @@ import type { PopulateColumn } from "../../pipeline/populate.js"; function buildRefreshInstructions(columns: PopulateColumn[]): string { const columnNames = columns.map((c) => c.name); + const dataExample = columnNames + .map((n) => `{"column": "${n}", "value": "value"}`) + .join(", "); const columnsDesc = columns .map( (c) => @@ -25,7 +28,7 @@ RULES: - If no "Previously found via" steps are provided, fall back to fetching the source URLs directly. - If a source returns a 404, timeout, or is blocked, note it and move to the next. - Compare the fetched data with the existing row data carefully. -- If data has MEANINGFULLY changed (not just formatting differences), call update_row with the FULL updated data object (all columns, not just changed ones), plus updated sources, row_summary, and how_found. +- If data has MEANINGFULLY changed (not just formatting differences), call update_row with the FULL updated row data (all columns, not just changed ones), plus updated sources, row_summary, and how_found. - If NO sources work (all 404/blocked), try ONE web search using the primary key values to find a current source. - If the data is unchanged, do NOT call update_row. Just report your findings. - Never fabricate values. If you can't verify a field, keep the existing value. @@ -33,7 +36,7 @@ RULES: TOOL CALL FORMAT — every tool call argument must be a JSON object wrapped in curly braces: fetch_page: {"url": "https://example.com"} search_web: {"query": "your search terms"} - update_row: {"rowId": "", "data": {${columnNames.map((n) => `"${n}": "value"`).join(", ")}}, "sources": ["https://..."], "row_summary": "one line about this entity", "how_found": "how you verified this data"} + update_row: {"rowId": "", "data": [${dataExample}], "sources": ["https://..."], "row_summary": "one line about this entity", "how_found": "how you verified this data"} WORKFLOW: 1. Fetch the provided source URLs (1-2 calls). diff --git a/backend/src/mastra/tools/dataset-tools.ts b/backend/src/mastra/tools/dataset-tools.ts index 1fc016e..e0109a6 100644 --- a/backend/src/mastra/tools/dataset-tools.ts +++ b/backend/src/mastra/tools/dataset-tools.ts @@ -58,6 +58,21 @@ const writeResultSchema = z.object({ const ROW_NOT_FOUND_MSG = "Row not found. It may have been deleted, or the id belongs to a different dataset. Use list_rows to see valid row ids."; +const rowDataCellSchema = z.object({ + column: z.string().min(1), + value: z.string(), +}); + +type RowDataCell = z.infer; + +function rowDataCellsToRecord(data: RowDataCell[]): Record { + const row: Record = {}; + for (const cell of data) { + row[cell.column] = cell.value; + } + return row; +} + function cleanDataKeys(data: Record): Record { const cleaned: Record = {}; for (const [key, value] of Object.entries(data)) { @@ -126,7 +141,12 @@ export function buildPopulateTools( description: "Insert a single row into the dataset you are populating. Call this each time you have a row ready — don't wait to batch them.", inputSchema: z.object({ - data: z.record(z.string(), z.any()), + data: z + .array(rowDataCellSchema) + .min(1) + .describe( + 'Row values as {"column": "column_name", "value": "cell value"} entries. Use an empty string for unknown values.', + ), sources: z .array(z.string()) .optional() @@ -142,14 +162,14 @@ export function buildPopulateTools( }), outputSchema: writeResultSchema, execute: async ({ data, sources, row_summary, how_found }) => { - if (!data || Object.keys(data).length === 0) + if (!data || data.length === 0) return { success: false, error: - 'data is required and must have at least one key. Pass an object like { "Column Name": value }.', + 'data is required and must include at least one entry like { "column": "column_name", "value": "cell value" }.', }; - const cleanedData = cleanDataKeys(data); + const cleanedData = cleanDataKeys(rowDataCellsToRecord(data)); console.log( `[insert_row] ${logCtx} cols=${Object.keys(cleanedData).length} sources=${sources?.length ?? 0}`, ); @@ -257,10 +277,15 @@ export function buildPopulateTools( const updateRowTool = createTool({ id: "update_row", description: - "Update an existing row by its ID. Pass the full updated data object. Changes are tracked in history.", + "Update an existing row by its ID. Pass the full updated row data. Changes are tracked in history.", inputSchema: z.object({ rowId: z.string(), - data: z.record(z.string(), z.any()), + data: z + .array(rowDataCellSchema) + .min(1) + .describe( + 'Full row values as {"column": "column_name", "value": "cell value"} entries. Use an empty string for unknown values.', + ), sources: z .array(z.string()) .optional() @@ -277,13 +302,13 @@ export function buildPopulateTools( outputSchema: writeResultSchema, execute: async ({ rowId, data, sources, row_summary, how_found }) => { if (!rowId) return { success: false, error: "rowId is required." }; - if (!data || Object.keys(data).length === 0) + if (!data || data.length === 0) return { success: false, - error: "data is required. Pass the full updated row data object.", + error: "data is required. Pass the full updated row data entries.", }; - const cleanedData = cleanDataKeys(data); + const cleanedData = cleanDataKeys(rowDataCellsToRecord(data)); console.log( `[update_row] ${logCtx} row=${rowId} cols=${Object.keys(cleanedData).length}`, ); diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 1b23764..0202100 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -8,6 +8,11 @@ import type { RunMetrics } from "../run-metrics.js"; import { getSignal } from "../../abort-registry.js"; import type { LlmProviderConfig } from "../../config/llm.js"; +const keyValueSchema = z.object({ + column: z.string().min(1), + value: z.string().min(1), +}); + const investigateInputSchema = z.object({ entity_hint: z .string() @@ -15,12 +20,10 @@ const investigateInputSchema = z.object({ "What entity to look for, e.g. 'head of GTM at Appcharge' or 'Starbucks coffee products on Amazon'", ), primary_keys: z - .record(z.string(), z.string()) - .refine((v) => Object.keys(v).length > 0, { - message: "primary_keys must include at least one primary-key value", - }) + .array(keyValueSchema) + .min(1, "primary_keys must include at least one primary-key value") .describe( - "REQUIRED: the primary key column value(s) for this entity. e.g. {\"Company Name\": \"Stripe\"} or {\"First Name\": \"John\", \"Last Name\": \"Doe\"}. You MUST provide at least the primary key values you have found.", + 'REQUIRED: primary key values as {"column": "column_name", "value": "value"} entries. e.g. [{"column": "company_name", "value": "Stripe"}]. You MUST provide at least the primary key values you have found.', ), context: z .string() @@ -112,8 +115,8 @@ export function buildSubagentTool( llmConfig, ); - const pkBlock = Object.entries(primary_keys) - .map(([k, v]) => `- ${k}: ${v}`) + const pkBlock = primary_keys + .map(({ column, value }) => `- ${column}: ${value}`) .join("\n"); const urlsBlock = urls && urls.length > 0 diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index 3673047..bd1c97e 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -174,7 +174,7 @@ const buildPromptStep = createStep({ const pkNote = pkColumns.length > 0 - ? `\nPrimary key column(s): ${pkColumns.map((c) => `"${c.name}"`).join(", ")}. When calling run_subagent, you MUST pass these values in the primary_keys field. The subagent will research and fill in the remaining columns.` + ? `\nPrimary key column(s): ${pkColumns.map((c) => `"${c.name}"`).join(", ")}. When calling run_subagent, you MUST pass these values in the primary_keys field as an array of {"column": "column_name", "value": "value"} entries. The subagent will research and fill in the remaining columns.` : ""; let manifestNote = ""; @@ -202,6 +202,7 @@ ${columnsDesc}${pkNote}${manifestNote}${strategyNote} Search the web broadly to find real entities that fit this dataset topic. For each lead you find, call run_subagent with the primary key values and any context/URLs you have found. +Example primary_keys format: [{"column": "company_name", "value": "Stripe"}] If run_subagent returns ROW_LIMIT_REACHED, stop immediately and do not make any more tool calls. Stop the populate run as soon as the dataset reaches ${inputData.maxRowCount} rows.`; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index d1ab510..8a72abe 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -10,7 +10,7 @@ const SYSTEM_PROMPT = `You are a data engineering assistant that converts natura Your job is to: 1. Identify the universe of entities the user wants to collect. Each entity becomes one row in the dataset. -2. Pick primary key column(s) — one or more columns whose combined values uniquely identify each row (no two legitimate rows should share the same values across all primary key columns in any case). Refrain from names unless necessary, as they may not always be unqiue (unless this is guarenteed). Otherwise use thigns like URLs or IDs that have a 100% guarentee of being unique. Set \`is_primary_key: true\` on each primary key column. Set \`primary_key\` to the column name if there is one, or an array of column names if there are multiple. Every primary key column must have \`nullable: false\` and \`is_enumerable: true\`. Prefer a single column when one naturally uniquely identifies each row. +2. Pick primary key column(s) — one or more columns whose combined values uniquely identify each row (no two legitimate rows should share the same values across all primary key columns in any case). Refrain from names unless necessary, as they may not always be unqiue (unless this is guarenteed). Otherwise use thigns like URLs or IDs that have a 100% guarentee of being unique. Set \`is_primary_key: true\` on each primary key column. Set \`primary_key\` to an array of primary key column names; use a one-item array for a single primary key. Every primary key column must have \`nullable: false\` and \`is_enumerable: true\`. Prefer a single column when one naturally uniquely identifies each row. 3. Choose useful columns. Each column captures one fact about the entity. Use snake_case names. Mark \`is_enumerable: true\` only on columns whose values can be used to list all rows (typically just the primary key, and occasionally one or two others when a source page lists them alongside the primary key). 4. Set \`retrieval_strategy\`: - \`search_fetch\` — the data lives on a static page or sitemap that can be fetched as HTML. diff --git a/backend/src/pipeline/types.ts b/backend/src/pipeline/types.ts index e0b95f9..ee9ed13 100644 --- a/backend/src/pipeline/types.ts +++ b/backend/src/pipeline/types.ts @@ -35,7 +35,7 @@ export const datasetSchemaSchema = z dataset_name: z.string().regex(snakeCase, "must be snake_case"), description: z.string().min(1), columns: z.array(columnDefinitionSchema).min(1), - primary_key: z.union([z.string(), z.array(z.string())]), + primary_key: z.array(z.string()).min(1), retrieval_strategy: retrievalStrategySchema, source_hint: z.string().min(1), }) @@ -61,9 +61,7 @@ export const datasetSchemaSchema = z } const pkNames = pkCols.map((c) => c.name); - const declaredPkRaw = Array.isArray(data.primary_key) - ? data.primary_key - : [data.primary_key]; + const declaredPkRaw = data.primary_key; const declaredPk = [...new Set(declaredPkRaw)]; if ( declaredPk.length !== declaredPkRaw.length || diff --git a/frontend/app/setup/page.tsx b/frontend/app/setup/page.tsx index 2814ad3..928d861 100644 --- a/frontend/app/setup/page.tsx +++ b/frontend/app/setup/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useMemo, useState, type ReactNode } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { useRouter } from "next/navigation"; import { CheckCircle2, @@ -11,24 +11,74 @@ import { } from "lucide-react"; import { getLocalSetupStatus, + getLlmProviderModels, + getModelConfig, saveLlmProviderConfig, + saveModelConfig, saveTinyFishApiKey, + type EffectiveModelConfig, type LlmProviderType, type LocalSetupStatus, + type OpenRouterModel, type ServiceSetupStatus, } from "@/lib/backend"; import { isLocalMode } from "@/lib/app-mode"; import { LlmProviderBrand, LlmProviderSelector, + displayBaseUrl, + llmProviderLabelForStatus, llmProviderOption, + localLlmPresetForBaseUrl, + type LlmProviderOptionValue, } from "@/components/settings/llm-providers"; +import { + beginOpenRouterOAuth, + useCanUseOpenRouterOAuth, +} from "@/lib/openrouter-oauth"; +import { LocalUtilityMenu } from "@/components/LocalUtilityMenu"; +import { ModelSideSheet } from "@/components/settings/ModelSideSheet"; +import { MODEL_ROLES, type ModelRole } from "@/components/settings/types"; +import { useAppAuth } from "@/lib/app-auth"; + +function modelListCacheKey(status: LocalSetupStatus | null): string { + const llm = status?.services.llm; + if (!llm) return ""; + return [ + llm.provider ?? "openrouter", + llm.baseUrl ?? "", + llm.defaultModel ?? "", + llm.verifiedAt ?? "", + ].join("|"); +} + +function emptyModelConfig(): EffectiveModelConfig { + return { + schemaInference: "", + populateOrchestrator: "", + investigateSubagent: "", + }; +} export default function SetupPage() { const router = useRouter(); + const { getToken } = useAppAuth(); const [status, setStatus] = useState(null); const [loading, setLoading] = useState(true); const [modal, setModal] = useState<"tinyfish" | "llm" | null>(null); + const [modelConfig, setModelConfig] = useState( + null, + ); + const [loadingModelConfig, setLoadingModelConfig] = useState(false); + const [modelError, setModelError] = useState(null); + const [activeModelRole, setActiveModelRole] = useState(null); + const [modelOptions, setModelOptions] = useState([]); + const [modelOptionsCacheKey, setModelOptionsCacheKey] = useState< + string | null + >(null); + const [refreshingModels, setRefreshingModels] = useState(false); + const [savingModel, setSavingModel] = useState(false); + const activeModelListCacheKeyRef = useRef(""); useEffect(() => { if (!isLocalMode) { @@ -41,7 +91,122 @@ export default function SetupPage() { .finally(() => setLoading(false)); }, [router]); + const activeModelListCacheKey = modelListCacheKey(status); + + useEffect(() => { + activeModelListCacheKeyRef.current = activeModelListCacheKey; + }, [activeModelListCacheKey]); + + useEffect(() => { + let active = true; + + if (!status?.services.llm.configured) return; + + async function loadModelConfig() { + setLoadingModelConfig(true); + setModelError(null); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + const config = await getModelConfig(token); + if (active) setModelConfig(config); + } catch (err) { + if (!active) return; + setModelConfig(emptyModelConfig()); + setModelError( + err instanceof Error ? err.message : "Failed to load model settings", + ); + } finally { + if (active) setLoadingModelConfig(false); + } + } + + void loadModelConfig(); + + return () => { + active = false; + }; + }, [ + getToken, + status?.services.llm.baseUrl, + status?.services.llm.configured, + status?.services.llm.provider, + status?.services.llm.verifiedAt, + ]); + + const loadProviderModels = useCallback( + async (force = false) => { + const cacheKey = activeModelListCacheKeyRef.current; + if (!force && modelOptionsCacheKey === cacheKey && modelOptions.length > 0) { + return; + } + + setRefreshingModels(true); + setModelError(null); + try { + const models = await getLlmProviderModels(); + if (activeModelListCacheKeyRef.current !== cacheKey) return; + setModelOptions(models); + setModelOptionsCacheKey(cacheKey); + } catch (err) { + if (activeModelListCacheKeyRef.current !== cacheKey) return; + setModelOptions([]); + setModelOptionsCacheKey(cacheKey); + setModelError( + err instanceof Error ? err.message : "Failed to load models", + ); + } finally { + if (activeModelListCacheKeyRef.current === cacheKey) { + setRefreshingModels(false); + } + } + }, + [modelOptions.length, modelOptionsCacheKey], + ); + + function modelForRole(role: ModelRole): string { + const key = role.key as keyof EffectiveModelConfig; + return modelConfig?.[key] ?? ""; + } + + function openModelSheet(role: ModelRole) { + setActiveModelRole(role); + void loadProviderModels(); + } + + async function saveModelForRole(role: ModelRole, modelId: string) { + const nextModelId = modelId.trim(); + if (!nextModelId) return; + + const key = role.key as keyof EffectiveModelConfig; + setSavingModel(true); + setModelError(null); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + await saveModelConfig({ [key]: nextModelId }, token); + setModelConfig((prev) => ({ + ...emptyModelConfig(), + ...prev, + [key]: nextModelId, + })); + setActiveModelRole(null); + } catch (err) { + setModelError( + err instanceof Error ? err.message : "Failed to save model", + ); + } finally { + setSavingModel(false); + } + } + const complete = status?.complete ?? false; + const modelSelectionRequired = + !!status?.services.llm.configured && !status.services.llm.defaultModel; + const modelsConfigured = + !modelSelectionRequired || + MODEL_ROLES.every((role) => modelForRole(role).trim().length > 0); + const canCompleteSetup = complete && modelsConfigured; if (loading) { return ( @@ -53,9 +218,12 @@ export default function SetupPage() { return (
-
- BigSet - BigSet +
+
+ BigSet + BigSet +
+
@@ -65,12 +233,11 @@ export default function SetupPage() { Connect your services

- Add TinyFish and your preferred LLM provider to start building - live datasets. + Add TinyFish and choose where BigSet should run model calls.

-
+
@@ -86,20 +253,28 @@ export default function SetupPage() { /> } - description="BigSet uses TinyFish's best-in-class search API to unlock real-time information." + description="Connect TinyFish for live search and source pages." status={status?.services.tinyfish} primaryLabel={ status?.services.tinyfish.configured ? "Update key" : "Add API key" } onPrimary={() => setModal("tinyfish")} helperHref="https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2" - helperLabel="Need a TinyFish key?" - helperDescription="Open the TinyFish API keys page" + helperLabel="Get your TinyFish API Key" /> } - description="BigSet uses your LLM provider for schema generation and dataset-building agents." + brand={ + + } + description="Choose the provider BigSet uses for schema generation and agents." status={status?.services.llm} primaryLabel={ status?.services.llm.configured @@ -107,21 +282,74 @@ export default function SetupPage() { : "Choose provider" } onPrimary={() => setModal("llm")} - helperHref="https://platform.openai.com/api-keys" - helperLabel="Bring your own model" - helperDescription="OpenAI, Anthropic, OpenRouter, or custom" />
+ {status?.services.llm.configured && ( +
+
+
+

+ Models +

+

+ {llmProviderLabelForStatus(status.services.llm) ?? + "Model provider"} +

+
+ {modelSelectionRequired && !modelsConfigured && ( + + Required + + )} +
+
+ {MODEL_ROLES.map((role) => { + const selectedModel = modelForRole(role); + return ( + + ); + })} +
+ {modelError && ( +
+ {modelError} +
+ )} +
+ )} +

- {complete + {complete && modelsConfigured ? "Everything is connected. You can start building datasets." + : complete && modelSelectionRequired + ? "Choose models to continue." : "Complete both connections to continue."}

); } @@ -165,21 +408,22 @@ function ServiceCard({ onPrimary: () => void; secondaryLabel?: string; onSecondary?: () => void; - helperHref: string; - helperLabel: string; - helperDescription: string; + helperHref?: string; + helperLabel?: string; + helperDescription?: string; }) { const connected = status?.configured ?? false; const detail = useMemo(() => { if (!connected) return "Not connected"; - if (status?.providerLabel) return status.providerLabel; + const llmLabel = llmProviderLabelForStatus(status); + if (llmLabel) return llmLabel; if (status?.connectionMethod === "oauth") return "Connected through OAuth"; if (status?.source === "env") return "Connected through .env"; return "Connected through API key"; - }, [connected, status?.connectionMethod, status?.providerLabel, status?.source]); + }, [connected, status]); return ( -
+
{brand}
@@ -197,7 +441,7 @@ function ServiceCard({ {description}

-
+
- - {helperLabel} {helperDescription} - - + {helperHref && helperLabel ? ( + + {helperLabel} + {helperDescription ? ` ${helperDescription}` : null} + + + ) : helperLabel ? ( +

+ {helperLabel}:{" "} + {helperDescription} +

+ ) : null}
); @@ -233,31 +485,54 @@ function ServiceCard({ function ApiKeyModal({ service, + status, onClose, onSaved, }: { service: "tinyfish" | "llm"; + status: LocalSetupStatus | null; onClose: () => void; onSaved: (status: LocalSetupStatus) => void; }) { + const initialProvider = initialLlmProviderSelection(status); const [apiKey, setApiKey] = useState(""); - const [provider, setProvider] = useState("openrouter"); - const [baseUrl, setBaseUrl] = useState(""); + const [provider, setProvider] = + useState(initialProvider); + const [baseUrl, setBaseUrl] = useState(() => + initialBaseUrl(status, initialProvider), + ); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); const isTinyFish = service === "tinyfish"; const providerCopy = llmProviderOption(provider); + const providerStatuses = status?.services.llmProviders; + const resolvedProvider = providerCopy.provider; + const selectedProviderStatus = providerStatuses?.[resolvedProvider]; + const selectedProviderConfigured = + selectedProviderStatus?.configured ?? + (status?.services.llm.configured && + status.services.llm.provider === resolvedProvider) ?? + false; + const selectedRequiresBaseUrl = providerCopy.requiresBaseUrl ?? false; + const selectedRequiresApiKey = providerCopy.requiresApiKey ?? resolvedProvider !== "custom"; + const selectedUsesPresetBaseUrl = !!providerCopy.defaultBaseUrl; + const showOpenRouterOAuth = useCanUseOpenRouterOAuth(); + const isCustomEndpoint = provider === "custom"; - function handleProviderChange(next: LlmProviderType) { + function handleProviderChange(next: LlmProviderOptionValue) { setProvider(next); - setBaseUrl(""); + setBaseUrl(initialBaseUrl(status, next)); + setApiKey(""); + setError(null); } async function handleSubmit() { if (saving) return; if (isTinyFish && !apiKey.trim()) return; - if (!isTinyFish && provider !== "custom" && !apiKey.trim()) return; - if (!isTinyFish && provider === "custom" && !baseUrl.trim()) { + if (!isTinyFish && selectedRequiresApiKey && !apiKey.trim() && !selectedProviderConfigured) { + return; + } + if (!isTinyFish && selectedRequiresBaseUrl && !baseUrl.trim() && !selectedProviderConfigured) { setError("Custom providers require a base URL"); return; } @@ -268,10 +543,16 @@ function ApiKeyModal({ const next = isTinyFish ? await saveTinyFishApiKey(apiKey.trim()) : await saveLlmProviderConfig({ - provider, - apiKey: apiKey.trim(), - defaultModel: llmProviderOption(provider).defaultModel, - baseUrl: provider === "custom" ? baseUrl.trim() : undefined, + provider: resolvedProvider, + apiKey: + selectedRequiresApiKey || provider === "custom" + ? apiKey.trim() + : "", + defaultModel: providerCopy.defaultModel, + baseUrl: + selectedRequiresBaseUrl && baseUrl.trim() + ? baseUrl.trim() + : undefined, }); onSaved(next); } catch (err) { @@ -284,14 +565,33 @@ function ApiKeyModal({ const helperHref = isTinyFish ? "https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2" : providerCopy.helperHref; - const helperLabel = !isTinyFish && provider === "custom" ? "Provider docs" : "Get a key"; + const helperLabel = isTinyFish + ? "Get your TinyFish API Key" + : isCustomEndpoint + ? "OpenAI API docs" + : selectedRequiresBaseUrl + ? "Provider docs" + : "Get a key"; + const showApiKeyHelper = + !!helperHref && (isTinyFish || selectedRequiresApiKey); const canSubmit = !saving && (isTinyFish ? !!apiKey.trim() - : provider === "custom" - ? !!baseUrl.trim() - : !!apiKey.trim()); + : selectedRequiresBaseUrl + ? !!baseUrl.trim() || selectedProviderConfigured + : !selectedRequiresApiKey || !!apiKey.trim() || selectedProviderConfigured); + const usingSavedProvider = + !isTinyFish && + selectedProviderConfigured && + !apiKey.trim() && + (!selectedRequiresBaseUrl || + !baseUrl.trim() || + baseUrl.trim() === displayBaseUrl(selectedProviderStatus?.baseUrl)); + const modalTitle = isTinyFish ? "TinyFish API key" : "Model provider"; + const modalDescription = isTinyFish + ? "BigSet verifies the key and stores it in your OS keychain." + : "Select a provider. BigSet stores local credentials in your OS keychain."; return (
@@ -304,16 +604,14 @@ function ApiKeyModal({
-
+
-

- {isTinyFish ? "TinyFish API key" : "LLM provider"} -

-

- BigSet checks the provider endpoint and stores the key in your OS keychain. -

+

{modalTitle}

+

{modalDescription}

-
+
{!isTinyFish && (
- Provider - + + Provider + +
)} - {!isTinyFish && provider === "custom" && ( -
-
- - {helperLabel} - - - -
+
+
); } + +function initialLlmProviderSelection( + status: LocalSetupStatus | null, +): LlmProviderOptionValue { + if (status?.services.llm.configured && status.services.llm.provider) { + if (status.services.llm.provider === "custom") { + return ( + localLlmPresetForBaseUrl(status.services.llm.baseUrl)?.value ?? "custom" + ); + } + return status.services.llm.provider; + } + + const savedProvider = (Object.entries(status?.services.llmProviders ?? {}) as [ + LlmProviderType, + ServiceSetupStatus, + ][]).find(([, providerStatus]) => providerStatus.configured)?.[0]; + + if (savedProvider === "custom") { + return ( + localLlmPresetForBaseUrl(status?.services.llmProviders?.custom?.baseUrl) + ?.value ?? "custom" + ); + } + + return savedProvider ?? "openrouter"; +} + +function initialBaseUrl( + status: LocalSetupStatus | null, + provider: LlmProviderOptionValue, +) { + const option = llmProviderOption(provider); + const savedBaseUrl = displayBaseUrl( + status?.services.llmProviders?.[option.provider]?.baseUrl, + ); + if (option.defaultBaseUrl) return savedBaseUrl || option.defaultBaseUrl; + if (option.provider === "custom") { + return savedBaseUrl; + } + return ""; +} diff --git a/frontend/components/settings/LocalCredentialsPanel.tsx b/frontend/components/settings/LocalCredentialsPanel.tsx index cbd4132..772f3e8 100644 --- a/frontend/components/settings/LocalCredentialsPanel.tsx +++ b/frontend/components/settings/LocalCredentialsPanel.tsx @@ -20,46 +20,52 @@ import { isLocalMode } from "@/lib/app-mode"; import { LlmProviderBrand, LlmProviderSelector, + displayBaseUrl, + llmProviderLabelForStatus, llmProviderOption, + localLlmPresetForBaseUrl, + type LlmProviderOptionValue, } from "@/components/settings/llm-providers"; +import { + beginOpenRouterOAuth, + useCanUseOpenRouterOAuth, +} from "@/lib/openrouter-oauth"; type ServiceName = "tinyfish" | "llm"; -const SERVICE_COPY = { +type ServiceCopy = { + modalTitle: string; + description: string; + inputPlaceholder: string; + modalDescription: string; + helperHref?: string; + helperLabel: string; + helperDescription: string; +}; + +const SERVICE_COPY: Record = { tinyfish: { - modalTitle: "TinyFish API key", + modalTitle: "Connect TinyFish", description: - "BigSet uses TinyFish's best-in-class search API to unlock real-time information.", + "Connect TinyFish for live search and source pages.", inputPlaceholder: "tf_...", modalDescription: "BigSet verifies the key and stores it in your OS keychain.", helperHref: "https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2", - helperLabel: "Need a TinyFish key?", - helperDescription: "Open the TinyFish API keys page", + helperLabel: "Get your TinyFish API Key", + helperDescription: "", }, llm: { - modalTitle: "LLM provider", + modalTitle: "Model provider", description: - "BigSet uses your LLM provider for schema generation and dataset-building agents.", + "Choose the provider BigSet uses for schema generation and agents.", inputPlaceholder: "API key", modalDescription: - "BigSet checks the provider endpoint and stores the key in your OS keychain.", - helperHref: "https://platform.openai.com/api-keys", - helperLabel: "Bring your own model", - helperDescription: "OpenAI, Anthropic, OpenRouter, or custom", + "Select a provider. BigSet stores local credentials in your OS keychain.", + helperLabel: "", + helperDescription: "", }, -} satisfies Record< - ServiceName, - { - modalTitle: string; - description: string; - inputPlaceholder: string; - modalDescription: string; - helperHref: string; - helperLabel: string; - helperDescription: string; - } ->; +}; export function LocalCredentialsPanel({ onStatusChange, @@ -119,7 +125,7 @@ export function LocalCredentialsPanel({ {loadError}
) : ( -
+
setModal(null)} onSaved={(next) => { setStatus(next); @@ -164,13 +171,25 @@ function CredentialCard({ const copy = SERVICE_COPY[service]; const connected = status?.configured ?? false; const detail = useCredentialDetail(status, loading); + const primaryLabel = + service === "llm" + ? connected + ? "Update provider" + : "Choose provider" + : connected + ? "Update key" + : "Add API key"; return ( -
+
- +

{detail}

@@ -181,24 +200,34 @@ function CredentialCard({ {copy.description}

-
+
- - {copy.helperLabel} {copy.helperDescription} - - + {copy.helperHref && copy.helperLabel ? ( + + {copy.helperLabel} + {copy.helperDescription ? ` ${copy.helperDescription}` : null} + + + ) : copy.helperLabel ? ( +

+ + {copy.helperLabel}: + {" "} + {copy.helperDescription} +

+ ) : null}
); @@ -207,9 +236,11 @@ function CredentialCard({ function ServiceBrand({ service, provider, + baseUrl, }: { service: ServiceName; provider?: LlmProviderType; + baseUrl?: string; }) { if (service === "tinyfish") { return ( @@ -228,7 +259,7 @@ function ServiceBrand({ ); } - return ; + return ; } function StatusLabel({ @@ -264,41 +295,65 @@ function useCredentialDetail( return useMemo(() => { if (loading) return "Checking connection..."; if (!status?.configured) return "Not connected"; - if (status.providerLabel) return status.providerLabel; + const llmLabel = llmProviderLabelForStatus(status); + if (llmLabel) return llmLabel; if (status.connectionMethod === "oauth") return "Connected through OAuth"; if (status.source === "env") return "Connected through .env"; return "Connected through API key"; - }, [loading, status?.configured, status?.connectionMethod, status?.providerLabel, status?.source]); + }, [loading, status]); } function ApiKeyModal({ service, + status, onClose, onSaved, }: { service: ServiceName; + status: LocalSetupStatus | null; onClose: () => void; onSaved: (status: LocalSetupStatus) => void; }) { + const initialProvider = initialLlmProviderSelection(status); const [apiKey, setApiKey] = useState(""); - const [provider, setProvider] = useState("openrouter"); - const [baseUrl, setBaseUrl] = useState(""); + const [provider, setProvider] = + useState(initialProvider); + const [baseUrl, setBaseUrl] = useState(() => + initialBaseUrl(status, initialProvider), + ); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); const copy = SERVICE_COPY[service]; const isTinyFish = service === "tinyfish"; const providerCopy = llmProviderOption(provider); - - function handleProviderChange(next: LlmProviderType) { + const providerStatuses = status?.services.llmProviders; + const resolvedProvider = providerCopy.provider; + const selectedProviderStatus = providerStatuses?.[resolvedProvider]; + const selectedProviderConfigured = + selectedProviderStatus?.configured ?? + (status?.services.llm.configured && + status.services.llm.provider === resolvedProvider) ?? + false; + const selectedRequiresBaseUrl = providerCopy.requiresBaseUrl ?? false; + const selectedRequiresApiKey = providerCopy.requiresApiKey ?? resolvedProvider !== "custom"; + const selectedUsesPresetBaseUrl = !!providerCopy.defaultBaseUrl; + const showOpenRouterOAuth = useCanUseOpenRouterOAuth(); + const isCustomEndpoint = provider === "custom"; + + function handleProviderChange(next: LlmProviderOptionValue) { setProvider(next); - setBaseUrl(""); + setBaseUrl(initialBaseUrl(status, next)); + setApiKey(""); + setError(null); } async function handleSubmit() { if (saving) return; if (isTinyFish && !apiKey.trim()) return; - if (!isTinyFish && provider !== "custom" && !apiKey.trim()) return; - if (!isTinyFish && provider === "custom" && !baseUrl.trim()) { + if (!isTinyFish && selectedRequiresApiKey && !apiKey.trim() && !selectedProviderConfigured) { + return; + } + if (!isTinyFish && selectedRequiresBaseUrl && !baseUrl.trim() && !selectedProviderConfigured) { setError("Custom providers require a base URL"); return; } @@ -309,10 +364,16 @@ function ApiKeyModal({ const next = isTinyFish ? await saveTinyFishApiKey(apiKey.trim()) : await saveLlmProviderConfig({ - provider, - apiKey: apiKey.trim(), - defaultModel: llmProviderOption(provider).defaultModel, - baseUrl: provider === "custom" ? baseUrl.trim() : undefined, + provider: resolvedProvider, + apiKey: + selectedRequiresApiKey || provider === "custom" + ? apiKey.trim() + : "", + defaultModel: providerCopy.defaultModel, + baseUrl: + selectedRequiresBaseUrl && baseUrl.trim() + ? baseUrl.trim() + : undefined, }); onSaved(next); } catch (err) { @@ -323,14 +384,29 @@ function ApiKeyModal({ } const helperHref = isTinyFish ? copy.helperHref : providerCopy.helperHref; - const helperLabel = !isTinyFish && provider === "custom" ? "Provider docs" : "Get a key"; + const helperLabel = isTinyFish + ? "Get your TinyFish API Key" + : isCustomEndpoint + ? "OpenAI API docs" + : selectedRequiresBaseUrl + ? "Provider docs" + : "Get a key"; + const showApiKeyHelper = + !!helperHref && (isTinyFish || selectedRequiresApiKey); const canSubmit = !saving && (isTinyFish ? !!apiKey.trim() - : provider === "custom" - ? !!baseUrl.trim() - : !!apiKey.trim()); + : selectedRequiresBaseUrl + ? !!baseUrl.trim() || selectedProviderConfigured + : !selectedRequiresApiKey || !!apiKey.trim() || selectedProviderConfigured); + const usingSavedProvider = + !isTinyFish && + selectedProviderConfigured && + !apiKey.trim() && + (!selectedRequiresBaseUrl || + !baseUrl.trim() || + baseUrl.trim() === displayBaseUrl(selectedProviderStatus?.baseUrl)); return (
@@ -343,12 +419,16 @@ function ApiKeyModal({
-
+
-

{copy.modalTitle}

-

{copy.modalDescription}

+

{copy.modalTitle}

+

+ {copy.modalDescription} +

-
+
{!isTinyFish && (
- Provider - + + Provider + +
)} - {!isTinyFish && provider === "custom" && ( -
-
- - {helperLabel} - - - -
+
+
); } + +function initialLlmProviderSelection( + status: LocalSetupStatus | null, +): LlmProviderOptionValue { + if (status?.services.llm.configured && status.services.llm.provider) { + if (status.services.llm.provider === "custom") { + return ( + localLlmPresetForBaseUrl(status.services.llm.baseUrl)?.value ?? "custom" + ); + } + return status.services.llm.provider; + } + + const savedProvider = (Object.entries(status?.services.llmProviders ?? {}) as [ + LlmProviderType, + ServiceSetupStatus, + ][]).find(([, providerStatus]) => providerStatus.configured)?.[0]; + + if (savedProvider === "custom") { + return ( + localLlmPresetForBaseUrl(status?.services.llmProviders?.custom?.baseUrl) + ?.value ?? "custom" + ); + } + + return savedProvider ?? "openrouter"; +} + +function initialBaseUrl( + status: LocalSetupStatus | null, + provider: LlmProviderOptionValue, +) { + const option = llmProviderOption(provider); + const savedBaseUrl = displayBaseUrl( + status?.services.llmProviders?.[option.provider]?.baseUrl, + ); + if (option.defaultBaseUrl) return savedBaseUrl || option.defaultBaseUrl; + if (option.provider === "custom") { + return savedBaseUrl; + } + return ""; +} + +function currentReturnPath() { + if (typeof window === "undefined") return "/setup"; + return `${window.location.pathname}${window.location.search}`; +} diff --git a/frontend/components/settings/llm-providers.tsx b/frontend/components/settings/llm-providers.tsx index 64e87b6..3271f30 100644 --- a/frontend/components/settings/llm-providers.tsx +++ b/frontend/components/settings/llm-providers.tsx @@ -1,111 +1,450 @@ "use client"; -import type { LlmProviderType } from "@/lib/backend"; +import { useEffect, useState } from "react"; +import { CircleHelp, Plug, TriangleAlert } from "lucide-react"; +import type { LlmProviderType, ServiceSetupStatus } from "@/lib/backend"; + +type LlmProviderCategory = "direct" | "router" | "local" | "custom"; +export type LlmProviderOptionValue = LlmProviderType; export type LlmProviderOption = { - value: LlmProviderType; + value: LlmProviderOptionValue; + provider: LlmProviderType; label: string; description: string; + category: LlmProviderCategory; + shortLabel: string; + capability: string; + authLabel: string; defaultModel: string; + defaultBaseUrl?: string; + requiresBaseUrl?: boolean; + requiresApiKey?: boolean; apiKeyPlaceholder: string; helperHref: string; iconSrc?: string; wordmarkSrc?: string; }; -export const LLM_PROVIDER_OPTIONS: LlmProviderOption[] = [ +export const LLM_PROVIDER_GROUPS: { + categories: LlmProviderCategory[]; + label: string; +}[] = [ { - value: "openrouter", - label: "OpenRouter", - description: "Use OpenRouter model slugs.", - defaultModel: "anthropic/claude-sonnet-4.6", - apiKeyPlaceholder: "sk-or-...", - helperHref: "https://openrouter.ai/settings/keys", - iconSrc: "/logos/providers/openrouter.svg", - wordmarkSrc: "/logos/providers/openrouter-wordmark.svg", + categories: ["router"], + label: "Router", + }, + { + categories: ["direct"], + label: "Direct", }, + { + categories: ["local"], + label: "Local", + }, + { + categories: ["custom"], + label: "Custom", + }, +]; + +export const LLM_PROVIDER_OPTIONS: LlmProviderOption[] = [ { value: "openai", + provider: "openai", label: "OpenAI", - description: "Use an OpenAI API key directly.", + description: "Use OpenAI models directly with your API key.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", defaultModel: "gpt-5.4-mini", apiKeyPlaceholder: "sk-...", helperHref: "https://platform.openai.com/api-keys", - iconSrc: "/logos/providers/openai.svg", + iconSrc: "/logos/providers/openai-icon.svg", wordmarkSrc: "/logos/providers/openai.svg", }, { value: "anthropic", + provider: "anthropic", label: "Anthropic", - description: "Use a Claude API key directly.", + description: "Use Claude models directly with your API key.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", defaultModel: "claude-sonnet-4-6", apiKeyPlaceholder: "sk-ant-...", helperHref: "https://console.anthropic.com/settings/keys", - iconSrc: "/logos/providers/anthropic.svg", + iconSrc: "/logos/providers/anthropic-icon.svg", wordmarkSrc: "/logos/providers/anthropic.svg", }, { - value: "custom", - label: "Custom", - description: "Use LM Studio or any OpenAI-compatible base URL.", + value: "google", + provider: "google", + label: "Google Gemini", + description: "Use Gemini models directly with your Google AI Studio API key.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "gemini-3.5-flash", + apiKeyPlaceholder: "AIza...", + helperHref: "https://aistudio.google.com/app/apikey", + iconSrc: "/logos/providers/google-g.svg", + }, + { + value: "xai", + provider: "xai", + label: "xAI", + description: "Use Grok models directly with your xAI API key.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "grok-4.3", + apiKeyPlaceholder: "xai-...", + helperHref: "https://console.x.ai/", + iconSrc: "/logos/providers/xai.svg", + }, + { + value: "deepseek", + provider: "deepseek", + label: "DeepSeek", + description: "Use DeepSeek chat and reasoning models directly.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "deepseek-chat", + apiKeyPlaceholder: "sk-...", + helperHref: "https://platform.deepseek.com/api_keys", + iconSrc: "/logos/providers/deepseek.svg", + }, + { + value: "qwen", + provider: "qwen", + label: "Qwen", + description: "Use Qwen models through Alibaba Cloud Model Studio.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "qwen-plus", + apiKeyPlaceholder: "sk-...", + helperHref: "https://modelstudio.console.alibabacloud.com/", + iconSrc: "/logos/providers/qwen.svg", + }, + { + value: "mistral", + provider: "mistral", + label: "Mistral AI", + description: "Use Mistral chat and reasoning models directly.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "mistral-large-latest", + apiKeyPlaceholder: "sk-...", + helperHref: "https://console.mistral.ai/api-keys/", + iconSrc: "/logos/providers/mistral-ai.svg", + }, + { + value: "groq", + provider: "groq", + label: "Groq", + description: "Use fast hosted open-weight models through GroqCloud.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "openai/gpt-oss-120b", + apiKeyPlaceholder: "gsk_...", + helperHref: "https://console.groq.com/keys", + iconSrc: "/logos/providers/groq.svg", + }, + { + value: "togetherai", + provider: "togetherai", + label: "Together.ai", + description: "Use Together.ai serverless open-source model hosting.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "Qwen/Qwen3.5-397B-A17B", + apiKeyPlaceholder: "tok_...", + helperHref: "https://api.together.ai/settings/api-keys", + iconSrc: "/logos/providers/together-ai.svg", + }, + { + value: "deepinfra", + provider: "deepinfra", + label: "DeepInfra", + description: "Use DeepInfra's hosted open-source model catalog.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + apiKeyPlaceholder: "sk-...", + helperHref: "https://deepinfra.com/dash/api_keys", + iconSrc: "/logos/providers/deepinfra.svg", + }, + { + value: "fireworks", + provider: "fireworks", + label: "Fireworks AI", + description: "Use Fireworks-hosted open-weight and fine-tuned models.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "accounts/fireworks/models/kimi-k2p5", + apiKeyPlaceholder: "fw_...", + helperHref: "https://fireworks.ai/account/api-keys", + iconSrc: "/logos/providers/fireworks-ai.svg", + }, + { + value: "huggingface", + provider: "huggingface", + label: "Hugging Face", + description: "Use Hugging Face Inference Providers and routed models.", + category: "direct", + shortLabel: "Direct API", + capability: "Hosted", + authLabel: "API key", + defaultModel: "deepseek-ai/DeepSeek-V3-0324", + apiKeyPlaceholder: "hf_...", + helperHref: "https://huggingface.co/settings/tokens", + iconSrc: "/logos/providers/huggingface.svg", + }, + { + value: "openrouter", + provider: "openrouter", + label: "OpenRouter", + description: "Route across many hosted model families with one account.", + category: "router", + shortLabel: "Model router", + capability: "Multi-provider", + authLabel: "API key or OAuth", + defaultModel: "anthropic/claude-sonnet-4.6", + apiKeyPlaceholder: "sk-or-...", + helperHref: "https://openrouter.ai/settings/keys", + iconSrc: "/logos/providers/openrouter.svg", + wordmarkSrc: "/logos/providers/openrouter-wordmark.svg", + }, + { + value: "ollama", + provider: "ollama", + label: "Ollama", + description: "Use Ollama's local OpenAI-compatible endpoint.", + category: "local", + shortLabel: "Local", + capability: "Local", + authLabel: "No key", defaultModel: "", - apiKeyPlaceholder: "Optional — leave blank for LM Studio", + defaultBaseUrl: "http://localhost:11434/v1", + requiresBaseUrl: true, + requiresApiKey: false, + apiKeyPlaceholder: "No key required", + helperHref: "https://github.com/ollama/ollama/blob/main/docs/openai.md", + iconSrc: "/logos/providers/ollama.svg", + }, + { + value: "lmstudio", + provider: "lmstudio", + label: "LM Studio", + description: "Use LM Studio's local OpenAI-compatible server.", + category: "local", + shortLabel: "Local", + capability: "Local", + authLabel: "No key", + defaultModel: "", + defaultBaseUrl: "http://localhost:1234/v1", + requiresBaseUrl: true, + requiresApiKey: false, + apiKeyPlaceholder: "No key required", helperHref: "https://lmstudio.ai/docs/app/api/endpoints/openai", + iconSrc: "/logos/providers/lmstudio.svg", + }, + { + value: "custom", + provider: "custom", + label: "Custom endpoint", + description: "Use another OpenAI-compatible base URL.", + category: "custom", + shortLabel: "OpenAI-compatible", + capability: "Local or hosted", + authLabel: "Optional key", + defaultModel: "", + requiresBaseUrl: true, + requiresApiKey: false, + apiKeyPlaceholder: "Optional for local endpoints", + helperHref: "https://platform.openai.com/docs/api-reference", }, ]; -export function llmProviderOption(value: LlmProviderType) { +const EXPERIMENTAL_PROVIDER_VALUES = new Set([ + "openai", + "anthropic", + "google", + "xai", + "deepseek", + "qwen", + "mistral", + "groq", + "togetherai", + "deepinfra", + "fireworks", + "huggingface", + "ollama", + "lmstudio", + "custom", +]); + +const LOCAL_MODEL_PROVIDER_VALUES = new Set([ + "ollama", + "lmstudio", +]); + +function isExperimentalProvider(value: LlmProviderOptionValue) { + return EXPERIMENTAL_PROVIDER_VALUES.has(value); +} + +function isLocalModelProvider(value: LlmProviderOptionValue) { + return LOCAL_MODEL_PROVIDER_VALUES.has(value); +} + +export function llmProviderOption(value: LlmProviderOptionValue) { return ( LLM_PROVIDER_OPTIONS.find((option) => option.value === value) ?? + LLM_PROVIDER_OPTIONS.find((option) => option.value === "openrouter") ?? LLM_PROVIDER_OPTIONS[0] ); } +export function displayBaseUrl(baseUrl?: string) { + return baseUrl?.replace("host.docker.internal", "localhost") ?? ""; +} + +export function localLlmPresetForBaseUrl(baseUrl?: string) { + const displayUrl = displayBaseUrl(baseUrl); + if (!displayUrl) return undefined; + + try { + const parsed = new URL(displayUrl); + if (parsed.port === "11434") { + return llmProviderOption("ollama"); + } + if (parsed.port === "1234") { + return llmProviderOption("lmstudio"); + } + } catch { + if (displayUrl.includes(":11434")) return llmProviderOption("ollama"); + if (displayUrl.includes(":1234")) return llmProviderOption("lmstudio"); + } + + return undefined; +} + +export function llmProviderLabelForStatus(status?: ServiceSetupStatus) { + if (status?.provider === "custom") { + return localLlmPresetForBaseUrl(status.baseUrl)?.label ?? "Custom endpoint"; + } + if (status?.provider) return llmProviderOption(status.provider).label; + return status?.providerLabel; +} + export function LlmProviderLogo({ provider, variant = "wordmark", className = "", }: { - provider: LlmProviderType; + provider: LlmProviderOptionValue; variant?: "icon" | "wordmark"; className?: string; }) { const option = llmProviderOption(provider); + const src = option.iconSrc ?? option.wordmarkSrc; + + if (variant === "icon") { + if (!src) { + return ( + + ); + } - if (provider === "custom") { return ( - - Custom - + {option.label} ); } - const src = variant === "icon" ? option.iconSrc : option.wordmarkSrc; + if (!src) { + return ( + + + ); + } return ( - {option.label} + + + + {option.label} + + ); } -export function LlmProviderBrand({ provider }: { provider?: LlmProviderType }) { +export function LlmProviderBrand({ + provider, + baseUrl, +}: { + provider?: LlmProviderType; + baseUrl?: string; +}) { if (provider) { + const option = + provider === "custom" + ? localLlmPresetForBaseUrl(baseUrl) ?? llmProviderOption("custom") + : llmProviderOption(provider); + return (
- +
); } return (
- - AI - - LLM Provider + Model provider
); } @@ -114,43 +453,126 @@ export function LlmProviderSelector({ value, onChange, }: { - value: LlmProviderType; - onChange: (provider: LlmProviderType) => void; + value: LlmProviderOptionValue; + onChange: (provider: LlmProviderOptionValue) => void; }) { + const [showExperimentalProviders, setShowExperimentalProviders] = + useState(false); + const orderedOptions = LLM_PROVIDER_GROUPS.flatMap((group) => + LLM_PROVIDER_OPTIONS.filter((option) => + group.categories.includes(option.category), + ), + ).filter( + (option) => + showExperimentalProviders || !isExperimentalProvider(option.value), + ); + + useEffect(() => { + if (!showExperimentalProviders && isExperimentalProvider(value)) { + onChange("openrouter"); + } + }, [onChange, showExperimentalProviders, value]); + + function handleExperimentalChange(checked: boolean) { + setShowExperimentalProviders(checked); + if (!checked && isExperimentalProvider(value)) { + onChange("openrouter"); + } + } + return ( -
- {LLM_PROVIDER_OPTIONS.map((option) => { - const selected = option.value === value; - - return ( -
-

{option.description}

- - ); - })} +
+
); } diff --git a/frontend/convex/localCredentials.ts b/frontend/convex/localCredentials.ts index eeafd3f..cc9ac5c 100644 --- a/frontend/convex/localCredentials.ts +++ b/frontend/convex/localCredentials.ts @@ -7,6 +7,18 @@ const serviceValidator = v.union( v.literal("openrouter"), v.literal("openai"), v.literal("anthropic"), + v.literal("google"), + v.literal("xai"), + v.literal("deepseek"), + v.literal("qwen"), + v.literal("mistral"), + v.literal("groq"), + v.literal("togetherai"), + v.literal("deepinfra"), + v.literal("fireworks"), + v.literal("huggingface"), + v.literal("ollama"), + v.literal("lmstudio"), v.literal("custom"), ); @@ -19,6 +31,18 @@ const llmProviderValidator = v.union( v.literal("openrouter"), v.literal("openai"), v.literal("anthropic"), + v.literal("google"), + v.literal("xai"), + v.literal("deepseek"), + v.literal("qwen"), + v.literal("mistral"), + v.literal("groq"), + v.literal("togetherai"), + v.literal("deepinfra"), + v.literal("fireworks"), + v.literal("huggingface"), + v.literal("ollama"), + v.literal("lmstudio"), v.literal("custom"), ); diff --git a/frontend/convex/modelConfig.ts b/frontend/convex/modelConfig.ts index dc47991..5d7480d 100644 --- a/frontend/convex/modelConfig.ts +++ b/frontend/convex/modelConfig.ts @@ -3,12 +3,40 @@ import type { MutationCtx, QueryCtx } from "./_generated/server.js"; import { v } from "convex/values"; import { getIdentity } from "./lib/authz.js"; -type LlmProvider = "openrouter" | "openai" | "anthropic" | "custom"; +type LlmProvider = + | "openrouter" + | "openai" + | "anthropic" + | "google" + | "xai" + | "deepseek" + | "qwen" + | "mistral" + | "groq" + | "togetherai" + | "deepinfra" + | "fireworks" + | "huggingface" + | "ollama" + | "lmstudio" + | "custom"; const providerValidator = v.union( v.literal("openrouter"), v.literal("openai"), v.literal("anthropic"), + v.literal("google"), + v.literal("xai"), + v.literal("deepseek"), + v.literal("qwen"), + v.literal("mistral"), + v.literal("groq"), + v.literal("togetherai"), + v.literal("deepinfra"), + v.literal("fireworks"), + v.literal("huggingface"), + v.literal("ollama"), + v.literal("lmstudio"), v.literal("custom"), ); diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index 458dcc5..f002c21 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -140,6 +140,18 @@ export default defineSchema({ v.literal("openrouter"), v.literal("openai"), v.literal("anthropic"), + v.literal("google"), + v.literal("xai"), + v.literal("deepseek"), + v.literal("qwen"), + v.literal("mistral"), + v.literal("groq"), + v.literal("togetherai"), + v.literal("deepinfra"), + v.literal("fireworks"), + v.literal("huggingface"), + v.literal("ollama"), + v.literal("lmstudio"), v.literal("custom") ) ), @@ -157,6 +169,18 @@ export default defineSchema({ v.literal("openrouter"), v.literal("openai"), v.literal("anthropic"), + v.literal("google"), + v.literal("xai"), + v.literal("deepseek"), + v.literal("qwen"), + v.literal("mistral"), + v.literal("groq"), + v.literal("togetherai"), + v.literal("deepinfra"), + v.literal("fireworks"), + v.literal("huggingface"), + v.literal("ollama"), + v.literal("lmstudio"), v.literal("custom") ), keychainAccount: v.optional(v.string()), @@ -172,6 +196,18 @@ export default defineSchema({ v.literal("openrouter"), v.literal("openai"), v.literal("anthropic"), + v.literal("google"), + v.literal("xai"), + v.literal("deepseek"), + v.literal("qwen"), + v.literal("mistral"), + v.literal("groq"), + v.literal("togetherai"), + v.literal("deepinfra"), + v.literal("fireworks"), + v.literal("huggingface"), + v.literal("ollama"), + v.literal("lmstudio"), v.literal("custom") ) ), diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index 09f7951..6178dba 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -2,7 +2,7 @@ export interface InferredSchema { dataset_name: string; description: string; columns: InferredColumn[]; - primary_key: string; + primary_key: string[]; retrieval_strategy: "search_fetch" | "browser" | "hybrid"; source_hint: string; } @@ -63,7 +63,23 @@ export interface OpenRouterModel { promptCost: number; } -export type LlmProviderType = "openrouter" | "openai" | "anthropic" | "custom"; +export type LlmProviderType = + | "openrouter" + | "openai" + | "anthropic" + | "google" + | "xai" + | "deepseek" + | "qwen" + | "mistral" + | "groq" + | "togetherai" + | "deepinfra" + | "fireworks" + | "huggingface" + | "ollama" + | "lmstudio" + | "custom"; export interface ServiceSetupStatus { configured: boolean; @@ -149,7 +165,7 @@ export async function saveOpenRouterApiKey( return saveLlmProviderConfig({ provider: "openrouter", apiKey, - defaultModel: "openai/gpt-5.4-mini", + defaultModel: "anthropic/claude-sonnet-4.6", }); } @@ -208,7 +224,7 @@ export async function getModelConfig(token: string): Promise { + if (!/^\d{1,3}$/.test(part)) return Number.NaN; + const value = Number(part); + return value >= 0 && value <= 255 ? value : Number.NaN; + }); + if (octets.some(Number.isNaN)) return false; + + const [first, second] = octets; + if (first === 0 || first === 10 || first === 127 || first === 192) { + return true; + } + if (first === 100 && second >= 64 && second <= 127) return true; + if (first === 169 && second === 254) return true; + if (first === 172 && second >= 16 && second <= 31) return true; + if (first === 198 && (second === 18 || second === 19)) return true; + + return false; +} + +function isLocalIpv6Hostname(hostname: string): boolean { + if (!hostname.includes(":")) return false; + if (hostname === "::1" || hostname === "0:0:0:0:0:0:0:1") return true; + + const firstSegment = Number.parseInt(hostname.split(":")[0] || "0", 16); + if (Number.isNaN(firstSegment)) return false; + + return (firstSegment & 0xfe00) === 0xfc00 || (firstSegment & 0xffc0) === 0xfe80; +} + +export function isLocalOpenRouterOAuthHostname(hostname: string): boolean { + const normalized = normalizedHostname(hostname); + if (!normalized) return true; + if (LOCAL_HOSTNAMES.has(normalized)) return true; + if (LOCAL_HOST_SUFFIXES.some((suffix) => normalized.endsWith(suffix))) { + return true; + } + if (!normalized.includes(".") && !normalized.includes(":")) return true; + + return isLocalIpv4Hostname(normalized) || isLocalIpv6Hostname(normalized); +} + +export function canUseOpenRouterOAuth(): boolean { + if (typeof window === "undefined") return false; + return !isLocalOpenRouterOAuthHostname(window.location.hostname); +} + +function subscribeToOpenRouterOAuthAvailability() { + return () => {}; +} + +function unavailableOnServer() { + return false; +} + +export function useCanUseOpenRouterOAuth(): boolean { + return useSyncExternalStore( + subscribeToOpenRouterOAuthAvailability, + canUseOpenRouterOAuth, + unavailableOnServer, + ); +} + export async function beginOpenRouterOAuth(returnTo = "/setup") { + if (!canUseOpenRouterOAuth()) return; + const verifier = randomVerifier(); const challenge = base64Url(await sha256(verifier)); sessionStorage.setItem(OPENROUTER_VERIFIER_KEY, verifier); diff --git a/frontend/public/logos/providers/anthropic-icon.svg b/frontend/public/logos/providers/anthropic-icon.svg new file mode 100644 index 0000000..88dc745 --- /dev/null +++ b/frontend/public/logos/providers/anthropic-icon.svg @@ -0,0 +1 @@ +Anthropic \ No newline at end of file diff --git a/frontend/public/logos/providers/deepinfra.svg b/frontend/public/logos/providers/deepinfra.svg new file mode 100644 index 0000000..925c139 --- /dev/null +++ b/frontend/public/logos/providers/deepinfra.svg @@ -0,0 +1,29 @@ + + DeepInfra + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/public/logos/providers/deepseek.svg b/frontend/public/logos/providers/deepseek.svg new file mode 100644 index 0000000..117e952 --- /dev/null +++ b/frontend/public/logos/providers/deepseek.svg @@ -0,0 +1 @@ +DeepSeek diff --git a/frontend/public/logos/providers/fireworks-ai.svg b/frontend/public/logos/providers/fireworks-ai.svg new file mode 100644 index 0000000..06da2e7 --- /dev/null +++ b/frontend/public/logos/providers/fireworks-ai.svg @@ -0,0 +1 @@ +Fireworks AI diff --git a/frontend/public/logos/providers/google-g.svg b/frontend/public/logos/providers/google-g.svg new file mode 100644 index 0000000..40c9063 --- /dev/null +++ b/frontend/public/logos/providers/google-g.svg @@ -0,0 +1 @@ +Google Gemini diff --git a/frontend/public/logos/providers/groq.svg b/frontend/public/logos/providers/groq.svg new file mode 100644 index 0000000..49a43fa --- /dev/null +++ b/frontend/public/logos/providers/groq.svg @@ -0,0 +1 @@ +Groq diff --git a/frontend/public/logos/providers/huggingface.svg b/frontend/public/logos/providers/huggingface.svg new file mode 100644 index 0000000..5992e36 --- /dev/null +++ b/frontend/public/logos/providers/huggingface.svg @@ -0,0 +1 @@ +Hugging Face diff --git a/frontend/public/logos/providers/lmstudio.svg b/frontend/public/logos/providers/lmstudio.svg new file mode 100644 index 0000000..a2a179f --- /dev/null +++ b/frontend/public/logos/providers/lmstudio.svg @@ -0,0 +1 @@ +LM Studio \ No newline at end of file diff --git a/frontend/public/logos/providers/mistral-ai.svg b/frontend/public/logos/providers/mistral-ai.svg new file mode 100644 index 0000000..b0af170 --- /dev/null +++ b/frontend/public/logos/providers/mistral-ai.svg @@ -0,0 +1 @@ +Mistral AI diff --git a/frontend/public/logos/providers/ollama.svg b/frontend/public/logos/providers/ollama.svg new file mode 100644 index 0000000..432f73e --- /dev/null +++ b/frontend/public/logos/providers/ollama.svg @@ -0,0 +1 @@ +Ollama \ No newline at end of file diff --git a/frontend/public/logos/providers/openai-icon.svg b/frontend/public/logos/providers/openai-icon.svg new file mode 100644 index 0000000..ebbdab0 --- /dev/null +++ b/frontend/public/logos/providers/openai-icon.svg @@ -0,0 +1 @@ +OpenAI \ No newline at end of file diff --git a/frontend/public/logos/providers/qwen.svg b/frontend/public/logos/providers/qwen.svg new file mode 100644 index 0000000..fbf9232 --- /dev/null +++ b/frontend/public/logos/providers/qwen.svg @@ -0,0 +1 @@ +Qwen diff --git a/frontend/public/logos/providers/together-ai.svg b/frontend/public/logos/providers/together-ai.svg new file mode 100644 index 0000000..8d0f75b --- /dev/null +++ b/frontend/public/logos/providers/together-ai.svg @@ -0,0 +1 @@ +Together.ai diff --git a/frontend/public/logos/providers/xai.svg b/frontend/public/logos/providers/xai.svg new file mode 100644 index 0000000..d8e9f54 --- /dev/null +++ b/frontend/public/logos/providers/xai.svg @@ -0,0 +1 @@ +xAI From ef9de138ec9769936794b92e9697090018694a86 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Thu, 11 Jun 2026 14:43:18 -0700 Subject: [PATCH 03/16] testing row extraction --- backend/package-lock.json | 13 + backend/package.json | 1 + backend/src/mastra/tools/investigate-tool.ts | 35 ++ .../src/row-extractors/try-row-extractor.ts | 582 ++++++++++++++++++ 4 files changed, 631 insertions(+) create mode 100644 backend/src/row-extractors/try-row-extractor.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index e231b48..8239c4e 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -18,6 +18,7 @@ "dotenv": "^16.4.0", "fastify": "^5.0.0", "fastify-plugin": "^5.1.0", + "playwright-core": "^1.60.0", "posthog-node": "^5.35.1", "resend": "^6.12.3", "zod": "^4.4.3" @@ -7623,6 +7624,18 @@ "pathe": "^2.0.1" } }, + "node_modules/playwright-core": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz", + "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/postal-mime": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-2.7.4.tgz", diff --git a/backend/package.json b/backend/package.json index 11df78b..0e161cb 100644 --- a/backend/package.json +++ b/backend/package.json @@ -20,6 +20,7 @@ "dotenv": "^16.4.0", "fastify": "^5.0.0", "fastify-plugin": "^5.1.0", + "playwright-core": "^1.60.0", "posthog-node": "^5.35.1", "resend": "^6.12.3", "zod": "^4.4.3" diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 0139aa4..65fc7e2 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -6,6 +6,7 @@ import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; import { getSignal } from "../../abort-registry.js"; +import { tryRowExtractor } from "../../row-extractors/try-row-extractor.js"; const investigateInputSchema = z.object({ entity_hint: z @@ -100,6 +101,40 @@ export function buildSubagentTool( } if (metrics) metrics.investigateCalls++; + + const extractorResult = await tryRowExtractor({ + datasetId: authorizedDatasetId, + columns, + primaryKeys: primary_keys, + urls, + context, + }); + if (extractorResult.status === "inserted") { + if (metrics) metrics.rowsInserted++; + console.log( + `[run_subagent] row extractor inserted entity="${entity_hint}" reason="${extractorResult.reason}"`, + ); + return { + inserted: true, + reason: extractorResult.reason, + row_summary: extractorResult.rowSummary, + clues: undefined, + }; + } + if (/duplicate/i.test(extractorResult.reason)) { + return { + inserted: false, + reason: extractorResult.reason, + row_summary: undefined, + clues: undefined, + }; + } + if (extractorResult.status === "failed") { + console.warn( + `[run_subagent] row extractor failed entity="${entity_hint}" reason="${extractorResult.reason}"`, + ); + } + console.log( `[run_subagent] spawning subagent user=${authContext.authorizedUserId} run=${authContext.workflowRunId} dataset=${authorizedDatasetId} entity="${entity_hint}" pk=${JSON.stringify(primary_keys)}`, ); diff --git a/backend/src/row-extractors/try-row-extractor.ts b/backend/src/row-extractors/try-row-extractor.ts new file mode 100644 index 0000000..8f61112 --- /dev/null +++ b/backend/src/row-extractors/try-row-extractor.ts @@ -0,0 +1,582 @@ +import { chromium, type Browser, type Page } from "playwright-core"; + +import { getSignal } from "../abort-registry.js"; +import { convex, internal } from "../convex.js"; +import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; +import { getTinyFishApiKey, tinyFishHeaders } from "../local-credentials.js"; +import type { PopulateColumn } from "../pipeline/populate.js"; + +type ExtractorStatus = "inserted" | "miss" | "failed"; + +export interface TryRowExtractorInput { + datasetId: string; + columns: PopulateColumn[]; + primaryKeys: Record; + urls?: string[]; + context?: string; +} + +export interface TryRowExtractorResult { + status: ExtractorStatus; + reason: string; + rowSummary?: string; + sources?: string[]; +} + +interface TinyFishBrowserSession { + session_id: string; + cdp_url: string; + base_url: string; +} + +interface GitHubRepoFacts { + owner: string; + repo: string; + fullName: string; + url: string; + description?: string; + stars?: number; + forks?: number; + watchers?: number; + issues?: number; + pullRequests?: number; + language?: string; + license?: string; + latestCommitAt?: string; + updatedAt?: string; + createdAt?: string; + homepage?: string; + archived?: boolean; +} + +interface RawGitHubRepoDomFacts { + description?: string; + stars?: string; + forks?: string; + watchers?: string; + issues?: string; + pullRequests?: string; + language?: string; + license?: string; + latestCommitAt?: string; + homepage?: string; + archived?: boolean; +} + +const ENABLED_VALUES = new Set(["1", "true", "yes", "on"]); +const GITHUB_HOSTS = new Set(["github.com", "www.github.com"]); +const BROWSER_TIMEOUT_MS = 45_000; +const CDP_CONNECT_TIMEOUT_MS = 45_000; +const BROWSER_ATTEMPTS = 2; + +export async function tryRowExtractor( + input: TryRowExtractorInput, +): Promise { + if (!ENABLED_VALUES.has((process.env.ROW_EXTRACTORS_ENABLED ?? "").toLowerCase())) { + return { status: "miss", reason: "row extractors are disabled" }; + } + + const url = firstCandidateUrl(input); + if (!url) return { status: "miss", reason: "no URL primary key or candidate URL" }; + + const repoRef = parseGitHubRepoUrl(url); + if (!repoRef) { + return { status: "miss", reason: `unsupported URL host: ${safeHost(url)}` }; + } + + try { + const facts = await extractGitHubRepoFacts(url, input.datasetId); + const row = buildGitHubRow(input.columns, input.primaryKeys, facts); + if (!row) { + return { + status: "miss", + reason: "GitHub extractor could not satisfy all requested columns", + }; + } + + await convex.mutation(internal.datasetRows.insert, { + datasetId: input.datasetId, + data: row, + sources: [facts.url], + rowSummary: facts.description + ? `${facts.fullName}: ${facts.description}` + : facts.fullName, + howFound: + "Opened the GitHub repository URL with TinyFish Browser and extracted repository facts from the rendered page.", + }); + + return { + status: "inserted", + reason: "Inserted by GitHub row extractor", + rowSummary: facts.description + ? `${facts.fullName}: ${facts.description}` + : facts.fullName, + sources: [facts.url], + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/duplicate/i.test(msg)) { + return { + status: "miss", + reason: `${msg} Move on to the next entity.`, + }; + } + return { status: "failed", reason: msg }; + } +} + +function firstCandidateUrl(input: TryRowExtractorInput): string | undefined { + const fromPrimaryKey = Object.values(input.primaryKeys).find((value) => + isHttpUrl(value), + ); + if (fromPrimaryKey) return normalizeUrl(fromPrimaryKey); + + const fromUrls = input.urls?.find(isHttpUrl); + if (fromUrls) return normalizeUrl(fromUrls); + + const fromContext = input.context?.match(/https?:\/\/[^\s)>"']+/i)?.[0]; + return fromContext ? normalizeUrl(fromContext) : undefined; +} + +function normalizeUrl(value: string): string { + return value.trim().replace(/[.,;:]+$/, ""); +} + +function isHttpUrl(value: string | undefined): value is string { + if (!value) return false; + try { + const parsed = new URL(normalizeUrl(value)); + return parsed.protocol === "http:" || parsed.protocol === "https:"; + } catch { + return false; + } +} + +function safeHost(value: string): string { + try { + return new URL(value).host; + } catch { + return "invalid-url"; + } +} + +function parseGitHubRepoUrl(value: string): { owner: string; repo: string } | null { + try { + const url = new URL(value); + if (!GITHUB_HOSTS.has(url.hostname.toLowerCase())) return null; + const [owner, repo] = url.pathname + .split("/") + .filter(Boolean) + .map((part) => part.trim()); + if (!owner || !repo) return null; + if (["orgs", "topics", "marketplace", "features"].includes(owner)) return null; + return { owner, repo: repo.replace(/\.git$/i, "") }; + } catch { + return null; + } +} + +async function extractGitHubRepoFacts( + url: string, + datasetId: string, +): Promise { + const apiKey = await getTinyFishApiKey(); + if (!apiKey) throw new Error("TINYFISH_API_KEY is not configured"); + + let lastError: unknown; + for (let attempt = 1; attempt <= BROWSER_ATTEMPTS; attempt++) { + try { + return await extractGitHubRepoFactsOnce(apiKey, url, datasetId); + } catch (err) { + lastError = err; + if (getSignal(datasetId)?.aborted || attempt === BROWSER_ATTEMPTS) break; + const msg = err instanceof Error ? err.message : String(err); + console.warn( + `[row_extractor] GitHub browser attempt ${attempt} failed; retrying: ${msg}`, + ); + } + } + + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +async function extractGitHubRepoFactsOnce( + apiKey: string, + url: string, + datasetId: string, +): Promise { + const session = await createTinyFishBrowserSession(apiKey, url, datasetId); + let browser: Browser | undefined; + try { + browser = await chromium.connectOverCDP(session.cdp_url, { + timeout: CDP_CONNECT_TIMEOUT_MS, + }); + const context = browser.contexts()[0] ?? (await browser.newContext()); + const page = context.pages()[0] ?? (await context.newPage()); + await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: BROWSER_TIMEOUT_MS, + }); + await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => { + // GitHub may keep long-lived requests open. DOMContentLoaded is enough. + }); + return await readGitHubRepoFacts(page); + } finally { + await browser?.close().catch(() => undefined); + } +} + +async function createTinyFishBrowserSession( + apiKey: string, + url: string, + datasetId: string, +): Promise { + const response = await withRunTimeoutSignal(datasetId, FETCH_TIMEOUT_MS, (signal) => + fetch("https://agent.tinyfish.ai/v1/browser", { + method: "POST", + headers: { + ...tinyFishHeaders(apiKey), + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ url }), + signal, + }), + ); + + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error( + `TinyFish Browser returned HTTP ${response.status}: ${body.slice(0, 200)}`, + ); + } + + const data = (await response.json()) as Partial; + if (!data.session_id || !data.cdp_url || !data.base_url) { + throw new Error("TinyFish Browser response did not include CDP connection details"); + } + + return { + session_id: data.session_id, + cdp_url: data.cdp_url, + base_url: data.base_url, + }; +} + +async function withRunTimeoutSignal( + datasetId: string, + timeoutMs: number, + operation: (signal: AbortSignal) => Promise, +): Promise { + const runSignal = getSignal(datasetId); + if (runSignal?.aborted) throw new DOMException("Run was stopped", "AbortError"); + + const controller = new AbortController(); + const timeout = setTimeout( + () => controller.abort(new DOMException("Timed out", "TimeoutError")), + timeoutMs, + ); + const abortFromRun = () => + controller.abort(runSignal?.reason ?? new DOMException("Run was stopped", "AbortError")); + + runSignal?.addEventListener("abort", abortFromRun, { once: true }); + try { + return await operation(controller.signal); + } finally { + clearTimeout(timeout); + runSignal?.removeEventListener("abort", abortFromRun); + } +} + +async function readGitHubRepoFacts(page: Page): Promise { + const url = page.url(); + const repoRef = parseGitHubRepoUrl(url); + if (!repoRef) throw new Error(`Not a GitHub repository page: ${url}`); + + const facts = (await page.evaluate(` + (() => { + const text = (selector) => + document.querySelector(selector)?.textContent?.trim() || undefined; + const attr = (selector, name) => + document.querySelector(selector)?.getAttribute(name) || undefined; + const firstCandidateText = (selector, predicate) => + Array.from(document.querySelectorAll(selector)) + .map((el) => el.textContent?.trim()) + .filter(Boolean) + .find((value) => !predicate || predicate(value)); + const language = () => + text("[itemprop=\\"programmingLanguage\\"]") ?? + text("a[href*=\\"search?l=\\"] span.color-fg-default.text-bold") ?? + text("a[href*=\\"search?l=\\"] .text-bold"); + const license = () => + firstCandidateText( + "a[href*=\\"LICENSE\\"], a[href*=\\"license\\"], [data-testid*=\\"license\\"]", + (value) => /licensed|MIT|Apache|BSD|GPL|MPL|ISC/i.test(value), + ) ?? + firstCandidateText( + "a[href*=\\"LICENSE\\"], a[href*=\\"license\\"], [data-testid*=\\"license\\"]", + (value) => !/^(license|view license)$/i.test(value), + ) ?? + firstCandidateText( + "a[href*=\\"LICENSE\\"], a[href*=\\"license\\"], [data-testid*=\\"license\\"]", + ) ?? + text("svg.octicon-law + span"); + const bodyText = document.body?.innerText ?? ""; + + return { + description: + text("[data-pjax=\\"#repo-content-pjax-container\\"] [itemprop=\\"about\\"]") ?? + text("[itemprop=\\"about\\"]") ?? + attr("meta[name='description']", "content"), + stars: + text("#repo-stars-counter-star") ?? + text("a[href$='/stargazers'] strong") ?? + text("a[href$='/stargazers']"), + forks: + text("#repo-network-counter") ?? + text("a[href$='/forks'] strong") ?? + text("a[href$='/forks']"), + watchers: + text("a[href$='/watchers'] strong") ?? + text("a[href$='/watchers']"), + issues: + text("#issues-tab span.Counter") ?? + text("a[href$=\\"/issues\\"] span.Counter") ?? + text("a[data-tab-item=\\"i1issues-tab\\"] span.Counter"), + pullRequests: + text("#pull-requests-tab span.Counter") ?? + text("a[href$=\\"/pulls\\"] span.Counter") ?? + text("a[data-tab-item=\\"i2pull-requests-tab\\"] span.Counter"), + language: language(), + license: license(), + latestCommitAt: + attr("relative-time[datetime]", "datetime") ?? + attr("time-ago[datetime]", "datetime"), + homepage: attr("[itemprop='url']", "href"), + archived: /This repository has been archived/i.test(bodyText), + }; + })() + `)) as RawGitHubRepoDomFacts; + + const apiFacts = await fetchGitHubApiFacts(page, repoRef.owner, repoRef.repo).catch( + () => undefined, + ); + + return { + owner: repoRef.owner, + repo: repoRef.repo, + fullName: `${repoRef.owner}/${repoRef.repo}`, + url, + description: apiFacts?.description ?? cleanOptionalText(facts.description), + stars: apiFacts?.stars ?? parseCompactNumber(facts.stars), + forks: apiFacts?.forks ?? parseCompactNumber(facts.forks), + watchers: apiFacts?.watchers ?? parseCompactNumber(facts.watchers), + issues: parseCompactNumber(facts.issues) ?? apiFacts?.issues, + pullRequests: parseCompactNumber(facts.pullRequests) ?? apiFacts?.pullRequests, + language: apiFacts?.language ?? cleanOptionalText(facts.language), + license: apiFacts?.license ?? cleanOptionalText(facts.license), + latestCommitAt: apiFacts?.latestCommitAt ?? facts.latestCommitAt, + updatedAt: apiFacts?.updatedAt, + createdAt: apiFacts?.createdAt, + homepage: apiFacts?.homepage ?? cleanOptionalText(facts.homepage), + archived: apiFacts?.archived ?? facts.archived, + }; +} + +async function fetchGitHubApiFacts( + page: Page, + owner: string, + repo: string, +): Promise> { + const response = await page.request.get( + `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, + { + headers: { + Accept: "application/vnd.github+json", + }, + timeout: FETCH_TIMEOUT_MS, + }, + ); + if (!response.ok()) { + throw new Error(`GitHub API returned HTTP ${response.status()}`); + } + + const data = (await response.json()) as { + description?: string | null; + stargazers_count?: number; + forks_count?: number; + watchers_count?: number; + open_issues_count?: number; + language?: string | null; + license?: { spdx_id?: string | null; name?: string | null } | null; + pushed_at?: string | null; + updated_at?: string | null; + created_at?: string | null; + homepage?: string | null; + archived?: boolean; + html_url?: string; + }; + + return { + url: data.html_url, + description: data.description ?? undefined, + stars: data.stargazers_count, + forks: data.forks_count, + watchers: data.watchers_count, + issues: data.open_issues_count, + language: data.language ?? undefined, + license: data.license?.spdx_id || data.license?.name || undefined, + latestCommitAt: data.pushed_at ?? undefined, + updatedAt: data.updated_at ?? undefined, + createdAt: data.created_at ?? undefined, + homepage: data.homepage || undefined, + archived: data.archived, + }; +} + +function buildGitHubRow( + columns: PopulateColumn[], + primaryKeys: Record, + facts: GitHubRepoFacts, +): Record | null { + const row: Record = {}; + + for (const column of columns) { + const pkValue = findPrimaryKeyValue(column.name, primaryKeys); + const rawValue = pkValue ?? valueForGitHubColumn(column.name, facts); + const value = coerceColumnValue(rawValue, column); + if (value === undefined) return null; + row[column.name] = value; + } + + return row; +} + +function findPrimaryKeyValue( + columnName: string, + primaryKeys: Record, +): string | undefined { + if (primaryKeys[columnName]) return primaryKeys[columnName]; + const normalizedColumn = normalizeFieldName(columnName); + const entry = Object.entries(primaryKeys).find( + ([key]) => normalizeFieldName(key) === normalizedColumn, + ); + return entry?.[1]; +} + +function valueForGitHubColumn( + columnName: string, + facts: GitHubRepoFacts, +): string | number | boolean | undefined { + const normalized = normalizeFieldName(columnName); + if (matches(normalized, ["repository_url", "repo_url", "github_url", "url", "link"])) { + return facts.url; + } + if (matches(normalized, ["repository_name", "repo_name"])) { + return facts.fullName; + } + if (matches(normalized, ["repository", "repo", "name"])) { + return facts.repo; + } + if (matches(normalized, ["full_name", "repository_full_name", "repo_full_name"])) { + return facts.fullName; + } + if (matches(normalized, ["owner", "organization", "org", "user"])) { + return facts.owner; + } + if (matches(normalized, ["description", "summary", "about"])) { + return facts.description; + } + if (matches(normalized, ["stars", "star_count", "stargazers", "stargazer_count"])) { + return facts.stars; + } + if (matches(normalized, ["forks", "fork_count"])) { + return facts.forks; + } + if (matches(normalized, ["watchers", "watcher_count"])) { + return facts.watchers; + } + if (matches(normalized, ["issues", "open_issues", "open_issue_count"])) { + return facts.issues; + } + if (matches(normalized, ["pull_requests", "open_pull_requests", "prs", "open_prs", "pr_count", "open_pr_count"])) { + return facts.pullRequests; + } + if (matches(normalized, ["language", "primary_language"])) { + return facts.language; + } + if (matches(normalized, ["license", "license_type", "license_spdx"])) { + return facts.license; + } + if (matches(normalized, ["latest_commit", "latest_commit_at", "last_commit", "pushed_at", "activity", "last_activity"])) { + return facts.latestCommitAt; + } + if (matches(normalized, ["updated", "updated_at", "last_updated"])) { + return facts.updatedAt; + } + if (matches(normalized, ["created", "created_at"])) { + return facts.createdAt; + } + if (matches(normalized, ["homepage", "website", "site"])) { + return facts.homepage; + } + if (matches(normalized, ["archived", "is_archived"])) { + return facts.archived; + } + return undefined; +} + +function coerceColumnValue( + value: string | number | boolean | undefined, + column: PopulateColumn, +): string | number | boolean | undefined { + if (value === undefined || value === "") return undefined; + switch (column.type) { + case "number": { + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + const parsed = Number(String(value).replace(/,/g, "")); + return Number.isFinite(parsed) ? parsed : undefined; + } + case "boolean": + if (typeof value === "boolean") return value; + if (/^(true|yes)$/i.test(String(value))) return true; + if (/^(false|no)$/i.test(String(value))) return false; + return undefined; + case "url": + return isHttpUrl(String(value)) ? normalizeUrl(String(value)) : undefined; + case "date": { + const date = new Date(String(value)); + return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); + } + case "text": + return String(value).trim(); + } +} + +function parseCompactNumber(value: string | undefined): number | undefined { + if (!value) return undefined; + const match = value.replace(/,/g, "").match(/([\d.]+)\s*([kmb])?/i); + if (!match) return undefined; + const base = Number(match[1]); + if (!Number.isFinite(base)) return undefined; + const suffix = match[2]?.toLowerCase(); + const multiplier = suffix === "k" ? 1_000 : suffix === "m" ? 1_000_000 : suffix === "b" ? 1_000_000_000 : 1; + return Math.round(base * multiplier); +} + +function cleanOptionalText(value: string | undefined | null): string | undefined { + const trimmed = value?.trim(); + return trimmed || undefined; +} + +function normalizeFieldName(value: string): string { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +function matches(value: string, candidates: string[]): boolean { + return candidates.includes(value); +} From 919e1a36621a5f32c671cffb31244d545a7a034f Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Thu, 11 Jun 2026 16:15:33 -0700 Subject: [PATCH 04/16] improve row extraction, especially for updates, especially for concurrency --- backend/src/config/models.ts | 59 +++++++ backend/src/env.ts | 5 + backend/src/index.ts | 12 ++ backend/src/mastra/tools/investigate-tool.ts | 1 + backend/src/mastra/workflows/populate.ts | 2 + backend/src/mastra/workflows/update.ts | 62 ++++++- .../src/row-extractors/try-row-extractor.ts | 155 ++++++++++++++++-- .../app/dashboard/settings/models/page.tsx | 107 +++++++++++- frontend/app/setup/page.tsx | 4 +- frontend/convex/modelConfig.ts | 12 ++ frontend/convex/schema.ts | 2 + frontend/lib/backend.ts | 48 +++++- 12 files changed, 445 insertions(+), 24 deletions(-) diff --git a/backend/src/config/models.ts b/backend/src/config/models.ts index 3da1baa..9b94095 100644 --- a/backend/src/config/models.ts +++ b/backend/src/config/models.ts @@ -35,6 +35,46 @@ export const DEFAULT_MODEL_IDS = { INVESTIGATE_SUBAGENT: env.INVESTIGATE_SUBAGENT_MODEL, } as const; +const ROW_EXTRACTOR_CONCURRENCY_MIN = 1; +export const ROW_EXTRACTOR_CONCURRENCY_MAX = 100; +const ROW_EXTRACTOR_BROWSER_ATTEMPTS_MIN = 1; +export const ROW_EXTRACTOR_BROWSER_ATTEMPTS_MAX = 10; + +export function normalizeRowExtractorConcurrency(value: unknown): number { + return normalizeIntegerSetting( + value, + 5, + ROW_EXTRACTOR_CONCURRENCY_MIN, + ROW_EXTRACTOR_CONCURRENCY_MAX, + ); +} + +export function normalizeRowExtractorBrowserAttempts(value: unknown): number { + return normalizeIntegerSetting( + value, + 2, + ROW_EXTRACTOR_BROWSER_ATTEMPTS_MIN, + ROW_EXTRACTOR_BROWSER_ATTEMPTS_MAX, + ); +} + +function normalizeIntegerSetting( + value: unknown, + fallback: number, + min: number, + max: number, +): number { + const parsed = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(max, Math.max(min, Math.trunc(parsed))); +} + +const DEFAULT_ROW_EXTRACTOR_CONCURRENCY = normalizeRowExtractorConcurrency( + env.ROW_EXTRACTOR_CONCURRENCY, +); +const DEFAULT_ROW_EXTRACTOR_BROWSER_ATTEMPTS = + normalizeRowExtractorBrowserAttempts(env.ROW_EXTRACTOR_BROWSER_ATTEMPTS); + const OPENAI_MODEL_EXCLUDE_PATTERNS = [ "audio", "babbage", @@ -454,6 +494,8 @@ export async function upsertModelConfig( schemaInference?: string; populateOrchestrator?: string; investigateSubagent?: string; + rowExtractorConcurrency?: number; + rowExtractorBrowserAttempts?: number; } ): Promise { const llmConfig = await getLlmProviderConfig(); @@ -463,6 +505,14 @@ export async function upsertModelConfig( schemaInference: config.schemaInference ?? undefined, populateOrchestrator: config.populateOrchestrator ?? undefined, investigateSubagent: config.investigateSubagent ?? undefined, + rowExtractorConcurrency: + config.rowExtractorConcurrency !== undefined + ? normalizeRowExtractorConcurrency(config.rowExtractorConcurrency) + : undefined, + rowExtractorBrowserAttempts: + config.rowExtractorBrowserAttempts !== undefined + ? normalizeRowExtractorBrowserAttempts(config.rowExtractorBrowserAttempts) + : undefined, }); } @@ -477,6 +527,8 @@ export async function getModelConfig( schemaInference: string; populateOrchestrator: string; investigateSubagent: string; + rowExtractorConcurrency: number; + rowExtractorBrowserAttempts: number; }> { const llmConfig = await getLlmProviderConfig(); const config = await convex.query(internal.modelConfig.getInternal, { @@ -502,6 +554,13 @@ export async function getModelConfig( DEFAULT_MODEL_IDS.INVESTIGATE_SUBAGENT, llmConfig, ), + rowExtractorConcurrency: normalizeRowExtractorConcurrency( + config?.rowExtractorConcurrency ?? DEFAULT_ROW_EXTRACTOR_CONCURRENCY, + ), + rowExtractorBrowserAttempts: normalizeRowExtractorBrowserAttempts( + config?.rowExtractorBrowserAttempts ?? + DEFAULT_ROW_EXTRACTOR_BROWSER_ATTEMPTS, + ), }; } diff --git a/backend/src/env.ts b/backend/src/env.ts index c02511b..4e3db63 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -53,6 +53,11 @@ export const env = { process.env.POPULATE_ORCHESTRATOR_MODEL ?? "qwen/qwen3.7-max", INVESTIGATE_SUBAGENT_MODEL: process.env.INVESTIGATE_SUBAGENT_MODEL ?? "qwen/qwen3.7-max", + ROW_EXTRACTOR_CONCURRENCY: numberFromEnv("ROW_EXTRACTOR_CONCURRENCY", 5), + ROW_EXTRACTOR_BROWSER_ATTEMPTS: numberFromEnv( + "ROW_EXTRACTOR_BROWSER_ATTEMPTS", + 2, + ), // Resend (transactional email). Optional — when RESEND_API_KEY is unset // the email module no-ops with a log line, so local dev works without diff --git a/backend/src/index.ts b/backend/src/index.ts index b2aca72..36855e9 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1147,11 +1147,21 @@ await fastify.register(async (instance) => { schemaInference?: string | null; populateOrchestrator?: string | null; investigateSubagent?: string | null; + rowExtractorConcurrency?: number | null; + rowExtractorBrowserAttempts?: number | null; }; const config = { schemaInference: typeof body.schemaInference === "string" ? body.schemaInference.trim() || undefined : undefined, populateOrchestrator: typeof body.populateOrchestrator === "string" ? body.populateOrchestrator.trim() || undefined : undefined, investigateSubagent: typeof body.investigateSubagent === "string" ? body.investigateSubagent.trim() || undefined : undefined, + rowExtractorConcurrency: + typeof body.rowExtractorConcurrency === "number" + ? body.rowExtractorConcurrency + : undefined, + rowExtractorBrowserAttempts: + typeof body.rowExtractorBrowserAttempts === "number" + ? body.rowExtractorBrowserAttempts + : undefined, }; const toValidate: Array<{ role: "schemaInference" | "populateOrchestrator" | "investigateSubagent"; slug: string }> = []; @@ -1178,6 +1188,8 @@ await fastify.register(async (instance) => { schemaInference: config.schemaInference, populateOrchestrator: config.populateOrchestrator, investigateSubagent: config.investigateSubagent, + rowExtractorConcurrency: config.rowExtractorConcurrency, + rowExtractorBrowserAttempts: config.rowExtractorBrowserAttempts, }); return { success: true }; } catch (err) { diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 3c078d5..a1f7034 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -115,6 +115,7 @@ export function buildSubagentTool( primaryKeys: primaryKeyRecord, urls, context, + browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, }); if (extractorResult.status === "inserted") { if (metrics) metrics.rowsInserted++; diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index bd1c97e..7173425 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -37,6 +37,8 @@ export const authContextSchema = z.object({ schemaInference: z.string().min(1), populateOrchestrator: z.string().min(1), investigateSubagent: z.string().min(1), + rowExtractorConcurrency: z.number().int().min(1).max(100).default(5), + rowExtractorBrowserAttempts: z.number().int().min(1).max(10).default(2), }), isBenchmark: z.boolean().optional(), }); diff --git a/backend/src/mastra/workflows/update.ts b/backend/src/mastra/workflows/update.ts index 4bb8ada..39da763 100644 --- a/backend/src/mastra/workflows/update.ts +++ b/backend/src/mastra/workflows/update.ts @@ -8,6 +8,7 @@ import { requireLlmProviderConfig } from "../../local-credentials.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; import { getSignal } from "../../abort-registry.js"; +import { tryRefreshRowExtractor } from "../../row-extractors/try-row-extractor.js"; export const updateInputSchema = datasetContextSchema.extend({ authContext: authContextSchema, @@ -69,8 +70,6 @@ const refreshOutputSchema = z.object({ errors: z.number(), }); -const MAX_CONCURRENT = 5; - async function processWithConcurrency( items: T[], handler: (item: T) => Promise, @@ -100,17 +99,62 @@ const refreshRowsStep = createStep({ const metrics = new RunMetrics(); const startedAt = Date.now(); - const llmConfig = await requireLlmProviderConfig(); + let llmConfigPromise: ReturnType | undefined; + const getLlmConfig = () => { + llmConfigPromise ??= requireLlmProviderConfig(); + return llmConfigPromise; + }; const pkColumns = columns.filter((c) => c.isPrimaryKey); + const maxConcurrent = authContext.modelConfig.rowExtractorConcurrency; async function processRow(row: z.infer) { try { + const primaryKeyRecord = Object.fromEntries( + pkColumns + .map((column) => [column.name, String(row.data[column.name] ?? "").trim()]) + .filter(([, value]) => value.length > 0), + ); + + const extractorResult = await tryRefreshRowExtractor({ + datasetId, + rowId: row._id, + columns, + primaryKeys: primaryKeyRecord, + existingData: row.data, + urls: row.sources, + context: [row.rowSummary, row.howFound].filter(Boolean).join("\n"), + browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, + }); + + if (extractorResult.status === "updated") { + updatedCount++; + metrics.rowsUpdated++; + console.log( + `[refresh-rows] Row ${row._id}: updated=true via=row_extractor reason="${extractorResult.reason}"`, + ); + return; + } + + if (extractorResult.status === "unchanged") { + console.log( + `[refresh-rows] Row ${row._id}: updated=false via=row_extractor reason="${extractorResult.reason}"`, + ); + return; + } + + if (extractorResult.status === "failed") { + if (getSignal(datasetId)?.aborted) throwAbortError(); + console.warn( + `[refresh-rows] Row ${row._id}: row extractor failed; falling back to refresh agent: ${extractorResult.reason}`, + ); + } + const agent = buildRefreshAgent( datasetId, authContext, columns, - llmConfig, + await getLlmConfig(), ); const pkBlock = @@ -191,9 +235,9 @@ ${row.howFound ? `\nPreviously found via: ${row.howFound}` : ""}`; } console.log( - `[refresh-rows] Processing ${rows.length} rows (max ${MAX_CONCURRENT} concurrent)`, + `[refresh-rows] Processing ${rows.length} rows (max ${maxConcurrent} concurrent)`, ); - await processWithConcurrency(rows, processRow, MAX_CONCURRENT); + await processWithConcurrency(rows, processRow, maxConcurrent); const finishedAt = Date.now(); // If the run was stopped mid-update, workers exited early via AbortError. @@ -242,6 +286,12 @@ ${row.howFound ? `\nPreviously found via: ${row.howFound}` : ""}`; }, }); +function throwAbortError(): never { + const err = new Error("Run was stopped"); + err.name = "AbortError"; + throw err; +} + export const updateWorkflow = createWorkflow({ id: "update-workflow", inputSchema: updateInputSchema, diff --git a/backend/src/row-extractors/try-row-extractor.ts b/backend/src/row-extractors/try-row-extractor.ts index 8f61112..29dda9f 100644 --- a/backend/src/row-extractors/try-row-extractor.ts +++ b/backend/src/row-extractors/try-row-extractor.ts @@ -6,7 +6,7 @@ import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; import { getTinyFishApiKey, tinyFishHeaders } from "../local-credentials.js"; import type { PopulateColumn } from "../pipeline/populate.js"; -type ExtractorStatus = "inserted" | "miss" | "failed"; +type ExtractorStatus = "inserted" | "updated" | "unchanged" | "miss" | "failed"; export interface TryRowExtractorInput { datasetId: string; @@ -14,6 +14,12 @@ export interface TryRowExtractorInput { primaryKeys: Record; urls?: string[]; context?: string; + browserAttempts?: number; +} + +export interface TryRefreshRowExtractorInput extends TryRowExtractorInput { + rowId: string; + existingData: Record; } export interface TryRowExtractorResult { @@ -67,7 +73,11 @@ const ENABLED_VALUES = new Set(["1", "true", "yes", "on"]); const GITHUB_HOSTS = new Set(["github.com", "www.github.com"]); const BROWSER_TIMEOUT_MS = 45_000; const CDP_CONNECT_TIMEOUT_MS = 45_000; -const BROWSER_ATTEMPTS = 2; +const DEFAULT_BROWSER_ATTEMPTS = 2; +const GITHUB_EXTRACTOR_HOW_FOUND = + "Opened the GitHub repository URL with TinyFish Browser and extracted repository facts from the rendered page."; +const GITHUB_REFRESH_HOW_FOUND = + "Refreshed the GitHub repository URL with TinyFish Browser and extracted repository facts from the rendered page."; export async function tryRowExtractor( input: TryRowExtractorInput, @@ -85,7 +95,11 @@ export async function tryRowExtractor( } try { - const facts = await extractGitHubRepoFacts(url, input.datasetId); + const facts = await extractGitHubRepoFacts( + url, + input.datasetId, + input.browserAttempts, + ); const row = buildGitHubRow(input.columns, input.primaryKeys, facts); if (!row) { return { @@ -98,19 +112,14 @@ export async function tryRowExtractor( datasetId: input.datasetId, data: row, sources: [facts.url], - rowSummary: facts.description - ? `${facts.fullName}: ${facts.description}` - : facts.fullName, - howFound: - "Opened the GitHub repository URL with TinyFish Browser and extracted repository facts from the rendered page.", + rowSummary: githubRowSummary(facts), + howFound: GITHUB_EXTRACTOR_HOW_FOUND, }); return { status: "inserted", reason: "Inserted by GitHub row extractor", - rowSummary: facts.description - ? `${facts.fullName}: ${facts.description}` - : facts.fullName, + rowSummary: githubRowSummary(facts), sources: [facts.url], }; } catch (err) { @@ -125,6 +134,66 @@ export async function tryRowExtractor( } } +export async function tryRefreshRowExtractor( + input: TryRefreshRowExtractorInput, +): Promise { + if (!ENABLED_VALUES.has((process.env.ROW_EXTRACTORS_ENABLED ?? "").toLowerCase())) { + return { status: "miss", reason: "row extractors are disabled" }; + } + + const url = firstCandidateUrl(input); + if (!url) return { status: "miss", reason: "no URL primary key or candidate URL" }; + + const repoRef = parseGitHubRepoUrl(url); + if (!repoRef) { + return { status: "miss", reason: `unsupported URL host: ${safeHost(url)}` }; + } + + try { + const facts = await extractGitHubRepoFacts( + url, + input.datasetId, + input.browserAttempts, + ); + const row = buildGitHubRow(input.columns, input.primaryKeys, facts); + if (!row) { + return { + status: "miss", + reason: "GitHub extractor could not satisfy all requested columns", + }; + } + + const changedColumns = changedColumnNames(row, input.existingData, input.columns); + if (changedColumns.length === 0) { + return { + status: "unchanged", + reason: "Verified unchanged by GitHub row extractor", + rowSummary: githubRowSummary(facts), + sources: [facts.url], + }; + } + + await convex.mutation(internal.datasetRows.update, { + id: input.rowId, + expectedDatasetId: input.datasetId, + data: row, + sources: [facts.url], + rowSummary: githubRowSummary(facts), + howFound: GITHUB_REFRESH_HOW_FOUND, + }); + + return { + status: "updated", + reason: `Updated by GitHub row extractor (${changedColumns.join(", ")})`, + rowSummary: githubRowSummary(facts), + sources: [facts.url], + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { status: "failed", reason: msg }; + } +} + function firstCandidateUrl(input: TryRowExtractorInput): string | undefined { const fromPrimaryKey = Object.values(input.primaryKeys).find((value) => isHttpUrl(value), @@ -179,20 +248,22 @@ function parseGitHubRepoUrl(value: string): { owner: string; repo: string } | nu async function extractGitHubRepoFacts( url: string, datasetId: string, + browserAttempts: number | undefined, ): Promise { const apiKey = await getTinyFishApiKey(); if (!apiKey) throw new Error("TINYFISH_API_KEY is not configured"); + const attempts = normalizedBrowserAttempts(browserAttempts); let lastError: unknown; - for (let attempt = 1; attempt <= BROWSER_ATTEMPTS; attempt++) { + for (let attempt = 1; attempt <= attempts; attempt++) { try { return await extractGitHubRepoFactsOnce(apiKey, url, datasetId); } catch (err) { lastError = err; - if (getSignal(datasetId)?.aborted || attempt === BROWSER_ATTEMPTS) break; + if (getSignal(datasetId)?.aborted || attempt === attempts) break; const msg = err instanceof Error ? err.message : String(err); console.warn( - `[row_extractor] GitHub browser attempt ${attempt} failed; retrying: ${msg}`, + `[row_extractor] GitHub browser attempt ${attempt}/${attempts} failed; retrying: ${msg}`, ); } } @@ -200,6 +271,13 @@ async function extractGitHubRepoFacts( throw lastError instanceof Error ? lastError : new Error(String(lastError)); } +function normalizedBrowserAttempts(value: number | undefined): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + return DEFAULT_BROWSER_ATTEMPTS; + } + return Math.min(10, Math.max(1, Math.trunc(value))); +} + async function extractGitHubRepoFactsOnce( apiKey: string, url: string, @@ -452,6 +530,55 @@ function buildGitHubRow( return row; } +function githubRowSummary(facts: GitHubRepoFacts): string { + return facts.description ? `${facts.fullName}: ${facts.description}` : facts.fullName; +} + +function changedColumnNames( + nextRow: Record, + existingData: Record, + columns: PopulateColumn[], +): string[] { + return columns + .filter((column) => !valuesEqualForColumn(nextRow[column.name], existingData[column.name], column)) + .map((column) => column.name); +} + +function valuesEqualForColumn( + nextValue: string | number | boolean | undefined, + existingValue: unknown, + column: PopulateColumn, +): boolean { + if (nextValue === undefined) return existingValue === undefined || existingValue === ""; + + switch (column.type) { + case "number": { + const existingNumber = + typeof existingValue === "number" + ? existingValue + : Number(String(existingValue ?? "").replace(/,/g, "")); + return Number.isFinite(existingNumber) && existingNumber === nextValue; + } + case "boolean": + if (typeof existingValue === "boolean") return existingValue === nextValue; + if (/^(true|yes)$/i.test(String(existingValue))) return nextValue === true; + if (/^(false|no)$/i.test(String(existingValue))) return nextValue === false; + return false; + case "date": { + const nextDate = new Date(String(nextValue)); + const existingDate = new Date(String(existingValue ?? "")); + return ( + Number.isFinite(nextDate.getTime()) && + Number.isFinite(existingDate.getTime()) && + nextDate.getTime() === existingDate.getTime() + ); + } + case "url": + case "text": + return String(existingValue ?? "").trim() === String(nextValue).trim(); + } +} + function findPrimaryKeyValue( columnName: string, primaryKeys: Record, diff --git a/frontend/app/dashboard/settings/models/page.tsx b/frontend/app/dashboard/settings/models/page.tsx index f376c89..47d695e 100644 --- a/frontend/app/dashboard/settings/models/page.tsx +++ b/frontend/app/dashboard/settings/models/page.tsx @@ -42,6 +42,9 @@ export default function ModelSettingsPage() { ); const activeModelListCacheKeyRef = useRef(activeModelListCacheKey); const [isSavingModel, setIsSavingModel] = useState(false); + const [isSavingExtractorSettings, setIsSavingExtractorSettings] = useState(false); + const [extractorConcurrency, setExtractorConcurrency] = useState(5); + const [extractorAttempts, setExtractorAttempts] = useState(2); const [modelConfigReloadKey, setModelConfigReloadKey] = useState(0); const needsOpenRouterCache = !isLocalMode || llmProvider === "openrouter"; @@ -97,6 +100,12 @@ export default function ModelSettingsPage() { .catch(() => setLlmProvider("openrouter")); }, [syncLlmProvider]); + useEffect(() => { + if (!effectiveConfig) return; + setExtractorConcurrency(effectiveConfig.rowExtractorConcurrency); + setExtractorAttempts(effectiveConfig.rowExtractorBrowserAttempts); + }, [effectiveConfig]); + const models: OpenRouterModel[] = convexModels ? convexModels.map((m) => ({ modelName: m.modelName, @@ -114,7 +123,7 @@ export default function ModelSettingsPage() { : []; function getSelectedModel(role: ModelRole): string { - return effectiveConfig?.[role.key as keyof typeof effectiveConfig] ?? ""; + return String(effectiveConfig?.[role.key as keyof typeof effectiveConfig] ?? ""); } async function saveModelForRole(role: ModelRole, modelId: string) { @@ -137,6 +146,34 @@ export default function ModelSettingsPage() { } } + async function saveExtractorSettings() { + setIsSavingExtractorSettings(true); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + await saveModelConfig( + { + rowExtractorConcurrency: extractorConcurrency, + rowExtractorBrowserAttempts: extractorAttempts, + }, + token, + ); + setEffectiveConfig((prev: EffectiveModelConfig | null) => + prev + ? { + ...prev, + rowExtractorConcurrency: extractorConcurrency, + rowExtractorBrowserAttempts: extractorAttempts, + } + : prev, + ); + } catch { + // we will add toast later + } finally { + setIsSavingExtractorSettings(false); + } + } + function openSideSheet(role: ModelRole) { const cacheKey = activeModelListCacheKeyRef.current || activeModelListCacheKey; if (sheetModels.length === 0 || sheetModelsCacheKey !== cacheKey) { @@ -157,7 +194,7 @@ export default function ModelSettingsPage() { const navItems = [ { - label: "Models", + label: "General", href: "/dashboard/settings/models", icon: ( @@ -219,6 +256,72 @@ export default function ModelSettingsPage() { )) )}
+ + {isLocalMode && !isLoading && ( +
+ + +
+ + + +
+ +
+ +
+
+ )}
{activeSheet && ( diff --git a/frontend/app/setup/page.tsx b/frontend/app/setup/page.tsx index 928d861..bc5217f 100644 --- a/frontend/app/setup/page.tsx +++ b/frontend/app/setup/page.tsx @@ -57,6 +57,8 @@ function emptyModelConfig(): EffectiveModelConfig { schemaInference: "", populateOrchestrator: "", investigateSubagent: "", + rowExtractorConcurrency: 5, + rowExtractorBrowserAttempts: 2, }; } @@ -166,7 +168,7 @@ export default function SetupPage() { function modelForRole(role: ModelRole): string { const key = role.key as keyof EffectiveModelConfig; - return modelConfig?.[key] ?? ""; + return String(modelConfig?.[key] ?? ""); } function openModelSheet(role: ModelRole) { diff --git a/frontend/convex/modelConfig.ts b/frontend/convex/modelConfig.ts index 5d7480d..83cd9f5 100644 --- a/frontend/convex/modelConfig.ts +++ b/frontend/convex/modelConfig.ts @@ -91,6 +91,8 @@ export const upsert = mutation({ schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), + rowExtractorConcurrency: v.optional(v.number()), + rowExtractorBrowserAttempts: v.optional(v.number()), }, handler: async (ctx, args) => { const identity = await getIdentity(ctx); @@ -104,10 +106,14 @@ export const upsert = mutation({ schemaInference?: string; populateOrchestrator?: string; investigateSubagent?: string; + rowExtractorConcurrency?: number; + rowExtractorBrowserAttempts?: number; } = { provider }; if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; + if (args.rowExtractorConcurrency !== undefined) patch.rowExtractorConcurrency = args.rowExtractorConcurrency; + if (args.rowExtractorBrowserAttempts !== undefined) patch.rowExtractorBrowserAttempts = args.rowExtractorBrowserAttempts; if (existing) { await ctx.db.patch(existing._id, patch); @@ -143,6 +149,8 @@ export const upsertInternal = internalMutation({ schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), + rowExtractorConcurrency: v.optional(v.number()), + rowExtractorBrowserAttempts: v.optional(v.number()), }, handler: async (ctx, args) => { const provider = args.provider ?? "openrouter"; @@ -153,10 +161,14 @@ export const upsertInternal = internalMutation({ schemaInference?: string; populateOrchestrator?: string; investigateSubagent?: string; + rowExtractorConcurrency?: number; + rowExtractorBrowserAttempts?: number; } = { provider }; if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; + if (args.rowExtractorConcurrency !== undefined) patch.rowExtractorConcurrency = args.rowExtractorConcurrency; + if (args.rowExtractorBrowserAttempts !== undefined) patch.rowExtractorBrowserAttempts = args.rowExtractorBrowserAttempts; if (existing) { await ctx.db.patch(existing._id, patch); diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index f002c21..b978e8f 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -158,6 +158,8 @@ export default defineSchema({ schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), + rowExtractorConcurrency: v.optional(v.number()), + rowExtractorBrowserAttempts: v.optional(v.number()), }) .index("by_user", ["userId"]) .index("by_user_provider", ["userId", "provider"]), diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index 6178dba..c05f54b 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -43,6 +43,8 @@ export interface EffectiveModelConfig { schemaInference: string; populateOrchestrator: string; investigateSubagent: string; + rowExtractorConcurrency: number; + rowExtractorBrowserAttempts: number; } /** @@ -53,6 +55,8 @@ export interface SavedModelConfig { schemaInference: string | null; populateOrchestrator: string | null; investigateSubagent: string | null; + rowExtractorConcurrency: number | null; + rowExtractorBrowserAttempts: number | null; } export interface OpenRouterModel { @@ -106,6 +110,48 @@ export interface LocalSetupStatus { const BACKEND_URL = process.env.NEXT_PUBLIC_BACKEND_URL || "http://localhost:3501"; +const DEFAULT_ROW_EXTRACTOR_CONCURRENCY = 5; +const DEFAULT_ROW_EXTRACTOR_BROWSER_ATTEMPTS = 2; + +function normalizeIntegerSetting( + value: unknown, + fallback: number, + min: number, + max: number, +): number { + const parsed = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(max, Math.max(min, Math.trunc(parsed))); +} + +function normalizeEffectiveModelConfig( + config: Partial | null | undefined, +): EffectiveModelConfig { + return { + schemaInference: + typeof config?.schemaInference === "string" ? config.schemaInference : "", + populateOrchestrator: + typeof config?.populateOrchestrator === "string" + ? config.populateOrchestrator + : "", + investigateSubagent: + typeof config?.investigateSubagent === "string" + ? config.investigateSubagent + : "", + rowExtractorConcurrency: normalizeIntegerSetting( + config?.rowExtractorConcurrency, + DEFAULT_ROW_EXTRACTOR_CONCURRENCY, + 1, + 100, + ), + rowExtractorBrowserAttempts: normalizeIntegerSetting( + config?.rowExtractorBrowserAttempts, + DEFAULT_ROW_EXTRACTOR_BROWSER_ATTEMPTS, + 1, + 10, + ), + }; +} async function errorMessage(res: Response): Promise { const body = await res.json().catch(() => null); @@ -214,7 +260,7 @@ export async function getModelConfig(token: string): Promise Date: Thu, 11 Jun 2026 16:25:39 -0700 Subject: [PATCH 05/16] Fix row extractor settings initialization --- frontend/app/dashboard/settings/models/page.tsx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/frontend/app/dashboard/settings/models/page.tsx b/frontend/app/dashboard/settings/models/page.tsx index 47d695e..eb3bdf3 100644 --- a/frontend/app/dashboard/settings/models/page.tsx +++ b/frontend/app/dashboard/settings/models/page.tsx @@ -79,7 +79,11 @@ export default function ModelSettingsPage() { return getModelConfig(token); }) .then((config) => { - if (active) setEffectiveConfig(config); + if (active) { + setEffectiveConfig(config); + setExtractorConcurrency(config.rowExtractorConcurrency); + setExtractorAttempts(config.rowExtractorBrowserAttempts); + } }) .catch(() => { if (active) setEffectiveConfig(null); @@ -100,12 +104,6 @@ export default function ModelSettingsPage() { .catch(() => setLlmProvider("openrouter")); }, [syncLlmProvider]); - useEffect(() => { - if (!effectiveConfig) return; - setExtractorConcurrency(effectiveConfig.rowExtractorConcurrency); - setExtractorAttempts(effectiveConfig.rowExtractorBrowserAttempts); - }, [effectiveConfig]); - const models: OpenRouterModel[] = convexModels ? convexModels.map((m) => ({ modelName: m.modelName, From d32f0ac85a99f4946cda70b07c17f8eb03f2e64a Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Thu, 11 Jun 2026 17:05:42 -0700 Subject: [PATCH 06/16] what --- CODIFICATION.md | 289 ++++ backend/src/config/llm.ts | 19 +- backend/src/config/models.ts | 17 +- backend/src/env.ts | 2 + backend/src/index.ts | 62 +- backend/src/mastra/agents/populate.ts | 9 + backend/src/mastra/tools/investigate-tool.ts | 17 + backend/src/mastra/workflows/populate.ts | 15 + backend/src/mastra/workflows/update.ts | 22 +- backend/src/pipeline/populate.ts | 5 + backend/src/pipeline/schema-inference.ts | 3 + backend/src/pipeline/types.ts | 2 + .../src/row-extractors/try-row-extractor.ts | 1252 ++++++++++++----- .../app/dashboard/settings/models/page.tsx | 12 +- frontend/app/dataset/new/page.tsx | 9 + frontend/components/settings/types.ts | 3 +- frontend/convex/_generated/api.d.ts | 2 + frontend/convex/datasetExtractors.ts | 95 ++ frontend/convex/datasets.ts | 5 + frontend/convex/modelConfig.ts | 6 + frontend/convex/schema.ts | 17 + frontend/lib/backend.ts | 11 +- 22 files changed, 1498 insertions(+), 376 deletions(-) create mode 100644 CODIFICATION.md create mode 100644 frontend/convex/datasetExtractors.ts diff --git a/CODIFICATION.md b/CODIFICATION.md new file mode 100644 index 0000000..3a15073 --- /dev/null +++ b/CODIFICATION.md @@ -0,0 +1,289 @@ +# BigSet Codification Notes + +This document summarizes the product and technical direction discussed for "codifying" recurring BigSet datasets. + +## Core Idea + +BigSet should use LLMs for ambiguity, but not for every repeated row extraction. + +For datasets with predictable primary keys and predictable target pages — for example: + +- Amazon product URLs → price, star rating, review count, availability +- YC company URLs → company metadata +- GitHub repository URLs → stars, language, license, activity +- App/package pages → ratings, pricing, installs, versions + +BigSet should compile or use a deterministic browser extractor once, then run it across all rows with TinyFish Browser. + +This turns BigSet from: + +```txt +LLM researches every row +``` + +into: + +```txt +LLM understands the dataset once → TinyFish Browser extracts many rows +``` + +## Product Motivation + +BigSet is by TinyFish, and should showcase why TinyFish subscriptions are valuable. + +The goal is not to minimize TinyFish usage. The goal is to produce better datasets while shifting repeatable web work away from OpenRouter token spend and toward TinyFish Browser/Agent usage. + +Better data sells TinyFish. + +## TinyFish Agent vs TinyFish Browser + +We decided to treat TinyFish Browser as the foundational primitive for codified extraction. + +### Browser API + +Use Browser when BigSet knows the page pattern and can drive it with Playwright/CDP. + +Good for: + +- Rendering JavaScript-heavy pages +- Reading dynamic DOM content +- Scrolling / clicking load-more buttons +- Extracting from network/XHR JSON +- Running the same extractor across many similar rows +- Reducing OpenRouter usage + +Browser is deterministic infrastructure. + +### Agent API + +Use Agent when the task is still fuzzy or requires web judgment. + +Good for: + +- Unknown navigation paths +- Multi-step workflows +- Sites where BigSet does not know what to click +- Fallback when a generated extractor fails +- One-off messy cases + +Agent is delegation. + +The preferred quality path is: + +```txt +Search → Fetch → Browser → Agent fallback +``` + +For codified datasets, the main runtime path should be: + +```txt +Browser first, Agent only when needed +``` + +## Codified Dataset / Extractor Compiler + +A codified dataset is one where BigSet can create a reusable extractor from the schema and a representative row. + +Example: + +```txt +Primary key: amazon_product_url +Columns: price, star_rating, review_count, availability +``` + +This should not require an LLM per row. A Playwright extractor can be generated, tested, persisted, and reused. + +## Schema Builder Responsibilities + +Schema inference should determine more than just columns. + +It should identify: + +1. Whether the dataset is plausibly codifiable +2. The primary key shape, especially URL-based keys +3. The likely target site/page type +4. Column type contracts +5. Validation requirements for each column +6. Regex or normalization expectations where useful + +Example column contract: + +```ts +{ + name: "star_rating", + type: "number", + nullable: true, + validation_regex: "^[0-5](\\.\\d)?$", + normalization_hint: "Extract a numeric rating like 4.6 from text such as '4.6 out of 5 stars'." +} +``` + +The schema builder should not itself need browser access. It should decide whether a dataset is a codification candidate. + +## Browser Probe + +If a dataset is codifiable, BigSet should run a browser probe on the first representative row/source URL. + +The browser probe should collect evidence such as: + +- Final URL +- Page title +- Visible text excerpts +- DOM excerpts around likely fields +- JSON-LD / structured data +- Candidate selectors +- Network/XHR JSON hints +- Screenshot if useful + +The probe output becomes context for the extractor-builder model. + +## Extractor Builder Model + +A dedicated model role should generate the Playwright extractor. + +Current model roles are: + +```txt +schemaInference +populateOrchestrator +investigateSubagent +``` + +We should add: + +```txt +extractorBuilder +``` + +This model writes and repairs extractor scripts using: + +- Dataset schema +- Column validation contracts +- First row input +- Browser probe artifacts +- Test failure feedback when applicable + +Default recommendation: + +```txt +extractorBuilder = anthropic/claude-sonnet-4.6 +``` + +Reason: extractor generation happens once per dataset/pattern, not once per row. Code quality matters more than token cost here. + +## Extractor Runtime + +Generated extractor code should be run against the first row, validated, and repaired if needed. + +Once it passes, it can be applied across all rows. + +The extractor returns JSON only. BigSet validates and writes to Convex. + +Example extractor contract: + +```ts +export async function extract({ page, input, helpers }) { + await page.goto(input.product_url, { waitUntil: "domcontentloaded" }); + + const title = await page.locator("#productTitle").textContent(); + const price = await page.locator(".a-price .a-offscreen").first().textContent(); + const rating = await page.locator("#acrPopover").getAttribute("title"); + + return { + data: { + product_url: input.product_url, + title: helpers.cleanText(title), + price: helpers.parsePrice(price), + star_rating: helpers.parseRating(rating), + }, + sources: [page.url()], + how_found: "Opened the product URL in TinyFish Browser and extracted fields from the rendered DOM.", + }; +} +``` + +## Security Stance + +We do not need heavyweight Docker isolation for the MVP. + +But generated code should not run inside the main backend process. + +Minimum acceptable runtime boundary: + +- Run generated code in a child Node process +- Empty environment +- No OpenRouter key +- No TinyFish API key +- No Convex admin key +- No keychain access +- No direct database writes +- Hard timeout +- Output-size limit +- Backend validates all returned data before writing + +The generated script can control only a single TinyFish Browser session via Playwright. + +Before execution, BigSet can reject obvious dangerous code patterns as defense-in-depth: + +- `fs` +- `child_process` +- `worker_threads` +- `process.env` +- `require` +- dynamic import +- `eval` +- `new Function` +- `http` / `https` / `net` + +This is not a perfect hostile-code sandbox, but it is enough for an MVP if no secrets are exposed and all writes go through backend validation. + +## Failure / Roadblock Behavior + +A codified extractor should stop or request repair when: + +- Required selectors return empty +- Validation fails repeatedly +- Multiple consecutive rows fail +- The site layout differs from the first row +- The page is blocked or shows CAPTCHA/access-denied content +- Browser session times out + +At that point BigSet can: + +1. Send failure context back to the extractor-builder for repair +2. Fall back to TinyFish Agent for that row +3. Fall back to the existing LLM investigate flow +4. Mark the row as needing review + +## Usage and Credits + +TinyFish currently does not expose a documented credits-remaining endpoint. + +For now BigSet can estimate usage: + +- Search: free / no credits per current docs +- Fetch: free / no credits per current docs +- Agent: roughly `num_of_steps` credits +- Browser: roughly `ceil(duration_seconds / 240)` credits + +When TinyFish exposes a credits/balance endpoint, BigSet can show real usage and remaining balance. + +## Main Decision + +BigSet should add a codification layer that compiles repeatable dataset extraction into reusable TinyFish Browser scripts. + +LLMs should be used to: + +- infer schemas +- decide if codification is possible +- generate or repair extractors +- handle ambiguous fallback cases + +TinyFish Browser should be used to: + +- execute repeatable extraction +- bypass bot detection +- render dynamic pages +- refresh data cheaply and reliably at scale + +This improves data quality, reduces repeated OpenRouter token usage, and makes BigSet a stronger showcase for TinyFish subscriptions. diff --git a/backend/src/config/llm.ts b/backend/src/config/llm.ts index cb603dc..54b1346 100644 --- a/backend/src/config/llm.ts +++ b/backend/src/config/llm.ts @@ -41,7 +41,8 @@ export type LlmProviderType = (typeof LLM_PROVIDER_TYPES)[number]; export type ModelRoleKey = | "schemaInference" | "populateOrchestrator" - | "investigateSubagent"; + | "investigateSubagent" + | "extractorBuilder"; export interface LlmProviderConfig { provider: LlmProviderType; @@ -85,81 +86,97 @@ export const LLM_PROVIDER_DEFAULT_MODELS_BY_ROLE: Record< schemaInference: env.SCHEMA_INFERENCE_MODEL, populateOrchestrator: env.POPULATE_ORCHESTRATOR_MODEL, investigateSubagent: env.INVESTIGATE_SUBAGENT_MODEL, + extractorBuilder: env.EXTRACTOR_BUILDER_MODEL, }, openai: { schemaInference: "gpt-5.4-mini", populateOrchestrator: "gpt-5.4-mini", investigateSubagent: "gpt-5.4-mini", + extractorBuilder: "gpt-5.4-mini", }, anthropic: { schemaInference: "claude-sonnet-4-6", populateOrchestrator: "claude-haiku-4-5-20251001", investigateSubagent: "claude-haiku-4-5-20251001", + extractorBuilder: "claude-haiku-4-5-20251001", }, google: { schemaInference: "gemini-3.5-flash", populateOrchestrator: "gemini-3.5-flash", investigateSubagent: "gemini-3.5-flash", + extractorBuilder: "gemini-3.5-flash", }, xai: { schemaInference: "grok-4.3", populateOrchestrator: "grok-4.3", investigateSubagent: "grok-4.3", + extractorBuilder: "grok-4.3", }, deepseek: { schemaInference: "deepseek-chat", populateOrchestrator: "deepseek-chat", investigateSubagent: "deepseek-chat", + extractorBuilder: "deepseek-chat", }, qwen: { schemaInference: "qwen-plus", populateOrchestrator: "qwen-plus", investigateSubagent: "qwen-plus", + extractorBuilder: "qwen-plus", }, mistral: { schemaInference: "mistral-large-latest", populateOrchestrator: "mistral-large-latest", investigateSubagent: "mistral-large-latest", + extractorBuilder: "mistral-large-latest", }, groq: { schemaInference: "openai/gpt-oss-120b", populateOrchestrator: "openai/gpt-oss-120b", investigateSubagent: "openai/gpt-oss-120b", + extractorBuilder: "openai/gpt-oss-120b", }, togetherai: { schemaInference: "Qwen/Qwen3.5-397B-A17B", populateOrchestrator: "Qwen/Qwen3.5-397B-A17B", investigateSubagent: "Qwen/Qwen3.5-397B-A17B", + extractorBuilder: "Qwen/Qwen3.5-397B-A17B", }, deepinfra: { schemaInference: "meta-llama/Llama-3.3-70B-Instruct-Turbo", populateOrchestrator: "meta-llama/Llama-3.3-70B-Instruct-Turbo", investigateSubagent: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + extractorBuilder: "meta-llama/Llama-3.3-70B-Instruct-Turbo", }, fireworks: { schemaInference: "accounts/fireworks/models/kimi-k2p5", populateOrchestrator: "accounts/fireworks/models/kimi-k2p5", investigateSubagent: "accounts/fireworks/models/kimi-k2p5", + extractorBuilder: "accounts/fireworks/models/kimi-k2p5", }, huggingface: { schemaInference: "deepseek-ai/DeepSeek-V3-0324", populateOrchestrator: "deepseek-ai/DeepSeek-V3-0324", investigateSubagent: "deepseek-ai/DeepSeek-V3-0324", + extractorBuilder: "deepseek-ai/DeepSeek-V3-0324", }, ollama: { schemaInference: "", populateOrchestrator: "", investigateSubagent: "", + extractorBuilder: "", }, lmstudio: { schemaInference: "", populateOrchestrator: "", investigateSubagent: "", + extractorBuilder: "", }, custom: { schemaInference: "", populateOrchestrator: "", investigateSubagent: "", + extractorBuilder: "", }, }; diff --git a/backend/src/config/models.ts b/backend/src/config/models.ts index 9b94095..727b80a 100644 --- a/backend/src/config/models.ts +++ b/backend/src/config/models.ts @@ -33,6 +33,7 @@ export const DEFAULT_MODEL_IDS = { SCHEMA_INFERENCE: env.SCHEMA_INFERENCE_MODEL, POPULATE_ORCHESTRATOR: env.POPULATE_ORCHESTRATOR_MODEL, INVESTIGATE_SUBAGENT: env.INVESTIGATE_SUBAGENT_MODEL, + EXTRACTOR_BUILDER: env.EXTRACTOR_BUILDER_MODEL, } as const; const ROW_EXTRACTOR_CONCURRENCY_MIN = 1; @@ -316,6 +317,7 @@ export const MODEL_ROLES = [ { key: "schemaInference", label: "Schema Inference" }, { key: "populateOrchestrator", label: "Populate Orchestrator" }, { key: "investigateSubagent", label: "Investigate Subagent" }, + { key: "extractorBuilder", label: "Extractor Builder" }, ] as const; /** @@ -463,7 +465,11 @@ export async function fetchModelsForCurrentLlmProvider(): Promise { const models = await getCachedModels(); const found = models.some((m) => m.canonicalSlug === slug); @@ -494,6 +500,7 @@ export async function upsertModelConfig( schemaInference?: string; populateOrchestrator?: string; investigateSubagent?: string; + extractorBuilder?: string; rowExtractorConcurrency?: number; rowExtractorBrowserAttempts?: number; } @@ -505,6 +512,7 @@ export async function upsertModelConfig( schemaInference: config.schemaInference ?? undefined, populateOrchestrator: config.populateOrchestrator ?? undefined, investigateSubagent: config.investigateSubagent ?? undefined, + extractorBuilder: config.extractorBuilder ?? undefined, rowExtractorConcurrency: config.rowExtractorConcurrency !== undefined ? normalizeRowExtractorConcurrency(config.rowExtractorConcurrency) @@ -527,6 +535,7 @@ export async function getModelConfig( schemaInference: string; populateOrchestrator: string; investigateSubagent: string; + extractorBuilder: string; rowExtractorConcurrency: number; rowExtractorBrowserAttempts: number; }> { @@ -554,6 +563,12 @@ export async function getModelConfig( DEFAULT_MODEL_IDS.INVESTIGATE_SUBAGENT, llmConfig, ), + extractorBuilder: modelForProvider( + config?.extractorBuilder, + "extractorBuilder", + DEFAULT_MODEL_IDS.EXTRACTOR_BUILDER, + llmConfig, + ), rowExtractorConcurrency: normalizeRowExtractorConcurrency( config?.rowExtractorConcurrency ?? DEFAULT_ROW_EXTRACTOR_CONCURRENCY, ), diff --git a/backend/src/env.ts b/backend/src/env.ts index 4e3db63..0ba4f7b 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -53,6 +53,8 @@ export const env = { process.env.POPULATE_ORCHESTRATOR_MODEL ?? "qwen/qwen3.7-max", INVESTIGATE_SUBAGENT_MODEL: process.env.INVESTIGATE_SUBAGENT_MODEL ?? "qwen/qwen3.7-max", + EXTRACTOR_BUILDER_MODEL: + process.env.EXTRACTOR_BUILDER_MODEL ?? "anthropic/claude-sonnet-4.6", ROW_EXTRACTOR_CONCURRENCY: numberFromEnv("ROW_EXTRACTOR_CONCURRENCY", 5), ROW_EXTRACTOR_BROWSER_ATTEMPTS: numberFromEnv( "ROW_EXTRACTOR_BROWSER_ATTEMPTS", diff --git a/backend/src/index.ts b/backend/src/index.ts index 36855e9..a75846f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -56,6 +56,14 @@ type DatasetUpdateBeginOutcome = | "already_building" | "already_updating"; type UpdateWorkflowRun = Awaited>; +type RuntimeModelConfig = { + schemaInference: string; + populateOrchestrator: string; + investigateSubagent: string; + extractorBuilder: string; + rowExtractorConcurrency: number; + rowExtractorBrowserAttempts: number; +}; const refreshCadenceSchema = z.enum(["manual", "30m", "6h", "12h", "daily", "weekly"]); const cliCreateDatasetSchema = z.object({ @@ -277,11 +285,7 @@ async function runUpdateWorkflowInBackground({ authorizedUserId: string; logger: FastifyBaseLogger; clerk: ClerkClient; - modelConfig: { - schemaInference: string; - populateOrchestrator: string; - investigateSubagent: string; - }; + modelConfig: RuntimeModelConfig; }): Promise { const datasetId = input.datasetId; // registerDataset is called by the route handler before void-ing this @@ -381,11 +385,7 @@ async function runScheduledUpdateWorkflowInBackground({ run: UpdateWorkflowRun; authorizedUserId: string; logger: FastifyBaseLogger; - modelConfig: { - schemaInference: string; - populateOrchestrator: string; - investigateSubagent: string; - }; + modelConfig: RuntimeModelConfig; }): Promise { const datasetId = input.datasetId; registerDataset(datasetId); @@ -455,11 +455,7 @@ async function runPopulateWorkflowInBackground({ authorizedUserId: string; logger: FastifyBaseLogger; clerk: ClerkClient; - modelConfig: { - schemaInference: string; - populateOrchestrator: string; - investigateSubagent: string; - }; + modelConfig: RuntimeModelConfig; }): Promise { const datasetId = input.datasetId; @@ -647,6 +643,8 @@ function startLocalRefreshScheduler( description: dataset.description, maxRowCount: dataset.maxRowCount ?? 100, columns: dataset.columns, + retrievalStrategy: dataset.retrievalStrategy, + sourceHint: dataset.sourceHint, }, run, authorizedUserId: dataset.ownerId, @@ -930,6 +928,9 @@ fastify.post("/cli/datasets", async (req, reply) => { type: mapSchemaTypeToDatasetType(column.type), description: column.retrieval_hint, isPrimaryKey: column.is_primary_key || undefined, + nullable: column.nullable, + validationRegex: column.validation_regex || undefined, + normalizationHint: column.normalization_hint || undefined, })); const datasetId = await convex.mutation( @@ -1047,6 +1048,8 @@ fastify.post("/cli/datasets/:datasetId/populate", async (req, reply) => { description: dataset.description, maxRowCount: dataset.maxRowCount ?? 100, columns: dataset.columns, + retrievalStrategy: dataset.retrievalStrategy, + sourceHint: dataset.sourceHint, }, run, controller, @@ -1147,6 +1150,7 @@ await fastify.register(async (instance) => { schemaInference?: string | null; populateOrchestrator?: string | null; investigateSubagent?: string | null; + extractorBuilder?: string | null; rowExtractorConcurrency?: number | null; rowExtractorBrowserAttempts?: number | null; }; @@ -1154,6 +1158,7 @@ await fastify.register(async (instance) => { schemaInference: typeof body.schemaInference === "string" ? body.schemaInference.trim() || undefined : undefined, populateOrchestrator: typeof body.populateOrchestrator === "string" ? body.populateOrchestrator.trim() || undefined : undefined, investigateSubagent: typeof body.investigateSubagent === "string" ? body.investigateSubagent.trim() || undefined : undefined, + extractorBuilder: typeof body.extractorBuilder === "string" ? body.extractorBuilder.trim() || undefined : undefined, rowExtractorConcurrency: typeof body.rowExtractorConcurrency === "number" ? body.rowExtractorConcurrency @@ -1164,10 +1169,18 @@ await fastify.register(async (instance) => { : undefined, }; - const toValidate: Array<{ role: "schemaInference" | "populateOrchestrator" | "investigateSubagent"; slug: string }> = []; + const toValidate: Array<{ + role: + | "schemaInference" + | "populateOrchestrator" + | "investigateSubagent" + | "extractorBuilder"; + slug: string; + }> = []; if (config.schemaInference) toValidate.push({ role: "schemaInference", slug: config.schemaInference }); if (config.populateOrchestrator) toValidate.push({ role: "populateOrchestrator", slug: config.populateOrchestrator }); if (config.investigateSubagent) toValidate.push({ role: "investigateSubagent", slug: config.investigateSubagent }); + if (config.extractorBuilder) toValidate.push({ role: "extractorBuilder", slug: config.extractorBuilder }); if (toValidate.length > 0) { try { @@ -1188,6 +1201,7 @@ await fastify.register(async (instance) => { schemaInference: config.schemaInference, populateOrchestrator: config.populateOrchestrator, investigateSubagent: config.investigateSubagent, + extractorBuilder: config.extractorBuilder, rowExtractorConcurrency: config.rowExtractorConcurrency, rowExtractorBrowserAttempts: config.rowExtractorBrowserAttempts, }); @@ -1288,6 +1302,8 @@ await fastify.register(async (instance) => { input: { ...parsed.data, maxRowCount: dataset.maxRowCount ?? parsed.data.maxRowCount, + retrievalStrategy: dataset.retrievalStrategy ?? parsed.data.retrievalStrategy, + sourceHint: dataset.sourceHint ?? parsed.data.sourceHint, }, run, controller, @@ -1345,6 +1361,13 @@ await fastify.register(async (instance) => { throw new Error(`Unexpected update claim outcome: ${updateOutcome}`); } + const dataset = await convex.query(internal.datasets.getInternal, { + id: parsed.data.datasetId, + }); + if (!dataset) { + return reply.code(404).send({ error: "Dataset not found" }); + } + let run: UpdateWorkflowRun; try { run = await updateWorkflow.createRun(); @@ -1364,7 +1387,12 @@ await fastify.register(async (instance) => { registerDataset(parsed.data.datasetId); void runUpdateWorkflowInBackground({ - input: parsed.data, + input: { + ...parsed.data, + maxRowCount: dataset.maxRowCount ?? parsed.data.maxRowCount, + retrievalStrategy: dataset.retrievalStrategy ?? parsed.data.retrievalStrategy, + sourceHint: dataset.sourceHint ?? parsed.data.sourceHint, + }, run, authorizedUserId: auth.userId, logger: req.log, diff --git a/backend/src/mastra/agents/populate.ts b/backend/src/mastra/agents/populate.ts index f5d9869..bbdffd0 100644 --- a/backend/src/mastra/agents/populate.ts +++ b/backend/src/mastra/agents/populate.ts @@ -6,6 +6,13 @@ import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; +export interface PopulateAgentDatasetContext { + datasetName: string; + description: string; + retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; + sourceHint?: string; +} + function buildInstructions(maxRowCount: number): string { return `You are an expert dataset builder. You conduct research using your web tools. You do broad research to see which rows to add, and then you spin up sub-agents that can do the deep research and fill in each row for you. @@ -42,6 +49,7 @@ export function buildPopulateAgent( columns: PopulateColumn[], llmConfig: LlmProviderConfig, maxRowCount: number, + datasetContext: PopulateAgentDatasetContext, metrics?: RunMetrics, ): Agent { const modelSlug = authContext.modelConfig!.populateOrchestrator; @@ -60,6 +68,7 @@ export function buildPopulateAgent( columns, llmConfig, maxRowCount, + datasetContext, metrics, ), }, diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index a1f7034..2fd4ab6 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -50,6 +50,13 @@ const investigateOutputSchema = z.object({ reason: z.string(), }); +interface DatasetContextForExtractor { + datasetName: string; + description: string; + retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; + sourceHint?: string; +} + function parseInvestigateResult( text: string, ): z.infer { @@ -82,6 +89,7 @@ export function buildSubagentTool( columns: PopulateColumn[], llmConfig: LlmProviderConfig, maxRowCount: number, + datasetContext: DatasetContextForExtractor, metrics?: RunMetrics, ) { return createTool({ @@ -115,7 +123,12 @@ export function buildSubagentTool( primaryKeys: primaryKeyRecord, urls, context, + datasetName: datasetContext.datasetName, + description: datasetContext.description, + retrievalStrategy: datasetContext.retrievalStrategy, + sourceHint: datasetContext.sourceHint, browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, + extractorBuilderModel: authContext.modelConfig.extractorBuilder, }); if (extractorResult.status === "inserted") { if (metrics) metrics.rowsInserted++; @@ -141,6 +154,10 @@ export function buildSubagentTool( console.warn( `[run_subagent] row extractor failed entity="${entity_hint}" reason="${extractorResult.reason}"`, ); + } else if (extractorResult.status === "miss") { + console.log( + `[run_subagent] row extractor missed entity="${entity_hint}" reason="${extractorResult.reason}"`, + ); } console.log( diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index 7173425..5867ebf 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -37,6 +37,7 @@ export const authContextSchema = z.object({ schemaInference: z.string().min(1), populateOrchestrator: z.string().min(1), investigateSubagent: z.string().min(1), + extractorBuilder: z.string().min(1), rowExtractorConcurrency: z.number().int().min(1).max(100).default(5), rowExtractorBrowserAttempts: z.number().int().min(1).max(10).default(2), }), @@ -159,6 +160,10 @@ const buildPromptOutputSchema = z.object({ authContext: authContextSchema, columns: z.array(populateColumnSchema), maxRowCount: z.number().int().min(1), + datasetName: z.string(), + description: z.string(), + retrievalStrategy: z.enum(["search_fetch", "browser", "hybrid"]).optional(), + sourceHint: z.string().optional(), }); const buildPromptStep = createStep({ @@ -217,6 +222,10 @@ Stop the populate run as soon as the dataset reaches ${inputData.maxRowCount} ro authContext: inputData.authContext, columns: inputData.columns, maxRowCount: inputData.maxRowCount, + datasetName: inputData.datasetName, + description: inputData.description, + retrievalStrategy: inputData.retrievalStrategy, + sourceHint: inputData.sourceHint, }; }, }); @@ -252,6 +261,12 @@ const agentStep = createStep({ inputData.columns, await requireLlmProviderConfig(), inputData.maxRowCount, + { + datasetName: inputData.datasetName, + description: inputData.description, + retrievalStrategy: inputData.retrievalStrategy, + sourceHint: inputData.sourceHint, + }, metrics, ); const abortSignal = getSignal(inputData.authorizedDatasetId); diff --git a/backend/src/mastra/workflows/update.ts b/backend/src/mastra/workflows/update.ts index 39da763..bf1d3b8 100644 --- a/backend/src/mastra/workflows/update.ts +++ b/backend/src/mastra/workflows/update.ts @@ -93,7 +93,16 @@ const refreshRowsStep = createStep({ inputSchema: markAndFetchOutputSchema, outputSchema: refreshOutputSchema, execute: async ({ inputData }) => { - const { datasetId, columns, authContext, rows } = inputData; + const { + datasetId, + columns, + authContext, + rows, + datasetName, + description, + retrievalStrategy, + sourceHint, + } = inputData; let updatedCount = 0; let errors = 0; @@ -124,7 +133,12 @@ const refreshRowsStep = createStep({ existingData: row.data, urls: row.sources, context: [row.rowSummary, row.howFound].filter(Boolean).join("\n"), + datasetName, + description, + retrievalStrategy, + sourceHint, browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, + extractorBuilderModel: authContext.modelConfig.extractorBuilder, }); if (extractorResult.status === "updated") { @@ -143,6 +157,12 @@ const refreshRowsStep = createStep({ return; } + if (extractorResult.status === "miss") { + console.log( + `[refresh-rows] Row ${row._id}: row extractor missed; falling back to refresh agent: ${extractorResult.reason}`, + ); + } + if (extractorResult.status === "failed") { if (getSignal(datasetId)?.aborted) throwAbortError(); console.warn( diff --git a/backend/src/pipeline/populate.ts b/backend/src/pipeline/populate.ts index 589db1e..b6e3b85 100644 --- a/backend/src/pipeline/populate.ts +++ b/backend/src/pipeline/populate.ts @@ -7,6 +7,9 @@ export const populateColumnSchema = z.object({ type: z.enum(["text", "number", "boolean", "url", "date"]), description: z.optional(z.string()), isPrimaryKey: z.optional(z.boolean()), + nullable: z.optional(z.boolean()), + validationRegex: z.optional(z.string()), + normalizationHint: z.optional(z.string()), }); export type PopulateColumn = z.infer; @@ -17,5 +20,7 @@ export const datasetContextSchema = z.object({ maxRowCount: z.number().int().min(1).max(FREE_TIER_MONTHLY_QUOTA).default(100), columns: z.array(populateColumnSchema).min(1), rowIds: z.array(z.string()).min(1).optional(), + retrievalStrategy: z.enum(["search_fetch", "browser", "hybrid"]).optional(), + sourceHint: z.string().optional(), }); export type DatasetContext = z.infer; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 8a72abe..1c1bb95 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -18,6 +18,7 @@ Your job is to: - \`hybrid\` — unclear; the pipeline will try search_fetch first and fall back to browser. 5. Set \`source_hint\` to a specific URL whenever possible (e.g. \`https://www.ycombinator.com/companies?industry=Fintech\`). Avoid vague descriptions. 6. Write a \`retrieval_hint\` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. +7. For each column where a value has a known shape, include \`validation_regex\` and \`normalization_hint\`. These are extractor contracts, not UI decoration. Examples: ratings, prices, dates, URL/slug shapes, repository slugs, app package names, counts, currencies, availability labels. Omit \`validation_regex\` only when the value is genuinely free-form text. Rules: @@ -25,6 +26,8 @@ Rules: - \`dataset_name\` must be snake_case. - All column \`name\` values must be snake_case and unique. - Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. +- Validation regexes must validate the normalized final value, not raw page text. Keep them practical and anchored, e.g. "^[0-5](\\\\.\\\\d)?$" for a normalized rating or "^[^/\\\\s]+/[^/\\\\s]+$" for an owner/repo slug. +- \`normalization_hint\` should tell the extractor how to convert raw page text into the stored value, e.g. "Convert '4.6 out of 5 stars' to '4.6'" or "Strip commas and convert 1.2k to 1200". - When a column is a scalar numeric rating (e.g. average score like 4.3/5 for restaurants, cafes, hotels, products, apps): name it generically (e.g. "rating" not "yelp_rating") and write a retrieval_hint explaining that review sites (Yelp, TripAdvisor, Google Maps) block direct page fetches, so the agent must extract ratings from **search result snippets**. The hint should say: "Search for \\" rating reviews\\" and include location terms only when location is part of the entity identity. Look for ratings in snippets from TripAdvisor (\\"rated X.X of 5\\"), Yelp search listings (\\"X.X (N reviews)\\"), or aggregator sites (Birdeye, joe.coffee, giftly, Uber Eats, menufyy). Do NOT try to fetch yelp.com or tripadvisor.com directly — they block automated access. Accept ratings from any reputable source." If including a rating column, also add a "rating_source" text column so the agent records where the rating came from. Do not rename review-count or review-text fields to "rating" — keep those as distinct columns (e.g. "review_count") when the user explicitly asks for them.`; async function getModel(modelSlug?: string) { diff --git a/backend/src/pipeline/types.ts b/backend/src/pipeline/types.ts index ee9ed13..25338fa 100644 --- a/backend/src/pipeline/types.ts +++ b/backend/src/pipeline/types.ts @@ -27,6 +27,8 @@ export const columnDefinitionSchema = z.object({ is_enumerable: z.boolean(), retrieval_hint: z.string(), nullable: z.boolean(), + validation_regex: z.string().optional(), + normalization_hint: z.string().optional(), }); export type ColumnDefinition = z.infer; diff --git a/backend/src/row-extractors/try-row-extractor.ts b/backend/src/row-extractors/try-row-extractor.ts index 29dda9f..7507499 100644 --- a/backend/src/row-extractors/try-row-extractor.ts +++ b/backend/src/row-extractors/try-row-extractor.ts @@ -1,20 +1,36 @@ import { chromium, type Browser, type Page } from "playwright-core"; +import { generateText } from "ai"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; import { getSignal } from "../abort-registry.js"; import { convex, internal } from "../convex.js"; import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; -import { getTinyFishApiKey, tinyFishHeaders } from "../local-credentials.js"; +import { + getTinyFishApiKey, + requireOpenRouterApiKey, + tinyFishHeaders, +} from "../local-credentials.js"; import type { PopulateColumn } from "../pipeline/populate.js"; +import { DEFAULT_MODEL_IDS } from "../config/models.js"; type ExtractorStatus = "inserted" | "updated" | "unchanged" | "miss" | "failed"; +type ExtractedValue = string | number | boolean; export interface TryRowExtractorInput { datasetId: string; + datasetName?: string; + description?: string; columns: PopulateColumn[]; primaryKeys: Record; urls?: string[]; context?: string; + retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; + sourceHint?: string; browserAttempts?: number; + extractorBuilderModel?: string; } export interface TryRefreshRowExtractorInput extends TryRowExtractorInput { @@ -35,49 +51,122 @@ interface TinyFishBrowserSession { base_url: string; } -interface GitHubRepoFacts { - owner: string; - repo: string; - fullName: string; - url: string; - description?: string; - stars?: number; - forks?: number; - watchers?: number; - issues?: number; - pullRequests?: number; - language?: string; - license?: string; - latestCommitAt?: string; - updatedAt?: string; - createdAt?: string; - homepage?: string; - archived?: boolean; -} - -interface RawGitHubRepoDomFacts { +interface PageEvidence { + finalUrl: string; + title?: string; description?: string; - stars?: string; - forks?: string; - watchers?: string; - issues?: string; - pullRequests?: string; - language?: string; - license?: string; - latestCommitAt?: string; - homepage?: string; - archived?: boolean; + candidates: Record; + bodyText: string; +} + +interface GenericExtraction { + data: Record; + sources: string[]; + rowSummary?: string; + extractedColumns: string[]; + missingColumns: string[]; +} + +interface GeneratedExtractorResult { + data?: Record; + sources?: unknown; + row_summary?: unknown; + rowSummary?: unknown; + how_found?: unknown; + howFound?: unknown; } const ENABLED_VALUES = new Set(["1", "true", "yes", "on"]); -const GITHUB_HOSTS = new Set(["github.com", "www.github.com"]); const BROWSER_TIMEOUT_MS = 45_000; const CDP_CONNECT_TIMEOUT_MS = 45_000; const DEFAULT_BROWSER_ATTEMPTS = 2; -const GITHUB_EXTRACTOR_HOW_FOUND = - "Opened the GitHub repository URL with TinyFish Browser and extracted repository facts from the rendered page."; -const GITHUB_REFRESH_HOW_FOUND = - "Refreshed the GitHub repository URL with TinyFish Browser and extracted repository facts from the rendered page."; +const EXTRACTOR_RUNNER_TIMEOUT_MS = 60_000; +const EXTRACTOR_SCRIPT_OUTPUT_LIMIT = 256_000; +const BACKEND_ROOT = fileURLToPath(new URL("../../", import.meta.url)); +const GENERIC_EXTRACTOR_HOW_FOUND = + "Opened the row target with TinyFish Browser and ran the dataset's generated Playwright extractor."; +const GENERIC_REFRESH_HOW_FOUND = + "Refreshed the row target with TinyFish Browser and ran the dataset's generated Playwright extractor."; +const EXTRACTOR_RUNNER_SOURCE = ` +import { chromium } from "playwright-core"; +import vm from "node:vm"; + +const chunks = []; +for await (const chunk of process.stdin) chunks.push(chunk); +const payload = JSON.parse(Buffer.concat(chunks).toString("utf8")); + +const browser = await chromium.connectOverCDP(payload.cdpUrl, { timeout: 45000 }); +let timeout; + +const cleanText = (value) => String(value ?? "").replace(/\\s+/g, " ").trim(); +const parseCompactNumber = (value) => { + const normalized = String(value ?? "").replace(/,/g, ""); + const match = normalized.match(/[$€£]?\\s*([\\d]+(?:\\.\\d+)?)\\s*([kmb])?/i) || normalized.match(/([\\d]+(?:\\.\\d+)?)/); + if (!match) return undefined; + const base = Number(match[1]); + if (!Number.isFinite(base)) return undefined; + const suffix = match[2]?.toLowerCase(); + const multiplier = suffix === "k" ? 1000 : suffix === "m" ? 1000000 : suffix === "b" ? 1000000000 : 1; + return Math.round(base * multiplier * 100) / 100; +}; +const helpers = { + cleanText, + parseNumber: parseCompactNumber, + parseCompactNumber, + parsePrice: parseCompactNumber, + parseRating: parseCompactNumber, + absoluteUrl: (value, base = payload.url) => { + try { + return new URL(String(value ?? ""), base).toString(); + } catch { + return ""; + } + }, +}; + +try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = context.pages()[0] ?? await context.newPage(); + await page.goto(payload.url, { waitUntil: "domcontentloaded", timeout: 45000 }); + await page.waitForLoadState("networkidle", { timeout: 10000 }).catch(() => {}); + + const sandbox = { + URL, + Date, + Math, + JSON, + RegExp, + String, + Number, + Boolean, + Array, + Object, + Promise, + setTimeout, + clearTimeout, + console: { log() {}, warn() {}, error() {} }, + }; + vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } }); + const compiled = new vm.Script(payload.script + "\\n;globalThis.__extract = extract;", { + filename: "generated-extractor.js", + }); + compiled.runInContext(sandbox, { timeout: 1000 }); + const extract = sandbox.__extract; + if (typeof extract !== "function") throw new Error("extract is not a function"); + + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error("extract timed out")), payload.timeoutMs); + }); + const result = await Promise.race([ + extract({ page, input: payload.input, helpers }), + timeoutPromise, + ]); + process.stdout.write(JSON.stringify(result ?? {})); +} finally { + if (timeout) clearTimeout(timeout); + await browser.close().catch(() => {}); +} +`; export async function tryRowExtractor( input: TryRowExtractorInput, @@ -86,41 +175,31 @@ export async function tryRowExtractor( return { status: "miss", reason: "row extractors are disabled" }; } - const url = firstCandidateUrl(input); - if (!url) return { status: "miss", reason: "no URL primary key or candidate URL" }; - - const repoRef = parseGitHubRepoUrl(url); - if (!repoRef) { - return { status: "miss", reason: `unsupported URL host: ${safeHost(url)}` }; - } + const url = initialBrowserUrl(input); + if (!url) return { status: "miss", reason: "no browser start URL or row context" }; try { - const facts = await extractGitHubRepoFacts( - url, - input.datasetId, - input.browserAttempts, - ); - const row = buildGitHubRow(input.columns, input.primaryKeys, facts); - if (!row) { + const extraction = await extractGenericRow(input, url); + if (!hasExtractedNonPrimaryValue(extraction, input.columns)) { return { status: "miss", - reason: "GitHub extractor could not satisfy all requested columns", + reason: `TinyFish Browser did not extract non-primary fields from ${safeHost(url)}`, }; } await convex.mutation(internal.datasetRows.insert, { datasetId: input.datasetId, - data: row, - sources: [facts.url], - rowSummary: githubRowSummary(facts), - howFound: GITHUB_EXTRACTOR_HOW_FOUND, + data: extraction.data, + sources: extraction.sources, + rowSummary: extraction.rowSummary, + howFound: GENERIC_EXTRACTOR_HOW_FOUND, }); return { status: "inserted", - reason: "Inserted by GitHub row extractor", - rowSummary: githubRowSummary(facts), - sources: [facts.url], + reason: `Inserted by generic browser extractor (${extraction.extractedColumns.join(", ")})`, + rowSummary: extraction.rowSummary, + sources: extraction.sources, }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -141,52 +220,46 @@ export async function tryRefreshRowExtractor( return { status: "miss", reason: "row extractors are disabled" }; } - const url = firstCandidateUrl(input); - if (!url) return { status: "miss", reason: "no URL primary key or candidate URL" }; - - const repoRef = parseGitHubRepoUrl(url); - if (!repoRef) { - return { status: "miss", reason: `unsupported URL host: ${safeHost(url)}` }; - } + const url = initialBrowserUrl(input); + if (!url) return { status: "miss", reason: "no browser start URL or row context" }; try { - const facts = await extractGitHubRepoFacts( - url, - input.datasetId, - input.browserAttempts, - ); - const row = buildGitHubRow(input.columns, input.primaryKeys, facts); - if (!row) { + const extraction = await extractGenericRow(input, url, input.existingData); + if (!hasExtractedNonPrimaryValue(extraction, input.columns)) { return { status: "miss", - reason: "GitHub extractor could not satisfy all requested columns", + reason: `TinyFish Browser did not extract non-primary fields from ${safeHost(url)}`, }; } - const changedColumns = changedColumnNames(row, input.existingData, input.columns); + const changedColumns = changedColumnNames( + extraction.data, + input.existingData, + input.columns, + ); if (changedColumns.length === 0) { return { status: "unchanged", - reason: "Verified unchanged by GitHub row extractor", - rowSummary: githubRowSummary(facts), - sources: [facts.url], + reason: "Verified unchanged by generic browser extractor", + rowSummary: extraction.rowSummary, + sources: extraction.sources, }; } await convex.mutation(internal.datasetRows.update, { id: input.rowId, expectedDatasetId: input.datasetId, - data: row, - sources: [facts.url], - rowSummary: githubRowSummary(facts), - howFound: GITHUB_REFRESH_HOW_FOUND, + data: extraction.data, + sources: extraction.sources, + rowSummary: extraction.rowSummary, + howFound: GENERIC_REFRESH_HOW_FOUND, }); return { status: "updated", - reason: `Updated by GitHub row extractor (${changedColumns.join(", ")})`, - rowSummary: githubRowSummary(facts), - sources: [facts.url], + reason: `Updated by generic browser extractor (${changedColumns.join(", ")})`, + rowSummary: extraction.rowSummary, + sources: extraction.sources, }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -194,23 +267,47 @@ export async function tryRefreshRowExtractor( } } -function firstCandidateUrl(input: TryRowExtractorInput): string | undefined { - const fromPrimaryKey = Object.values(input.primaryKeys).find((value) => - isHttpUrl(value), - ); - if (fromPrimaryKey) return normalizeUrl(fromPrimaryKey); +function initialBrowserUrl(input: TryRowExtractorInput): string | undefined { + const explicitUrl = [ + ...Object.values(input.primaryKeys), + ...(input.urls ?? []), + input.sourceHint, + input.context?.match(/https?:\/\/[^\s)>"']+/i)?.[0], + ] + .map(coerceHttpUrl) + .find((value): value is string => Boolean(value)); + if (explicitUrl) return explicitUrl; - const fromUrls = input.urls?.find(isHttpUrl); - if (fromUrls) return normalizeUrl(fromUrls); + const searchQuery = [ + input.sourceHint, + input.datasetName, + input.description, + ...Object.values(input.primaryKeys), + ] + .map((value) => value?.trim()) + .filter((value): value is string => Boolean(value)) + .join(" ") + .slice(0, 500); - const fromContext = input.context?.match(/https?:\/\/[^\s)>"']+/i)?.[0]; - return fromContext ? normalizeUrl(fromContext) : undefined; + if (!searchQuery) return undefined; + return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}`; } function normalizeUrl(value: string): string { return value.trim().replace(/[.,;:]+$/, ""); } +function coerceHttpUrl(value: string | undefined): string | undefined { + if (!value) return undefined; + const normalized = normalizeUrl(value); + if (isHttpUrl(normalized)) return normalized; + if (/^[a-z0-9.-]+\.[a-z]{2,}(?:\/[^\s]*)?$/i.test(normalized)) { + const withScheme = `https://${normalized}`; + return isHttpUrl(withScheme) ? withScheme : undefined; + } + return undefined; +} + function isHttpUrl(value: string | undefined): value is string { if (!value) return false; try { @@ -229,46 +326,49 @@ function safeHost(value: string): string { } } -function parseGitHubRepoUrl(value: string): { owner: string; repo: string } | null { - try { - const url = new URL(value); - if (!GITHUB_HOSTS.has(url.hostname.toLowerCase())) return null; - const [owner, repo] = url.pathname - .split("/") - .filter(Boolean) - .map((part) => part.trim()); - if (!owner || !repo) return null; - if (["orgs", "topics", "marketplace", "features"].includes(owner)) return null; - return { owner, repo: repo.replace(/\.git$/i, "") }; - } catch { - return null; - } -} - -async function extractGitHubRepoFacts( +async function extractGenericRow( + input: TryRowExtractorInput, url: string, - datasetId: string, - browserAttempts: number | undefined, -): Promise { + existingData?: Record, +): Promise { const apiKey = await getTinyFishApiKey(); if (!apiKey) throw new Error("TINYFISH_API_KEY is not configured"); - const attempts = normalizedBrowserAttempts(browserAttempts); + const script = await getOrBuildExtractorScript(apiKey, input, url); + const attempts = normalizedBrowserAttempts(input.browserAttempts); let lastError: unknown; for (let attempt = 1; attempt <= attempts; attempt++) { try { - return await extractGitHubRepoFactsOnce(apiKey, url, datasetId); + const raw = await runGeneratedExtractorScript(apiKey, input, url, script); + return buildRowFromExtractorResult(input, url, raw, existingData); } catch (err) { lastError = err; - if (getSignal(datasetId)?.aborted || attempt === attempts) break; + if (getSignal(input.datasetId)?.aborted || attempt === attempts) break; const msg = err instanceof Error ? err.message : String(err); console.warn( - `[row_extractor] GitHub browser attempt ${attempt}/${attempts} failed; retrying: ${msg}`, + `[row_extractor] Browser attempt ${attempt}/${attempts} failed; retrying: ${msg}`, ); } } - throw lastError instanceof Error ? lastError : new Error(String(lastError)); + const failure = lastError instanceof Error ? lastError.message : String(lastError); + try { + console.warn( + `[row_extractor] generated extractor failed; requesting repair: ${failure}`, + ); + const repaired = await repairExtractorAfterRuntimeFailure( + apiKey, + input, + url, + script, + failure, + ); + const raw = await runGeneratedExtractorScript(apiKey, input, url, repaired); + return buildRowFromExtractorResult(input, url, raw, existingData); + } catch (repairErr) { + await markExtractorFailed(input, url, repairErr).catch(() => undefined); + throw repairErr instanceof Error ? repairErr : new Error(String(repairErr)); + } } function normalizedBrowserAttempts(value: number | undefined): number { @@ -278,11 +378,65 @@ function normalizedBrowserAttempts(value: number | undefined): number { return Math.min(10, Math.max(1, Math.trunc(value))); } -async function extractGitHubRepoFactsOnce( +async function getOrBuildExtractorScript( apiKey: string, + input: TryRowExtractorInput, url: string, +): Promise { + const siteKey = siteKeyForInput(input, url); + const columnsHash = hashColumns(input.columns); + const existing = (await convex.query(internal.datasetExtractors.getActive, { + datasetId: input.datasetId, + siteKey, + columnsHash, + })) as { script?: string } | null; + if (existing?.script) return existing.script; + + const evidence = await probePage(apiKey, input.datasetId, url); + const modelSlug = input.extractorBuilderModel ?? DEFAULT_MODEL_IDS.EXTRACTOR_BUILDER; + const script = await buildExtractorScript(input, url, evidence, modelSlug); + const validation = await validateExtractorScript(apiKey, input, url, script); + if (validation.ok) { + await convex.mutation(internal.datasetExtractors.upsert, { + datasetId: input.datasetId, + siteKey, + columnsHash, + script, + model: modelSlug, + probeSummary: summarizeEvidenceForStorage(evidence), + }); + return script; + } + + const repaired = await repairExtractorScript( + input, + url, + evidence, + script, + validation.reason, + modelSlug, + ); + const repairedValidation = await validateExtractorScript(apiKey, input, url, repaired); + if (!repairedValidation.ok) { + throw new Error(`generated extractor failed validation: ${repairedValidation.reason}`); + } + + await convex.mutation(internal.datasetExtractors.upsert, { + datasetId: input.datasetId, + siteKey, + columnsHash, + script: repaired, + model: modelSlug, + probeSummary: summarizeEvidenceForStorage(evidence), + }); + return repaired; +} + +async function probePage( + apiKey: string, datasetId: string, -): Promise { + url: string, +): Promise { const session = await createTinyFishBrowserSession(apiKey, url, datasetId); let browser: Browser | undefined; try { @@ -296,9 +450,10 @@ async function extractGitHubRepoFactsOnce( timeout: BROWSER_TIMEOUT_MS, }); await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => { - // GitHub may keep long-lived requests open. DOMContentLoaded is enough. + // Many modern sites keep long-lived requests open. DOMContentLoaded is enough. }); - return await readGitHubRepoFacts(page); + + return await readPageEvidence(page); } finally { await browser?.close().catch(() => undefined); } @@ -366,176 +521,607 @@ async function withRunTimeoutSignal( } } -async function readGitHubRepoFacts(page: Page): Promise { - const url = page.url(); - const repoRef = parseGitHubRepoUrl(url); - if (!repoRef) throw new Error(`Not a GitHub repository page: ${url}`); - - const facts = (await page.evaluate(` - (() => { - const text = (selector) => - document.querySelector(selector)?.textContent?.trim() || undefined; - const attr = (selector, name) => - document.querySelector(selector)?.getAttribute(name) || undefined; - const firstCandidateText = (selector, predicate) => - Array.from(document.querySelectorAll(selector)) - .map((el) => el.textContent?.trim()) - .filter(Boolean) - .find((value) => !predicate || predicate(value)); - const language = () => - text("[itemprop=\\"programmingLanguage\\"]") ?? - text("a[href*=\\"search?l=\\"] span.color-fg-default.text-bold") ?? - text("a[href*=\\"search?l=\\"] .text-bold"); - const license = () => - firstCandidateText( - "a[href*=\\"LICENSE\\"], a[href*=\\"license\\"], [data-testid*=\\"license\\"]", - (value) => /licensed|MIT|Apache|BSD|GPL|MPL|ISC/i.test(value), - ) ?? - firstCandidateText( - "a[href*=\\"LICENSE\\"], a[href*=\\"license\\"], [data-testid*=\\"license\\"]", - (value) => !/^(license|view license)$/i.test(value), - ) ?? - firstCandidateText( - "a[href*=\\"LICENSE\\"], a[href*=\\"license\\"], [data-testid*=\\"license\\"]", - ) ?? - text("svg.octicon-law + span"); - const bodyText = document.body?.innerText ?? ""; +async function readPageEvidence(page: Page): Promise { + const finalUrl = page.url(); + const evidence = await page.evaluate(() => { + const normalize = (value: string) => + value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); + const clean = (value: unknown) => + String(value ?? "") + .replace(/\s+/g, " ") + .trim(); + const candidates: Record = {}; + const push = (key: unknown, value: unknown) => { + const normalizedKey = normalize(clean(key)); + const cleanedValue = clean(value); + if (!normalizedKey || !cleanedValue || cleanedValue.length > 2_000) return; + const list = candidates[normalizedKey] ?? []; + if (list.length >= 8 || list.includes(cleanedValue)) return; + candidates[normalizedKey] = [...list, cleanedValue]; + }; + const text = (selector: string) => + clean(document.querySelector(selector)?.textContent ?? ""); + const attr = (selector: string, name: string) => + clean(document.querySelector(selector)?.getAttribute(name) ?? ""); - return { - description: - text("[data-pjax=\\"#repo-content-pjax-container\\"] [itemprop=\\"about\\"]") ?? - text("[itemprop=\\"about\\"]") ?? - attr("meta[name='description']", "content"), - stars: - text("#repo-stars-counter-star") ?? - text("a[href$='/stargazers'] strong") ?? - text("a[href$='/stargazers']"), - forks: - text("#repo-network-counter") ?? - text("a[href$='/forks'] strong") ?? - text("a[href$='/forks']"), - watchers: - text("a[href$='/watchers'] strong") ?? - text("a[href$='/watchers']"), - issues: - text("#issues-tab span.Counter") ?? - text("a[href$=\\"/issues\\"] span.Counter") ?? - text("a[data-tab-item=\\"i1issues-tab\\"] span.Counter"), - pullRequests: - text("#pull-requests-tab span.Counter") ?? - text("a[href$=\\"/pulls\\"] span.Counter") ?? - text("a[data-tab-item=\\"i2pull-requests-tab\\"] span.Counter"), - language: language(), - license: license(), - latestCommitAt: - attr("relative-time[datetime]", "datetime") ?? - attr("time-ago[datetime]", "datetime"), - homepage: attr("[itemprop='url']", "href"), - archived: /This repository has been archived/i.test(bodyText), - }; - })() - `)) as RawGitHubRepoDomFacts; + const title = + text("h1") || + attr("meta[property='og:title']", "content") || + attr("meta[name='twitter:title']", "content") || + clean(document.title); + const description = + attr("meta[name='description']", "content") || + attr("meta[property='og:description']", "content") || + attr("meta[name='twitter:description']", "content"); - const apiFacts = await fetchGitHubApiFacts(page, repoRef.owner, repoRef.repo).catch( - () => undefined, - ); + push("title", title); + push("name", title); + push("description", description); + push("summary", description); + push("url", location.href); + push("canonical", attr("link[rel='canonical']", "href")); - return { - owner: repoRef.owner, - repo: repoRef.repo, - fullName: `${repoRef.owner}/${repoRef.repo}`, + document.querySelectorAll("meta[name][content], meta[property][content]").forEach((el) => { + const key = el.getAttribute("name") || el.getAttribute("property"); + push(key, el.getAttribute("content")); + }); + + const walkJson = (value: unknown, path: string[] = []) => { + if (value == null) return; + if (Array.isArray(value)) { + value.slice(0, 40).forEach((item, index) => walkJson(item, [...path, String(index)])); + return; + } + if (typeof value === "object") { + for (const [key, nested] of Object.entries(value as Record)) { + walkJson(nested, [...path, key]); + } + return; + } + const leaf = clean(value); + if (!leaf) return; + const key = path[path.length - 1] ?? ""; + push(key, leaf); + push(path.filter((part) => !/^\d+$/.test(part)).join("_"), leaf); + }; + + document.querySelectorAll("script[type*='ld+json']").forEach((script) => { + try { + walkJson(JSON.parse(script.textContent || "")); + } catch { + // Ignore malformed structured data. + } + }); + + document.querySelectorAll("table tr").forEach((row) => { + const cells = Array.from(row.querySelectorAll("th,td")) + .map((cell) => clean(cell.textContent)) + .filter(Boolean); + if (cells.length >= 2) push(cells[0], cells.slice(1).join(" ")); + }); + + document.querySelectorAll("dt").forEach((dt) => { + let sibling = dt.nextElementSibling; + while (sibling && sibling.tagName.toLowerCase() !== "dd") { + sibling = sibling.nextElementSibling; + } + if (sibling) push(dt.textContent, sibling.textContent); + }); + + document.querySelectorAll("li,p,div,span").forEach((el) => { + const value = clean(el.textContent); + if (value.length < 4 || value.length > 500) return; + const match = value.match(/^([^:|\n]{2,80})\s*[:|]\s*(.{1,350})$/); + if (match) push(match[1], match[2]); + }); + + document.querySelectorAll("a[href]").forEach((el) => { + const label = clean(el.textContent) || clean(el.getAttribute("aria-label")); + const href = clean((el as HTMLAnchorElement).href); + if (!href || !/^https?:\/\//i.test(href)) return; + push(label || "link", href); + if (/website|homepage|site/i.test(label)) push("website", href); + }); + + return { + title, + description, + candidates, + bodyText: clean(document.body?.innerText ?? "").slice(0, 60_000), + }; + }); + + return { ...evidence, finalUrl }; +} + +async function buildExtractorScript( + input: TryRowExtractorInput, + url: string, + evidence: PageEvidence, + modelSlug: string, +): Promise { + const openrouter = createOpenRouter({ + apiKey: await requireOpenRouterApiKey(), + baseURL: process.env.OPENROUTER_BASE_URL, + }); + const result = await generateText({ + model: openrouter(modelSlug), + prompt: extractorBuilderPrompt(input, url, evidence), + maxOutputTokens: 4_000, + abortSignal: getSignal(input.datasetId), + }); + return sanitizeGeneratedScript(result.text); +} + +async function repairExtractorScript( + input: TryRowExtractorInput, + url: string, + evidence: PageEvidence, + script: string, + failure: string, + modelSlug: string, +): Promise { + const openrouter = createOpenRouter({ + apiKey: await requireOpenRouterApiKey(), + baseURL: process.env.OPENROUTER_BASE_URL, + }); + const result = await generateText({ + model: openrouter(modelSlug), + prompt: `${extractorBuilderPrompt(input, url, evidence)} + +The previous extractor failed validation. + +Failure: +${failure} + +Previous extractor: +\`\`\`js +${script} +\`\`\` + +Return a repaired extractor only.`, + maxOutputTokens: 4_000, + abortSignal: getSignal(input.datasetId), + }); + return sanitizeGeneratedScript(result.text); +} + +async function repairExtractorAfterRuntimeFailure( + apiKey: string, + input: TryRowExtractorInput, + url: string, + script: string, + failure: string, +): Promise { + const evidence = await probePage(apiKey, input.datasetId, url); + const modelSlug = input.extractorBuilderModel ?? DEFAULT_MODEL_IDS.EXTRACTOR_BUILDER; + const repaired = await repairExtractorScript( + input, url, - description: apiFacts?.description ?? cleanOptionalText(facts.description), - stars: apiFacts?.stars ?? parseCompactNumber(facts.stars), - forks: apiFacts?.forks ?? parseCompactNumber(facts.forks), - watchers: apiFacts?.watchers ?? parseCompactNumber(facts.watchers), - issues: parseCompactNumber(facts.issues) ?? apiFacts?.issues, - pullRequests: parseCompactNumber(facts.pullRequests) ?? apiFacts?.pullRequests, - language: apiFacts?.language ?? cleanOptionalText(facts.language), - license: apiFacts?.license ?? cleanOptionalText(facts.license), - latestCommitAt: apiFacts?.latestCommitAt ?? facts.latestCommitAt, - updatedAt: apiFacts?.updatedAt, - createdAt: apiFacts?.createdAt, - homepage: apiFacts?.homepage ?? cleanOptionalText(facts.homepage), - archived: apiFacts?.archived ?? facts.archived, - }; + evidence, + script, + failure, + modelSlug, + ); + const validation = await validateExtractorScript(apiKey, input, url, repaired); + if (!validation.ok) { + throw new Error(`repaired extractor failed validation: ${validation.reason}`); + } + + await convex.mutation(internal.datasetExtractors.upsert, { + datasetId: input.datasetId, + siteKey: siteKeyForInput(input, url), + columnsHash: hashColumns(input.columns), + script: repaired, + model: modelSlug, + probeSummary: summarizeEvidenceForStorage(evidence), + }); + return repaired; } -async function fetchGitHubApiFacts( - page: Page, - owner: string, - repo: string, -): Promise> { - const response = await page.request.get( - `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, - { - headers: { - Accept: "application/vnd.github+json", +async function markExtractorFailed( + input: TryRowExtractorInput, + url: string, + err: unknown, +): Promise { + await convex.mutation(internal.datasetExtractors.markFailed, { + datasetId: input.datasetId, + siteKey: siteKeyForInput(input, url), + columnsHash: hashColumns(input.columns), + error: err instanceof Error ? err.message : String(err), + }); +} + +function extractorBuilderPrompt( + input: TryRowExtractorInput, + url: string, + evidence: PageEvidence, +): string { + const columns = input.columns + .map( + (column) => { + const details = [ + column.isPrimaryKey ? "PRIMARY KEY" : undefined, + column.nullable === undefined ? undefined : `nullable=${column.nullable}`, + column.validationRegex ? `validation_regex=${JSON.stringify(column.validationRegex)}` : undefined, + column.normalizationHint ? `normalization_hint=${JSON.stringify(column.normalizationHint)}` : undefined, + column.description ? `retrieval_hint=${JSON.stringify(column.description)}` : undefined, + ].filter(Boolean); + return `- ${JSON.stringify(column.name)} (${column.type})${details.length > 0 ? ` [${details.join("; ")}]` : ""}`; }, - timeout: FETCH_TIMEOUT_MS, + ) + .join("\n"); + const primaryKeys = Object.entries(input.primaryKeys) + .map(([key, value]) => `- ${key}: ${value}`) + .join("\n") || "(none)"; + return `You are BigSet's extractorBuilder. Generate one reusable Playwright extractor for this dataset/page pattern. + +Dataset: +- Name: ${input.datasetName ?? ""} +- Description: ${input.description ?? ""} +- Retrieval strategy: ${input.retrievalStrategy ?? ""} +- Source hint: ${input.sourceHint ?? ""} + +Dataset columns: +${columns} + +Primary key values for the representative row: +${primaryKeys} + +Browser start URL: +${url} + +Browser probe: +${summarizeEvidenceForPrompt(evidence)} + +Requirements: +- Return only JavaScript. No Markdown. +- Define exactly: async function extract({ page, input, helpers }) { ... } +- Do not import anything. +- Do not use require, process, fs, child_process, fetch, eval, new Function, Node http/https/net modules, or direct database/network APIs. +- Use only the provided Playwright page, input, and helpers. +- The primary key values are arbitrary row input. Do not assume they are URLs. +- You may construct navigation URLs from input.primaryKeys, top-level primary key fields, input.sourceHint, input.urls, dataset name, or context. Example: a primary key like "tinyfish-io/bigset" may need page.goto(\`https://github.com/\${input.primaryKeys.repo_slug}\`). +- Navigate with page.goto(..., { waitUntil: "domcontentloaded" }) if needed. input.startUrl/input.url is only the browser start URL, not necessarily the row URL. +- Return an object: { data, sources, row_summary, how_found }. +- data must include every dataset column. Preserve primary key values from input.primaryKeys. +- Use "" for unknown values. Never fabricate. +- Returned values must be normalized to each column's type contract. +- If a column has validation_regex, the final returned value must match it after normalization. Use normalization_hint as the guide for converting page text to the canonical value. +- If a required selector/value is empty or does not match validation_regex, return "" for that field rather than guessing; BigSet will reject/repair failed validation. +- Prefer stable DOM selectors, JSON-LD, meta tags, tables, definition lists, and visible labels. +- The extractor must be reusable for other rows on the same site/page pattern.`; +} + +function summarizeEvidenceForPrompt(evidence: PageEvidence): string { + const candidates = Object.entries(evidence.candidates) + .slice(0, 80) + .map(([key, values]) => `- ${key}: ${values.slice(0, 3).join(" | ")}`) + .join("\n"); + return `Final URL: ${evidence.finalUrl} +Title: ${evidence.title ?? ""} +Description: ${evidence.description ?? ""} + +Candidate fields: +${candidates || "(none)"} + +Visible text excerpt: +${evidence.bodyText.slice(0, 8_000)}`; +} + +function summarizeEvidenceForStorage(evidence: PageEvidence): string { + return [ + `url=${evidence.finalUrl}`, + evidence.title ? `title=${evidence.title}` : undefined, + evidence.description ? `description=${evidence.description}` : undefined, + ] + .filter(Boolean) + .join("\n") + .slice(0, 2_000); +} + +function sanitizeGeneratedScript(text: string): string { + const fenced = text.match(/```(?:js|javascript)?\s*([\s\S]*?)```/i); + const raw = (fenced?.[1] ?? text) + .trim() + .replace(/\bexport\s+async\s+function\s+extract\b/, "async function extract") + .replace(/\bexport\s+function\s+extract\b/, "function extract"); + rejectDangerousScript(raw); + if (!/\basync\s+function\s+extract\s*\(/.test(raw) && !/\bfunction\s+extract\s*\(/.test(raw)) { + throw new Error("generated extractor did not define function extract"); + } + return raw; +} + +function rejectDangerousScript(script: string): void { + const dangerousPatterns = [ + /\bimport\s*(?:\(|[^("'])/i, + /\brequire\s*\(/i, + /\bprocess\b/i, + /\bfs\b/i, + /\bchild_process\b/i, + /\bworker_threads\b/i, + /\beval\s*\(/i, + /\bnew\s+Function\b/i, + /\bfetch\s*\(/i, + /\bXMLHttpRequest\b/i, + /\bWebSocket\b/i, + /\bnode:(?:http|https|net)\b/i, + /\bconvex\b/i, + ]; + const match = dangerousPatterns.find((pattern) => pattern.test(script)); + if (match) { + throw new Error(`generated extractor contains a blocked pattern: ${match}`); + } +} + +async function validateExtractorScript( + apiKey: string, + input: TryRowExtractorInput, + url: string, + script: string, +): Promise<{ ok: true } | { ok: false; reason: string }> { + try { + const raw = await runGeneratedExtractorScript(apiKey, input, url, script); + const extraction = buildRowFromExtractorResult(input, url, raw); + if (!hasExtractedNonPrimaryValue(extraction, input.columns)) { + return { ok: false, reason: "extractor returned no non-primary values" }; + } + return { ok: true }; + } catch (err) { + return { + ok: false, + reason: err instanceof Error ? err.message : String(err), + }; + } +} + +async function runGeneratedExtractorScript( + apiKey: string, + input: TryRowExtractorInput, + url: string, + script: string, +): Promise { + rejectDangerousScript(script); + const session = await createTinyFishBrowserSession(apiKey, url, input.datasetId); + const payload = JSON.stringify({ + cdpUrl: session.cdp_url, + url, + script, + input: { + ...input.primaryKeys, + url, + startUrl: url, + primaryKeys: input.primaryKeys, + urls: input.urls ?? [], + columns: input.columns, + datasetName: input.datasetName ?? "", + description: input.description ?? "", + retrievalStrategy: input.retrievalStrategy ?? "", + sourceHint: input.sourceHint ?? "", + context: input.context ?? "", }, - ); - if (!response.ok()) { - throw new Error(`GitHub API returned HTTP ${response.status()}`); + timeoutMs: EXTRACTOR_RUNNER_TIMEOUT_MS, + }); + + return await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--input-type=module", "-e", EXTRACTOR_RUNNER_SOURCE], { + cwd: BACKEND_ROOT, + env: {}, + stdio: ["pipe", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error("generated extractor timed out")); + }, EXTRACTOR_RUNNER_TIMEOUT_MS + 5_000); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (stdout.length > EXTRACTOR_SCRIPT_OUTPUT_LIMIT) { + child.kill("SIGKILL"); + reject(new Error("generated extractor output limit exceeded")); + } + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", (err) => { + clearTimeout(timeout); + reject(err); + }); + child.on("close", (code) => { + clearTimeout(timeout); + if (code !== 0) { + reject(new Error(stderr.trim() || `generated extractor exited ${code}`)); + return; + } + try { + resolve(JSON.parse(stdout) as GeneratedExtractorResult); + } catch (err) { + reject(new Error(`generated extractor returned invalid JSON: ${err}`)); + } + }); + child.stdin.end(payload); + }); +} + +function buildRowFromExtractorResult( + input: TryRowExtractorInput, + url: string, + result: GeneratedExtractorResult, + existingData?: Record, +): GenericExtraction { + if (!result || typeof result !== "object" || !result.data || typeof result.data !== "object") { + throw new Error("generated extractor did not return a data object"); } - const data = (await response.json()) as { - description?: string | null; - stargazers_count?: number; - forks_count?: number; - watchers_count?: number; - open_issues_count?: number; - language?: string | null; - license?: { spdx_id?: string | null; name?: string | null } | null; - pushed_at?: string | null; - updated_at?: string | null; - created_at?: string | null; - homepage?: string | null; - archived?: boolean; - html_url?: string; - }; + const data: Record = {}; + const extractedColumns: string[] = []; + const missingColumns: string[] = []; + const invalidColumns: string[] = []; + + for (const column of input.columns) { + const pkValue = findPrimaryKeyValue(column.name, input.primaryKeys); + if (pkValue !== undefined) { + const value = coerceColumnValue(pkValue, column, url) ?? pkValue; + const validationError = validateColumnValue(value, column); + if (validationError) { + throw new Error(`primary key "${column.name}" failed validation: ${validationError}`); + } + data[column.name] = value; + extractedColumns.push(column.name); + continue; + } + + const rawValue = result.data[column.name]; + const value = coerceColumnValue(rawValueToExtractedValue(rawValue), column, url); + if (value !== undefined) { + const validationError = validateColumnValue(value, column); + if (!validationError) { + data[column.name] = value; + extractedColumns.push(column.name); + continue; + } + invalidColumns.push(`${column.name}: ${validationError}`); + } + + if (existingData && existingData[column.name] !== undefined) { + data[column.name] = normalizeStoredValue(existingData[column.name]); + } else { + data[column.name] = ""; + } + missingColumns.push(column.name); + } + + if (invalidColumns.length > 0) { + throw new Error( + `generated extractor returned values that failed validation: ${invalidColumns.join("; ")}`, + ); + } + + const missingRequiredColumns = input.columns + .filter((column) => columnRequiresValue(column) && missingColumns.includes(column.name)) + .map((column) => column.name); + if (missingRequiredColumns.length > 0) { + throw new Error( + `generated extractor missed required columns: ${missingRequiredColumns.join(", ")}`, + ); + } + + const sources = Array.isArray(result.sources) + ? result.sources.filter((source): source is string => typeof source === "string" && isHttpUrl(source)) + : []; + const summary = String(result.row_summary ?? result.rowSummary ?? "").trim().slice(0, 500); return { - url: data.html_url, - description: data.description ?? undefined, - stars: data.stargazers_count, - forks: data.forks_count, - watchers: data.watchers_count, - issues: data.open_issues_count, - language: data.language ?? undefined, - license: data.license?.spdx_id || data.license?.name || undefined, - latestCommitAt: data.pushed_at ?? undefined, - updatedAt: data.updated_at ?? undefined, - createdAt: data.created_at ?? undefined, - homepage: data.homepage || undefined, - archived: data.archived, + data, + sources: sources.length > 0 ? sources : [url], + rowSummary: summary || url, + extractedColumns, + missingColumns, }; } -function buildGitHubRow( +function hasExtractedNonPrimaryValue( + extraction: GenericExtraction, columns: PopulateColumn[], - primaryKeys: Record, - facts: GitHubRepoFacts, -): Record | null { - const row: Record = {}; - - for (const column of columns) { - const pkValue = findPrimaryKeyValue(column.name, primaryKeys); - const rawValue = pkValue ?? valueForGitHubColumn(column.name, facts); - const value = coerceColumnValue(rawValue, column); - if (value === undefined) return null; - row[column.name] = value; +): boolean { + const pkColumns = new Set( + columns.filter((column) => column.isPrimaryKey).map((column) => column.name), + ); + return extraction.extractedColumns.some((columnName) => !pkColumns.has(columnName)); +} + +function rawValueToExtractedValue(value: unknown): ExtractedValue | undefined { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "boolean") return value; + if (typeof value === "string") return value; + if (value == null) return undefined; + return String(value); +} + +function columnRequiresValue(column: PopulateColumn): boolean { + return column.nullable === false; +} + +function validateColumnValue( + value: ExtractedValue, + column: PopulateColumn, +): string | undefined { + const pattern = column.validationRegex?.trim(); + if (!pattern) return undefined; + + const text = String(value).trim(); + if (!text) { + return column.nullable === true ? undefined : "empty value"; + } + + let regex: RegExp; + try { + regex = compileValidationRegex(pattern); + } catch (err) { + return `invalid schema validation regex ${JSON.stringify(pattern)}: ${ + err instanceof Error ? err.message : String(err) + }`; } - return row; + if (!regex.test(text)) { + return `value ${JSON.stringify(text).slice(0, 120)} does not match ${JSON.stringify(pattern)}`; + } + return undefined; } -function githubRowSummary(facts: GitHubRepoFacts): string { - return facts.description ? `${facts.fullName}: ${facts.description}` : facts.fullName; +function compileValidationRegex(pattern: string): RegExp { + const literal = pattern.match(/^\/([\s\S]*)\/([a-z]*)$/i); + if (!literal) return new RegExp(pattern); + const flags = Array.from(new Set(literal[2].replace(/[gy]/g, "").split(""))).join(""); + return new RegExp(literal[1], flags); +} + +function siteKeyForUrl(value: string): string { + try { + const url = new URL(value); + const firstPathSegment = url.pathname.split("/").filter(Boolean)[0]; + return [url.hostname.toLowerCase(), firstPathSegment].filter(Boolean).join("/"); + } catch { + return "invalid-url"; + } +} + +function siteKeyForInput(input: TryRowExtractorInput, browserStartUrl: string): string { + const sourceUrl = [ + input.sourceHint, + ...(input.urls ?? []), + ...Object.values(input.primaryKeys), + ] + .map(coerceHttpUrl) + .find((value): value is string => Boolean(value)); + return siteKeyForUrl(sourceUrl ?? browserStartUrl); +} + +function hashColumns(columns: PopulateColumn[]): string { + const payload = columns.map((column) => ({ + name: column.name, + type: column.type, + description: column.description ?? "", + isPrimaryKey: Boolean(column.isPrimaryKey), + nullable: column.nullable, + validationRegex: column.validationRegex ?? "", + normalizationHint: column.normalizationHint ?? "", + })); + return createHash("sha256").update(JSON.stringify(payload)).digest("hex"); } function changedColumnNames( - nextRow: Record, + nextRow: Record, existingData: Record, columns: PopulateColumn[], ): string[] { @@ -545,7 +1131,7 @@ function changedColumnNames( } function valuesEqualForColumn( - nextValue: string | number | boolean | undefined, + nextValue: ExtractedValue | undefined, existingValue: unknown, column: PopulateColumn, ): boolean { @@ -553,17 +1139,25 @@ function valuesEqualForColumn( switch (column.type) { case "number": { + const nextNumber = + typeof nextValue === "number" + ? nextValue + : Number(String(nextValue ?? "").replace(/,/g, "")); const existingNumber = typeof existingValue === "number" ? existingValue : Number(String(existingValue ?? "").replace(/,/g, "")); - return Number.isFinite(existingNumber) && existingNumber === nextValue; + return ( + Number.isFinite(nextNumber) && + Number.isFinite(existingNumber) && + existingNumber === nextNumber + ); } case "boolean": if (typeof existingValue === "boolean") return existingValue === nextValue; if (/^(true|yes)$/i.test(String(existingValue))) return nextValue === true; if (/^(false|no)$/i.test(String(existingValue))) return nextValue === false; - return false; + return String(existingValue ?? "").trim() === String(nextValue).trim(); case "date": { const nextDate = new Date(String(nextValue)); const existingDate = new Date(String(existingValue ?? "")); @@ -591,86 +1185,26 @@ function findPrimaryKeyValue( return entry?.[1]; } -function valueForGitHubColumn( - columnName: string, - facts: GitHubRepoFacts, -): string | number | boolean | undefined { - const normalized = normalizeFieldName(columnName); - if (matches(normalized, ["repository_url", "repo_url", "github_url", "url", "link"])) { - return facts.url; - } - if (matches(normalized, ["repository_name", "repo_name"])) { - return facts.fullName; - } - if (matches(normalized, ["repository", "repo", "name"])) { - return facts.repo; - } - if (matches(normalized, ["full_name", "repository_full_name", "repo_full_name"])) { - return facts.fullName; - } - if (matches(normalized, ["owner", "organization", "org", "user"])) { - return facts.owner; - } - if (matches(normalized, ["description", "summary", "about"])) { - return facts.description; - } - if (matches(normalized, ["stars", "star_count", "stargazers", "stargazer_count"])) { - return facts.stars; - } - if (matches(normalized, ["forks", "fork_count"])) { - return facts.forks; - } - if (matches(normalized, ["watchers", "watcher_count"])) { - return facts.watchers; - } - if (matches(normalized, ["issues", "open_issues", "open_issue_count"])) { - return facts.issues; - } - if (matches(normalized, ["pull_requests", "open_pull_requests", "prs", "open_prs", "pr_count", "open_pr_count"])) { - return facts.pullRequests; - } - if (matches(normalized, ["language", "primary_language"])) { - return facts.language; - } - if (matches(normalized, ["license", "license_type", "license_spdx"])) { - return facts.license; - } - if (matches(normalized, ["latest_commit", "latest_commit_at", "last_commit", "pushed_at", "activity", "last_activity"])) { - return facts.latestCommitAt; - } - if (matches(normalized, ["updated", "updated_at", "last_updated"])) { - return facts.updatedAt; - } - if (matches(normalized, ["created", "created_at"])) { - return facts.createdAt; - } - if (matches(normalized, ["homepage", "website", "site"])) { - return facts.homepage; - } - if (matches(normalized, ["archived", "is_archived"])) { - return facts.archived; - } - return undefined; -} - function coerceColumnValue( value: string | number | boolean | undefined, column: PopulateColumn, -): string | number | boolean | undefined { + baseUrl: string, +): ExtractedValue | undefined { if (value === undefined || value === "") return undefined; switch (column.type) { case "number": { if (typeof value === "number") return Number.isFinite(value) ? value : undefined; - const parsed = Number(String(value).replace(/,/g, "")); - return Number.isFinite(parsed) ? parsed : undefined; + return parseCompactNumber(String(value)); } case "boolean": if (typeof value === "boolean") return value; - if (/^(true|yes)$/i.test(String(value))) return true; - if (/^(false|no)$/i.test(String(value))) return false; + if (/^(true|yes|available|in stock|active)$/i.test(String(value).trim())) return true; + if (/^(false|no|unavailable|out of stock|inactive)$/i.test(String(value).trim())) { + return false; + } return undefined; case "url": - return isHttpUrl(String(value)) ? normalizeUrl(String(value)) : undefined; + return normalizeMaybeRelativeUrl(String(value), baseUrl); case "date": { const date = new Date(String(value)); return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); @@ -680,20 +1214,38 @@ function coerceColumnValue( } } +function normalizeMaybeRelativeUrl( + value: string, + baseUrl: string, +): string | undefined { + try { + const url = new URL(value, baseUrl); + return url.protocol === "http:" || url.protocol === "https:" + ? url.toString() + : undefined; + } catch { + return undefined; + } +} + +function normalizeStoredValue(value: unknown): ExtractedValue { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "boolean") return value; + return String(value ?? ""); +} + function parseCompactNumber(value: string | undefined): number | undefined { if (!value) return undefined; - const match = value.replace(/,/g, "").match(/([\d.]+)\s*([kmb])?/i); + const normalized = value.replace(/,/g, ""); + const match = + normalized.match(/[$€£]?\s*([\d]+(?:\.\d+)?)\s*([kmb])?/i) ?? + normalized.match(/([\d]+(?:\.\d+)?)/); if (!match) return undefined; const base = Number(match[1]); if (!Number.isFinite(base)) return undefined; const suffix = match[2]?.toLowerCase(); const multiplier = suffix === "k" ? 1_000 : suffix === "m" ? 1_000_000 : suffix === "b" ? 1_000_000_000 : 1; - return Math.round(base * multiplier); -} - -function cleanOptionalText(value: string | undefined | null): string | undefined { - const trimmed = value?.trim(); - return trimmed || undefined; + return Math.round(base * multiplier * 100) / 100; } function normalizeFieldName(value: string): string { @@ -703,7 +1255,3 @@ function normalizeFieldName(value: string): string { .replace(/[^a-z0-9]+/g, "_") .replace(/^_+|_+$/g, ""); } - -function matches(value: string, candidates: string[]): boolean { - return candidates.includes(value); -} diff --git a/frontend/app/dashboard/settings/models/page.tsx b/frontend/app/dashboard/settings/models/page.tsx index eb3bdf3..81aac6f 100644 --- a/frontend/app/dashboard/settings/models/page.tsx +++ b/frontend/app/dashboard/settings/models/page.tsx @@ -81,8 +81,16 @@ export default function ModelSettingsPage() { .then((config) => { if (active) { setEffectiveConfig(config); - setExtractorConcurrency(config.rowExtractorConcurrency); - setExtractorAttempts(config.rowExtractorBrowserAttempts); + setExtractorConcurrency( + Number.isFinite(config.rowExtractorConcurrency) + ? config.rowExtractorConcurrency + : 5, + ); + setExtractorAttempts( + Number.isFinite(config.rowExtractorBrowserAttempts) + ? config.rowExtractorBrowserAttempts + : 2, + ); } }) .catch(() => { diff --git a/frontend/app/dataset/new/page.tsx b/frontend/app/dataset/new/page.tsx index 63b8ebf..4dbed96 100644 --- a/frontend/app/dataset/new/page.tsx +++ b/frontend/app/dataset/new/page.tsx @@ -22,6 +22,9 @@ interface ProposedColumn { type: ColumnType; description: string; isPrimaryKey: boolean; + nullable?: boolean; + validationRegex?: string; + normalizationHint?: string; } type Step = "describe" | "generating" | "review"; @@ -52,6 +55,9 @@ function mapBackendColumn(col: InferredColumn, index: number): ProposedColumn { type: BACKEND_TYPE_MAP[col.type], description: col.retrieval_hint, isPrimaryKey: col.is_primary_key, + nullable: col.nullable, + validationRegex: col.validation_regex, + normalizationHint: col.normalization_hint, }; } @@ -197,6 +203,9 @@ export default function NewDatasetPage() { type: c.type, description: c.description || undefined, isPrimaryKey: c.isPrimaryKey || undefined, + nullable: c.nullable, + validationRegex: c.validationRegex || undefined, + normalizationHint: c.normalizationHint || undefined, })), retrievalStrategy: retrievalStrategy ?? undefined, sourceHint: sourceHint || undefined, diff --git a/frontend/components/settings/types.ts b/frontend/components/settings/types.ts index e04d2fc..3ef7d1b 100644 --- a/frontend/components/settings/types.ts +++ b/frontend/components/settings/types.ts @@ -16,4 +16,5 @@ export const MODEL_ROLES: ModelRole[] = [ { key: "schemaInference", label: "Schema Inference", description: "Used to generate dataset schema from natural language" }, { key: "populateOrchestrator", label: "Populate Orchestrator", description: "Coordinates row population workflow" }, { key: "investigateSubagent", label: "Investigate Subagent", description: "Researches individual entities" }, -]; \ No newline at end of file + { key: "extractorBuilder", label: "Extractor Builder", description: "Writes reusable Playwright extractors from browser probes" }, +]; diff --git a/frontend/convex/_generated/api.d.ts b/frontend/convex/_generated/api.d.ts index bc8dc15..b09c911 100644 --- a/frontend/convex/_generated/api.d.ts +++ b/frontend/convex/_generated/api.d.ts @@ -8,6 +8,7 @@ * @module */ +import type * as datasetExtractors from "../datasetExtractors.js"; import type * as datasetRows from "../datasetRows.js"; import type * as datasets from "../datasets.js"; import type * as lib_authz from "../lib/authz.js"; @@ -27,6 +28,7 @@ import type { } from "convex/server"; declare const fullApi: ApiFromModules<{ + datasetExtractors: typeof datasetExtractors; datasetRows: typeof datasetRows; datasets: typeof datasets; "lib/authz": typeof lib_authz; diff --git a/frontend/convex/datasetExtractors.ts b/frontend/convex/datasetExtractors.ts new file mode 100644 index 0000000..8782318 --- /dev/null +++ b/frontend/convex/datasetExtractors.ts @@ -0,0 +1,95 @@ +import { internalMutation, internalQuery } from "./_generated/server.js"; +import { v } from "convex/values"; + +export const getActive = internalQuery({ + args: { + datasetId: v.id("datasets"), + siteKey: v.string(), + columnsHash: v.string(), + }, + handler: async (ctx, args) => { + const extractor = await ctx.db + .query("datasetExtractors") + .withIndex("by_dataset_site", (q) => + q.eq("datasetId", args.datasetId).eq("siteKey", args.siteKey), + ) + .first(); + + if ( + !extractor || + extractor.status !== "active" || + extractor.columnsHash !== args.columnsHash + ) { + return null; + } + + return extractor; + }, +}); + +export const upsert = internalMutation({ + args: { + datasetId: v.id("datasets"), + siteKey: v.string(), + columnsHash: v.string(), + script: v.string(), + model: v.optional(v.string()), + probeSummary: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const now = Date.now(); + const existing = await ctx.db + .query("datasetExtractors") + .withIndex("by_dataset_site", (q) => + q.eq("datasetId", args.datasetId).eq("siteKey", args.siteKey), + ) + .first(); + + const patch = { + columnsHash: args.columnsHash, + script: args.script, + status: "active" as const, + model: args.model, + probeSummary: args.probeSummary, + lastError: undefined, + updatedAt: now, + }; + + if (existing) { + await ctx.db.patch(existing._id, patch); + return existing._id; + } + + return await ctx.db.insert("datasetExtractors", { + datasetId: args.datasetId, + siteKey: args.siteKey, + createdAt: now, + ...patch, + }); + }, +}); + +export const markFailed = internalMutation({ + args: { + datasetId: v.id("datasets"), + siteKey: v.string(), + columnsHash: v.string(), + error: v.string(), + }, + handler: async (ctx, args) => { + const existing = await ctx.db + .query("datasetExtractors") + .withIndex("by_dataset_site", (q) => + q.eq("datasetId", args.datasetId).eq("siteKey", args.siteKey), + ) + .first(); + if (!existing || existing.columnsHash !== args.columnsHash) return null; + + await ctx.db.patch(existing._id, { + status: "failed", + lastError: args.error.slice(0, 1_000), + updatedAt: Date.now(), + }); + return existing._id; + }, +}); diff --git a/frontend/convex/datasets.ts b/frontend/convex/datasets.ts index 791671b..829e604 100644 --- a/frontend/convex/datasets.ts +++ b/frontend/convex/datasets.ts @@ -31,6 +31,9 @@ const columnValidator = v.object({ ), description: v.optional(v.string()), isPrimaryKey: v.optional(v.boolean()), + nullable: v.optional(v.boolean()), + validationRegex: v.optional(v.string()), + normalizationHint: v.optional(v.string()), }); function refreshCadenceFromLegacyLabel( @@ -282,6 +285,8 @@ export const claimScheduledRefreshInternal = internalMutation({ datasetName: dataset.name, description: dataset.description, columns: dataset.columns, + retrievalStrategy: dataset.retrievalStrategy, + sourceHint: dataset.sourceHint, ownerId: dataset.ownerId, maxRowCount: dataset.maxRowCount ?? DEFAULT_MAX_ROW_COUNT, }, diff --git a/frontend/convex/modelConfig.ts b/frontend/convex/modelConfig.ts index 83cd9f5..a6223ea 100644 --- a/frontend/convex/modelConfig.ts +++ b/frontend/convex/modelConfig.ts @@ -91,6 +91,7 @@ export const upsert = mutation({ schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), + extractorBuilder: v.optional(v.string()), rowExtractorConcurrency: v.optional(v.number()), rowExtractorBrowserAttempts: v.optional(v.number()), }, @@ -106,12 +107,14 @@ export const upsert = mutation({ schemaInference?: string; populateOrchestrator?: string; investigateSubagent?: string; + extractorBuilder?: string; rowExtractorConcurrency?: number; rowExtractorBrowserAttempts?: number; } = { provider }; if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; + if (args.extractorBuilder !== undefined) patch.extractorBuilder = args.extractorBuilder; if (args.rowExtractorConcurrency !== undefined) patch.rowExtractorConcurrency = args.rowExtractorConcurrency; if (args.rowExtractorBrowserAttempts !== undefined) patch.rowExtractorBrowserAttempts = args.rowExtractorBrowserAttempts; @@ -149,6 +152,7 @@ export const upsertInternal = internalMutation({ schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), + extractorBuilder: v.optional(v.string()), rowExtractorConcurrency: v.optional(v.number()), rowExtractorBrowserAttempts: v.optional(v.number()), }, @@ -161,12 +165,14 @@ export const upsertInternal = internalMutation({ schemaInference?: string; populateOrchestrator?: string; investigateSubagent?: string; + extractorBuilder?: string; rowExtractorConcurrency?: number; rowExtractorBrowserAttempts?: number; } = { provider }; if (args.schemaInference !== undefined) patch.schemaInference = args.schemaInference; if (args.populateOrchestrator !== undefined) patch.populateOrchestrator = args.populateOrchestrator; if (args.investigateSubagent !== undefined) patch.investigateSubagent = args.investigateSubagent; + if (args.extractorBuilder !== undefined) patch.extractorBuilder = args.extractorBuilder; if (args.rowExtractorConcurrency !== undefined) patch.rowExtractorConcurrency = args.rowExtractorConcurrency; if (args.rowExtractorBrowserAttempts !== undefined) patch.rowExtractorBrowserAttempts = args.rowExtractorBrowserAttempts; diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index b978e8f..da29b7f 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -65,6 +65,9 @@ export default defineSchema({ ), description: v.optional(v.string()), isPrimaryKey: v.optional(v.boolean()), + nullable: v.optional(v.boolean()), + validationRegex: v.optional(v.string()), + normalizationHint: v.optional(v.string()), }) ), retrievalStrategy: v.optional( @@ -158,12 +161,26 @@ export default defineSchema({ schemaInference: v.optional(v.string()), populateOrchestrator: v.optional(v.string()), investigateSubagent: v.optional(v.string()), + extractorBuilder: v.optional(v.string()), rowExtractorConcurrency: v.optional(v.number()), rowExtractorBrowserAttempts: v.optional(v.number()), }) .index("by_user", ["userId"]) .index("by_user_provider", ["userId", "provider"]), + datasetExtractors: defineTable({ + datasetId: v.id("datasets"), + siteKey: v.string(), + columnsHash: v.string(), + script: v.string(), + status: v.union(v.literal("active"), v.literal("failed")), + model: v.optional(v.string()), + probeSummary: v.optional(v.string()), + lastError: v.optional(v.string()), + createdAt: v.number(), + updatedAt: v.number(), + }).index("by_dataset_site", ["datasetId", "siteKey"]), + localCredentials: defineTable({ service: v.union( v.literal("tinyfish"), diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index c05f54b..9e92ada 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -15,6 +15,8 @@ export interface InferredColumn { is_enumerable: boolean; retrieval_hint: string; nullable: boolean; + validation_regex?: string; + normalization_hint?: string; } export interface PopulateColumn { @@ -22,6 +24,9 @@ export interface PopulateColumn { type: "text" | "number" | "boolean" | "url" | "date"; description?: string; isPrimaryKey?: boolean; + nullable?: boolean; + validationRegex?: string; + normalizationHint?: string; } export interface PopulateStartResult { @@ -36,13 +41,14 @@ export interface WorkflowResult { /** * The effective model config — always complete, never null. - * schemaInference / populateOrchestrator / investigateSubagent are always strings + * schemaInference / populateOrchestrator / investigateSubagent / extractorBuilder are always strings * (user preference or system default from env). */ export interface EffectiveModelConfig { schemaInference: string; populateOrchestrator: string; investigateSubagent: string; + extractorBuilder: string; rowExtractorConcurrency: number; rowExtractorBrowserAttempts: number; } @@ -55,6 +61,7 @@ export interface SavedModelConfig { schemaInference: string | null; populateOrchestrator: string | null; investigateSubagent: string | null; + extractorBuilder: string | null; rowExtractorConcurrency: number | null; rowExtractorBrowserAttempts: number | null; } @@ -138,6 +145,8 @@ function normalizeEffectiveModelConfig( typeof config?.investigateSubagent === "string" ? config.investigateSubagent : "", + extractorBuilder: + typeof config?.extractorBuilder === "string" ? config.extractorBuilder : "", rowExtractorConcurrency: normalizeIntegerSetting( config?.rowExtractorConcurrency, DEFAULT_ROW_EXTRACTOR_CONCURRENCY, From d29b6243d94517f4720e945187be45d2880a3086 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Fri, 12 Jun 2026 13:34:20 -0700 Subject: [PATCH 07/16] mostly working --- README.md | 2 +- backend/prompts/schema-inference.txt | 5 + backend/src/config/agent-output-tokens.ts | 6 + backend/src/index.ts | 73 +- backend/src/mastra/agents/populate.ts | 3 +- backend/src/mastra/tools/investigate-tool.ts | 27 +- backend/src/mastra/workflows/populate.ts | 18 +- backend/src/mastra/workflows/update.ts | 10 +- backend/src/pipeline/codification.ts | 183 +++++ backend/src/pipeline/populate.ts | 18 + backend/src/pipeline/schema-inference.ts | 9 +- backend/src/pipeline/types.ts | 37 + .../src/row-extractors/try-row-extractor.ts | 736 ++++++++++++++---- frontend/app/dataset/new/page.tsx | 29 +- frontend/app/setup/page.tsx | 1 + frontend/convex/datasetExtractors.ts | 6 +- frontend/convex/datasets.ts | 46 ++ frontend/convex/schema.ts | 29 + frontend/lib/backend.ts | 33 + makefiles/Makefile | 4 +- 20 files changed, 1088 insertions(+), 187 deletions(-) create mode 100644 backend/src/config/agent-output-tokens.ts create mode 100644 backend/src/pipeline/codification.ts diff --git a/README.md b/README.md index 4b6a7e9..7537cf1 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,7 @@ This is idempotent; safe to run multiple times. 7. **Configures Convex auth** — sets `BIGSET_LOCAL_MODE=1` for the local app. 8. **Deploys Convex schema** — pushes the table schema and functions from `frontend/convex/` to the running instance. 9. **Starts remaining services** — brings up the frontend, backend, and Mastra. These read the now-populated `.env` including the admin key. -10. **Streams logs** — tails all container logs so you can see what's happening. `Ctrl+C` to stop watching (containers keep running). +10. **Streams app logs** — tails frontend, backend, and Mastra logs. `Ctrl+C` to stop watching (containers keep running). Convex logs are still available with `docker compose -f docker-compose.dev.yml logs -f convex`. ### Commands diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt index 8f5a720..cae42a8 100644 --- a/backend/prompts/schema-inference.txt +++ b/backend/prompts/schema-inference.txt @@ -11,6 +11,11 @@ Your job is to: - `hybrid` — unclear; the pipeline will try search_fetch first and fall back to browser. 5. Set `source_hint` to a specific URL whenever possible (e.g. `https://www.ycombinator.com/companies?industry=Fintech`). Avoid vague descriptions. 6. Write a `retrieval_hint` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. +7. Set `codification_profile` conservatively: + - `disabled` for broad web research, arbitrary unrelated domains, snippet-only work, or known-blocked sources. + - `candidate` when rows likely share stable page families and a reusable Playwright extractor might work after seeing a representative row. + - `required` when one authoritative browser-heavy source/directory is clearly the intended repeated extraction path. + - `unknown` only when evidence is genuinely insufficient; prefer `disabled` for broad web datasets. Rules: diff --git a/backend/src/config/agent-output-tokens.ts b/backend/src/config/agent-output-tokens.ts new file mode 100644 index 0000000..ffdd8ec --- /dev/null +++ b/backend/src/config/agent-output-tokens.ts @@ -0,0 +1,6 @@ +export const AGENT_MAX_OUTPUT_TOKENS = { + POPULATE_ORCHESTRATOR: 20_000, + INVESTIGATE_SUBAGENT: 16_000, + REFRESH_AGENT: 12_000, + EXTRACTOR_BUILDER: 20_000, +} as const; diff --git a/backend/src/index.ts b/backend/src/index.ts index a75846f..35b32b6 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -7,6 +7,10 @@ import { env } from "./env.js"; import clerkAuthPlugin, { requireAuth, getUserEmail } from "./clerk-auth.js"; import { inferSchema } from "./pipeline/schema-inference.js"; import { datasetContextSchema, type DatasetContext } from "./pipeline/populate.js"; +import { + normalizeCodificationProfile, + schemaCodificationProfileToRuntime, +} from "./pipeline/codification.js"; import { populateWorkflow } from "./mastra/workflows/populate.js"; import { updateWorkflow } from "./mastra/workflows/update.js"; import { convex, internal } from "./convex.js"; @@ -133,6 +137,30 @@ async function beginDatasetPopulate( return claim.outcome; } +async function withCodificationProfile( + input: DatasetContext, + logger: FastifyBaseLogger, +): Promise { + if (input.codificationProfile) return input; + + const codificationProfile = normalizeCodificationProfile(undefined, input); + try { + await convex.mutation(internal.datasets.setCodificationProfileInternal, { + id: input.datasetId, + codificationProfile, + }); + } catch (err) { + logger.warn( + { err, datasetId: input.datasetId }, + "Failed to persist legacy codification profile; continuing with in-memory profile", + ); + } + return { + ...input, + codificationProfile, + }; +} + async function sendDatasetReadyNotification({ logger, clerk, @@ -635,9 +663,8 @@ function startLocalRefreshScheduler( const dataset = claim.dataset; const { getModelConfig } = await import("./config/models.js"); const modelConfig = await getModelConfig(dataset.ownerId); - - void runScheduledUpdateWorkflowInBackground({ - input: { + const input = await withCodificationProfile( + { datasetId: dataset.datasetId, datasetName: dataset.datasetName, description: dataset.description, @@ -645,7 +672,13 @@ function startLocalRefreshScheduler( columns: dataset.columns, retrievalStrategy: dataset.retrievalStrategy, sourceHint: dataset.sourceHint, + codificationProfile: dataset.codificationProfile, }, + logger, + ); + + void runScheduledUpdateWorkflowInBackground({ + input, run, authorizedUserId: dataset.ownerId, logger, @@ -944,6 +977,7 @@ fastify.post("/cli/datasets", async (req, reply) => { columns, retrievalStrategy: schema.retrieval_strategy, sourceHint: schema.source_hint, + codificationProfile: schemaCodificationProfileToRuntime(schema.codification_profile), }, ); @@ -1040,9 +1074,8 @@ fastify.post("/cli/datasets/:datasetId/populate", async (req, reply) => { const modelConfig = await getModelConfig(ownerId); const run = await populateWorkflow.createRun(); const controller = registerDataset(dataset._id); - - void runPopulateWorkflowInBackground({ - input: { + const input = await withCodificationProfile( + { datasetId: dataset._id, datasetName: dataset.name, description: dataset.description, @@ -1050,7 +1083,13 @@ fastify.post("/cli/datasets/:datasetId/populate", async (req, reply) => { columns: dataset.columns, retrievalStrategy: dataset.retrievalStrategy, sourceHint: dataset.sourceHint, + codificationProfile: dataset.codificationProfile, }, + req.log, + ); + + void runPopulateWorkflowInBackground({ + input, run, controller, authorizedUserId: ownerId, @@ -1297,14 +1336,19 @@ await fastify.register(async (instance) => { // arriving before registerDataset runs inside the background function // would incorrectly force-transition an active run to "failed". const controller = registerDataset(parsed.data.datasetId); - - void runPopulateWorkflowInBackground({ - input: { + const input = await withCodificationProfile( + { ...parsed.data, maxRowCount: dataset.maxRowCount ?? parsed.data.maxRowCount, retrievalStrategy: dataset.retrievalStrategy ?? parsed.data.retrievalStrategy, sourceHint: dataset.sourceHint ?? parsed.data.sourceHint, + codificationProfile: dataset.codificationProfile ?? parsed.data.codificationProfile, }, + req.log, + ); + + void runPopulateWorkflowInBackground({ + input, run, controller, authorizedUserId: auth.userId, @@ -1385,14 +1429,19 @@ await fastify.register(async (instance) => { // arriving before registerDataset runs inside the background function // would incorrectly force-transition an active run to "failed". registerDataset(parsed.data.datasetId); - - void runUpdateWorkflowInBackground({ - input: { + const input = await withCodificationProfile( + { ...parsed.data, maxRowCount: dataset.maxRowCount ?? parsed.data.maxRowCount, retrievalStrategy: dataset.retrievalStrategy ?? parsed.data.retrievalStrategy, sourceHint: dataset.sourceHint ?? parsed.data.sourceHint, + codificationProfile: dataset.codificationProfile ?? parsed.data.codificationProfile, }, + req.log, + ); + + void runUpdateWorkflowInBackground({ + input, run, authorizedUserId: auth.userId, logger: req.log, diff --git a/backend/src/mastra/agents/populate.ts b/backend/src/mastra/agents/populate.ts index bbdffd0..7ce8cab 100644 --- a/backend/src/mastra/agents/populate.ts +++ b/backend/src/mastra/agents/populate.ts @@ -3,7 +3,7 @@ import { createLanguageModel, type LlmProviderConfig } from "../../config/llm.js import { buildSubagentTool } from "../tools/investigate-tool.js"; import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; -import type { PopulateColumn } from "../../pipeline/populate.js"; +import type { CodificationProfile, PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; export interface PopulateAgentDatasetContext { @@ -11,6 +11,7 @@ export interface PopulateAgentDatasetContext { description: string; retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; sourceHint?: string; + codificationProfile?: CodificationProfile; } function buildInstructions(maxRowCount: number): string { diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 2fd4ab6..8432611 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -3,11 +3,12 @@ import { z } from "zod"; import { convex, internal } from "../../convex.js"; import { buildInvestigateAgent } from "../agents/investigate.js"; import type { AuthContext } from "../workflows/populate.js"; -import type { PopulateColumn } from "../../pipeline/populate.js"; +import type { CodificationProfile, PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; import { getSignal } from "../../abort-registry.js"; import { tryRowExtractor } from "../../row-extractors/try-row-extractor.js"; import type { LlmProviderConfig } from "../../config/llm.js"; +import { AGENT_MAX_OUTPUT_TOKENS } from "../../config/agent-output-tokens.js"; const keyValueSchema = z.object({ column: z.string().min(1), @@ -55,6 +56,7 @@ interface DatasetContextForExtractor { description: string; retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; sourceHint?: string; + codificationProfile?: CodificationProfile; } function parseInvestigateResult( @@ -73,6 +75,12 @@ function parseInvestigateResult( }; } +function isIncompleteExtractorFailure(reason: string): boolean { + return /generated extractor (missed columns|missed required columns|returned values that failed validation)|repaired extractor failed validation|extractor returned no non-primary values/i.test( + reason, + ); +} + /** * Build the run_subagent tool scoped to one dataset. * @@ -127,6 +135,7 @@ export function buildSubagentTool( description: datasetContext.description, retrievalStrategy: datasetContext.retrievalStrategy, sourceHint: datasetContext.sourceHint, + codificationProfile: datasetContext.codificationProfile, browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, extractorBuilderModel: authContext.modelConfig.extractorBuilder, }); @@ -154,6 +163,14 @@ export function buildSubagentTool( console.warn( `[run_subagent] row extractor failed entity="${entity_hint}" reason="${extractorResult.reason}"`, ); + if (isIncompleteExtractorFailure(extractorResult.reason)) { + return { + inserted: false, + reason: `Browser extractor did not produce a complete row: ${extractorResult.reason}`, + row_summary: undefined, + clues: "Improve the reusable browser extractor or choose a schema/source where every column is visible.", + }; + } } else if (extractorResult.status === "miss") { console.log( `[run_subagent] row extractor missed entity="${entity_hint}" reason="${extractorResult.reason}"`, @@ -191,7 +208,13 @@ Context (partial data already found): ${context}${urlsBlock}${notesBlock}`; const abortSignal = getSignal(authorizedDatasetId); - const result = await agent.generate(prompt, { abortSignal, maxSteps: 25 }); + const result = await agent.generate(prompt, { + abortSignal, + maxSteps: 25, + modelSettings: { + maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.INVESTIGATE_SUBAGENT, + }, + }); if (metrics) { // Use result.toolCalls (the flat accumulated list across all steps) rather // than iterating result.steps[n].toolCalls. The per-step arrays are snapshots diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index 5867ebf..7f7746e 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -1,10 +1,15 @@ import { createStep, createWorkflow } from "@mastra/core/workflows"; import { z } from "zod"; import { generateText } from "ai"; -import { datasetContextSchema, populateColumnSchema } from "../../pipeline/populate.js"; +import { + codificationProfileSchema, + datasetContextSchema, + populateColumnSchema, +} from "../../pipeline/populate.js"; import { convex, internal } from "../../convex.js"; import { DEFAULT_MODEL_IDS } from "../../config/models.js"; import { createLanguageModel } from "../../config/llm.js"; +import { AGENT_MAX_OUTPUT_TOKENS } from "../../config/agent-output-tokens.js"; import { requireLlmProviderConfig } from "../../local-credentials.js"; import { buildPopulateAgent } from "../agents/populate.js"; import { RunMetrics } from "../run-metrics.js"; @@ -164,6 +169,7 @@ const buildPromptOutputSchema = z.object({ description: z.string(), retrievalStrategy: z.enum(["search_fetch", "browser", "hybrid"]).optional(), sourceHint: z.string().optional(), + codificationProfile: z.optional(codificationProfileSchema), }); const buildPromptStep = createStep({ @@ -226,6 +232,7 @@ Stop the populate run as soon as the dataset reaches ${inputData.maxRowCount} ro description: inputData.description, retrievalStrategy: inputData.retrievalStrategy, sourceHint: inputData.sourceHint, + codificationProfile: inputData.codificationProfile, }; }, }); @@ -266,11 +273,18 @@ const agentStep = createStep({ description: inputData.description, retrievalStrategy: inputData.retrievalStrategy, sourceHint: inputData.sourceHint, + codificationProfile: inputData.codificationProfile, }, metrics, ); const abortSignal = getSignal(inputData.authorizedDatasetId); - const result = await agent.generate(inputData.prompt, { abortSignal, maxSteps: 80 }); + const result = await agent.generate(inputData.prompt, { + abortSignal, + maxSteps: 80, + modelSettings: { + maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.POPULATE_ORCHESTRATOR, + }, + }); metrics.addOrchestratorResult(result); // Use result.toolCalls (flat accumulated list) — same reasoning as investigate-tool.ts. metrics.countToolCalls(result.toolCalls ?? []); diff --git a/backend/src/mastra/workflows/update.ts b/backend/src/mastra/workflows/update.ts index bf1d3b8..2a2a2a7 100644 --- a/backend/src/mastra/workflows/update.ts +++ b/backend/src/mastra/workflows/update.ts @@ -9,6 +9,7 @@ import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; import { getSignal } from "../../abort-registry.js"; import { tryRefreshRowExtractor } from "../../row-extractors/try-row-extractor.js"; +import { AGENT_MAX_OUTPUT_TOKENS } from "../../config/agent-output-tokens.js"; export const updateInputSchema = datasetContextSchema.extend({ authContext: authContextSchema, @@ -137,6 +138,7 @@ const refreshRowsStep = createStep({ description, retrievalStrategy, sourceHint, + codificationProfile: inputData.codificationProfile, browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, extractorBuilderModel: authContext.modelConfig.extractorBuilder, }); @@ -205,7 +207,13 @@ ${row.rowSummary ? `\nPrevious summary: ${row.rowSummary}` : ""} ${row.howFound ? `\nPreviously found via: ${row.howFound}` : ""}`; const abortSignal = getSignal(datasetId); - const result = await agent.generate(prompt, { abortSignal, maxSteps: 10 }); + const result = await agent.generate(prompt, { + abortSignal, + maxSteps: 10, + modelSettings: { + maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.REFRESH_AGENT, + }, + }); // Accumulate token usage into the investigate tier (refresh agents map // to the investigate tier so the runStats schema needs no new columns). diff --git a/backend/src/pipeline/codification.ts b/backend/src/pipeline/codification.ts new file mode 100644 index 0000000..ff75946 --- /dev/null +++ b/backend/src/pipeline/codification.ts @@ -0,0 +1,183 @@ +import type { + CodificationProfile, + PopulateColumn, +} from "./populate.js"; +import type { + CodificationProfile as SchemaCodificationProfile, +} from "./types.js"; + +interface CodificationClassificationInput { + datasetName?: string; + description?: string; + columns: PopulateColumn[]; + retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; + sourceHint?: string; +} + +const PROFILE_VERSION = 1; + +export function schemaCodificationProfileToRuntime( + profile: SchemaCodificationProfile, +): CodificationProfile { + return { + version: PROFILE_VERSION, + mode: profile.mode, + reason: profile.reason, + primaryKeyShape: profile.primary_key_shape, + families: profile.families.map((family) => ({ + label: family.label, + sourceHost: family.source_host, + sourcePathPrefix: family.source_path_prefix, + urlTemplate: family.url_template, + primaryKeyRegex: family.primary_key_regex, + })), + }; +} + +export function normalizeCodificationProfile( + profile: CodificationProfile | undefined, + input: CodificationClassificationInput, +): CodificationProfile { + return profile ?? classifyCodificationProfile(input); +} + +export function shouldAttemptCodification(profile: CodificationProfile): boolean { + return profile.mode === "candidate" || profile.mode === "required"; +} + +export function classifyCodificationProfile( + input: CodificationClassificationInput, +): CodificationProfile { + const primaryKeyShape = inferPrimaryKeyShape(input.columns); + const sourceUrl = firstHttpUrl(input.sourceHint); + const sourceFamily = sourceUrl ? familyFromUrl(sourceUrl) : undefined; + const retrievalStrategy = input.retrievalStrategy ?? "search_fetch"; + const searchableText = [ + input.datasetName, + input.description, + input.sourceHint, + ...input.columns.map((column) => `${column.name} ${column.description ?? ""}`), + ] + .join(" ") + .toLowerCase(); + + if (!sourceUrl && primaryKeyShape !== "url") { + return disabledProfile( + primaryKeyShape, + "No source URL or URL-shaped primary key; legacy metadata only supports broad investigation.", + ); + } + + if ( + !sourceUrl && + /\b(across|around|from)\s+the\s+web\b|\bsearch\s+the\s+web\b|\bany\s+source\b/.test( + searchableText, + ) + ) { + return disabledProfile( + primaryKeyShape, + "Prompt describes broad web research rather than one stable page family.", + ); + } + + if (primaryKeyShape === "url") { + return { + version: PROFILE_VERSION, + mode: "candidate", + reason: "Primary key is URL-shaped, so rows can be routed by page family.", + primaryKeyShape, + families: sourceFamily ? [sourceFamily] : [], + }; + } + + if (sourceFamily && (primaryKeyShape === "slug" || primaryKeyShape === "id")) { + return { + version: PROFILE_VERSION, + mode: retrievalStrategy === "browser" ? "required" : "candidate", + reason: "Dataset has a source URL and structured primary keys that may map to one page family.", + primaryKeyShape, + families: [sourceFamily], + }; + } + + if (sourceFamily && retrievalStrategy === "browser") { + return { + version: PROFILE_VERSION, + mode: "candidate", + reason: "Browser retrieval with an explicit source URL may support a reusable extractor.", + primaryKeyShape, + families: [sourceFamily], + }; + } + + return disabledProfile( + primaryKeyShape, + "Legacy metadata does not expose a stable URL, slug, ID, or browser source family.", + ); +} + +function disabledProfile( + primaryKeyShape: CodificationProfile["primaryKeyShape"], + reason: string, +): CodificationProfile { + return { + version: PROFILE_VERSION, + mode: "disabled", + reason, + primaryKeyShape, + families: [], + }; +} + +function inferPrimaryKeyShape( + columns: PopulateColumn[], +): CodificationProfile["primaryKeyShape"] { + const pkColumns = columns.filter((column) => column.isPrimaryKey); + if (pkColumns.length === 0) return "unknown"; + const shapes = new Set(pkColumns.map(inferColumnShape)); + if (shapes.size === 1) return [...shapes][0] ?? "unknown"; + return "mixed"; +} + +function inferColumnShape(column: PopulateColumn): CodificationProfile["primaryKeyShape"] { + const name = column.name.toLowerCase(); + const description = (column.description ?? "").toLowerCase(); + const regex = (column.validationRegex ?? "").toLowerCase(); + const text = `${name} ${description} ${regex}`; + + if (column.type === "url" || /\burl\b|https\?:|https?:/.test(text)) return "url"; + if (/\bslug\b|\bhandle\b|\bpath\b|\brepo\b|\bpackage\b|\busername\b/.test(text)) { + return "slug"; + } + if (/\bid\b|_id\b|\bidentifier\b|\buuid\b/.test(text)) return "id"; + if (/\bname\b|\btitle\b/.test(text)) return "name"; + return "unknown"; +} + +function firstHttpUrl(value: string | undefined): URL | undefined { + if (!value) return undefined; + const match = value.match(/https?:\/\/[^\s)>"']+/i); + if (!match) return undefined; + try { + return new URL(match[0].replace(/[.,;:]+$/, "")); + } catch { + return undefined; + } +} + +function familyFromUrl(url: URL): CodificationProfile["families"][number] { + const pathPrefix = url.pathname.split("/").filter(Boolean)[0]; + return { + label: sanitizeFamilyLabel([url.hostname, pathPrefix].filter(Boolean).join("_")), + sourceHost: url.hostname.toLowerCase(), + sourcePathPrefix: pathPrefix ? `/${pathPrefix}` : undefined, + }; +} + +function sanitizeFamilyLabel(value: string): string { + const label = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); + return /^[a-z]/.test(label) ? label : `site_${label || "unknown"}`; +} diff --git a/backend/src/pipeline/populate.ts b/backend/src/pipeline/populate.ts index b6e3b85..1a1bb37 100644 --- a/backend/src/pipeline/populate.ts +++ b/backend/src/pipeline/populate.ts @@ -13,6 +13,23 @@ export const populateColumnSchema = z.object({ }); export type PopulateColumn = z.infer; +export const codificationProfileSchema = z.object({ + version: z.literal(1), + mode: z.enum(["disabled", "candidate", "required", "unknown"]), + reason: z.string(), + primaryKeyShape: z.enum(["url", "slug", "name", "id", "mixed", "unknown"]), + families: z.array( + z.object({ + label: z.string(), + sourceHost: z.optional(z.string()), + sourcePathPrefix: z.optional(z.string()), + urlTemplate: z.optional(z.string()), + primaryKeyRegex: z.optional(z.string()), + }), + ), +}); +export type CodificationProfile = z.infer; + export const datasetContextSchema = z.object({ datasetId: z.string().min(1), datasetName: z.string(), @@ -22,5 +39,6 @@ export const datasetContextSchema = z.object({ rowIds: z.array(z.string()).min(1).optional(), retrievalStrategy: z.enum(["search_fetch", "browser", "hybrid"]).optional(), sourceHint: z.string().optional(), + codificationProfile: z.optional(codificationProfileSchema), }); export type DatasetContext = z.infer; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 1c1bb95..9bd1be5 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -19,6 +19,12 @@ Your job is to: 5. Set \`source_hint\` to a specific URL whenever possible (e.g. \`https://www.ycombinator.com/companies?industry=Fintech\`). Avoid vague descriptions. 6. Write a \`retrieval_hint\` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. 7. For each column where a value has a known shape, include \`validation_regex\` and \`normalization_hint\`. These are extractor contracts, not UI decoration. Examples: ratings, prices, dates, URL/slug shapes, repository slugs, app package names, counts, currencies, availability labels. Omit \`validation_regex\` only when the value is genuinely free-form text. +8. Set \`codification_profile\`. This is a cheap schema-time decision about whether BigSet should attempt to compile a reusable Playwright extractor later. + - \`mode: "disabled"\` when rows will come from broad web search, arbitrary unrelated domains, search snippets, or sources that are known to block browser automation for the requested fields. + - \`mode: "candidate"\` when rows likely share one or more stable page families and a reusable browser script may work after seeing a representative row. + - \`mode: "required"\` when the dataset is clearly tied to one authoritative browser-heavy source or directory where repeated extraction is the intended path. + - \`mode: "unknown"\` only when the prompt/schema gives too little evidence; prefer \`disabled\` over \`unknown\` for broad web datasets to avoid expensive extractor attempts. + Include \`primary_key_shape\` as one of \`url\`, \`slug\`, \`name\`, \`id\`, \`mixed\`, or \`unknown\`. Include \`families\` for known source/page families. Use snake_case labels. For URL templates, use column placeholders like \`https://github.com/{repo_slug}\`. Rules: @@ -28,6 +34,7 @@ Rules: - Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. - Validation regexes must validate the normalized final value, not raw page text. Keep them practical and anchored, e.g. "^[0-5](\\\\.\\\\d)?$" for a normalized rating or "^[^/\\\\s]+/[^/\\\\s]+$" for an owner/repo slug. - \`normalization_hint\` should tell the extractor how to convert raw page text into the stored value, e.g. "Convert '4.6 out of 5 stars' to '4.6'" or "Strip commas and convert 1.2k to 1200". +- \`codification_profile\` should be conservative. Do not mark arbitrary company/person/place/product research as codifiable just because pages exist on the web. Mark it codifiable only when rows can be routed to stable page families from the primary key, source_hint, or obvious URL templates. - When a column is a scalar numeric rating (e.g. average score like 4.3/5 for restaurants, cafes, hotels, products, apps): name it generically (e.g. "rating" not "yelp_rating") and write a retrieval_hint explaining that review sites (Yelp, TripAdvisor, Google Maps) block direct page fetches, so the agent must extract ratings from **search result snippets**. The hint should say: "Search for \\" rating reviews\\" and include location terms only when location is part of the entity identity. Look for ratings in snippets from TripAdvisor (\\"rated X.X of 5\\"), Yelp search listings (\\"X.X (N reviews)\\"), or aggregator sites (Birdeye, joe.coffee, giftly, Uber Eats, menufyy). Do NOT try to fetch yelp.com or tripadvisor.com directly — they block automated access. Accept ratings from any reputable source." If including a rating column, also add a "rating_source" text column so the agent records where the rating came from. Do not rename review-count or review-text fields to "rating" — keep those as distinct columns (e.g. "review_count") when the user explicitly asks for them.`; async function getModel(modelSlug?: string) { @@ -58,7 +65,7 @@ async function callOnce( model, output: Output.object({ schema: datasetSchemaSchema }), system: SYSTEM_PROMPT, - maxOutputTokens: 4096, + maxOutputTokens: 5000, prompt, }); if (!output) throw new Error("Model did not generate a valid schema object"); diff --git a/backend/src/pipeline/types.ts b/backend/src/pipeline/types.ts index 25338fa..9aa9e6a 100644 --- a/backend/src/pipeline/types.ts +++ b/backend/src/pipeline/types.ts @@ -32,6 +32,42 @@ export const columnDefinitionSchema = z.object({ }); export type ColumnDefinition = z.infer; +export const codificationModeSchema = z.enum([ + "disabled", + "candidate", + "required", + "unknown", +]); +export type CodificationMode = z.infer; + +export const primaryKeyShapeSchema = z.enum([ + "url", + "slug", + "name", + "id", + "mixed", + "unknown", +]); +export type PrimaryKeyShape = z.infer; + +export const codificationFamilySchema = z.object({ + label: z.string().regex(snakeCase, "must be snake_case"), + source_host: z.string().optional(), + source_path_prefix: z.string().optional(), + url_template: z.string().optional(), + primary_key_regex: z.string().optional(), +}); +export type CodificationFamily = z.infer; + +export const codificationProfileSchema = z.object({ + version: z.literal(1), + mode: codificationModeSchema, + reason: z.string().min(1), + primary_key_shape: primaryKeyShapeSchema, + families: z.array(codificationFamilySchema), +}); +export type CodificationProfile = z.infer; + export const datasetSchemaSchema = z .object({ dataset_name: z.string().regex(snakeCase, "must be snake_case"), @@ -40,6 +76,7 @@ export const datasetSchemaSchema = z primary_key: z.array(z.string()).min(1), retrieval_strategy: retrievalStrategySchema, source_hint: z.string().min(1), + codification_profile: codificationProfileSchema, }) .superRefine((data, ctx) => { const names = data.columns.map((c) => c.name); diff --git a/backend/src/row-extractors/try-row-extractor.ts b/backend/src/row-extractors/try-row-extractor.ts index 7507499..8d89601 100644 --- a/backend/src/row-extractors/try-row-extractor.ts +++ b/backend/src/row-extractors/try-row-extractor.ts @@ -1,19 +1,28 @@ import { chromium, type Browser, type Page } from "playwright-core"; -import { generateText } from "ai"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { Agent } from "@mastra/core/agent"; +import { createTool } from "@mastra/core/tools"; import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; +import { z } from "zod"; import { getSignal } from "../abort-registry.js"; import { convex, internal } from "../convex.js"; import { FETCH_TIMEOUT_MS } from "../fetch-timeout.js"; import { getTinyFishApiKey, - requireOpenRouterApiKey, + requireLlmProviderConfig, tinyFishHeaders, } from "../local-credentials.js"; -import type { PopulateColumn } from "../pipeline/populate.js"; +import { + createLanguageModel, +} from "../config/llm.js"; +import { AGENT_MAX_OUTPUT_TOKENS } from "../config/agent-output-tokens.js"; +import type { CodificationProfile, PopulateColumn } from "../pipeline/populate.js"; +import { + normalizeCodificationProfile, + shouldAttemptCodification, +} from "../pipeline/codification.js"; import { DEFAULT_MODEL_IDS } from "../config/models.js"; type ExtractorStatus = "inserted" | "updated" | "unchanged" | "miss" | "failed"; @@ -29,6 +38,7 @@ export interface TryRowExtractorInput { context?: string; retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; sourceHint?: string; + codificationProfile?: CodificationProfile; browserAttempts?: number; extractorBuilderModel?: string; } @@ -76,17 +86,36 @@ interface GeneratedExtractorResult { howFound?: unknown; } -const ENABLED_VALUES = new Set(["1", "true", "yes", "on"]); +interface DetailedExtractorTestResult { + ok: boolean; + reason: string; + extraction?: GenericExtraction; +} + +interface AgenticExtractorBuildResult { + script: string; + probeSummary: string; +} + +interface ExtractorSmokeTestCase { + input: TryRowExtractorInput; + url: string; + source: "memory" | "persisted"; +} + const BROWSER_TIMEOUT_MS = 45_000; const CDP_CONNECT_TIMEOUT_MS = 45_000; const DEFAULT_BROWSER_ATTEMPTS = 2; const EXTRACTOR_RUNNER_TIMEOUT_MS = 60_000; const EXTRACTOR_SCRIPT_OUTPUT_LIMIT = 256_000; +const EXTRACTOR_BUILDER_MAX_STEPS = 24; const BACKEND_ROOT = fileURLToPath(new URL("../../", import.meta.url)); const GENERIC_EXTRACTOR_HOW_FOUND = "Opened the row target with TinyFish Browser and ran the dataset's generated Playwright extractor."; const GENERIC_REFRESH_HOW_FOUND = "Refreshed the row target with TinyFish Browser and ran the dataset's generated Playwright extractor."; +const inFlightExtractorBuilds = new Map>(); +const extractorSmokeTests = new Map(); const EXTRACTOR_RUNNER_SOURCE = ` import { chromium } from "playwright-core"; import vm from "node:vm"; @@ -171,8 +200,12 @@ try { export async function tryRowExtractor( input: TryRowExtractorInput, ): Promise { - if (!ENABLED_VALUES.has((process.env.ROW_EXTRACTORS_ENABLED ?? "").toLowerCase())) { - return { status: "miss", reason: "row extractors are disabled" }; + const codificationProfile = normalizeCodificationProfile(input.codificationProfile, input); + if (!shouldAttemptCodification(codificationProfile)) { + return { + status: "miss", + reason: `codification profile is ${codificationProfile.mode}: ${codificationProfile.reason}`, + }; } const url = initialBrowserUrl(input); @@ -216,8 +249,12 @@ export async function tryRowExtractor( export async function tryRefreshRowExtractor( input: TryRefreshRowExtractorInput, ): Promise { - if (!ENABLED_VALUES.has((process.env.ROW_EXTRACTORS_ENABLED ?? "").toLowerCase())) { - return { status: "miss", reason: "row extractors are disabled" }; + const codificationProfile = normalizeCodificationProfile(input.codificationProfile, input); + if (!shouldAttemptCodification(codificationProfile)) { + return { + status: "miss", + reason: `codification profile is ${codificationProfile.mode}: ${codificationProfile.reason}`, + }; } const url = initialBrowserUrl(input); @@ -340,7 +377,9 @@ async function extractGenericRow( for (let attempt = 1; attempt <= attempts; attempt++) { try { const raw = await runGeneratedExtractorScript(apiKey, input, url, script); - return buildRowFromExtractorResult(input, url, raw, existingData); + const extraction = buildRowFromExtractorResult(input, url, raw, existingData); + rememberExtractorSmokeTest(input, url); + return extraction; } catch (err) { lastError = err; if (getSignal(input.datasetId)?.aborted || attempt === attempts) break; @@ -364,9 +403,14 @@ async function extractGenericRow( failure, ); const raw = await runGeneratedExtractorScript(apiKey, input, url, repaired); - return buildRowFromExtractorResult(input, url, raw, existingData); + const extraction = buildRowFromExtractorResult(input, url, raw, existingData); + rememberExtractorSmokeTest(input, url); + return extraction; } catch (repairErr) { - await markExtractorFailed(input, url, repairErr).catch(() => undefined); + const repairMsg = repairErr instanceof Error ? repairErr.message : String(repairErr); + console.warn( + `[row_extractor] repair failed; keeping cached extractor available for future improvement: ${repairMsg}`, + ); throw repairErr instanceof Error ? repairErr : new Error(String(repairErr)); } } @@ -385,6 +429,7 @@ async function getOrBuildExtractorScript( ): Promise { const siteKey = siteKeyForInput(input, url); const columnsHash = hashColumns(input.columns); + const buildKey = extractorBuildKey(input.datasetId, siteKey, columnsHash); const existing = (await convex.query(internal.datasetExtractors.getActive, { datasetId: input.datasetId, siteKey, @@ -392,9 +437,30 @@ async function getOrBuildExtractorScript( })) as { script?: string } | null; if (existing?.script) return existing.script; - const evidence = await probePage(apiKey, input.datasetId, url); + const inFlightBuild = inFlightExtractorBuilds.get(buildKey); + if (inFlightBuild) { + console.log(`[row_extractor] waiting for in-flight extractor build for ${siteKey}`); + return await inFlightBuild; + } + + const buildPromise = buildAndPersistExtractorScript(apiKey, input, url, siteKey, columnsHash) + .finally(() => { + inFlightExtractorBuilds.delete(buildKey); + }); + inFlightExtractorBuilds.set(buildKey, buildPromise); + return await buildPromise; +} + +async function buildAndPersistExtractorScript( + apiKey: string, + input: TryRowExtractorInput, + url: string, + siteKey: string, + columnsHash: string, +): Promise { const modelSlug = input.extractorBuilderModel ?? DEFAULT_MODEL_IDS.EXTRACTOR_BUILDER; - const script = await buildExtractorScript(input, url, evidence, modelSlug); + const built = await buildExtractorScriptWithAgent(apiKey, input, url, modelSlug); + const script = built.script; const validation = await validateExtractorScript(apiKey, input, url, script); if (validation.ok) { await convex.mutation(internal.datasetExtractors.upsert, { @@ -403,20 +469,22 @@ async function getOrBuildExtractorScript( columnsHash, script, model: modelSlug, - probeSummary: summarizeEvidenceForStorage(evidence), + probeSummary: built.probeSummary, }); return script; } - const repaired = await repairExtractorScript( + const repaired = await buildExtractorScriptWithAgent( + apiKey, input, url, - evidence, - script, - validation.reason, modelSlug, + { + previousScript: script, + failure: validation.reason, + }, ); - const repairedValidation = await validateExtractorScript(apiKey, input, url, repaired); + const repairedValidation = await validateExtractorScript(apiKey, input, url, repaired.script); if (!repairedValidation.ok) { throw new Error(`generated extractor failed validation: ${repairedValidation.reason}`); } @@ -425,11 +493,134 @@ async function getOrBuildExtractorScript( datasetId: input.datasetId, siteKey, columnsHash, - script: repaired, + script: repaired.script, model: modelSlug, - probeSummary: summarizeEvidenceForStorage(evidence), + probeSummary: repaired.probeSummary, + }); + return repaired.script; +} + +function extractorBuildKey( + datasetId: string, + siteKey: string, + columnsHash: string, +): string { + return `${datasetId}:${siteKey}:${columnsHash}`; +} + +function extractorBuildKeyForInput(input: TryRowExtractorInput, url: string): string { + return extractorBuildKey( + input.datasetId, + siteKeyForInput(input, url), + hashColumns(input.columns), + ); +} + +function rememberExtractorSmokeTest(input: TryRowExtractorInput, url: string): void { + extractorSmokeTests.set(extractorBuildKeyForInput(input, url), { + input: cloneSmokeTestInput(input), + url, + source: "memory", + }); +} + +async function getExtractorSmokeTest( + input: TryRowExtractorInput, + url: string, +): Promise { + const key = extractorBuildKeyForInput(input, url); + const memoryCase = extractorSmokeTests.get(key); + if (memoryCase && !samePrimaryKeys(memoryCase.input.primaryKeys, input.primaryKeys)) { + return memoryCase; + } + + const rows = (await convex.query(internal.datasetRows.listInternal, { + datasetId: input.datasetId, + })) as Array<{ + _id?: string; + data?: Record; + sources?: string[]; + rowSummary?: string; + howFound?: string; + }>; + + const currentRowId = "rowId" in input ? input.rowId : undefined; + for (const row of rows) { + if (currentRowId && row._id === currentRowId) continue; + if (!isBrowserExtractedRow(row.howFound)) continue; + if (!row.data || !storedRowHasCompleteValues(row.data, input.columns)) continue; + + const primaryKeys = primaryKeysFromStoredRow(row.data, input.columns); + if (Object.keys(primaryKeys).length === 0) continue; + if (samePrimaryKeys(primaryKeys, input.primaryKeys)) continue; + + const smokeInput = cloneSmokeTestInput({ + ...input, + primaryKeys, + urls: row.sources, + context: [row.rowSummary, row.howFound].filter(Boolean).join("\n"), + }); + const smokeUrl = initialBrowserUrl(smokeInput) ?? url; + return { input: smokeInput, url: smokeUrl, source: "persisted" }; + } + + return undefined; +} + +function cloneSmokeTestInput(input: TryRowExtractorInput): TryRowExtractorInput { + return { + datasetId: input.datasetId, + datasetName: input.datasetName, + description: input.description, + columns: input.columns, + primaryKeys: { ...input.primaryKeys }, + urls: input.urls ? [...input.urls] : undefined, + context: input.context, + retrievalStrategy: input.retrievalStrategy, + sourceHint: input.sourceHint, + codificationProfile: input.codificationProfile, + browserAttempts: input.browserAttempts, + extractorBuilderModel: input.extractorBuilderModel, + }; +} + +function isBrowserExtractedRow(howFound: string | undefined): boolean { + return Boolean(howFound && /generated Playwright extractor/i.test(howFound)); +} + +function storedRowHasCompleteValues( + data: Record, + columns: PopulateColumn[], +): boolean { + return columns.every((column) => { + const value = normalizeStoredValue(data[column.name]); + return hasCompleteColumnValue(value) && !validateColumnValue(value, column); + }); +} + +function primaryKeysFromStoredRow( + data: Record, + columns: PopulateColumn[], +): Record { + return Object.fromEntries( + columns + .filter((column) => column.isPrimaryKey) + .map((column) => [column.name, String(data[column.name] ?? "").trim()]) + .filter(([, value]) => value.length > 0), + ); +} + +function samePrimaryKeys( + left: Record, + right: Record, +): boolean { + const leftEntries = Object.entries(left).sort(([a], [b]) => a.localeCompare(b)); + const rightEntries = Object.entries(right).sort(([a], [b]) => a.localeCompare(b)); + if (leftEntries.length !== rightEntries.length) return false; + return leftEntries.every(([key, value], index) => { + const [rightKey, rightValue] = rightEntries[index]!; + return key === rightKey && value === rightValue; }); - return repaired; } async function probePage( @@ -638,56 +829,178 @@ async function readPageEvidence(page: Page): Promise { return { ...evidence, finalUrl }; } -async function buildExtractorScript( +async function buildExtractorScriptWithAgent( + apiKey: string, input: TryRowExtractorInput, url: string, - evidence: PageEvidence, modelSlug: string, -): Promise { - const openrouter = createOpenRouter({ - apiKey: await requireOpenRouterApiKey(), - baseURL: process.env.OPENROUTER_BASE_URL, - }); - const result = await generateText({ - model: openrouter(modelSlug), - prompt: extractorBuilderPrompt(input, url, evidence), - maxOutputTokens: 4_000, - abortSignal: getSignal(input.datasetId), + repairContext?: { + previousScript: string; + failure: string; + }, +): Promise { + const llmConfig = await requireLlmProviderConfig(); + let acceptedScript: string | undefined; + let declinedReason: string | undefined; + let lastEvidence: PageEvidence | undefined; + let lastProbeSummary = `url=${url}`; + + const inspectBrowserPageTool = createTool({ + id: "inspect_browser_page", + description: + "Open a URL with TinyFish Browser and return page evidence: final URL, title, meta description, structured candidate fields, and visible text excerpt. Use this whenever page structure is unclear.", + inputSchema: z.object({ + url: z + .string() + .optional() + .describe("Optional URL to inspect. If omitted, inspects the browser start URL."), + }), + outputSchema: z.object({ + finalUrl: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + candidateFields: z.array( + z.object({ + key: z.string(), + values: z.array(z.string()), + }), + ), + visibleTextExcerpt: z.string(), + error: z.string().optional(), + }), + execute: async ({ url: requestedUrl }) => { + try { + const targetUrl = coerceHttpUrl(requestedUrl) ?? url; + const evidence = await probePage(apiKey, input.datasetId, targetUrl); + lastEvidence = evidence; + lastProbeSummary = summarizeEvidenceForStorage(evidence); + return evidenceForTool(evidence); + } catch (err) { + return { + candidateFields: [], + visibleTextExcerpt: "", + error: err instanceof Error ? err.message : String(err), + }; + } + }, }); - return sanitizeGeneratedScript(result.text); -} -async function repairExtractorScript( - input: TryRowExtractorInput, - url: string, - evidence: PageEvidence, - script: string, - failure: string, - modelSlug: string, -): Promise { - const openrouter = createOpenRouter({ - apiKey: await requireOpenRouterApiKey(), - baseURL: process.env.OPENROUTER_BASE_URL, + const testExtractorTool = createTool({ + id: "test_extractor", + description: + "Run a candidate Playwright extractor in BigSet's sandbox against the representative row and return validation feedback. Call this repeatedly while improving the script.", + inputSchema: z.object({ + script: z.string().describe("The full JavaScript extractor script defining async function extract({ page, input, helpers })."), + }), + outputSchema: z.object({ + ok: z.boolean(), + reason: z.string(), + extractedColumns: z.array(z.string()).optional(), + missingColumns: z.array(z.string()).optional(), + rowSummary: z.string().optional(), + sources: z.array(z.string()).optional(), + dataPreview: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), + }), + execute: async ({ script }) => { + const result = await testExtractorScriptDetailed(apiKey, input, url, script); + return detailedTestResultForTool(result); + }, }); - const result = await generateText({ - model: openrouter(modelSlug), - prompt: `${extractorBuilderPrompt(input, url, evidence)} -The previous extractor failed validation. + const finishExtractorTool = createTool({ + id: "finish_extractor", + description: + "Finalize a candidate extractor. This runs the same sandbox validation as test_extractor and only accepts the script if it passes.", + inputSchema: z.object({ + script: z.string().describe("The full JavaScript extractor script to persist."), + notes: z.string().optional().describe("Brief notes about the page pattern this script supports."), + }), + outputSchema: z.object({ + ok: z.boolean(), + reason: z.string(), + extractedColumns: z.array(z.string()).optional(), + missingColumns: z.array(z.string()).optional(), + }), + execute: async ({ script }) => { + const result = await testExtractorScriptDetailed(apiKey, input, url, script); + if (result.ok) { + acceptedScript = sanitizeGeneratedScript(script); + } + return { + ok: result.ok, + reason: result.reason, + extractedColumns: result.extraction?.extractedColumns, + missingColumns: result.extraction?.missingColumns, + }; + }, + }); -Failure: -${failure} + const declineExtractorTool = createTool({ + id: "decline_extractor", + description: + "Use this when the representative row is not a good fit for a reusable Playwright extractor, for example arbitrary unrelated URLs, CAPTCHA/blocking, or no stable page family.", + inputSchema: z.object({ + reason: z.string().describe("Concrete reason this row/site pattern should fall back to the normal investigate agent."), + category: z + .enum([ + "mixed_unrelated_urls", + "blocked_or_captcha", + "no_stable_page_pattern", + "insufficient_row_context", + "other", + ]) + .describe("Why codification is not appropriate."), + }), + outputSchema: z.object({ + ok: z.boolean(), + reason: z.string(), + category: z.string(), + }), + execute: async ({ reason, category }) => { + declinedReason = `${category}: ${reason}`; + return { ok: true, reason, category }; + }, + }); -Previous extractor: -\`\`\`js -${script} -\`\`\` + const agent = new Agent({ + id: "extractor-builder-agent", + name: "Extractor Builder Agent", + instructions: extractorBuilderAgentInstructions(), + model: createLanguageModel(llmConfig, modelSlug), + tools: { + inspect_browser_page: inspectBrowserPageTool, + test_extractor: testExtractorTool, + finish_extractor: finishExtractorTool, + decline_extractor: declineExtractorTool, + }, + }); -Return a repaired extractor only.`, - maxOutputTokens: 4_000, + const prompt = extractorBuilderAgentPrompt(input, url, repairContext); + const result = await agent.generate(prompt, { abortSignal: getSignal(input.datasetId), + maxSteps: EXTRACTOR_BUILDER_MAX_STEPS, + modelSettings: { + maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.EXTRACTOR_BUILDER, + }, }); - return sanitizeGeneratedScript(result.text); + + if (acceptedScript) { + console.log( + `[extractor_builder] accepted script after ${result.steps?.length ?? "?"} steps for ${siteKeyForInput(input, url)}`, + ); + return { + script: acceptedScript, + probeSummary: lastEvidence ? summarizeEvidenceForStorage(lastEvidence) : lastProbeSummary, + }; + } + + if (declinedReason) { + throw new Error(`extractor builder declined codification: ${declinedReason}`); + } + + throw new Error( + `extractor builder did not finish a validated script within ${EXTRACTOR_BUILDER_MAX_STEPS} steps`, + ); } async function repairExtractorAfterRuntimeFailure( @@ -697,74 +1010,197 @@ async function repairExtractorAfterRuntimeFailure( script: string, failure: string, ): Promise { - const evidence = await probePage(apiKey, input.datasetId, url); const modelSlug = input.extractorBuilderModel ?? DEFAULT_MODEL_IDS.EXTRACTOR_BUILDER; - const repaired = await repairExtractorScript( + const repaired = await buildExtractorScriptWithAgent( + apiKey, input, url, - evidence, - script, - failure, modelSlug, + { + previousScript: script, + failure, + }, ); - const validation = await validateExtractorScript(apiKey, input, url, repaired); + const validation = await validateExtractorScript(apiKey, input, url, repaired.script); if (!validation.ok) { throw new Error(`repaired extractor failed validation: ${validation.reason}`); } + const siteKey = siteKeyForInput(input, url); + const columnsHash = hashColumns(input.columns); + const smokeTest = await getExtractorSmokeTest(input, url); + if (smokeTest) { + const regression = await testExtractorScriptDetailed( + apiKey, + smokeTest.input, + smokeTest.url, + repaired.script, + ); + if (!regression.ok) { + throw new Error( + `repaired extractor regressed ${smokeTest.source} known-good row: ${regression.reason}`, + ); + } + console.log( + `[row_extractor] repaired extractor passed ${smokeTest.source} regression test for ${siteKey}`, + ); + } else { + console.warn( + `[row_extractor] repaired extractor has no known-good regression row for ${siteKey}; using for this row without updating cache`, + ); + return repaired.script; + } + await convex.mutation(internal.datasetExtractors.upsert, { datasetId: input.datasetId, - siteKey: siteKeyForInput(input, url), - columnsHash: hashColumns(input.columns), - script: repaired, + siteKey, + columnsHash, + script: repaired.script, model: modelSlug, - probeSummary: summarizeEvidenceForStorage(evidence), + probeSummary: repaired.probeSummary, }); - return repaired; + return repaired.script; } -async function markExtractorFailed( +function evidenceForTool(evidence: PageEvidence): { + finalUrl: string; + title?: string; + description?: string; + candidateFields: Array<{ key: string; values: string[] }>; + visibleTextExcerpt: string; +} { + return { + finalUrl: evidence.finalUrl, + title: evidence.title, + description: evidence.description, + candidateFields: Object.entries(evidence.candidates) + .slice(0, 100) + .map(([key, values]) => ({ + key, + values: values.slice(0, 6), + })), + visibleTextExcerpt: evidence.bodyText.slice(0, 16_000), + }; +} + +async function testExtractorScriptDetailed( + apiKey: string, input: TryRowExtractorInput, url: string, - err: unknown, -): Promise { - await convex.mutation(internal.datasetExtractors.markFailed, { - datasetId: input.datasetId, - siteKey: siteKeyForInput(input, url), - columnsHash: hashColumns(input.columns), - error: err instanceof Error ? err.message : String(err), - }); + script: string, +): Promise { + try { + const sanitized = sanitizeGeneratedScript(script); + const raw = await runGeneratedExtractorScript(apiKey, input, url, sanitized); + const extraction = buildRowFromExtractorResult(input, url, raw); + if (!hasExtractedNonPrimaryValue(extraction, input.columns)) { + return { + ok: false, + reason: "extractor returned no non-primary values", + extraction, + }; + } + return { ok: true, reason: "passed", extraction }; + } catch (err) { + return { + ok: false, + reason: err instanceof Error ? err.message : String(err), + }; + } } -function extractorBuilderPrompt( +function detailedTestResultForTool(result: DetailedExtractorTestResult): { + ok: boolean; + reason: string; + extractedColumns?: string[]; + missingColumns?: string[]; + rowSummary?: string; + sources?: string[]; + dataPreview?: Record; +} { + return { + ok: result.ok, + reason: result.reason, + extractedColumns: result.extraction?.extractedColumns, + missingColumns: result.extraction?.missingColumns, + rowSummary: result.extraction?.rowSummary, + sources: result.extraction?.sources, + dataPreview: result.extraction?.data, + }; +} + +function extractorBuilderAgentInstructions(): string { + return `You are BigSet's autonomous extractor builder agent. + +Your job is to build a reusable Playwright extractor script for one dataset/page family. You have browser inspection and sandbox testing tools. Use whichever tools fit the situation; do not follow a rigid checklist. + +Critical behavior: +- Inspect pages when structure, source URLs, or primary key semantics are unclear. +- Write and test candidate scripts with test_extractor. Iterate on validation errors. +- Finish only by calling finish_extractor with the final script. A final text response without finish_extractor is failure. +- If the dataset is not a good fit for reusable browser codification, call decline_extractor with a concrete reason. + +When to decline: +- The representative primary keys/URLs point at arbitrary unrelated domains with no shared page pattern. +- The page is blocked by login, paywall, CAPTCHA, bot detection, or inaccessible content. +- The row context is too ambiguous to construct or find a stable browser target. +- You cannot produce a complete row with every dataset column populated. + +Important modeling rules: +- Primary keys are arbitrary row identifiers. They do not have to be URLs. +- You may construct navigation URLs from input.primaryKeys, top-level primary key fields, input.urls, input.sourceHint, dataset name, description, retrieval strategy, and context. +- Example: a primary key like "tinyfish-io/bigset" may require page.goto("https://github.com/" + input.primaryKeys.repo_slug). Do the equivalent for any site family you infer. +- Multiple known page families are allowed. For example, if rows consistently contain either Yelp or Google Maps URLs, write one reusable script that branches based on the available URL/domain. If rows are arbitrary unrelated URLs, decline. +- Respect validation_regex exactly. Normalize values before returning them, using normalization_hint and column descriptions. + +Script contract: +- Return only a full JavaScript script in tool arguments, never Markdown fences. +- Define exactly: async function extract({ page, input, helpers }) { ... } +- Do not import anything. +- Do not use require, process, fs, child_process, fetch, XMLHttpRequest, WebSocket, eval, new Function, Node http/https/net modules, direct database APIs, or Convex APIs. +- Use only the provided Playwright page, input, and helpers. +- You may use page.goto, locators, evaluate, textContent, getAttribute, and other Playwright page APIs. +- Prefer stable DOM sources: JSON-LD, meta tags, tables, definition lists, visible labels, aria labels, canonical links, and semantically named selectors. +- Return { data, sources, row_summary, how_found }. +- data must include every dataset column. Preserve primary key values from input.primaryKeys. +- Every dataset column must have a real non-empty value after normalization. Nullable columns are still required for codified browser extraction. +- Returned sources must be HTTP URLs you actually used. +- If any column cannot be extracted or normalized to satisfy validation_regex, keep investigating with the browser tools or decline. Do not finish a partial extractor.`; +} + +function extractorBuilderAgentPrompt( input: TryRowExtractorInput, url: string, - evidence: PageEvidence, + repairContext?: { + previousScript: string; + failure: string; + }, ): string { const columns = input.columns - .map( - (column) => { - const details = [ - column.isPrimaryKey ? "PRIMARY KEY" : undefined, - column.nullable === undefined ? undefined : `nullable=${column.nullable}`, - column.validationRegex ? `validation_regex=${JSON.stringify(column.validationRegex)}` : undefined, - column.normalizationHint ? `normalization_hint=${JSON.stringify(column.normalizationHint)}` : undefined, - column.description ? `retrieval_hint=${JSON.stringify(column.description)}` : undefined, - ].filter(Boolean); - return `- ${JSON.stringify(column.name)} (${column.type})${details.length > 0 ? ` [${details.join("; ")}]` : ""}`; - }, - ) + .map((column) => columnPromptLine(column)) .join("\n"); const primaryKeys = Object.entries(input.primaryKeys) - .map(([key, value]) => `- ${key}: ${value}`) + .map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`) .join("\n") || "(none)"; - return `You are BigSet's extractorBuilder. Generate one reusable Playwright extractor for this dataset/page pattern. + const urls = (input.urls ?? []).map((value) => `- ${value}`).join("\n") || "(none)"; + const repair = repairContext + ? ` +You are repairing a previous extractor. The previous script failed with: +${repairContext.failure} + +Previous script: +${repairContext.previousScript.slice(0, 20_000)} +` + : ""; + + return `Build a reusable Playwright extractor for this BigSet dataset. Dataset: - Name: ${input.datasetName ?? ""} - Description: ${input.description ?? ""} - Retrieval strategy: ${input.retrievalStrategy ?? ""} - Source hint: ${input.sourceHint ?? ""} +- Context: ${input.context ?? ""} Dataset columns: ${columns} @@ -772,45 +1208,30 @@ ${columns} Primary key values for the representative row: ${primaryKeys} -Browser start URL: -${url} - -Browser probe: -${summarizeEvidenceForPrompt(evidence)} +Candidate URLs from row context: +${urls} -Requirements: -- Return only JavaScript. No Markdown. -- Define exactly: async function extract({ page, input, helpers }) { ... } -- Do not import anything. -- Do not use require, process, fs, child_process, fetch, eval, new Function, Node http/https/net modules, or direct database/network APIs. -- Use only the provided Playwright page, input, and helpers. -- The primary key values are arbitrary row input. Do not assume they are URLs. -- You may construct navigation URLs from input.primaryKeys, top-level primary key fields, input.sourceHint, input.urls, dataset name, or context. Example: a primary key like "tinyfish-io/bigset" may need page.goto(\`https://github.com/\${input.primaryKeys.repo_slug}\`). -- Navigate with page.goto(..., { waitUntil: "domcontentloaded" }) if needed. input.startUrl/input.url is only the browser start URL, not necessarily the row URL. -- Return an object: { data, sources, row_summary, how_found }. -- data must include every dataset column. Preserve primary key values from input.primaryKeys. -- Use "" for unknown values. Never fabricate. -- Returned values must be normalized to each column's type contract. -- If a column has validation_regex, the final returned value must match it after normalization. Use normalization_hint as the guide for converting page text to the canonical value. -- If a required selector/value is empty or does not match validation_regex, return "" for that field rather than guessing; BigSet will reject/repair failed validation. -- Prefer stable DOM selectors, JSON-LD, meta tags, tables, definition lists, and visible labels. -- The extractor must be reusable for other rows on the same site/page pattern.`; -} - -function summarizeEvidenceForPrompt(evidence: PageEvidence): string { - const candidates = Object.entries(evidence.candidates) - .slice(0, 80) - .map(([key, values]) => `- ${key}: ${values.slice(0, 3).join(" | ")}`) - .join("\n"); - return `Final URL: ${evidence.finalUrl} -Title: ${evidence.title ?? ""} -Description: ${evidence.description ?? ""} +Representative browser start URL: +${url} -Candidate fields: -${candidates || "(none)"} +Use the tools to inspect, test, revise, finish, or decline. The goal is a validated script, not a prose answer.${repair}`; +} -Visible text excerpt: -${evidence.bodyText.slice(0, 8_000)}`; +function columnPromptLine(column: PopulateColumn): string { + const details = [ + column.isPrimaryKey ? "PRIMARY KEY" : undefined, + column.nullable === undefined ? undefined : `nullable=${column.nullable}`, + column.validationRegex + ? `validation_regex=${JSON.stringify(column.validationRegex)}` + : undefined, + column.normalizationHint + ? `normalization_hint=${JSON.stringify(column.normalizationHint)}` + : undefined, + column.description ? `retrieval_hint=${JSON.stringify(column.description)}` : undefined, + ].filter(Boolean); + return `- ${JSON.stringify(column.name)} (${column.type})${ + details.length > 0 ? ` [${details.join("; ")}]` : "" + }`; } function summarizeEvidenceForStorage(evidence: PageEvidence): string { @@ -865,19 +1286,8 @@ async function validateExtractorScript( url: string, script: string, ): Promise<{ ok: true } | { ok: false; reason: string }> { - try { - const raw = await runGeneratedExtractorScript(apiKey, input, url, script); - const extraction = buildRowFromExtractorResult(input, url, raw); - if (!hasExtractedNonPrimaryValue(extraction, input.columns)) { - return { ok: false, reason: "extractor returned no non-primary values" }; - } - return { ok: true }; - } catch (err) { - return { - ok: false, - reason: err instanceof Error ? err.message : String(err), - }; - } + const result = await testExtractorScriptDetailed(apiKey, input, url, script); + return result.ok ? { ok: true } : { ok: false, reason: result.reason }; } async function runGeneratedExtractorScript( @@ -983,7 +1393,7 @@ function buildRowFromExtractorResult( const rawValue = result.data[column.name]; const value = coerceColumnValue(rawValueToExtractedValue(rawValue), column, url); - if (value !== undefined) { + if (value !== undefined && hasCompleteColumnValue(value)) { const validationError = validateColumnValue(value, column); if (!validationError) { data[column.name] = value; @@ -994,10 +1404,18 @@ function buildRowFromExtractorResult( } if (existingData && existingData[column.name] !== undefined) { - data[column.name] = normalizeStoredValue(existingData[column.name]); - } else { - data[column.name] = ""; + const storedValue = normalizeStoredValue(existingData[column.name]); + if (hasCompleteColumnValue(storedValue)) { + const validationError = validateColumnValue(storedValue, column); + if (!validationError) { + data[column.name] = storedValue; + continue; + } + invalidColumns.push(`${column.name}: stored value ${validationError}`); + } } + + data[column.name] = ""; missingColumns.push(column.name); } @@ -1007,12 +1425,9 @@ function buildRowFromExtractorResult( ); } - const missingRequiredColumns = input.columns - .filter((column) => columnRequiresValue(column) && missingColumns.includes(column.name)) - .map((column) => column.name); - if (missingRequiredColumns.length > 0) { + if (missingColumns.length > 0) { throw new Error( - `generated extractor missed required columns: ${missingRequiredColumns.join(", ")}`, + `generated extractor missed columns: ${missingColumns.join(", ")}`, ); } @@ -1048,10 +1463,6 @@ function rawValueToExtractedValue(value: unknown): ExtractedValue | undefined { return String(value); } -function columnRequiresValue(column: PopulateColumn): boolean { - return column.nullable === false; -} - function validateColumnValue( value: ExtractedValue, column: PopulateColumn, @@ -1210,10 +1621,15 @@ function coerceColumnValue( return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); } case "text": - return String(value).trim(); + return String(value).trim() || undefined; } } +function hasCompleteColumnValue(value: ExtractedValue): boolean { + if (typeof value === "string") return value.trim().length > 0; + return true; +} + function normalizeMaybeRelativeUrl( value: string, baseUrl: string, diff --git a/frontend/app/dataset/new/page.tsx b/frontend/app/dataset/new/page.tsx index 4dbed96..9a48f6f 100644 --- a/frontend/app/dataset/new/page.tsx +++ b/frontend/app/dataset/new/page.tsx @@ -6,7 +6,12 @@ import Link from "next/link"; import { useMutation, useQuery } from "convex/react"; import { api } from "@/convex/_generated/api"; import { EVENTS, track } from "@/lib/analytics"; -import { inferSchema, type InferredColumn } from "@/lib/backend"; +import { + inferSchema, + type CodificationProfile, + type InferredCodificationProfile, + type InferredColumn, +} from "@/lib/backend"; import { useAppAuth, useAppConvexAuth } from "@/lib/app-auth"; import { REFRESH_CADENCE_OPTIONS, @@ -61,6 +66,24 @@ function mapBackendColumn(col: InferredColumn, index: number): ProposedColumn { }; } +function mapCodificationProfile( + profile: InferredCodificationProfile, +): CodificationProfile { + return { + version: 1, + mode: profile.mode, + reason: profile.reason, + primaryKeyShape: profile.primary_key_shape, + families: profile.families.map((family) => ({ + label: family.label, + sourceHost: family.source_host, + sourcePathPrefix: family.source_path_prefix, + urlTemplate: family.url_template, + primaryKeyRegex: family.primary_key_regex, + })), + }; +} + function TypeSelector({ value, onChange }: { value: ColumnType; onChange: (v: ColumnType) => void }) { return (
@@ -100,6 +123,8 @@ export default function NewDatasetPage() { "search_fetch" | "browser" | "hybrid" | null >(null); const [sourceHint, setSourceHint] = useState(""); + const [codificationProfile, setCodificationProfile] = + useState(null); const { getToken } = useAppAuth(); const createDataset = useMutation(api.datasets.create); @@ -149,6 +174,7 @@ export default function NewDatasetPage() { ); setRetrievalStrategy(schema.retrieval_strategy); setSourceHint(schema.source_hint); + setCodificationProfile(mapCodificationProfile(schema.codification_profile)); track(EVENTS.DATASET_SCHEMA_GENERATED, { column_count: schema.columns.length, }); @@ -209,6 +235,7 @@ export default function NewDatasetPage() { })), retrievalStrategy: retrievalStrategy ?? undefined, sourceHint: sourceHint || undefined, + codificationProfile: codificationProfile ?? undefined, }); } catch (err) { const message = err instanceof Error ? err.message : "Failed to create dataset"; diff --git a/frontend/app/setup/page.tsx b/frontend/app/setup/page.tsx index bc5217f..7d85f24 100644 --- a/frontend/app/setup/page.tsx +++ b/frontend/app/setup/page.tsx @@ -57,6 +57,7 @@ function emptyModelConfig(): EffectiveModelConfig { schemaInference: "", populateOrchestrator: "", investigateSubagent: "", + extractorBuilder: "", rowExtractorConcurrency: 5, rowExtractorBrowserAttempts: 2, }; diff --git a/frontend/convex/datasetExtractors.ts b/frontend/convex/datasetExtractors.ts index 8782318..d6d365a 100644 --- a/frontend/convex/datasetExtractors.ts +++ b/frontend/convex/datasetExtractors.ts @@ -15,11 +15,7 @@ export const getActive = internalQuery({ ) .first(); - if ( - !extractor || - extractor.status !== "active" || - extractor.columnsHash !== args.columnsHash - ) { + if (!extractor || extractor.columnsHash !== args.columnsHash) { return null; } diff --git a/frontend/convex/datasets.ts b/frontend/convex/datasets.ts index 829e604..7a9d4a7 100644 --- a/frontend/convex/datasets.ts +++ b/frontend/convex/datasets.ts @@ -36,6 +36,34 @@ const columnValidator = v.object({ normalizationHint: v.optional(v.string()), }); +const codificationProfileValidator = v.object({ + version: v.literal(1), + mode: v.union( + v.literal("disabled"), + v.literal("candidate"), + v.literal("required"), + v.literal("unknown"), + ), + reason: v.string(), + primaryKeyShape: v.union( + v.literal("url"), + v.literal("slug"), + v.literal("name"), + v.literal("id"), + v.literal("mixed"), + v.literal("unknown"), + ), + families: v.array( + v.object({ + label: v.string(), + sourceHost: v.optional(v.string()), + sourcePathPrefix: v.optional(v.string()), + urlTemplate: v.optional(v.string()), + primaryKeyRegex: v.optional(v.string()), + }), + ), +}); + function refreshCadenceFromLegacyLabel( legacyCadence: string | undefined, fallback: RefreshCadence, @@ -287,6 +315,7 @@ export const claimScheduledRefreshInternal = internalMutation({ columns: dataset.columns, retrievalStrategy: dataset.retrievalStrategy, sourceHint: dataset.sourceHint, + codificationProfile: dataset.codificationProfile, ownerId: dataset.ownerId, maxRowCount: dataset.maxRowCount ?? DEFAULT_MAX_ROW_COUNT, }, @@ -389,6 +418,7 @@ export const create = mutation({ ) ), sourceHint: v.optional(v.string()), + codificationProfile: v.optional(codificationProfileValidator), }, handler: async (ctx, args) => { const identity = await requireIdentity(ctx); @@ -427,6 +457,7 @@ export const createForOwnerInternal = internalMutation({ ) ), sourceHint: v.optional(v.string()), + codificationProfile: v.optional(codificationProfileValidator), }, handler: async (ctx, args) => { assertNotReservedOwner(args.ownerId); @@ -467,6 +498,21 @@ export const getOwnedInternal = internalQuery({ }, }); +export const setCodificationProfileInternal = internalMutation({ + args: { + id: v.id("datasets"), + codificationProfile: codificationProfileValidator, + }, + handler: async (ctx, args) => { + const dataset = await ctx.db.get(args.id); + if (!dataset) return { outcome: "not_found" as const }; + await ctx.db.patch(dataset._id, { + codificationProfile: args.codificationProfile, + }); + return { outcome: "updated" as const }; + }, +}); + export const updateRefreshSettings = mutation({ args: { id: v.id("datasets"), diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index da29b7f..97700c4 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -70,6 +70,35 @@ export default defineSchema({ normalizationHint: v.optional(v.string()), }) ), + codificationProfile: v.optional( + v.object({ + version: v.literal(1), + mode: v.union( + v.literal("disabled"), + v.literal("candidate"), + v.literal("required"), + v.literal("unknown") + ), + reason: v.string(), + primaryKeyShape: v.union( + v.literal("url"), + v.literal("slug"), + v.literal("name"), + v.literal("id"), + v.literal("mixed"), + v.literal("unknown") + ), + families: v.array( + v.object({ + label: v.string(), + sourceHost: v.optional(v.string()), + sourcePathPrefix: v.optional(v.string()), + urlTemplate: v.optional(v.string()), + primaryKeyRegex: v.optional(v.string()), + }) + ), + }) + ), retrievalStrategy: v.optional( v.union( v.literal("search_fetch"), diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index 9e92ada..163d9a3 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -5,6 +5,39 @@ export interface InferredSchema { primary_key: string[]; retrieval_strategy: "search_fetch" | "browser" | "hybrid"; source_hint: string; + codification_profile: InferredCodificationProfile; +} + +export interface InferredCodificationProfile { + version: 1; + mode: "disabled" | "candidate" | "required" | "unknown"; + reason: string; + primary_key_shape: "url" | "slug" | "name" | "id" | "mixed" | "unknown"; + families: InferredCodificationFamily[]; +} + +export interface InferredCodificationFamily { + label: string; + source_host?: string; + source_path_prefix?: string; + url_template?: string; + primary_key_regex?: string; +} + +export interface CodificationProfile { + version: 1; + mode: "disabled" | "candidate" | "required" | "unknown"; + reason: string; + primaryKeyShape: "url" | "slug" | "name" | "id" | "mixed" | "unknown"; + families: CodificationFamily[]; +} + +export interface CodificationFamily { + label: string; + sourceHost?: string; + sourcePathPrefix?: string; + urlTemplate?: string; + primaryKeyRegex?: string; } export interface InferredColumn { diff --git a/makefiles/Makefile b/makefiles/Makefile index 2eccc1a..1870a16 100644 --- a/makefiles/Makefile +++ b/makefiles/Makefile @@ -33,7 +33,9 @@ dev: validate-dev-env install-deps start-local-keychain @echo " Mastra Studio: http://localhost:4111" @echo " Convex Dashboard: http://localhost:6791" @echo "" - docker compose -f docker-compose.dev.yml logs -f + @echo "Streaming app logs (frontend, backend, mastra)." + @echo "For Convex logs: docker compose -f docker-compose.dev.yml logs -f convex" + docker compose -f docker-compose.dev.yml logs -f frontend backend mastra validate-dev-env: @if [ ! -f .env ]; then \ From f69036f7d3776f8890af3f17cd5ca0c760846964 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Mon, 15 Jun 2026 13:10:09 -0700 Subject: [PATCH 08/16] hopefully much more robust and stuff --- CODIFICATION.md | 289 -------- backend/prompts/schema-inference.txt | 5 +- backend/src/env.ts | 9 + backend/src/mastra/agents/investigate.ts | 32 +- backend/src/mastra/tools/dataset-tools.ts | 127 +++- backend/src/mastra/tools/investigate-tool.ts | 202 +++++- backend/src/mastra/workflows/populate.ts | 10 +- backend/src/pipeline/codification.ts | 105 ++- backend/src/pipeline/schema-inference.ts | 4 +- .../src/row-extractors/try-row-extractor.ts | 644 ++++++++++++++++-- docker-compose.dev.yml | 1 + frontend/app/dataset/[id]/page.tsx | 13 +- frontend/app/dataset/new/page.tsx | 2 +- frontend/components/SideSheet.tsx | 47 +- frontend/components/table/types.ts | 1 + frontend/convex/datasetRows.ts | 3 + frontend/convex/schema.ts | 1 + 17 files changed, 1070 insertions(+), 425 deletions(-) delete mode 100644 CODIFICATION.md diff --git a/CODIFICATION.md b/CODIFICATION.md deleted file mode 100644 index 3a15073..0000000 --- a/CODIFICATION.md +++ /dev/null @@ -1,289 +0,0 @@ -# BigSet Codification Notes - -This document summarizes the product and technical direction discussed for "codifying" recurring BigSet datasets. - -## Core Idea - -BigSet should use LLMs for ambiguity, but not for every repeated row extraction. - -For datasets with predictable primary keys and predictable target pages — for example: - -- Amazon product URLs → price, star rating, review count, availability -- YC company URLs → company metadata -- GitHub repository URLs → stars, language, license, activity -- App/package pages → ratings, pricing, installs, versions - -BigSet should compile or use a deterministic browser extractor once, then run it across all rows with TinyFish Browser. - -This turns BigSet from: - -```txt -LLM researches every row -``` - -into: - -```txt -LLM understands the dataset once → TinyFish Browser extracts many rows -``` - -## Product Motivation - -BigSet is by TinyFish, and should showcase why TinyFish subscriptions are valuable. - -The goal is not to minimize TinyFish usage. The goal is to produce better datasets while shifting repeatable web work away from OpenRouter token spend and toward TinyFish Browser/Agent usage. - -Better data sells TinyFish. - -## TinyFish Agent vs TinyFish Browser - -We decided to treat TinyFish Browser as the foundational primitive for codified extraction. - -### Browser API - -Use Browser when BigSet knows the page pattern and can drive it with Playwright/CDP. - -Good for: - -- Rendering JavaScript-heavy pages -- Reading dynamic DOM content -- Scrolling / clicking load-more buttons -- Extracting from network/XHR JSON -- Running the same extractor across many similar rows -- Reducing OpenRouter usage - -Browser is deterministic infrastructure. - -### Agent API - -Use Agent when the task is still fuzzy or requires web judgment. - -Good for: - -- Unknown navigation paths -- Multi-step workflows -- Sites where BigSet does not know what to click -- Fallback when a generated extractor fails -- One-off messy cases - -Agent is delegation. - -The preferred quality path is: - -```txt -Search → Fetch → Browser → Agent fallback -``` - -For codified datasets, the main runtime path should be: - -```txt -Browser first, Agent only when needed -``` - -## Codified Dataset / Extractor Compiler - -A codified dataset is one where BigSet can create a reusable extractor from the schema and a representative row. - -Example: - -```txt -Primary key: amazon_product_url -Columns: price, star_rating, review_count, availability -``` - -This should not require an LLM per row. A Playwright extractor can be generated, tested, persisted, and reused. - -## Schema Builder Responsibilities - -Schema inference should determine more than just columns. - -It should identify: - -1. Whether the dataset is plausibly codifiable -2. The primary key shape, especially URL-based keys -3. The likely target site/page type -4. Column type contracts -5. Validation requirements for each column -6. Regex or normalization expectations where useful - -Example column contract: - -```ts -{ - name: "star_rating", - type: "number", - nullable: true, - validation_regex: "^[0-5](\\.\\d)?$", - normalization_hint: "Extract a numeric rating like 4.6 from text such as '4.6 out of 5 stars'." -} -``` - -The schema builder should not itself need browser access. It should decide whether a dataset is a codification candidate. - -## Browser Probe - -If a dataset is codifiable, BigSet should run a browser probe on the first representative row/source URL. - -The browser probe should collect evidence such as: - -- Final URL -- Page title -- Visible text excerpts -- DOM excerpts around likely fields -- JSON-LD / structured data -- Candidate selectors -- Network/XHR JSON hints -- Screenshot if useful - -The probe output becomes context for the extractor-builder model. - -## Extractor Builder Model - -A dedicated model role should generate the Playwright extractor. - -Current model roles are: - -```txt -schemaInference -populateOrchestrator -investigateSubagent -``` - -We should add: - -```txt -extractorBuilder -``` - -This model writes and repairs extractor scripts using: - -- Dataset schema -- Column validation contracts -- First row input -- Browser probe artifacts -- Test failure feedback when applicable - -Default recommendation: - -```txt -extractorBuilder = anthropic/claude-sonnet-4.6 -``` - -Reason: extractor generation happens once per dataset/pattern, not once per row. Code quality matters more than token cost here. - -## Extractor Runtime - -Generated extractor code should be run against the first row, validated, and repaired if needed. - -Once it passes, it can be applied across all rows. - -The extractor returns JSON only. BigSet validates and writes to Convex. - -Example extractor contract: - -```ts -export async function extract({ page, input, helpers }) { - await page.goto(input.product_url, { waitUntil: "domcontentloaded" }); - - const title = await page.locator("#productTitle").textContent(); - const price = await page.locator(".a-price .a-offscreen").first().textContent(); - const rating = await page.locator("#acrPopover").getAttribute("title"); - - return { - data: { - product_url: input.product_url, - title: helpers.cleanText(title), - price: helpers.parsePrice(price), - star_rating: helpers.parseRating(rating), - }, - sources: [page.url()], - how_found: "Opened the product URL in TinyFish Browser and extracted fields from the rendered DOM.", - }; -} -``` - -## Security Stance - -We do not need heavyweight Docker isolation for the MVP. - -But generated code should not run inside the main backend process. - -Minimum acceptable runtime boundary: - -- Run generated code in a child Node process -- Empty environment -- No OpenRouter key -- No TinyFish API key -- No Convex admin key -- No keychain access -- No direct database writes -- Hard timeout -- Output-size limit -- Backend validates all returned data before writing - -The generated script can control only a single TinyFish Browser session via Playwright. - -Before execution, BigSet can reject obvious dangerous code patterns as defense-in-depth: - -- `fs` -- `child_process` -- `worker_threads` -- `process.env` -- `require` -- dynamic import -- `eval` -- `new Function` -- `http` / `https` / `net` - -This is not a perfect hostile-code sandbox, but it is enough for an MVP if no secrets are exposed and all writes go through backend validation. - -## Failure / Roadblock Behavior - -A codified extractor should stop or request repair when: - -- Required selectors return empty -- Validation fails repeatedly -- Multiple consecutive rows fail -- The site layout differs from the first row -- The page is blocked or shows CAPTCHA/access-denied content -- Browser session times out - -At that point BigSet can: - -1. Send failure context back to the extractor-builder for repair -2. Fall back to TinyFish Agent for that row -3. Fall back to the existing LLM investigate flow -4. Mark the row as needing review - -## Usage and Credits - -TinyFish currently does not expose a documented credits-remaining endpoint. - -For now BigSet can estimate usage: - -- Search: free / no credits per current docs -- Fetch: free / no credits per current docs -- Agent: roughly `num_of_steps` credits -- Browser: roughly `ceil(duration_seconds / 240)` credits - -When TinyFish exposes a credits/balance endpoint, BigSet can show real usage and remaining balance. - -## Main Decision - -BigSet should add a codification layer that compiles repeatable dataset extraction into reusable TinyFish Browser scripts. - -LLMs should be used to: - -- infer schemas -- decide if codification is possible -- generate or repair extractors -- handle ambiguous fallback cases - -TinyFish Browser should be used to: - -- execute repeatable extraction -- bypass bot detection -- render dynamic pages -- refresh data cheaply and reliably at scale - -This improves data quality, reduces repeated OpenRouter token usage, and makes BigSet a stronger showcase for TinyFish subscriptions. diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt index cae42a8..1f3b7b0 100644 --- a/backend/prompts/schema-inference.txt +++ b/backend/prompts/schema-inference.txt @@ -12,10 +12,11 @@ Your job is to: 5. Set `source_hint` to a specific URL whenever possible (e.g. `https://www.ycombinator.com/companies?industry=Fintech`). Avoid vague descriptions. 6. Write a `retrieval_hint` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. 7. Set `codification_profile` conservatively: - - `disabled` for broad web research, arbitrary unrelated domains, snippet-only work, or known-blocked sources. - - `candidate` when rows likely share stable page families and a reusable Playwright extractor might work after seeing a representative row. + - `disabled` for broad web research, arbitrary unrelated domains, or snippet-only work with no stable page family. + - `candidate` when rows likely share stable page families and a reusable Playwright extractor might work after seeing a representative row. Use `candidate`, not `disabled`, when the only concern is that the stable source has anti-bot or automation-blocking reputation. BigSet uses TinyFish Browser for interactive browser access, and TinyFish can often get through surfaces that plain fetches or commodity browser automation cannot; the extractor builder can inspect a real page and decline if it is actually blocked. - `required` when one authoritative browser-heavy source/directory is clearly the intended repeated extraction path. - `unknown` only when evidence is genuinely insufficient; prefer `disabled` for broad web datasets. + - For marketplace/catalog identifiers with deterministic product pages, include URL-template families instead of disabling codification. Use the site/schema's actual identifier and route shape; do not hardcode a source-specific template unless it is implied by the user prompt or discovered source hint. Rules: diff --git a/backend/src/env.ts b/backend/src/env.ts index 0ba4f7b..85abe13 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -18,6 +18,11 @@ function numberFromEnv(name: string, fallback: number): number { return Number.isFinite(parsed) ? parsed : fallback; } +function positiveIntegerFromEnv(name: string, fallback: number): number { + const parsed = numberFromEnv(name, fallback); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + export const env = { PROD: process.env.PROD, IS_PROD: process.env.PROD === "1", @@ -51,6 +56,10 @@ export const env = { process.env.SCHEMA_INFERENCE_MODEL ?? "anthropic/claude-sonnet-4.6", POPULATE_ORCHESTRATOR_MODEL: process.env.POPULATE_ORCHESTRATOR_MODEL ?? "qwen/qwen3.7-max", + POPULATE_ORCHESTRATOR_MAX_STEPS: positiveIntegerFromEnv( + "POPULATE_ORCHESTRATOR_MAX_STEPS", + 80, + ), INVESTIGATE_SUBAGENT_MODEL: process.env.INVESTIGATE_SUBAGENT_MODEL ?? "qwen/qwen3.7-max", EXTRACTOR_BUILDER_MODEL: diff --git a/backend/src/mastra/agents/investigate.ts b/backend/src/mastra/agents/investigate.ts index ac80b73..81c7ffe 100644 --- a/backend/src/mastra/agents/investigate.ts +++ b/backend/src/mastra/agents/investigate.ts @@ -5,6 +5,17 @@ import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; +interface InvestigateAgentOptions { + insertDefaults?: { + data?: Record; + lockColumns?: string[]; + sources?: string[]; + cellSources?: Record; + rowSummary?: string; + howFoundPrefix?: string; + }; +} + function buildInvestigateInstructions(columns: PopulateColumn[]): string { const columnNames = columns.map((c) => c.name); const dataExample = columnNames @@ -13,32 +24,35 @@ function buildInvestigateInstructions(columns: PopulateColumn[]): string { const columnsDesc = columns .map( (c) => - `- "${c.name}" (${c.type})${c.isPrimaryKey ? " [PRIMARY KEY]" : ""}${c.description ? `: ${c.description}` : ""}`, + `- "${c.name}" (${c.type})${c.isPrimaryKey ? " [PRIMARY KEY]" : ""}${c.nullable === false ? " [REQUIRED]" : c.nullable === true ? " [OPTIONAL]" : ""}${c.validationRegex ? ` validation_regex=${JSON.stringify(c.validationRegex)}` : ""}${c.normalizationHint ? ` normalization_hint=${JSON.stringify(c.normalizationHint)}` : ""}${c.description ? `: ${c.description}` : ""}`, ) .join("\n"); - return `You research one entity and insert one row. Be fast — you have very few steps. + return `You research one entity and insert one row. Work efficiently, but do not sacrifice data quality. Columns: ${columnsDesc} RULES: - Do NOT fetch the same URL twice. If a fetch worked, use the data you got. -- You have at most 6 tool calls total. Budget them: 1 fetch + 1 search + 1 fetch + 1 insert = done. -- ALWAYS insert a row, even if some fields are incomplete. Use "" for unknown fields. Partial real data is better than no row. +- Try to stay under 6 tool calls when that is enough, but do not stop early if more searching/fetching is needed to verify requested columns. +- Your goal is to fill every column from real, source-backed evidence. Some columns may be impossible to verify; use "" only after a reasonable targeted attempt. - Never fabricate values. Use "" for anything you cannot verify. +- If the prompt says some values were already verified by browser extraction, preserve them exactly. Focus tool calls on unresolved columns. +- Optional/nullable columns should still be researched. Optional only means the row can be inserted blank if you cannot verify the value. +- Normalize values to the schema contract before inserting. Follow validation_regex and normalization_hint when present. - insert_row rejects duplicates based on primary key columns. If you get a "Duplicate" error, do NOT retry — report INSERTED: false and move on. TOOL CALL FORMAT — every tool call argument must be a JSON object wrapped in curly braces: search_web: {"query": "your search terms"} fetch_page: {"url": "https://example.com"} - insert_row: {"data": [${dataExample}], "sources": ["https://url-you-fetched.com"], "row_summary": "one line about this entity", "how_found": "step by step guide on how to extract the data so an agent in the future can do it too"} + insert_row: {"data": [${dataExample}], "sources": ["https://url-you-fetched.com"], "cell_sources": [{"column": "column_name", "sources": ["https://url-that-justifies-this-cell.com"]}], "row_summary": "one line about this entity", "how_found": "step by step guide on how to extract the data so an agent in the future can do it too"} WORKFLOW: 1. Fetch 1-2 of the provided URLs to get real data (if URLs were given). -2. If you need more, run ONE search and fetch the best result. -3. Call insert_row with whatever real data you have. Use "" for missing fields. - Include "sources" (URLs you fetched), "row_summary" (one line about this entity), and "how_found" (a step by step guide on how you found this data. eg, 1. fetch the contents of this url "", 2. Look for the pricing field, and title name field, 3. etc...) +2. For unresolved columns, run targeted searches/fetches until the value is verified, clearly unavailable, or further tool calls are unlikely to help. +3. Call insert_row with the best verified row you can produce. Use "" for any field that remains unknown after the targeted attempt. + Include "sources" (URLs you fetched for the row), "cell_sources" (only URLs that justify the exact cell value for that column), "row_summary" (one line about this entity), and "how_found" (a step by step guide on how you found this data. eg, 1. fetch the contents of this url "", 2. Look for the pricing field, and title name field, 3. etc...) 4. Write your final response: INSERTED: true/false SUMMARY: one line @@ -59,12 +73,14 @@ export function buildInvestigateAgent( authContext: AuthContext, columns: PopulateColumn[], llmConfig: LlmProviderConfig, + options: InvestigateAgentOptions = {}, ): Agent { const modelSlug = authContext.modelConfig!.investigateSubagent; const { insert_row } = buildPopulateTools( authorizedDatasetId, authContext, + options, ); return new Agent({ id: "investigate-agent", diff --git a/backend/src/mastra/tools/dataset-tools.ts b/backend/src/mastra/tools/dataset-tools.ts index e0109a6..a249281 100644 --- a/backend/src/mastra/tools/dataset-tools.ts +++ b/backend/src/mastra/tools/dataset-tools.ts @@ -65,6 +65,26 @@ const rowDataCellSchema = z.object({ type RowDataCell = z.infer; +const cellSourcesSchema = z.object({ + column: z.string().min(1), + sources: z.array(z.string()).min(1), +}); + +type CellSourcesInput = z.infer; + +interface InsertDefaults { + data?: Record; + lockColumns?: string[]; + sources?: string[]; + cellSources?: Record; + rowSummary?: string; + howFoundPrefix?: string; +} + +interface PopulateToolOptions { + insertDefaults?: InsertDefaults; +} + function rowDataCellsToRecord(data: RowDataCell[]): Record { const row: Record = {}; for (const cell of data) { @@ -81,6 +101,65 @@ function cleanDataKeys(data: Record): Record { return cleaned; } +function hasMeaningfulValue(value: unknown): boolean { + if (value === undefined || value === null) return false; + if (typeof value === "string") return value.trim().length > 0; + return true; +} + +function uniqueStrings(values: Array): string[] { + return [...new Set(values.filter((value): value is string => Boolean(value)))]; +} + +function cellSourcesToRecord( + entries: CellSourcesInput[] | undefined, +): Record | undefined { + if (!entries || entries.length === 0) return undefined; + + const output: Record = {}; + for (const entry of entries) { + const column = entry.column.replace(/^["`]+|["`]+$/g, ""); + const sources = uniqueStrings(entry.sources); + if (column && sources.length > 0) output[column] = sources; + } + return Object.keys(output).length > 0 ? output : undefined; +} + +function mergeCellSources( + defaults: Record | undefined, + proposed: Record | undefined, +): Record | undefined { + const merged: Record = {}; + for (const [column, sources] of Object.entries(defaults ?? {})) { + const cleanSources = uniqueStrings(sources); + if (cleanSources.length > 0) merged[column] = cleanSources; + } + for (const [column, sources] of Object.entries(proposed ?? {})) { + const cleanSources = uniqueStrings([...(merged[column] ?? []), ...sources]); + if (cleanSources.length > 0) merged[column] = cleanSources; + } + return Object.keys(merged).length > 0 ? merged : undefined; +} + +function mergeInsertData( + proposedData: Record, + defaults: InsertDefaults | undefined, +): Record { + if (!defaults?.data) return proposedData; + + const merged = cleanDataKeys({ + ...defaults.data, + ...proposedData, + }); + + for (const column of defaults.lockColumns ?? []) { + const value = defaults.data[column]; + if (hasMeaningfulValue(value)) merged[column] = value; + } + + return merged; +} + /** * One place that records a refused capability check, so the log line + * PostHog event always carry the same fields. Called from update_row / @@ -117,6 +196,7 @@ function recordCapabilityViolation(params: { export function buildPopulateTools( authorizedDatasetId: string, authContext: AuthContext, + options: PopulateToolOptions = {}, ) { if (!authorizedDatasetId) { // Fail loud at construction time — never silently fall back to an @@ -151,6 +231,12 @@ export function buildPopulateTools( .array(z.string()) .optional() .describe("URLs you visited or used to gather data for this row"), + cell_sources: z + .array(cellSourcesSchema) + .optional() + .describe( + 'Per-cell source URLs as {"column": "column_name", "sources": ["https://..."]}. Only include URLs that justify that exact cell value.', + ), row_summary: z .string() .optional() @@ -161,7 +247,7 @@ export function buildPopulateTools( .describe("Brief description of how you found and verified this data"), }), outputSchema: writeResultSchema, - execute: async ({ data, sources, row_summary, how_found }) => { + execute: async ({ data, sources, cell_sources, row_summary, how_found }) => { if (!data || data.length === 0) return { success: false, @@ -169,17 +255,35 @@ export function buildPopulateTools( 'data is required and must include at least one entry like { "column": "column_name", "value": "cell value" }.', }; - const cleanedData = cleanDataKeys(rowDataCellsToRecord(data)); + const cleanedData = mergeInsertData( + cleanDataKeys(rowDataCellsToRecord(data)), + options.insertDefaults, + ); + const mergedSources = uniqueStrings([ + ...(options.insertDefaults?.sources ?? []), + ...(sources ?? []), + ]); + const mergedCellSources = mergeCellSources( + options.insertDefaults?.cellSources, + cellSourcesToRecord(cell_sources), + ); + const mergedRowSummary = + row_summary ?? options.insertDefaults?.rowSummary; + const mergedHowFound = uniqueStrings([ + options.insertDefaults?.howFoundPrefix, + how_found, + ]).join("\n"); console.log( - `[insert_row] ${logCtx} cols=${Object.keys(cleanedData).length} sources=${sources?.length ?? 0}`, + `[insert_row] ${logCtx} cols=${Object.keys(cleanedData).length} sources=${mergedSources.length}`, ); try { await convex.mutation(internal.datasetRows.insert, { datasetId: authorizedDatasetId, data: cleanedData, - ...(sources !== undefined ? { sources } : {}), - ...(row_summary !== undefined ? { rowSummary: row_summary } : {}), - ...(how_found !== undefined ? { howFound: how_found } : {}), + ...(mergedSources.length > 0 ? { sources: mergedSources } : {}), + ...(mergedCellSources ? { cellSources: mergedCellSources } : {}), + ...(mergedRowSummary !== undefined ? { rowSummary: mergedRowSummary } : {}), + ...(mergedHowFound ? { howFound: mergedHowFound } : {}), }); return { success: true }; } catch (err) { @@ -290,6 +394,12 @@ export function buildPopulateTools( .array(z.string()) .optional() .describe("Updated source URLs where this data was verified"), + cell_sources: z + .array(cellSourcesSchema) + .optional() + .describe( + 'Updated per-cell source URLs as {"column": "column_name", "sources": ["https://..."]}. Only include URLs that justify that exact cell value.', + ), row_summary: z .string() .optional() @@ -300,7 +410,7 @@ export function buildPopulateTools( .describe("Brief description of how the updated data was found"), }), outputSchema: writeResultSchema, - execute: async ({ rowId, data, sources, row_summary, how_found }) => { + execute: async ({ rowId, data, sources, cell_sources, row_summary, how_found }) => { if (!rowId) return { success: false, error: "rowId is required." }; if (!data || data.length === 0) return { @@ -318,6 +428,9 @@ export function buildPopulateTools( expectedDatasetId: authorizedDatasetId, data: cleanedData, ...(sources !== undefined ? { sources } : {}), + ...(cell_sources !== undefined + ? { cellSources: cellSourcesToRecord(cell_sources) ?? {} } + : {}), ...(row_summary !== undefined ? { rowSummary: row_summary } : {}), ...(how_found !== undefined ? { howFound: how_found } : {}), }); diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 8432611..0b03d38 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -6,7 +6,10 @@ import type { AuthContext } from "../workflows/populate.js"; import type { CodificationProfile, PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; import { getSignal } from "../../abort-registry.js"; -import { tryRowExtractor } from "../../row-extractors/try-row-extractor.js"; +import { + tryRowExtractorDraft, + type RowExtractorDraftResult, +} from "../../row-extractors/try-row-extractor.js"; import type { LlmProviderConfig } from "../../config/llm.js"; import { AGENT_MAX_OUTPUT_TOKENS } from "../../config/agent-output-tokens.js"; @@ -75,10 +78,88 @@ function parseInvestigateResult( }; } -function isIncompleteExtractorFailure(reason: string): boolean { - return /generated extractor (missed columns|missed required columns|returned values that failed validation)|repaired extractor failed validation|extractor returned no non-primary values/i.test( - reason, +function hasMeaningfulValue(value: unknown): boolean { + if (value === undefined || value === null) return false; + if (typeof value === "string") return value.trim().length > 0; + return true; +} + +function filledColumnNames( + draft: RowExtractorDraftResult | undefined, + columns: PopulateColumn[], +): string[] { + if (!draft?.data) return []; + const primaryKeyColumns = new Set( + columns.filter((column) => column.isPrimaryKey).map((column) => column.name), ); + return Object.entries(draft.data) + .filter(([column, value]) => { + if (!hasMeaningfulValue(value)) return false; + if (primaryKeyColumns.has(column)) return true; + return (draft.cellSources?.[column]?.length ?? 0) > 0; + }) + .map(([column]) => column); +} + +function formatRowData(data: Record | undefined): string { + if (!data) return "(none)"; + const lines = Object.entries(data) + .filter(([, value]) => hasMeaningfulValue(value)) + .map(([column, value]) => `- ${column}: ${JSON.stringify(value)}`); + return lines.length > 0 ? lines.join("\n") : "(none)"; +} + +function pickRowData( + data: Record | undefined, + columns: string[], +): Record | undefined { + if (!data) return undefined; + const picked: Record = {}; + for (const column of columns) { + const value = data[column]; + if (hasMeaningfulValue(value)) picked[column] = value; + } + return Object.keys(picked).length > 0 ? picked : undefined; +} + +function formatUnresolvedColumns( + columns: PopulateColumn[], + draft: RowExtractorDraftResult | undefined, + lockedColumns: string[] = [], +): string { + if (!draft) return "(none)"; + const locked = new Set(lockedColumns); + const fallbackNames = columns + .filter((column) => !column.isPrimaryKey) + .filter((column) => { + if (locked.has(column.name)) return false; + return draft.missingColumns?.includes(column.name) || hasMeaningfulValue(draft.data?.[column.name]); + }) + .map((column) => column.name); + if (fallbackNames.length === 0) return "(none)"; + + const columnByName = new Map(columns.map((column) => [column.name, column])); + return fallbackNames + .map((name) => { + const column = columnByName.get(name); + const requiredness = column?.nullable === false ? "required" : "optional"; + const status = draft.columnStatuses?.[name]; + const missingCellSource = + hasMeaningfulValue(draft.data?.[name]) && (draft.cellSources?.[name]?.length ?? 0) === 0; + const detail = [ + `${requiredness}`, + missingCellSource ? "browser_status=unverified_cell_source" : undefined, + status?.status ? `browser_status=${status.status}` : undefined, + status?.reason ? `reason=${JSON.stringify(status.reason)}` : undefined, + column?.normalizationHint + ? `normalization=${JSON.stringify(column.normalizationHint)}` + : undefined, + ] + .filter(Boolean) + .join("; "); + return `- ${name} (${detail})`; + }) + .join("\n"); } /** @@ -125,7 +206,7 @@ export function buildSubagentTool( primary_keys.map(({ column, value }) => [column, value]), ); - const extractorResult = await tryRowExtractor({ + const extractorResult = await tryRowExtractorDraft({ datasetId: authorizedDatasetId, columns, primaryKeys: primaryKeyRecord, @@ -139,18 +220,6 @@ export function buildSubagentTool( browserAttempts: authContext.modelConfig.rowExtractorBrowserAttempts, extractorBuilderModel: authContext.modelConfig.extractorBuilder, }); - if (extractorResult.status === "inserted") { - if (metrics) metrics.rowsInserted++; - console.log( - `[run_subagent] row extractor inserted entity="${entity_hint}" reason="${extractorResult.reason}"`, - ); - return { - inserted: true, - reason: extractorResult.reason, - row_summary: extractorResult.rowSummary, - clues: undefined, - }; - } if (/duplicate/i.test(extractorResult.reason)) { return { inserted: false, @@ -159,18 +228,51 @@ export function buildSubagentTool( clues: undefined, }; } - if (extractorResult.status === "failed") { + + if (extractorResult.status === "extracted") { + const missingColumns = extractorResult.missingColumns ?? []; + if (missingColumns.length === 0) { + try { + await convex.mutation(internal.datasetRows.insert, { + datasetId: authorizedDatasetId, + data: extractorResult.data ?? {}, + sources: extractorResult.sources, + cellSources: extractorResult.cellSources, + rowSummary: extractorResult.rowSummary, + howFound: + "Opened the row target with TinyFish Browser and ran the dataset's generated Playwright extractor. No fallback columns were unresolved.", + }); + if (metrics) metrics.rowsInserted++; + console.log( + `[run_subagent] row extractor inserted complete row entity="${entity_hint}" reason="${extractorResult.reason}"`, + ); + return { + inserted: true, + reason: extractorResult.reason, + row_summary: extractorResult.rowSummary, + clues: undefined, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/duplicate/i.test(msg)) { + return { + inserted: false, + reason: `${msg} Move on to the next entity.`, + row_summary: undefined, + clues: undefined, + }; + } + throw err; + } + } + + console.log( + `[run_subagent] row extractor drafted entity="${entity_hint}" filled=${extractorResult.extractedColumns?.length ?? 0} unresolved=${missingColumns.length}`, + ); + } else if (extractorResult.status === "failed") { console.warn( `[run_subagent] row extractor failed entity="${entity_hint}" reason="${extractorResult.reason}"`, ); - if (isIncompleteExtractorFailure(extractorResult.reason)) { - return { - inserted: false, - reason: `Browser extractor did not produce a complete row: ${extractorResult.reason}`, - row_summary: undefined, - clues: "Improve the reusable browser extractor or choose a schema/source where every column is visible.", - }; - } } else if (extractorResult.status === "miss") { console.log( `[run_subagent] row extractor missed entity="${entity_hint}" reason="${extractorResult.reason}"`, @@ -181,11 +283,37 @@ export function buildSubagentTool( `[run_subagent] spawning subagent user=${authContext.authorizedUserId} run=${authContext.workflowRunId} dataset=${authorizedDatasetId} entity="${entity_hint}" pk=${JSON.stringify(primary_keys)}`, ); + const browserFilledValues = + extractorResult.status === "extracted" ? extractorResult.data : undefined; + const browserFilledColumns = + extractorResult.status === "extracted" + ? filledColumnNames(extractorResult, columns) + : []; + const browserLockedValues = + extractorResult.status === "extracted" + ? pickRowData(browserFilledValues, browserFilledColumns) + : undefined; + const browserSources = + extractorResult.status === "extracted" ? extractorResult.sources : undefined; + const agent = buildInvestigateAgent( authorizedDatasetId, authContext, columns, llmConfig, + extractorResult.status === "extracted" + ? { + insertDefaults: { + data: browserLockedValues, + lockColumns: browserFilledColumns, + sources: browserSources, + cellSources: extractorResult.cellSources, + rowSummary: extractorResult.rowSummary, + howFoundPrefix: + "Browser extraction pass ran first. The insert tool preserved browser-verified values and merged fallback research for unresolved columns.", + }, + } + : {}, ); const pkBlock = primary_keys @@ -196,6 +324,26 @@ export function buildSubagentTool( ? `\nUseful URLs to start from:\n${urls.map((u) => `- ${u}`).join("\n")}` : ""; const notesBlock = notes ? `\nAdditional notes: ${notes}` : ""; + const browserDraftBlock = + extractorResult.status === "extracted" + ? ` + +Browser extraction already verified these values. Do not spend tool calls re-verifying or changing them; insert_row will preserve them: +${formatRowData(browserLockedValues)} + +Unresolved columns to research now. Try every listed column, including optional ones. If a value still cannot be verified, insert "" for that column and explain why: +${formatUnresolvedColumns(columns, extractorResult, browserFilledColumns)} + +Browser sources already used: +${(browserSources ?? []).map((u) => `- ${u}`).join("\n") || "(none)"} +` + : ` + +Browser extraction did not produce a usable draft for this row: +- status: ${extractorResult.status} +- reason: ${extractorResult.reason} + +Research all non-primary columns normally.`; const prompt = `Research this entity and insert a row if you find real, verified data. @@ -205,7 +353,7 @@ Primary key values (MUST be included in insert_row): ${pkBlock} Context (partial data already found): -${context}${urlsBlock}${notesBlock}`; +${context}${urlsBlock}${notesBlock}${browserDraftBlock}`; const abortSignal = getSignal(authorizedDatasetId); const result = await agent.generate(prompt, { diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index 7f7746e..2f9ba56 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -15,6 +15,7 @@ import { buildPopulateAgent } from "../agents/populate.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; import { getSignal } from "../../abort-registry.js"; +import { env } from "../../env.js"; /** * Server-set auth/run context threaded through every step. @@ -93,7 +94,7 @@ const enumerateStep = createStep({ const columnsDesc = inputData.columns .map( (c) => - `- "${c.name}" (${c.type})${c.isPrimaryKey ? " [PK]" : ""}${c.description ? `: ${c.description}` : ""}`, + `- "${c.name}" (${c.type})${c.isPrimaryKey ? " [PK]" : ""}${c.nullable === false ? " [REQUIRED]" : c.nullable === true ? " [OPTIONAL]" : ""}${c.validationRegex ? ` validation_regex=${JSON.stringify(c.validationRegex)}` : ""}${c.normalizationHint ? ` normalization_hint=${JSON.stringify(c.normalizationHint)}` : ""}${c.description ? `: ${c.description}` : ""}`, ) .join("\n"); @@ -181,7 +182,7 @@ const buildPromptStep = createStep({ const columnsDesc = inputData.columns .map( (c) => - `- "${c.name}" (${c.type})${c.isPrimaryKey ? " [PRIMARY KEY]" : ""}${c.description ? `: ${c.description}` : ""}`, + `- "${c.name}" (${c.type})${c.isPrimaryKey ? " [PRIMARY KEY]" : ""}${c.nullable === false ? " [REQUIRED]" : c.nullable === true ? " [OPTIONAL]" : ""}${c.validationRegex ? ` validation_regex=${JSON.stringify(c.validationRegex)}` : ""}${c.normalizationHint ? ` normalization_hint=${JSON.stringify(c.normalizationHint)}` : ""}${c.description ? `: ${c.description}` : ""}`, ) .join("\n"); @@ -278,9 +279,12 @@ const agentStep = createStep({ metrics, ); const abortSignal = getSignal(inputData.authorizedDatasetId); + console.log( + `[populate-agent] Running orchestrator with maxSteps=${env.POPULATE_ORCHESTRATOR_MAX_STEPS}`, + ); const result = await agent.generate(inputData.prompt, { abortSignal, - maxSteps: 80, + maxSteps: env.POPULATE_ORCHESTRATOR_MAX_STEPS, modelSettings: { maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.POPULATE_ORCHESTRATOR, }, diff --git a/backend/src/pipeline/codification.ts b/backend/src/pipeline/codification.ts index ff75946..c61f214 100644 --- a/backend/src/pipeline/codification.ts +++ b/backend/src/pipeline/codification.ts @@ -10,6 +10,9 @@ interface CodificationClassificationInput { datasetName?: string; description?: string; columns: PopulateColumn[]; + primaryKeys?: Record; + urls?: string[]; + context?: string; retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; sourceHint?: string; } @@ -41,8 +44,13 @@ export function normalizeCodificationProfile( return profile ?? classifyCodificationProfile(input); } -export function shouldAttemptCodification(profile: CodificationProfile): boolean { - return profile.mode === "candidate" || profile.mode === "required"; +export function shouldAttemptCodification( + profile: CodificationProfile, + input?: CodificationClassificationInput, +): boolean { + if (profile.mode === "candidate" || profile.mode === "required") return true; + if (!input || (profile.mode !== "disabled" && profile.mode !== "unknown")) return false; + return hasConcreteCodificationRoute(profile, input); } export function classifyCodificationProfile( @@ -149,11 +157,85 @@ function inferColumnShape(column: PopulateColumn): CodificationProfile["primaryK if (/\bslug\b|\bhandle\b|\bpath\b|\brepo\b|\bpackage\b|\busername\b/.test(text)) { return "slug"; } - if (/\bid\b|_id\b|\bidentifier\b|\buuid\b/.test(text)) return "id"; + if (/\bid\b|_id\b|\bidentifier\b|\buuid\b|\bisbn\b|\bsku\b/.test(text)) { + return "id"; + } if (/\bname\b|\btitle\b/.test(text)) return "name"; return "unknown"; } +function hasConcreteCodificationRoute( + profile: CodificationProfile, + input: CodificationClassificationInput, +): boolean { + const hasUsableTemplate = profile.families.some( + (family) => family.urlTemplate && hasTemplateValues(family.urlTemplate, input), + ); + if (hasUsableTemplate) return true; + + const rowUrl = firstHttpUrl( + [ + ...Object.values(input.primaryKeys ?? {}), + ...(input.urls ?? []), + input.context, + ].join(" "), + ); + const broadResearch = isBroadResearchDisableReason(profile.reason); + if (rowUrl && !broadResearch) return true; + + const sourceUrl = firstHttpUrl(input.sourceHint); + const primaryKeyShape = + profile.primaryKeyShape === "unknown" + ? inferPrimaryKeyShape(input.columns) + : profile.primaryKeyShape; + const structuredPrimaryKey = + primaryKeyShape === "id" || + primaryKeyShape === "slug" || + primaryKeyShape === "url"; + if (sourceUrl && structuredPrimaryKey) return true; + if (broadResearch) return false; + + const accessRiskOnly = /\b(block|blocked|captcha|bot|automation|browser|fetch|access|degrad)/i.test( + profile.reason, + ); + return accessRiskOnly && (structuredPrimaryKey || hasIdentifierLikePrimaryKey(input.primaryKeys)); +} + +function isBroadResearchDisableReason(reason: string): boolean { + return /\b(broad web|arbitrary unrelated|unrelated domains|snippet-only|search snippets)\b/i.test( + reason, + ); +} + +function hasTemplateValues( + template: string, + input: CodificationClassificationInput, +): boolean { + const placeholders = [...template.matchAll(/\{([a-zA-Z0-9_]+)\}/g)].map( + (match) => match[1], + ); + if (placeholders.length === 0) return false; + return placeholders.every((placeholder) => + Boolean(findPrimaryKeyValue(placeholder, input.primaryKeys ?? {})), + ); +} + +function hasIdentifierLikePrimaryKey( + primaryKeys: Record | undefined, +): boolean { + const values = Object.values(primaryKeys ?? {}) + .map((value) => value.trim()) + .filter(Boolean); + if (values.length === 0) return false; + + return values.every((value) => { + if (/\s/.test(value)) return false; + if (/^https?:\/\//i.test(value)) return true; + if (!/^[a-z0-9._~:/?#\[\]@!$&'()*+,;=%-]{6,}$/i.test(value)) return false; + return /[0-9._~:/?#\[\]@!$&'()*+,;=%-]/.test(value); + }); +} + function firstHttpUrl(value: string | undefined): URL | undefined { if (!value) return undefined; const match = value.match(/https?:\/\/[^\s)>"']+/i); @@ -181,3 +263,20 @@ function sanitizeFamilyLabel(value: string): string { .replace(/^_+|_+$/g, ""); return /^[a-z]/.test(label) ? label : `site_${label || "unknown"}`; } + +function findPrimaryKeyValue( + columnName: string | undefined, + primaryKeys: Record, +): string | undefined { + if (!columnName) return undefined; + if (primaryKeys[columnName]) return primaryKeys[columnName]; + const normalizedColumn = normalizeFieldName(columnName); + const entry = Object.entries(primaryKeys).find( + ([key]) => normalizeFieldName(key) === normalizedColumn, + ); + return entry?.[1]; +} + +function normalizeFieldName(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]+/g, ""); +} diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 9bd1be5..3ad92d9 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -20,8 +20,9 @@ Your job is to: 6. Write a \`retrieval_hint\` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. 7. For each column where a value has a known shape, include \`validation_regex\` and \`normalization_hint\`. These are extractor contracts, not UI decoration. Examples: ratings, prices, dates, URL/slug shapes, repository slugs, app package names, counts, currencies, availability labels. Omit \`validation_regex\` only when the value is genuinely free-form text. 8. Set \`codification_profile\`. This is a cheap schema-time decision about whether BigSet should attempt to compile a reusable Playwright extractor later. - - \`mode: "disabled"\` when rows will come from broad web search, arbitrary unrelated domains, search snippets, or sources that are known to block browser automation for the requested fields. + - \`mode: "disabled"\` when rows will come from broad web search, arbitrary unrelated domains, or search snippets with no stable page family. - \`mode: "candidate"\` when rows likely share one or more stable page families and a reusable browser script may work after seeing a representative row. + - Use \`mode: "candidate"\`, not \`disabled\`, when the only concern is that the stable source has anti-bot or automation-blocking reputation. BigSet uses TinyFish Browser for interactive browser access, and TinyFish can often get through surfaces that plain fetches or commodity browser automation cannot. The extractor builder will inspect a real page and decline if it is actually blocked. - \`mode: "required"\` when the dataset is clearly tied to one authoritative browser-heavy source or directory where repeated extraction is the intended path. - \`mode: "unknown"\` only when the prompt/schema gives too little evidence; prefer \`disabled\` over \`unknown\` for broad web datasets to avoid expensive extractor attempts. Include \`primary_key_shape\` as one of \`url\`, \`slug\`, \`name\`, \`id\`, \`mixed\`, or \`unknown\`. Include \`families\` for known source/page families. Use snake_case labels. For URL templates, use column placeholders like \`https://github.com/{repo_slug}\`. @@ -35,6 +36,7 @@ Rules: - Validation regexes must validate the normalized final value, not raw page text. Keep them practical and anchored, e.g. "^[0-5](\\\\.\\\\d)?$" for a normalized rating or "^[^/\\\\s]+/[^/\\\\s]+$" for an owner/repo slug. - \`normalization_hint\` should tell the extractor how to convert raw page text into the stored value, e.g. "Convert '4.6 out of 5 stars' to '4.6'" or "Strip commas and convert 1.2k to 1200". - \`codification_profile\` should be conservative. Do not mark arbitrary company/person/place/product research as codifiable just because pages exist on the web. Mark it codifiable only when rows can be routed to stable page families from the primary key, source_hint, or obvious URL templates. +- For marketplace/catalog identifiers with deterministic product pages, prefer a URL-template family over disabling codification. Use the site/schema's actual identifier and route shape; do not hardcode a source-specific template unless it is implied by the user prompt or discovered source hint. - When a column is a scalar numeric rating (e.g. average score like 4.3/5 for restaurants, cafes, hotels, products, apps): name it generically (e.g. "rating" not "yelp_rating") and write a retrieval_hint explaining that review sites (Yelp, TripAdvisor, Google Maps) block direct page fetches, so the agent must extract ratings from **search result snippets**. The hint should say: "Search for \\" rating reviews\\" and include location terms only when location is part of the entity identity. Look for ratings in snippets from TripAdvisor (\\"rated X.X of 5\\"), Yelp search listings (\\"X.X (N reviews)\\"), or aggregator sites (Birdeye, joe.coffee, giftly, Uber Eats, menufyy). Do NOT try to fetch yelp.com or tripadvisor.com directly — they block automated access. Accept ratings from any reputable source." If including a rating column, also add a "rating_source" text column so the agent records where the rating came from. Do not rename review-count or review-text fields to "rating" — keep those as distinct columns (e.g. "review_count") when the user explicitly asks for them.`; async function getModel(modelSlug?: string) { diff --git a/backend/src/row-extractors/try-row-extractor.ts b/backend/src/row-extractors/try-row-extractor.ts index 8d89601..bef2e5e 100644 --- a/backend/src/row-extractors/try-row-extractor.ts +++ b/backend/src/row-extractors/try-row-extractor.ts @@ -27,6 +27,20 @@ import { DEFAULT_MODEL_IDS } from "../config/models.js"; type ExtractorStatus = "inserted" | "updated" | "unchanged" | "miss" | "failed"; type ExtractedValue = string | number | boolean; +type ColumnExtractionStatus = + | "extracted" + | "derived" + | "not_present_on_page" + | "blocked" + | "ambiguous" + | "validation_failed" + | "fallback_needed" + | "missing"; + +interface ColumnStatus { + status: ColumnExtractionStatus; + reason?: string; +} export interface TryRowExtractorInput { datasetId: string; @@ -55,6 +69,20 @@ export interface TryRowExtractorResult { sources?: string[]; } +export interface RowExtractorDraftResult { + status: "extracted" | "miss" | "failed"; + reason: string; + data?: Record; + rowSummary?: string; + sources?: string[]; + cellSources?: Record; + extractedColumns?: string[]; + missingColumns?: string[]; + requiredMissingColumns?: string[]; + optionalMissingColumns?: string[]; + columnStatuses?: Record; +} + interface TinyFishBrowserSession { session_id: string; cdp_url: string; @@ -73,17 +101,25 @@ interface GenericExtraction { data: Record; sources: string[]; rowSummary?: string; + cellSources: Record; extractedColumns: string[]; missingColumns: string[]; + requiredMissingColumns: string[]; + optionalMissingColumns: string[]; + columnStatuses: Record; } interface GeneratedExtractorResult { data?: Record; sources?: unknown; + cell_sources?: unknown; + cellSources?: unknown; row_summary?: unknown; rowSummary?: unknown; how_found?: unknown; howFound?: unknown; + column_status?: unknown; + columnStatus?: unknown; } interface DetailedExtractorTestResult { @@ -108,7 +144,8 @@ const CDP_CONNECT_TIMEOUT_MS = 45_000; const DEFAULT_BROWSER_ATTEMPTS = 2; const EXTRACTOR_RUNNER_TIMEOUT_MS = 60_000; const EXTRACTOR_SCRIPT_OUTPUT_LIMIT = 256_000; -const EXTRACTOR_BUILDER_MAX_STEPS = 24; +const EXTRACTOR_BUILDER_MAX_STEPS = 80; +const EXTRACTOR_REPAIR_MAX_STEPS = 24; const BACKEND_ROOT = fileURLToPath(new URL("../../", import.meta.url)); const GENERIC_EXTRACTOR_HOW_FOUND = "Opened the row target with TinyFish Browser and ran the dataset's generated Playwright extractor."; @@ -200,8 +237,58 @@ try { export async function tryRowExtractor( input: TryRowExtractorInput, ): Promise { + const draft = await tryRowExtractorDraft(input); + if (draft.status !== "extracted") { + return { + status: draft.status, + reason: draft.reason, + rowSummary: draft.rowSummary, + sources: draft.sources, + }; + } + + if ((draft.missingColumns ?? []).length > 0) { + return { + status: "miss", + reason: `browser extractor left unresolved columns for fallback: ${(draft.missingColumns ?? []).join(", ")}`, + rowSummary: draft.rowSummary, + sources: draft.sources, + }; + } + + try { + await convex.mutation(internal.datasetRows.insert, { + datasetId: input.datasetId, + data: draft.data ?? {}, + sources: draft.sources, + cellSources: draft.cellSources, + rowSummary: draft.rowSummary, + howFound: GENERIC_EXTRACTOR_HOW_FOUND, + }); + + return { + status: "inserted", + reason: `Inserted by generic browser extractor (${(draft.extractedColumns ?? []).join(", ")})`, + rowSummary: draft.rowSummary, + sources: draft.sources, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/duplicate/i.test(msg)) { + return { + status: "miss", + reason: `${msg} Move on to the next entity.`, + }; + } + return { status: "failed", reason: msg }; + } +} + +export async function tryRowExtractorDraft( + input: TryRowExtractorInput, +): Promise { const codificationProfile = normalizeCodificationProfile(input.codificationProfile, input); - if (!shouldAttemptCodification(codificationProfile)) { + if (!shouldAttemptCodification(codificationProfile, input)) { return { status: "miss", reason: `codification profile is ${codificationProfile.mode}: ${codificationProfile.reason}`, @@ -220,19 +307,21 @@ export async function tryRowExtractor( }; } - await convex.mutation(internal.datasetRows.insert, { - datasetId: input.datasetId, - data: extraction.data, - sources: extraction.sources, - rowSummary: extraction.rowSummary, - howFound: GENERIC_EXTRACTOR_HOW_FOUND, - }); - return { - status: "inserted", - reason: `Inserted by generic browser extractor (${extraction.extractedColumns.join(", ")})`, + status: "extracted", + reason: + extraction.missingColumns.length > 0 + ? `Browser extractor filled ${extraction.extractedColumns.length} columns and left ${extraction.missingColumns.length} for fallback` + : `Browser extractor filled all ${extraction.extractedColumns.length} columns`, + data: extraction.data, rowSummary: extraction.rowSummary, sources: extraction.sources, + cellSources: extraction.cellSources, + extractedColumns: extraction.extractedColumns, + missingColumns: extraction.missingColumns, + requiredMissingColumns: extraction.requiredMissingColumns, + optionalMissingColumns: extraction.optionalMissingColumns, + columnStatuses: extraction.columnStatuses, }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -250,7 +339,7 @@ export async function tryRefreshRowExtractor( input: TryRefreshRowExtractorInput, ): Promise { const codificationProfile = normalizeCodificationProfile(input.codificationProfile, input); - if (!shouldAttemptCodification(codificationProfile)) { + if (!shouldAttemptCodification(codificationProfile, input)) { return { status: "miss", reason: `codification profile is ${codificationProfile.mode}: ${codificationProfile.reason}`, @@ -288,6 +377,7 @@ export async function tryRefreshRowExtractor( expectedDatasetId: input.datasetId, data: extraction.data, sources: extraction.sources, + cellSources: extraction.cellSources, rowSummary: extraction.rowSummary, howFound: GENERIC_REFRESH_HOW_FOUND, }); @@ -305,12 +395,7 @@ export async function tryRefreshRowExtractor( } function initialBrowserUrl(input: TryRowExtractorInput): string | undefined { - const explicitUrl = [ - ...Object.values(input.primaryKeys), - ...(input.urls ?? []), - input.sourceHint, - input.context?.match(/https?:\/\/[^\s)>"']+/i)?.[0], - ] + const explicitUrl = browserStartUrlCandidates(input) .map(coerceHttpUrl) .find((value): value is string => Boolean(value)); if (explicitUrl) return explicitUrl; @@ -330,6 +415,78 @@ function initialBrowserUrl(input: TryRowExtractorInput): string | undefined { return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}`; } +function browserStartUrlCandidates(input: TryRowExtractorInput): Array { + return uniqueCandidateValues([ + ...Object.values(input.primaryKeys), + ...(input.urls ?? []), + ...renderCodificationTemplateUrls(input), + ...extractHttpUrls(input.sourceHint), + ...extractHttpUrls(input.context), + input.sourceHint, + ]); +} + +function renderCodificationTemplateUrls(input: TryRowExtractorInput): string[] { + const urls: string[] = []; + for (const family of input.codificationProfile?.families ?? []) { + if (!family.urlTemplate) continue; + const rendered = renderUrlTemplate(family.urlTemplate, input.primaryKeys); + if (rendered) urls.push(rendered); + } + return urls; +} + +function renderUrlTemplate( + template: string, + primaryKeys: Record, +): string | undefined { + let missing = false; + const rendered = template.replace( + /\{([a-zA-Z0-9_]+)\}/g, + (match: string, key: string, offset: number) => { + const value = findPrimaryKeyValue(key, primaryKeys)?.trim(); + if (!value) { + missing = true; + return ""; + } + return encodeTemplateValue(template, offset, value); + }, + ); + return missing ? undefined : rendered; +} + +function encodeTemplateValue(template: string, placeholderOffset: number, value: string): string { + if (placeholderIsInQueryOrHash(template, placeholderOffset)) { + return encodeURIComponent(value); + } + return value.split("/").map(encodeURIComponent).join("/"); +} + +function placeholderIsInQueryOrHash(template: string, placeholderOffset: number): boolean { + const queryIndex = template.indexOf("?"); + const hashIndex = template.indexOf("#"); + const boundaryIndexes = [queryIndex, hashIndex].filter((index) => index >= 0); + return boundaryIndexes.length > 0 && Math.min(...boundaryIndexes) < placeholderOffset; +} + +function extractHttpUrls(value: string | undefined): string[] { + if (!value) return []; + return [...value.matchAll(/https?:\/\/[^\s)>"']+/gi)].map((match) => normalizeUrl(match[0])); +} + +function uniqueCandidateValues(values: Array): Array { + const seen = new Set(); + const output: Array = []; + for (const value of values) { + const key = value?.trim(); + if (!key) continue; + if (seen.has(key)) continue; + seen.add(key); + output.push(value); + } + return output; +} + function normalizeUrl(value: string): string { return value.trim().replace(/[.,;:]+$/, ""); } @@ -371,13 +528,19 @@ async function extractGenericRow( const apiKey = await getTinyFishApiKey(); if (!apiKey) throw new Error("TINYFISH_API_KEY is not configured"); + const siteKey = siteKeyForInput(input, url); const script = await getOrBuildExtractorScript(apiKey, input, url); const attempts = normalizedBrowserAttempts(input.browserAttempts); let lastError: unknown; for (let attempt = 1; attempt <= attempts; attempt++) { try { + console.log( + `[row_extractor] running generated Playwright extractor for ${siteKey} attempt=${attempt}/${attempts}`, + ); const raw = await runGeneratedExtractorScript(apiKey, input, url, script); const extraction = buildRowFromExtractorResult(input, url, raw, existingData); + const qualityIssue = extractorQualityIssue(extraction, input.columns); + if (qualityIssue) throw new Error(qualityIssue); rememberExtractorSmokeTest(input, url); return extraction; } catch (err) { @@ -402,8 +565,11 @@ async function extractGenericRow( script, failure, ); + console.log(`[row_extractor] running repaired Playwright extractor for ${siteKey}`); const raw = await runGeneratedExtractorScript(apiKey, input, url, repaired); const extraction = buildRowFromExtractorResult(input, url, raw, existingData); + const qualityIssue = extractorQualityIssue(extraction, input.columns); + if (qualityIssue) throw new Error(qualityIssue); rememberExtractorSmokeTest(input, url); return extraction; } catch (repairErr) { @@ -435,7 +601,10 @@ async function getOrBuildExtractorScript( siteKey, columnsHash, })) as { script?: string } | null; - if (existing?.script) return existing.script; + if (existing?.script) { + console.log(`[row_extractor] using cached generated Playwright extractor for ${siteKey}`); + return existing.script; + } const inFlightBuild = inFlightExtractorBuilds.get(buildKey); if (inFlightBuild) { @@ -696,19 +865,57 @@ async function withRunTimeoutSignal( if (runSignal?.aborted) throw new DOMException("Run was stopped", "AbortError"); const controller = new AbortController(); - const timeout = setTimeout( - () => controller.abort(new DOMException("Timed out", "TimeoutError")), - timeoutMs, - ); - const abortFromRun = () => - controller.abort(runSignal?.reason ?? new DOMException("Run was stopped", "AbortError")); + let timeout: ReturnType; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + const reason = new DOMException( + `Timed out after ${Math.round(timeoutMs / 1000)}s`, + "TimeoutError", + ); + controller.abort(reason); + reject(reason); + }, timeoutMs); + }); + let abortFromRun: (() => void) | undefined; + const abortPromise = new Promise((_resolve, reject) => { + abortFromRun = () => { + const reason = runSignal?.reason ?? new DOMException("Run was stopped", "AbortError"); + controller.abort(reason); + reject(reason); + }; + + runSignal?.addEventListener("abort", abortFromRun, { once: true }); + }); + try { + return await Promise.race([operation(controller.signal), timeoutPromise, abortPromise]); + } finally { + clearTimeout(timeout!); + if (abortFromRun) runSignal?.removeEventListener("abort", abortFromRun); + } +} + +async function withRunAbortSignal( + datasetId: string, + operation: (signal: AbortSignal) => Promise, +): Promise { + const runSignal = getSignal(datasetId); + if (runSignal?.aborted) throw new DOMException("Run was stopped", "AbortError"); + + const controller = new AbortController(); + let abortFromRun: (() => void) | undefined; + const abortPromise = new Promise((_resolve, reject) => { + abortFromRun = () => { + const reason = runSignal?.reason ?? new DOMException("Run was stopped", "AbortError"); + controller.abort(reason); + reject(reason); + }; + runSignal?.addEventListener("abort", abortFromRun, { once: true }); + }); - runSignal?.addEventListener("abort", abortFromRun, { once: true }); try { - return await operation(controller.signal); + return await Promise.race([operation(controller.signal), abortPromise]); } finally { - clearTimeout(timeout); - runSignal?.removeEventListener("abort", abortFromRun); + if (abortFromRun) runSignal?.removeEventListener("abort", abortFromRun); } } @@ -838,12 +1045,24 @@ async function buildExtractorScriptWithAgent( previousScript: string; failure: string; }, + options: { + maxSteps?: number; + phase?: string; + } = {}, ): Promise { const llmConfig = await requireLlmProviderConfig(); let acceptedScript: string | undefined; + let acceptedFromTest = false; let declinedReason: string | undefined; let lastEvidence: PageEvidence | undefined; let lastProbeSummary = `url=${url}`; + const siteKey = siteKeyForInput(input, url); + const phase = options.phase ?? (repairContext ? "repair" : "build"); + const maxSteps = options.maxSteps ?? EXTRACTOR_BUILDER_MAX_STEPS; + + console.log( + `[extractor_builder] ${phase} started for ${siteKey} maxSteps=${maxSteps}`, + ); const inspectBrowserPageTool = createTool({ id: "inspect_browser_page", @@ -871,6 +1090,7 @@ async function buildExtractorScriptWithAgent( execute: async ({ url: requestedUrl }) => { try { const targetUrl = coerceHttpUrl(requestedUrl) ?? url; + console.log(`[extractor_builder] ${phase} inspecting ${targetUrl}`); const evidence = await probePage(apiKey, input.datasetId, targetUrl); lastEvidence = evidence; lastProbeSummary = summarizeEvidenceForStorage(evidence); @@ -897,12 +1117,22 @@ async function buildExtractorScriptWithAgent( reason: z.string(), extractedColumns: z.array(z.string()).optional(), missingColumns: z.array(z.string()).optional(), + requiredMissingColumns: z.array(z.string()).optional(), + optionalMissingColumns: z.array(z.string()).optional(), rowSummary: z.string().optional(), sources: z.array(z.string()).optional(), dataPreview: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), }), execute: async ({ script }) => { + console.log(`[extractor_builder] ${phase} testing candidate for ${siteKey}`); const result = await testExtractorScriptDetailed(apiKey, input, url, script); + if (result.ok) { + acceptedScript = sanitizeGeneratedScript(script); + acceptedFromTest = true; + } + console.log( + `[extractor_builder] ${phase} test ${result.ok ? "passed" : "failed"} for ${siteKey}: ${reasonForLog(result.reason)}`, + ); return detailedTestResultForTool(result); }, }); @@ -920,20 +1150,28 @@ async function buildExtractorScriptWithAgent( reason: z.string(), extractedColumns: z.array(z.string()).optional(), missingColumns: z.array(z.string()).optional(), + requiredMissingColumns: z.array(z.string()).optional(), + optionalMissingColumns: z.array(z.string()).optional(), }), execute: async ({ script }) => { + console.log(`[extractor_builder] ${phase} finishing candidate for ${siteKey}`); const result = await testExtractorScriptDetailed(apiKey, input, url, script); if (result.ok) { acceptedScript = sanitizeGeneratedScript(script); } - return { - ok: result.ok, - reason: result.reason, - extractedColumns: result.extraction?.extractedColumns, - missingColumns: result.extraction?.missingColumns, - }; - }, - }); + console.log( + `[extractor_builder] ${phase} finish ${result.ok ? "accepted" : "rejected"} for ${siteKey}: ${reasonForLog(result.reason)}`, + ); + return { + ok: result.ok, + reason: result.reason, + extractedColumns: result.extraction?.extractedColumns, + missingColumns: result.extraction?.missingColumns, + requiredMissingColumns: result.extraction?.requiredMissingColumns, + optionalMissingColumns: result.extraction?.optionalMissingColumns, + }; + }, +}); const declineExtractorTool = createTool({ id: "decline_extractor", @@ -976,17 +1214,37 @@ async function buildExtractorScriptWithAgent( }); const prompt = extractorBuilderAgentPrompt(input, url, repairContext); - const result = await agent.generate(prompt, { - abortSignal: getSignal(input.datasetId), - maxSteps: EXTRACTOR_BUILDER_MAX_STEPS, - modelSettings: { - maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.EXTRACTOR_BUILDER, - }, - }); + let result: Awaited>; + try { + result = await withRunAbortSignal(input.datasetId, (abortSignal) => + agent.generate(prompt, { + abortSignal, + maxSteps, + modelSettings: { + maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS.EXTRACTOR_BUILDER, + }, + }), + ); + } catch (err) { + if (acceptedScript) { + console.warn( + `[extractor_builder] ${phase} returning ${acceptedFromTest ? "tested" : "finished"} accepted script for ${siteKey} after agent error: ${reasonForLog(err instanceof Error ? err.message : String(err))}`, + ); + return { + script: acceptedScript, + probeSummary: lastEvidence ? summarizeEvidenceForStorage(lastEvidence) : lastProbeSummary, + }; + } + throw err; + } + + console.log( + `[extractor_builder] ${phase} ended for ${siteKey} steps=${result.steps?.length ?? "?"}`, + ); if (acceptedScript) { console.log( - `[extractor_builder] accepted script after ${result.steps?.length ?? "?"} steps for ${siteKeyForInput(input, url)}`, + `[extractor_builder] accepted script after ${result.steps?.length ?? "?"} steps for ${siteKey}`, ); return { script: acceptedScript, @@ -999,7 +1257,7 @@ async function buildExtractorScriptWithAgent( } throw new Error( - `extractor builder did not finish a validated script within ${EXTRACTOR_BUILDER_MAX_STEPS} steps`, + `extractor builder did not finish a validated script within ${maxSteps} steps`, ); } @@ -1020,6 +1278,10 @@ async function repairExtractorAfterRuntimeFailure( previousScript: script, failure, }, + { + maxSteps: EXTRACTOR_REPAIR_MAX_STEPS, + phase: "repair", + }, ); const validation = await validateExtractorScript(apiKey, input, url, repaired.script); if (!validation.ok) { @@ -1100,6 +1362,22 @@ async function testExtractorScriptDetailed( extraction, }; } + const qualityIssue = extractorQualityIssue(extraction, input.columns); + if (qualityIssue) { + return { + ok: false, + reason: qualityIssue, + extraction, + }; + } + const hardcodedValues = hardcodedExtractorValues(sanitized, extraction, input, url); + if (hardcodedValues.length > 0) { + return { + ok: false, + reason: `extractor appears to hardcode representative-row values: ${hardcodedValues.join(", ")}`, + extraction, + }; + } return { ok: true, reason: "passed", extraction }; } catch (err) { return { @@ -1114,6 +1392,8 @@ function detailedTestResultForTool(result: DetailedExtractorTestResult): { reason: string; extractedColumns?: string[]; missingColumns?: string[]; + requiredMissingColumns?: string[]; + optionalMissingColumns?: string[]; rowSummary?: string; sources?: string[]; dataPreview?: Record; @@ -1123,12 +1403,19 @@ function detailedTestResultForTool(result: DetailedExtractorTestResult): { reason: result.reason, extractedColumns: result.extraction?.extractedColumns, missingColumns: result.extraction?.missingColumns, + requiredMissingColumns: result.extraction?.requiredMissingColumns, + optionalMissingColumns: result.extraction?.optionalMissingColumns, rowSummary: result.extraction?.rowSummary, sources: result.extraction?.sources, dataPreview: result.extraction?.data, }; } +function reasonForLog(reason: string): string { + const normalized = reason.replace(/\s+/g, " ").trim(); + return normalized.length > 500 ? `${normalized.slice(0, 500)}...` : normalized; +} + function extractorBuilderAgentInstructions(): string { return `You are BigSet's autonomous extractor builder agent. @@ -1136,22 +1423,27 @@ Your job is to build a reusable Playwright extractor script for one dataset/page Critical behavior: - Inspect pages when structure, source URLs, or primary key semantics are unclear. +- Do not decline because a source has an anti-bot reputation. BigSet uses TinyFish Browser for interactive browser access; inspect the representative page and judge the actual session result. - Write and test candidate scripts with test_extractor. Iterate on validation errors. - Finish only by calling finish_extractor with the final script. A final text response without finish_extractor is failure. - If the dataset is not a good fit for reusable browser codification, call decline_extractor with a concrete reason. +- Treat this as a general page-family compiler. Do not optimize for any named site. The page family may be a marketplace listing, social group, product page, blog post archive, repository page, directory profile, forum thread, app-store listing, or any other accessible browser surface. When to decline: - The representative primary keys/URLs point at arbitrary unrelated domains with no shared page pattern. -- The page is blocked by login, paywall, CAPTCHA, bot detection, or inaccessible content. +- The representative page remains blocked by login, paywall, CAPTCHA, bot detection, or inaccessible content after a TinyFish Browser inspection attempt. - The row context is too ambiguous to construct or find a stable browser target. -- You cannot produce a complete row with every dataset column populated. +- TinyFish Browser can access the page but the requested values require actions/data outside the accessible page family and cannot be reliably derived. Important modeling rules: - Primary keys are arbitrary row identifiers. They do not have to be URLs. - You may construct navigation URLs from input.primaryKeys, top-level primary key fields, input.urls, input.sourceHint, dataset name, description, retrieval strategy, and context. -- Example: a primary key like "tinyfish-io/bigset" may require page.goto("https://github.com/" + input.primaryKeys.repo_slug). Do the equivalent for any site family you infer. -- Multiple known page families are allowed. For example, if rows consistently contain either Yelp or Google Maps URLs, write one reusable script that branches based on the available URL/domain. If rows are arbitrary unrelated URLs, decline. +- If the source/schema imply a URL template, build the row URL from the primary key or identifier. This applies to any source family: handles, slugs, product IDs, post paths, item IDs, listing IDs, package names, tickers, and similar identifiers. +- Search is a last resort when a deterministic source URL can be built. +- Multiple known page families are allowed. If rows consistently contain URLs from a small set of stable domains or path patterns, write one reusable script that branches based on the available URL/domain. If rows are arbitrary unrelated URLs, decline. - Respect validation_regex exactly. Normalize values before returning them, using normalization_hint and column descriptions. +- Attempt every column, including nullable/optional columns. Nullable means the final row may survive if all methods fail, not that you should skip the field. +- You may derive values from page structure instead of finding literal labels. For example: count repeated cards in a relevant section, infer booleans from the presence of controls/badges, parse counts from aria labels, or normalize visible variants into the schema's final format. Script contract: - Return only a full JavaScript script in tool arguments, never Markdown fences. @@ -1161,11 +1453,35 @@ Script contract: - Use only the provided Playwright page, input, and helpers. - You may use page.goto, locators, evaluate, textContent, getAttribute, and other Playwright page APIs. - Prefer stable DOM sources: JSON-LD, meta tags, tables, definition lists, visible labels, aria labels, canonical links, and semantically named selectors. -- Return { data, sources, row_summary, how_found }. -- data must include every dataset column. Preserve primary key values from input.primaryKeys. -- Every dataset column must have a real non-empty value after normalization. Nullable columns are still required for codified browser extraction. +- Return { data, column_status, cell_sources, sources, row_summary, how_found }. +- data should include every dataset column that Playwright can extract or derive. Preserve primary key values from input.primaryKeys. +- column_status should include every dataset column. Use status "extracted" or "derived" for values in data. Use "not_present_on_page", "blocked", "ambiguous", "validation_failed", or "fallback_needed" for unresolved values, with a short reason. +- cell_sources should map each extracted/derived column to the exact HTTP URL(s) that justify that specific value. Do not put row-level sources here. - Returned sources must be HTTP URLs you actually used. -- If any column cannot be extracted or normalized to satisfy validation_regex, keep investigating with the browser tools or decline. Do not finish a partial extractor.`; +- If any column cannot be extracted or normalized to satisfy validation_regex, keep investigating with the browser tools. If it still cannot be solved from the browser page family, mark it in column_status for fallback instead of fabricating a value. + +Starter-code reference only. This shape will not work on the target site until you inspect the actual page and replace the URL construction/selectors: +async function extract({ page, input, helpers }) { + const id = String(input.primaryKeys.item_id ?? input.item_id ?? "").trim(); + const startUrl = input.urls[0] || input.sourceHint || input.startUrl; + await page.goto(startUrl, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 10000 }).catch(() => {}); + const read = async (locator) => helpers.cleanText(await locator.first().textContent().catch(() => "")); + const title = await read(page.locator("h1")); + return { + data: { item_id: id, title }, + column_status: { + item_id: { status: "extracted" }, + title: title ? { status: "extracted" } : { status: "fallback_needed", reason: "No h1/title equivalent found after inspection" } + }, + cell_sources: title ? { title: [page.url()] } : {}, + sources: [page.url()], + row_summary: title || id, + how_found: "Reference shape only; actual extractor must describe the inspected selectors and derived values." + }; +} + +Never hardcode representative-row values in the script. Literal values from the first tested row, such as a specific product name, profile URL, rating, count, or homepage, make the extractor invalid unless they come from input primary keys or are source-family constants like a host prefix.`; } function extractorBuilderAgentPrompt( @@ -1376,7 +1692,12 @@ function buildRowFromExtractorResult( const data: Record = {}; const extractedColumns: string[] = []; const missingColumns: string[] = []; - const invalidColumns: string[] = []; + const requiredMissingColumns: string[] = []; + const optionalMissingColumns: string[] = []; + const rawColumnStatuses = normalizeColumnStatuses( + result.column_status ?? result.columnStatus, + ); + const columnStatuses: Record = {}; for (const column of input.columns) { const pkValue = findPrimaryKeyValue(column.name, input.primaryKeys); @@ -1388,6 +1709,10 @@ function buildRowFromExtractorResult( } data[column.name] = value; extractedColumns.push(column.name); + columnStatuses[column.name] = { + status: rawColumnStatuses[column.name]?.status ?? "extracted", + reason: rawColumnStatuses[column.name]?.reason, + }; continue; } @@ -1398,9 +1723,16 @@ function buildRowFromExtractorResult( if (!validationError) { data[column.name] = value; extractedColumns.push(column.name); + columnStatuses[column.name] = { + status: rawColumnStatuses[column.name]?.status ?? "extracted", + reason: rawColumnStatuses[column.name]?.reason, + }; continue; } - invalidColumns.push(`${column.name}: ${validationError}`); + columnStatuses[column.name] = { + status: "validation_failed", + reason: validationError, + }; } if (existingData && existingData[column.name] !== undefined) { @@ -1409,39 +1741,44 @@ function buildRowFromExtractorResult( const validationError = validateColumnValue(storedValue, column); if (!validationError) { data[column.name] = storedValue; + columnStatuses[column.name] = { + status: rawColumnStatuses[column.name]?.status ?? "extracted", + reason: rawColumnStatuses[column.name]?.reason ?? "Preserved existing validated value.", + }; continue; } - invalidColumns.push(`${column.name}: stored value ${validationError}`); } } data[column.name] = ""; missingColumns.push(column.name); - } - - if (invalidColumns.length > 0) { - throw new Error( - `generated extractor returned values that failed validation: ${invalidColumns.join("; ")}`, - ); - } - - if (missingColumns.length > 0) { - throw new Error( - `generated extractor missed columns: ${missingColumns.join(", ")}`, - ); + if (isRequiredColumn(column)) { + requiredMissingColumns.push(column.name); + } else { + optionalMissingColumns.push(column.name); + } + columnStatuses[column.name] = columnStatuses[column.name] ?? rawColumnStatuses[column.name] ?? { + status: "fallback_needed", + reason: "The browser extractor did not return a validated value.", + }; } const sources = Array.isArray(result.sources) ? result.sources.filter((source): source is string => typeof source === "string" && isHttpUrl(source)) : []; + const cellSources = normalizeCellSources(result.cell_sources ?? result.cellSources); const summary = String(result.row_summary ?? result.rowSummary ?? "").trim().slice(0, 500); return { data, sources: sources.length > 0 ? sources : [url], rowSummary: summary || url, + cellSources, extractedColumns, missingColumns, + requiredMissingColumns, + optionalMissingColumns, + columnStatuses, }; } @@ -1455,6 +1792,167 @@ function hasExtractedNonPrimaryValue( return extraction.extractedColumns.some((columnName) => !pkColumns.has(columnName)); } +function extractorQualityIssue( + extraction: GenericExtraction, + columns: PopulateColumn[], +): string | undefined { + const nonPrimaryColumns = columns.filter((column) => !column.isPrimaryKey); + if (nonPrimaryColumns.length === 0) return undefined; + + const extracted = new Set(extraction.extractedColumns); + const missing = new Set(extraction.missingColumns); + const extractedNonPrimary = nonPrimaryColumns.filter((column) => extracted.has(column.name)); + const missingCellSources = extractedNonPrimary.filter( + (column) => (extraction.cellSources[column.name]?.length ?? 0) === 0, + ); + if (missingCellSources.length > 0) { + return `generated extractor omitted cell_sources for extracted columns: ${missingCellSources.map((column) => column.name).join(", ")}`; + } + + const missingRequired = nonPrimaryColumns.filter( + (column) => isRequiredColumn(column) && missing.has(column.name), + ); + if (missingRequired.length > 0) { + return `generated extractor missed required columns: ${missingRequired.map((column) => column.name).join(", ")}`; + } + + const minimum = minimumBrowserExtractedColumnCount(nonPrimaryColumns.length); + if (extractedNonPrimary.length < minimum) { + const missingColumns = nonPrimaryColumns + .filter((column) => !extracted.has(column.name)) + .map((column) => column.name); + return [ + `generated extractor coverage too low: extracted ${extractedNonPrimary.length}/${nonPrimaryColumns.length} non-primary columns`, + `minimum=${minimum}`, + `missing=${missingColumns.join(", ") || "(none)"}`, + ].join("; "); + } + + return undefined; +} + +function minimumBrowserExtractedColumnCount(nonPrimaryColumnCount: number): number { + if (nonPrimaryColumnCount <= 1) return nonPrimaryColumnCount; + return Math.floor(nonPrimaryColumnCount / 2) + 1; +} + +function hardcodedExtractorValues( + script: string, + extraction: GenericExtraction, + input: TryRowExtractorInput, + url: string, +): string[] { + const scriptText = script.toLowerCase(); + const pkColumns = new Set( + input.columns.filter((column) => column.isPrimaryKey).map((column) => column.name), + ); + const sourceUrls = new Set( + [url, ...(input.urls ?? []), input.sourceHint] + .filter((value): value is string => Boolean(value)) + .map((value) => value.toLowerCase()), + ); + const violations: string[] = []; + + for (const [columnName, value] of Object.entries(extraction.data)) { + if (pkColumns.has(columnName)) continue; + if (typeof value !== "string") continue; + const normalized = value.trim(); + if (normalized.length < 8) continue; + const lowerValue = normalized.toLowerCase(); + if (sourceUrls.has(lowerValue)) continue; + if (!scriptText.includes(lowerValue)) continue; + violations.push(`${columnName}=${JSON.stringify(normalized.slice(0, 120))}`); + } + + return violations.slice(0, 8); +} + +function isRequiredColumn(column: PopulateColumn): boolean { + return column.isPrimaryKey === true || column.nullable === false; +} + +function normalizeColumnStatuses(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + + const statuses: Record = {}; + for (const [column, rawStatus] of Object.entries(value as Record)) { + const normalized = + typeof rawStatus === "string" + ? { status: normalizeColumnStatus(rawStatus) } + : rawStatus && typeof rawStatus === "object" + ? { + status: normalizeColumnStatus( + String((rawStatus as { status?: unknown }).status ?? "fallback_needed"), + ), + reason: + typeof (rawStatus as { reason?: unknown }).reason === "string" + ? String((rawStatus as { reason?: unknown }).reason).slice(0, 300) + : undefined, + } + : { status: "fallback_needed" as const }; + statuses[column] = normalized; + } + return statuses; +} + +function normalizeColumnStatus(value: string): ColumnExtractionStatus { + const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_"); + switch (normalized) { + case "extracted": + case "derived": + case "not_present_on_page": + case "blocked": + case "ambiguous": + case "validation_failed": + case "fallback_needed": + case "missing": + return normalized; + case "not_found": + case "unavailable": + case "not_visible": + return "not_present_on_page"; + default: + return "fallback_needed"; + } +} + +function normalizeCellSources(value: unknown): Record { + if (!value) return {}; + + const output: Record = {}; + const add = (column: string, sources: unknown) => { + const normalizedColumn = column.trim().replace(/^["`]+|["`]+$/g, ""); + if (!normalizedColumn) return; + const sourceValues = Array.isArray(sources) ? sources : [sources]; + const cleaned = [ + ...new Set( + sourceValues.filter((source): source is string => { + return typeof source === "string" && isHttpUrl(source); + }), + ), + ]; + if (cleaned.length > 0) output[normalizedColumn] = cleaned; + }; + + if (Array.isArray(value)) { + for (const entry of value) { + if (!entry || typeof entry !== "object") continue; + const column = (entry as { column?: unknown }).column; + if (typeof column !== "string") continue; + add(column, (entry as { sources?: unknown; source?: unknown }).sources ?? (entry as { source?: unknown }).source); + } + return output; + } + + if (typeof value === "object") { + for (const [column, sources] of Object.entries(value as Record)) { + add(column, sources); + } + } + + return output; +} + function rawValueToExtractedValue(value: unknown): ExtractedValue | undefined { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "boolean") return value; @@ -1508,11 +2006,7 @@ function siteKeyForUrl(value: string): string { } function siteKeyForInput(input: TryRowExtractorInput, browserStartUrl: string): string { - const sourceUrl = [ - input.sourceHint, - ...(input.urls ?? []), - ...Object.values(input.primaryKeys), - ] + const sourceUrl = browserStartUrlCandidates(input) .map(coerceHttpUrl) .find((value): value is string => Boolean(value)); return siteKeyForUrl(sourceUrl ?? browserStartUrl); diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3506ef9..ecd496d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -48,6 +48,7 @@ services: # When unset, backend analytics module no-ops. POSTHOG_KEY: ${NEXT_PUBLIC_POSTHOG_KEY:-} POSTHOG_HOST: ${NEXT_PUBLIC_POSTHOG_HOST:-https://us.i.posthog.com} + POPULATE_ORCHESTRATOR_MAX_STEPS: ${POPULATE_ORCHESTRATOR_MAX_STEPS:-80} REFRESH_SCHEDULER_ENABLED: ${REFRESH_SCHEDULER_ENABLED:-true} REFRESH_SCHEDULER_POLL_MS: ${REFRESH_SCHEDULER_POLL_MS:-60000} REFRESH_SCHEDULER_BATCH_SIZE: ${REFRESH_SCHEDULER_BATCH_SIZE:-5} diff --git a/frontend/app/dataset/[id]/page.tsx b/frontend/app/dataset/[id]/page.tsx index 697addd..37ca9c9 100644 --- a/frontend/app/dataset/[id]/page.tsx +++ b/frontend/app/dataset/[id]/page.tsx @@ -42,7 +42,8 @@ export default function DatasetPage() { const [cellDetail, setCellDetail] = useState<{ column: DatasetColumn; value: unknown; - sources?: string[]; + cellSources?: string[]; + rowSources?: string[]; } | null>(null); const datasetId = params.id as Id<"datasets">; @@ -103,7 +104,12 @@ export default function DatasetPage() { const col = dataset.columns.find((c) => c.name === columnName); if (!col) return; const row = rows.find((r) => r._id === rowId); - setCellDetail({ column: col, value, sources: row?.sources }); + setCellDetail({ + column: col, + value, + cellSources: row?.cellSources?.[columnName], + rowSources: row?.sources, + }); }, [dataset, rows]); const openedFired = useRef(null); @@ -461,7 +467,8 @@ export default function DatasetPage() { )} diff --git a/frontend/app/dataset/new/page.tsx b/frontend/app/dataset/new/page.tsx index 9a48f6f..77c5646 100644 --- a/frontend/app/dataset/new/page.tsx +++ b/frontend/app/dataset/new/page.tsx @@ -56,7 +56,7 @@ const DEFAULT_MAX_ROW_COUNT = 100; function mapBackendColumn(col: InferredColumn, index: number): ProposedColumn { return { id: String(index + 1), - name: col.display_name, + name: col.name, type: BACKEND_TYPE_MAP[col.type], description: col.retrieval_hint, isPrimaryKey: col.is_primary_key, diff --git a/frontend/components/SideSheet.tsx b/frontend/components/SideSheet.tsx index 215d477..09bb917 100644 --- a/frontend/components/SideSheet.tsx +++ b/frontend/components/SideSheet.tsx @@ -117,8 +117,8 @@ export function SideSheet({ open, onClose, children }: SideSheetProps) { interface CellDetailProps { column: DatasetColumn; value: unknown; - /** Row-level sources stored by the populate agent. */ - sources?: string[]; + cellSources?: string[]; + rowSources?: string[]; } function isValidHttpUrl(src: string): boolean { @@ -130,7 +130,7 @@ function isValidHttpUrl(src: string): boolean { } } -export function CellDetail({ column, value, sources }: CellDetailProps) { +export function CellDetail({ column, value, cellSources, rowSources }: CellDetailProps) { const [copied, setCopied] = useState(false); const copyTimerRef = useRef | null>(null); const displayValue = value == null || value === "" ? "—" : String(value); @@ -192,14 +192,49 @@ export function CellDetail({ column, value, sources }: CellDetailProps) {
- {/* Sources */} - {sources && sources.length > 0 && ( + {/* Cell sources */} + {cellSources && cellSources.length > 0 && (

Sources

    - {sources.map((src, i) => ( + {cellSources.map((src, i) => ( +
  • + {isValidHttpUrl(src) ? ( + + + {src} + + ) : ( + + + {src} + + )} +
    • + ))} +
+
+ )} + + {/* Row sources */} + {rowSources && rowSources.length > 0 && ( +
+

+ Row Sources +

+

+ These URLs were recorded for the row, not as proof for this specific cell. +

+