From 7510b06386cf37faf84ca3444a017559ff9fe81b Mon Sep 17 00:00:00 2001 From: siddarth Date: Wed, 3 Jun 2026 22:23:18 +0530 Subject: [PATCH 01/10] fix: strip carriage returns from .env values in Makefile On Windows, .env files use CRLF line endings. cut -d= -f2- includes the trailing \r in the extracted value, so CLERK_JWT_ISSUER_DOMAIN gets stored in Convex as https://example.clerk.accounts.dev\r which silently breaks authentication with a cryptic JWT error. Fix: pipe through tr -d '\r' in ensure-admin-key, convex-env, and convex-push targets. --- makefiles/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/makefiles/Makefile b/makefiles/Makefile index 81c3d6b..b8a6cbb 100644 --- a/makefiles/Makefile +++ b/makefiles/Makefile @@ -84,7 +84,7 @@ validate-dev-env: fi ensure-admin-key: - @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2-)"; \ + @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ needs_gen=false; \ if [[ -z "$$admin_key" ]]; then \ needs_gen=true; \ @@ -115,8 +115,8 @@ ensure-admin-key: convex-env: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } - @issuer="$$(grep '^CLERK_JWT_ISSUER_DOMAIN=' .env | cut -d= -f2-)"; \ - admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2-)"; \ + @issuer="$$(grep '^CLERK_JWT_ISSUER_DOMAIN=' .env | cut -d= -f2- | tr -d '\r')"; \ + admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ if [[ -z "$$issuer" || "$$issuer" == "https://your-app.clerk.accounts.dev" ]]; then \ echo "Error: CLERK_JWT_ISSUER_DOMAIN must be your Clerk issuer URL in root .env"; \ exit 1; \ @@ -132,7 +132,7 @@ convex-env: convex-push: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } - @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2-)"; \ + @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ if [[ -z "$$admin_key" ]]; then \ echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY is missing in root .env"; \ echo "Run make dev to generate it automatically."; \ From 80b6db4bfc45fc1d3ffa9a03c926b7f68c18274f Mon Sep 17 00:00:00 2001 From: Mengzhe Li <129459076+MMeteorL@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:43:42 -0700 Subject: [PATCH 02/10] Add stop population/update feature (#107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add stop population/update feature Adds a Stop button in Settings that cancels in-flight populate or update runs, preserving all collected rows and transitioning the dataset to 'live'. Implementation: - abort-registry.ts: module-level Map shared across the process - /stop route: looks up the active runId for a dataset, fires AbortController.abort() - AbortSignal threaded via abortSignal parameter to every agent.generate() call (orchestrator, subagent, refresh agents) - run_subagent tool re-throws AbortError so cancellation propagates up to the orchestrator - Update workflow: processRow re-throws AbortError so workers exit early; a post-concurrency check bulk-clears any remaining pending row updateStatus fields - Background runners detect controller.signal.aborted in catch, set status → 'live', send ready notification with collected row count - Convex: new clearAllPendingUpdateStatus internal mutation to bulk-reset pending shimmers on stop - runStats still saved via existing finally block (recorded as stopped with errorMsg "Stopped by user") Co-Authored-By: Claude Sonnet 4.6 * Simplify abort registry and remove dead code Three refactors to the stop-population implementation: 1. Key abort-registry by datasetId instead of workflowRunId Convex's atomic claim already guarantees at most one active run per dataset, so datasetId is a valid unique key. This eliminates the activeDatasets Map that existed only to bridge datasetId → runId → AbortController, and simplifies all call sites (steps already have authorizedDatasetId/datasetId in scope). 2. Extract finaliseRunAsLive() helper Both background runners had identical ~12-line blocks to handle a user stop: query dataset, set status "live", count rows, conditionally email. Extracted into a shared helper. 3. Remove dead abort catch in runUpdateWorkflowInBackground When a stop fires during an update, processRow re-throws AbortError, Promise.allSettled winds down the workers, refreshRowsStep detects signal.aborted, clears pending rows, and returns normally. Mastra sees a successful step, run.start() returns {status:"success"}, and the existing success path handles the live transition. The controller.signal.aborted branch in the catch was unreachable. Co-Authored-By: Claude Sonnet 4.6 * Address CodeRabbit review and fix stuck-run scenarios Backend fixes: - finaliseRunAsLive failure: if the live-status mutation fails after a user stop, fall back to setting status "failed" so the dataset always leaves "building". Previously a failed finalisation returned without touching the status, leaving the dataset permanently stuck with no registry entry for /stop to act on. - investigate-tool maxSteps: fix 10 → 25 to match the contract in CLAUDE.md - enumerateStep abort: thread abortSignal into generateText so a stop pressed during enumeration (a ~10-token LLM call) cancels immediately rather than waiting for the call to complete. Re-throws AbortError so Mastra marks the step failed and the background runner's abort-detection fires. Frontend fixes: - stopping latch: don't clear stopping in handleStop's finally. Instead, a useEffect clears it when dataset.status transitions out of "building"/ "updating". This keeps the Stop button in "Stopping…"/disabled state until Convex confirms the run has actually finished, preventing the re-enable flash and duplicate stop requests during the cleanup window. Only clears immediately on fetch error (run was never reached). - backend.ts: extract backendPost helper — 4 functions were duplicating the same fetch/auth-header/error-parse boilerplate. - clearAllPendingUpdateStatus: add scale note about Convex per-mutation document limits. Co-Authored-By: Claude Sonnet 4.6 * Revert maxSteps change and backendPost refactor maxSteps in investigate-tool.ts: revert 25 → 10 to match main. The CodeRabbit comment cited CLAUDE.md describing the extract-tool-pipeline architecture where subagents use 25 steps — not applicable on main. backendPost helper: revert to individual fetch functions. The refactor adds a layer of indirection without making the stop feature safer or simpler. Code quality cleanups belong in a separate PR. Co-Authored-By: Claude Sonnet 4.6 * Fix clearAllPendingUpdateStatus full-dataset scan Add a compound index by_dataset_update_status (["datasetId", "updateStatus"]) to datasetRows so the mutation can query only the pending rows directly rather than scanning the entire dataset table. Before: collect() all rows for the dataset, filter in TS, patch pending. After: index query returns only pending rows; no other rows are scanned. Requires make convex-push to deploy the schema change. Co-Authored-By: Claude Sonnet 4.6 * Fix AbortError re-throw regression causing 0-row populate runs Tighten all three AbortError re-throw guards to check getSignal(datasetId)?.aborted before propagating. Without this, spurious network AbortErrors (undici ECONNRESET, dropped SSE streams, SDK-internal cleanup aborts) were re-thrown as if the user had pressed Stop. In investigate-tool.ts this caused the orchestrator's agent.generate() to receive a tool-thrown AbortError and exit early with a graceful empty result, producing a successful workflow run with 0 rows inserted. The fix restores the original structured-failure return path for all non-user aborts so the orchestrator can log the failure and continue with other entities. Co-Authored-By: Claude Sonnet 4.6 * Fix react-hooks/set-state-in-effect lint error on stopping latch Calling setStopping(false) synchronously inside a useEffect body triggers the react-hooks/set-state-in-effect rule, blocking the frontend lint gate. Remove the effect entirely. The stopping latch only needs to reset before the *next* populate or update run starts. Both handlePopulate and handleUpdate are already guarded against running while the dataset is busy, so they only fire after the status has settled to live/failed. Adding setStopping(false) at the top of each handler — right before setPopulating/setUpdating — is both correct and semantically clearer: "starting a new run discards any leftover stop-latch from the prior one." While the dataset is non-busy, the Stop button is hidden entirely (replaced by Update/Clear & Populate), so the stale stopping=true value is invisible and harmless until those handlers run. Co-Authored-By: Claude Sonnet 4.6 * Fix stop-button latch, orphaned-dataset recovery, and TOCTOU race - Frontend: hoist isDatasetBusy before the loading guard (optional chaining) so the useEffect dep array never hits the TDZ during the loading state. Add a useEffect + setTimeout to clear the `stopping` latch once Convex confirms the dataset has left the busy state, satisfying the react-hooks/set-state-in-effect lint rule. - Backend (/stop): detect orphaned datasets (busy in Convex but no abort registry entry) and force-transition them to "failed" so they are never permanently stuck after a server restart. - Backend (TOCTOU): move registerDataset() from inside the background async functions to the route handlers, before the void call. This closes the race window where a /stop arriving before the background function ran registerDataset would incorrectly trigger the orphan path against a live run. Co-Authored-By: Claude Sonnet 4.6 * Fix stop cleanup edge cases * Move stop action to dataset toolbar --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Simantak Dabhade Co-authored-by: Simantak Dabhade <67303107+simantak-dabhade@users.noreply.github.com> --- backend/src/abort-registry.ts | 53 +++++++ backend/src/index.ts | 158 +++++++++++++++++++ backend/src/mastra/tools/investigate-tool.ts | 10 +- backend/src/mastra/workflows/populate.ts | 17 +- backend/src/mastra/workflows/update.ts | 24 ++- frontend/app/dataset/[id]/page.tsx | 63 +++++++- frontend/convex/datasetRows.ts | 28 ++++ frontend/convex/schema.ts | 6 +- frontend/lib/analytics.ts | 1 + frontend/lib/backend.ts | 22 +++ 10 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 backend/src/abort-registry.ts diff --git a/backend/src/abort-registry.ts b/backend/src/abort-registry.ts new file mode 100644 index 0000000..3911ea3 --- /dev/null +++ b/backend/src/abort-registry.ts @@ -0,0 +1,53 @@ +/** + * Module-level registry mapping datasetId → AbortController. + * + * Allows the /stop HTTP route to cancel an in-flight populate or update + * workflow. The AbortSignal is retrieved inside Mastra workflow steps + * (which receive no signal parameter from the framework) via `getSignal()` + * and passed explicitly to each `agent.generate()` call. + * + * Keyed by datasetId because: + * - The /stop route knows the datasetId (from the request body). + * - Convex's atomic claim guarantees at most one active run per dataset, + * so datasetId uniquely identifies the in-flight run. + * - Workflow steps have authorizedDatasetId in their inputData — no + * separate workflowRunId lookup needed. + * + * Design notes: + * - `abortDataset()` fires the signal but does NOT remove the entry. + * The background runner's `finally` block calls `deregisterDataset()` + * so the catch block can still read `controller.signal.aborted` to + * distinguish a user stop from a genuine failure. + * - All operations are synchronous and safe within a single Node.js process. + */ + +const controllers = new Map(); + +/** Register an active run for a dataset and return its AbortController. */ +export function registerDataset(datasetId: string): AbortController { + const controller = new AbortController(); + controllers.set(datasetId, controller); + return controller; +} + +/** Retrieve the AbortSignal for a dataset's active run (undefined if not running). */ +export function getSignal(datasetId: string): AbortSignal | undefined { + return controllers.get(datasetId)?.signal; +} + +/** + * Fire the abort signal for a dataset's active run. + * Does NOT remove the entry — deregisterDataset() handles that. + * Returns true if a run was found and aborted; false if the dataset is idle. + */ +export function abortDataset(datasetId: string): boolean { + const controller = controllers.get(datasetId); + if (!controller) return false; + controller.abort(); + return true; +} + +/** Remove a dataset from the registry. Call in the background runner's finally block. */ +export function deregisterDataset(datasetId: string): void { + controllers.delete(datasetId); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 57c75e9..61c7314 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -13,6 +13,7 @@ import { sendTransactionalEmail } from "./email/send.js"; import { datasetReadyTemplate } from "./email/templates/dataset-ready.js"; import { capture, shutdown as shutdownAnalytics } from "./analytics/posthog.js"; import { EVENTS } from "./analytics/events.js"; +import { registerDataset, deregisterDataset, abortDataset } from "./abort-registry.js"; /** Domain part of an email, for analytics (we never log full addresses). */ function emailDomain(email: string): string { @@ -141,6 +142,45 @@ async function sendDatasetReadyNotification({ } } +/** + * Shared stop-success path: set the dataset live, send the ready email. + * + * Called by both background runners when the user presses Stop. Populate + * only emails when at least one row was collected (a stopped run with 0 + * rows is live but empty). Update always emails regardless of count. + */ +async function finaliseRunAsLive({ + logger, + clerk, + datasetId, + authorizedUserId, + workflowType = "populate", +}: { + logger: FastifyBaseLogger; + clerk: ClerkClient; + datasetId: string; + authorizedUserId: string; + workflowType?: "populate" | "update"; +}): Promise { + const currentDataset = await convex.query(internal.datasets.getInternal, { id: datasetId }); + if (!currentDataset) return; + + await setDatasetPopulateStatus(datasetId, "live"); + + const rowCount = await convex.query(internal.datasetRows.countByDataset, { datasetId }); + if (workflowType === "update" || rowCount > 0) { + await sendDatasetReadyNotification({ + logger, + clerk, + userId: authorizedUserId, + datasetId, + datasetName: currentDataset.name, + rowCount, + workflowType, + }); + } +} + async function beginDatasetUpdate( datasetId: string, ownerId: string, @@ -172,6 +212,9 @@ async function runUpdateWorkflowInBackground({ }; }): Promise { const datasetId = input.datasetId; + // registerDataset is called by the route handler before void-ing this + // function, so the registry entry is guaranteed visible the moment the + // 202 response is sent. No call needed here. try { const result = await run.start({ @@ -224,6 +267,11 @@ async function runUpdateWorkflowInBackground({ workflowType: "update", }); } catch (err) { + // Note: a user-triggered stop is NOT handled here. The update workflow's + // refreshRowsStep detects the abort internally, clears pending row + // statuses, and returns normally — so run.start() returns { status: + // "success" } and the success path above handles the live transition. + // This catch only fires on genuine failures. const lastStatusError = statusErrorMessage(err); logger.error({ err, datasetId }, "Update background workflow failed"); @@ -245,6 +293,8 @@ async function runUpdateWorkflowInBackground({ "Failed to transition dataset status to 'failed' after update", ); } + } finally { + deregisterDataset(datasetId); } } @@ -266,6 +316,7 @@ async function runScheduledUpdateWorkflowInBackground({ }; }): Promise { const datasetId = input.datasetId; + registerDataset(datasetId); try { const result = await run.start({ @@ -312,12 +363,15 @@ async function runScheduledUpdateWorkflowInBackground({ "Failed to record scheduled refresh failure", ); } + } finally { + deregisterDataset(datasetId); } } async function runPopulateWorkflowInBackground({ input, run, + controller, authorizedUserId, logger, clerk, @@ -325,6 +379,7 @@ async function runPopulateWorkflowInBackground({ }: { input: DatasetContext; run: PopulateWorkflowRun; + controller: AbortController; authorizedUserId: string; logger: FastifyBaseLogger; clerk: ClerkClient; @@ -389,6 +444,25 @@ async function runPopulateWorkflowInBackground({ rowCount, }); } catch (err) { + if (controller.signal.aborted) { + // User pressed Stop — treat whatever was collected as the final dataset. + logger.info({ datasetId }, "Populate workflow stopped by user; transitioning to live"); + try { + await finaliseRunAsLive({ logger, clerk, datasetId, authorizedUserId }); + } catch (stopErr) { + logger.error({ err: stopErr, datasetId }, "Failed to finalise stopped populate run; marking as failed"); + // Ensure the dataset always leaves "building" — without this fallback, + // a failed finalisation leaves the dataset with no active registry entry + // and no way for /stop to act on it again. + try { + await setDatasetPopulateStatus(datasetId, "failed", "Workflow stopped but could not be finalised"); + } catch (fallbackErr) { + logger.error({ err: fallbackErr, datasetId }, "Could not update dataset status after stop finalisation failure"); + } + } + return; + } + const lastStatusError = statusErrorMessage(err); logger.error( { err, datasetId }, @@ -414,6 +488,8 @@ async function runPopulateWorkflowInBackground({ "Failed to transition dataset status to 'failed'", ); } + } finally { + deregisterDataset(datasetId); } } @@ -704,9 +780,16 @@ await fastify.register(async (instance) => { return reply.code(502).send({ error: "Failed to populate dataset. Please try again." }); } + // Register before void-ing so the abort-registry entry is visible the + // instant the 202 is sent, closing the TOCTOU window where a /stop + // arriving before registerDataset runs inside the background function + // would incorrectly force-transition an active run to "failed". + const controller = registerDataset(parsed.data.datasetId); + void runPopulateWorkflowInBackground({ input: parsed.data, run, + controller, authorizedUserId: auth.userId, logger: req.log, clerk: req.server.clerk, @@ -772,6 +855,12 @@ await fastify.register(async (instance) => { const { getModelConfig } = await import("./config/models.js"); const modelConfig = await getModelConfig(auth.userId); + // Register before void-ing so the abort-registry entry is visible the + // instant the 202 is sent, closing the TOCTOU window where a /stop + // arriving before registerDataset runs inside the background function + // would incorrectly force-transition an active run to "failed". + registerDataset(parsed.data.datasetId); + void runUpdateWorkflowInBackground({ input: parsed.data, run, @@ -791,6 +880,75 @@ await fastify.register(async (instance) => { return reply.code(502).send({ error: "Failed to update dataset. Please try again." }); } }); + + instance.post("/stop", async (req, reply) => { + const body = req.body as { datasetId?: string }; + if (!body?.datasetId || typeof body.datasetId !== "string") { + return reply.code(400).send({ error: "datasetId is required" }); + } + + const auth = req.auth; + if (!auth) { + return reply.code(401).send({ error: "Authentication required" }); + } + + try { + const dataset = await convex.query(internal.datasets.getInternal, { + id: body.datasetId, + }); + if (!dataset) { + return reply.code(404).send({ error: "Dataset not found" }); + } + if (dataset.ownerId !== auth.userId) { + return reply.code(403).send({ error: "Not authorized to stop this dataset" }); + } + if (dataset.status !== "building" && dataset.status !== "updating") { + return reply.code(409).send({ error: "Dataset is not currently running" }); + } + + const aborted = abortDataset(body.datasetId); + if (!aborted) { + // No registered signal despite the dataset being "building"/"updating". + // The normal finish path always sets a terminal status in Convex + // *before* calling deregisterDataset(), so if the status is still + // busy here, no running process owns this dataset — it was orphaned + // by a server restart. Force-transition to "failed" so the dataset + // is no longer stuck. + req.log.warn( + { datasetId: body.datasetId }, + "Stop requested for orphaned dataset (no active run registered); forcing to failed", + ); + try { + if (dataset.status === "updating") { + await convex.mutation(internal.datasetRows.clearAllPendingUpdateStatus, { + datasetId: body.datasetId, + }); + } + await setDatasetPopulateStatus( + body.datasetId, + "failed", + "Run interrupted: server restarted while building/updating", + ); + } catch (statusErr) { + req.log.error( + { err: statusErr, datasetId: body.datasetId }, + "Failed to force-transition orphaned dataset to failed", + ); + } + return reply.code(200).send({ success: true }); + } + req.log.info({ datasetId: body.datasetId }, "Stop requested"); + + return reply.code(202).send({ success: true }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes("validator") || msg.includes("Invalid")) { + return reply.code(400).send({ error: "Invalid datasetId" }); + } + req.log.error(err, "Stop failed"); + return reply.code(502).send({ error: "Failed to stop dataset run. Please try again." }); + } + }); }); try { diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 2be1016..534ed7f 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -5,6 +5,7 @@ import { buildInvestigateAgent } from "../agents/investigate.js"; import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; +import { getSignal } from "../../abort-registry.js"; const MAX_DATASET_ROWS = 100; @@ -128,7 +129,8 @@ ${pkBlock} Context (partial data already found): ${context}${urlsBlock}${notesBlock}`; - const result = await agent.generate(prompt, { maxSteps: 10 }); + const abortSignal = getSignal(authorizedDatasetId); + const result = await agent.generate(prompt, { abortSignal, maxSteps: 10 }); if (metrics) { // Use result.toolCalls (the flat accumulated list across all steps) rather // than iterating result.steps[n].toolCalls. The per-step arrays are snapshots @@ -151,6 +153,12 @@ ${context}${urlsBlock}${notesBlock}`; ); return parsed; } catch (err) { + // Only propagate an AbortError if OUR signal was actually fired (i.e. + // the user pressed Stop). Network errors in Node.js can also surface as + // AbortError — re-throwing those would cause the orchestrator's + // agent.generate() to exit early and return a graceful empty result, + // producing a "0 rows" run without any user action. + if (err instanceof Error && err.name === "AbortError" && getSignal(authorizedDatasetId)?.aborted) throw err; const msg = err instanceof Error ? err.message : String(err); console.error(`[run_subagent] subagent error entity="${entity_hint}" err=${msg}`); return { diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index ea06e0e..a831616 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -8,6 +8,7 @@ import { DEFAULT_MODEL_IDS } from "../../config/models.js"; import { buildPopulateAgent } from "../agents/populate.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; +import { getSignal } from "../../abort-registry.js"; /** * Server-set auth/run context threaded through every step. @@ -116,6 +117,7 @@ Respond with EXACTLY one word: scraper or search`; model: openrouter(modelSlug), prompt: classificationPrompt, maxOutputTokens: 10, + abortSignal: getSignal(inputData.datasetId), }); const answer = result.text.trim().toLowerCase(); if (answer === "scraper" || answer === "search") { @@ -124,6 +126,9 @@ Respond with EXACTLY one word: scraper or search`; console.warn(`[enumerate] Unexpected classification "${answer}", defaulting to "search"`); } } catch (err) { + // Only re-throw if OUR signal was actually fired. A spurious network + // AbortError should fall through and default to "search" as before. + if (err instanceof Error && err.name === "AbortError" && getSignal(inputData.datasetId)?.aborted) throw err; const msg = err instanceof Error ? err.message : String(err); console.error(`[enumerate] Classification failed: ${msg}, defaulting to "search"`); } @@ -243,14 +248,22 @@ const agentStep = createStep({ inputData.columns, metrics, ); - const result = await agent.generate(inputData.prompt, { maxSteps: 80 }); + const abortSignal = getSignal(inputData.authorizedDatasetId); + const result = await agent.generate(inputData.prompt, { abortSignal, maxSteps: 80 }); metrics.addOrchestratorResult(result); // Use result.toolCalls (flat accumulated list) — same reasoning as investigate-tool.ts. metrics.countToolCalls(result.toolCalls ?? []); return { text: result.text }; } catch (err) { status = "error"; - errorMsg = err instanceof Error ? err.message : String(err); + // Label user-initiated stops clearly in runStats; treat spurious network + // AbortErrors (signal not fired) as regular failures so the error message + // doesn't mislead operators into thinking the user pressed Stop. + if (err instanceof Error && err.name === "AbortError" && getSignal(inputData.authorizedDatasetId)?.aborted) { + errorMsg = "Stopped by user"; + } else { + errorMsg = err instanceof Error ? err.message : String(err); + } console.error(`[populate-agent] agent.generate failed: ${errorMsg}`); throw err; } finally { diff --git a/backend/src/mastra/workflows/update.ts b/backend/src/mastra/workflows/update.ts index 70c8739..45e3421 100644 --- a/backend/src/mastra/workflows/update.ts +++ b/backend/src/mastra/workflows/update.ts @@ -6,6 +6,7 @@ import { buildRefreshAgent } from "../agents/refresh.js"; import { authContextSchema } from "./populate.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; +import { getSignal } from "../../abort-registry.js"; export const updateInputSchema = datasetContextSchema.extend({ authContext: authContextSchema, @@ -132,7 +133,8 @@ ${sourcesBlock} ${row.rowSummary ? `\nPrevious summary: ${row.rowSummary}` : ""} ${row.howFound ? `\nPreviously found via: ${row.howFound}` : ""}`; - const result = await agent.generate(prompt, { maxSteps: 10 }); + const abortSignal = getSignal(datasetId); + const result = await agent.generate(prompt, { abortSignal, maxSteps: 10 }); // Accumulate token usage into the investigate tier (refresh agents map // to the investigate tier so the runStats schema needs no new columns). @@ -156,6 +158,10 @@ ${row.howFound ? `\nPreviously found via: ${row.howFound}` : ""}`; `[refresh-rows] Row ${row._id}: updated=${updated} steps=${result.steps?.length ?? "?"} toolCalls=${(result.toolCalls as any[])?.length ?? "?"}`, ); } catch (err) { + // Only re-throw if OUR signal was actually fired. Spurious network + // AbortErrors must not terminate a worker — they should be counted as + // row errors so the rest of the dataset continues refreshing. + if (err instanceof Error && err.name === "AbortError" && getSignal(datasetId)?.aborted) throw err; const msg = err instanceof Error ? err.message : String(err); console.error( `[refresh-rows] Row ${row._id} failed: ${msg}`, @@ -182,6 +188,22 @@ ${row.howFound ? `\nPreviously found via: ${row.howFound}` : ""}`; ); await processWithConcurrency(rows, processRow, MAX_CONCURRENT); const finishedAt = Date.now(); + + // If the run was stopped mid-update, workers exited early via AbortError. + // Rows that were never processed still have updateStatus:"pending". + // Clear them now so the UI doesn't show stale shimmer indicators. + const abortSignal = getSignal(datasetId); + if (abortSignal?.aborted) { + console.log(`[refresh-rows] Run was stopped — clearing remaining pending row statuses`); + try { + await convex.mutation(internal.datasetRows.clearAllPendingUpdateStatus, { + datasetId, + }); + } catch (cleanupErr) { + console.error(`[refresh-rows] Failed to clear pending update statuses: ${cleanupErr}`); + } + } + console.log( `[refresh-rows] Done: ${updatedCount} updated, ${errors} errors, ${rows.length - updatedCount - errors} unchanged`, ); diff --git a/frontend/app/dataset/[id]/page.tsx b/frontend/app/dataset/[id]/page.tsx index adba9a0..da7164f 100644 --- a/frontend/app/dataset/[id]/page.tsx +++ b/frontend/app/dataset/[id]/page.tsx @@ -12,7 +12,7 @@ import { useSelection } from "@/components/table/use-selection"; import { useTheme } from "@/components/ThemeToggle"; import { StatusBadge } from "@/components/dataset/StatusBadge"; import { downloadCSV, downloadXLSX } from "@/lib/export"; -import { populate, update } from "@/lib/backend"; +import { populate, update, stopPopulation } from "@/lib/backend"; import { EVENTS, captureException, track } from "@/lib/analytics"; import { REFRESH_CADENCE_OPTIONS, @@ -30,6 +30,7 @@ export default function DatasetPage() { const [exporting, setExporting] = useState<"csv" | "xlsx" | null>(null); const [populating, setPopulating] = useState(false); const [updating, setUpdating] = useState(false); + const [stopping, setStopping] = useState(false); const [exportOpen, setExportOpen] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false); const [confirmPopulate, setConfirmPopulate] = useState(false); @@ -52,6 +53,8 @@ export default function DatasetPage() { const handlePopulate = useCallback(async () => { if (!dataset || populating || dataset.status === "building") return; + // A new run is starting — discard any lingering stop-latch from the previous run. + setStopping(false); setPopulating(true); try { const token = await getToken(); @@ -143,6 +146,8 @@ export default function DatasetPage() { async function handleUpdate() { if (!dataset || updating || dataset.status === "building" || dataset.status === "updating") return; + // A new run is starting — discard any lingering stop-latch from the previous run. + setStopping(false); setUpdating(true); try { const token = await getToken(); @@ -195,6 +200,47 @@ export default function DatasetPage() { } } + async function handleStop() { + if (!dataset || stopping) return; + if (dataset.status !== "building" && dataset.status !== "updating") return; + setStopping(true); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + await stopPopulation(dataset._id, token); + track(EVENTS.DATASET_STOP_REQUESTED, { + datasetId: dataset._id, + status: dataset.status, + }); + // Do NOT clear stopping here. Keep it true until Convex confirms the + // dataset has left the busy state (via the isDatasetBusy effect below). + // This prevents the button from briefly re-enabling during the cleanup window. + } catch (err) { + console.error("[stop] failed", err); + captureException(err, { + operation: "dataset_stop", + datasetId: dataset._id, + }); + // Request failed — clear immediately so the user can retry. + setStopping(false); + } + } + + // Computed before the loading guard so the useEffect below can depend on it + // without hitting the TDZ. Optional chaining handles the pre-load undefined state. + const isDatasetBusy = dataset?.status === "building" || dataset?.status === "updating"; + + // Clear the stop latch once the dataset is confirmed not-busy by Convex. + // Using setTimeout defers the setState out of the synchronous effect body, + // satisfying the react-hooks/set-state-in-effect lint rule while preserving + // the same semantic: latch clears on the next tick after the status transition. + useEffect(() => { + if (!isDatasetBusy) { + const id = setTimeout(() => setStopping(false), 0); + return () => clearTimeout(id); + } + }, [isDatasetBusy]); + if (authLoading || dataset === undefined || rows === undefined) { return (
@@ -208,7 +254,6 @@ export default function DatasetPage() { // the "Dataset not found" UI. const exportDisabled = exporting !== null || rows.length === 0; - const isDatasetBusy = dataset.status === "building" || dataset.status === "updating"; const isOwner = userId === dataset.ownerId; const displayDataset = { ...dataset, @@ -217,6 +262,8 @@ export default function DatasetPage() { }; const updateDisabled = updating || isDatasetBusy; const populateDisabled = populating || isDatasetBusy; + const stopDisabled = stopping || !isDatasetBusy; + const stopLabel = stopping ? "Stopping…" : "Stop"; const updateLabel = dataset.status === "updating" ? "Updating…" : dataset.status === "building" @@ -267,6 +314,18 @@ export default function DatasetPage() { onExport={(fmt) => { setExportOpen(false); handleExport(fmt); }} /> + {isDatasetBusy && ( + + )} + setSettingsOpen((o) => !o)} diff --git a/frontend/convex/datasetRows.ts b/frontend/convex/datasetRows.ts index db609f7..0536f3b 100644 --- a/frontend/convex/datasetRows.ts +++ b/frontend/convex/datasetRows.ts @@ -311,6 +311,34 @@ export const clearUpdateStatus = internalMutation({ }, }); +/** + * Bulk-clear all pending update statuses for a dataset. + * + * Called when a user stops an in-flight update workflow. Workers exit early + * via AbortError, so rows they never reached still have `updateStatus: + * "pending"`. This clears them so the UI doesn't show stale shimmer + * indicators after the run is marked live. + * + * Uses the `by_dataset_update_status` compound index so only the pending + * rows are scanned — the query never touches rows that have already been + * processed (updateStatus === undefined). + */ +export const clearAllPendingUpdateStatus = internalMutation({ + args: { datasetId: v.id("datasets") }, + handler: async (ctx, args) => { + const pendingRows = await ctx.db + .query("datasetRows") + .withIndex("by_dataset_update_status", (q) => + q.eq("datasetId", args.datasetId).eq("updateStatus", "pending"), + ) + .collect(); + for (const row of pendingRows) { + await ctx.db.patch(row._id, { updateStatus: undefined }); + } + return pendingRows.length; + }, +}); + /** * Admin-only row listing for a dataset. Used by the populate agent's * `list_rows` tool to see what's already been inserted in the dataset diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index 7907d6c..d1c1888 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -86,7 +86,11 @@ export default defineSchema({ howFound: v.optional(v.string()), updateStatus: v.optional(v.literal("pending")), scrapeScript: v.optional(v.string()), - }).index("by_dataset", ["datasetId"]), + }) + .index("by_dataset", ["datasetId"]) + // Compound index used by clearAllPendingUpdateStatus to scan only the rows + // that need clearing without a full-dataset read. + .index("by_dataset_update_status", ["datasetId", "updateStatus"]), datasetHistory: defineTable({ datasetRowId: v.id("datasetRows"), diff --git a/frontend/lib/analytics.ts b/frontend/lib/analytics.ts index bb2cb4a..a2a2757 100644 --- a/frontend/lib/analytics.ts +++ b/frontend/lib/analytics.ts @@ -34,6 +34,7 @@ export const EVENTS = { DATASET_EXPORTED: "dataset_exported", DATASET_POPULATE_STARTED: "dataset_populate_started", DATASET_UPDATE_STARTED: "dataset_update_started", + DATASET_STOP_REQUESTED: "dataset_stop_requested", // Creation flow DATASET_CREATION_STARTED: "dataset_creation_started", diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index 9406a18..a4f316d 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -259,3 +259,25 @@ export async function update( return res.json(); } + +export async function stopPopulation( + datasetId: string, + token: string, +): Promise<{ success: boolean }> { + const res = await fetch(`${BACKEND_URL}/stop`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ datasetId }), + }); + + if (!res.ok) { + const body = await res.json().catch(() => null); + const message = body?.error || `Backend error (${res.status})`; + throw new Error(message); + } + + return res.json(); +} From 7549613f0027fed92418f226fa3d5049cc1894c0 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Tue, 2 Jun 2026 15:55:14 -0700 Subject: [PATCH 03/10] Subscribe to theme changes (#118) Co-authored-by: Simantak Dabhade <67303107+simantak-dabhade@users.noreply.github.com> --- frontend/components/ThemeToggle.tsx | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/components/ThemeToggle.tsx b/frontend/components/ThemeToggle.tsx index 346a509..c185991 100644 --- a/frontend/components/ThemeToggle.tsx +++ b/frontend/components/ThemeToggle.tsx @@ -32,11 +32,19 @@ function applyTheme(theme: Theme): void { function subscribeToThemeChange(onThemeChange: () => void): () => void { if (typeof window === "undefined") return () => {}; - window.addEventListener("storage", onThemeChange); - window.addEventListener(THEME_CHANGED_EVENT, onThemeChange); + const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); + function syncTheme() { + applyTheme(readEffectiveTheme()); + onThemeChange(); + } + + window.addEventListener("storage", syncTheme); + window.addEventListener(THEME_CHANGED_EVENT, syncTheme); + mediaQuery.addEventListener("change", syncTheme); return () => { - window.removeEventListener("storage", onThemeChange); - window.removeEventListener(THEME_CHANGED_EVENT, onThemeChange); + window.removeEventListener("storage", syncTheme); + window.removeEventListener(THEME_CHANGED_EVENT, syncTheme); + mediaQuery.removeEventListener("change", syncTheme); }; } From 1b841cc8252ae4839e014a6b5279aa86024f5870 Mon Sep 17 00:00:00 2001 From: Mengzhe Li <129459076+MMeteorL@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:23:36 -0700 Subject: [PATCH 04/10] Feature/cell expand side sheet (#115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add cell expand side sheet with sources Hovering any table cell reveals a Maximize2 icon. Clicking it opens a slide-in panel showing the column name, type, description, full cell value with a copy button, and (when available) the row-level sources the populate agent saved alongside the data. Changes: - SideSheet.tsx: new component with backdrop, Escape/Tab trap, scroll lock, and slide-in animation. CellDetail sub-component shows column metadata, value (with inline "Copied" feedback), and sources as clickable external links. - types.ts: add optional `sources?: string[]` to DatasetRow - DataRow.tsx: `group` + Maximize2 button on hover per cell; `onCellExpand` callback added to DataRowData interface. - DatasetTable.tsx: accept and forward `onCellExpand` prop. - page.tsx: cellDetail state, row sources lookup, SideSheet render. - globals.css: `slide-in` keyframe + `.animate-slide-in` utility. Co-Authored-By: Claude Sonnet 4.6 * Fix lucide-react module-not-found by using inline SVGs lucide-react is in bun.lock but not installed in the running Docker container's node_modules. Existing components (ColumnHeader, ColumnIcon, ThemeToggle) all use inline SVGs — follow the same pattern instead of adding a runtime dependency. Replaced: Copy, Check, ExternalLink, X in SideSheet.tsx and Maximize2 in DataRow.tsx with self-contained SVG components. Co-Authored-By: Claude Sonnet 4.6 * Fix perf regression, simplify state shape, remove dead CSS Performance: onCellExpand was an inline arrow on , busting the itemData useMemo on every parent render and causing react-window to re-render all visible rows unnecessarily. Replaced with a useCallback (handleCellExpand) so the reference is stable. Simplification: cellDetail state now stores the resolved DatasetColumn object at click time instead of a columnName string. This eliminates the render-time columns.find() call and the IIFE pattern in JSX — the SideSheet children reduce to a plain conditional render. Cleanup: removed @keyframes slide-in and .animate-slide-in from globals.css — those were added for the side-sheet variant and are unused now that the UI is a centered modal. Co-Authored-By: Claude Sonnet 4.6 * Address CodeRabbit review: XSS guard, timer cleanup, a11y, lint fixes - SideSheet: validate source URLs to http/https before rendering as to prevent javascript:/data: XSS; invalid URLs fall back to plain text - SideSheet: store setTimeout ID in a ref and clear on unmount to avoid calling setCopied after the component is gone; promote handleCopy to useCallback - SideSheet: add aria-labelledby on dialog + id on heading for screen readers - SideSheet: use URL string as list key instead of array index - DataRow: move floorWidth import above IconMaximize2 declaration (import/first) - DataRow: reveal expand button on keyboard focus-visible, not only on hover Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Simantak Dabhade <67303107+simantak-dabhade@users.noreply.github.com> --- frontend/app/dataset/[id]/page.tsx | 26 +++ frontend/components/SideSheet.tsx | 228 +++++++++++++++++++++ frontend/components/table/DataRow.tsx | 24 ++- frontend/components/table/DatasetTable.tsx | 5 +- frontend/components/table/types.ts | 1 + 5 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 frontend/components/SideSheet.tsx diff --git a/frontend/app/dataset/[id]/page.tsx b/frontend/app/dataset/[id]/page.tsx index da7164f..d9ee1b6 100644 --- a/frontend/app/dataset/[id]/page.tsx +++ b/frontend/app/dataset/[id]/page.tsx @@ -9,6 +9,8 @@ import { api } from "@/convex/_generated/api"; import type { Id } from "@/convex/_generated/dataModel"; import { DatasetTable } from "@/components/table"; import { useSelection } from "@/components/table/use-selection"; +import { SideSheet, CellDetail } from "@/components/SideSheet"; +import type { DatasetColumn } from "@/components/table/types"; import { useTheme } from "@/components/ThemeToggle"; import { StatusBadge } from "@/components/dataset/StatusBadge"; import { downloadCSV, downloadXLSX } from "@/lib/export"; @@ -35,6 +37,11 @@ export default function DatasetPage() { const [settingsOpen, setSettingsOpen] = useState(false); const [confirmPopulate, setConfirmPopulate] = useState(false); const [savingRefreshCadence, setSavingRefreshCadence] = useState(false); + const [cellDetail, setCellDetail] = useState<{ + column: DatasetColumn; + value: unknown; + sources?: string[]; + } | null>(null); const datasetId = params.id as Id<"datasets">; const dataset = useQuery( @@ -83,6 +90,14 @@ export default function DatasetPage() { } }, [dataset, populating, getToken]); + const handleCellExpand = useCallback((columnName: string, value: unknown, rowId: string) => { + if (!dataset || !rows) return; + const col = dataset.columns.find((c) => c.name === columnName); + if (!col) return; + const row = rows.find((r) => r._id === rowId); + setCellDetail({ column: col, value, sources: row?.sources }); + }, [dataset, rows]); + const openedFired = useRef(null); const autoPopulateFired = useRef(null); useEffect(() => { @@ -385,8 +400,19 @@ export default function DatasetPage() { rows={rows} datasetId={datasetId} selection={selection} + onCellExpand={handleCellExpand} /> + setCellDetail(null)}> + {cellDetail && ( + + )} + + {confirmPopulate && (
+ {/* Backdrop */} +
+ {/* Modal */} +
+
+

Cell Detail

+ +
+
{children}
+
+
+ ); +} + +/* ------------------------------------------------------------------ */ +/* Content */ +/* ------------------------------------------------------------------ */ + +interface CellDetailProps { + column: DatasetColumn; + value: unknown; + /** Row-level sources stored by the populate agent. */ + sources?: string[]; +} + +function isValidHttpUrl(src: string): boolean { + try { + const { protocol } = new URL(src); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} + +export function CellDetail({ column, value, sources }: CellDetailProps) { + const [copied, setCopied] = useState(false); + const copyTimerRef = useRef | null>(null); + const displayValue = value == null || value === "" ? "—" : String(value); + + // Clear any pending timer when the component unmounts to avoid calling + // setCopied on an already-gone component. + useEffect(() => { + return () => { + if (copyTimerRef.current != null) clearTimeout(copyTimerRef.current); + }; + }, []); + + const handleCopy = useCallback(async () => { + try { + await navigator.clipboard.writeText(value == null ? "" : String(value)); + setCopied(true); + if (copyTimerRef.current != null) clearTimeout(copyTimerRef.current); + copyTimerRef.current = setTimeout(() => setCopied(false), 2000); + } catch { + // Clipboard API unavailable (e.g. non-HTTPS dev); silently ignore. + } + }, [value]); + + return ( + + ); +} diff --git a/frontend/components/table/DataRow.tsx b/frontend/components/table/DataRow.tsx index 1fe0974..2d54661 100644 --- a/frontend/components/table/DataRow.tsx +++ b/frontend/components/table/DataRow.tsx @@ -7,12 +7,21 @@ import type { DatasetRow, DatasetColumn } from "./types"; import { CellValue } from "./CellValue"; import { floorWidth } from "./utils"; +function IconMaximize2() { + return ( + + ); +} + export interface DataRowData { rows: Row[]; columns: DatasetColumn[]; columnWidths: number[]; isSelected: (id: string) => boolean; toggleRow: (id: string, shiftKey: boolean) => void; + onCellExpand: (columnName: string, value: unknown, rowId: string) => void; isBuilding: boolean; pendingRowIds: Set; flashingCells: Set; @@ -27,7 +36,7 @@ function DataRowImpl({ index: number; style: CSSProperties; }) { - const { rows, columns, columnWidths, isSelected, toggleRow, isBuilding, pendingRowIds, flashingCells } = data; + const { rows, columns, columnWidths, isSelected, toggleRow, onCellExpand, isBuilding, pendingRowIds, flashingCells } = data; const row = rows[index]; if (!row) { @@ -112,7 +121,7 @@ function DataRowImpl({
+ {isPending &&
}
); diff --git a/frontend/components/table/DatasetTable.tsx b/frontend/components/table/DatasetTable.tsx index f921875..e74c2bb 100644 --- a/frontend/components/table/DatasetTable.tsx +++ b/frontend/components/table/DatasetTable.tsx @@ -85,11 +85,13 @@ export function DatasetTable({ rows, datasetId, selection, + onCellExpand, }: { dataset: DatasetMeta; rows: DatasetRow[]; datasetId: string; selection: Selection; + onCellExpand: (columnName: string, value: unknown, rowId: string) => void; }) { const tableContainerRef = useRef(null); const previousResizingColumnIdRef = useRef(false); @@ -168,11 +170,12 @@ export function DatasetTable({ columnWidths, isSelected: selection.has, toggleRow, + onCellExpand, isBuilding, pendingRowIds, flashingCells, }), - [tableRows, dataset.columns, columnWidths, selection.has, toggleRow, isBuilding, pendingRowIds, flashingCells], + [tableRows, dataset.columns, columnWidths, selection.has, toggleRow, onCellExpand, isBuilding, pendingRowIds, flashingCells], ); return ( diff --git a/frontend/components/table/types.ts b/frontend/components/table/types.ts index 5fc380c..6eff009 100644 --- a/frontend/components/table/types.ts +++ b/frontend/components/table/types.ts @@ -28,5 +28,6 @@ export interface DatasetRow { _id: string; _creationTime: number; data: Record; + sources?: string[]; updateStatus?: "pending"; } From 4a8bf4cc05a828fc7d02849c6079e1fcfdc6c148 Mon Sep 17 00:00:00 2001 From: "tinyfish-github-terraform[bot]" <184532149+tinyfish-github-terraform[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 19:05:36 +0000 Subject: [PATCH 05/10] Update secrets-scanner.yml workflow --- .github/workflows/secrets-scanner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/secrets-scanner.yml b/.github/workflows/secrets-scanner.yml index d419edf..ad955d2 100644 --- a/.github/workflows/secrets-scanner.yml +++ b/.github/workflows/secrets-scanner.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 - name: TruffleHog OSS From 8e04dddc44e8367d38b2beda756985545e8228b9 Mon Sep 17 00:00:00 2001 From: siddarth Date: Thu, 4 Jun 2026 14:27:14 +0530 Subject: [PATCH 06/10] Merge upstream/main into main --- frontend/app/dataset/[id]/page.tsx | 94 +++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/frontend/app/dataset/[id]/page.tsx b/frontend/app/dataset/[id]/page.tsx index d9ee1b6..dec5dfe 100644 --- a/frontend/app/dataset/[id]/page.tsx +++ b/frontend/app/dataset/[id]/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useParams } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import Link from "next/link"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useMutation, useQuery, useConvexAuth } from "convex/react"; @@ -25,6 +25,7 @@ import type { ProfileUser } from "@/lib/profile-user"; export default function DatasetPage() { const params = useParams(); + const router = useRouter(); const { isLoading: authLoading } = useConvexAuth(); const { userId, getToken } = useAuth(); const { user } = useUser(); @@ -36,6 +37,7 @@ export default function DatasetPage() { const [exportOpen, setExportOpen] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false); const [confirmPopulate, setConfirmPopulate] = useState(false); + const [confirmDelete, setConfirmDelete] = useState(false); const [savingRefreshCadence, setSavingRefreshCadence] = useState(false); const [cellDetail, setCellDetail] = useState<{ column: DatasetColumn; @@ -53,6 +55,7 @@ export default function DatasetPage() { authLoading ? "skip" : { datasetId }, ); const updateRefreshSettings = useMutation(api.datasets.updateRefreshSettings); + const removeDataset = useMutation(api.datasets.remove); const rowIds = useMemo(() => (rows ?? []).map((r) => r._id), [rows]); const selection = useSelection(rowIds); @@ -241,6 +244,17 @@ export default function DatasetPage() { } } + async function handleDelete() { + if (!dataset) return; + try { + await removeDataset({ id: dataset._id }); + router.push("/dashboard"); + } catch (err) { + console.error("[delete] failed", err); + captureException(err, { operation: "dataset_delete", datasetId: dataset._id }); + } + } + // Computed before the loading guard so the useEffect below can depend on it // without hitting the TDZ. Optional chaining handles the pre-load undefined state. const isDatasetBusy = dataset?.status === "building" || dataset?.status === "updating"; @@ -255,7 +269,6 @@ export default function DatasetPage() { return () => clearTimeout(id); } }, [isDatasetBusy]); - if (authLoading || dataset === undefined || rows === undefined) { return (
@@ -361,6 +374,7 @@ export default function DatasetPage() { handlePopulate(); } }} + onDelete={isOwner ? () => { setSettingsOpen(false); setConfirmDelete(true); } : undefined} />
@@ -423,6 +437,17 @@ export default function DatasetPage() { onCancel={() => setConfirmPopulate(false)} /> )} + + {confirmDelete && ( + { + setConfirmDelete(false); + handleDelete(); + }} + onCancel={() => setConfirmDelete(false)} + /> + )}
); } @@ -523,6 +548,7 @@ function SettingsDropdown({ onRefreshCadenceChange, onUpdate, onPopulate, + onDelete, }: { open: boolean; onToggle: () => void; @@ -536,6 +562,7 @@ function SettingsDropdown({ onRefreshCadenceChange: (refreshCadence: RefreshCadence) => void; onUpdate: () => void; onPopulate: () => void; + onDelete?: () => void; }) { const ref = useRef(null); @@ -589,6 +616,7 @@ function SettingsDropdown({
+ {onDelete && ( +
+ +
+ )}
)}
@@ -735,3 +774,54 @@ function ConfirmPopulateModal({
); } + +/* ------------------------------------------------------------------ */ +/* Confirm delete modal */ +/* ------------------------------------------------------------------ */ + +function ConfirmDeleteModal({ + datasetName, + onConfirm, + onCancel, +}: { + datasetName: string; + onConfirm: () => void; + onCancel: () => void; +}) { + useEffect(() => { + function handleKey(e: KeyboardEvent) { + if (e.key === "Escape") onCancel(); + } + document.addEventListener("keydown", handleKey); + return () => document.removeEventListener("keydown", handleKey); + }, [onCancel]); + + return ( +
{ if (e.target === e.currentTarget) onCancel(); }} + role="presentation" + > +
+

+ Delete “{datasetName}”? +

+

All rows will be permanently deleted. This can't be undone.

+
+ + +
+
+
+ ); +} From c694133e02bd233b54175f2a0a40f48460c53692 Mon Sep 17 00:00:00 2001 From: Pranav Janakiraman <104693003+pranavjana@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:37:48 -0700 Subject: [PATCH 07/10] Allow custom dataset max rows (#128) * Cap dataset population at 100 rows * Handle row cap count failures in subagent tool * Mention row limit sentinel in populate prompt * Allow custom dataset max rows * Allow editing dataset max rows * Address max rows review feedback --- backend/src/index.ts | 13 +- backend/src/mastra/agents/populate.ts | 12 +- backend/src/mastra/tools/investigate-tool.ts | 9 +- backend/src/mastra/workflows/populate.ts | 5 +- backend/src/pipeline/populate.ts | 3 + frontend/app/dataset/[id]/page.tsx | 219 +++++++++++-------- frontend/app/dataset/new/page.tsx | 52 ++++- frontend/convex/datasetRows.ts | 7 +- frontend/convex/datasets.ts | 37 +++- frontend/convex/schema.ts | 3 + frontend/lib/backend.ts | 3 +- 11 files changed, 251 insertions(+), 112 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 61c7314..26c72ac 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -568,6 +568,7 @@ function startLocalRefreshScheduler( datasetId: dataset.datasetId, datasetName: dataset.datasetName, description: dataset.description, + maxRowCount: dataset.maxRowCount ?? 100, columns: dataset.columns, }, run, @@ -768,6 +769,13 @@ await fastify.register(async (instance) => { throw new Error(`Unexpected populate claim outcome: ${populateOutcome}`); } + const dataset = await convex.query(internal.datasets.getInternal, { + id: parsed.data.datasetId, + }); + if (!dataset) { + return reply.code(404).send({ error: "Dataset not found" }); + } + const { getModelConfig } = await import("./config/models.js"); const modelConfig = await getModelConfig(auth.userId); @@ -787,7 +795,10 @@ await fastify.register(async (instance) => { const controller = registerDataset(parsed.data.datasetId); void runPopulateWorkflowInBackground({ - input: parsed.data, + input: { + ...parsed.data, + maxRowCount: dataset.maxRowCount ?? parsed.data.maxRowCount, + }, run, controller, authorizedUserId: auth.userId, diff --git a/backend/src/mastra/agents/populate.ts b/backend/src/mastra/agents/populate.ts index 85edf53..155492a 100644 --- a/backend/src/mastra/agents/populate.ts +++ b/backend/src/mastra/agents/populate.ts @@ -10,9 +10,10 @@ const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY!, }); -const INSTRUCTIONS = `You are an expert dataset builder. You conduct research using your web tools. +function buildInstructions(maxRowCount: number): string { + return `You are an expert dataset builder. You conduct research using your web tools. You do broad research to see which rows to add, and then you spin up sub-agents that can do the deep research and fill in each row for you. -Your job is to make sure you dispatch and manage your army of sub agents to build up a dataset with 100 rows in it. Stop as soon as the dataset reaches 100 rows. +Your job is to make sure you dispatch and manage your army of sub agents to build up a dataset with ${maxRowCount} rows in it. Stop as soon as the dataset reaches ${maxRowCount} rows. WORKFLOW: 1. Understand the data that is is needed and do some research to find places on the web where this data may be obvious and easy to find, collect these links to see what the task of scraping the web is going to look like. @@ -22,12 +23,13 @@ If the dataset is to look at YC Companies, collect links for the YC Startup regi 3. See what the subagent reports back with, if all good and it gives you some information, use that to give better instuctions to subsequent sub agents. -Keep going until you have 100 rows, then finish immediately. If run_subagent reports ROW_LIMIT_REACHED, stop calling tools and finish the run. +Keep going until you have ${maxRowCount} rows, then finish immediately. If run_subagent reports ROW_LIMIT_REACHED, stop calling tools and finish the run. This process should become faster overtime as you just find new rows to go and build, and you keep invoking sub agents in parallel to fill them in. Duplicates are rejected automatically based on primary key columns. If a subagent reports a duplicate, don't re-investigate the same entity — move on to a new one. `; +} /** * Build the orchestrator Agent for a populate run. @@ -42,6 +44,7 @@ export function buildPopulateAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], + maxRowCount: number, metrics?: RunMetrics, ): Agent { const modelSlug = authContext.modelConfig!.populateOrchestrator; @@ -49,7 +52,7 @@ export function buildPopulateAgent( return new Agent({ id: "populate-agent", name: "Dataset Populate Orchestrator", - instructions: INSTRUCTIONS, + instructions: buildInstructions(maxRowCount), model: openrouter(modelSlug), tools: { search_web: searchWebTool, @@ -58,6 +61,7 @@ export function buildPopulateAgent( authorizedDatasetId, authContext, columns, + maxRowCount, metrics, ), }, diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 534ed7f..c1d6b18 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -7,8 +7,6 @@ import type { PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; import { getSignal } from "../../abort-registry.js"; -const MAX_DATASET_ROWS = 100; - const investigateInputSchema = z.object({ entity_hint: z .string() @@ -77,6 +75,7 @@ export function buildSubagentTool( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], + maxRowCount: number, metrics?: RunMetrics, ) { return createTool({ @@ -90,10 +89,10 @@ export function buildSubagentTool( const rowCount = await convex.query(internal.datasetRows.countByDataset, { datasetId: authorizedDatasetId, }); - if (rowCount >= MAX_DATASET_ROWS) { + if (rowCount >= maxRowCount) { return { inserted: false, - reason: `ROW_LIMIT_REACHED: BigSet datasets are capped at ${MAX_DATASET_ROWS} rows. Stop calling run_subagent and finish the run.`, + reason: `ROW_LIMIT_REACHED: this BigSet dataset is capped at ${maxRowCount} rows. Stop calling run_subagent and finish the run.`, row_summary: undefined, clues: undefined, }; @@ -130,7 +129,7 @@ Context (partial data already found): ${context}${urlsBlock}${notesBlock}`; const abortSignal = getSignal(authorizedDatasetId); - const result = await agent.generate(prompt, { abortSignal, maxSteps: 10 }); + const result = await agent.generate(prompt, { abortSignal, maxSteps: 25 }); if (metrics) { // Use result.toolCalls (the flat accumulated list across all steps) rather // than iterating result.steps[n].toolCalls. The per-step arrays are snapshots diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index a831616..35db3b1 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -157,6 +157,7 @@ const buildPromptOutputSchema = z.object({ authorizedDatasetId: z.string(), authContext: authContextSchema, columns: z.array(populateColumnSchema), + maxRowCount: z.number().int().min(1), }); const buildPromptStep = createStep({ @@ -203,7 +204,7 @@ ${columnsDesc}${pkNote}${manifestNote}${strategyNote} Search the web broadly to find real entities that fit this dataset topic. For each lead you find, call run_subagent with the primary key values and any context/URLs you have found. If run_subagent returns ROW_LIMIT_REACHED, stop immediately and do not make any more tool calls. -Stop the populate run as soon as the dataset reaches 100 rows.`; +Stop the populate run as soon as the dataset reaches ${inputData.maxRowCount} rows.`; console.log( `[build-prompt] Built prompt for ${inputData.datasetName} (${inputData.columns.length} columns, strategy=${inputData.enumerationStrategy})`, @@ -213,6 +214,7 @@ Stop the populate run as soon as the dataset reaches 100 rows.`; authorizedDatasetId: inputData.datasetId, authContext: inputData.authContext, columns: inputData.columns, + maxRowCount: inputData.maxRowCount, }; }, }); @@ -246,6 +248,7 @@ const agentStep = createStep({ inputData.authorizedDatasetId, inputData.authContext, inputData.columns, + inputData.maxRowCount, metrics, ); const abortSignal = getSignal(inputData.authorizedDatasetId); diff --git a/backend/src/pipeline/populate.ts b/backend/src/pipeline/populate.ts index 55d37aa..589db1e 100644 --- a/backend/src/pipeline/populate.ts +++ b/backend/src/pipeline/populate.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +const FREE_TIER_MONTHLY_QUOTA = 2500; + export const populateColumnSchema = z.object({ name: z.string(), type: z.enum(["text", "number", "boolean", "url", "date"]), @@ -12,6 +14,7 @@ export const datasetContextSchema = z.object({ datasetId: z.string().min(1), datasetName: z.string(), description: z.string(), + maxRowCount: z.number().int().min(1).max(FREE_TIER_MONTHLY_QUOTA).default(100), columns: z.array(populateColumnSchema).min(1), rowIds: z.array(z.string()).min(1).optional(), }); diff --git a/frontend/app/dataset/[id]/page.tsx b/frontend/app/dataset/[id]/page.tsx index dec5dfe..d28abda 100644 --- a/frontend/app/dataset/[id]/page.tsx +++ b/frontend/app/dataset/[id]/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useParams, useRouter } from "next/navigation"; +import { useParams } from "next/navigation"; import Link from "next/link"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useMutation, useQuery, useConvexAuth } from "convex/react"; @@ -25,8 +25,7 @@ import type { ProfileUser } from "@/lib/profile-user"; export default function DatasetPage() { const params = useParams(); - const router = useRouter(); - const { isLoading: authLoading } = useConvexAuth(); + const { isLoading: authLoading, isAuthenticated } = useConvexAuth(); const { userId, getToken } = useAuth(); const { user } = useUser(); const { signOut } = useClerk(); @@ -37,8 +36,9 @@ export default function DatasetPage() { const [exportOpen, setExportOpen] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false); const [confirmPopulate, setConfirmPopulate] = useState(false); - const [confirmDelete, setConfirmDelete] = useState(false); const [savingRefreshCadence, setSavingRefreshCadence] = useState(false); + const [savingMaxRowCount, setSavingMaxRowCount] = useState(false); + const [maxRowCountSaveError, setMaxRowCountSaveError] = useState(null); const [cellDetail, setCellDetail] = useState<{ column: DatasetColumn; value: unknown; @@ -55,7 +55,11 @@ export default function DatasetPage() { authLoading ? "skip" : { datasetId }, ); const updateRefreshSettings = useMutation(api.datasets.updateRefreshSettings); - const removeDataset = useMutation(api.datasets.remove); + const updateMaxRowCount = useMutation(api.datasets.updateMaxRowCount); + const usage = useQuery( + api.quota.getMy, + isAuthenticated ? {} : "skip", + ); const rowIds = useMemo(() => (rows ?? []).map((r) => r._id), [rows]); const selection = useSelection(rowIds); @@ -74,6 +78,7 @@ export default function DatasetPage() { dataset._id, dataset.name, dataset.description, + dataset.maxRowCount ?? 100, dataset.columns, token, ); @@ -218,6 +223,45 @@ export default function DatasetPage() { } } + async function handleMaxRowCountChange(maxRowCount: number) { + if (!dataset || savingMaxRowCount || userId !== dataset.ownerId) return; + if (!Number.isInteger(maxRowCount) || maxRowCount < 1) { + setMaxRowCountSaveError("Max rows must be a whole number greater than 0."); + captureException(new Error("Invalid max row count"), { + operation: "dataset_max_row_count_update", + datasetId: dataset._id, + }); + return; + } + if (usage && maxRowCount > usage.remaining) { + setMaxRowCountSaveError( + `Max rows cannot exceed your remaining monthly quota of ${usage.remaining.toLocaleString()} row operations.`, + ); + return; + } + + setMaxRowCountSaveError(null); + setSavingMaxRowCount(true); + try { + await updateMaxRowCount({ + id: dataset._id, + maxRowCount, + }); + setMaxRowCountSaveError(null); + } catch (err) { + console.error("[max rows] failed", err); + setMaxRowCountSaveError( + err instanceof Error ? err.message : "Failed to update max rows.", + ); + captureException(err, { + operation: "dataset_max_row_count_update", + datasetId: dataset._id, + }); + } finally { + setSavingMaxRowCount(false); + } + } + async function handleStop() { if (!dataset || stopping) return; if (dataset.status !== "building" && dataset.status !== "updating") return; @@ -244,17 +288,6 @@ export default function DatasetPage() { } } - async function handleDelete() { - if (!dataset) return; - try { - await removeDataset({ id: dataset._id }); - router.push("/dashboard"); - } catch (err) { - console.error("[delete] failed", err); - captureException(err, { operation: "dataset_delete", datasetId: dataset._id }); - } - } - // Computed before the loading guard so the useEffect below can depend on it // without hitting the TDZ. Optional chaining handles the pre-load undefined state. const isDatasetBusy = dataset?.status === "building" || dataset?.status === "updating"; @@ -269,6 +302,7 @@ export default function DatasetPage() { return () => clearTimeout(id); } }, [isDatasetBusy]); + if (authLoading || dataset === undefined || rows === undefined) { return (
@@ -287,6 +321,7 @@ export default function DatasetPage() { ...dataset, refreshCadence: dataset.refreshCadence ?? "daily", refreshEnabled: dataset.refreshEnabled ?? true, + maxRowCount: dataset.maxRowCount ?? 100, }; const updateDisabled = updating || isDatasetBusy; const populateDisabled = populating || isDatasetBusy; @@ -360,11 +395,16 @@ export default function DatasetPage() { onClose={() => setSettingsOpen(false)} refreshCadence={displayDataset.refreshCadence} refreshCadenceDisabled={!isOwner || savingRefreshCadence} + maxRowCount={displayDataset.maxRowCount} + maxRowCountRemaining={usage?.remaining} + maxRowCountSaveError={maxRowCountSaveError} + maxRowCountDisabled={!isOwner || savingMaxRowCount} updateLabel={updateLabel} updateDisabled={updateDisabled} populateLabel={populateLabel} populateDisabled={populateDisabled} onRefreshCadenceChange={handleRefreshCadenceChange} + onMaxRowCountChange={handleMaxRowCountChange} onUpdate={() => { setSettingsOpen(false); handleUpdate(); }} onPopulate={() => { setSettingsOpen(false); @@ -374,7 +414,6 @@ export default function DatasetPage() { handlePopulate(); } }} - onDelete={isOwner ? () => { setSettingsOpen(false); setConfirmDelete(true); } : undefined} />
@@ -437,17 +476,6 @@ export default function DatasetPage() { onCancel={() => setConfirmPopulate(false)} /> )} - - {confirmDelete && ( - { - setConfirmDelete(false); - handleDelete(); - }} - onCancel={() => setConfirmDelete(false)} - /> - )}
); } @@ -541,30 +569,50 @@ function SettingsDropdown({ onClose, refreshCadence, refreshCadenceDisabled, + maxRowCount, + maxRowCountRemaining, + maxRowCountSaveError, + maxRowCountDisabled, updateLabel, updateDisabled, populateLabel, populateDisabled, onRefreshCadenceChange, + onMaxRowCountChange, onUpdate, onPopulate, - onDelete, }: { open: boolean; onToggle: () => void; onClose: () => void; refreshCadence: RefreshCadence; refreshCadenceDisabled: boolean; + maxRowCount: number; + maxRowCountRemaining?: number; + maxRowCountSaveError: string | null; + maxRowCountDisabled: boolean; updateLabel: string; updateDisabled: boolean; populateLabel: string; populateDisabled: boolean; onRefreshCadenceChange: (refreshCadence: RefreshCadence) => void; + onMaxRowCountChange: (maxRowCount: number) => void; onUpdate: () => void; onPopulate: () => void; - onDelete?: () => void; }) { const ref = useRef(null); + const [maxRowCountInput, setMaxRowCountInput] = useState(String(maxRowCount)); + const parsedMaxRowCount = Number(maxRowCountInput); + const maxRowCountValidationError = + !maxRowCountInput.trim() + ? "Required" + : !Number.isInteger(parsedMaxRowCount) || parsedMaxRowCount < 1 + ? "Use a whole number" + : maxRowCountRemaining !== undefined && parsedMaxRowCount > maxRowCountRemaining + ? `Max ${maxRowCountRemaining.toLocaleString()}` + : null; + const maxRowCountChanged = + Number.isInteger(parsedMaxRowCount) && parsedMaxRowCount !== maxRowCount; useEffect(() => { if (!open) return; @@ -575,6 +623,10 @@ function SettingsDropdown({ return () => document.removeEventListener("mousedown", handleClick); }, [open, onClose]); + useEffect(() => { + if (!open) setMaxRowCountInput(String(maxRowCount)); + }, [maxRowCount, open]); + return (
+
+
+ Max rows +
+
+ setMaxRowCountInput(e.currentTarget.value)} + onBlur={() => { + if (!maxRowCountInput.trim()) return; + const value = Number(maxRowCountInput); + if (Number.isInteger(value) && value >= 1) { + setMaxRowCountInput(String(value)); + } + }} + className="min-w-0 flex-1 rounded-lg border border-border bg-background px-2 py-1.5 text-xs text-foreground outline-none transition-colors focus:border-foreground/30 disabled:opacity-50" + /> + +
+

+ {maxRowCountValidationError ?? + maxRowCountSaveError ?? + (maxRowCountRemaining !== undefined + ? `${maxRowCountRemaining.toLocaleString()} row operations available` + : "Applies to the next populate run")} +

+
Refresh cadence @@ -616,7 +710,6 @@ function SettingsDropdown({
- {onDelete && ( -
- -
- )}
)}
@@ -774,54 +856,3 @@ function ConfirmPopulateModal({ ); } - -/* ------------------------------------------------------------------ */ -/* Confirm delete modal */ -/* ------------------------------------------------------------------ */ - -function ConfirmDeleteModal({ - datasetName, - onConfirm, - onCancel, -}: { - datasetName: string; - onConfirm: () => void; - onCancel: () => void; -}) { - useEffect(() => { - function handleKey(e: KeyboardEvent) { - if (e.key === "Escape") onCancel(); - } - document.addEventListener("keydown", handleKey); - return () => document.removeEventListener("keydown", handleKey); - }, [onCancel]); - - return ( -
{ if (e.target === e.currentTarget) onCancel(); }} - role="presentation" - > -
-

- Delete “{datasetName}”? -

-

All rows will be permanently deleted. This can't be undone.

-
- - -
-
-
- ); -} diff --git a/frontend/app/dataset/new/page.tsx b/frontend/app/dataset/new/page.tsx index 285cbde..bb4dded 100644 --- a/frontend/app/dataset/new/page.tsx +++ b/frontend/app/dataset/new/page.tsx @@ -4,7 +4,7 @@ import { useEffect, useState, useRef } from "react"; import { useRouter } from "next/navigation"; import Link from "next/link"; import { useAuth } from "@clerk/nextjs"; -import { useMutation, useConvexAuth } from "convex/react"; +import { useMutation, useQuery, useConvexAuth } from "convex/react"; import { api } from "@/convex/_generated/api"; import { EVENTS, track } from "@/lib/analytics"; import { inferSchema, type InferredColumn } from "@/lib/backend"; @@ -43,6 +43,8 @@ const BACKEND_TYPE_MAP: Record = { boolean: "boolean", }; +const DEFAULT_MAX_ROW_COUNT = 100; + function mapBackendColumn(col: InferredColumn, index: number): ProposedColumn { return { id: String(index + 1), @@ -81,6 +83,9 @@ export default function NewDatasetPage() { const [step, setStep] = useState("describe"); const [prompt, setPrompt] = useState(""); const [refreshCadence, setRefreshCadence] = useState("daily"); + const [maxRowCountInput, setMaxRowCountInput] = useState( + String(DEFAULT_MAX_ROW_COUNT), + ); const [columns, setColumns] = useState([]); const [datasetName, setDatasetName] = useState(""); const [isCreating, setIsCreating] = useState(false); @@ -92,6 +97,10 @@ export default function NewDatasetPage() { const { getToken } = useAuth(); const createDataset = useMutation(api.datasets.create); + const usage = useQuery( + api.quota.getMy, + isAuthenticated ? {} : "skip", + ); // Page-view event: fires once when the wizard becomes visible (after // auth resolves and the user is authenticated; we don't want to fire @@ -163,6 +172,17 @@ export default function NewDatasetPage() { async function handleConfirm() { if (isCreating) return; + const maxRowCount = Number(maxRowCountInput); + if (!Number.isInteger(maxRowCount) || maxRowCount < 1) { + setError("Max rows must be a whole number greater than 0."); + return; + } + if (usage && maxRowCount > usage.remaining) { + setError( + `Max rows cannot exceed your remaining monthly quota of ${usage.remaining.toLocaleString()} row operations.`, + ); + return; + } setIsCreating(true); setError(null); let datasetId: string; @@ -171,6 +191,7 @@ export default function NewDatasetPage() { name: datasetName, description: prompt, refreshCadence, + maxRowCount, columns: columns.map((c) => ({ name: c.name, type: c.type, @@ -195,6 +216,7 @@ export default function NewDatasetPage() { datasetId, column_count: columns.length, refreshCadence, + maxRowCount, }); } catch {} router.push(`/dataset/${datasetId}`); @@ -320,6 +342,34 @@ export default function NewDatasetPage() { ))} + +
+ + setMaxRowCountInput(e.currentTarget.value)} + onBlur={() => { + if (!maxRowCountInput.trim()) return; + const value = Number(maxRowCountInput); + if (Number.isInteger(value) && value >= 1) { + setMaxRowCountInput(String(value)); + } + }} + className="w-36 rounded-lg border border-border bg-surface px-4 py-2.5 text-sm font-medium outline-none focus:border-foreground/30 transition-colors" + /> + {usage && ( +

+ Up to {usage.remaining.toLocaleString()} row operations available this month. +

+ )} +
diff --git a/frontend/convex/datasetRows.ts b/frontend/convex/datasetRows.ts index 0536f3b..eac39ea 100644 --- a/frontend/convex/datasetRows.ts +++ b/frontend/convex/datasetRows.ts @@ -5,7 +5,7 @@ import type { Id } from "./_generated/dataModel.js"; import { assertRowInDataset, loadReadableDataset } from "./lib/authz.js"; import { consumeQuotaForDataset } from "./lib/quota.js"; -const MAX_DATASET_ROWS = 100; +const DEFAULT_MAX_DATASET_ROWS = 100; /** * Authoritative row count for a dataset. O(N), so use only on the slow @@ -77,9 +77,10 @@ export const insert = internalMutation({ typeof dataset.rowCount === "number" ? dataset.rowCount : await actualRowCount(ctx, args.datasetId); - if (previousCount >= MAX_DATASET_ROWS) { + const maxRowCount = dataset.maxRowCount ?? DEFAULT_MAX_DATASET_ROWS; + if (previousCount >= maxRowCount) { throw new Error( - `Row limit reached: BigSet datasets are capped at ${MAX_DATASET_ROWS} rows. Stop inserting rows and finish the run.`, + `Row limit reached: this BigSet dataset is capped at ${maxRowCount} rows. Stop inserting rows and finish the run.`, ); } diff --git a/frontend/convex/datasets.ts b/frontend/convex/datasets.ts index 4f34619..d295327 100644 --- a/frontend/convex/datasets.ts +++ b/frontend/convex/datasets.ts @@ -13,7 +13,7 @@ import { loadReadableDataset, requireIdentity, } from "./lib/authz.js"; -import { requireQuotaRemaining } from "./lib/quota.js"; +import { FREE_TIER_MONTHLY_QUOTA, requireQuotaRemaining } from "./lib/quota.js"; import { nextRefreshAtFor, refreshCadenceValidator, @@ -62,6 +62,19 @@ function refreshCadenceFromLegacyLabel( } const PREVIEW_ROW_COUNT = 5; +const DEFAULT_MAX_ROW_COUNT = 100; + +function validateMaxRowCount(maxRowCount: number): void { + if ( + !Number.isInteger(maxRowCount) || + maxRowCount < 1 || + maxRowCount > FREE_TIER_MONTHLY_QUOTA + ) { + throw new Error( + `Max row count must be a whole number between 1 and ${FREE_TIER_MONTHLY_QUOTA}.`, + ); + } +} async function attachPreview(ctx: QueryCtx, dataset: Doc<"datasets">) { // Mini-table preview: just the first N rows. `.take` keeps the @@ -270,6 +283,7 @@ export const claimScheduledRefreshInternal = internalMutation({ description: dataset.description, columns: dataset.columns, ownerId: dataset.ownerId, + maxRowCount: dataset.maxRowCount ?? DEFAULT_MAX_ROW_COUNT, }, }; }, @@ -360,6 +374,7 @@ export const create = mutation({ name: v.string(), description: v.string(), refreshCadence: refreshCadenceValidator, + maxRowCount: v.number(), columns: v.array(columnValidator), retrievalStrategy: v.optional( v.union( @@ -373,10 +388,11 @@ export const create = mutation({ handler: async (ctx, args) => { const identity = await requireIdentity(ctx); assertNotReservedOwner(identity.subject); + validateMaxRowCount(args.maxRowCount); // Block dataset creation at full exhaustion — a dataset you can't // populate is just clutter. Row generation later will re-check, so // this is a UX safeguard, not the only line of defense. - await requireQuotaRemaining(ctx, identity.subject, 1); + await requireQuotaRemaining(ctx, identity.subject, args.maxRowCount); return await ctx.db.insert("datasets", { ...args, @@ -408,6 +424,23 @@ export const updateRefreshSettings = mutation({ }, }); +export const updateMaxRowCount = mutation({ + args: { + id: v.id("datasets"), + maxRowCount: v.number(), + }, + handler: async (ctx, args) => { + const dataset = await loadOwnedDataset(ctx, args.id); + validateMaxRowCount(args.maxRowCount); + const currentRowCount = dataset.rowCount ?? 0; + const additionalRowsNeeded = Math.max(0, args.maxRowCount - currentRowCount); + await requireQuotaRemaining(ctx, dataset.ownerId, additionalRowsNeeded); + await ctx.db.patch(dataset._id, { + maxRowCount: args.maxRowCount, + }); + }, +}); + export const backfillRefreshSettings = internalMutation({ args: { defaultCadence: v.optional(refreshCadenceValidator), diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index d1c1888..4e7b5f4 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -50,6 +50,9 @@ export default defineSchema({ // with rows created before this field existed — write paths self-heal // on first hit, and `datasets.backfillRowCounts` migrates all at once. rowCount: v.optional(v.number()), + // User-selected target/limit for populate runs. Optional so existing + // datasets keep the legacy 100-row behavior until touched. + maxRowCount: v.optional(v.number()), columns: v.array( v.object({ name: v.string(), diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index a4f316d..f023a35 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -211,6 +211,7 @@ export async function populate( datasetId: string, datasetName: string, description: string, + maxRowCount: number, columns: PopulateColumn[], token: string, ): Promise { @@ -220,7 +221,7 @@ export async function populate( "Content-Type": "application/json", Authorization: `Bearer ${token}`, }, - body: JSON.stringify({ datasetId, datasetName: datasetName, description, columns }), + body: JSON.stringify({ datasetId, datasetName, description, maxRowCount, columns }), }); if (!res.ok) { From b1a4a901557c5253b0e52caa45bc56e3d2ae3669 Mon Sep 17 00:00:00 2001 From: Simantak Dabhade <67303107+simantak-dabhade@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:10:58 -0700 Subject: [PATCH 08/10] Update README.md (#132) --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index e0c89e7..f63efee 100644 --- a/README.md +++ b/README.md @@ -269,8 +269,24 @@ We'd love your feedback, ideas, or help building — come say hi: - 🗣 **Twitter:** [@not_simantak](https://x.com/not_simantak) for the unfiltered version - 🐛 **GitHub Issues:** [Report bugs or request features](https://github.com/tinyfish-io/bigset/issues) + +## Star History + +

+ + Star History Chart + +

+ + ## 🤝 Contributing + + + + +^ This awesome team is behing BigSet! We'd love to have you on board :) + Contributions are very welcome — whether it's code, feedback, or just telling us what datasets you'd want to build. 1. Fork the repo From 52a47e46cc28217c74118642b15e1aa36ef7ca3e Mon Sep 17 00:00:00 2001 From: siddarth Date: Sat, 6 Jun 2026 19:23:46 +0530 Subject: [PATCH 09/10] refactor: centralize .env reads in env_value helper Replace scattered cut/tr patterns with a single env_value() shell function defined inline in each recipe. All .env reads are now CRLF-safe by default, not just the Convex-specific ones. --- makefiles/Makefile | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/makefiles/Makefile b/makefiles/Makefile index b8a6cbb..5fc8075 100644 --- a/makefiles/Makefile +++ b/makefiles/Makefile @@ -31,10 +31,11 @@ dev: validate-dev-env install-deps validate-dev-env: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } - @missing=""; \ + @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ + missing=""; \ check_env() { \ key="$$1"; placeholder="$$2"; \ - value="$$(grep "^$$key=" .env | cut -d= -f2-)"; \ + value="$$(env_value "$$key")"; \ if [[ -z "$$value" || "$$value" == "$$placeholder" || "$$value" == *"..."* ]]; then \ missing="$$missing $$key"; \ fi; \ @@ -84,7 +85,12 @@ validate-dev-env: fi ensure-admin-key: +<<<<<<< Updated upstream @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ +======= + @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ + admin_key="$$(env_value CONVEX_SELF_HOSTED_ADMIN_KEY)"; \ +>>>>>>> Stashed changes needs_gen=false; \ if [[ -z "$$admin_key" ]]; then \ needs_gen=true; \ @@ -115,8 +121,14 @@ ensure-admin-key: convex-env: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } +<<<<<<< Updated upstream @issuer="$$(grep '^CLERK_JWT_ISSUER_DOMAIN=' .env | cut -d= -f2- | tr -d '\r')"; \ admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ +======= + @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ + issuer="$$(env_value CLERK_JWT_ISSUER_DOMAIN)"; \ + admin_key="$$(env_value CONVEX_SELF_HOSTED_ADMIN_KEY)"; \ +>>>>>>> Stashed changes if [[ -z "$$issuer" || "$$issuer" == "https://your-app.clerk.accounts.dev" ]]; then \ echo "Error: CLERK_JWT_ISSUER_DOMAIN must be your Clerk issuer URL in root .env"; \ exit 1; \ @@ -132,7 +144,12 @@ convex-env: convex-push: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } +<<<<<<< Updated upstream @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ +======= + @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ + admin_key="$$(env_value CONVEX_SELF_HOSTED_ADMIN_KEY)"; \ +>>>>>>> Stashed changes if [[ -z "$$admin_key" ]]; then \ echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY is missing in root .env"; \ echo "Run make dev to generate it automatically."; \ From 7baad01f2b8f45e142547496a90eecdf3829bd0c Mon Sep 17 00:00:00 2001 From: siddarth Date: Sat, 6 Jun 2026 19:37:28 +0530 Subject: [PATCH 10/10] fix: resolve merge conflicts in Makefile --- makefiles/Makefile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/makefiles/Makefile b/makefiles/Makefile index 5fc8075..2eb758a 100644 --- a/makefiles/Makefile +++ b/makefiles/Makefile @@ -85,12 +85,8 @@ validate-dev-env: fi ensure-admin-key: -<<<<<<< Updated upstream - @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ -======= @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ admin_key="$$(env_value CONVEX_SELF_HOSTED_ADMIN_KEY)"; \ ->>>>>>> Stashed changes needs_gen=false; \ if [[ -z "$$admin_key" ]]; then \ needs_gen=true; \ @@ -121,14 +117,9 @@ ensure-admin-key: convex-env: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } -<<<<<<< Updated upstream - @issuer="$$(grep '^CLERK_JWT_ISSUER_DOMAIN=' .env | cut -d= -f2- | tr -d '\r')"; \ - admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ -======= @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ issuer="$$(env_value CLERK_JWT_ISSUER_DOMAIN)"; \ admin_key="$$(env_value CONVEX_SELF_HOSTED_ADMIN_KEY)"; \ ->>>>>>> Stashed changes if [[ -z "$$issuer" || "$$issuer" == "https://your-app.clerk.accounts.dev" ]]; then \ echo "Error: CLERK_JWT_ISSUER_DOMAIN must be your Clerk issuer URL in root .env"; \ exit 1; \ @@ -144,12 +135,8 @@ convex-env: convex-push: @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } -<<<<<<< Updated upstream - @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2- | tr -d '\r')"; \ -======= @env_value() { grep "^$$1=" .env | tail -n1 | cut -d= -f2- | tr -d '\r'; }; \ admin_key="$$(env_value CONVEX_SELF_HOSTED_ADMIN_KEY)"; \ ->>>>>>> Stashed changes if [[ -z "$$admin_key" ]]; then \ echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY is missing in root .env"; \ echo "Run make dev to generate it automatically."; \