fix: resolve Makefile merge and centralize .env CRLF handling

.github/workflows/secrets-scanner.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
         with:
           fetch-depth: 0
       - name: TruffleHog OSS

README.md

@@ -269,8 +269,24 @@ We'd love your feedback, ideas, or help building — come say hi:
 - 🗣 **Twitter:** [@not_simantak](https://x.com/not_simantak) for the unfiltered version
 - 🐛 **GitHub Issues:** [Report bugs or request features](https://github.com/tinyfish-io/bigset/issues)
 
+
+## Star History
+
+<p align="center">
+  <a href="https://www.star-history.com/#tinyfish-io/bigset&type=date">
+    <img src="https://api.star-history.com/svg?repos=tinyfish-io/bigset&type=date&legend=top-left" alt="Star History Chart">
+  </a>
+</p>
+
+
 ## 🤝 Contributing
 
+<a href="https://github.com/tinyfish-io/bigset/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=tinyfish-io/bigset" />
+</a>
+
+^ This awesome team is behing BigSet! We'd love to have you on board :) 
+
 Contributions are very welcome — whether it's code, feedback, or just telling us what datasets you'd want to build.
 
 1. Fork the repo

backend/src/abort-registry.ts

@@ -0,0 +1,53 @@
+/**
+ * Module-level registry mapping datasetId → AbortController.
+ *
+ * Allows the /stop HTTP route to cancel an in-flight populate or update
+ * workflow. The AbortSignal is retrieved inside Mastra workflow steps
+ * (which receive no signal parameter from the framework) via `getSignal()`
+ * and passed explicitly to each `agent.generate()` call.
+ *
+ * Keyed by datasetId because:
+ *   - The /stop route knows the datasetId (from the request body).
+ *   - Convex's atomic claim guarantees at most one active run per dataset,
+ *     so datasetId uniquely identifies the in-flight run.
+ *   - Workflow steps have authorizedDatasetId in their inputData — no
+ *     separate workflowRunId lookup needed.
+ *
+ * Design notes:
+ *   - `abortDataset()` fires the signal but does NOT remove the entry.
+ *     The background runner's `finally` block calls `deregisterDataset()`
+ *     so the catch block can still read `controller.signal.aborted` to
+ *     distinguish a user stop from a genuine failure.
+ *   - All operations are synchronous and safe within a single Node.js process.
+ */
+
+const controllers = new Map<string, AbortController>();
+
+/** Register an active run for a dataset and return its AbortController. */
+export function registerDataset(datasetId: string): AbortController {
+  const controller = new AbortController();
+  controllers.set(datasetId, controller);
+  return controller;
+}
+
+/** Retrieve the AbortSignal for a dataset's active run (undefined if not running). */
+export function getSignal(datasetId: string): AbortSignal | undefined {
+  return controllers.get(datasetId)?.signal;
+}
+
+/**
+ * Fire the abort signal for a dataset's active run.
+ * Does NOT remove the entry — deregisterDataset() handles that.
+ * Returns true if a run was found and aborted; false if the dataset is idle.
+ */
+export function abortDataset(datasetId: string): boolean {
+  const controller = controllers.get(datasetId);
+  if (!controller) return false;
+  controller.abort();
+  return true;
+}
+
+/** Remove a dataset from the registry. Call in the background runner's finally block. */
+export function deregisterDataset(datasetId: string): void {
+  controllers.delete(datasetId);
+}

backend/src/index.ts

@@ -13,6 +13,7 @@ import { sendTransactionalEmail } from "./email/send.js";
 import { datasetReadyTemplate } from "./email/templates/dataset-ready.js";
 import { capture, shutdown as shutdownAnalytics } from "./analytics/posthog.js";
 import { EVENTS } from "./analytics/events.js";
+import { registerDataset, deregisterDataset, abortDataset } from "./abort-registry.js";
 
 /** Domain part of an email, for analytics (we never log full addresses). */
 function emailDomain(email: string): string {
@@ -141,6 +142,45 @@ async function sendDatasetReadyNotification({
   }
 }
 
+/**
+ * Shared stop-success path: set the dataset live, send the ready email.
+ *
+ * Called by both background runners when the user presses Stop. Populate
+ * only emails when at least one row was collected (a stopped run with 0
+ * rows is live but empty). Update always emails regardless of count.
+ */
+async function finaliseRunAsLive({
+  logger,
+  clerk,
+  datasetId,
+  authorizedUserId,
+  workflowType = "populate",
+}: {
+  logger: FastifyBaseLogger;
+  clerk: ClerkClient;
+  datasetId: string;
+  authorizedUserId: string;
+  workflowType?: "populate" | "update";
+}): Promise<void> {
+  const currentDataset = await convex.query(internal.datasets.getInternal, { id: datasetId });
+  if (!currentDataset) return;
+
+  await setDatasetPopulateStatus(datasetId, "live");
+
+  const rowCount = await convex.query(internal.datasetRows.countByDataset, { datasetId });
+  if (workflowType === "update" || rowCount > 0) {
+    await sendDatasetReadyNotification({
+      logger,
+      clerk,
+      userId: authorizedUserId,
+      datasetId,
+      datasetName: currentDataset.name,
+      rowCount,
+      workflowType,
+    });
+  }
+}
+
 async function beginDatasetUpdate(
   datasetId: string,
   ownerId: string,
@@ -172,6 +212,9 @@ async function runUpdateWorkflowInBackground({
   };
 }): Promise<void> {
   const datasetId = input.datasetId;
+  // registerDataset is called by the route handler before void-ing this
+  // function, so the registry entry is guaranteed visible the moment the
+  // 202 response is sent. No call needed here.
 
   try {
     const result = await run.start({
@@ -224,6 +267,11 @@ async function runUpdateWorkflowInBackground({
       workflowType: "update",
     });
   } catch (err) {
+    // Note: a user-triggered stop is NOT handled here. The update workflow's
+    // refreshRowsStep detects the abort internally, clears pending row
+    // statuses, and returns normally — so run.start() returns { status:
+    // "success" } and the success path above handles the live transition.
+    // This catch only fires on genuine failures.
     const lastStatusError = statusErrorMessage(err);
     logger.error({ err, datasetId }, "Update background workflow failed");
 
@@ -245,6 +293,8 @@ async function runUpdateWorkflowInBackground({
         "Failed to transition dataset status to 'failed' after update",
       );
     }
+  } finally {
+    deregisterDataset(datasetId);
   }
 }
 
@@ -266,6 +316,7 @@ async function runScheduledUpdateWorkflowInBackground({
   };
 }): Promise<void> {
   const datasetId = input.datasetId;
+  registerDataset(datasetId);
 
   try {
     const result = await run.start({
@@ -312,19 +363,23 @@ async function runScheduledUpdateWorkflowInBackground({
         "Failed to record scheduled refresh failure",
       );
     }
+  } finally {
+    deregisterDataset(datasetId);
   }
 }
 
 async function runPopulateWorkflowInBackground({
   input,
   run,
+  controller,
   authorizedUserId,
   logger,
   clerk,
   modelConfig,
 }: {
   input: DatasetContext;
   run: PopulateWorkflowRun;
+  controller: AbortController;
   authorizedUserId: string;
   logger: FastifyBaseLogger;
   clerk: ClerkClient;
@@ -389,6 +444,25 @@ async function runPopulateWorkflowInBackground({
       rowCount,
     });
   } catch (err) {
+    if (controller.signal.aborted) {
+      // User pressed Stop — treat whatever was collected as the final dataset.
+      logger.info({ datasetId }, "Populate workflow stopped by user; transitioning to live");
+      try {
+        await finaliseRunAsLive({ logger, clerk, datasetId, authorizedUserId });
+      } catch (stopErr) {
+        logger.error({ err: stopErr, datasetId }, "Failed to finalise stopped populate run; marking as failed");
+        // Ensure the dataset always leaves "building" — without this fallback,
+        // a failed finalisation leaves the dataset with no active registry entry
+        // and no way for /stop to act on it again.
+        try {
+          await setDatasetPopulateStatus(datasetId, "failed", "Workflow stopped but could not be finalised");
+        } catch (fallbackErr) {
+          logger.error({ err: fallbackErr, datasetId }, "Could not update dataset status after stop finalisation failure");
+        }
+      }
+      return;
+    }
+
     const lastStatusError = statusErrorMessage(err);
     logger.error(
       { err, datasetId },
@@ -414,6 +488,8 @@ async function runPopulateWorkflowInBackground({
         "Failed to transition dataset status to 'failed'",
       );
     }
+  } finally {
+    deregisterDataset(datasetId);
   }
 }
 
@@ -492,6 +568,7 @@ function startLocalRefreshScheduler(
             datasetId: dataset.datasetId,
             datasetName: dataset.datasetName,
             description: dataset.description,
+            maxRowCount: dataset.maxRowCount ?? 100,
             columns: dataset.columns,
           },
           run,
@@ -692,6 +769,13 @@ await fastify.register(async (instance) => {
         throw new Error(`Unexpected populate claim outcome: ${populateOutcome}`);
       }
 
+      const dataset = await convex.query(internal.datasets.getInternal, {
+        id: parsed.data.datasetId,
+      });
+      if (!dataset) {
+        return reply.code(404).send({ error: "Dataset not found" });
+      }
+
       const { getModelConfig } = await import("./config/models.js");
       const modelConfig = await getModelConfig(auth.userId);
 
@@ -704,9 +788,19 @@ await fastify.register(async (instance) => {
         return reply.code(502).send({ error: "Failed to populate dataset. Please try again." });
       }
 
+      // Register before void-ing so the abort-registry entry is visible the
+      // instant the 202 is sent, closing the TOCTOU window where a /stop
+      // arriving before registerDataset runs inside the background function
+      // would incorrectly force-transition an active run to "failed".
+      const controller = registerDataset(parsed.data.datasetId);
+
       void runPopulateWorkflowInBackground({
-        input: parsed.data,
+        input: {
+          ...parsed.data,
+          maxRowCount: dataset.maxRowCount ?? parsed.data.maxRowCount,
+        },
         run,
+        controller,
         authorizedUserId: auth.userId,
         logger: req.log,
         clerk: req.server.clerk,
@@ -772,6 +866,12 @@ await fastify.register(async (instance) => {
       const { getModelConfig } = await import("./config/models.js");
       const modelConfig = await getModelConfig(auth.userId);
 
+      // Register before void-ing so the abort-registry entry is visible the
+      // instant the 202 is sent, closing the TOCTOU window where a /stop
+      // arriving before registerDataset runs inside the background function
+      // would incorrectly force-transition an active run to "failed".
+      registerDataset(parsed.data.datasetId);
+
       void runUpdateWorkflowInBackground({
         input: parsed.data,
         run,
@@ -791,6 +891,75 @@ await fastify.register(async (instance) => {
       return reply.code(502).send({ error: "Failed to update dataset. Please try again." });
     }
   });
+
+  instance.post("/stop", async (req, reply) => {
+    const body = req.body as { datasetId?: string };
+    if (!body?.datasetId || typeof body.datasetId !== "string") {
+      return reply.code(400).send({ error: "datasetId is required" });
+    }
+
+    const auth = req.auth;
+    if (!auth) {
+      return reply.code(401).send({ error: "Authentication required" });
+    }
+
+    try {
+      const dataset = await convex.query(internal.datasets.getInternal, {
+        id: body.datasetId,
+      });
+      if (!dataset) {
+        return reply.code(404).send({ error: "Dataset not found" });
+      }
+      if (dataset.ownerId !== auth.userId) {
+        return reply.code(403).send({ error: "Not authorized to stop this dataset" });
+      }
+      if (dataset.status !== "building" && dataset.status !== "updating") {
+        return reply.code(409).send({ error: "Dataset is not currently running" });
+      }
+
+      const aborted = abortDataset(body.datasetId);
+      if (!aborted) {
+        // No registered signal despite the dataset being "building"/"updating".
+        // The normal finish path always sets a terminal status in Convex
+        // *before* calling deregisterDataset(), so if the status is still
+        // busy here, no running process owns this dataset — it was orphaned
+        // by a server restart. Force-transition to "failed" so the dataset
+        // is no longer stuck.
+        req.log.warn(
+          { datasetId: body.datasetId },
+          "Stop requested for orphaned dataset (no active run registered); forcing to failed",
+        );
+        try {
+          if (dataset.status === "updating") {
+            await convex.mutation(internal.datasetRows.clearAllPendingUpdateStatus, {
+              datasetId: body.datasetId,
+            });
+          }
+          await setDatasetPopulateStatus(
+            body.datasetId,
+            "failed",
+            "Run interrupted: server restarted while building/updating",
+          );
+        } catch (statusErr) {
+          req.log.error(
+            { err: statusErr, datasetId: body.datasetId },
+            "Failed to force-transition orphaned dataset to failed",
+          );
+        }
+        return reply.code(200).send({ success: true });
+      }
+      req.log.info({ datasetId: body.datasetId }, "Stop requested");
+
+      return reply.code(202).send({ success: true });
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (msg.includes("validator") || msg.includes("Invalid")) {
+        return reply.code(400).send({ error: "Invalid datasetId" });
+      }
+      req.log.error(err, "Stop failed");
+      return reply.code(502).send({ error: "Failed to stop dataset run. Please try again." });
+    }
+  });
 });
 
 try {