Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ If you want a completely fresh start: `make clean` then `make dev`.
| Data Collection | [TinyFish](https://www.tinyfish.ai?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2) APIs (Search, Fetch, Browser) |
| AI orchestration | [Mastra](https://mastra.ai) workflows + [Vercel AI SDK](https://sdk.vercel.ai) + [OpenRouter](https://openrouter.ai) → Claude Sonnet (schema inference + populate agent) |
| Table view | [TanStack Table](https://tanstack.com/table) + [react-window](https://github.com/bvaughn/react-window) virtualization |
| Exports | CSV (built-in) + XLSX ([SheetJS](https://sheetjs.com), dynamic-imported) |
| Exports | CSV (built-in) + XLSX ([write-excel-file](https://www.npmjs.com/package/write-excel-file), dynamic-imported) |
| Analytics | [PostHog](https://posthog.com) — events, session replay, error tracking (optional) |

## 📁 Project Structure
Expand Down
30 changes: 15 additions & 15 deletions backend/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions frontend/app/dashboard/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,18 @@ export default function DashboardPage() {
const mine = useQuery(
api.datasets.listMine,
isAuthenticated ? {} : "skip",
);
) as DatasetCardData[] | undefined;
const showCurated = !isLocalMode;
const curated = useQuery(
api.datasets.listPublic,
showCurated ? {} : "skip",
);
) as DatasetCardData[] | undefined;

// Quota limits are cloud-only. Local mode can create datasets without this gate.
const usage = useQuery(
api.quota.getMy,
!isLocalMode && isAuthenticated ? {} : "skip",
);
) as { remaining: number } | undefined;
const atLimit = !isLocalMode && usage !== undefined && usage.remaining === 0;

// Fire dashboard_viewed once per mount when both queries have resolved,
Expand All @@ -61,7 +61,7 @@ export default function DashboardPage() {

const { filteredMine, filteredCurated } = useMemo(() => {
const q = search.trim().toLowerCase();
const apply = (list: typeof mine) =>
const apply = (list: DatasetCardData[] | undefined) =>
(list ?? []).filter((ds) =>
!q ||
ds.name.toLowerCase().includes(q) ||
Expand Down Expand Up @@ -187,7 +187,7 @@ export default function DashboardPage() {
eyebrow="Yours"
heading="Datasets you own"
isLoading={mine === undefined}
datasets={filteredMine as unknown as DatasetCardData[]}
datasets={filteredMine}
emptyState={
search
? `No datasets of yours match "${search}".`
Expand All @@ -205,7 +205,7 @@ export default function DashboardPage() {
eyebrow="Curated by BigSet"
heading="Explore live datasets"
isLoading={curated === undefined}
datasets={filteredCurated as unknown as DatasetCardData[]}
datasets={filteredCurated}
emptyState={
search
? `No curated datasets match "${search}".`
Expand Down
17 changes: 13 additions & 4 deletions frontend/app/dataset/[id]/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useMutation, useQuery } from "convex/react";
import { api } from "@/convex/_generated/api";
import type { Id } from "@/convex/_generated/dataModel";
import { DatasetTable } from "@/components/table";
import { DatasetTable, type DatasetMeta, type DatasetRow } from "@/components/table";
import { useSelection } from "@/components/table/use-selection";
import { SideSheet, CellDetail } from "@/components/SideSheet";
import type { DatasetColumn } from "@/components/table/types";
Expand All @@ -23,6 +23,15 @@ import {
import type { ProfileUser } from "@/lib/profile-user";
import { useAppAuth, useAppClerk, useAppConvexAuth, useAppUser } from "@/lib/app-auth";

type DatasetDetail = DatasetMeta & {
_id: Id<"datasets">;
ownerId: string;
rowCount?: number;
maxRowCount?: number;
seedKey?: string;
visibility?: "public" | "private";
};

export default function DatasetPage() {
const params = useParams();
const { isLoading: authLoading, isAuthenticated } = useAppConvexAuth();
Expand All @@ -49,11 +58,11 @@ export default function DatasetPage() {
const dataset = useQuery(
api.datasets.get,
authLoading ? "skip" : { id: datasetId },
);
) as DatasetDetail | undefined;
const rows = useQuery(
api.datasetRows.listByDataset,
authLoading ? "skip" : { datasetId },
);
) as DatasetRow[] | undefined;
const updateRefreshSettings = useMutation(api.datasets.updateRefreshSettings);
const updateMaxRowCount = useMutation(api.datasets.updateMaxRowCount);
const usage = useQuery(
Expand Down Expand Up @@ -168,7 +177,7 @@ export default function DatasetPage() {
}

async function handleUpdate() {
if (!dataset || updating || dataset.status === "building" || dataset.status === "updating") return;
if (!dataset || !rows || updating || dataset.status === "building" || dataset.status === "updating") return;
// A new run is starting — discard any lingering stop-latch from the previous run.
setStopping(false);
setUpdating(true);
Expand Down
45 changes: 35 additions & 10 deletions frontend/app/dataset/new/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,23 @@ function mapBackendColumn(col: InferredColumn, index: number): ProposedColumn {
};
}

function getSchemaValidationError(datasetName: string, columns: ProposedColumn[]): string | null {
if (!datasetName.trim()) return "Dataset name is required.";
if (columns.length === 0) return "Add at least one column.";

const seenColumnNames = new Set<string>();
for (const column of columns) {
const normalizedName = column.name.trim().toLowerCase();
if (!normalizedName) return "Every column needs a name.";
if (seenColumnNames.has(normalizedName)) {
return `Column names must be unique. "${column.name.trim()}" is duplicated.`;
}
seenColumnNames.add(normalizedName);
}

return null;
}

function TypeSelector({ value, onChange }: { value: ColumnType; onChange: (v: ColumnType) => void }) {
return (
<div className="relative inline-flex">
Expand Down Expand Up @@ -172,6 +189,12 @@ export default function NewDatasetPage() {

async function handleConfirm() {
if (isCreating) return;
const validationError = getSchemaValidationError(datasetName, columns);
if (validationError) {
setError(validationError);
return;
}

const maxRowCount = Number(maxRowCountInput);
if (!Number.isInteger(maxRowCount) || maxRowCount < 1) {
setError("Max rows must be a whole number greater than 0.");
Expand All @@ -186,20 +209,22 @@ export default function NewDatasetPage() {
setIsCreating(true);
setError(null);
let datasetId: string;
const normalizedColumns = columns.map((column) => ({
name: column.name.trim(),
type: column.type,
description: column.description.trim() || undefined,
isPrimaryKey: column.isPrimaryKey || undefined,
}));

try {
datasetId = await createDataset({
name: datasetName,
description: prompt,
name: datasetName.trim(),
description: prompt.trim(),
refreshCadence,
maxRowCount,
columns: columns.map((c) => ({
name: c.name,
type: c.type,
description: c.description || undefined,
isPrimaryKey: c.isPrimaryKey || undefined,
})),
columns: normalizedColumns,
retrievalStrategy: retrievalStrategy ?? undefined,
sourceHint: sourceHint || undefined,
sourceHint: sourceHint.trim() || undefined,
});
} catch (err) {
const message = err instanceof Error ? err.message : "Failed to create dataset";
Expand All @@ -214,7 +239,7 @@ export default function NewDatasetPage() {
try {
track(EVENTS.DATASET_CREATED, {
datasetId,
column_count: columns.length,
column_count: normalizedColumns.length,
refreshCadence,
maxRowCount,
});
Expand Down
4 changes: 2 additions & 2 deletions frontend/app/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const PUBLIC_GRID_COUNT = 9;

export default function Home() {
const { isAuthenticated, isLoading } = useAppConvexAuth();
const publicDatasets = useQuery(api.datasets.listPublic, {});
const publicDatasets = useQuery(api.datasets.listPublic, {}) as DatasetCardData[] | undefined;

// Fire once when the landing page actually displays to an anonymous
// visitor. Skip if we'll immediately redirect them to the dashboard.
Expand Down Expand Up @@ -103,7 +103,7 @@ export default function Home() {
{shownDatasets.map((ds) => (
<DatasetCard
key={ds._id}
dataset={ds as unknown as DatasetCardData}
dataset={ds}
/>
))}
</div>
Expand Down
Loading