From 4cdf47d32f78e38351aea06c10a5e7d6f83f52d1 Mon Sep 17 00:00:00 2001 From: pritpatel2412 Date: Thu, 4 Jun 2026 14:46:02 +0530 Subject: [PATCH] feat: paginate runStats queries, fix verify-authz on Windows, and resolve React hook state effect warning --- .../table/use-row-change-detection.ts | 14 +++-- frontend/convex/runStats.ts | 54 +++++++++++++------ frontend/convex/schema.ts | 2 + scripts/verify-authz.sh | 19 +++++-- 4 files changed, 64 insertions(+), 25 deletions(-) diff --git a/frontend/components/table/use-row-change-detection.ts b/frontend/components/table/use-row-change-detection.ts index 49ce219..e823372 100644 --- a/frontend/components/table/use-row-change-detection.ts +++ b/frontend/components/table/use-row-change-detection.ts @@ -53,11 +53,15 @@ export function useRowChangeDetection(rows: DatasetRow[]) { prevRowsRef.current = nextMap; if (newFlashes.size > 0) { - setFlashingCells((prev) => { - const merged = new Set(prev); - for (const key of newFlashes) merged.add(key); - return merged; - }); + const updateTimer = setTimeout(() => { + setFlashingCells((prev) => { + const merged = new Set(prev); + for (const key of newFlashes) merged.add(key); + return merged; + }); + flashTimersRef.current.delete(updateTimer); + }, 0); + flashTimersRef.current.add(updateTimer); const timer = setTimeout(() => { setFlashingCells((prev) => { diff --git a/frontend/convex/runStats.ts b/frontend/convex/runStats.ts index d1165ec..c225c40 100644 --- a/frontend/convex/runStats.ts +++ b/frontend/convex/runStats.ts @@ -1,6 +1,9 @@ import { internalMutation, internalQuery } from "./_generated/server.js"; import { v } from "convex/values"; +const DEFAULT_PAGE_SIZE = 50; +const MAX_PAGE_SIZE = 200; + /** * Insert a populate-run metrics record. * @@ -68,33 +71,52 @@ export const getByWorkflowRunId = internalQuery({ }); /** - * List all runs for a dataset, newest first. - * TODO: paginate — .collect() loads all docs into memory and will degrade - * as run history grows. Add cursor-based pagination when this is exposed - * to the frontend or run counts become large. + * List runs for a dataset, newest first. + * Cursor-based pagination keeps memory bounded as run history grows. */ export const listByDataset = internalQuery({ - args: { datasetId: v.string() }, + args: { + datasetId: v.string(), + cursor: v.optional(v.string()), + limit: v.optional(v.number()), + }, handler: async (ctx, args) => { - const runs = await ctx.db + const limit = Math.min(args.limit ?? DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE); + const { page, isDone, continueCursor } = await ctx.db .query("runStats") - .withIndex("by_dataset", (q) => q.eq("datasetId", args.datasetId)) - .collect(); - return runs.sort((a, b) => b.startedAt - a.startedAt); + .withIndex("by_dataset_started_at", (q) => + q.eq("datasetId", args.datasetId), + ) + .order("desc") + .paginate({ + cursor: args.cursor ?? null, + numItems: limit, + }); + + return { runs: page, isDone, continueCursor }; }, }); /** - * List all runs for a user, newest first. - * TODO: paginate — same concern as listByDataset above. + * List runs for a user, newest first. */ export const listByUser = internalQuery({ - args: { userId: v.string() }, + args: { + userId: v.string(), + cursor: v.optional(v.string()), + limit: v.optional(v.number()), + }, handler: async (ctx, args) => { - const runs = await ctx.db + const limit = Math.min(args.limit ?? DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE); + const { page, isDone, continueCursor } = await ctx.db .query("runStats") - .withIndex("by_user", (q) => q.eq("userId", args.userId)) - .collect(); - return runs.sort((a, b) => b.startedAt - a.startedAt); + .withIndex("by_user_started_at", (q) => q.eq("userId", args.userId)) + .order("desc") + .paginate({ + cursor: args.cursor ?? null, + numItems: limit, + }); + + return { runs: page, isDone, continueCursor }; }, }); diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index d1c1888..5918710 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -170,6 +170,8 @@ export default defineSchema({ rowsUpdated: v.optional(v.number()), }) .index("by_dataset", ["datasetId"]) + .index("by_dataset_started_at", ["datasetId", "startedAt"]) .index("by_user", ["userId"]) + .index("by_user_started_at", ["userId", "startedAt"]) .index("by_workflow_run", ["workflowRunId"]), }); diff --git a/scripts/verify-authz.sh b/scripts/verify-authz.sh index b3496a1..3dd457e 100644 --- a/scripts/verify-authz.sh +++ b/scripts/verify-authz.sh @@ -6,6 +6,17 @@ # bash scripts/verify-authz.sh set -u +# Use python3 if available, fallback to python (common on Windows) +PYTHON="python3" +if ! command -v python3 &>/dev/null; then + if command -v python &>/dev/null; then + PYTHON="python" + else + echo "Error: python3 or python is required to run this script." + exit 1 + fi +fi + CONVEX="${CONVEX_URL:-http://localhost:3210}" FRONTEND="${FRONTEND_URL:-http://localhost:3500}" FAIL=0 @@ -34,11 +45,11 @@ mutation() { } assert_success() { - python3 -c "import json,sys; d=json.load(sys.stdin); print('PASS' if d.get('status')=='success' else 'FAIL: '+d.get('errorMessage','?')[:60])" + $PYTHON -c "import json,sys; d=json.load(sys.stdin); print('PASS' if d.get('status')=='success' else 'FAIL: '+d.get('errorMessage','?')[:60])" } assert_error_contains() { local needle="$1" - python3 -c "import json,sys; d=json.load(sys.stdin); print('PASS' if '$needle' in d.get('errorMessage','') else 'FAIL: '+d.get('errorMessage','?')[:80])" + $PYTHON -c "import json,sys; d=json.load(sys.stdin); print('PASS' if '$needle' in d.get('errorMessage','') else 'FAIL: '+d.get('errorMessage','?')[:80])" } echo "════════════════════════════════════════════════════════════════" @@ -47,7 +58,7 @@ echo " convex=$CONVEX frontend=$FRONTEND" echo "════════════════════════════════════════════════════════════════" PUB_ID=$(query '{"path":"datasets:listPublic","args":{},"format":"json"}' \ - | python3 -c "import json,sys; print(json.load(sys.stdin)['value'][0]['_id'])") + | $PYTHON -c "import json,sys; print(json.load(sys.stdin)['value'][0]['_id'])") if [ -z "${PUB_ID:-}" ]; then echo "No public dataset found. Seed curated data first (publicSeed:seedPublicDatasets)." exit 1 @@ -65,7 +76,7 @@ section "Anonymous WRITES — must all be rejected" run_test "anon datasets.listMine -> Not authenticated" \ "$(query '{"path":"datasets:listMine","args":{},"format":"json"}' | assert_error_contains 'Not authenticated')" run_test "anon datasets.create -> Not authenticated" \ - "$(mutation '{"path":"datasets:create","args":{"name":"x","description":"x","cadence":"daily","columns":[]},"format":"json"}' | assert_error_contains 'Not authenticated')" + "$(mutation '{"path":"datasets:create","args":{"name":"x","description":"x","refreshCadence":"daily","columns":[]},"format":"json"}' | assert_error_contains 'Not authenticated')" run_test "anon datasets.updateStatus -> Not authenticated" \ "$(mutation "{\"path\":\"datasets:updateStatus\",\"args\":{\"id\":\"$PUB_ID\",\"status\":\"paused\"},\"format\":\"json\"}" | assert_error_contains 'Not authenticated')" run_test "anon datasets.remove -> Not authenticated" \