From f8f815b7078839e791fd22d9b33b87cb470d28cc Mon Sep 17 00:00:00 2001 From: huyplb Date: Sun, 25 Oct 2026 18:48:27 -0600 Subject: [PATCH] feat(data-migrate): apply up to 2,000 row ops, one cap shared by screen and route The browser capped Data migrate at 500 ops and the route refused more than 500, each holding its own copy. DATA_MIGRATE_MAX_OPS (2,000, in @foxschema/shared) now drives both, matching the compare window so every difference a compare run finds can be applied from the screen that found it. Past it, the Server Beam pointer stays. The restore snapshot is built from the destination rows already loaded (no key IN-list query), so 2,000 rows need no chunking; the 10 MB body limit holds them. Phase B of docs/plans/2026-10-17-data-compare-and-sql-tracer.md. Co-Authored-By: Claude Opus 5.5 --- .../sql-editor/components/DataMigrateBar.tsx | 2 +- .../sql-editor/components/ResultsPanel.tsx | 4 +-- .../sql-editor/lib/resultRowDiff.test.ts | 13 +++++----- .../features/sql-editor/lib/resultRowDiff.ts | 4 ++- .../data-migrate/data-migrate.routes.test.ts | 26 +++++++++++++++++++ .../data-migrate/data-migrate.routes.ts | 5 ++-- packages/shared/src/data-migrate.ts | 14 ++++++++++ packages/shared/src/index.ts | 1 + 8 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 packages/shared/src/data-migrate.ts diff --git a/apps/web/src/frontend/features/sql-editor/components/DataMigrateBar.tsx b/apps/web/src/frontend/features/sql-editor/components/DataMigrateBar.tsx index 6bc3744b2..f98624859 100644 --- a/apps/web/src/frontend/features/sql-editor/components/DataMigrateBar.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/DataMigrateBar.tsx @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 * * Side-by-side data migrate: key-based insert/update/delete onto a destination - * grid (≤500 ops). Larger sets toast with Server Beam instructions. + * grid (≤ DATA_MIGRATE_ROW_CAP ops, 2,000). Larger sets toast with Server Beam instructions. */ import React, { useEffect, useMemo, useState } from 'react'; import { createPortal } from 'react-dom'; diff --git a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx index 0c5204c2c..1b62a1fee 100644 --- a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx @@ -56,7 +56,7 @@ import { compareKeyAlignedGrids, type AlignRowOp, } from '@/features/sql-editor/lib/resultKeyAlign'; -import { migrateGridsAreComplete } from '@/features/sql-editor/lib/resultRowDiff'; +import { DATA_MIGRATE_ROW_CAP, migrateGridsAreComplete } from '@/features/sql-editor/lib/resultRowDiff'; import { sideBySideSectionCount } from '@/features/sql-editor/lib/resultsSections'; import { detectTriggerManagedColumns } from '@/features/sql-editor/lib/triggerManagedColumns'; import { resolvePeekKeyColumns } from '@/features/sql-editor/lib/rowDml'; @@ -1517,7 +1517,7 @@ const SideBySideStatementSection: React.FC<{ Sync column (on by default for differing rows).{' '} Sync scroll keeps all grids aligned;{' '} CSV all downloads every grid in one file. Migrate - needs Add / Edit / Delete checked (enabled when diffs exist) plus Sync rows. Cap: 500 ops — + needs Add / Edit / Delete checked (enabled when diffs exist) plus Sync rows. Cap: {DATA_MIGRATE_ROW_CAP.toLocaleString()} ops — larger sets use Server Beam.

) : null; diff --git a/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.test.ts b/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.test.ts index dc6a0c83a..96560f904 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.test.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.test.ts @@ -207,15 +207,15 @@ describe('migrateKeysSafeForMutatingOps', () => { }); describe('selectMigrateOps', () => { - it('respects checkboxes and caps at 500', () => { - const inserts = Array.from({ length: 300 }, (_, i) => ({ + it('respects checkboxes and caps at the shared limit (2,000)', () => { + const inserts = Array.from({ length: 1200 }, (_, i) => ({ op: 'insert' as const, keyLabel: `id=${i}`, sourceRow: [i], })); - const updates = Array.from({ length: 300 }, (_, i) => ({ + const updates = Array.from({ length: 900 }, (_, i) => ({ op: 'update' as const, - keyLabel: `id=${i + 1000}`, + keyLabel: `id=${i + 5000}`, sourceRow: [i], destRow: [i], })); @@ -225,7 +225,7 @@ describe('selectMigrateOps', () => { deletes: [], skippedNullKeys: 0, duplicateKeys: 0, - totalOps: 600, + totalOps: 2100, unresolved: 0, }; const selected = selectMigrateOps( @@ -233,7 +233,8 @@ describe('selectMigrateOps', () => { { insert: true, update: true, delete: false }, DATA_MIGRATE_ROW_CAP ); - expect(selected.uncappedCount).toBe(600); + expect(DATA_MIGRATE_ROW_CAP).toBe(2000); + expect(selected.uncappedCount).toBe(2100); expect(selected.truncated).toBe(true); expect(selected.ops).toHaveLength(DATA_MIGRATE_ROW_CAP); expect(selected.ops.every((o) => o.op === 'insert' || o.op === 'update')).toBe( diff --git a/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.ts b/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.ts index b3d3d9b25..cc863f686 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/resultRowDiff.ts @@ -9,9 +9,11 @@ import { resultValuesEqual, type ResultGridLike } from './resultDataDiff'; import { normalizeResultKey } from '@/features/sql-editor/lib/resultValueKey'; +import { DATA_MIGRATE_MAX_OPS } from '@foxschema/shared'; import type { PeekKeyColumn } from './rowDml'; -export const DATA_MIGRATE_ROW_CAP = 500; +/** The route refuses more than this (`DATA_MIGRATE_MAX_OPS`, shared with the server). */ +export const DATA_MIGRATE_ROW_CAP = DATA_MIGRATE_MAX_OPS; export type RowDiffOp = 'insert' | 'update' | 'delete'; diff --git a/packages/server/src/features/data-migrate/data-migrate.routes.test.ts b/packages/server/src/features/data-migrate/data-migrate.routes.test.ts index 62a1d5916..d9e58b35c 100644 --- a/packages/server/src/features/data-migrate/data-migrate.routes.test.ts +++ b/packages/server/src/features/data-migrate/data-migrate.routes.test.ts @@ -11,6 +11,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { fakeReply, routeHandler } from '../../platform/http/route-doubles'; import { createDataMigrateRoutes } from './data-migrate.routes'; import { executeDataMigrateOps } from './data-migrate-execute'; +import { DATA_MIGRATE_MAX_OPS } from '@foxschema/shared'; vi.mock('./data-migrate-execute', () => ({ executeDataMigrateOps: vi.fn(async () => ({ results: [] })) })); @@ -37,6 +38,31 @@ describe('POST /data-migrate/execute', () => { expect(executeDataMigrateOps).not.toHaveBeenCalled(); }); + it('takes up to DATA_MIGRATE_MAX_OPS (2,000) ops, and refuses one more', async () => { + const handler = routeHandler( + createDataMigrateRoutes({ + resolveRef: vi.fn().mockResolvedValue({ dialect: 'sqlite', option: {}, schema: '' }), + dataMigrateHistory: {} as never, + }), + 'POST', + '/data-migrate/execute' + ); + const ops = (n: number) => + Array.from({ length: n }, (_, i) => ({ op: 'insert', key: `id=${i}`, sql: `INSERT INTO t (id) VALUES (${i})` })); + const call = async (n: number) => { + const { reply, sent } = fakeReply(); + await handler({ body: { connectionId: 'c1', ops: ops(n) }, userId: 'u1', appRole: 'admin' } as never, reply as never); + return sent; + }; + expect(DATA_MIGRATE_MAX_OPS).toBe(2000); + const over = await call(2001); + expect(over.status).toBe(400); + expect(over.body.error).toMatch(/At most 2000 ops/); + expect(executeDataMigrateOps).not.toHaveBeenCalled(); + await call(2000); + expect(executeDataMigrateOps).toHaveBeenCalledOnce(); + }); + it('counts the statements inside a code fence', async () => { const handler = routeHandler( createDataMigrateRoutes({ diff --git a/packages/server/src/features/data-migrate/data-migrate.routes.ts b/packages/server/src/features/data-migrate/data-migrate.routes.ts index 2243d0d64..c57df73f8 100644 --- a/packages/server/src/features/data-migrate/data-migrate.routes.ts +++ b/packages/server/src/features/data-migrate/data-migrate.routes.ts @@ -14,6 +14,7 @@ import { requirePermissions, denyUnless } from '../../platform/authorization/rba import type { AuthedRequest } from '../../platform/http/types'; import type { ConnectionRef } from '../../platform/connections/resolve'; import { rateLimit } from '../../platform/guards/rate-limit'; +import { DATA_MIGRATE_MAX_OPS } from '@foxschema/shared'; import { MAX_STATEMENT_LENGTH } from '../sql-editor'; import type { Permission } from '@foxschema/shared'; import { CATEGORY_PERMISSION, DATAGRID_ACTION_PERMISSION } from '@foxschema/shared'; @@ -52,8 +53,8 @@ export function createDataMigrateRoutes(deps: DataMigrateRouteDeps): Router { sendError(res, 'invalid_input', 'ops[] is required.'); return; } - if (body.ops.length > 500) { - sendError(res, 'invalid_input', 'At most 500 ops per data migrate.'); + if (body.ops.length > DATA_MIGRATE_MAX_OPS) { + sendError(res, 'invalid_input', `At most ${DATA_MIGRATE_MAX_OPS} ops per data migrate.`); return; } // The connection first: each op is classified as its database reads it. diff --git a/packages/shared/src/data-migrate.ts b/packages/shared/src/data-migrate.ts new file mode 100644 index 000000000..66a6901f7 --- /dev/null +++ b/packages/shared/src/data-migrate.ts @@ -0,0 +1,14 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * The most row operations one Data migrate sends. The browser caps what it + * offers and the route refuses more, from this one number: the two used to + * hold their own copies (500 each), and a change to one would have left the + * screen offering what the server refused. + * + * It matches the compare window (2,000 rows per side), so every difference a + * compare run finds can be applied from the screen that found it. + */ +export const DATA_MIGRATE_MAX_OPS = 2000; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 2fd4c042e..361deb561 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -18,3 +18,4 @@ export * from './lokee-wire'; export * from './nav'; export * from './password-policy'; export * from './migration-status'; +export * from './data-migrate';