From 99e3dec2512e894bb7a00debb386a01b59db4893 Mon Sep 17 00:00:00 2001 From: huyplb Date: Sun, 25 Oct 2026 18:42:14 -0600 Subject: [PATCH] feat(data-compare): compare 2,000 rows of both sides in key order, paged and filtered The grids page each side on its own with OFFSET, so their pages never line up and a one-sided row on a cut-short side could only be 'unresolved'. Compare 2,000 rows reads both sides again ordered by the key columns (the order goes on the outer query: /sql/execute takes orderBy, plain column names only), 2,001 rows each to learn whether more follow. - A row found on one side only is classified when the other side holds everything, or (numeric keys) when its key sorts at or before that side's last key. Text keys stay unresolved past a cut: engines sort them by their own collation. - Filter chips with counts for the whole window (Changed, Only in source, Only in target, Unresolved, Matching), browser-side pages of 50-500, and Next / Previous difference across pages. Sync boxes keep their state. - Data migrate reads the compared window; it still writes only when both sides are complete (raising its cap is Phase B). Phase A of docs/plans/2026-10-17-data-compare-and-sql-tracer.md. Co-Authored-By: Claude Opus 5.5 --- apps/e2e/scripts/run-all.mjs | 1 + apps/e2e/src/generated/test-ids.ts | 36 +++ .../tests/sql-editor-compare-window.test.ts | 162 ++++++++++ .../sql-editor/components/ResultsPanel.tsx | 283 ++++++++++++++++-- .../sql-editor/components/useCompareWindow.ts | 90 ++++++ .../sql-editor/lib/compareView.test.ts | 75 +++++ .../features/sql-editor/lib/compareView.ts | 119 ++++++++ .../sql-editor/lib/resultKeyAlign.test.ts | 38 +++ .../features/sql-editor/lib/resultKeyAlign.ts | 54 +++- apps/web/src/frontend/shared/api/sqlApi.ts | 3 + docs/testing/TEST_IDS.md | 14 +- .../src/features/sql-editor/editor.routes.ts | 14 +- .../sql-editor/sql-execute.service.test.ts | 19 ++ .../sql-editor/sql-execute.service.ts | 15 +- .../sql-editor/sql-page-wrap.service.test.ts | 37 +++ .../sql-editor/sql-page-wrap.service.ts | 52 ++++ 16 files changed, 981 insertions(+), 31 deletions(-) create mode 100644 apps/e2e/src/tests/sql-editor-compare-window.test.ts create mode 100644 apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts create mode 100644 apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts create mode 100644 apps/web/src/frontend/features/sql-editor/lib/compareView.ts diff --git a/apps/e2e/scripts/run-all.mjs b/apps/e2e/scripts/run-all.mjs index 3fcdc9739..6ca5e22c7 100644 --- a/apps/e2e/scripts/run-all.mjs +++ b/apps/e2e/scripts/run-all.mjs @@ -134,6 +134,7 @@ const ALWAYS = [ 'src/tests/sql-editor-peek-stacking.test.ts', 'src/tests/sql-editor-data-peek-grid.test.ts', 'src/tests/sql-editor-compare-multi-target.test.ts', + 'src/tests/sql-editor-compare-window.test.ts', 'src/tests/sql-editor-join-typing.test.ts', ].join(' '), label: 'SQL Editor', diff --git a/apps/e2e/src/generated/test-ids.ts b/apps/e2e/src/generated/test-ids.ts index f0e27f0ee..ad81ef3cb 100644 --- a/apps/e2e/src/generated/test-ids.ts +++ b/apps/e2e/src/generated/test-ids.ts @@ -2239,10 +2239,16 @@ export const TestIds = { sqlResultCompareClose: (statementIndex: string | number) => `sql-result-compare-close-${statementIndex}`, /** select */ sqlResultCompareDest: (statementIndex: string | number) => `sql-result-compare-dest-${statementIndex}`, + /** button · Difference › */ + sqlResultCompareDiffNext: (statementIndex: string | number) => `sql-result-compare-diff-next-${statementIndex}`, + /** button · ‹ Difference */ + sqlResultCompareDiffPrev: (statementIndex: string | number) => `sql-result-compare-diff-prev-${statementIndex}`, /** button · Export all compare grids as one CSV (columns prefixed by Source / Target) */ sqlResultCompareExportCsv: (statementIndex: string | number) => `sql-result-compare-export-csv-${statementIndex}`, /** button · Export all compare grids as one CSV */ sqlResultCompareExportCsvModal: (statementIndex: string | number) => `sql-result-compare-export-csv-modal-${statementIndex}`, + /** button */ + sqlResultCompareFilter: (f: string | number, statementIndex: string | number) => `sql-result-compare-filter-${f}-${statementIndex}`, /** p */ sqlResultCompareHint: (statementIndex: string | number) => `sql-result-compare-hint-${statementIndex}`, /** span */ @@ -2253,6 +2259,14 @@ export const TestIds = { sqlResultCompareMaximizedHint: (statementIndex: string | number) => `sql-result-compare-maximized-hint-${statementIndex}`, /** div */ sqlResultCompareModal: (statementIndex: string | number) => `sql-result-compare-modal-${statementIndex}`, + /** span */ + sqlResultComparePageLabel: (statementIndex: string | number) => `sql-result-compare-page-label-${statementIndex}`, + /** button · Next */ + sqlResultComparePageNext: (statementIndex: string | number) => `sql-result-compare-page-next-${statementIndex}`, + /** button · Prev */ + sqlResultComparePagePrev: (statementIndex: string | number) => `sql-result-compare-page-prev-${statementIndex}`, + /** select · Rows per compare page */ + sqlResultComparePageSize: (statementIndex: string | number) => `sql-result-compare-page-size-${statementIndex}`, /** input */ sqlResultCompareSkipTrigger: (statementIndex: string | number) => `sql-result-compare-skip-trigger-${statementIndex}`, /** input */ @@ -2263,6 +2277,16 @@ export const TestIds = { sqlResultCompareToggle: (statementIndex: string | number) => `sql-result-compare-toggle-${statementIndex}`, /** div */ sqlResultCompareToolbar: (statementIndex: string | number) => `sql-result-compare-toolbar-${statementIndex}`, + /** div */ + sqlResultCompareView: (statementIndex: string | number) => `sql-result-compare-view-${statementIndex}`, + /** button · Back to loaded rows */ + sqlResultCompareWindowClear: (statementIndex: string | number) => `sql-result-compare-window-clear-${statementIndex}`, + /** span */ + sqlResultCompareWindowError: (statementIndex: string | number) => `sql-result-compare-window-error-${statementIndex}`, + /** button · Compare rows */ + sqlResultCompareWindowRun: (statementIndex: string | number) => `sql-result-compare-window-run-${statementIndex}`, + /** span */ + sqlResultCompareWindowSummary: (statementIndex: string | number) => `sql-result-compare-window-summary-${statementIndex}`, /** section */ sqlResultCred: (connectionId: string | number) => `sql-result-cred-${connectionId}`, /** button · Close maximized results */ @@ -4823,18 +4847,30 @@ export type TestId = | `sql-result-compare-baseline-${string}` | `sql-result-compare-close-${string}` | `sql-result-compare-dest-${string}` + | `sql-result-compare-diff-next-${string}` + | `sql-result-compare-diff-prev-${string}` | `sql-result-compare-export-csv-${string}` | `sql-result-compare-export-csv-modal-${string}` + | `sql-result-compare-filter-${string}-${string}` | `sql-result-compare-hint-${string}` | `sql-result-compare-legend-${string}` | `sql-result-compare-maximize-${string}` | `sql-result-compare-maximized-hint-${string}` | `sql-result-compare-modal-${string}` + | `sql-result-compare-page-label-${string}` + | `sql-result-compare-page-next-${string}` + | `sql-result-compare-page-prev-${string}` + | `sql-result-compare-page-size-${string}` | `sql-result-compare-skip-trigger-${string}` | `sql-result-compare-sync-scroll-${string}` | `sql-result-compare-sync-scroll-modal-${string}` | `sql-result-compare-toggle-${string}` | `sql-result-compare-toolbar-${string}` + | `sql-result-compare-view-${string}` + | `sql-result-compare-window-clear-${string}` + | `sql-result-compare-window-error-${string}` + | `sql-result-compare-window-run-${string}` + | `sql-result-compare-window-summary-${string}` | `sql-result-cred-${string}` | `sql-result-cred-close-${string}` | `sql-result-cred-maximize-${string}` diff --git a/apps/e2e/src/tests/sql-editor-compare-window.test.ts b/apps/e2e/src/tests/sql-editor-compare-window.test.ts new file mode 100644 index 000000000..59944e76d --- /dev/null +++ b/apps/e2e/src/tests/sql-editor-compare-window.test.ts @@ -0,0 +1,162 @@ +/** + * Compare 2,000 rows: both sides read again in key order, every row inside the + * window classified, and the result paged and filtered in the browser. + * + * The grids page each side on its own with OFFSET, so their pages do not line + * up and a one-sided row on a cut-short side can only be "unresolved". A + * compare run reads the first 2,000 rows of both sides by key instead. + * + * Fixture (every count below follows from it): + * source ids 1–2,500, name n + * target the same, without 7, 1,500 and 2,300; names changed at 10, 20 and + * 1,990; plus 2,600 and 2,601 + * In the window, source ends at id 2,000 and target at 2,002, so: + * Changed 3 · Only in source 2 (7, 1,500) · Only in target 0 · + * Unresolved 2 (2,001 and 2,002, past the source's last key) · + * Matching 1,995 · All 2,002 + * + * Two SQLite files under a per-run directory. Requires the web app + API and + * a `sqlite3` CLI on PATH. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execFileSync, execSync } from 'node:child_process'; +import { mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Page } from 'playwright'; +import { buildDriver, quitDriver } from '../helpers/driver.js'; +import { saveScreenshot } from '../helpers/screenshot.js'; +import { byTestId } from '../helpers/test-ids.js'; +import { AppPage } from '../pages/AppPage.js'; +import { SqlEditorPage } from '../pages/SqlEditorPage.js'; + +const RUN = Date.now().toString(36); +const DIR = `/tmp/foxschema-e2e-compare-window-${RUN}`; +const SOURCE = { file: join(DIR, 'source.db'), name: `E2E Window Src ${RUN}` }; +const TARGET = { file: join(DIR, 'target.db'), name: `E2E Window Tgt ${RUN}` }; + +function hasSqlite3(): boolean { + try { + execSync('which sqlite3', { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function seed(path: string, target: boolean): void { + const missing = new Set([7, 1500, 2300]); + const changed = new Set([10, 20, 1990]); + const rows: string[] = []; + for (let id = 1; id <= 2500; id++) { + if (target && missing.has(id)) continue; + rows.push(`(${id}, '${target && changed.has(id) ? `changed${id}` : `n${id}`}')`); + } + if (target) rows.push("(2600, 'n2600')", "(2601, 'n2601')"); + execFileSync('sqlite3', [path], { + input: `CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO items (id, name) VALUES ${rows.join(',')};`, + }); +} + +describe.skipIf(!hasSqlite3())('SQL Editor · Compare 2,000 rows', () => { + let driver: Page; + let sql: SqlEditorPage; + const label = (id: string) => driver.locator(byTestId(id as never)).innerText(); + + beforeAll(async () => { + rmSync(DIR, { recursive: true, force: true }); + mkdirSync(DIR, { recursive: true }); + seed(SOURCE.file, false); + seed(TARGET.file, true); + + driver = await buildDriver(); + sql = new SqlEditorPage(driver); + await new AppPage(driver).open(); + await sql.resetPersistedEditorState(); + await driver.reload(); + await driver.waitForSelector(byTestId('toolbar')); + await sql.addSqliteCredential(SOURCE.name, SOURCE.file); + await sql.addSqliteCredential(TARGET.name, TARGET.file); + }, 180_000); + + afterAll(async () => { + if (driver) await quitDriver(driver); + rmSync(DIR, { recursive: true, force: true }); + }); + + it('compares the first 2,000 rows of both sides by key, classifying every row the window can decide', async () => { + await sql.openView(); + await sql.checkConnection(SOURCE.name); + await sql.checkConnection(TARGET.name); + await sql.setSql('SELECT id, name FROM items;'); + await sql.run(); + await sql.waitForResults(); + await sql.setLayoutSideBySide(); + await sql.compareToggle(0).click(); + + // The source is the first destination checked; the target, the other. + const baseline = await driver.locator(byTestId('sql-result-compare-baseline-0')).inputValue(); + const sourceValue = await driver.evaluate( + (name) => + [...(document.querySelector('[data-testid="sql-result-compare-baseline-0"]') as HTMLSelectElement).options].find( + (o) => o.textContent?.includes(name) + )?.value, + SOURCE.name + ); + if (sourceValue && sourceValue !== baseline) { + await driver.locator(byTestId('sql-result-compare-baseline-0')).selectOption(sourceValue); + } + + await driver.locator(byTestId('sql-result-compare-window-run-0')).click(); + await driver.waitForSelector(byTestId('sql-result-compare-window-summary-0'), { timeout: 60_000 }); + expect(await label('sql-result-compare-window-summary-0')).toMatch( + /Compared 2,000 \(more follow\) source and 2,000 \(more follow\) target rows, ordered by id/ + ); + + const count = async (f: string) => { + const text = await label(`sql-result-compare-filter-${f}-0`); + return Number(text.replace(/[^\d]/g, '')); + }; + expect({ + changed: await count('changed'), + source: await count('source'), + target: await count('target'), + unresolved: await count('unresolved'), + matching: await count('matching'), + all: await count('all'), + }).toEqual({ changed: 3, source: 2, target: 0, unresolved: 2, matching: 1995, all: 2002 }); + await saveScreenshot(driver, 'compare-window-summary'); + }, 180_000); + + it('filters to the changed rows and pages the rest in the browser', async () => { + await driver.locator(byTestId('sql-result-compare-filter-changed-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 1 of 1'); + const keys = await driver.evaluate(() => + [...document.querySelectorAll('[data-testid="sql-data-grid"]')][0]! + .querySelectorAll('tbody tr [data-testid="sql-row-num"]').length + ); + expect(keys).toBe(3); + + await driver.locator(byTestId('sql-result-compare-filter-all-0')).click(); + await driver.locator(byTestId('sql-result-compare-page-size-0')).selectOption('100'); + expect(await label('sql-result-compare-page-label-0')).toBe('page 1 of 21'); + }, 60_000); + + it('steps from one difference to the next across pages of matching rows', async () => { + // Differences sit at ids 7, 10 and 20 (page 1), 1,500 (page 15) and 1,990 (page 20). + await driver.locator(byTestId('sql-result-compare-diff-next-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 15 of 21'); + await driver.locator(byTestId('sql-result-compare-diff-next-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 20 of 21'); + expect(await driver.locator(byTestId('sql-result-compare-diff-next-0')).isDisabled()).toBe(true); + await driver.locator(byTestId('sql-result-compare-diff-prev-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 15 of 21'); + await saveScreenshot(driver, 'compare-window-difference'); + }, 60_000); + + it('goes back to the loaded pages, where a cut-short side leaves one-sided rows unresolved', async () => { + await driver.locator(byTestId('sql-result-compare-window-clear-0')).click(); + await driver.waitForSelector(byTestId('sql-result-compare-window-run-0')); + expect(await driver.locator(byTestId('sql-result-compare-window-summary-0')).count()).toBe(0); + }, 60_000); +}); diff --git a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx index 0c5204c2c..b5bec7b7e 100644 --- a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx @@ -42,6 +42,7 @@ import { import { attributeResultColumns, collapsedColumnsFor, + isSubsetDialect, rowKeyFor, tablesInOrigins, } from '@foxschema/ui-shared'; @@ -57,6 +58,19 @@ import { type AlignRowOp, } from '@/features/sql-editor/lib/resultKeyAlign'; import { migrateGridsAreComplete } from '@/features/sql-editor/lib/resultRowDiff'; +import { + COMPARE_PAGE_SIZES, + COMPARE_WINDOW_ROWS, + DEFAULT_COMPARE_PAGE_SIZE, + compareFilterCounts, + comparePage, + filteredRowIndexes, + lastRowKey, + pageOfNextDifference, + sliceAligned, + type CompareFilter, +} from '@/features/sql-editor/lib/compareView'; +import { useCompareWindow } from './useCompareWindow'; import { sideBySideSectionCount } from '@/features/sql-editor/lib/resultsSections'; import { detectTriggerManagedColumns } from '@/features/sql-editor/lib/triggerManagedColumns'; import { resolvePeekKeyColumns } from '@/features/sql-editor/lib/rowDml'; @@ -1027,6 +1041,15 @@ const SideBySideStatementSection: React.FC<{ const sourceGrid = okGrids.find((g) => g.connectionId === baselineId); const destGrid = okGrids.find((g) => g.connectionId === destId); + /** A new run of the statement makes any compare window describe old rows. */ + const windowResetKey = okGrids + .map((g) => `${g.connectionId}:${g.result.ok ? `${g.result.durationMs}:${g.result.rowCount}` : 'x'}`) + .join(','); + const compareWindow = useCompareWindow(`${statementIndex}|${windowResetKey}`); + const [compareFilter, setCompareFilter] = useState('all'); + const [comparePageIndex, setComparePageIndex] = useState(0); + const [comparePageSize, setComparePageSize] = useState(DEFAULT_COMPARE_PAGE_SIZE); + const defaultKeys = useMemo(() => { if (!sourceGrid?.result.ok) return [] as string[]; const cols = sourceGrid.result.columns; @@ -1118,6 +1141,17 @@ const SideBySideStatementSection: React.FC<{ // instead of joining it into a string. const effectiveKeys = keyNames.length ? keyNames : defaultKeys; + /** The compare window, while it still describes this pair, by these keys. */ + const windowState = compareWindow.state; + const windowOn = + compareActive && + windowState.status === 'ready' && + windowState.sourceId === baselineId && + windowState.destId === destId && + windowState.keys.join('\0') === effectiveKeys.join('\0') + ? windowState + : null; + /** * Whether each side holds its whole result: page 1, no next page, nothing * truncated. The same test Data migrate applies before it will write. @@ -1146,13 +1180,49 @@ const SideBySideStatementSection: React.FC<{ const keyAligned = useMemo(() => { if (!compareActive || !sourceGrid?.result.ok || !destGrid?.result.ok) return null; if (effectiveKeys.length === 0) return null; + if (windowOn) { + // Both windows are in key order: a side cut short at the window ends at + // a known key, and a one-sided row before it is classified after all. + const sourceMore = Boolean(windowOn.source.hasNext || windowOn.source.truncated); + const destMore = Boolean(windowOn.dest.hasNext || windowOn.dest.truncated); + return alignResultGridsByKey( + { columns: windowOn.source.columns, rows: windowOn.source.rows }, + { columns: windowOn.dest.columns, rows: windowOn.dest.rows }, + effectiveKeys, + { + ...ignoreOpts, + leftComplete: !sourceMore, + rightComplete: !destMore, + leftWindowEnd: sourceMore ? lastRowKey(windowOn.source, effectiveKeys) : undefined, + rightWindowEnd: destMore ? lastRowKey(windowOn.dest, effectiveKeys) : undefined, + } + ); + } return alignResultGridsByKey( { columns: sourceGrid.result.columns, rows: sourceGrid.result.rows }, { columns: destGrid.result.columns, rows: destGrid.result.rows }, effectiveKeys, { ...ignoreOpts, leftComplete: sourceComplete, rightComplete: destComplete } ); - }, [compareActive, sourceGrid, destGrid, effectiveKeys, ignoreOpts, sourceComplete, destComplete]); + }, [compareActive, sourceGrid, destGrid, effectiveKeys, ignoreOpts, sourceComplete, destComplete, windowOn]); + + const filterCounts = useMemo(() => (keyAligned ? compareFilterCounts(keyAligned) : null), [keyAligned]); + const filteredRows = useMemo( + () => (keyAligned ? filteredRowIndexes(keyAligned, compareFilter) : []), + [keyAligned, compareFilter] + ); + const pageView = useMemo( + () => comparePage(filteredRows, comparePageIndex, comparePageSize), + [filteredRows, comparePageIndex, comparePageSize] + ); + /** What the grids show: this page of the filtered rows. Counts stay the whole set's. */ + const viewAligned = useMemo( + () => (keyAligned ? sliceAligned(keyAligned, pageView.rows) : null), + [keyAligned, pageView] + ); + useEffect(() => { + setComparePageIndex(0); + }, [keyAligned, compareFilter, comparePageSize]); const { diffByConnection, badgeByConnection, legendBits, displayItems } = useMemo(() => { const diffByConnection: Record = {}; @@ -1184,7 +1254,7 @@ const SideBySideStatementSection: React.FC<{ const pair = compareKeyAlignedGrids( baselineGrid, destGridLike, - keyAligned, + viewAligned ?? keyAligned, ignoreOpts ); diffByConnection[baselineId] = pair.baseline; @@ -1200,7 +1270,9 @@ const SideBySideStatementSection: React.FC<{ if (keyAligned.unresolvedCount > 0) { legendBits.push( `${keyAligned.unresolvedCount} on one side only — the other side's result is cut ` + - 'off, so they may be further down; raise Rows/page to classify them' + (windowOn + ? 'off past its last key, so they may be further down' + : `off, so they may be further down; Compare ${COMPARE_WINDOW_ROWS.toLocaleString()} rows classifies them`) ); } if (keyAligned.duplicateKeys > 0) { @@ -1238,8 +1310,8 @@ const SideBySideStatementSection: React.FC<{ ...item, result: { ...item.result, - rows: keyAligned.leftRows, - rowCount: keyAligned.leftRows.length, + rows: (viewAligned ?? keyAligned).leftRows, + rowCount: (viewAligned ?? keyAligned).leftRows.length, }, }; } @@ -1248,8 +1320,8 @@ const SideBySideStatementSection: React.FC<{ ...item, result: { ...item.result, - rows: keyAligned.rightRows, - rowCount: keyAligned.rightRows.length, + rows: (viewAligned ?? keyAligned).rightRows, + rowCount: (viewAligned ?? keyAligned).rightRows.length, }, }; } @@ -1354,23 +1426,27 @@ const SideBySideStatementSection: React.FC<{ baselineId, destId, keyAligned, + viewAligned, + windowOn, ignoreOpts, triggerIgnoreColumns, ]); const rowSyncByConnection = useMemo(() => { - if (!compareActive || !keyAligned || !destId) return undefined; + // By the grids' own row index, so from the page being shown. + const shown = viewAligned ?? keyAligned; + if (!compareActive || !shown || !destId) return undefined; return { [destId]: { isChecked: (rowIdx: number): boolean | null => { - const op = keyAligned.rowOps[rowIdx]; + const op = shown.rowOps[rowIdx]; if (op === 'match' || op === 'unresolved') return null; - const label = keyAligned.rowKeyLabels[rowIdx]; + const label = shown.rowKeyLabels[rowIdx]; if (!label) return false; return selectedSyncKeys.has(label); }, onToggle: (rowIdx: number, checked: boolean) => { - const label = keyAligned.rowKeyLabels[rowIdx]; + const label = shown.rowKeyLabels[rowIdx]; if (!label) return; setSelectedSyncKeys((prev) => { const next = new Set(prev); @@ -1381,7 +1457,7 @@ const SideBySideStatementSection: React.FC<{ }, }, }; - }, [compareActive, keyAligned, destId, selectedSyncKeys]); + }, [compareActive, keyAligned, viewAligned, destId, selectedSyncKeys]); const insertServerBeamSample = () => { const sample = buildSampleBookmarks().find((b) => b.id === 'sample-server-beam-chunked'); @@ -1475,27 +1551,34 @@ const SideBySideStatementSection: React.FC<{ compareActive && sourceGrid?.result.ok && destGrid?.result.ok ? ( ) : null; + const windowSupported = Boolean( + sourceGrid?.statementSql && + destGrid?.statementSql && + !isSubsetDialect(sourceGrid.dialect) && + !isSubsetDialect(destGrid.dialect) && + effectiveKeys.length > 0 + ); + const runCompareWindow = () => { + if (!sourceGrid?.statementSql || !destGrid?.statementSql) return; + void compareWindow.run( + { connectionId: sourceGrid.connectionId, sql: sourceGrid.statementSql }, + { connectionId: destGrid.connectionId, sql: destGrid.statementSql }, + effectiveKeys + ); + }; + const rowsLabel = (r: { rows: unknown[] ; hasNext?: boolean; truncated?: boolean }) => + `${r.rows.length.toLocaleString()}${r.hasNext || r.truncated ? ' (more follow)' : ''}`; + const FILTER_LABELS: Record = { + all: 'All', + changed: 'Changed', + source: 'Only in source', + target: 'Only in target', + unresolved: 'Unresolved', + matching: 'Matching', + }; + const nextDiffPage = keyAligned + ? pageOfNextDifference(keyAligned, filteredRows, pageView.page, comparePageSize, 1) + : null; + const prevDiffPage = keyAligned + ? pageOfNextDifference(keyAligned, filteredRows, pageView.page, comparePageSize, -1) + : null; + const navBtn = + 'rounded-md border border-slate-700 px-2 py-0.5 text-slate-200 hover:border-sky-500/40 disabled:opacity-40 disabled:hover:border-slate-700'; + + const compareViewBar = + compareActive && keyAligned && filterCounts ? ( +
+ {windowOn ? ( + <> + + Compared {rowsLabel(windowOn.source)} source and {rowsLabel(windowOn.dest)} target rows, ordered by{' '} + {windowOn.keys.join(', ')} + + + + ) : ( + + )} + {windowState.status === 'error' && ( + + {windowState.error} + + )} + + {(Object.keys(FILTER_LABELS) as CompareFilter[]).map((f) => ( + + ))} + + + + + + page {pageView.page + 1} of {pageView.pageCount} + + + + + +
+ ) : null; + const compareHint = compareActive ? (

{migrateBar} + {compareViewBar} {compareGrids(false)} {compareHint} @@ -1774,6 +2012,7 @@ const SideBySideStatementSection: React.FC<{

{migrateBar} + {compareViewBar}
{compareGrids(true)}
{compareHint}
diff --git a/apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts b/apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts new file mode 100644 index 000000000..246fc8e60 --- /dev/null +++ b/apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts @@ -0,0 +1,90 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * A compare run: the statement read again on the source and the target, in + * key order, COMPARE_WINDOW_ROWS (+1) rows each. The pages a grid shows are + * read independently with OFFSET and do not line up, so comparing them leaves + * most one-sided rows unresolved; two windows in the same key order do. + * + * Held here, beside the pane that asked, not in the editor store: the grids' + * own pages stay as they are, and a new run of the statement drops it. + */ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { executeSql, type SqlStatementResult } from '@/shared/api/sqlApi'; +import { getSessionPassword } from '@/shared/lib/sessionPasswords'; +import { useSqlEditorStore } from '@/features/sql-editor/state/useSqlEditorStore'; +import { COMPARE_WINDOW_ROWS } from '@/features/sql-editor/lib/compareView'; + +type OkResult = Extract; + +export type CompareWindowState = + | { status: 'idle' } + | { status: 'loading'; keys: string[] } + | { status: 'ready'; keys: string[]; sourceId: string; destId: string; source: OkResult; dest: OkResult } + | { status: 'error'; keys: string[]; error: string }; + +export interface CompareWindowSide { + connectionId: string; + sql: string; +} + +export function useCompareWindow( + /** Changes when the statement runs again: the windows then describe old results. */ + resetKey: unknown +): { + state: CompareWindowState; + run: (source: CompareWindowSide, dest: CompareWindowSide, keys: string[]) => Promise; + clear: () => void; +} { + const [state, setState] = useState({ status: 'idle' }); + const sessionPasswords = useSqlEditorStore((s) => s.sessionPasswords); + /** Which run is current: a slow older one must not overwrite a newer answer. */ + const runId = useRef(0); + + useEffect(() => { + runId.current += 1; + setState({ status: 'idle' }); + }, [resetKey]); + + const run = useCallback( + async (source: CompareWindowSide, dest: CompareWindowSide, keys: string[]) => { + const id = ++runId.current; + setState({ status: 'loading', keys }); + const read = async (side: CompareWindowSide): Promise => { + const password = getSessionPassword(side.connectionId) || sessionPasswords[side.connectionId] || undefined; + const { results } = await executeSql( + { connectionId: side.connectionId, password }, + [side.sql], + COMPARE_WINDOW_ROWS, + 0, + undefined, + { orderBy: keys } + ); + const one = results[0]; + if (!one) throw new Error('No result returned'); + if (!one.ok) throw new Error(one.error); + return one; + }; + try { + const [s, d] = await Promise.all([read(source), read(dest)]); + if (id === runId.current) { + setState({ status: 'ready', keys, sourceId: source.connectionId, destId: dest.connectionId, source: s, dest: d }); + } + } catch (error: unknown) { + if (id === runId.current) { + setState({ status: 'error', keys, error: error instanceof Error ? error.message : String(error) }); + } + } + }, + [sessionPasswords] + ); + + const clear = useCallback(() => { + runId.current += 1; + setState({ status: 'idle' }); + }, []); + + return { state, run, clear }; +} diff --git a/apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts b/apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts new file mode 100644 index 000000000..68f336c81 --- /dev/null +++ b/apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts @@ -0,0 +1,75 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + */ +import { describe, expect, it } from 'vitest'; +import type { AlignRowOp, KeyAlignedGrids } from './resultKeyAlign'; +import { + compareFilterCounts, + comparePage, + filteredRowIndexes, + lastRowKey, + pageOfNextDifference, + sliceAligned, +} from './compareView'; + +const aligned = (ops: AlignRowOp[]): KeyAlignedGrids => ({ + keyNames: ['id'], + leftRows: ops.map((_, i) => [i]), + rightRows: ops.map((_, i) => [i]), + leftGap: ops.map((op) => op === 'insert'), + rightGap: ops.map((op) => op === 'delete'), + rowOps: ops, + rowKeyLabels: ops.map((_, i) => `id=${i}`), + matchCount: 0, + updateCount: 0, + insertCount: 0, + deleteCount: 0, + unresolvedCount: 0, + duplicateKeys: 0, +}); + +describe('compare view', () => { + it('counts each filter over the whole window', () => { + expect(compareFilterCounts(aligned(['match', 'update', 'delete', 'insert', 'unresolved', 'match']))).toEqual({ + all: 6, + changed: 1, + source: 1, + target: 1, + unresolved: 1, + matching: 2, + }); + }); + + it('pages the filtered rows, clamping the page into range', () => { + const rows = filteredRowIndexes(aligned(['match', 'update', 'match', 'update', 'update']), 'changed'); + expect(rows).toEqual([1, 3, 4]); + expect(comparePage(rows, 0, 2)).toEqual({ rows: [1, 3], page: 0, pageCount: 2 }); + expect(comparePage(rows, 9, 2)).toEqual({ rows: [4], page: 1, pageCount: 2 }); + expect(comparePage([], 0, 100)).toEqual({ rows: [], page: 0, pageCount: 1 }); + }); + + it('steps to the page holding the next or previous difference, skipping pages of matches', () => { + const ops: AlignRowOp[] = ['update', ...Array(9).fill('match'), 'insert', 'match']; + const a = aligned(ops); + const rows = filteredRowIndexes(a, 'all'); + expect(pageOfNextDifference(a, rows, 0, 3, 1)).toBe(3); // row 10, page 3 + expect(pageOfNextDifference(a, rows, 3, 3, 1)).toBeNull(); + expect(pageOfNextDifference(a, rows, 3, 3, -1)).toBe(0); + }); + + it('slices the aligned pair to the page, keeping the key labels the Sync boxes use', () => { + const view = sliceAligned(aligned(['match', 'update', 'delete']), [2, 1]); + expect(view.rowOps).toEqual(['delete', 'update']); + expect(view.rowKeyLabels).toEqual(['id=2', 'id=1']); + expect(view.rightGap).toEqual([true, false]); + }); + + it('reads the last row’s key, case-insensitively, or nothing when a key column is missing', () => { + const result = { columns: ['ID', 'name'], rows: [[1, 'a'], [7, 'b']] }; + expect(lastRowKey(result, ['id'])).toEqual([7]); + expect(lastRowKey(result, ['missing'])).toBeUndefined(); + expect(lastRowKey({ columns: ['id'], rows: [] }, ['id'])).toBeUndefined(); + }); +}); diff --git a/apps/web/src/frontend/features/sql-editor/lib/compareView.ts b/apps/web/src/frontend/features/sql-editor/lib/compareView.ts new file mode 100644 index 000000000..f7ab31a6e --- /dev/null +++ b/apps/web/src/frontend/features/sql-editor/lib/compareView.ts @@ -0,0 +1,119 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * The compare view's filters and pages, over a key-aligned pair already in + * memory. A compare run holds at most two windows of COMPARE_WINDOW_ROWS, so + * paging is done here, never by asking the databases again. + */ +import type { AlignRowOp, KeyAlignedGrids } from './resultKeyAlign'; + +/** Rows per side a compare run reads: a fixed window, not a setting. */ +export const COMPARE_WINDOW_ROWS = 2000; + +export const COMPARE_PAGE_SIZES = [50, 100, 200, 500] as const; +export const DEFAULT_COMPARE_PAGE_SIZE = 100; + +/** + * What a filter keeps, in migrate's words: a source-only row is one migrate + * adds to the target, a target-only row one it deletes. + */ +export type CompareFilter = 'all' | 'changed' | 'source' | 'target' | 'unresolved' | 'matching'; + +const KEEPS: Record boolean> = { + all: () => true, + changed: (op) => op === 'update', + source: (op) => op === 'delete', + target: (op) => op === 'insert', + unresolved: (op) => op === 'unresolved', + matching: (op) => op === 'match', +}; + +const isDifference = (op: AlignRowOp): boolean => op === 'update' || op === 'insert' || op === 'delete'; + +/** How many rows each filter keeps, over the whole window. */ +export function compareFilterCounts(aligned: Pick): Record { + const counts = { all: 0, changed: 0, source: 0, target: 0, unresolved: 0, matching: 0 }; + for (const op of aligned.rowOps) { + for (const f of Object.keys(KEEPS) as CompareFilter[]) if (KEEPS[f](op)) counts[f] += 1; + } + return counts; +} + +/** Aligned row indexes the filter keeps, in aligned order. */ +export function filteredRowIndexes(aligned: Pick, filter: CompareFilter): number[] { + const keep = KEEPS[filter]; + const out: number[] = []; + aligned.rowOps.forEach((op, i) => { + if (keep(op)) out.push(i); + }); + return out; +} + +export interface ComparePage { + /** Aligned row indexes on this page. */ + rows: number[]; + /** Zero-based, clamped into range. */ + page: number; + pageCount: number; +} + +export function comparePage(rows: readonly number[], page: number, size: number): ComparePage { + const pageCount = Math.max(1, Math.ceil(rows.length / size)); + const p = Math.min(Math.max(0, page), pageCount - 1); + return { rows: rows.slice(p * size, p * size + size), page: p, pageCount }; +} + +/** + * The page holding the next (or previous) difference beyond this page, among + * the filtered rows; null when there is none that way. + */ +export function pageOfNextDifference( + aligned: Pick, + rows: readonly number[], + page: number, + size: number, + direction: 1 | -1 +): number | null { + if (direction === 1) { + for (let pos = (page + 1) * size; pos < rows.length; pos++) { + if (isDifference(aligned.rowOps[rows[pos]!]!)) return Math.floor(pos / size); + } + return null; + } + for (let pos = page * size - 1; pos >= 0; pos--) { + if (isDifference(aligned.rowOps[rows[pos]!]!)) return Math.floor(pos / size); + } + return null; +} + +/** + * The aligned pair cut down to `rows`, for display: tints, Sync checkboxes and + * the grids all read it by its own row index. Counts stay the whole window's. + */ +export function sliceAligned(aligned: KeyAlignedGrids, rows: readonly number[]): KeyAlignedGrids { + const pick = (xs: T[]) => rows.map((i) => xs[i]!); + return { + ...aligned, + leftRows: pick(aligned.leftRows), + rightRows: pick(aligned.rightRows), + leftGap: pick(aligned.leftGap), + rightGap: pick(aligned.rightGap), + rowOps: pick(aligned.rowOps), + rowKeyLabels: pick(aligned.rowKeyLabels), + }; +} + +/** The key values of a result's last row, or undefined when a key column is missing. */ +export function lastRowKey( + result: { columns: string[]; rows: unknown[][] }, + keyNames: readonly string[] +): unknown[] | undefined { + const last = result.rows[result.rows.length - 1]; + if (!last) return undefined; + const lower = result.columns.map((c) => c.toLowerCase()); + const idx = keyNames.map((k) => lower.indexOf(k.toLowerCase())); + if (idx.some((i) => i < 0)) return undefined; + return idx.map((i) => last[i]); +} diff --git a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts index ed5a59f93..ecc5640d1 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts @@ -178,3 +178,41 @@ describe('alignResultGridsByKey on a partial grid', () => { expect(aligned.rowOps).toEqual(['unresolved', 'unresolved', 'insert']); }); }); + +describe('alignResultGridsByKey in a compare run (both sides read in key order, cut at a window)', () => { + const grid = (ids: (number | string)[]) => ({ columns: ['id', 'v'], rows: ids.map((id) => [id, 'x']) }); + + it('classifies a one-sided row at or before the cut-short side’s last key, and only past it leaves it unresolved', () => { + // Target has more rows than the window: it was cut at id 50. + const aligned = alignResultGridsByKey(grid([10, 20, 40, 60, 70]), grid([10, 30, 40, 50]), ['id'], { + rightComplete: false, + rightWindowEnd: [50], + })!; + const opOf = (id: number) => aligned.rowOps[aligned.rowKeyLabels.indexOf(`id=${id}`)]; + expect(opOf(20)).toBe('delete'); // 20 < 50: the target would have shown it + expect(opOf(60)).toBe('unresolved'); // past 50: it may be further down + expect(opOf(70)).toBe('unresolved'); + expect(opOf(30)).toBe('insert'); // the source is complete + expect(aligned).toMatchObject({ deleteCount: 1, insertCount: 2, unresolvedCount: 2, matchCount: 2 }); + }); + + it('compares numbers by value (2 before 10), including bigints sent as text', () => { + const aligned = alignResultGridsByKey( + grid(['2', '9007199254740993']), + grid(['10']), + ['id'], + { rightComplete: false, rightWindowEnd: ['9007199254740992'] } + )!; + expect(aligned.rowOps[aligned.rowKeyLabels.indexOf('id=2')]).toBe('delete'); + expect(aligned.rowOps[aligned.rowKeyLabels.indexOf('id=9007199254740993')]).toBe('unresolved'); + }); + + it('leaves text keys unresolved: the engine’s collation decides their order, not JavaScript', () => { + const aligned = alignResultGridsByKey(grid(['a', 'b']), grid(['c']), ['id'], { + rightComplete: false, + rightWindowEnd: ['z'], + })!; + expect(aligned.unresolvedCount).toBe(2); + expect(aligned.deleteCount).toBe(0); + }); +}); diff --git a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts index b98390947..7aa533f2c 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts @@ -118,6 +118,15 @@ export function alignResultGridsByKey( */ leftComplete?: boolean; rightComplete?: boolean; + /** + * For a side read in key order and cut short (a compare run): the key of + * its last row. A row found only on the other side whose key sorts at or + * before it cannot be further down, so it is classified after all. Only + * numeric keys are compared: engines sort text by their own collation, and + * guessing that order would classify a row the other side does have. + */ + leftWindowEnd?: unknown[]; + rightWindowEnd?: unknown[]; } ): KeyAlignedGrids | null { if (keyNames.length === 0) return null; @@ -206,7 +215,7 @@ export function alignResultGridsByKey( leftGap.push(false); rightGap.push(true); rowKeyLabels.push(keyLabelForRow(lRow, leftKeys)); - if (rightComplete) { + if (rightComplete || withinWindow(lRow, leftKeys, opts?.rightWindowEnd)) { rowOps.push('delete'); deleteCount += 1; } else { @@ -224,7 +233,7 @@ export function alignResultGridsByKey( leftGap.push(true); rightGap.push(false); rowKeyLabels.push(keyLabelForRow(rRow, rightKeys)); - if (leftComplete) { + if (leftComplete || withinWindow(rRow, rightKeys, opts?.leftWindowEnd)) { rowOps.push('insert'); insertCount += 1; } else { @@ -250,6 +259,47 @@ export function alignResultGridsByKey( }; } +const DIGITS = /^\d+$/; + +/** An integer or decimal in text (a bigint arrives as text): `-12`, `3.5`. */ +function isNumericText(text: string): boolean { + const unsigned = text.startsWith('-') ? text.slice(1) : text; + const [whole, fraction, extra] = unsigned.split('.'); + return extra === undefined && DIGITS.test(whole ?? '') && (fraction === undefined || DIGITS.test(fraction)); +} + +/** a < b → negative, a > b → positive; null when either is not a number. */ +function compareNumeric(a: unknown, b: unknown): number | null { + const as = typeof a === 'bigint' ? a.toString() : typeof a === 'number' && Number.isFinite(a) ? String(a) : a; + const bs = typeof b === 'bigint' ? b.toString() : typeof b === 'number' && Number.isFinite(b) ? String(b) : b; + if (typeof as !== 'string' || typeof bs !== 'string' || !isNumericText(as) || !isNumericText(bs)) return null; + if (!as.includes('.') && !bs.includes('.')) { + const x = BigInt(as); + const y = BigInt(bs); + return x < y ? -1 : x > y ? 1 : 0; + } + return Number(as) - Number(bs); +} + +/** + * True when `row`'s key sorts at or before `windowEnd`, compared column by + * column the way ORDER BY does. False when there is no window end or any key + * part is not a number: the row stays unresolved. + */ +function withinWindow( + row: unknown[], + keys: ReturnType, + windowEnd: unknown[] | undefined +): boolean { + if (!windowEnd || windowEnd.length !== keys.length) return false; + for (let i = 0; i < keys.length; i++) { + const cmp = compareNumeric(row[keys[i]!.resultIndex], windowEnd[i]); + if (cmp === null) return false; + if (cmp !== 0) return cmp < 0; + } + return true; +} + function nonKeyDiffer( leftRow: unknown[], rightRow: unknown[], diff --git a/apps/web/src/frontend/shared/api/sqlApi.ts b/apps/web/src/frontend/shared/api/sqlApi.ts index e902cdc94..904939059 100644 --- a/apps/web/src/frontend/shared/api/sqlApi.ts +++ b/apps/web/src/frontend/shared/api/sqlApi.ts @@ -35,6 +35,8 @@ export async function executeSql( opts?: { datagridAction?: 'insert' | 'update' | 'delete'; seek?: { columns: string[]; values: unknown[]; descending?: boolean | boolean[] }; + /** Order each page by these columns (a compare run reads both sides this way). */ + orderBy?: string[]; /** Leave the statements after a failure unrun; `results` then ends at it. */ stopOnError?: boolean; /** One transaction for the whole batch; a failure rolls back what already ran. */ @@ -49,6 +51,7 @@ export async function executeSql( params, ...(opts?.datagridAction ? { datagridAction: opts.datagridAction } : {}), ...(opts?.seek ? { seek: opts.seek } : {}), + ...(opts?.orderBy?.length ? { orderBy: opts.orderBy } : {}), ...(opts?.stopOnError ? { stopOnError: true } : {}), ...(opts?.transaction ? { transaction: true } : {}), }); diff --git a/docs/testing/TEST_IDS.md b/docs/testing/TEST_IDS.md index 52216ef36..9bdb8d3e6 100644 --- a/docs/testing/TEST_IDS.md +++ b/docs/testing/TEST_IDS.md @@ -8,7 +8,7 @@ or `TestIds` for the same tree with autocomplete. `{name}` marks a part filled i time; "in FilterPicker" marks an ID a shared component draws from the prop written here. How to write a test with them: [WRITING_E2E.md](WRITING_E2E.md). -Every control has a test ID: **1000** buttons, text boxes, selects and textareas. +Every control has a test ID: **1008** buttons, text boxes, selects and textareas. ## access @@ -1203,18 +1203,30 @@ Every control has a test ID: **1000** buttons, text boxes, selects and textareas - `sql-result-compare-baseline-{statementIndex}` · select - `sql-result-compare-close-{statementIndex}` · button · Close (Esc) - `sql-result-compare-dest-{statementIndex}` · select + - `sql-result-compare-diff-next-{statementIndex}` · button · Difference › + - `sql-result-compare-diff-prev-{statementIndex}` · button · ‹ Difference - `sql-result-compare-export-csv-{statementIndex}` · button · Export all compare grids as one CSV (columns prefixed by Source / Target) - `sql-result-compare-export-csv-modal-{statementIndex}` · button · Export all compare grids as one CSV + - `sql-result-compare-filter-{f}-{statementIndex}` · button - `sql-result-compare-hint-{statementIndex}` · p - `sql-result-compare-legend-{statementIndex}` · span - `sql-result-compare-maximize-{statementIndex}` · button · Maximize compare - `sql-result-compare-maximized-hint-{statementIndex}` · p - `sql-result-compare-modal-{statementIndex}` · div + - `sql-result-compare-page-label-{statementIndex}` · span + - `sql-result-compare-page-next-{statementIndex}` · button · Next + - `sql-result-compare-page-prev-{statementIndex}` · button · Prev + - `sql-result-compare-page-size-{statementIndex}` · select · Rows per compare page - `sql-result-compare-skip-trigger-{statementIndex}` · input - `sql-result-compare-sync-scroll-{statementIndex}` · input - `sql-result-compare-sync-scroll-modal-{statementIndex}` · input - `sql-result-compare-toggle-{statementIndex}` · input - `sql-result-compare-toolbar-{statementIndex}` · div + - `sql-result-compare-view-{statementIndex}` · div + - `sql-result-compare-window-clear-{statementIndex}` · button · Back to loaded rows + - `sql-result-compare-window-error-{statementIndex}` · span + - `sql-result-compare-window-run-{statementIndex}` · button · Compare rows + - `sql-result-compare-window-summary-{statementIndex}` · span - `sql-result-cred-{connectionId}` · section - `sql-result-cred-close-{connectionId}` · button · Close maximized results - `sql-result-cred-maximize-{connectionId}` · button · Maximize results diff --git a/packages/server/src/features/sql-editor/editor.routes.ts b/packages/server/src/features/sql-editor/editor.routes.ts index 362503b1a..4624e6a5d 100644 --- a/packages/server/src/features/sql-editor/editor.routes.ts +++ b/packages/server/src/features/sql-editor/editor.routes.ts @@ -30,7 +30,7 @@ import { statementVerb, } from '@foxschema/sql'; import { isSingleSqlStatement } from '../../platform/statements/single-statement'; -import { clampOffset, parseSqlSeek } from './sql-page-wrap.service'; +import { clampOffset, parseSqlOrderBy, parseSqlSeek } from './sql-page-wrap.service'; import { makeBeamCellQueryRunner, makeCellQueryRunner } from './code-cell-query.service'; import type { CellQueryRunner } from './code-cell-execute.service'; import { parseBeamEndpoints } from '@foxschema/shared'; @@ -81,12 +81,14 @@ export function createEditorRoutes(deps: EditorRouteDeps): Router { // called next(), so the request hung with no error and no response. const writeIdempotency = idempotency(); router.post('/sql/execute', sqlExecuteLimiter, writeIdempotency, async (req: AppRequest, res: FastifyReply) => { - const { statements, maxRows, offset, params, datagridAction, seek, stopOnError, transaction, ...ref } = req.body as ConnectionRef & { + const { statements, maxRows, offset, params, datagridAction, seek, orderBy, stopOnError, transaction, ...ref } = req.body as ConnectionRef & { statements?: unknown; maxRows?: unknown; offset?: unknown; params?: unknown; seek?: unknown; + /** Order each page by these columns (a compare run). */ + orderBy?: unknown; /** Leave the statements after a failure unrun (results end at it). */ stopOnError?: unknown; /** Wrap the batch in one transaction and roll it back on the first failure. */ @@ -195,6 +197,11 @@ export function createEditorRoutes(deps: EditorRouteDeps): Router { sendError(res, 'invalid_input', parsedSeek.error); return; } + const parsedOrder = parseSqlOrderBy(orderBy); + if (!parsedOrder.ok) { + sendError(res, 'invalid_input', parsedOrder.error); + return; + } try { // Apply the saved connection's schema (CURRENT SCHEMA / search_path) so // unqualified names like ORDERS resolve to DEMO.ORDERS, not USER.ORDERS. @@ -208,7 +215,8 @@ export function createEditorRoutes(deps: EditorRouteDeps): Router { (params as unknown[][] | undefined) ?? [], parsedSeek.value, stopOnError === true, - transaction === true + transaction === true, + parsedOrder.value ); res.send({ results }); } catch (error: unknown) { diff --git a/packages/server/src/features/sql-editor/sql-execute.service.test.ts b/packages/server/src/features/sql-editor/sql-execute.service.test.ts index e33e71f72..d394c5744 100644 --- a/packages/server/src/features/sql-editor/sql-execute.service.test.ts +++ b/packages/server/src/features/sql-editor/sql-execute.service.test.ts @@ -184,6 +184,25 @@ describe('runStatements against a real SQLite file', () => { expect(results[0]).toMatchObject({ ok: true, rowCount: 2, truncated: true, hasNext: true }); }); + it('orders a page by the given keys and says whether more rows follow (a compare run)', async () => { + const results = await runStatements( + 'sqlite', + { connectionString: dbPath }, + ['SELECT id, name FROM t ORDER BY name DESC;'], + 2, + undefined, + 0, + [], + undefined, + false, + false, + ['id'] + ); + expect(results[0]).toMatchObject({ ok: true, hasNext: true }); + const ids = (results[0] as { rows: unknown[][] }).rows.map((r) => Number(r[0])); + expect(ids).toEqual([...ids].sort((a, b) => a - b)); + }); + it('pages SELECT with OFFSET (page 1 is not page 0)', async () => { const page0 = await runStatements( 'sqlite', diff --git a/packages/server/src/features/sql-editor/sql-execute.service.ts b/packages/server/src/features/sql-editor/sql-execute.service.ts index 0a022bcca..394766dc0 100644 --- a/packages/server/src/features/sql-editor/sql-execute.service.ts +++ b/packages/server/src/features/sql-editor/sql-execute.service.ts @@ -1,6 +1,6 @@ import { ConnectionFactory, getAdapter, type ConnectionOptions } from '@foxschema/db'; import { autoAliasSelectColumns, errorMessage, isSubsetDialect } from '@foxschema/sql'; -import { isPageableStatement, syntaxProbeSql, trimPageProbe, wrapSqlForPage, wrapSqlForSeek, type SqlSeek } from './sql-page-wrap.service'; +import { isPageableStatement, syntaxProbeSql, trimPageProbe, wrapSqlForOrderedPage, wrapSqlForPage, wrapSqlForSeek, type SqlSeek } from './sql-page-wrap.service'; /** * Helpers behind POST /api/sql/execute (SQL Editor). One request = one @@ -138,7 +138,13 @@ export async function runStatements( * Engines that auto-commit DDL still report each statement; the rollback is * best-effort there. */ - transaction = false + transaction = false, + /** + * Order each page by these columns (outer ORDER BY, `wrapSqlForOrderedPage`). + * A compare run reads both sides this way, so the same keys line up. + * Ignored with `seek`, and by the SQL-subset stores (Redis, MongoDB). + */ + orderBy?: readonly string[] ): Promise { const schemaName = (schema ?? option.schema)?.trim() || ''; const optionWithSchema: ConnectionOptions = schemaName @@ -212,7 +218,10 @@ export async function runStatements( continue; } - let paged = wrapSqlForPage(sql, dialect, offset, maxRows); + let paged = + orderBy?.length && !seek && !isSubsetDialect(dialect) + ? wrapSqlForOrderedPage(sql, dialect, orderBy, offset, maxRows) + : wrapSqlForPage(sql, dialect, offset, maxRows); if (seek && offset === 0) { const keyed = wrapSqlForSeek(sql, dialect, seek, maxRows, params.length); if ('error' in keyed) { diff --git a/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts b/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts index e786de224..1ff7fad4e 100644 --- a/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts +++ b/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts @@ -2,11 +2,48 @@ import { describe, expect, it } from 'vitest'; import { hasTopLevelOrderBy, isPageableStatement, + parseSqlOrderBy, trimPageProbe, wrapSqlForPage, + wrapSqlForOrderedPage, wrapSqlForSeek, } from './sql-page-wrap.service'; +describe('ordered pages (a compare run)', () => { + it('orders the outer query by the keys, in each engine\'s paging syntax, fetching one extra row', () => { + expect(wrapSqlForOrderedPage('SELECT * FROM t;', 'postgres', ['id'], 0, 2000)).toBe( + 'SELECT * FROM (SELECT * FROM t) AS fox_page ORDER BY "id" LIMIT 2001 OFFSET 0' + ); + expect(wrapSqlForOrderedPage('SELECT * FROM t', 'mysql', ['a', 'b'], 0, 10)).toBe( + 'SELECT * FROM (SELECT * FROM t) AS fox_page ORDER BY `a`, `b` LIMIT 11 OFFSET 0' + ); + expect(wrapSqlForOrderedPage('SELECT * FROM t', 'oracle', ['ID'], 0, 10)).toMatch( + /ORDER BY "ID" OFFSET 0 ROWS FETCH NEXT 11 ROWS ONLY$/ + ); + expect(wrapSqlForOrderedPage('SELECT * FROM t', 'db2', ['ID'], 0, 10)).toMatch( + /ORDER BY "ID" OFFSET 0 ROWS FETCH FIRST 11 ROWS ONLY$/ + ); + }); + + it('on SQL Server drops a plain top-level ORDER BY, which a derived table refuses', () => { + expect(wrapSqlForOrderedPage('SELECT id FROM t ORDER BY name', 'sqlserver', ['id'], 0, 10)).toBe( + 'SELECT * FROM (SELECT id FROM t) AS fox_page ORDER BY [id] OFFSET 0 ROWS FETCH NEXT 11 ROWS ONLY' + ); + // One that pages itself stays: it is legal there, and it limits the rows. + expect( + wrapSqlForOrderedPage('SELECT id FROM t ORDER BY id OFFSET 5 ROWS', 'azuresql', ['id'], 0, 10) + ).toContain('ORDER BY id OFFSET 5 ROWS) AS fox_page'); + }); + + it('takes only plain column names, never SQL', () => { + expect(parseSqlOrderBy(undefined)).toEqual({ ok: true, value: undefined }); + expect(parseSqlOrderBy(['id', 'tenant_id'])).toEqual({ ok: true, value: ['id', 'tenant_id'] }); + for (const bad of [[], 'id', ['id; DROP TABLE t'], ['1=1 --'], [3], Array.from({ length: 9 }, (_, i) => `c${i}`)]) { + expect(parseSqlOrderBy(bad).ok, JSON.stringify(bad)).toBe(false); + } + }); +}); + describe('paging Redis and MongoDB', () => { // They read SQL through a parser with no subqueries: a nested page was // refused ("Unsupported clause") for every Redis and MongoDB SELECT. diff --git a/packages/server/src/features/sql-editor/sql-page-wrap.service.ts b/packages/server/src/features/sql-editor/sql-page-wrap.service.ts index c9e874d16..4796f02e6 100644 --- a/packages/server/src/features/sql-editor/sql-page-wrap.service.ts +++ b/packages/server/src/features/sql-editor/sql-page-wrap.service.ts @@ -12,6 +12,7 @@ import { isPageableStatement } from '@foxschema/db'; import { + findTopLevelOrderByIndex, isSafeSeekColumn, isSubsetDialect, isTsqlDialect, @@ -195,6 +196,57 @@ export function wrapSqlForPage( return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} LIMIT ${fetchLimit} OFFSET ${offset}`; } +/** Longest key a compare run orders by: more columns than this is not a key. */ +const MAX_ORDER_COLUMNS = 8; + +/** + * `orderBy` from a request: plain column names, or nothing. Anything else is + * refused, never quoted into SQL as given. + */ +export function parseSqlOrderBy(raw: unknown): { ok: true; value?: string[] } | { ok: false; error: string } { + if (raw == null) return { ok: true, value: undefined }; + if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_ORDER_COLUMNS) { + return { ok: false, error: `orderBy must be 1 to ${MAX_ORDER_COLUMNS} column names` }; + } + if (!raw.every((c) => typeof c === 'string' && isSafeSeekColumn(c))) { + return { ok: false, error: 'orderBy must be plain column names' }; + } + return { ok: true, value: raw as string[] }; +} + +/** + * One page of `sql` in `orderBy` order: the order goes on the outer query, + * where every engine honours it (an ORDER BY inside a derived table is not + * guaranteed to survive, and SQL Server refuses one without TOP or OFFSET — + * so on SQL Server a plain top-level ORDER BY of the statement is dropped; + * the outer one decides anyway). Fetches `limit + 1` so the caller learns + * whether more rows follow. A compare run reads both sides this way. + */ +export function wrapSqlForOrderedPage( + sql: string, + dialect: string, + orderBy: readonly string[], + offset: number, + limit: number +): string { + const d = dialect.toLowerCase(); + let inner = sql.trim().replace(/;+\s*$/, ''); + const fetchLimit = limit + 1; + const orderSql = orderBy.map((c) => quoteSqlIdentifier(c, dialect)).join(', '); + if (isTsqlDialect(d)) { + const at = findTopLevelOrderByIndex(inner); + if (at >= 0 && !/\boffset\b/i.test(inner.slice(at))) inner = inner.slice(0, at).trimEnd(); + return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} ORDER BY ${orderSql} OFFSET ${offset} ROWS FETCH NEXT ${fetchLimit} ROWS ONLY`; + } + if (d === 'oracle') { + return `SELECT * FROM (${inner}) ${PAGE_ALIAS} ORDER BY ${orderSql} OFFSET ${offset} ROWS FETCH NEXT ${fetchLimit} ROWS ONLY`; + } + if (d === 'db2') { + return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} ORDER BY ${orderSql} OFFSET ${offset} ROWS FETCH FIRST ${fetchLimit} ROWS ONLY`; + } + return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} ORDER BY ${orderSql} LIMIT ${fetchLimit} OFFSET ${offset}`; +} + export interface SqlSeek { columns: string[]; values: unknown[];