diff --git a/apps/e2e/scripts/run-all.mjs b/apps/e2e/scripts/run-all.mjs index 673061805..d21c4aa66 100644 --- a/apps/e2e/scripts/run-all.mjs +++ b/apps/e2e/scripts/run-all.mjs @@ -134,6 +134,7 @@ const ALWAYS = [ 'src/tests/sql-editor-peek-stacking.test.ts', 'src/tests/sql-editor-data-peek-grid.test.ts', 'src/tests/sql-editor-compare-multi-target.test.ts', + 'src/tests/sql-editor-compare-window.test.ts', 'src/tests/sql-editor-join-typing.test.ts', ].join(' '), label: 'SQL Editor', diff --git a/apps/e2e/src/generated/test-ids.ts b/apps/e2e/src/generated/test-ids.ts index f0e27f0ee..ad81ef3cb 100644 --- a/apps/e2e/src/generated/test-ids.ts +++ b/apps/e2e/src/generated/test-ids.ts @@ -2239,10 +2239,16 @@ export const TestIds = { sqlResultCompareClose: (statementIndex: string | number) => `sql-result-compare-close-${statementIndex}`, /** select */ sqlResultCompareDest: (statementIndex: string | number) => `sql-result-compare-dest-${statementIndex}`, + /** button · Difference › */ + sqlResultCompareDiffNext: (statementIndex: string | number) => `sql-result-compare-diff-next-${statementIndex}`, + /** button · ‹ Difference */ + sqlResultCompareDiffPrev: (statementIndex: string | number) => `sql-result-compare-diff-prev-${statementIndex}`, /** button · Export all compare grids as one CSV (columns prefixed by Source / Target) */ sqlResultCompareExportCsv: (statementIndex: string | number) => `sql-result-compare-export-csv-${statementIndex}`, /** button · Export all compare grids as one CSV */ sqlResultCompareExportCsvModal: (statementIndex: string | number) => `sql-result-compare-export-csv-modal-${statementIndex}`, + /** button */ + sqlResultCompareFilter: (f: string | number, statementIndex: string | number) => `sql-result-compare-filter-${f}-${statementIndex}`, /** p */ sqlResultCompareHint: (statementIndex: string | number) => `sql-result-compare-hint-${statementIndex}`, /** span */ @@ -2253,6 +2259,14 @@ export const TestIds = { sqlResultCompareMaximizedHint: (statementIndex: string | number) => `sql-result-compare-maximized-hint-${statementIndex}`, /** div */ sqlResultCompareModal: (statementIndex: string | number) => `sql-result-compare-modal-${statementIndex}`, + /** span */ + sqlResultComparePageLabel: (statementIndex: string | number) => `sql-result-compare-page-label-${statementIndex}`, + /** button · Next */ + sqlResultComparePageNext: (statementIndex: string | number) => `sql-result-compare-page-next-${statementIndex}`, + /** button · Prev */ + sqlResultComparePagePrev: (statementIndex: string | number) => `sql-result-compare-page-prev-${statementIndex}`, + /** select · Rows per compare page */ + sqlResultComparePageSize: (statementIndex: string | number) => `sql-result-compare-page-size-${statementIndex}`, /** input */ sqlResultCompareSkipTrigger: (statementIndex: string | number) => `sql-result-compare-skip-trigger-${statementIndex}`, /** input */ @@ -2263,6 +2277,16 @@ export const TestIds = { sqlResultCompareToggle: (statementIndex: string | number) => `sql-result-compare-toggle-${statementIndex}`, /** div */ sqlResultCompareToolbar: (statementIndex: string | number) => `sql-result-compare-toolbar-${statementIndex}`, + /** div */ + sqlResultCompareView: (statementIndex: string | number) => `sql-result-compare-view-${statementIndex}`, + /** button · Back to loaded rows */ + sqlResultCompareWindowClear: (statementIndex: string | number) => `sql-result-compare-window-clear-${statementIndex}`, + /** span */ + sqlResultCompareWindowError: (statementIndex: string | number) => `sql-result-compare-window-error-${statementIndex}`, + /** button · Compare rows */ + sqlResultCompareWindowRun: (statementIndex: string | number) => `sql-result-compare-window-run-${statementIndex}`, + /** span */ + sqlResultCompareWindowSummary: (statementIndex: string | number) => `sql-result-compare-window-summary-${statementIndex}`, /** section */ sqlResultCred: (connectionId: string | number) => `sql-result-cred-${connectionId}`, /** button · Close maximized results */ @@ -4823,18 +4847,30 @@ export type TestId = | `sql-result-compare-baseline-${string}` | `sql-result-compare-close-${string}` | `sql-result-compare-dest-${string}` + | `sql-result-compare-diff-next-${string}` + | `sql-result-compare-diff-prev-${string}` | `sql-result-compare-export-csv-${string}` | `sql-result-compare-export-csv-modal-${string}` + | `sql-result-compare-filter-${string}-${string}` | `sql-result-compare-hint-${string}` | `sql-result-compare-legend-${string}` | `sql-result-compare-maximize-${string}` | `sql-result-compare-maximized-hint-${string}` | `sql-result-compare-modal-${string}` + | `sql-result-compare-page-label-${string}` + | `sql-result-compare-page-next-${string}` + | `sql-result-compare-page-prev-${string}` + | `sql-result-compare-page-size-${string}` | `sql-result-compare-skip-trigger-${string}` | `sql-result-compare-sync-scroll-${string}` | `sql-result-compare-sync-scroll-modal-${string}` | `sql-result-compare-toggle-${string}` | `sql-result-compare-toolbar-${string}` + | `sql-result-compare-view-${string}` + | `sql-result-compare-window-clear-${string}` + | `sql-result-compare-window-error-${string}` + | `sql-result-compare-window-run-${string}` + | `sql-result-compare-window-summary-${string}` | `sql-result-cred-${string}` | `sql-result-cred-close-${string}` | `sql-result-cred-maximize-${string}` diff --git a/apps/e2e/src/tests/sql-editor-compare-window.test.ts b/apps/e2e/src/tests/sql-editor-compare-window.test.ts new file mode 100644 index 000000000..59944e76d --- /dev/null +++ b/apps/e2e/src/tests/sql-editor-compare-window.test.ts @@ -0,0 +1,162 @@ +/** + * Compare 2,000 rows: both sides read again in key order, every row inside the + * window classified, and the result paged and filtered in the browser. + * + * The grids page each side on its own with OFFSET, so their pages do not line + * up and a one-sided row on a cut-short side can only be "unresolved". A + * compare run reads the first 2,000 rows of both sides by key instead. + * + * Fixture (every count below follows from it): + * source ids 1–2,500, name n + * target the same, without 7, 1,500 and 2,300; names changed at 10, 20 and + * 1,990; plus 2,600 and 2,601 + * In the window, source ends at id 2,000 and target at 2,002, so: + * Changed 3 · Only in source 2 (7, 1,500) · Only in target 0 · + * Unresolved 2 (2,001 and 2,002, past the source's last key) · + * Matching 1,995 · All 2,002 + * + * Two SQLite files under a per-run directory. Requires the web app + API and + * a `sqlite3` CLI on PATH. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execFileSync, execSync } from 'node:child_process'; +import { mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Page } from 'playwright'; +import { buildDriver, quitDriver } from '../helpers/driver.js'; +import { saveScreenshot } from '../helpers/screenshot.js'; +import { byTestId } from '../helpers/test-ids.js'; +import { AppPage } from '../pages/AppPage.js'; +import { SqlEditorPage } from '../pages/SqlEditorPage.js'; + +const RUN = Date.now().toString(36); +const DIR = `/tmp/foxschema-e2e-compare-window-${RUN}`; +const SOURCE = { file: join(DIR, 'source.db'), name: `E2E Window Src ${RUN}` }; +const TARGET = { file: join(DIR, 'target.db'), name: `E2E Window Tgt ${RUN}` }; + +function hasSqlite3(): boolean { + try { + execSync('which sqlite3', { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function seed(path: string, target: boolean): void { + const missing = new Set([7, 1500, 2300]); + const changed = new Set([10, 20, 1990]); + const rows: string[] = []; + for (let id = 1; id <= 2500; id++) { + if (target && missing.has(id)) continue; + rows.push(`(${id}, '${target && changed.has(id) ? `changed${id}` : `n${id}`}')`); + } + if (target) rows.push("(2600, 'n2600')", "(2601, 'n2601')"); + execFileSync('sqlite3', [path], { + input: `CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO items (id, name) VALUES ${rows.join(',')};`, + }); +} + +describe.skipIf(!hasSqlite3())('SQL Editor · Compare 2,000 rows', () => { + let driver: Page; + let sql: SqlEditorPage; + const label = (id: string) => driver.locator(byTestId(id as never)).innerText(); + + beforeAll(async () => { + rmSync(DIR, { recursive: true, force: true }); + mkdirSync(DIR, { recursive: true }); + seed(SOURCE.file, false); + seed(TARGET.file, true); + + driver = await buildDriver(); + sql = new SqlEditorPage(driver); + await new AppPage(driver).open(); + await sql.resetPersistedEditorState(); + await driver.reload(); + await driver.waitForSelector(byTestId('toolbar')); + await sql.addSqliteCredential(SOURCE.name, SOURCE.file); + await sql.addSqliteCredential(TARGET.name, TARGET.file); + }, 180_000); + + afterAll(async () => { + if (driver) await quitDriver(driver); + rmSync(DIR, { recursive: true, force: true }); + }); + + it('compares the first 2,000 rows of both sides by key, classifying every row the window can decide', async () => { + await sql.openView(); + await sql.checkConnection(SOURCE.name); + await sql.checkConnection(TARGET.name); + await sql.setSql('SELECT id, name FROM items;'); + await sql.run(); + await sql.waitForResults(); + await sql.setLayoutSideBySide(); + await sql.compareToggle(0).click(); + + // The source is the first destination checked; the target, the other. + const baseline = await driver.locator(byTestId('sql-result-compare-baseline-0')).inputValue(); + const sourceValue = await driver.evaluate( + (name) => + [...(document.querySelector('[data-testid="sql-result-compare-baseline-0"]') as HTMLSelectElement).options].find( + (o) => o.textContent?.includes(name) + )?.value, + SOURCE.name + ); + if (sourceValue && sourceValue !== baseline) { + await driver.locator(byTestId('sql-result-compare-baseline-0')).selectOption(sourceValue); + } + + await driver.locator(byTestId('sql-result-compare-window-run-0')).click(); + await driver.waitForSelector(byTestId('sql-result-compare-window-summary-0'), { timeout: 60_000 }); + expect(await label('sql-result-compare-window-summary-0')).toMatch( + /Compared 2,000 \(more follow\) source and 2,000 \(more follow\) target rows, ordered by id/ + ); + + const count = async (f: string) => { + const text = await label(`sql-result-compare-filter-${f}-0`); + return Number(text.replace(/[^\d]/g, '')); + }; + expect({ + changed: await count('changed'), + source: await count('source'), + target: await count('target'), + unresolved: await count('unresolved'), + matching: await count('matching'), + all: await count('all'), + }).toEqual({ changed: 3, source: 2, target: 0, unresolved: 2, matching: 1995, all: 2002 }); + await saveScreenshot(driver, 'compare-window-summary'); + }, 180_000); + + it('filters to the changed rows and pages the rest in the browser', async () => { + await driver.locator(byTestId('sql-result-compare-filter-changed-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 1 of 1'); + const keys = await driver.evaluate(() => + [...document.querySelectorAll('[data-testid="sql-data-grid"]')][0]! + .querySelectorAll('tbody tr [data-testid="sql-row-num"]').length + ); + expect(keys).toBe(3); + + await driver.locator(byTestId('sql-result-compare-filter-all-0')).click(); + await driver.locator(byTestId('sql-result-compare-page-size-0')).selectOption('100'); + expect(await label('sql-result-compare-page-label-0')).toBe('page 1 of 21'); + }, 60_000); + + it('steps from one difference to the next across pages of matching rows', async () => { + // Differences sit at ids 7, 10 and 20 (page 1), 1,500 (page 15) and 1,990 (page 20). + await driver.locator(byTestId('sql-result-compare-diff-next-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 15 of 21'); + await driver.locator(byTestId('sql-result-compare-diff-next-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 20 of 21'); + expect(await driver.locator(byTestId('sql-result-compare-diff-next-0')).isDisabled()).toBe(true); + await driver.locator(byTestId('sql-result-compare-diff-prev-0')).click(); + expect(await label('sql-result-compare-page-label-0')).toBe('page 15 of 21'); + await saveScreenshot(driver, 'compare-window-difference'); + }, 60_000); + + it('goes back to the loaded pages, where a cut-short side leaves one-sided rows unresolved', async () => { + await driver.locator(byTestId('sql-result-compare-window-clear-0')).click(); + await driver.waitForSelector(byTestId('sql-result-compare-window-run-0')); + expect(await driver.locator(byTestId('sql-result-compare-window-summary-0')).count()).toBe(0); + }, 60_000); +}); diff --git a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx index 0c5204c2c..b5bec7b7e 100644 --- a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx @@ -42,6 +42,7 @@ import { import { attributeResultColumns, collapsedColumnsFor, + isSubsetDialect, rowKeyFor, tablesInOrigins, } from '@foxschema/ui-shared'; @@ -57,6 +58,19 @@ import { type AlignRowOp, } from '@/features/sql-editor/lib/resultKeyAlign'; import { migrateGridsAreComplete } from '@/features/sql-editor/lib/resultRowDiff'; +import { + COMPARE_PAGE_SIZES, + COMPARE_WINDOW_ROWS, + DEFAULT_COMPARE_PAGE_SIZE, + compareFilterCounts, + comparePage, + filteredRowIndexes, + lastRowKey, + pageOfNextDifference, + sliceAligned, + type CompareFilter, +} from '@/features/sql-editor/lib/compareView'; +import { useCompareWindow } from './useCompareWindow'; import { sideBySideSectionCount } from '@/features/sql-editor/lib/resultsSections'; import { detectTriggerManagedColumns } from '@/features/sql-editor/lib/triggerManagedColumns'; import { resolvePeekKeyColumns } from '@/features/sql-editor/lib/rowDml'; @@ -1027,6 +1041,15 @@ const SideBySideStatementSection: React.FC<{ const sourceGrid = okGrids.find((g) => g.connectionId === baselineId); const destGrid = okGrids.find((g) => g.connectionId === destId); + /** A new run of the statement makes any compare window describe old rows. */ + const windowResetKey = okGrids + .map((g) => `${g.connectionId}:${g.result.ok ? `${g.result.durationMs}:${g.result.rowCount}` : 'x'}`) + .join(','); + const compareWindow = useCompareWindow(`${statementIndex}|${windowResetKey}`); + const [compareFilter, setCompareFilter] = useState('all'); + const [comparePageIndex, setComparePageIndex] = useState(0); + const [comparePageSize, setComparePageSize] = useState(DEFAULT_COMPARE_PAGE_SIZE); + const defaultKeys = useMemo(() => { if (!sourceGrid?.result.ok) return [] as string[]; const cols = sourceGrid.result.columns; @@ -1118,6 +1141,17 @@ const SideBySideStatementSection: React.FC<{ // instead of joining it into a string. const effectiveKeys = keyNames.length ? keyNames : defaultKeys; + /** The compare window, while it still describes this pair, by these keys. */ + const windowState = compareWindow.state; + const windowOn = + compareActive && + windowState.status === 'ready' && + windowState.sourceId === baselineId && + windowState.destId === destId && + windowState.keys.join('\0') === effectiveKeys.join('\0') + ? windowState + : null; + /** * Whether each side holds its whole result: page 1, no next page, nothing * truncated. The same test Data migrate applies before it will write. @@ -1146,13 +1180,49 @@ const SideBySideStatementSection: React.FC<{ const keyAligned = useMemo(() => { if (!compareActive || !sourceGrid?.result.ok || !destGrid?.result.ok) return null; if (effectiveKeys.length === 0) return null; + if (windowOn) { + // Both windows are in key order: a side cut short at the window ends at + // a known key, and a one-sided row before it is classified after all. + const sourceMore = Boolean(windowOn.source.hasNext || windowOn.source.truncated); + const destMore = Boolean(windowOn.dest.hasNext || windowOn.dest.truncated); + return alignResultGridsByKey( + { columns: windowOn.source.columns, rows: windowOn.source.rows }, + { columns: windowOn.dest.columns, rows: windowOn.dest.rows }, + effectiveKeys, + { + ...ignoreOpts, + leftComplete: !sourceMore, + rightComplete: !destMore, + leftWindowEnd: sourceMore ? lastRowKey(windowOn.source, effectiveKeys) : undefined, + rightWindowEnd: destMore ? lastRowKey(windowOn.dest, effectiveKeys) : undefined, + } + ); + } return alignResultGridsByKey( { columns: sourceGrid.result.columns, rows: sourceGrid.result.rows }, { columns: destGrid.result.columns, rows: destGrid.result.rows }, effectiveKeys, { ...ignoreOpts, leftComplete: sourceComplete, rightComplete: destComplete } ); - }, [compareActive, sourceGrid, destGrid, effectiveKeys, ignoreOpts, sourceComplete, destComplete]); + }, [compareActive, sourceGrid, destGrid, effectiveKeys, ignoreOpts, sourceComplete, destComplete, windowOn]); + + const filterCounts = useMemo(() => (keyAligned ? compareFilterCounts(keyAligned) : null), [keyAligned]); + const filteredRows = useMemo( + () => (keyAligned ? filteredRowIndexes(keyAligned, compareFilter) : []), + [keyAligned, compareFilter] + ); + const pageView = useMemo( + () => comparePage(filteredRows, comparePageIndex, comparePageSize), + [filteredRows, comparePageIndex, comparePageSize] + ); + /** What the grids show: this page of the filtered rows. Counts stay the whole set's. */ + const viewAligned = useMemo( + () => (keyAligned ? sliceAligned(keyAligned, pageView.rows) : null), + [keyAligned, pageView] + ); + useEffect(() => { + setComparePageIndex(0); + }, [keyAligned, compareFilter, comparePageSize]); const { diffByConnection, badgeByConnection, legendBits, displayItems } = useMemo(() => { const diffByConnection: Record = {}; @@ -1184,7 +1254,7 @@ const SideBySideStatementSection: React.FC<{ const pair = compareKeyAlignedGrids( baselineGrid, destGridLike, - keyAligned, + viewAligned ?? keyAligned, ignoreOpts ); diffByConnection[baselineId] = pair.baseline; @@ -1200,7 +1270,9 @@ const SideBySideStatementSection: React.FC<{ if (keyAligned.unresolvedCount > 0) { legendBits.push( `${keyAligned.unresolvedCount} on one side only — the other side's result is cut ` + - 'off, so they may be further down; raise Rows/page to classify them' + (windowOn + ? 'off past its last key, so they may be further down' + : `off, so they may be further down; Compare ${COMPARE_WINDOW_ROWS.toLocaleString()} rows classifies them`) ); } if (keyAligned.duplicateKeys > 0) { @@ -1238,8 +1310,8 @@ const SideBySideStatementSection: React.FC<{ ...item, result: { ...item.result, - rows: keyAligned.leftRows, - rowCount: keyAligned.leftRows.length, + rows: (viewAligned ?? keyAligned).leftRows, + rowCount: (viewAligned ?? keyAligned).leftRows.length, }, }; } @@ -1248,8 +1320,8 @@ const SideBySideStatementSection: React.FC<{ ...item, result: { ...item.result, - rows: keyAligned.rightRows, - rowCount: keyAligned.rightRows.length, + rows: (viewAligned ?? keyAligned).rightRows, + rowCount: (viewAligned ?? keyAligned).rightRows.length, }, }; } @@ -1354,23 +1426,27 @@ const SideBySideStatementSection: React.FC<{ baselineId, destId, keyAligned, + viewAligned, + windowOn, ignoreOpts, triggerIgnoreColumns, ]); const rowSyncByConnection = useMemo(() => { - if (!compareActive || !keyAligned || !destId) return undefined; + // By the grids' own row index, so from the page being shown. + const shown = viewAligned ?? keyAligned; + if (!compareActive || !shown || !destId) return undefined; return { [destId]: { isChecked: (rowIdx: number): boolean | null => { - const op = keyAligned.rowOps[rowIdx]; + const op = shown.rowOps[rowIdx]; if (op === 'match' || op === 'unresolved') return null; - const label = keyAligned.rowKeyLabels[rowIdx]; + const label = shown.rowKeyLabels[rowIdx]; if (!label) return false; return selectedSyncKeys.has(label); }, onToggle: (rowIdx: number, checked: boolean) => { - const label = keyAligned.rowKeyLabels[rowIdx]; + const label = shown.rowKeyLabels[rowIdx]; if (!label) return; setSelectedSyncKeys((prev) => { const next = new Set(prev); @@ -1381,7 +1457,7 @@ const SideBySideStatementSection: React.FC<{ }, }, }; - }, [compareActive, keyAligned, destId, selectedSyncKeys]); + }, [compareActive, keyAligned, viewAligned, destId, selectedSyncKeys]); const insertServerBeamSample = () => { const sample = buildSampleBookmarks().find((b) => b.id === 'sample-server-beam-chunked'); @@ -1475,27 +1551,34 @@ const SideBySideStatementSection: React.FC<{ compareActive && sourceGrid?.result.ok && destGrid?.result.ok ? ( ) : null; + const windowSupported = Boolean( + sourceGrid?.statementSql && + destGrid?.statementSql && + !isSubsetDialect(sourceGrid.dialect) && + !isSubsetDialect(destGrid.dialect) && + effectiveKeys.length > 0 + ); + const runCompareWindow = () => { + if (!sourceGrid?.statementSql || !destGrid?.statementSql) return; + void compareWindow.run( + { connectionId: sourceGrid.connectionId, sql: sourceGrid.statementSql }, + { connectionId: destGrid.connectionId, sql: destGrid.statementSql }, + effectiveKeys + ); + }; + const rowsLabel = (r: { rows: unknown[] ; hasNext?: boolean; truncated?: boolean }) => + `${r.rows.length.toLocaleString()}${r.hasNext || r.truncated ? ' (more follow)' : ''}`; + const FILTER_LABELS: Record = { + all: 'All', + changed: 'Changed', + source: 'Only in source', + target: 'Only in target', + unresolved: 'Unresolved', + matching: 'Matching', + }; + const nextDiffPage = keyAligned + ? pageOfNextDifference(keyAligned, filteredRows, pageView.page, comparePageSize, 1) + : null; + const prevDiffPage = keyAligned + ? pageOfNextDifference(keyAligned, filteredRows, pageView.page, comparePageSize, -1) + : null; + const navBtn = + 'rounded-md border border-slate-700 px-2 py-0.5 text-slate-200 hover:border-sky-500/40 disabled:opacity-40 disabled:hover:border-slate-700'; + + const compareViewBar = + compareActive && keyAligned && filterCounts ? ( +
+ {windowOn ? ( + <> + + Compared {rowsLabel(windowOn.source)} source and {rowsLabel(windowOn.dest)} target rows, ordered by{' '} + {windowOn.keys.join(', ')} + + + + ) : ( + + )} + {windowState.status === 'error' && ( + + {windowState.error} + + )} + + {(Object.keys(FILTER_LABELS) as CompareFilter[]).map((f) => ( + + ))} + + + + + + page {pageView.page + 1} of {pageView.pageCount} + + + + + +
+ ) : null; + const compareHint = compareActive ? (

{migrateBar} + {compareViewBar} {compareGrids(false)} {compareHint} @@ -1774,6 +2012,7 @@ const SideBySideStatementSection: React.FC<{

{migrateBar} + {compareViewBar}
{compareGrids(true)}
{compareHint}
diff --git a/apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts b/apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts new file mode 100644 index 000000000..246fc8e60 --- /dev/null +++ b/apps/web/src/frontend/features/sql-editor/components/useCompareWindow.ts @@ -0,0 +1,90 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * A compare run: the statement read again on the source and the target, in + * key order, COMPARE_WINDOW_ROWS (+1) rows each. The pages a grid shows are + * read independently with OFFSET and do not line up, so comparing them leaves + * most one-sided rows unresolved; two windows in the same key order do. + * + * Held here, beside the pane that asked, not in the editor store: the grids' + * own pages stay as they are, and a new run of the statement drops it. + */ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { executeSql, type SqlStatementResult } from '@/shared/api/sqlApi'; +import { getSessionPassword } from '@/shared/lib/sessionPasswords'; +import { useSqlEditorStore } from '@/features/sql-editor/state/useSqlEditorStore'; +import { COMPARE_WINDOW_ROWS } from '@/features/sql-editor/lib/compareView'; + +type OkResult = Extract; + +export type CompareWindowState = + | { status: 'idle' } + | { status: 'loading'; keys: string[] } + | { status: 'ready'; keys: string[]; sourceId: string; destId: string; source: OkResult; dest: OkResult } + | { status: 'error'; keys: string[]; error: string }; + +export interface CompareWindowSide { + connectionId: string; + sql: string; +} + +export function useCompareWindow( + /** Changes when the statement runs again: the windows then describe old results. */ + resetKey: unknown +): { + state: CompareWindowState; + run: (source: CompareWindowSide, dest: CompareWindowSide, keys: string[]) => Promise; + clear: () => void; +} { + const [state, setState] = useState({ status: 'idle' }); + const sessionPasswords = useSqlEditorStore((s) => s.sessionPasswords); + /** Which run is current: a slow older one must not overwrite a newer answer. */ + const runId = useRef(0); + + useEffect(() => { + runId.current += 1; + setState({ status: 'idle' }); + }, [resetKey]); + + const run = useCallback( + async (source: CompareWindowSide, dest: CompareWindowSide, keys: string[]) => { + const id = ++runId.current; + setState({ status: 'loading', keys }); + const read = async (side: CompareWindowSide): Promise => { + const password = getSessionPassword(side.connectionId) || sessionPasswords[side.connectionId] || undefined; + const { results } = await executeSql( + { connectionId: side.connectionId, password }, + [side.sql], + COMPARE_WINDOW_ROWS, + 0, + undefined, + { orderBy: keys } + ); + const one = results[0]; + if (!one) throw new Error('No result returned'); + if (!one.ok) throw new Error(one.error); + return one; + }; + try { + const [s, d] = await Promise.all([read(source), read(dest)]); + if (id === runId.current) { + setState({ status: 'ready', keys, sourceId: source.connectionId, destId: dest.connectionId, source: s, dest: d }); + } + } catch (error: unknown) { + if (id === runId.current) { + setState({ status: 'error', keys, error: error instanceof Error ? error.message : String(error) }); + } + } + }, + [sessionPasswords] + ); + + const clear = useCallback(() => { + runId.current += 1; + setState({ status: 'idle' }); + }, []); + + return { state, run, clear }; +} diff --git a/apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts b/apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts new file mode 100644 index 000000000..68f336c81 --- /dev/null +++ b/apps/web/src/frontend/features/sql-editor/lib/compareView.test.ts @@ -0,0 +1,75 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + */ +import { describe, expect, it } from 'vitest'; +import type { AlignRowOp, KeyAlignedGrids } from './resultKeyAlign'; +import { + compareFilterCounts, + comparePage, + filteredRowIndexes, + lastRowKey, + pageOfNextDifference, + sliceAligned, +} from './compareView'; + +const aligned = (ops: AlignRowOp[]): KeyAlignedGrids => ({ + keyNames: ['id'], + leftRows: ops.map((_, i) => [i]), + rightRows: ops.map((_, i) => [i]), + leftGap: ops.map((op) => op === 'insert'), + rightGap: ops.map((op) => op === 'delete'), + rowOps: ops, + rowKeyLabels: ops.map((_, i) => `id=${i}`), + matchCount: 0, + updateCount: 0, + insertCount: 0, + deleteCount: 0, + unresolvedCount: 0, + duplicateKeys: 0, +}); + +describe('compare view', () => { + it('counts each filter over the whole window', () => { + expect(compareFilterCounts(aligned(['match', 'update', 'delete', 'insert', 'unresolved', 'match']))).toEqual({ + all: 6, + changed: 1, + source: 1, + target: 1, + unresolved: 1, + matching: 2, + }); + }); + + it('pages the filtered rows, clamping the page into range', () => { + const rows = filteredRowIndexes(aligned(['match', 'update', 'match', 'update', 'update']), 'changed'); + expect(rows).toEqual([1, 3, 4]); + expect(comparePage(rows, 0, 2)).toEqual({ rows: [1, 3], page: 0, pageCount: 2 }); + expect(comparePage(rows, 9, 2)).toEqual({ rows: [4], page: 1, pageCount: 2 }); + expect(comparePage([], 0, 100)).toEqual({ rows: [], page: 0, pageCount: 1 }); + }); + + it('steps to the page holding the next or previous difference, skipping pages of matches', () => { + const ops: AlignRowOp[] = ['update', ...Array(9).fill('match'), 'insert', 'match']; + const a = aligned(ops); + const rows = filteredRowIndexes(a, 'all'); + expect(pageOfNextDifference(a, rows, 0, 3, 1)).toBe(3); // row 10, page 3 + expect(pageOfNextDifference(a, rows, 3, 3, 1)).toBeNull(); + expect(pageOfNextDifference(a, rows, 3, 3, -1)).toBe(0); + }); + + it('slices the aligned pair to the page, keeping the key labels the Sync boxes use', () => { + const view = sliceAligned(aligned(['match', 'update', 'delete']), [2, 1]); + expect(view.rowOps).toEqual(['delete', 'update']); + expect(view.rowKeyLabels).toEqual(['id=2', 'id=1']); + expect(view.rightGap).toEqual([true, false]); + }); + + it('reads the last row’s key, case-insensitively, or nothing when a key column is missing', () => { + const result = { columns: ['ID', 'name'], rows: [[1, 'a'], [7, 'b']] }; + expect(lastRowKey(result, ['id'])).toEqual([7]); + expect(lastRowKey(result, ['missing'])).toBeUndefined(); + expect(lastRowKey({ columns: ['id'], rows: [] }, ['id'])).toBeUndefined(); + }); +}); diff --git a/apps/web/src/frontend/features/sql-editor/lib/compareView.ts b/apps/web/src/frontend/features/sql-editor/lib/compareView.ts new file mode 100644 index 000000000..f7ab31a6e --- /dev/null +++ b/apps/web/src/frontend/features/sql-editor/lib/compareView.ts @@ -0,0 +1,119 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * The compare view's filters and pages, over a key-aligned pair already in + * memory. A compare run holds at most two windows of COMPARE_WINDOW_ROWS, so + * paging is done here, never by asking the databases again. + */ +import type { AlignRowOp, KeyAlignedGrids } from './resultKeyAlign'; + +/** Rows per side a compare run reads: a fixed window, not a setting. */ +export const COMPARE_WINDOW_ROWS = 2000; + +export const COMPARE_PAGE_SIZES = [50, 100, 200, 500] as const; +export const DEFAULT_COMPARE_PAGE_SIZE = 100; + +/** + * What a filter keeps, in migrate's words: a source-only row is one migrate + * adds to the target, a target-only row one it deletes. + */ +export type CompareFilter = 'all' | 'changed' | 'source' | 'target' | 'unresolved' | 'matching'; + +const KEEPS: Record boolean> = { + all: () => true, + changed: (op) => op === 'update', + source: (op) => op === 'delete', + target: (op) => op === 'insert', + unresolved: (op) => op === 'unresolved', + matching: (op) => op === 'match', +}; + +const isDifference = (op: AlignRowOp): boolean => op === 'update' || op === 'insert' || op === 'delete'; + +/** How many rows each filter keeps, over the whole window. */ +export function compareFilterCounts(aligned: Pick): Record { + const counts = { all: 0, changed: 0, source: 0, target: 0, unresolved: 0, matching: 0 }; + for (const op of aligned.rowOps) { + for (const f of Object.keys(KEEPS) as CompareFilter[]) if (KEEPS[f](op)) counts[f] += 1; + } + return counts; +} + +/** Aligned row indexes the filter keeps, in aligned order. */ +export function filteredRowIndexes(aligned: Pick, filter: CompareFilter): number[] { + const keep = KEEPS[filter]; + const out: number[] = []; + aligned.rowOps.forEach((op, i) => { + if (keep(op)) out.push(i); + }); + return out; +} + +export interface ComparePage { + /** Aligned row indexes on this page. */ + rows: number[]; + /** Zero-based, clamped into range. */ + page: number; + pageCount: number; +} + +export function comparePage(rows: readonly number[], page: number, size: number): ComparePage { + const pageCount = Math.max(1, Math.ceil(rows.length / size)); + const p = Math.min(Math.max(0, page), pageCount - 1); + return { rows: rows.slice(p * size, p * size + size), page: p, pageCount }; +} + +/** + * The page holding the next (or previous) difference beyond this page, among + * the filtered rows; null when there is none that way. + */ +export function pageOfNextDifference( + aligned: Pick, + rows: readonly number[], + page: number, + size: number, + direction: 1 | -1 +): number | null { + if (direction === 1) { + for (let pos = (page + 1) * size; pos < rows.length; pos++) { + if (isDifference(aligned.rowOps[rows[pos]!]!)) return Math.floor(pos / size); + } + return null; + } + for (let pos = page * size - 1; pos >= 0; pos--) { + if (isDifference(aligned.rowOps[rows[pos]!]!)) return Math.floor(pos / size); + } + return null; +} + +/** + * The aligned pair cut down to `rows`, for display: tints, Sync checkboxes and + * the grids all read it by its own row index. Counts stay the whole window's. + */ +export function sliceAligned(aligned: KeyAlignedGrids, rows: readonly number[]): KeyAlignedGrids { + const pick = (xs: T[]) => rows.map((i) => xs[i]!); + return { + ...aligned, + leftRows: pick(aligned.leftRows), + rightRows: pick(aligned.rightRows), + leftGap: pick(aligned.leftGap), + rightGap: pick(aligned.rightGap), + rowOps: pick(aligned.rowOps), + rowKeyLabels: pick(aligned.rowKeyLabels), + }; +} + +/** The key values of a result's last row, or undefined when a key column is missing. */ +export function lastRowKey( + result: { columns: string[]; rows: unknown[][] }, + keyNames: readonly string[] +): unknown[] | undefined { + const last = result.rows[result.rows.length - 1]; + if (!last) return undefined; + const lower = result.columns.map((c) => c.toLowerCase()); + const idx = keyNames.map((k) => lower.indexOf(k.toLowerCase())); + if (idx.some((i) => i < 0)) return undefined; + return idx.map((i) => last[i]); +} diff --git a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts index ed5a59f93..ecc5640d1 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.test.ts @@ -178,3 +178,41 @@ describe('alignResultGridsByKey on a partial grid', () => { expect(aligned.rowOps).toEqual(['unresolved', 'unresolved', 'insert']); }); }); + +describe('alignResultGridsByKey in a compare run (both sides read in key order, cut at a window)', () => { + const grid = (ids: (number | string)[]) => ({ columns: ['id', 'v'], rows: ids.map((id) => [id, 'x']) }); + + it('classifies a one-sided row at or before the cut-short side’s last key, and only past it leaves it unresolved', () => { + // Target has more rows than the window: it was cut at id 50. + const aligned = alignResultGridsByKey(grid([10, 20, 40, 60, 70]), grid([10, 30, 40, 50]), ['id'], { + rightComplete: false, + rightWindowEnd: [50], + })!; + const opOf = (id: number) => aligned.rowOps[aligned.rowKeyLabels.indexOf(`id=${id}`)]; + expect(opOf(20)).toBe('delete'); // 20 < 50: the target would have shown it + expect(opOf(60)).toBe('unresolved'); // past 50: it may be further down + expect(opOf(70)).toBe('unresolved'); + expect(opOf(30)).toBe('insert'); // the source is complete + expect(aligned).toMatchObject({ deleteCount: 1, insertCount: 2, unresolvedCount: 2, matchCount: 2 }); + }); + + it('compares numbers by value (2 before 10), including bigints sent as text', () => { + const aligned = alignResultGridsByKey( + grid(['2', '9007199254740993']), + grid(['10']), + ['id'], + { rightComplete: false, rightWindowEnd: ['9007199254740992'] } + )!; + expect(aligned.rowOps[aligned.rowKeyLabels.indexOf('id=2')]).toBe('delete'); + expect(aligned.rowOps[aligned.rowKeyLabels.indexOf('id=9007199254740993')]).toBe('unresolved'); + }); + + it('leaves text keys unresolved: the engine’s collation decides their order, not JavaScript', () => { + const aligned = alignResultGridsByKey(grid(['a', 'b']), grid(['c']), ['id'], { + rightComplete: false, + rightWindowEnd: ['z'], + })!; + expect(aligned.unresolvedCount).toBe(2); + expect(aligned.deleteCount).toBe(0); + }); +}); diff --git a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts index b98390947..7aa533f2c 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/resultKeyAlign.ts @@ -118,6 +118,15 @@ export function alignResultGridsByKey( */ leftComplete?: boolean; rightComplete?: boolean; + /** + * For a side read in key order and cut short (a compare run): the key of + * its last row. A row found only on the other side whose key sorts at or + * before it cannot be further down, so it is classified after all. Only + * numeric keys are compared: engines sort text by their own collation, and + * guessing that order would classify a row the other side does have. + */ + leftWindowEnd?: unknown[]; + rightWindowEnd?: unknown[]; } ): KeyAlignedGrids | null { if (keyNames.length === 0) return null; @@ -206,7 +215,7 @@ export function alignResultGridsByKey( leftGap.push(false); rightGap.push(true); rowKeyLabels.push(keyLabelForRow(lRow, leftKeys)); - if (rightComplete) { + if (rightComplete || withinWindow(lRow, leftKeys, opts?.rightWindowEnd)) { rowOps.push('delete'); deleteCount += 1; } else { @@ -224,7 +233,7 @@ export function alignResultGridsByKey( leftGap.push(true); rightGap.push(false); rowKeyLabels.push(keyLabelForRow(rRow, rightKeys)); - if (leftComplete) { + if (leftComplete || withinWindow(rRow, rightKeys, opts?.leftWindowEnd)) { rowOps.push('insert'); insertCount += 1; } else { @@ -250,6 +259,47 @@ export function alignResultGridsByKey( }; } +const DIGITS = /^\d+$/; + +/** An integer or decimal in text (a bigint arrives as text): `-12`, `3.5`. */ +function isNumericText(text: string): boolean { + const unsigned = text.startsWith('-') ? text.slice(1) : text; + const [whole, fraction, extra] = unsigned.split('.'); + return extra === undefined && DIGITS.test(whole ?? '') && (fraction === undefined || DIGITS.test(fraction)); +} + +/** a < b → negative, a > b → positive; null when either is not a number. */ +function compareNumeric(a: unknown, b: unknown): number | null { + const as = typeof a === 'bigint' ? a.toString() : typeof a === 'number' && Number.isFinite(a) ? String(a) : a; + const bs = typeof b === 'bigint' ? b.toString() : typeof b === 'number' && Number.isFinite(b) ? String(b) : b; + if (typeof as !== 'string' || typeof bs !== 'string' || !isNumericText(as) || !isNumericText(bs)) return null; + if (!as.includes('.') && !bs.includes('.')) { + const x = BigInt(as); + const y = BigInt(bs); + return x < y ? -1 : x > y ? 1 : 0; + } + return Number(as) - Number(bs); +} + +/** + * True when `row`'s key sorts at or before `windowEnd`, compared column by + * column the way ORDER BY does. False when there is no window end or any key + * part is not a number: the row stays unresolved. + */ +function withinWindow( + row: unknown[], + keys: ReturnType, + windowEnd: unknown[] | undefined +): boolean { + if (!windowEnd || windowEnd.length !== keys.length) return false; + for (let i = 0; i < keys.length; i++) { + const cmp = compareNumeric(row[keys[i]!.resultIndex], windowEnd[i]); + if (cmp === null) return false; + if (cmp !== 0) return cmp < 0; + } + return true; +} + function nonKeyDiffer( leftRow: unknown[], rightRow: unknown[], diff --git a/apps/web/src/frontend/shared/api/sqlApi.ts b/apps/web/src/frontend/shared/api/sqlApi.ts index e902cdc94..904939059 100644 --- a/apps/web/src/frontend/shared/api/sqlApi.ts +++ b/apps/web/src/frontend/shared/api/sqlApi.ts @@ -35,6 +35,8 @@ export async function executeSql( opts?: { datagridAction?: 'insert' | 'update' | 'delete'; seek?: { columns: string[]; values: unknown[]; descending?: boolean | boolean[] }; + /** Order each page by these columns (a compare run reads both sides this way). */ + orderBy?: string[]; /** Leave the statements after a failure unrun; `results` then ends at it. */ stopOnError?: boolean; /** One transaction for the whole batch; a failure rolls back what already ran. */ @@ -49,6 +51,7 @@ export async function executeSql( params, ...(opts?.datagridAction ? { datagridAction: opts.datagridAction } : {}), ...(opts?.seek ? { seek: opts.seek } : {}), + ...(opts?.orderBy?.length ? { orderBy: opts.orderBy } : {}), ...(opts?.stopOnError ? { stopOnError: true } : {}), ...(opts?.transaction ? { transaction: true } : {}), }); diff --git a/docs/testing/TEST_IDS.md b/docs/testing/TEST_IDS.md index 52216ef36..9bdb8d3e6 100644 --- a/docs/testing/TEST_IDS.md +++ b/docs/testing/TEST_IDS.md @@ -8,7 +8,7 @@ or `TestIds` for the same tree with autocomplete. `{name}` marks a part filled i time; "in FilterPicker" marks an ID a shared component draws from the prop written here. How to write a test with them: [WRITING_E2E.md](WRITING_E2E.md). -Every control has a test ID: **1000** buttons, text boxes, selects and textareas. +Every control has a test ID: **1008** buttons, text boxes, selects and textareas. ## access @@ -1203,18 +1203,30 @@ Every control has a test ID: **1000** buttons, text boxes, selects and textareas - `sql-result-compare-baseline-{statementIndex}` · select - `sql-result-compare-close-{statementIndex}` · button · Close (Esc) - `sql-result-compare-dest-{statementIndex}` · select + - `sql-result-compare-diff-next-{statementIndex}` · button · Difference › + - `sql-result-compare-diff-prev-{statementIndex}` · button · ‹ Difference - `sql-result-compare-export-csv-{statementIndex}` · button · Export all compare grids as one CSV (columns prefixed by Source / Target) - `sql-result-compare-export-csv-modal-{statementIndex}` · button · Export all compare grids as one CSV + - `sql-result-compare-filter-{f}-{statementIndex}` · button - `sql-result-compare-hint-{statementIndex}` · p - `sql-result-compare-legend-{statementIndex}` · span - `sql-result-compare-maximize-{statementIndex}` · button · Maximize compare - `sql-result-compare-maximized-hint-{statementIndex}` · p - `sql-result-compare-modal-{statementIndex}` · div + - `sql-result-compare-page-label-{statementIndex}` · span + - `sql-result-compare-page-next-{statementIndex}` · button · Next + - `sql-result-compare-page-prev-{statementIndex}` · button · Prev + - `sql-result-compare-page-size-{statementIndex}` · select · Rows per compare page - `sql-result-compare-skip-trigger-{statementIndex}` · input - `sql-result-compare-sync-scroll-{statementIndex}` · input - `sql-result-compare-sync-scroll-modal-{statementIndex}` · input - `sql-result-compare-toggle-{statementIndex}` · input - `sql-result-compare-toolbar-{statementIndex}` · div + - `sql-result-compare-view-{statementIndex}` · div + - `sql-result-compare-window-clear-{statementIndex}` · button · Back to loaded rows + - `sql-result-compare-window-error-{statementIndex}` · span + - `sql-result-compare-window-run-{statementIndex}` · button · Compare rows + - `sql-result-compare-window-summary-{statementIndex}` · span - `sql-result-cred-{connectionId}` · section - `sql-result-cred-close-{connectionId}` · button · Close maximized results - `sql-result-cred-maximize-{connectionId}` · button · Maximize results diff --git a/packages/server/src/features/sql-editor/editor.routes.ts b/packages/server/src/features/sql-editor/editor.routes.ts index 362503b1a..4624e6a5d 100644 --- a/packages/server/src/features/sql-editor/editor.routes.ts +++ b/packages/server/src/features/sql-editor/editor.routes.ts @@ -30,7 +30,7 @@ import { statementVerb, } from '@foxschema/sql'; import { isSingleSqlStatement } from '../../platform/statements/single-statement'; -import { clampOffset, parseSqlSeek } from './sql-page-wrap.service'; +import { clampOffset, parseSqlOrderBy, parseSqlSeek } from './sql-page-wrap.service'; import { makeBeamCellQueryRunner, makeCellQueryRunner } from './code-cell-query.service'; import type { CellQueryRunner } from './code-cell-execute.service'; import { parseBeamEndpoints } from '@foxschema/shared'; @@ -81,12 +81,14 @@ export function createEditorRoutes(deps: EditorRouteDeps): Router { // called next(), so the request hung with no error and no response. const writeIdempotency = idempotency(); router.post('/sql/execute', sqlExecuteLimiter, writeIdempotency, async (req: AppRequest, res: FastifyReply) => { - const { statements, maxRows, offset, params, datagridAction, seek, stopOnError, transaction, ...ref } = req.body as ConnectionRef & { + const { statements, maxRows, offset, params, datagridAction, seek, orderBy, stopOnError, transaction, ...ref } = req.body as ConnectionRef & { statements?: unknown; maxRows?: unknown; offset?: unknown; params?: unknown; seek?: unknown; + /** Order each page by these columns (a compare run). */ + orderBy?: unknown; /** Leave the statements after a failure unrun (results end at it). */ stopOnError?: unknown; /** Wrap the batch in one transaction and roll it back on the first failure. */ @@ -195,6 +197,11 @@ export function createEditorRoutes(deps: EditorRouteDeps): Router { sendError(res, 'invalid_input', parsedSeek.error); return; } + const parsedOrder = parseSqlOrderBy(orderBy); + if (!parsedOrder.ok) { + sendError(res, 'invalid_input', parsedOrder.error); + return; + } try { // Apply the saved connection's schema (CURRENT SCHEMA / search_path) so // unqualified names like ORDERS resolve to DEMO.ORDERS, not USER.ORDERS. @@ -208,7 +215,8 @@ export function createEditorRoutes(deps: EditorRouteDeps): Router { (params as unknown[][] | undefined) ?? [], parsedSeek.value, stopOnError === true, - transaction === true + transaction === true, + parsedOrder.value ); res.send({ results }); } catch (error: unknown) { diff --git a/packages/server/src/features/sql-editor/sql-execute.service.test.ts b/packages/server/src/features/sql-editor/sql-execute.service.test.ts index e33e71f72..d394c5744 100644 --- a/packages/server/src/features/sql-editor/sql-execute.service.test.ts +++ b/packages/server/src/features/sql-editor/sql-execute.service.test.ts @@ -184,6 +184,25 @@ describe('runStatements against a real SQLite file', () => { expect(results[0]).toMatchObject({ ok: true, rowCount: 2, truncated: true, hasNext: true }); }); + it('orders a page by the given keys and says whether more rows follow (a compare run)', async () => { + const results = await runStatements( + 'sqlite', + { connectionString: dbPath }, + ['SELECT id, name FROM t ORDER BY name DESC;'], + 2, + undefined, + 0, + [], + undefined, + false, + false, + ['id'] + ); + expect(results[0]).toMatchObject({ ok: true, hasNext: true }); + const ids = (results[0] as { rows: unknown[][] }).rows.map((r) => Number(r[0])); + expect(ids).toEqual([...ids].sort((a, b) => a - b)); + }); + it('pages SELECT with OFFSET (page 1 is not page 0)', async () => { const page0 = await runStatements( 'sqlite', diff --git a/packages/server/src/features/sql-editor/sql-execute.service.ts b/packages/server/src/features/sql-editor/sql-execute.service.ts index 0a022bcca..394766dc0 100644 --- a/packages/server/src/features/sql-editor/sql-execute.service.ts +++ b/packages/server/src/features/sql-editor/sql-execute.service.ts @@ -1,6 +1,6 @@ import { ConnectionFactory, getAdapter, type ConnectionOptions } from '@foxschema/db'; import { autoAliasSelectColumns, errorMessage, isSubsetDialect } from '@foxschema/sql'; -import { isPageableStatement, syntaxProbeSql, trimPageProbe, wrapSqlForPage, wrapSqlForSeek, type SqlSeek } from './sql-page-wrap.service'; +import { isPageableStatement, syntaxProbeSql, trimPageProbe, wrapSqlForOrderedPage, wrapSqlForPage, wrapSqlForSeek, type SqlSeek } from './sql-page-wrap.service'; /** * Helpers behind POST /api/sql/execute (SQL Editor). One request = one @@ -138,7 +138,13 @@ export async function runStatements( * Engines that auto-commit DDL still report each statement; the rollback is * best-effort there. */ - transaction = false + transaction = false, + /** + * Order each page by these columns (outer ORDER BY, `wrapSqlForOrderedPage`). + * A compare run reads both sides this way, so the same keys line up. + * Ignored with `seek`, and by the SQL-subset stores (Redis, MongoDB). + */ + orderBy?: readonly string[] ): Promise { const schemaName = (schema ?? option.schema)?.trim() || ''; const optionWithSchema: ConnectionOptions = schemaName @@ -212,7 +218,10 @@ export async function runStatements( continue; } - let paged = wrapSqlForPage(sql, dialect, offset, maxRows); + let paged = + orderBy?.length && !seek && !isSubsetDialect(dialect) + ? wrapSqlForOrderedPage(sql, dialect, orderBy, offset, maxRows) + : wrapSqlForPage(sql, dialect, offset, maxRows); if (seek && offset === 0) { const keyed = wrapSqlForSeek(sql, dialect, seek, maxRows, params.length); if ('error' in keyed) { diff --git a/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts b/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts index e786de224..1ff7fad4e 100644 --- a/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts +++ b/packages/server/src/features/sql-editor/sql-page-wrap.service.test.ts @@ -2,11 +2,48 @@ import { describe, expect, it } from 'vitest'; import { hasTopLevelOrderBy, isPageableStatement, + parseSqlOrderBy, trimPageProbe, wrapSqlForPage, + wrapSqlForOrderedPage, wrapSqlForSeek, } from './sql-page-wrap.service'; +describe('ordered pages (a compare run)', () => { + it('orders the outer query by the keys, in each engine\'s paging syntax, fetching one extra row', () => { + expect(wrapSqlForOrderedPage('SELECT * FROM t;', 'postgres', ['id'], 0, 2000)).toBe( + 'SELECT * FROM (SELECT * FROM t) AS fox_page ORDER BY "id" LIMIT 2001 OFFSET 0' + ); + expect(wrapSqlForOrderedPage('SELECT * FROM t', 'mysql', ['a', 'b'], 0, 10)).toBe( + 'SELECT * FROM (SELECT * FROM t) AS fox_page ORDER BY `a`, `b` LIMIT 11 OFFSET 0' + ); + expect(wrapSqlForOrderedPage('SELECT * FROM t', 'oracle', ['ID'], 0, 10)).toMatch( + /ORDER BY "ID" OFFSET 0 ROWS FETCH NEXT 11 ROWS ONLY$/ + ); + expect(wrapSqlForOrderedPage('SELECT * FROM t', 'db2', ['ID'], 0, 10)).toMatch( + /ORDER BY "ID" OFFSET 0 ROWS FETCH FIRST 11 ROWS ONLY$/ + ); + }); + + it('on SQL Server drops a plain top-level ORDER BY, which a derived table refuses', () => { + expect(wrapSqlForOrderedPage('SELECT id FROM t ORDER BY name', 'sqlserver', ['id'], 0, 10)).toBe( + 'SELECT * FROM (SELECT id FROM t) AS fox_page ORDER BY [id] OFFSET 0 ROWS FETCH NEXT 11 ROWS ONLY' + ); + // One that pages itself stays: it is legal there, and it limits the rows. + expect( + wrapSqlForOrderedPage('SELECT id FROM t ORDER BY id OFFSET 5 ROWS', 'azuresql', ['id'], 0, 10) + ).toContain('ORDER BY id OFFSET 5 ROWS) AS fox_page'); + }); + + it('takes only plain column names, never SQL', () => { + expect(parseSqlOrderBy(undefined)).toEqual({ ok: true, value: undefined }); + expect(parseSqlOrderBy(['id', 'tenant_id'])).toEqual({ ok: true, value: ['id', 'tenant_id'] }); + for (const bad of [[], 'id', ['id; DROP TABLE t'], ['1=1 --'], [3], Array.from({ length: 9 }, (_, i) => `c${i}`)]) { + expect(parseSqlOrderBy(bad).ok, JSON.stringify(bad)).toBe(false); + } + }); +}); + describe('paging Redis and MongoDB', () => { // They read SQL through a parser with no subqueries: a nested page was // refused ("Unsupported clause") for every Redis and MongoDB SELECT. diff --git a/packages/server/src/features/sql-editor/sql-page-wrap.service.ts b/packages/server/src/features/sql-editor/sql-page-wrap.service.ts index c9e874d16..4796f02e6 100644 --- a/packages/server/src/features/sql-editor/sql-page-wrap.service.ts +++ b/packages/server/src/features/sql-editor/sql-page-wrap.service.ts @@ -12,6 +12,7 @@ import { isPageableStatement } from '@foxschema/db'; import { + findTopLevelOrderByIndex, isSafeSeekColumn, isSubsetDialect, isTsqlDialect, @@ -195,6 +196,57 @@ export function wrapSqlForPage( return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} LIMIT ${fetchLimit} OFFSET ${offset}`; } +/** Longest key a compare run orders by: more columns than this is not a key. */ +const MAX_ORDER_COLUMNS = 8; + +/** + * `orderBy` from a request: plain column names, or nothing. Anything else is + * refused, never quoted into SQL as given. + */ +export function parseSqlOrderBy(raw: unknown): { ok: true; value?: string[] } | { ok: false; error: string } { + if (raw == null) return { ok: true, value: undefined }; + if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_ORDER_COLUMNS) { + return { ok: false, error: `orderBy must be 1 to ${MAX_ORDER_COLUMNS} column names` }; + } + if (!raw.every((c) => typeof c === 'string' && isSafeSeekColumn(c))) { + return { ok: false, error: 'orderBy must be plain column names' }; + } + return { ok: true, value: raw as string[] }; +} + +/** + * One page of `sql` in `orderBy` order: the order goes on the outer query, + * where every engine honours it (an ORDER BY inside a derived table is not + * guaranteed to survive, and SQL Server refuses one without TOP or OFFSET — + * so on SQL Server a plain top-level ORDER BY of the statement is dropped; + * the outer one decides anyway). Fetches `limit + 1` so the caller learns + * whether more rows follow. A compare run reads both sides this way. + */ +export function wrapSqlForOrderedPage( + sql: string, + dialect: string, + orderBy: readonly string[], + offset: number, + limit: number +): string { + const d = dialect.toLowerCase(); + let inner = sql.trim().replace(/;+\s*$/, ''); + const fetchLimit = limit + 1; + const orderSql = orderBy.map((c) => quoteSqlIdentifier(c, dialect)).join(', '); + if (isTsqlDialect(d)) { + const at = findTopLevelOrderByIndex(inner); + if (at >= 0 && !/\boffset\b/i.test(inner.slice(at))) inner = inner.slice(0, at).trimEnd(); + return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} ORDER BY ${orderSql} OFFSET ${offset} ROWS FETCH NEXT ${fetchLimit} ROWS ONLY`; + } + if (d === 'oracle') { + return `SELECT * FROM (${inner}) ${PAGE_ALIAS} ORDER BY ${orderSql} OFFSET ${offset} ROWS FETCH NEXT ${fetchLimit} ROWS ONLY`; + } + if (d === 'db2') { + return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} ORDER BY ${orderSql} OFFSET ${offset} ROWS FETCH FIRST ${fetchLimit} ROWS ONLY`; + } + return `SELECT * FROM (${inner}) AS ${PAGE_ALIAS} ORDER BY ${orderSql} LIMIT ${fetchLimit} OFFSET ${offset}`; +} + export interface SqlSeek { columns: string[]; values: unknown[];