From ccccad0131c8e7ab8b46b7725027b93943ff8b1e Mon Sep 17 00:00:00 2001 From: huyplb Date: Sun, 25 Oct 2026 09:33:31 -0600 Subject: [PATCH 1/3] wip: Redis keys as tables; subset paging; cross-dialect fixes --- apps/e2e/scripts/run-all.mjs | 2 + apps/e2e/src/generated/test-ids.ts | 12 + apps/e2e/src/tests/redis-browse.test.ts | 159 +++++ apps/e2e/src/tests/redis-editor.test.ts | 634 ++++++++++++++++++ .../compare/components/CompareToolbar.tsx | 2 +- .../features/compare/state/useSyncStore.ts | 2 +- .../sql-editor/components/DataGrid.tsx | 5 +- .../sql-editor/components/ResultsPanel.tsx | 10 +- .../sql-editor/components/SqlEditorView.tsx | 28 +- .../components/SqlSchemaExplorer.tsx | 61 +- .../sql-editor/state/useSqlEditorStore.ts | 10 +- .../components/BackupServerActions.tsx | 2 +- .../utilities/components/CloneTableModal.tsx | 2 +- .../components/DatabaseAccessModal.tsx | 2 +- .../components/IndexManagementModal.tsx | 2 +- .../components/ServerInsightsModal.tsx | 2 +- docs/testing/TEST_IDS.md | 4 + .../providers/mongodb/mongodb.adapter.test.ts | 34 + .../src/providers/mongodb/mongodb.adapter.ts | 4 + .../db/src/providers/provider-registry.ts | 4 + .../src/providers/redis/redis.adapter.test.ts | 83 +++ .../db/src/providers/redis/redis.adapter.ts | 88 ++- .../providers/redis/redis.client-double.ts | 158 +++++ .../db/src/providers/redis/redis.it.test.ts | 65 +- .../providers/redis/redis.keyspace.test.ts | 203 ++++++ .../db/src/providers/redis/redis.keyspace.ts | 361 ++++++++++ .../providers/redis/redis.provider.test.ts | 58 ++ .../db/src/providers/redis/redis.provider.ts | 129 ++++ .../history/live-capture.service.test.ts | 38 ++ .../features/history/live-capture.service.ts | 7 + .../sql-editor/sql-execute.service.test.ts | 16 + .../sql-editor/sql-execute.service.ts | 9 +- .../sql-editor/sql-page-wrap.service.test.ts | 10 + .../sql-editor/sql-page-wrap.service.ts | 6 + .../sql/src/cores/connection-auth.test.ts | 12 + packages/sql/src/cores/connection-auth.ts | 14 +- packages/sql/src/cores/schema-to-tables.ts | 1 + packages/sql/src/index.ts | 2 +- .../sql/src/interfaces/schema.interface.ts | 8 + .../modules/sql-editor/select-alias.test.ts | 11 + .../src/modules/sql-editor/select-alias.ts | 4 + .../src/modules/sql-editor/sql-subset.test.ts | 120 +++- .../sql/src/modules/sql-editor/sql-subset.ts | 201 +++++- packages/ui-shared/src/index.ts | 2 +- 44 files changed, 2501 insertions(+), 86 deletions(-) create mode 100644 apps/e2e/src/tests/redis-browse.test.ts create mode 100644 apps/e2e/src/tests/redis-editor.test.ts create mode 100644 packages/db/src/providers/redis/redis.client-double.ts create mode 100644 packages/db/src/providers/redis/redis.keyspace.test.ts create mode 100644 packages/db/src/providers/redis/redis.keyspace.ts create mode 100644 packages/db/src/providers/redis/redis.provider.test.ts create mode 100644 packages/db/src/providers/redis/redis.provider.ts create mode 100644 packages/server/src/features/history/live-capture.service.test.ts diff --git a/apps/e2e/scripts/run-all.mjs b/apps/e2e/scripts/run-all.mjs index 307f47991..6b31030e4 100644 --- a/apps/e2e/scripts/run-all.mjs +++ b/apps/e2e/scripts/run-all.mjs @@ -123,6 +123,8 @@ const ALWAYS = [ 'src/tests/sql-editor-code-cell-faker.test.ts', 'src/tests/lazy-loading.test.ts', 'src/tests/sql-editor-result-edit.test.ts', + 'src/tests/redis-browse.test.ts', + 'src/tests/redis-editor.test.ts', 'src/tests/sql-editor-peek-row-form.test.ts', 'src/tests/sql-editor-peek-stacking.test.ts', 'src/tests/sql-editor-compare-multi-target.test.ts', diff --git a/apps/e2e/src/generated/test-ids.ts b/apps/e2e/src/generated/test-ids.ts index 0fb3edf09..0172a68c1 100644 --- a/apps/e2e/src/generated/test-ids.ts +++ b/apps/e2e/src/generated/test-ids.ts @@ -1867,6 +1867,8 @@ export const TestIds = { sqlGridCtxCopyValues: 'sql-grid-ctx-copy-values', /** button · Open row */ sqlGridCtxOpenRow: 'sql-grid-ctx-open-row', + /** div */ + sqlGridError: 'sql-grid-error', /** button · Export CSV */ sqlGridExportBtn: 'sql-grid-export-btn', /** button · Export CSV */ @@ -2221,6 +2223,8 @@ export const TestIds = { sqlResultCredMaximizedHint: (connectionId: string | number) => `sql-result-cred-maximized-hint-${connectionId}`, /** div */ sqlResultCredModal: (connectionId: string | number) => `sql-result-cred-modal-${connectionId}`, + /** div */ + sqlResultError: 'sql-result-error', /** button · Maximize results */ sqlResultMaximize: (statementIndex: string | number) => `sql-result-maximize-${statementIndex}`, /** p */ @@ -2253,6 +2257,8 @@ export const TestIds = { sqlResultsSideBySide: 'sql-results-side-by-side', /** div */ sqlResultsWarnings: 'sql-results-warnings', + /** div */ + sqlRunError: 'sql-run-error', }, SelectColumnPicker: { /** button · SELECT * */ @@ -2374,6 +2380,8 @@ export const TestIds = { /** button */ sqlExplorerParamInsert: (name: string | number, name2: string | number) => `sql-explorer-param-insert-${name}-${name2}`, /** span */ + sqlExplorerStoreType: (name: string | number) => `sql-explorer-store-type-${name}`, + /** span */ sqlExplorerTableStats: (name: string | number) => `sql-explorer-table-stats-${name}`, /** button */ sqlExplorerToggle: (name: string | number) => `sql-explorer-toggle-${name}`, @@ -4648,6 +4656,7 @@ export type TestId = | `sql-explorer-from-${string}` | `sql-explorer-object-${string}` | `sql-explorer-param-insert-${string}-${string}` + | `sql-explorer-store-type-${string}` | `sql-explorer-table-stats-${string}` | `sql-explorer-toggle-${string}` | `sql-filter-op-${string}` @@ -4682,6 +4691,7 @@ export type TestId = | 'sql-grid-ctx-copy-headers-only' | 'sql-grid-ctx-copy-values' | 'sql-grid-ctx-open-row' + | 'sql-grid-error' | 'sql-grid-export-btn' | 'sql-grid-export-csv' | 'sql-grid-export-json' @@ -4751,6 +4761,7 @@ export type TestId = | `sql-result-cred-maximize-${string}` | `sql-result-cred-maximized-hint-${string}` | `sql-result-cred-modal-${string}` + | 'sql-result-error' | `sql-result-maximize-${string}` | `sql-result-maximized-hint-${string}` | `sql-result-modal-${string}` @@ -4772,6 +4783,7 @@ export type TestId = | `sql-row-selected-${string}` | `sql-row-sync-check-${string}` | 'sql-run-btn' + | 'sql-run-error' | 'sql-runs-clear' | 'sql-runs-close' | 'sql-runs-drawer' diff --git a/apps/e2e/src/tests/redis-browse.test.ts b/apps/e2e/src/tests/redis-browse.test.ts new file mode 100644 index 000000000..a51f9d949 --- /dev/null +++ b/apps/e2e/src/tests/redis-browse.test.ts @@ -0,0 +1,159 @@ +/** + * Redis in the SQL editor, in a real browser against the running server. + * + * Three things were broken here, and this suite is what they look like fixed: + * + * - the schema explorer listed nothing ("No provider registered for dialect: + * redis"), so no key was browsable at all; + * - every SELECT failed with "Paging failed … Unsupported clause", because the + * editor nested it in a page wrapper the Redis translator cannot read; + * - a list, set, sorted set or stream read as `{ id, type }` only, and a JSON + * document as one `value` holding the whole text. + * + * Seeds its own keys under a per-run prefix in the dev container's Redis and + * removes them afterwards. Skips when that container is not reachable. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import type { Page } from 'playwright'; +import { buildDriver, quitDriver, waitFor, BASE_URL } from '../helpers/driver.js'; +import { sessionCookie } from '../helpers/app-session.js'; +import { saveScreenshot } from '../helpers/screenshot.js'; +import { byTestId } from '../helpers/test-ids.js'; +import { AppPage } from '../pages/AppPage.js'; +import { SqlEditorPage } from '../pages/SqlEditorPage.js'; + +const API_URL = process.env.E2E_API_URL ?? 'http://127.0.0.1:3210'; +const RUN = Date.now().toString(36); +const P = `e2e${RUN}`; +const NAME = `E2E Redis ${RUN}`; +const KEYS = { + users: `${P}-users`, + docs: `${P}-docs`, + jobs: `${P}-jobs`, + board: `${P}-board`, +}; + +function redis(...args: string[]): string { + return execFileSync('docker', ['exec', '-i', 'foxschema-redis', 'redis-cli', ...args], { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }); +} + +function redisReachable(): boolean { + try { + return redis('PING').trim() === 'PONG'; + } catch { + return false; + } +} + +describe.skipIf(!redisReachable())('SQL Editor · Redis keys as tables', () => { + let admin = ''; + let connectionId = ''; + let driver: Page; + let sql: SqlEditorPage; + + const api = async (method: string, path: string, body?: unknown) => { + const res = await fetch(`${API_URL}/api${path}`, { + method, + headers: { ...(body !== undefined ? { 'content-type': 'application/json' } : {}), cookie: admin }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + return { status: res.status, json: (await res.json().catch(() => null)) as any }; + }; + + beforeAll(async () => { + redis('HSET', `${KEYS.users}:1`, 'name', 'Ana', 'plan', 'free'); + redis('HSET', `${KEYS.users}:2`, 'name', 'Bo', 'plan', 'pro'); + redis('SET', `${KEYS.docs}:a`, JSON.stringify({ title: 'Hello', meta: { v: 1 } })); + redis('RPUSH', KEYS.jobs, JSON.stringify({ job: 'send', to: 'ana@x' }), JSON.stringify({ job: 'bill', to: 'bo@x' })); + redis('ZADD', KEYS.board, '20', 'bo', '10', 'al'); + + admin = await sessionCookie(); + const saved = await api('POST', '/connections', { + name: NAME, + dialect: 'redis', + option: { host: '127.0.0.1', port: 6379, schema: '0' }, + }); + expect(saved.status, JSON.stringify(saved.json)).toBe(200); + connectionId = saved.json.connection.id; + + driver = await buildDriver(); + sql = new SqlEditorPage(driver); + await new AppPage(driver).open(); + await sql.resetPersistedEditorState(); + await driver.reload(); + await driver.waitForSelector(byTestId('toolbar')); + await sql.openView(); + await sql.checkConnection(NAME); + }, 180_000); + + afterAll(async () => { + if (driver) await quitDriver(driver); + if (connectionId) await api('DELETE', `/connections/${connectionId}`); + for (const key of [`${KEYS.users}:1`, `${KEYS.users}:2`, `${KEYS.docs}:a`, KEYS.jobs, KEYS.board]) { + redis('DEL', key); + } + }); + + it('the schema explorer lists prefixes and collection keys, each with its type', async () => { + await sql.ensureSidebarSectionOpen('schema'); + for (const [name, type] of [ + [`${KEYS.users}`, 'hash'], + [`${KEYS.docs}`, 'json'], + [KEYS.jobs, 'list'], + [KEYS.board, 'zset'], + ] as const) { + const badge = byTestId(`sql-explorer-store-type-${name}`); + await waitFor(driver, badge, 30_000); + expect((await driver.locator(badge).innerText()).toLowerCase()).toBe(type); + } + // A key has no DDL: no table blueprint for any of them, and no New table. + expect(await driver.locator(`${byTestId('sql-schema-explorer')} ${byTestId('sql-open-blueprint')}`).count()).toBe(0); + expect(await driver.locator(byTestId('sql-schema-new-table')).count()).toBe(0); + await saveScreenshot(driver, 'redis-explorer'); + }, 120_000); + + it('a SELECT on a list key runs and shows its elements as rows', async () => { + await sql.setSql(`SELECT * FROM "${KEYS.jobs}";`); + await sql.run(); + await sql.waitForResults(); + const text = await sql.resultsText(); + expect(text).not.toMatch(/Paging failed|Unsupported clause/); + for (const cell of ['send', 'bill', 'ana@x', 'bo@x']) expect(text).toContain(cell); + await saveScreenshot(driver, 'redis-list-select'); + }, 120_000); + + it('a JSON document reads as its fields, and a sorted set by score', async () => { + await sql.setSql(`SELECT * FROM ${KEYS.docs};`); + await sql.run(); + await sql.waitForResults(); + const doc = await sql.resultsText(); + expect(doc).toContain('Hello'); + expect(doc).toContain('{"v":1}'); + + await sql.setSql(`SELECT member, score FROM "${KEYS.board}";`); + await sql.run(); + await sql.waitForResults(); + const board = await sql.resultsText(); + expect(board.indexOf('al')).toBeGreaterThan(-1); + expect(board.indexOf('al')).toBeLessThan(board.indexOf('bo')); + }, 120_000); + + it('Data peek opens a collection key from the explorer', async () => { + const label = driver.locator(byTestId('sql-schema-explorer')).getByText(KEYS.jobs, { exact: true }).first(); + await label.click({ modifiers: [process.platform === 'darwin' ? 'Meta' : 'Control'] }); + await waitFor(driver, byTestId('data-peek'), 30_000); + await driver.waitForFunction( + () => /send/.test(document.querySelector('[data-testid="data-peek"]')?.textContent ?? ''), + undefined, + { timeout: 30_000 } + ); + const peek = await driver.locator(byTestId('data-peek')).innerText(); + expect(peek).toContain('bill'); + expect(peek).not.toMatch(/Paging failed|Unsupported clause/); + await saveScreenshot(driver, 'redis-peek-list'); + }, 120_000); +}); diff --git a/apps/e2e/src/tests/redis-editor.test.ts b/apps/e2e/src/tests/redis-editor.test.ts new file mode 100644 index 000000000..0dea25e12 --- /dev/null +++ b/apps/e2e/src/tests/redis-editor.test.ts @@ -0,0 +1,634 @@ +/** + * Redis in the SQL editor, end to end: basic SQL, every built-in sample, + * Compare data with Data migrate, and Data peek with its row editor. + * + * Three Redis databases in the dev container, all under a per-run key prefix: + * + * db 0 "Redis A" the main data set: a hash prefix, JSON documents, a list, + * a set, a sorted set and a stream + * db 1 "Redis B" the same hash prefix, differing from A, for Compare data + * db 2 "Redis S" a sandbox the samples run against, so a sample that writes + * cannot touch A or B + * + * Every assertion reads the rows the grid shows, or the keys Redis holds + * afterwards (`redis-cli`), never just the absence of an error. + * + * Requires the web app + API and the dev Redis container (`foxschema-redis`). + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { appendFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { Page } from 'playwright'; +import { buildDriver, quitDriver, clickWhen, waitFor } from '../helpers/driver.js'; +import { sessionCookie } from '../helpers/app-session.js'; +import { saveScreenshot } from '../helpers/screenshot.js'; +import { byTestId } from '../helpers/test-ids.js'; +import { AppPage } from '../pages/AppPage.js'; +import { SqlEditorPage } from '../pages/SqlEditorPage.js'; + +const API_URL = process.env.E2E_API_URL ?? 'http://127.0.0.1:3210'; +const RUN = Date.now().toString(36); +const P = `rt${RUN}`; +const T = { + users: `${P}users`, + docs: `${P}docs`, + jobs: `${P}jobs`, + tags: `${P}tags`, + board: `${P}board`, + log: `${P}log`, +}; +const CONN = { a: `Redis A ${RUN}`, b: `Redis B ${RUN}`, s: `Redis S ${RUN}` }; +/** One line per sample as it finishes (see the samples test). */ +const SAMPLE_LOG = process.env.FOX_SAMPLE_LOG ?? join(tmpdir(), 'fox-redis-samples.log'); + +function redis(db: number, ...args: string[]): string { + return execFileSync('docker', ['exec', '-i', 'foxschema-redis', 'redis-cli', '-n', String(db), ...args], { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }).trim(); +} + +function redisReachable(): boolean { + try { + return redis(0, 'PING') === 'PONG'; + } catch { + return false; + } +} + +/** Every key in `db` matching `pattern`. */ +function keys(db: number, pattern: string): string[] { + const out = redis(db, '--scan', '--pattern', pattern); + return out ? out.split('\n').filter(Boolean).sort() : []; +} + +/** A hash as an object. */ +function hash(db: number, key: string): Record { + const flat = redis(db, 'HGETALL', key).split('\n').filter((l) => l !== ''); + const out: Record = {}; + for (let i = 0; i + 1 < flat.length; i += 2) out[flat[i]!] = flat[i + 1]!; + return out; +} + +function seed(): void { + const users: Array<[string, string, string]> = [ + ['1', 'Ana', 'free'], + ['2', 'Bo', 'pro'], + ['3', 'Cy', 'pro'], + ['4', 'Di', 'free'], + ['5', 'Ed', 'team'], + ]; + for (const [id, name, plan] of users) redis(0, 'HSET', `${T.users}:${id}`, 'name', name, 'plan', plan); + redis(0, 'SET', `${T.docs}:a`, JSON.stringify({ title: 'Hello', tags: ['x'], n: 1 })); + redis(0, 'SET', `${T.docs}:b`, JSON.stringify({ title: 'World', n: 2 })); + redis(0, 'RPUSH', T.jobs, JSON.stringify({ job: 'send', to: 'ana' }), JSON.stringify({ job: 'bill', to: 'bo' }), 'plain-entry'); + redis(0, 'SADD', T.tags, 'red', 'green', 'blue'); + redis(0, 'ZADD', T.board, '30', 'cy', '10', 'al', '20', 'bo'); + redis(0, 'XADD', T.log, '*', 'level', 'info', 'msg', 'started'); + redis(0, 'XADD', T.log, '*', 'level', 'warn', 'msg', 'slow'); + + // B: user 1 the same, 2 changed, 6 only here; 3–5 only in A. + redis(1, 'HSET', `${T.users}:1`, 'name', 'Ana', 'plan', 'free'); + redis(1, 'HSET', `${T.users}:2`, 'name', 'Bo', 'plan', 'free'); + redis(1, 'HSET', `${T.users}:6`, 'name', 'Fay', 'plan', 'pro'); +} + +function cleanup(): void { + for (const db of [0, 1, 2]) for (const k of keys(db, `${P}*`)) redis(db, 'DEL', k); +} + +describe.skipIf(!redisReachable())('SQL Editor · Redis', () => { + let admin = ''; + const connectionIds: string[] = []; + let driver: Page; + let sql: SqlEditorPage; + /** Keys in the samples' sandbox before they ran, so only what they made is removed. */ + let sandboxBefore = new Set(); + + const api = async (method: string, path: string, body?: unknown) => { + const res = await fetch(`${API_URL}/api${path}`, { + method, + headers: { ...(body !== undefined ? { 'content-type': 'application/json' } : {}), cookie: admin }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + return { status: res.status, json: (await res.json().catch(() => null)) as any }; + }; + + /** Tick exactly these destinations. */ + async function destinations(...names: string[]): Promise { + await sql.ensureSidebarSectionOpen('destinations'); + for (const name of Object.values(CONN)) { + const box = driver.locator(byTestId(`sql-conn-check-${name}`)); + await box.waitFor({ timeout: 15_000 }); + const want = names.includes(name); + if ((await box.isChecked()) !== want) await box.click(); + // Plain polling: this also runs in beforeAll, where expect.poll is not allowed. + const deadline = Date.now() + 10_000; + while ((await box.isChecked()) !== want) { + if (Date.now() > deadline) throw new Error(`${name} did not become ${want ? 'checked' : 'unchecked'}`); + await driver.waitForTimeout(200); + } + } + } + + /** + * Run `text`; confirm the Safe mode prompt when it is a write. A statement + * with no WHERE also needs its acknowledgement ticked before Confirm enables. + */ + async function runSql(text: string, opts: { write?: boolean } = {}): Promise { + await sql.setSql(text); + await sql.run(); + if (opts.write) { + const ack = driver.locator(byTestId('sql-safe-no-where-ack-check')); + if (await ack.isVisible({ timeout: 2_000 }).catch(() => false)) await ack.check(); + await sql.confirmWriteIfShown(); + } + await sql.waitForResults(60_000); + } + + /** The rows of every result grid on screen, as cell text, without the # column. */ + const gridRows = (): Promise => + driver.evaluate(() => + [...document.querySelectorAll('[data-testid="sql-data-grid"] tbody tr')] + .filter((tr) => tr.querySelector('[data-testid="sql-row-num"]')) + .map((tr) => [...tr.querySelectorAll('td')].slice(1).map((td) => (td.textContent ?? '').trim())) + ); + + /** Column headers of the first result grid. */ + const gridColumns = (): Promise => + driver.evaluate(() => + [...(document.querySelector('[data-testid="sql-data-grid"]')?.querySelectorAll('[data-testid="sql-col-header"]') ?? [])].map( + // A header reads "▲nametext": sort arrow, name, type badge (uppercase by CSS only). + (th) => (th.textContent ?? '').trim().replace(/^[^A-Za-z_]+/, '').replace(/(text|num|bool|date|json)$/i, '') + ) + ); + + /** Error text the results show, statement errors and run errors alike. */ + const errors = (): Promise => + driver.evaluate(() => + [ + ...document.querySelectorAll( + '[data-testid="sql-grid-error"], [data-testid="sql-result-error"], [data-testid="sql-run-error"]' + ), + ].map((e) => + (e.textContent ?? '').trim() + ) + ); + + /** + * Wait for a run to finish, confirming a write when Safe mode asks (the + * sandbox database takes it). True when it settled within `timeoutMs`. + * The results container never mounts for a run refused before it starts, + * so waiting on it would sit out the whole timeout for every refusal. + */ + async function settle(timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + const busy = () => + driver.evaluate(() => + Boolean(document.querySelector('[data-testid="sql-run-btn"] .animate-spin, [data-testid="sql-results-running"]')) + ); + // A run refused before it starts never spins; give it 3 s to show it began. + const startBy = Date.now() + 3_000; + while (!(await busy()) && Date.now() < startBy) { + if (await driver.locator(byTestId('sql-write-confirm')).isVisible()) break; + await driver.waitForTimeout(100); + } + while (Date.now() < deadline) { + if (await driver.locator(byTestId('sql-write-confirm')).isVisible()) { + const ack = driver.locator(byTestId('sql-safe-no-where-ack-check')); + if (await ack.isVisible()) await ack.check(); + await clickWhen(driver, byTestId('sql-write-confirm-btn')); + await driver.waitForTimeout(300); + continue; + } + if (!(await busy())) return true; + await driver.waitForTimeout(250); + } + return false; + } + + beforeAll(async () => { + cleanup(); + seed(); + sandboxBefore = new Set(keys(2, '*')); + admin = await sessionCookie(); + for (const [name, db] of [ + [CONN.a, '0'], + [CONN.b, '1'], + [CONN.s, '2'], + ] as const) { + const saved = await api('POST', '/connections', { name, dialect: 'redis', option: { host: '127.0.0.1', port: 6379, schema: db } }); + expect(saved.status, JSON.stringify(saved.json)).toBe(200); + connectionIds.push(saved.json.connection.id); + } + + driver = await buildDriver(); + sql = new SqlEditorPage(driver); + await new AppPage(driver).open(); + await sql.resetPersistedEditorState(); + await driver.reload(); + await driver.waitForSelector(byTestId('toolbar')); + await sql.openView(); + }, 180_000); + + afterAll(async () => { + if (driver) await quitDriver(driver); + for (const id of connectionIds) await api('DELETE', `/connections/${id}`); + cleanup(); + for (const k of keys(2, '*')) if (!sandboxBefore.has(k)) redis(2, 'DEL', k); + }); + + // ── Basic SQL ────────────────────────────────────────────────────────── + + describe('basic SQL', () => { + beforeAll(async () => { + await destinations(CONN.a); + }, 60_000); + + it('SELECT * reads every hash under a prefix, its fields as columns', async () => { + await runSql(`SELECT * FROM ${T.users};`); + expect(await errors()).toEqual([]); + const rows = await gridRows(); + expect(rows).toHaveLength(5); + expect(new Set(await gridColumns())).toEqual(new Set(['id', 'name', 'plan'])); + expect(rows.map((r) => r.join(' ')).join('\n')).toMatch(/Ana[\s\S]*Ed|Ed[\s\S]*Ana/); + }, 60_000); + + it('selects columns, filters by equality, and reads one key by id', async () => { + await runSql(`SELECT id, name FROM ${T.users} WHERE plan = 'pro';`); + expect(await gridColumns()).toEqual(['id', 'name']); + expect((await gridRows()).map((r) => r[1]).sort()).toEqual(['Bo', 'Cy']); + + await runSql(`SELECT * FROM ${T.users} WHERE id = '4';`); + const rows = await gridRows(); + expect(rows).toHaveLength(1); + expect(rows[0]).toContain('Di'); + }, 60_000); + + it('honours LIMIT and OFFSET', async () => { + await runSql(`SELECT * FROM ${T.users} LIMIT 2;`); + expect(await gridRows()).toHaveLength(2); + await runSql(`SELECT * FROM ${T.users} LIMIT 10 OFFSET 3;`); + expect(await gridRows()).toHaveLength(2); + }, 60_000); + + it('pages the grid with Rows/page, Next and Prev', async () => { + await driver.fill(byTestId('sql-max-rows'), '2'); + try { + await runSql(`SELECT * FROM ${T.users};`); + const seen = new Set(); + const page1 = await gridRows(); + expect(page1).toHaveLength(2); + page1.forEach((r) => seen.add(r[0]!)); + expect(await sql.pageNextEnabled()).toBe(true); + + await sql.clickPageNext(); + await expect.poll(async () => (await gridRows())[0]?.[0]).not.toBe(page1[0]![0]); + (await gridRows()).forEach((r) => seen.add(r[0]!)); + await sql.clickPageNext(); + await expect.poll(async () => (await gridRows()).length).toBe(1); + (await gridRows()).forEach((r) => seen.add(r[0]!)); + expect([...seen].sort()).toEqual(['1', '2', '3', '4', '5']); + expect(await sql.pageNextEnabled()).toBe(false); + + await sql.clickPagePrev(); + await expect.poll(async () => (await gridRows()).length).toBe(2); + } finally { + await driver.fill(byTestId('sql-max-rows'), '200'); + } + }, 90_000); + + it('INSERT, UPDATE and DELETE write the hash — and nothing else', async () => { + await runSql(`INSERT INTO ${T.users} (id, name, plan) VALUES ('9', 'Zed', 'free');`, { write: true }); + expect(await errors()).toEqual([]); + expect(hash(0, `${T.users}:9`)).toEqual({ name: 'Zed', plan: 'free' }); + + await runSql(`UPDATE ${T.users} SET plan = 'pro' WHERE id = '9';`, { write: true }); + expect(hash(0, `${T.users}:9`)).toEqual({ name: 'Zed', plan: 'pro' }); + + // A non-id predicate that does not hold leaves the row alone. + await runSql(`UPDATE ${T.users} SET plan = 'team' WHERE id = '9' AND name = 'Nobody';`, { write: true }); + expect(hash(0, `${T.users}:9`).plan).toBe('pro'); + + await runSql(`DELETE FROM ${T.users} WHERE id = '9';`, { write: true }); + expect(keys(0, `${T.users}:9`)).toEqual([]); + expect(keys(0, `${T.users}:*`)).toHaveLength(5); + }, 90_000); + + it.each([ + ['a DELETE with no WHERE', (t: typeof T) => `DELETE FROM ${t.users};`, true], + ['a range predicate', (t: typeof T) => `SELECT * FROM ${t.users} WHERE id > 1;`, false], + ['a join', (t: typeof T) => `SELECT * FROM ${t.users} u JOIN ${t.docs} d ON u.id = d.id;`, false], + ['an OR', (t: typeof T) => `SELECT * FROM ${t.users} WHERE id = '1' OR id = '2';`, false], + ['an aggregate', (t: typeof T) => `SELECT COUNT(*) FROM ${t.users};`, false], + ] as const)('refuses %s with a message, and changes nothing', async (_label, build, write) => { + await runSql(build(T), { write }); + const errs = await errors(); + expect(errs.length, 'no error shown').toBeGreaterThan(0); + expect(errs.join('\n')).not.toMatch(/Paging failed/); + expect(keys(0, `${T.users}:*`)).toHaveLength(5); + }, 60_000); + + it('reads a list, a set, a sorted set and a stream as tables of their elements', async () => { + await runSql(`SELECT * FROM "${T.jobs}";`); + expect(await gridColumns()).toEqual(['index', 'job', 'to', 'value']); + // A JSON element fills `job`; the plain one fills `value` (the grid prints NULL for the other). + const cell = (v?: string) => (v && v !== 'NULL' ? v : ''); + expect((await gridRows()).map((r) => cell(r[1]) || cell(r[3]))).toEqual(['send', 'bill', 'plain-entry']); + + await runSql(`SELECT * FROM "${T.tags}";`); + expect((await gridRows()).map((r) => r[0]).sort()).toEqual(['blue', 'green', 'red']); + + await runSql(`SELECT member, score FROM "${T.board}";`); + expect((await gridRows()).map((r) => r[0])).toEqual(['al', 'bo', 'cy']); + + await runSql(`SELECT level, msg FROM "${T.log}" WHERE level = 'warn';`); + expect(await gridRows()).toEqual([['warn', 'slow']]); + }, 90_000); + + it('reads JSON documents as their fields', async () => { + await runSql(`SELECT id, title, n, tags FROM ${T.docs};`); + const rows = await gridRows(); + expect(rows.map((r) => r[1]).sort()).toEqual(['Hello', 'World']); + expect(rows.find((r) => r[1] === 'Hello')?.[3]).toBe('["x"]'); + }, 60_000); + + it('refuses to write a collection or a JSON document, and leaves them unchanged', async () => { + await runSql(`INSERT INTO "${T.jobs}" (id, job) VALUES ('9', 'x');`, { write: true }); + expect((await errors()).join('\n')).toMatch(/read-only/); + expect(redis(0, 'LLEN', T.jobs)).toBe('3'); + + await runSql(`UPDATE ${T.docs} SET title = 'x' WHERE id = 'a';`, { write: true }); + expect((await errors()).join('\n')).toMatch(/only hash keys are updated/); + expect(JSON.parse(redis(0, 'GET', `${T.docs}:a`)).title).toBe('Hello'); + await saveScreenshot(driver, 'redis-editor-refusal'); + }, 60_000); + }); + + // ── Samples ─────────────────────────────────────────────────────────── + + describe('built-in samples', () => { + type Outcome = { id: string; title: string; ok: boolean; error: string }; + const outcomes: Outcome[] = []; + + beforeAll(async () => { + writeFileSync(SAMPLE_LOG, ''); + await destinations(CONN.s); + await sql.ensureSidebarSectionOpen('bookmarks'); + await clickWhen(driver, byTestId('sql-bookmark-install-samples')); + }, 60_000); + + it('runs every sample against Redis: each finishes, and a failure says why', async () => { + // The editor persists 400 ms after a change, under a key scoped to the + // open workspace (`foxschema-sql-editor:`); read whichever holds them. + await driver.waitForTimeout(600); + const samples: Array<{ id: string; title: string; sql: string }> = await driver.evaluate(() => { + window.dispatchEvent(new Event('pagehide')); + for (const key of Object.keys(localStorage).filter((k) => k.startsWith('foxschema-sql-editor'))) { + const found = (JSON.parse(localStorage.getItem(key) ?? '{}')?.state?.bookmarks ?? []).filter( + (b: { id: string }) => b.id.startsWith('sample-') + ); + if (found.length) return found; + } + return []; + }); + expect(samples.length).toBeGreaterThan(30); + + const mod = process.platform === 'darwin' ? 'Meta' : 'Control'; + for (const s of samples) { + // The sample's own text, not typed: typing runs Monaco's auto-indent + // and bracket closing over it, and takes a minute for the long ones. + await sql.waitForMonaco(); + await driver.locator('.monaco-editor').first().click(); + await driver.keyboard.press(`${mod}+KeyA`); + await driver.keyboard.insertText(s.sql); + await driver.waitForTimeout(350); + // With nothing selected, Run runs the statement at the caret — here the + // last cell. Select the whole buffer so every cell runs, in order. + await driver.keyboard.press(`${mod}+KeyA`); + await sql.run(); + const settled = await settle(45_000); + const errs = settled ? await errors() : ['HUNG: still running after 45 s']; + const o = { id: s.id, title: s.title, ok: errs.length === 0, error: errs.join(' | ').slice(0, 200) }; + outcomes.push(o); + // Written as it goes, so a run that times out still says how far it got. + appendFileSync(SAMPLE_LOG, `${o.ok ? 'OK ' : 'FAIL'} ${o.id}${o.ok ? '' : ` — ${o.error}`}\n`); + if (!settled) { + // Nothing stops a run from the toolbar: start the editor over. + await driver.reload(); + await driver.waitForSelector(byTestId('toolbar')); + await sql.openView(); + await destinations(CONN.s); + } + } + + // Printed for the report: which samples run on Redis, and why the rest stop. + console.log( + '\nSamples on Redis:\n' + + outcomes.map((o) => `${o.ok ? 'OK ' : 'FAIL'} ${o.id}${o.ok ? '' : ` — ${o.error}`}`).join('\n') + ); + + // None may hang, and none may stop without saying why. + // None may hang, hit the paging bug, or stop without saying why. + for (const o of outcomes) { + expect(o.error, o.id).not.toMatch(/^HUNG|Paging failed|Unsupported clause: \* FROM \(/); + if (!o.ok) expect(o.error.length, `${o.id} failed silently`).toBeGreaterThan(0); + } + // Code-only samples (no SQL, no network) never touch the store: they must run. + for (const id of ['sample-js-pure-grid', 'sample-js-try-catch', 'sample-js-faker-random-data']) { + const o = outcomes.find((x) => x.id === id); + if (o) expect(o.ok, `${id}: ${o.error}`).toBe(true); + } + // The A/B databases were never touched by a sample. + expect(keys(0, `${T.users}:*`)).toHaveLength(5); + }, 1_800_000); + }); + + // ── Compare data ────────────────────────────────────────────────────── + + describe('Compare data and Data migrate', () => { + /** Pick the Compare Source by its Destination name. */ + async function chooseSource(name: string): Promise { + const select = sql.compareBaselineSelect(0); + const value = await select.locator('option', { hasText: name }).first().getAttribute('value'); + if (!value) throw new Error(`no Compare source named ${name}`); + await select.selectOption(value); + } + + beforeAll(async () => { + await destinations(CONN.a, CONN.b); + }, 60_000); + + it('aligns the two databases by key and counts each kind of difference', async () => { + await runSql(`SELECT * FROM ${T.users};`); + expect(await errors()).toEqual([]); + await sql.setLayoutSideBySide(); + await waitFor(driver, byTestId('sql-result-compare-toggle-0'), 20_000); + if (!(await sql.compareToggle(0).isChecked())) await sql.compareToggle(0).click(); + await waitFor(driver, byTestId('sql-data-migrate-bar-0'), 20_000); + // Source defaults to whichever Destination is listed first; make it A. + await chooseSource(CONN.a); + const direction = async () => { + const bar = await driver.locator(byTestId('sql-data-migrate-bar-0')).innerText(); + const arrow = bar.indexOf('→'); + return bar.indexOf(CONN.a) < arrow && bar.indexOf(CONN.b) > arrow ? 'A→B' : 'B→A'; + }; + await expect.poll(direction, { timeout: 10_000 }).toBe('A→B'); + + // Spelled out so the test-ID catalog can check each one exists. + const CHIPS = { + add: 'sql-data-migrate-chip-add-0', + edit: 'sql-data-migrate-chip-edit-0', + delete: 'sql-data-migrate-chip-delete-0', + } as const; + const chip = async (kind: keyof typeof CHIPS) => + Number(((await driver.locator(byTestId(CHIPS[kind])).innerText()).match(/\d+/) ?? ['0'])[0]); + // A → B: 3, 4 and 5 are only in A (add), 2 differs (edit), 6 is only in B (delete). + await expect.poll(() => chip('add'), { timeout: 15_000 }).toBe(3); + expect(await chip('edit')).toBe(1); + expect(await chip('delete')).toBe(1); + expect(await sql.diffCellCount()).toBeGreaterThan(0); + await saveScreenshot(driver, 'redis-compare'); + }, 90_000); + + it('applies the migrate to B, after which B holds exactly what A does', async () => { + await clickWhen(driver, byTestId('sql-data-migrate-apply-0')); + // Data migrate asks through its own Safe mode dialog, not the editor's. + await clickWhen(driver, byTestId('sql-data-migrate-safe-confirm-apply')); + await waitFor(driver, byTestId('sql-data-migrate-progress'), 20_000); + const failures = driver.locator(byTestId('sql-data-migrate-failures-0')); + await expect + .poll( + async () => + (await failures.count()) ? `FAILED: ${await failures.innerText()}` : keys(1, `${T.users}:*`).join(','), + { timeout: 30_000 } + ) + .toBe(keys(0, `${T.users}:*`).join(',')); + await saveScreenshot(driver, 'redis-migrate-applied'); + // The progress panel gives way to a toast once the run ends. + const progress = driver.locator(byTestId('sql-data-migrate-progress-close')); + if (await progress.isVisible()) await progress.click(); + for (const id of ['1', '2', '3', '4', '5']) { + expect(hash(1, `${T.users}:${id}`), id).toEqual(hash(0, `${T.users}:${id}`)); + } + expect(keys(1, `${T.users}:6`)).toEqual([]); + }, 90_000); + + it('compares clean afterwards', async () => { + await runSql(`SELECT * FROM ${T.users};`); + await waitFor(driver, byTestId('sql-result-compare-toggle-0'), 20_000); + if (!(await sql.compareToggle(0).isChecked())) await sql.compareToggle(0).click(); + await expect.poll(() => sql.diffCellCount(), { timeout: 15_000 }).toBe(0); + await sql.setLayoutByCredential(); + }, 60_000); + }); + + // ── Data peek ───────────────────────────────────────────────────────── + + describe('Data peek', () => { + const peekRows = (): Promise => + driver.evaluate(() => + [...document.querySelectorAll('[data-testid^="data-peek-grid-"] [data-testid="sql-data-grid"] tbody tr')] + .filter((tr) => tr.querySelector('[data-testid="sql-row-num"]')) + .map((tr) => [...tr.querySelectorAll('td')].slice(1).map((td) => (td.textContent ?? '').trim())) + ); + + async function peek(table: string): Promise { + if (await driver.locator(byTestId('data-peek')).count()) { + await clickWhen(driver, byTestId('data-peek-close')); + await driver.locator(byTestId('data-peek')).waitFor({ state: 'detached' }); + } + await sql.ensureSidebarSectionOpen('schema'); + const explorer = driver.locator(byTestId('sql-schema-explorer')); + await waitFor(driver, byTestId(`sql-explorer-store-type-${table}`), 30_000); + await explorer.getByText(table, { exact: true }).first().click({ modifiers: [process.platform === 'darwin' ? 'Meta' : 'Control'] }); + await waitFor(driver, byTestId('data-peek'), 30_000); + await expect.poll(async () => (await peekRows()).length, { timeout: 30_000 }).toBeGreaterThan(0); + } + + beforeAll(async () => { + await destinations(CONN.a); + }, 60_000); + + it('peeks a hash prefix, filters it with WHERE, and pages it', async () => { + await peek(T.users); + expect(await peekRows()).toHaveLength(5); + + const where = driver.locator('[data-testid^="data-peek-where-"]').first(); + await where.fill("plan = 'pro'"); + await where.press('Enter'); + await expect.poll(async () => (await peekRows()).map((r) => r[1]).sort(), { timeout: 15_000 }).toEqual(['Bo', 'Cy']); + await driver.locator('[data-testid^="data-peek-where-clear-"]').first().click(); + await expect.poll(async () => (await peekRows()).length, { timeout: 15_000 }).toBe(5); + + const limit = driver.locator('[data-testid^="data-peek-limit-"]').first(); + await limit.fill('2'); + await limit.press('Enter'); + await expect.poll(async () => (await peekRows()).length, { timeout: 15_000 }).toBe(2); + const first = (await peekRows())[0]![0]; + await driver.locator(`[data-testid^="data-peek-grid-"] ${byTestId('sql-page-next')}`).first().click(); + await expect.poll(async () => (await peekRows())[0]?.[0], { timeout: 15_000 }).not.toBe(first); + await limit.fill('50'); + await limit.press('Enter'); + await expect.poll(async () => (await peekRows()).length, { timeout: 15_000 }).toBe(5); + }, 120_000); + + it('adds, edits and deletes a hash row through the row editor', async () => { + // Add + await clickWhen(driver, '[data-testid^="data-peek-add-"]'); + const form = driver.locator(byTestId('peek-row-editor')); + await form.waitFor({ state: 'visible', timeout: 10_000 }); + await form.locator(byTestId('peek-row-field-id')).fill('8'); + await form.locator(byTestId('peek-row-field-name')).fill('Gus'); + await form.locator(byTestId('peek-row-field-plan')).fill('free'); + await clickWhen(driver, byTestId('peek-row-submit')); + await clickWhen(driver, byTestId('peek-row-save')); + await sql.confirmWriteIfShown(); + await form.waitFor({ state: 'detached', timeout: 15_000 }); + expect(hash(0, `${T.users}:8`)).toEqual({ name: 'Gus', plan: 'free' }); + await expect.poll(async () => (await peekRows()).length, { timeout: 15_000 }).toBe(6); + + // Edit: select Gus's row, change the plan. + const gusRow = async () => (await peekRows()).findIndex((r) => r[1] === 'Gus'); + await expect.poll(gusRow).toBeGreaterThanOrEqual(0); + await driver + .locator(`[data-testid^="data-peek-grid-"] [data-testid="sql-row-${await gusRow()}"] td`) + .nth(2) + .click(); + await clickWhen(driver, '[data-testid^="data-peek-edit-"]'); + await form.waitFor({ state: 'visible', timeout: 10_000 }); + await form.locator(byTestId('peek-row-field-plan')).fill('team'); + await clickWhen(driver, byTestId('peek-row-submit')); + await clickWhen(driver, byTestId('peek-row-save')); + await sql.confirmWriteIfShown(); + await form.waitFor({ state: 'detached', timeout: 15_000 }); + await expect.poll(() => hash(0, `${T.users}:8`).plan, { timeout: 15_000 }).toBe('team'); + + // Delete + await expect.poll(gusRow).toBeGreaterThanOrEqual(0); + await driver + .locator(`[data-testid^="data-peek-grid-"] [data-testid="sql-row-${await gusRow()}"] td`) + .nth(2) + .click(); + await clickWhen(driver, '[data-testid^="data-peek-delete-"]'); + await sql.confirmWriteIfShown(); + await expect.poll(() => keys(0, `${T.users}:8`).length, { timeout: 15_000 }).toBe(0); + await expect.poll(async () => (await peekRows()).length, { timeout: 15_000 }).toBe(5); + await saveScreenshot(driver, 'redis-peek-crud'); + }, 180_000); + + it('peeks a list and a JSON prefix read-only, with no row editing offered', async () => { + await peek(T.jobs); + expect((await peekRows()).length).toBe(3); + expect(await driver.locator('[data-testid^="data-peek-add-"]').count()).toBe(0); + expect(await driver.locator(byTestId('data-peek')).innerText()).toMatch(/Read-only/); + + await peek(T.docs); + expect((await peekRows()).map((r) => r.join(' ')).join('\n')).toMatch(/Hello[\s\S]*World|World[\s\S]*Hello/); + expect(await driver.locator('[data-testid^="data-peek-add-"]').count()).toBe(0); + await clickWhen(driver, byTestId('data-peek-close')); + }, 120_000); + }); +}); diff --git a/apps/web/src/frontend/features/compare/components/CompareToolbar.tsx b/apps/web/src/frontend/features/compare/components/CompareToolbar.tsx index 6a7b58f51..b4ceeece6 100644 --- a/apps/web/src/frontend/features/compare/components/CompareToolbar.tsx +++ b/apps/web/src/frontend/features/compare/components/CompareToolbar.tsx @@ -117,7 +117,7 @@ export const CompareToolbarStart: React.FC = () => { const selectSavedConnection = (side: 'source' | 'target', id: string) => { const conn = connections.find((c) => c.id === id); - if (conn && !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod)) { + if (conn && !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username)) { const cfg = side === 'source' ? sourceConfig : targetConfig; const existing = getSessionPassword(id) || diff --git a/apps/web/src/frontend/features/compare/state/useSyncStore.ts b/apps/web/src/frontend/features/compare/state/useSyncStore.ts index d473fe6a2..1a06758cb 100644 --- a/apps/web/src/frontend/features/compare/state/useSyncStore.ts +++ b/apps/web/src/frontend/features/compare/state/useSyncStore.ts @@ -285,7 +285,7 @@ export const useSyncStore = create()( const canRetest = !!( conn.hasPassword || sessionPassword || - !connectionNeedsSecret(conn.dialect, conn.authMethod) + !connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) ); if (side === 'source') { patch.selectedSourceConnectionId = id; diff --git a/apps/web/src/frontend/features/sql-editor/components/DataGrid.tsx b/apps/web/src/frontend/features/sql-editor/components/DataGrid.tsx index 5761b9035..a6b7a793d 100644 --- a/apps/web/src/frontend/features/sql-editor/components/DataGrid.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/DataGrid.tsx @@ -924,7 +924,10 @@ export const DataGrid: React.FC<{ onRefresh={onRefresh} toolbarExtra={toolbarExtra} /> -
+
{result.error}
diff --git a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx index 31bb5edfa..f865b41d5 100644 --- a/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/ResultsPanel.tsx @@ -620,7 +620,10 @@ const PaneBody: React.FC<{ )}
-
+
{item.error}
@@ -1870,7 +1873,10 @@ const CredentialResultsSection: React.FC<{ {run.status === 'error' && ( -
+
{run.error}
diff --git a/apps/web/src/frontend/features/sql-editor/components/SqlEditorView.tsx b/apps/web/src/frontend/features/sql-editor/components/SqlEditorView.tsx index 1b7790b6f..5f5da4b56 100644 --- a/apps/web/src/frontend/features/sql-editor/components/SqlEditorView.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/SqlEditorView.tsx @@ -1,4 +1,5 @@ import React, { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { isSubsetDialect } from '@foxschema/ui-shared'; import { Loader2, Play, @@ -174,6 +175,8 @@ export const SqlEditorView: React.FC = () => { const [dragOver, setDragOver] = useState(null); const secretsPanelRef = useRef(null); const schemaExplorerRef = useRef(null); + /** The browsed connection's dialect: Redis and MongoDB have no tables to create. */ + const [explorerDialect, setExplorerDialect] = useState(null); const [secretsRefreshing, setSecretsRefreshing] = useState(false); const [runsOpen, setRunsOpen] = useState(false); @@ -516,20 +519,22 @@ export const SqlEditorView: React.FC = () => { height={sectionHeights.schema} onResizeHeight={(h) => setSectionHeight('schema', h)} actions={ - + explorerDialect && isSubsetDialect(explorerDialect) ? undefined : ( + + ) } {...drag} > - + ); default: @@ -541,6 +546,7 @@ export const SqlEditorView: React.FC = () => { canEditorBookmarks, canEditorVariables, canVariablesRead, + explorerDialect, canEditorSecrets, canSecretsView, canEditorSchema, diff --git a/apps/web/src/frontend/features/sql-editor/components/SqlSchemaExplorer.tsx b/apps/web/src/frontend/features/sql-editor/components/SqlSchemaExplorer.tsx index 481571603..8c8858312 100644 --- a/apps/web/src/frontend/features/sql-editor/components/SqlSchemaExplorer.tsx +++ b/apps/web/src/frontend/features/sql-editor/components/SqlSchemaExplorer.tsx @@ -11,7 +11,7 @@ import { import { useSyncStore } from '@/features/compare'; import { SCHEMA_ROUTINE_SCOPE, useSqlEditorStore } from '@/features/sql-editor/state/useSqlEditorStore'; import { connectionPickerOption, FilterPicker } from '@/shared/components/FilterPicker'; -import { quoteIdentifierIfNeeded } from '@foxschema/ui-shared'; +import { isSubsetDialect, quoteIdentifierIfNeeded } from '@foxschema/ui-shared'; import { getProviderSettings } from '@/shared/lib/provider-settings'; import { TYPE_META } from '@/features/schema-diff'; import { @@ -94,10 +94,13 @@ function writeStoredExplorerId(id: string): void { * destinations moves the explorer onto a checked credential when the current * one is no longer checked. */ -export const SqlSchemaExplorer = forwardRef(function SqlSchemaExplorer( - _props, - ref -) { +export const SqlSchemaExplorer = forwardRef< + SqlSchemaExplorerHandle, + { + /** The dialect of the connection being browsed, so the header can offer only what fits it. */ + onDialectChange?: (dialect: string | null) => void; + } +>(function SqlSchemaExplorer({ onDialectChange }, ref) { const connections = useSyncStore((s) => s.connections); const connectionsLoaded = useSyncStore((s) => s.connectionsLoaded); const tabs = useSqlEditorStore((s) => s.tabs); @@ -220,6 +223,11 @@ export const SqlSchemaExplorer = forwardRef(function Sq const conn = connections.find((c) => c.id === explorerId); const sessionPassword = explorerId ? sessionPasswords[explorerId] : undefined; + const browsedDialect = conn?.dialect ?? null; + + useEffect(() => { + onDialectChange?.(browsedDialect); + }, [browsedDialect, onDialectChange]); useEffect(() => { if (!explorerId || entry?.status !== 'ready' || !conn) { @@ -231,7 +239,7 @@ export const SqlSchemaExplorer = forwardRef(function Sq return; } // File dialects carry no password; waiting for one never resolved. - if (!conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod) && !sessionPassword) return; + if (!conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && !sessionPassword) return; let cancelled = false; void fetchDbaUtility( { connectionId: explorerId, password: sessionPassword }, @@ -398,13 +406,15 @@ export const SqlSchemaExplorer = forwardRef(function Sq schema={conn?.schema} sizeGroup={sizeOf(t.name, conn?.schema)} onOpenBlueprint={ - t.objectType === 'TABLE' || t.objectType === 'MQT' + // A Redis key has no DDL to edit: no blueprint for a store type. + (t.objectType === 'TABLE' || t.objectType === 'MQT') && !t.storeType ? () => { setBlueprintMode('edit'); setBlueprintTable(t); } : undefined } + canInsertFrom={t.objectType === 'TABLE' || t.objectType === 'MQT'} canPeek={ t.objectType === 'TABLE' || t.objectType === 'MQT' || @@ -444,6 +454,18 @@ export const SqlSchemaExplorer = forwardRef(function Sq ); }); +/** What each Redis store type means, on the badge's tooltip. */ +const STORE_TYPE_HINT: Record = { + hash: 'Hashes: one key per row, its fields as columns', + json: 'JSON documents: one key per row, top-level fields as columns', + string: 'Strings: one key per row, the value in one column', + mixed: 'Keys of several types under this prefix', + list: 'A list: one element per row, in list order', + set: 'A set: one member per row', + zset: 'A sorted set: one member per row, with its score', + stream: 'A stream: one entry per row, by id', +}; + const ObjectNode: React.FC<{ table: TableSchema; open: boolean; @@ -455,6 +477,8 @@ const ObjectNode: React.FC<{ onOpenBlueprint?: () => void; /** Data peek is only meaningful for row-bearing objects (not routines). */ canPeek?: boolean; + /** Offer "From": the object reads as rows (a table, an MQT, a Redis key). */ + canInsertFrom?: boolean; onPeek?: (tableName: string) => void; onOpenSource?: () => void; }> = ({ @@ -467,9 +491,13 @@ const ObjectNode: React.FC<{ onOpenBlueprint, onOpenSource, canPeek, + canInsertFrom, onPeek, }) => { const meta = TYPE_META[table.objectType] ?? TYPE_META.TABLE; + // Redis and MongoDB read SQL through a small parser: one bare table name, no + // alias and no `alias.column` (sql-subset.ts refuses both). + const subset = isSubsetDialect(dialect); const insertName = quoteIdentifierIfNeeded(table.name, dialect); const schemaName = schema?.trim(); const qualifiedName = @@ -489,6 +517,10 @@ const ObjectNode: React.FC<{ insertAtCursor(routineInsertText(qualifiedName, table.objectType, params)); return; } + if (subset) { + insertAtCursor(`${insertName} `); + return; + } // Tables/views: add into FROM with an auto alias (e.g. users u). const ok = mutateSql((sql) => addTableWithAliasToFrom(sql, qualifiedName).sql); if (!ok) insertAtCursor(`${qualifiedName} `); @@ -526,7 +558,7 @@ const ObjectNode: React.FC<{ /** Add `alias.column` into the active SELECT list (or insert at cursor). */ const insertColumnIntoSelect = (colName: string) => { const col = quoteIdentifierIfNeeded(colName, dialect); - const expr = `${resolveTableAlias()}.${col}`; + const expr = subset ? col : `${resolveTableAlias()}.${col}`; const ok = mutateSql((sql) => insertIntoSelectList(sql, expr)); if (!ok) insertAtCursor(`${expr}, `); }; @@ -579,6 +611,15 @@ const ObjectNode: React.FC<{ {table.name} + {table.storeType && ( + + {table.storeType} + + )} {isRoutine && params.length > 0 && ( ({params.length}) @@ -597,10 +638,10 @@ const ObjectNode: React.FC<{ {/* One group, so the actions wrap as a unit instead of splitting apart. */} - {onOpenBlueprint && ( + {canInsertFrom && ( + {canCreateTable && ( + + )} + {canCreateTable && ( + + )}
)} @@ -421,15 +435,13 @@ export const SqlSchemaExplorer = forwardRef< schema={conn?.schema} sizeGroup={sizeOf(t.name, conn?.schema)} onOpenBlueprint={ - // A Redis key has no DDL to edit: no blueprint for a store type. - (t.objectType === 'TABLE' || t.objectType === 'MQT') && !t.storeType + (t.objectType === 'TABLE' || t.objectType === 'MQT') && canCreateTable ? () => { setBlueprintMode('edit'); setBlueprintTable(t); } : undefined } - canInsertFrom={t.objectType === 'TABLE' || t.objectType === 'MQT'} canPeek={ t.objectType === 'TABLE' || t.objectType === 'MQT' || @@ -492,8 +504,6 @@ const ObjectNode: React.FC<{ onOpenBlueprint?: () => void; /** Data peek is only meaningful for row-bearing objects (not routines). */ canPeek?: boolean; - /** Offer "From": the object reads as rows (a table, an MQT, a Redis key). */ - canInsertFrom?: boolean; onPeek?: (tableName: string) => void; onOpenSource?: () => void; }> = ({ @@ -506,10 +516,11 @@ const ObjectNode: React.FC<{ onOpenBlueprint, onOpenSource, canPeek, - canInsertFrom, onPeek, }) => { const meta = TYPE_META[table.objectType] ?? TYPE_META.TABLE; + /** Offer "From": the object reads as rows (a table, an MQT, a Redis key). */ + const canInsertFrom = table.objectType === 'TABLE' || table.objectType === 'MQT'; // Redis and MongoDB read SQL through a small parser: one bare table name, no // alias and no `alias.column` (sql-subset.ts refuses both). const subset = isSubsetDialect(dialect); diff --git a/apps/web/src/frontend/features/sql-editor/state/useSqlEditorStore.ts b/apps/web/src/frontend/features/sql-editor/state/useSqlEditorStore.ts index e723359f0..aa0aee0dc 100644 --- a/apps/web/src/frontend/features/sql-editor/state/useSqlEditorStore.ts +++ b/apps/web/src/frontend/features/sql-editor/state/useSqlEditorStore.ts @@ -83,7 +83,7 @@ import { type SqlVariableKind, type VariableOverride, } from '@/shared/lib/sql-variables'; -import { connectionNeedsSecret, dialectHasTransactions } from '@/shared/lib/provider-settings'; +import { dialectHasTransactions, needsSessionPassword } from '@/shared/lib/provider-settings'; import { useSyncStore } from '@/features/compare'; import { SQL_EDITOR_PERSIST_KEY, SQL_EDITOR_PERSIST_VERSION, useRecentQueries } from './recentQueries'; import { workspaceEditorStorage } from '@/shared/lib/editorWorkspaceStorage'; @@ -677,12 +677,7 @@ export const useSqlEditorStore = create()( // A file database has no password to be missing. Prompting for one // left the destination checkbox refusing to tick, so SQLite could not // be run against at all. - if ( - conn && - !conn.hasPassword && - connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && - !sessionPasswordFor(id, sessionPasswords) - ) { + if (conn && needsSessionPassword(conn, sessionPasswordFor(id, sessionPasswords))) { set({ pendingPassword: { id, @@ -702,12 +697,7 @@ export const useSqlEditorStore = create()( const tab = tabs.find((t) => t.id === activeTabId); if (!tab || !id) return; const conn = useSyncStore.getState().connections.find((c) => c.id === id); - if ( - conn && - !conn.hasPassword && - connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && - !sessionPasswordFor(id, sessionPasswords) - ) { + if (conn && needsSessionPassword(conn, sessionPasswordFor(id, sessionPasswords))) { set({ pendingPassword: { id, @@ -907,11 +897,7 @@ export const useSqlEditorStore = create()( } const { sessionPasswords } = get(); - if ( - !conn.hasPassword && - connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && - !sessionPasswordFor(connectionId, sessionPasswords) - ) { + if (needsSessionPassword(conn, sessionPasswordFor(connectionId, sessionPasswords))) { set({ pendingPassword: { id: connectionId, @@ -1071,12 +1057,7 @@ export const useSqlEditorStore = create()( connections = [LOCAL_CODE_RUN_TARGET]; } - const needingPassword = connections.find( - (c) => - !c.hasPassword && - connectionNeedsSecret(c.dialect, c.authMethod, c.username) && - !sessionPasswordFor(c.id, sessionPasswords) - ); + const needingPassword = connections.find((c) => needsSessionPassword(c, sessionPasswordFor(c.id, sessionPasswords))); if (needingPassword) { set({ pendingPassword: { diff --git a/apps/web/src/frontend/features/utilities/components/BackupServerActions.tsx b/apps/web/src/frontend/features/utilities/components/BackupServerActions.tsx index 367123266..b042928b0 100644 --- a/apps/web/src/frontend/features/utilities/components/BackupServerActions.tsx +++ b/apps/web/src/frontend/features/utilities/components/BackupServerActions.tsx @@ -25,7 +25,7 @@ import { useAuthStore } from '@/platform/authStore'; import { useSqlEditorStore } from '@/features/sql-editor/state'; import type { useSyncStore } from '@/features/compare'; import { executeSql } from '@/shared/api/sqlApi'; -import { connectionNeedsSecret } from '@/shared/lib/provider-settings'; +import { needsSessionPassword } from '@/shared/lib/provider-settings'; import { dialectLabel } from '@/shared/lib/dialectLabel'; type Connection = ReturnType['connections'][number]; @@ -52,10 +52,7 @@ export const BackupServerActions: React.FC<{ const [history, setHistory] = useState<'idle' | 'loading' | BackupHistoryEntry[] | { error: string }>('idle'); const historySql = backupHistoryQuery({ dialect: connection.dialect, database: connection.database ?? '', schema: connection.schema }); - const needsPassword = - !connection.hasPassword && - connectionNeedsSecret(connection.dialect, connection.authMethod, connection.username) && - !sessionPasswords[connection.id]; + const needsPassword = needsSessionPassword(connection, sessionPasswords[connection.id]); const ref = { connectionId: connection.id, password: sessionPasswords[connection.id] || undefined, diff --git a/apps/web/src/frontend/features/utilities/components/CloneTableModal.tsx b/apps/web/src/frontend/features/utilities/components/CloneTableModal.tsx index f4cb29e8b..8e0931541 100644 --- a/apps/web/src/frontend/features/utilities/components/CloneTableModal.tsx +++ b/apps/web/src/frontend/features/utilities/components/CloneTableModal.tsx @@ -15,7 +15,7 @@ import { useSyncStore } from '@/features/compare'; import { useSqlEditorStore } from '@/features/sql-editor/state'; import { useUiStore } from '@/platform/uiStore'; import type { TableSchema } from '@/shared/lib/types'; -import { PROVIDER_SETTINGS, connectionNeedsSecret } from '@/shared/lib/provider-settings'; +import { PROVIDER_SETTINGS, needsSessionPassword } from '@/shared/lib/provider-settings'; import { insertAtCursor } from '@/features/sql-editor'; import { WriteConfirmDialog } from '@/features/sql-editor/ui'; import { SQL_ICON_STROKE } from '@/shared/lib/iconStyle'; @@ -111,7 +111,7 @@ export const CloneTableModal: React.FC = ({ const conn = connections.find((c) => c.id === connectionId) || null; // File dialects carry no password; asking for one blocked the utility outright. const needsPassword = Boolean( - conn && !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && !sessionPasswords[connectionId] + conn && needsSessionPassword(conn, sessionPasswords[connectionId]) ); const cache = connectionId ? schemaCache[connectionId] : undefined; diff --git a/apps/web/src/frontend/features/utilities/components/DatabaseAccessModal.tsx b/apps/web/src/frontend/features/utilities/components/DatabaseAccessModal.tsx index 162169162..250a7dfac 100644 --- a/apps/web/src/frontend/features/utilities/components/DatabaseAccessModal.tsx +++ b/apps/web/src/frontend/features/utilities/components/DatabaseAccessModal.tsx @@ -40,7 +40,7 @@ import { runAccessSql } from '@/shared/api/accessSql'; import { useSyncStore } from '@/features/compare'; import { useSqlEditorStore } from '@/features/sql-editor/state'; import { useAuthStore } from '@/platform/authStore'; -import { PROVIDER_SETTINGS, connectionNeedsSecret } from '@/shared/lib/provider-settings'; +import { PROVIDER_SETTINGS, needsSessionPassword } from '@/shared/lib/provider-settings'; import { DbAccessPermissionSections } from '@/features/access'; import { sectionLabelCls } from '@/shared/components/surfaces'; import { connectionOptionLabel } from '@/shared/lib/dialectLabel'; @@ -260,7 +260,7 @@ export const DatabaseAccessModal: React.FC = ({ const conn = connections.find((c) => c.id === connectionId); // File dialects carry no password; asking for one blocked the utility outright. const needsPassword = Boolean( - conn && !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && !sessionPasswords[connectionId] + conn && needsSessionPassword(conn, sessionPasswords[connectionId]) ); const dialect = conn?.dialect ?? ''; const support = dialect ? dialectSupportsDbAccess(dialect) : null; diff --git a/apps/web/src/frontend/features/utilities/components/IndexManagementModal.tsx b/apps/web/src/frontend/features/utilities/components/IndexManagementModal.tsx index db2673945..8c0547c8a 100644 --- a/apps/web/src/frontend/features/utilities/components/IndexManagementModal.tsx +++ b/apps/web/src/frontend/features/utilities/components/IndexManagementModal.tsx @@ -49,7 +49,7 @@ import { useSyncStore } from '@/features/compare'; import { useSqlEditorStore } from '@/features/sql-editor/state'; import { useAuthStore } from '@/platform/authStore'; import type { IndexInfo, TableSchema } from '@/shared/lib/types'; -import { PROVIDER_SETTINGS, connectionNeedsSecret } from '@/shared/lib/provider-settings'; +import { PROVIDER_SETTINGS, needsSessionPassword } from '@/shared/lib/provider-settings'; import { DEFAULT_INDEX_MGMT_SORT, averageFragmentation, @@ -208,7 +208,7 @@ export const IndexManagementModal: React.FC = ({ const conn = connections.find((c) => c.id === connectionId) || null; // File dialects carry no password; asking for one blocked the utility outright. const needsPassword = Boolean( - conn && !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && !sessionPasswords[connectionId] + conn && needsSessionPassword(conn, sessionPasswords[connectionId]) ); const cache = connectionId ? schemaCache[connectionId] : undefined; /** diff --git a/apps/web/src/frontend/features/utilities/components/ServerInsightsModal.tsx b/apps/web/src/frontend/features/utilities/components/ServerInsightsModal.tsx index 26a7d32cb..409aa50e9 100644 --- a/apps/web/src/frontend/features/utilities/components/ServerInsightsModal.tsx +++ b/apps/web/src/frontend/features/utilities/components/ServerInsightsModal.tsx @@ -26,7 +26,7 @@ import { dialectSupportsDbaUtility, formatBytes, formatPct, type DbaUtilityKind import { fetchDbaUtility, type DbaUtilityResponse } from '@/shared/api/schemaApi'; import { useSyncStore } from '@/features/compare'; import { useSqlEditorStore } from '@/features/sql-editor/state'; -import { PROVIDER_SETTINGS, connectionNeedsSecret } from '@/shared/lib/provider-settings'; +import { PROVIDER_SETTINGS, needsSessionPassword } from '@/shared/lib/provider-settings'; import { IndexManagementModal } from './IndexManagementModal'; import { sectionLabelCls } from '@/shared/components/surfaces'; import { connectionOptionLabel } from '@/shared/lib/dialectLabel'; @@ -110,7 +110,7 @@ export const ServerInsightsModal: React.FC = ({ const conn = connections.find((c) => c.id === connectionId) || null; // File dialects carry no password; asking for one blocked the utility outright. const needsPassword = Boolean( - conn && !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && !sessionPasswords[connectionId] + conn && needsSessionPassword(conn, sessionPasswords[connectionId]) ); const support = useMemo( () => dialectSupportsDbaUtility(conn?.dialect || '', tab), diff --git a/apps/web/src/frontend/shared/lib/provider-settings-facade.test.ts b/apps/web/src/frontend/shared/lib/provider-settings-facade.test.ts index eee6db8bc..652489454 100644 --- a/apps/web/src/frontend/shared/lib/provider-settings-facade.test.ts +++ b/apps/web/src/frontend/shared/lib/provider-settings-facade.test.ts @@ -32,6 +32,7 @@ describe('provider-settings is a facade over @foxschema/sql', () => { expect(facade.isFileDialect).toBe(core.isFileDialect); expect(facade.DEFAULT_PORTS).toBe(core.DEFAULT_PORTS); expect(facade.connectionNeedsSecret).toBe(core.connectionNeedsSecret); + expect(facade.needsSessionPassword).toBe(core.needsSessionPassword); }); it('exposes every dialect the canonical registry has', () => { diff --git a/apps/web/src/frontend/shared/lib/provider-settings.ts b/apps/web/src/frontend/shared/lib/provider-settings.ts index a026030c3..2acc4c209 100644 --- a/apps/web/src/frontend/shared/lib/provider-settings.ts +++ b/apps/web/src/frontend/shared/lib/provider-settings.ts @@ -36,6 +36,7 @@ export { dialectHasTransactions, DEFAULT_PORTS, connectionNeedsSecret, + needsSessionPassword, authMethodsForDialect, dialectOffersAuthMethods, normalizeAuthMethod, diff --git a/packages/db/src/providers/redis/redis.adapter.test.ts b/packages/db/src/providers/redis/redis.adapter.test.ts index c27251623..e0f91de16 100644 --- a/packages/db/src/providers/redis/redis.adapter.test.ts +++ b/packages/db/src/providers/redis/redis.adapter.test.ts @@ -4,7 +4,9 @@ * SPDX-License-Identifier: Apache-2.0 */ import { describe, expect, it } from 'vitest'; -import { redisRowMatchesPredicates } from './redis.adapter.js'; +import { pageSubsetSelect } from '@foxschema/sql'; +import { redisAdapter, redisRowMatchesPredicates } from './redis.adapter.js'; +import { RedisClientDouble } from './redis.client-double.js'; describe('redisRowMatchesPredicates', () => { it('matches when every non-id predicate holds', () => { @@ -45,11 +47,7 @@ describe('redisRowMatchesPredicates', () => { }); describe('redis adapter · every key type through the SQL subset', () => { - // Imported here so the predicate tests above stay free of the double. - const setup = async () => { - const { RedisClientDouble } = await import('./redis.client-double.js'); - const { redisAdapter } = await import('./redis.adapter.js'); - const { pageSubsetSelect } = await import('@foxschema/sql'); + const setup = () => { const c = new RedisClientDouble() .set('users:1', { type: 'hash', value: { name: 'Ana', plan: 'free' } }) .set('users:2', { type: 'hash', value: { name: 'Bo', plan: 'pro' } }) @@ -64,11 +62,11 @@ describe('redis adapter · every key type through the SQL subset', () => { ], }); const q = (sql: string, params: unknown[] = []) => redisAdapter.query(c as never, sql, params) as Promise[]>; - return { c, q, pageSubsetSelect }; + return { c, q }; }; it('reads a list key as a table of its elements', async () => { - const { q } = await setup(); + const { q } = setup(); expect(await q('SELECT * FROM "queue:jobs"')).toEqual([ { index: 0, job: 'send' }, { index: 1, job: 'bill' }, @@ -77,13 +75,13 @@ describe('redis adapter · every key type through the SQL subset', () => { }); it('pages and projects a collection table', async () => { - const { q } = await setup(); + const { q } = setup(); expect(await q('SELECT member, score FROM board LIMIT 1 OFFSET 1')).toEqual([{ member: 'bo', score: 20 }]); expect(await q(`SELECT job FROM "queue:jobs" WHERE job = 'bill'`)).toEqual([{ job: 'bill' }]); }); it('pages a prefix table with OFFSET and keeps only the selected columns', async () => { - const { q } = await setup(); + const { q } = setup(); const names = (await q('SELECT id, name FROM users LIMIT 2 OFFSET 1')).map((r) => r.name); expect(names).toHaveLength(2); const all = (await q('SELECT id, name FROM users')).map((r) => r.name).sort(); @@ -92,7 +90,7 @@ describe('redis adapter · every key type through the SQL subset', () => { }); it('runs the page the editor now sends, instead of refusing it', async () => { - const { q, pageSubsetSelect } = await setup(); + const { q } = setup(); // What the editor's page wrap used to send — a nested SELECT — is refused. await expect(q('SELECT * FROM (SELECT * FROM users) AS fox_page LIMIT 201 OFFSET 0')).rejects.toThrow(/Unsupported clause/); expect(await q(pageSubsetSelect('SELECT * FROM "queue:jobs"', 0, 201))).toHaveLength(3); @@ -102,19 +100,19 @@ describe('redis adapter · every key type through the SQL subset', () => { }); it('reads nothing, and asks the server nothing, for an empty page', async () => { - const { c, q } = await setup(); + const { c, q } = setup(); expect(await q('SELECT * FROM users LIMIT 0')).toEqual([]); expect(c.calls).toEqual([]); }); it('reads a JSON document as fields, and refuses to update it field by field', async () => { - const { q } = await setup(); + const { q } = setup(); expect(await q(`SELECT * FROM docs WHERE id = 'a'`)).toEqual([{ id: 'a', title: 'Hello' }]); await expect(q(`UPDATE docs SET title = 'x' WHERE id = 'a'`)).rejects.toThrow(/only hash keys are updated/); }); it('refuses writes to a collection table, and still writes hashes', async () => { - const { q, c } = await setup(); + const { q, c } = setup(); for (const sql of [ `INSERT INTO "queue:jobs" (id, job) VALUES ('9', 'x')`, `UPDATE "queue:jobs" SET job = 'x' WHERE id = '0'`, @@ -125,4 +123,15 @@ describe('redis adapter · every key type through the SQL subset', () => { expect(c.keys.get('queue:jobs')).toMatchObject({ type: 'list' }); expect(await q(`UPDATE users SET plan = 'pro' WHERE id = '1'`)).toEqual([{ rowCount: 1, key: 'users:1' }]); }); + + it('an UPDATE asks for the table’s type and the key’s once each', async () => { + const { q, c } = setup(); + await q(`UPDATE users SET plan = 'pro' WHERE id = '2'`); + expect(c.calls.filter((x) => x === 'TYPE').length).toBe(2); + }); + + it('pages a prefix scan with a filter, skipping matches for the offset', async () => { + const { q } = setup(); + expect(await q(`SELECT id FROM users WHERE plan = 'pro' LIMIT 1 OFFSET 1`)).toEqual([{ id: '3' }]); + }); }); diff --git a/packages/db/src/providers/redis/redis.adapter.ts b/packages/db/src/providers/redis/redis.adapter.ts index 5b2f4ce86..d836f2126 100644 --- a/packages/db/src/providers/redis/redis.adapter.ts +++ b/packages/db/src/providers/redis/redis.adapter.ts @@ -1,7 +1,7 @@ import { type ConnectionOptions, type DriverAdapter, parseSqlSubset, subsetValue } from '@foxschema/sql'; import { credentialedCacheKey } from '../../cores/pool-cache.js'; import { requireDriver } from '../../cores/driver-loader.js'; -import { COLLECTION_TYPES, JSON_TYPE, readCollection, readKeyRow } from './redis.keyspace.js'; +import { COLLECTION_TYPES, JSON_TYPE, readCollection, readKey, scanBatches, takePage } from './redis.keyspace.js'; /** * Redis behind the SQL editor, for the cache-maintenance case. @@ -99,11 +99,6 @@ class RedisAdapter implements DriverAdapter { return `${table}:${String(id)}`; } - /** Read one key as a row, whatever type it holds. */ - private readRow(client: any, table: string, key: string): Promise | null> { - return readKeyRow(client, table, key); - } - /** The collection type when `table` names a list, set, sorted set or stream key itself. */ private async collectionType(client: any, table: string): Promise { const type = String(await client.type(table)); @@ -150,8 +145,7 @@ class RedisAdapter implements DriverAdapter { await readCollection(client, intent.table, collection, { offset: intent.offset ?? 0, limit: intent.limit ?? 100, - matches: (row) => redisRowMatchesPredicates(row, intent.where, params), - filtered: intent.where.length > 0, + matches: intent.where.length ? (row) => redisRowMatchesPredicates(row, intent.where, params) : undefined, }) ) as T[]; } @@ -159,43 +153,28 @@ class RedisAdapter implements DriverAdapter { const id = idFrom(intent.where); const extra = extraWhere(intent.where); if (id !== undefined) { - const row = await this.readRow(client, intent.table, this.keyFor(intent.table, id)); + const row = (await readKey(client, intent.table, this.keyFor(intent.table, id)))?.row; if (!row || (intent.offset ?? 0) > 0) return [] as T[]; // Non-id predicates must still hold — otherwise `WHERE id = 1 AND // status = 'active'` would return the inactive row. if (!redisRowMatchesPredicates(row, extra, params)) return [] as T[]; return project([row]) as T[]; } - // No id: scan the prefix. Bounded by LIMIT so browsing a large cache - // cannot walk the whole keyspace. Filter *before* counting toward the - // limit, or `WHERE name = ? LIMIT 2` would sample the first two keys - // and then discard non-matches. - const limit = intent.limit ?? 100; - let skip = intent.offset ?? 0; - const rows: Record[] = []; - // node-redis v6 yields a *batch* of keys per iteration; older clients - // yielded one key. Normalising both is not defensive noise — assuming - // the single-key shape made String(['a','b']) the key, so every scan - // silently returned nothing. - outer: for await (const batch of client.scanIterator({ - MATCH: `${intent.table}:*`, - COUNT: 200, - })) { - const keys: string[] = Array.isArray(batch) - ? batch.map((k) => String(k)) - : [String(batch)]; - for (const k of keys) { - const row = await this.readRow(client, intent.table, k); - if (!row) continue; - if (!redisRowMatchesPredicates(row, extra, params)) continue; - if (skip > 0) { - skip--; - continue; - } - rows.push(row); - if (rows.length >= limit) break outer; + // No id: scan the prefix, a SCAN batch at a time with its keys read + // together (node-redis pipelines them). takePage filters *before* + // counting toward LIMIT, or `WHERE name = ? LIMIT 2` would sample the + // first two keys and then discard non-matches, and it stops after + // MAX_SCAN keys, so a WHERE that matches nothing cannot walk a cache. + const table = intent.table; + async function* prefixRows(): AsyncIterable[]> { + for await (const keys of scanBatches(client.scanIterator({ MATCH: `${table}:*`, COUNT: 200 }))) { + const reads = await Promise.all(keys.map((k) => readKey(client, table, k))); + yield reads.flatMap((read) => (read ? [read.row] : [])); } } + const rows = await takePage(prefixRows(), intent.offset ?? 0, intent.limit ?? 100, (row) => + redisRowMatchesPredicates(row, extra, params) + ); return project(rows) as T[]; } case 'insert': { @@ -218,20 +197,21 @@ class RedisAdapter implements DriverAdapter { if (id === undefined) { throw new Error('Redis: UPDATE needs `id` in the WHERE clause to address a key.'); } - await this.refuseCollectionWrite(client, intent.table); const key = this.keyFor(intent.table, id); - const row = await this.readRow(client, intent.table, key); - if (!row) return [{ rowCount: 0 }] as T[]; + const [, read] = await Promise.all([ + this.refuseCollectionWrite(client, intent.table), + readKey(client, intent.table, key), + ]); + if (!read) return [{ rowCount: 0 }] as T[]; // A JSON document or a string reads as fields too, but HSET on it is // WRONGTYPE. Say what it is instead of failing with that. - const type = String(await client.type(key)); - if (type !== 'hash') { - const what = type === JSON_TYPE ? 'JSON document' : type; + if (read.type !== 'hash') { + const what = read.type === JSON_TYPE ? 'JSON document' : read.type; throw new Error(`Redis: "${key}" is a ${what}; only hash keys are updated field by field.`); } // Must honour every AND predicate. Ignoring `status = 'active'` here // would overwrite a key the WHERE clause excluded — silent data loss. - if (!redisRowMatchesPredicates(row, extraWhere(intent.where), params)) { + if (!redisRowMatchesPredicates(read.row, extraWhere(intent.where), params)) { return [{ rowCount: 0 }] as T[]; } const set: Record = {}; @@ -248,9 +228,12 @@ class RedisAdapter implements DriverAdapter { if (id === undefined) { throw new Error('Redis: DELETE needs `id` in the WHERE clause to address a key.'); } - await this.refuseCollectionWrite(client, intent.table); const key = this.keyFor(intent.table, id); - const row = await this.readRow(client, intent.table, key); + const [, read] = await Promise.all([ + this.refuseCollectionWrite(client, intent.table), + readKey(client, intent.table, key), + ]); + const row = read?.row; if (!row) return [{ rowCount: 0 }] as T[]; // Same rule as UPDATE: `DELETE … WHERE id = 1 AND status = 'active'` // must not remove an inactive row. diff --git a/packages/db/src/providers/redis/redis.client-double.ts b/packages/db/src/providers/redis/redis.client-double.ts index 80d88ea3b..4cff43cb0 100644 --- a/packages/db/src/providers/redis/redis.client-double.ts +++ b/packages/db/src/providers/redis/redis.client-double.ts @@ -11,6 +11,7 @@ * `redis.it.test.ts` is the proof against a real server; this is what lets the * shapes of every key type be tested without one. */ +import { escapeRegExp } from '@foxschema/sql'; type Entry = | { type: 'string'; value: string } @@ -33,7 +34,7 @@ function globToRegExp(pattern: string): RegExp { for (const ch of pattern) { if (ch === '*') out += '.*'; else if (ch === '?') out += '.'; - else out += ch.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + else out += escapeRegExp(ch); } return new RegExp(`^${out}$`); } diff --git a/packages/db/src/providers/redis/redis.it.test.ts b/packages/db/src/providers/redis/redis.it.test.ts index 5e39ce954..3a2ea3017 100644 --- a/packages/db/src/providers/redis/redis.it.test.ts +++ b/packages/db/src/providers/redis/redis.it.test.ts @@ -1,6 +1,6 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { redisAdapter } from './redis.adapter.js'; -import { sampleKeyspace } from './redis.keyspace.js'; +import { sampleKeyspace, scanBatches } from './redis.keyspace.js'; /** * Runs against a real Redis. Skipped unless FOX_REDIS_URL is set, so the normal @@ -25,10 +25,9 @@ maybe('redis adapter against a live server', () => { }); afterAll(async () => { - // Leave the keyspace as we found it. node-redis yields a batch of keys per - // step; `String(batch)` used to join them into one key that matched none. - for await (const batch of conn.scanIterator({ MATCH: `${table}*`, COUNT: 100 })) { - for (const key of Array.isArray(batch) ? batch : [batch]) await conn.del(String(key)); + // Leave the keyspace as we found it. + for await (const keys of scanBatches(conn.scanIterator({ MATCH: `${table}*`, COUNT: 100 }))) { + if (keys.length) await conn.del(keys); } await redisAdapter.closeAll(); }); diff --git a/packages/db/src/providers/redis/redis.keyspace.test.ts b/packages/db/src/providers/redis/redis.keyspace.test.ts index 82e33e029..f60abcf52 100644 --- a/packages/db/src/providers/redis/redis.keyspace.test.ts +++ b/packages/db/src/providers/redis/redis.keyspace.test.ts @@ -13,11 +13,12 @@ import { MAX_SCAN, jsonObjectFields, readCollection, - readKeyRow, + readKey, sampleKeyspace, } from './redis.keyspace.js'; -const all = () => true; +/** The row a key reads as, or null. */ +const readKeyRow = async (c: RedisClientDouble, table: string, key: string) => (await readKey(c, table, key))?.row ?? null; describe('jsonObjectFields', () => { it('reads a JSON object’s top-level fields, nested values as JSON text', () => { @@ -37,7 +38,7 @@ describe('jsonObjectFields', () => { }); }); -describe('readKeyRow', () => { +describe('readKey', () => { it('reads a hash as its fields, a plain string as one value', async () => { const c = new RedisClientDouble() .set('users:1', { type: 'hash', value: { name: 'Ana' } }) @@ -74,19 +75,36 @@ describe('readCollection', () => { it('pages a list by index, across chunk boundaries, without reading it whole', async () => { const c = new RedisClientDouble().set('l', { type: 'list', value: list(1200) }); - const page = await readCollection(c, 'l', 'list', { offset: 990, limit: 20, matches: all, filtered: false }); + const page = await readCollection(c, 'l', 'list', { offset: 990, limit: 20 }); expect(page.map((r) => r.index)).toEqual(Array.from({ length: 20 }, (_, i) => 990 + i)); expect(page[0]).toEqual({ index: 990, value: 'v990' }); - // Started at the offset itself: one LRANGE (990..1489 holds all 20), not two from 0. + // Started at the offset itself: one LRANGE, not two from 0. expect(c.calls.filter((x) => x === 'LRANGE').length).toBe(1); }); + it('reads no more of a list or stream than an unfiltered page shows', async () => { + const c = new RedisClientDouble() + .set('l', { type: 'list', value: list(1200) }) + .set('s', { type: 'stream', value: list(1200).map((v, i) => ({ id: `1-${String(i).padStart(4, '0')}`, message: { v } })) }); + const ranges: number[][] = []; + const lRange = c.lRange.bind(c); + c.lRange = (key, start, stop) => (ranges.push([start, stop]), lRange(key, start, stop)); + const counts: number[] = []; + const xRange = c.xRange.bind(c); + c.xRange = (key, from, to, opts) => (counts.push(opts.COUNT), xRange(key, from, to, opts)); + + await readCollection(c, 'l', 'list', { offset: 990, limit: 20 }); + await readCollection(c, 's', 'stream', { offset: 10, limit: 20 }); + expect(ranges).toEqual([[990, 1009]]); + expect(counts).toEqual([30]); + }); + it('expands a list of JSON objects into their fields', async () => { const c = new RedisClientDouble().set('events', { type: 'list', value: ['{"kind":"click","x":1}', 'not json', '{"kind":"view"}'], }); - expect(await readCollection(c, 'events', 'list', { offset: 0, limit: 10, matches: all, filtered: false })).toEqual([ + expect(await readCollection(c, 'events', 'list', { offset: 0, limit: 10 })).toEqual([ { index: 0, kind: 'click', x: 1 }, { index: 1, value: 'not json' }, { index: 2, kind: 'view' }, @@ -99,7 +117,6 @@ describe('readCollection', () => { offset: 1, limit: 2, matches: (r) => r.value === 'a', - filtered: true, }); expect(page).toEqual([ { index: 2, value: 'a' }, @@ -109,7 +126,7 @@ describe('readCollection', () => { it('stops filtering after MAX_SCAN elements instead of walking a huge list', async () => { const c = new RedisClientDouble().set('big', { type: 'list', value: list(MAX_SCAN + 5_000) }); - const page = await readCollection(c, 'big', 'list', { offset: 0, limit: 10, matches: () => false, filtered: true }); + const page = await readCollection(c, 'big', 'list', { offset: 0, limit: 10, matches: () => false }); expect(page).toEqual([]); expect(c.calls.filter((x) => x === 'LRANGE').length).toBeLessThanOrEqual(MAX_SCAN / 500 + 1); }); @@ -123,7 +140,7 @@ describe('readCollection', () => { { value: 'cy', score: 30 }, ], }); - expect(await readCollection(c, 'board', 'zset', { offset: 1, limit: 5, matches: all, filtered: false })).toEqual([ + expect(await readCollection(c, 'board', 'zset', { offset: 1, limit: 5 })).toEqual([ { member: 'bo', score: 20 }, { member: 'cy', score: 30 }, ]); @@ -132,7 +149,7 @@ describe('readCollection', () => { it('reads a set’s members a scan batch at a time', async () => { const members = Array.from({ length: 700 }, (_, i) => `m${i}`); const c = new RedisClientDouble().set('tags', { type: 'set', value: members }); - const page = await readCollection(c, 'tags', 'set', { offset: 600, limit: 50, matches: all, filtered: false }); + const page = await readCollection(c, 'tags', 'set', { offset: 600, limit: 50 }); expect(page.map((r) => r.member)).toEqual(members.slice(600, 650)); }); @@ -142,14 +159,14 @@ describe('readCollection', () => { message: { n: String(i) }, })); const c = new RedisClientDouble().set('log', { type: 'stream', value: entries }); - const page = await readCollection(c, 'log', 'stream', { offset: 498, limit: 4, matches: all, filtered: false }); + const page = await readCollection(c, 'log', 'stream', { offset: 498, limit: 4 }); expect(page.map((r) => r.n)).toEqual(['498', '499', '500', '501']); expect(page[0]).toEqual({ entry_id: entries[498]!.id, n: '498' }); }); it('reads nothing for an empty page', async () => { const c = new RedisClientDouble().set('l', { type: 'list', value: ['a'] }); - expect(await readCollection(c, 'l', 'list', { offset: 0, limit: 0, matches: all, filtered: false })).toEqual([]); + expect(await readCollection(c, 'l', 'list', { offset: 0, limit: 0 })).toEqual([]); expect(c.calls).toEqual([]); }); }); @@ -191,6 +208,8 @@ describe('sampleKeyspace', () => { // hash with no prefix has no `prefix:*` keys to read. expect(byName.queue).toBeUndefined(); expect(byName.lonely).toBeUndefined(); + // One TYPE per key: the sampled rows reuse the types already read. + expect(c.calls.filter((x) => x === 'TYPE').length).toBe(c.keys.size); }); it('says when the keyspace was larger than the sample', async () => { diff --git a/packages/db/src/providers/redis/redis.keyspace.ts b/packages/db/src/providers/redis/redis.keyspace.ts index 8e7df8943..d58e675f3 100644 --- a/packages/db/src/providers/redis/redis.keyspace.ts +++ b/packages/db/src/providers/redis/redis.keyspace.ts @@ -23,31 +23,47 @@ * SMEMBERS, no LRANGE 0 -1). */ -/** Key types that read as a table of their own rather than as one row. */ -export const COLLECTION_TYPES: ReadonlySet = new Set(['list', 'set', 'zset', 'stream']); - /** What `TYPE` answers for a RedisJSON document (the module, or Redis 8). */ export const JSON_TYPE = 'ReJSON-RL'; -/** Elements fetched per command while reading a collection. */ +/** Elements fetched per command while reading a collection, and keys per pipelined batch. */ const CHUNK = 500; /** * Most elements one query examines while filtering or skipping. A WHERE on a * list has to read elements to test them; this keeps one page of the editor - * from walking a million-element list. + * from walking a million-element list, or a WHERE that matches nothing from + * walking a whole cache. */ export const MAX_SCAN = 50_000; type Row = Record; +/** + * A node-redis scan iterator (SCAN, SSCAN) a batch of names at a time. + * node-redis v6 yields a batch per step; older clients yielded one name. + * Assuming the single shape made `String(['a','b'])` the key, so every scan + * silently returned nothing. + */ +export async function* scanBatches(iterator: AsyncIterable): AsyncIterable { + for await (const batch of iterator) yield (Array.isArray(batch) ? batch : [batch]).map(String); +} + +/** `fn` over every item, CHUNK at a time: node-redis pipelines commands issued together, so a batch is one round trip. */ +async function inBatches(items: readonly T[], fn: (item: T) => Promise): Promise { + const out: R[] = []; + for (let i = 0; i < items.length; i += CHUNK) out.push(...(await Promise.all(items.slice(i, i + CHUNK).map(fn)))); + return out; +} + /** * A JSON object's top-level fields as cell values, or null when `text` is not * a JSON object. Nested objects and arrays become JSON text: a grid cell holds * one value, and flattening `a.b.c` into columns would invent a schema. */ export function jsonObjectFields(text: string | null | undefined): Row | null { - if (typeof text !== 'string' || text.trimStart()[0] !== '{') return null; + // A regex, not trimStart(): that copies the whole value to read one character. + if (typeof text !== 'string' || !/^\s*\{/.test(text)) return null; let parsed: unknown; try { parsed = JSON.parse(text); @@ -73,126 +89,172 @@ async function readJsonDocument(client: any, key: string): Promise { - switch (type) { - case 'list': - return Number(await client.lLen(key)); - case 'set': - return Number(await client.sCard(key)); - case 'zset': - return Number(await client.zCard(key)); - case 'stream': - return Number(await client.xLen(key)); - default: - return null; +/** A list element or stream entry: its JSON fields when it is a JSON object, else the raw value. */ +function elementRow(lead: Row, value: unknown): Row { + const fields = typeof value === 'string' ? jsonObjectFields(value) : null; + return fields ? { ...lead, ...fields } : { ...lead, value }; +} + +/** LRANGE / ZRANGE `size` at a time from index `start`. */ +async function* rangeChunks( + fetch: (from: number, to: number) => Promise, + toRow: (item: T, index: number) => Row, + start: number, + size: number +): AsyncIterable { + for (let at = start; ; at += size) { + const items = await fetch(at, at + size - 1); + if (items.length > 0) yield items.map((item, i) => toRow(item, at + i)); + if (items.length < size) return; } } +/** One collection type: how a key of it reads as a table. */ +interface CollectionSpec { + /** The columns before JSON elements widen the table, with their types (text unless named). */ + columns: string[]; + columnTypes?: Record; + length(client: any, key: string): Promise; + /** Rows `size` at a time, from element `start` when `indexed`, else from the first. */ + chunks(client: any, key: string, start: number, size: number): AsyncIterable; + /** LRANGE / ZRANGE start at an element index, so an unfiltered page skips nothing. */ + indexed: boolean; +} + +/** Every key type that reads as a table of its own rather than as one row. */ +const COLLECTIONS = new Map([ + [ + 'list', + { + columns: ['index', 'value'], + columnTypes: { index: 'integer' }, + indexed: true, + length: (c, k) => c.lLen(k), + chunks: (c, k, start, size) => + rangeChunks((a, b) => c.lRange(k, a, b), (v, i) => elementRow({ index: i }, v), start, size), + }, + ], + [ + 'zset', + { + columns: ['member', 'score'], + columnTypes: { score: 'double' }, + indexed: true, + length: (c, k) => c.zCard(k), + chunks: (c, k, start, size) => + rangeChunks<{ value: string; score: number }>( + (a, b) => c.zRangeWithScores(k, a, b), + (m) => ({ member: m.value, score: m.score }), + start, + size + ), + }, + ], + [ + 'set', + { + columns: ['member'], + indexed: false, + length: (c, k) => c.sCard(k), + // SSCAN's COUNT is only a hint, so a set reads CHUNK per step whatever the page. + async *chunks(c, k) { + for await (const members of scanBatches(c.sScanIterator(k, { COUNT: CHUNK }))) { + yield members.map((member) => ({ member })); + } + }, + }, + ], + [ + 'stream', + { + columns: ['entry_id'], + indexed: false, + length: (c, k) => c.xLen(k), + async *chunks(c, k, _start, size) { + let from = '-'; + for (;;) { + const entries: Array<{ id: string; message: Record }> = await c.xRange(k, from, '+', { + COUNT: size, + }); + if (entries.length > 0) { + yield entries.map((e) => ({ entry_id: e.id, ...e.message })); + // Exclusive start (`(id`) so the last entry is not read twice. + from = `(${entries[entries.length - 1]!.id}`; + } + if (entries.length < size) return; + } + }, + }, + ], +]); + +/** Key types that read as a table of their own rather than as one row. */ +export const COLLECTION_TYPES: ReadonlySet = new Set(COLLECTIONS.keys()); + +/** A column's type as the explorer shows it. Redis stores text; a list's index and a sorted set's score are numbers. */ +export function columnType(column: string, storeType: string): string { + return COLLECTIONS.get(storeType)?.columnTypes?.[column] ?? 'text'; +} + /** `users:7` under the table `users` → `7`; a key outside it keeps its name. */ function idWithin(table: string, key: string): string { return key.startsWith(`${table}:`) ? key.slice(table.length + 1) : key; } -/** What a key read as: the row, and which shape it took. */ -interface KeyRead { - row: Row | null; - shape: 'hash' | 'json' | 'string' | 'collection' | 'none'; +/** A key read as a row, the type it holds, and the shape the row took. */ +export interface KeyRead { + row: Row; + type: string; + shape: 'hash' | 'json' | 'string' | 'collection'; } -async function readKey(client: any, table: string, key: string): Promise { - const type = String(await client.type(key)); +/** + * One key of the prefix table `table` as a row, or null when it no longer + * exists. Pass `type` when it is already known, to skip the TYPE lookup. + * + * A collection key under a prefix still gets a row — its type and size — so the + * prefix lists everything under it; its elements are read through its own + * table (the key's full name). + */ +export async function readKey(client: any, table: string, key: string, type?: string): Promise { + const t = type ?? String(await client.type(key)); const id = idWithin(table, key); - switch (type) { + switch (t) { case 'none': - return { row: null, shape: 'none' }; + return null; case 'hash': { const hash = await client.hGetAll(key); - if (!hash || Object.keys(hash).length === 0) return { row: null, shape: 'none' }; - return { row: { id, ...hash }, shape: 'hash' }; + if (!hash || Object.keys(hash).length === 0) return null; + return { row: { id, ...hash }, type: t, shape: 'hash' }; } case 'string': { const value = await client.get(key); - if (value === null) return { row: null, shape: 'none' }; + if (value === null) return null; const fields = jsonObjectFields(value); - return fields ? { row: { id, ...fields }, shape: 'json' } : { row: { id, value }, shape: 'string' }; + return fields ? { row: { id, ...fields }, type: t, shape: 'json' } : { row: { id, value }, type: t, shape: 'string' }; } case JSON_TYPE: { const text = await readJsonDocument(client, key); - if (text === null) return { row: { id, type }, shape: 'json' }; + if (text === null) return { row: { id, type: t }, type: t, shape: 'json' }; const fields = jsonObjectFields(text); - return { row: fields ? { id, ...fields } : { id, value: text }, shape: 'json' }; + return { row: fields ? { id, ...fields } : { id, value: text }, type: t, shape: 'json' }; } - default: - return { row: { id, type, length: await collectionLength(client, key, type) }, shape: 'collection' }; - } -} - -/** - * One key of a prefix table as a row, or null when it no longer exists. - * - * A collection key under a prefix still gets a row — its type and size — so the - * prefix lists everything under it; its elements are read through its own - * table (the key's full name). - */ -export async function readKeyRow(client: any, table: string, key: string): Promise { - return (await readKey(client, table, key)).row; -} - -/** A list element or stream entry: its JSON fields when it is a JSON object, else the raw value. */ -function elementRow(lead: Row, value: unknown): Row { - const fields = typeof value === 'string' ? jsonObjectFields(value) : null; - return fields ? { ...lead, ...fields } : { ...lead, value }; -} - -async function* listChunks(client: any, key: string, start: number): AsyncIterable { - for (let at = start; ; at += CHUNK) { - const items: string[] = await client.lRange(key, at, at + CHUNK - 1); - if (items.length > 0) yield items.map((v, i) => elementRow({ index: at + i }, v)); - if (items.length < CHUNK) return; - } -} - -async function* sortedSetChunks(client: any, key: string, start: number): AsyncIterable { - for (let at = start; ; at += CHUNK) { - const items: Array<{ value: string; score: number }> = await client.zRangeWithScores(key, at, at + CHUNK - 1); - if (items.length > 0) yield items.map((m) => ({ member: m.value, score: m.score })); - if (items.length < CHUNK) return; - } -} - -async function* setChunks(client: any, key: string): AsyncIterable { - // node-redis yields a batch of members per step; older clients yielded one. - for await (const batch of client.sScanIterator(key, { COUNT: CHUNK })) { - const members = Array.isArray(batch) ? batch : [batch]; - yield members.map((m: unknown) => ({ member: String(m) })); - } -} - -async function* streamChunks(client: any, key: string): AsyncIterable { - let from = '-'; - for (;;) { - const entries: Array<{ id: string; message: Record }> = await client.xRange(key, from, '+', { - COUNT: CHUNK, - }); - if (entries.length > 0) { - yield entries.map((e) => ({ entry_id: e.id, ...e.message })); - // Exclusive start (`(id`) so the last entry is not read twice. - from = `(${entries[entries.length - 1]!.id}`; + default: { + const spec = COLLECTIONS.get(t); + return { row: { id, type: t, length: spec ? Number(await spec.length(client, key)) : null }, type: t, shape: 'collection' }; } - if (entries.length < CHUNK) return; } } /** * `limit` rows after skipping `offset` matching ones, examining at most - * `MAX_SCAN` elements. + * `MAX_SCAN` rows. */ -async function takePage( +export async function takePage( chunks: AsyncIterable, offset: number, limit: number, - matches: (row: Row) => boolean + matches: (row: Row) => boolean = () => true ): Promise { const out: Row[] = []; let skipped = 0; @@ -213,49 +275,24 @@ async function takePage( } /** - * A page of a collection key's elements. + * A page of a collection key's elements, those `matches` keeps when given. * - * Unfiltered lists and sorted sets start reading at `offset` itself (they are - * indexed); a filter, a set or a stream has to read from the start and skip. + * An unfiltered list or sorted set starts reading at `offset` itself (they are + * indexed); a filter, a set or a stream reads from the start and skips. */ export async function readCollection( client: any, key: string, type: string, - opts: { offset: number; limit: number; matches: (row: Row) => boolean; filtered: boolean } + opts: { offset: number; limit: number; matches?: (row: Row) => boolean } ): Promise { - const { offset, limit, matches, filtered } = opts; - if (limit <= 0) return []; - const indexedStart = filtered ? 0 : offset; - const skip = filtered ? offset : 0; - switch (type) { - case 'list': - return takePage(listChunks(client, key, indexedStart), skip, limit, matches); - case 'zset': - return takePage(sortedSetChunks(client, key, indexedStart), skip, limit, matches); - case 'set': - return takePage(setChunks(client, key), offset, limit, matches); - case 'stream': - return takePage(streamChunks(client, key), offset, limit, matches); - default: - return []; - } -} - -/** The columns a collection table has before any JSON elements widen it. */ -export function collectionColumns(type: string): string[] { - switch (type) { - case 'list': - return ['index', 'value']; - case 'set': - return ['member']; - case 'zset': - return ['member', 'score']; - case 'stream': - return ['entry_id']; - default: - return []; - } + const { offset, limit, matches } = opts; + const spec = COLLECTIONS.get(type); + if (!spec || limit <= 0) return []; + const start = spec.indexed && !matches ? offset : 0; + // Unfiltered, the page needs only the elements up to its end: never read more per command. + const size = matches ? CHUNK : Math.min(CHUNK, offset - start + limit); + return takePage(spec.chunks(client, key, start, size), offset - start, limit, matches); } export interface KeyspaceTable { @@ -290,72 +327,62 @@ export async function sampleKeyspace( const keys: string[] = []; let sampled = false; - outer: for await (const batch of client.scanIterator({ COUNT: CHUNK })) { - for (const k of Array.isArray(batch) ? batch : [batch]) { + outer: for await (const batch of scanBatches(client.scanIterator({ COUNT: CHUNK }))) { + for (const k of batch) { if (keys.length >= maxKeys) { sampled = true; break outer; } - keys.push(String(k)); + keys.push(k); } } - // node-redis pipelines commands issued together, so one round trip per batch. - const types: string[] = []; - for (let i = 0; i < keys.length; i += CHUNK) { - const batch = keys.slice(i, i + CHUNK); - types.push(...(await Promise.all(batch.map((k) => client.type(k).then(String))))); - } + const types = await inBatches(keys, async (k): Promise => String(await client.type(k))); - const prefixes = new Map }>(); + const prefixes = new Map>(); const collections: KeyspaceTable[] = []; keys.forEach((key, i) => { const type = types[i]!; if (type === 'none') return; - if (COLLECTION_TYPES.has(type)) { - collections.push({ name: key, storeType: type, kind: 'collection', columns: collectionColumns(type) }); + const spec = COLLECTIONS.get(type); + if (spec) { + collections.push({ name: key, storeType: type, kind: 'collection', columns: [...spec.columns] }); return; } const colon = key.indexOf(':'); if (colon <= 0) return; const prefix = key.slice(0, colon); - const entry = prefixes.get(prefix) ?? { keys: [], types: new Set() }; - entry.keys.push(key); - entry.types.add(type); - prefixes.set(prefix, entry); + prefixes.set(prefix, [...(prefixes.get(prefix) ?? []), { key, type }]); + }); + + // Up to `rowsPerTable` keys of every prefix, read together with the types already known. + const ordered = [...prefixes.entries()].sort(([a], [b]) => a.localeCompare(b)); + const samples = ordered.flatMap(([prefix, entries]) => entries.slice(0, rowsPerTable).map((e) => ({ prefix, ...e }))); + const reads = await inBatches(samples, (s) => readKey(client, s.prefix, s.key, s.type)); + const byPrefix = new Map(); + samples.forEach((s, i) => { + const read = reads[i]; + if (read) byPrefix.set(s.prefix, [...(byPrefix.get(s.prefix) ?? []), read]); }); - const tables: KeyspaceTable[] = []; - for (const [prefix, entry] of [...prefixes.entries()].sort(([a], [b]) => a.localeCompare(b))) { + const tables: KeyspaceTable[] = ordered.map(([prefix]) => { const columns = new Set(['id']); const kinds = new Set(); - for (const key of entry.keys.slice(0, rowsPerTable)) { - const { row, shape } = await readKey(client, prefix, key); - if (!row) continue; + for (const { row, shape } of byPrefix.get(prefix) ?? []) { kinds.add(shape); for (const field of Object.keys(row)) columns.add(field); } - tables.push({ - name: prefix, - storeType: kinds.size === 1 ? [...kinds][0]! : 'mixed', - kind: 'prefix', - columns: [...columns], - }); - } + return { name: prefix, storeType: kinds.size === 1 ? [...kinds][0]! : 'mixed', kind: 'prefix', columns: [...columns] }; + }); - for (const table of collections.sort((a, b) => a.name.localeCompare(b.name))) { - if (table.storeType === 'list' || table.storeType === 'stream') { - // JSON elements widen the table: their fields become columns. - const head = await readCollection(client, table.name, table.storeType, { - offset: 0, - limit: 20, - matches: () => true, - filtered: false, - }); - const columns = new Set(table.columns); - for (const row of head) for (const field of Object.keys(row)) columns.add(field); - table.columns = [...columns]; - } - tables.push(table); - } + // JSON elements widen a list or stream: the fields of its first elements become columns. + collections.sort((a, b) => a.name.localeCompare(b.name)); + const widen = collections.filter((t) => t.storeType === 'list' || t.storeType === 'stream'); + const heads = await inBatches(widen, (t) => readCollection(client, t.name, t.storeType, { offset: 0, limit: 20 })); + widen.forEach((table, i) => { + const columns = new Set(table.columns); + for (const row of heads[i]!) for (const field of Object.keys(row)) columns.add(field); + table.columns = [...columns]; + }); + tables.push(...collections); return { tables, sampled }; } diff --git a/packages/db/src/providers/redis/redis.provider.ts b/packages/db/src/providers/redis/redis.provider.ts index 2d8f28d1c..f77aa4538 100644 --- a/packages/db/src/providers/redis/redis.provider.ts +++ b/packages/db/src/providers/redis/redis.provider.ts @@ -27,17 +27,10 @@ import { type TableSchema, } from '@foxschema/sql'; import { redisAdapter } from './redis.adapter.js'; -import { sampleKeyspace, type KeyspaceSample } from './redis.keyspace.js'; +import { columnType, sampleKeyspace, type KeyspaceSample } from './redis.keyspace.js'; const DIALECT = 'redis'; -/** Column types as the explorer shows them. Redis stores text; index and score are numbers. */ -function columnType(name: string, storeType: string): string { - if (name === 'index' && storeType === 'list') return 'integer'; - if (name === 'score' && storeType === 'zset') return 'double'; - return 'text'; -} - /** Database numbers that hold keys, from `INFO keyspace` (`db3:keys=12,…`). */ export function databasesFromInfo(info: string): string[] { const dbs = new Set(['0']); diff --git a/packages/server/src/features/history/history.routes.ts b/packages/server/src/features/history/history.routes.ts index 8e0c471a9..dce983bfa 100644 --- a/packages/server/src/features/history/history.routes.ts +++ b/packages/server/src/features/history/history.routes.ts @@ -19,6 +19,7 @@ import { targetKey, targetLocks } from '../../platform/guards/target-lock'; import { LOKEE_FULL_SCOPE } from './lokee-scope'; import type { ConnectionRef } from '../../platform/connections/resolve'; import type { ConnectionOptions, MigrationModule } from '@foxschema/db'; +import { schemaCompareSupport } from '@foxschema/sql'; import { sendError, sendThrown } from '../../platform/http/respond'; import type { ForceMigrateErrorCode, LokeeRevertErrorCode } from '@foxschema/shared'; @@ -77,6 +78,14 @@ export function createHistoryRoutes(deps: HistoryRouteDeps): Router { scopeOf(req as AuthedRequest), body ); + // A force migrate writes DDL in the target's dialect. Redis and MongoDB + // have none: their tables are a keyspace sample, and `resolveDialect` + // answers Db2 for a name it does not know. + const support = schemaCompareSupport(dialect); + if (!support.supported) { + sendError(res, 'invalid_input', support.reason); + return null; + } return { versionId, dialect, option, schema }; } catch (error: unknown) { sendError(res, 'invalid_input', error instanceof Error ? error.message : 'Invalid connection'); diff --git a/packages/server/src/features/history/reversal-refusals.test.ts b/packages/server/src/features/history/reversal-refusals.test.ts index 6ee447866..8a66cc405 100644 --- a/packages/server/src/features/history/reversal-refusals.test.ts +++ b/packages/server/src/features/history/reversal-refusals.test.ts @@ -120,3 +120,25 @@ describe('when the install requires migrations committed to Git', () => { expect(execute).not.toHaveBeenCalled(); }); }); + +describe('force-migrate onto an engine without a SQL dialect', () => { + // Redis and MongoDB load as a keyspace sample, and `resolveDialect` answers + // Db2 for a name it does not know: a plan here would be Db2 DDL for Redis. + it.each(['/lokee/databases/:id/force-migrate/plan', FORCE])('%s refuses a Redis target before reading it', async (path) => { + const { d, execute } = deps('safe'); + d.resolveRef.mockResolvedValue({ dialect: 'redis', option: { host: 'h', database: '0' }, schema: '' }); + const { reply, sent } = fakeReply(); + await handlerFor(d, path)( + { + body: { connectionId: 'c', versionId: 'v1', confirmLossy: true, confirmForce: true }, + params: { id: 'db1' }, + userId: 'u1', + workspaceId: 'w1', + } as never, + reply as never + ); + expect(sent.status).toBe(400); + expect(d.loadScopedTables).not.toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/sql/src/cores/connection-auth.test.ts b/packages/sql/src/cores/connection-auth.test.ts index 1872d5426..8d10bca88 100644 --- a/packages/sql/src/cores/connection-auth.test.ts +++ b/packages/sql/src/cores/connection-auth.test.ts @@ -8,6 +8,7 @@ import { assertWindowsAccount, authMethodsForDialect, connectionNeedsSecret, + needsSessionPassword, normalizeAuthMethod, parseWindowsAccount, passwordFieldLabel, @@ -56,6 +57,15 @@ describe('connection auth methods', () => { expect(connectionNeedsSecret('postgres', 'password', '')).toBe(true); }); + it('asks for a password only when none is saved or given this session, and the engine needs one', () => { + expect(needsSessionPassword({ dialect: 'postgres' })).toBe(true); + expect(needsSessionPassword({ dialect: 'postgres', hasPassword: true })).toBe(false); + expect(needsSessionPassword({ dialect: 'postgres' }, 'typed-this-session')).toBe(false); + expect(needsSessionPassword({ dialect: 'sqlite' })).toBe(false); + expect(needsSessionPassword({ dialect: 'redis' })).toBe(false); + expect(needsSessionPassword({ dialect: 'redis', username: 'app' })).toBe(true); + }); + it('parses DOMAIN\\user and an explicit domain field', () => { expect(parseWindowsAccount('CONTOSO\\alice')).toEqual({ domain: 'CONTOSO', userName: 'alice' }); expect(parseWindowsAccount('alice', 'CONTOSO')).toEqual({ domain: 'CONTOSO', userName: 'alice' }); diff --git a/packages/sql/src/cores/connection-auth.ts b/packages/sql/src/cores/connection-auth.ts index eb16a94c0..c81e2ea1e 100644 --- a/packages/sql/src/cores/connection-auth.ts +++ b/packages/sql/src/cores/connection-auth.ts @@ -81,6 +81,23 @@ export function connectionNeedsSecret( return true; } +/** The fields of a saved connection the password rule reads. */ +export interface SecretGateConnection { + dialect: string; + authMethod?: string | null; + username?: string | null; + hasPassword?: boolean; +} + +/** + * Whether using this connection must first ask for its password: the engine + * needs a secret (`connectionNeedsSecret`), none was saved with it, and none + * was given this session. The one copy of the rule every prompt and gate reads. + */ +export function needsSessionPassword(conn: SecretGateConnection, sessionPassword?: string | null): boolean { + return !conn.hasPassword && connectionNeedsSecret(conn.dialect, conn.authMethod, conn.username) && !sessionPassword; +} + export function parseWindowsAccount( username?: string, domain?: string diff --git a/packages/sql/src/index.ts b/packages/sql/src/index.ts index 5916d09f1..cf9ea0949 100644 --- a/packages/sql/src/index.ts +++ b/packages/sql/src/index.ts @@ -403,6 +403,8 @@ export { authMethodsForDialect, dialectOffersAuthMethods, connectionNeedsSecret, + needsSessionPassword, + type SecretGateConnection, parseWindowsAccount, assertWindowsAccount, passwordFieldLabel, diff --git a/packages/sql/src/modules/sql-editor/sql-subset.ts b/packages/sql/src/modules/sql-editor/sql-subset.ts index 9eb812421..b9a6947b6 100644 --- a/packages/sql/src/modules/sql-editor/sql-subset.ts +++ b/packages/sql/src/modules/sql-editor/sql-subset.ts @@ -35,6 +35,9 @@ * functions and no expressions here. Not "not yet" — admitting any of them * means the intent can no longer be checked by inspection. */ +import { isKnownEngine } from '../capabilities/feature-reasons.js'; +import { hasSqlDialect } from '../dialect/sql-dialect-keys.js'; +import { splitTopLevel } from './column-origin.js'; export interface SubsetColumnEq { column: string; @@ -83,26 +86,7 @@ function unquote(raw: string): string { * as the table `v1:users"`, which matched nothing. */ function tableName(raw: string): string { - const parts: string[] = []; - let current = ''; - let close: string | null = null; - for (const ch of raw.trim()) { - if (close) { - current += ch; - if (ch === close) close = null; - continue; - } - if (ch === '"' || ch === '`') close = ch; - else if (ch === '[') close = ']'; - if (ch === '.' && !close) { - parts.push(current); - current = ''; - continue; - } - current += ch; - } - parts.push(current); - const names = parts.map(unquote).filter(Boolean); + const names = splitTopLevel(raw.trim(), '.').map(unquote).filter(Boolean); return names[names.length - 1] ?? ''; } @@ -217,27 +201,14 @@ const SEGMENT = /^[A-Za-z_$][\w$]*$/; */ function innerOfParens(text: string): string | null { if (!text.startsWith('(') || !text.endsWith(')')) return null; - let depth = 0; - let inQuote: string | null = null; - for (let i = 0; i < text.length; i++) { - const ch = text[i]!; - if (inQuote) { - if (ch === inQuote) { - if (text[i + 1] === inQuote) i++; - else inQuote = null; - } - continue; - } - if (ch === "'" || ch === '"' || ch === '`') inQuote = ch; - else if (ch === '(') depth++; - else if (ch === ')') { - depth--; - if (depth === 0) return i === text.length - 1 ? text.slice(1, -1) : null; - } - } - return null; + const inner = text.slice(1, -1); + // splitTop refuses unbalanced text: in `a = 1) AND (b = 2` a `)` closes early. + return splitTop(inner, NEVER) ? inner : null; } +/** A separator that never matches, for splitTop as a balance check. */ +const NEVER = /(?!)/; + /** `a = ? AND b = ?` → equality pairs, or null if anything else appears. */ function parseEqualityList( text: string, @@ -474,12 +445,14 @@ export function subsetValue(value: SubsetValue, params: readonly unknown[]): unk return value.kind === 'literal' ? value.value : (params[value.index] ?? null); } -/** Stores Fox reaches through this subset rather than through SQL. */ -const SUBSET_DIALECTS: ReadonlySet = new Set(['redis', 'mongodb']); - -/** Whether `dialect` is read through `parseSqlSubset` instead of an SQL engine. */ +/** + * Whether `dialect` is read through `parseSqlSubset` instead of an SQL engine: + * every engine Fox connects to that has no SQL dialect (Redis, MongoDB). Derived + * from the registry, the way schema compare decides, so a new store needs no + * list of its own here. + */ export function isSubsetDialect(dialect: string): boolean { - return SUBSET_DIALECTS.has((dialect || '').toLowerCase()); + return isKnownEngine(dialect) && !hasSqlDialect(dialect); } /** diff --git a/packages/ui-shared/src/index.ts b/packages/ui-shared/src/index.ts index 776623c41..2a69f08f6 100644 --- a/packages/ui-shared/src/index.ts +++ b/packages/ui-shared/src/index.ts @@ -42,7 +42,7 @@ export type { Dialect } from '@foxschema/sql'; // Connection strings, sign-in methods, error text, and small helpers. export { DEFAULT_PORTS, assertWindowsAccount, authMethodsForDialect, buildConnectionString, - connectionNeedsSecret, dialectOffersAuthMethods, errorMessage, escapeRegExp, nonSecretFingerprint, + connectionNeedsSecret, needsSessionPassword, dialectOffersAuthMethods, errorMessage, escapeRegExp, nonSecretFingerprint, normalizeAuthMethod, parseWindowsAccount, passwordFieldLabel, resolveAuthMethod, withConnectionString, } from '@foxschema/sql'; @@ -59,7 +59,7 @@ export type { // What each engine supports, and why not when it does not. export { - dialectFeatureReason, dialectFeatures, schemaCompareBlocker, supportsDialectFeature, + dialectFeatureReason, dialectFeatures, hasSqlDialect, schemaCompareBlocker, supportsDialectFeature, } from '@foxschema/sql'; export type { DialectFeature, FeatureSupport } from '@foxschema/sql'; From 90416f7bc70e351353b1c6d1be0d9b5b4cfcb892 Mon Sep 17 00:00:00 2001 From: huyplb Date: Sun, 25 Oct 2026 11:23:42 -0600 Subject: [PATCH 3/3] perf(redis): group sampled keys by push, not by copying the list each time Co-Authored-By: Claude Opus 5.5 --- packages/db/src/providers/redis/redis.keyspace.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/db/src/providers/redis/redis.keyspace.ts b/packages/db/src/providers/redis/redis.keyspace.ts index d58e675f3..a468fb940 100644 --- a/packages/db/src/providers/redis/redis.keyspace.ts +++ b/packages/db/src/providers/redis/redis.keyspace.ts @@ -351,7 +351,9 @@ export async function sampleKeyspace( const colon = key.indexOf(':'); if (colon <= 0) return; const prefix = key.slice(0, colon); - prefixes.set(prefix, [...(prefixes.get(prefix) ?? []), { key, type }]); + const entries = prefixes.get(prefix); + if (entries) entries.push({ key, type }); + else prefixes.set(prefix, [{ key, type }]); }); // Up to `rowsPerTable` keys of every prefix, read together with the types already known. @@ -361,7 +363,10 @@ export async function sampleKeyspace( const byPrefix = new Map(); samples.forEach((s, i) => { const read = reads[i]; - if (read) byPrefix.set(s.prefix, [...(byPrefix.get(s.prefix) ?? []), read]); + if (!read) return; + const list = byPrefix.get(s.prefix); + if (list) list.push(read); + else byPrefix.set(s.prefix, [read]); }); const tables: KeyspaceTable[] = ordered.map(([prefix]) => {