diff --git a/apps/cli/src/commands/drivers.ts b/apps/cli/src/commands/drivers.ts index 50189aabb..7cb9f76f4 100644 --- a/apps/cli/src/commands/drivers.ts +++ b/apps/cli/src/commands/drivers.ts @@ -126,25 +126,38 @@ export async function runDriversInstall(name: string): Promise { const mono = monorepoRootFromWeb(webRoot); const spec = entry.pin ? `${entry.pkg}@${entry.pin}` : entry.pkg; - // ibm_db must land where @foxschema/db can require it (not only apps/web). - const workspacePkg = entry.pkg === 'ibm_db' ? '@foxschema/db' : '@foxschema/web'; - const prefixRoot = - entry.pkg === 'ibm_db' - ? (() => { - try { - return dirname(require.resolve('@foxschema/db/package.json')); - } catch { - return webRoot; - } - })() - : webRoot; + // Drivers are required from @foxschema/db; installing into @foxschema/web + // leaves packages/db unable to resolve the module. + const workspacePkg = '@foxschema/db'; + const prefixRoot = (() => { + try { + return dirname(require.resolve('@foxschema/db/package.json')); + } catch { + return webRoot; + } + })(); + + const cwd = mono ?? prefixRoot; + const version = entry.pin ?? 'latest'; // Prefer workspace install in a checkout; otherwise --prefix into the package tree. // ibm_db must run install scripts or clidriver never downloads. + // Writing onto dependencies first: `npm install -w @foxschema/db` is a + // no-op when the package is already an optional peer. + if (mono) { + await new Promise((resolve, reject) => { + const child = spawn('npm', ['pkg', 'set', `dependencies.${entry.pkg}=${version}`, '-w', workspacePkg], { + cwd: mono, + stdio: 'inherit', + shell: process.platform === 'win32', + }); + child.on('error', reject); + child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`npm pkg set exited ${code}`)))); + }); + } const npmArgs = mono - ? ['install', spec, '--foreground-scripts', '-w', workspacePkg] - : ['install', spec, '--foreground-scripts', '--prefix', prefixRoot]; - const cwd = mono ?? prefixRoot; + ? ['install', '--foreground-scripts', '-w', workspacePkg] + : ['install', spec, '--save', '--foreground-scripts', '--prefix', prefixRoot]; await new Promise((resolve, reject) => { const child = spawn('npm', npmArgs, { diff --git a/apps/cli/src/format/__tests__/friendlyError.test.ts b/apps/cli/src/format/__tests__/friendlyError.test.ts index fde59b0d4..82b880193 100644 --- a/apps/cli/src/format/__tests__/friendlyError.test.ts +++ b/apps/cli/src/format/__tests__/friendlyError.test.ts @@ -20,6 +20,10 @@ describe('friendlyError', () => { expect(friendlyError(new Error('connect ETIMEDOUT 10.0.0.5:1433'))).toContain('timed out'); }); + it('rewrites ECONNRESET', () => { + expect(friendlyError(new Error('read ECONNRESET'))).toMatch(/closed the connection before login \(ECONNRESET\)/); + }); + it('rewrites Postgres auth failure', () => { expect(friendlyError(new Error('password authentication failed for user "foo"'))).toBe( 'Login failed — check the username and password on this connection.' diff --git a/apps/cli/src/format/friendlyError.ts b/apps/cli/src/format/friendlyError.ts index 58e182e33..a928e057b 100644 --- a/apps/cli/src/format/friendlyError.ts +++ b/apps/cli/src/format/friendlyError.ts @@ -19,6 +19,11 @@ const PATTERNS: Pattern[] = [ render: (m) => `Can't resolve host "${m[1]}" — check the hostname.`, }, { test: /ETIMEDOUT/i, render: () => 'Connection timed out — check the host, port, and any firewall/VPN.' }, + { + test: /ECONNRESET/i, + render: () => + 'The database closed the connection before login (ECONNRESET) — often a paused cluster, firewall, TLS mismatch, or a local Docker database that is listening but not ready.', + }, { test: /password authentication failed|access denied for user|login failed for user|ORA-01017|ORA-01005|SQLCODE=-1403|SQL30082/i, render: () => 'Login failed — check the username and password on this connection.', diff --git a/apps/e2e/src/generated/test-ids.ts b/apps/e2e/src/generated/test-ids.ts index 6519e7c76..2294adc70 100644 --- a/apps/e2e/src/generated/test-ids.ts +++ b/apps/e2e/src/generated/test-ids.ts @@ -756,6 +756,20 @@ export const TestIds = { /** div */ demoBanner: 'demo-banner', }, + DriverInventoryCard: { + /** section · Drivers */ + homeDrivers: 'home-drivers', + /** p */ + homeDriversActionError: 'home-drivers-action-error', + /** button */ + homeDriversCheck: 'home-drivers-check', + /** p */ + homeDriversError: 'home-drivers-error', + /** li */ + homeDriversLoading: 'home-drivers-loading', + /** p */ + homeDriversQueue: 'home-drivers-queue', + }, featureRegistry: { /** data · Snapshots */ syncPaneHistoryBtn: 'sync-pane-history-btn', @@ -773,13 +787,13 @@ export const TestIds = { HomeView: { /** button · Search workspaces and recents ⌘K */ homeCommandPalette: 'home-command-palette', - /** button */ - homeConnection: (id: string | number) => `home-connection-${id}`, /** section · Connections by provider */ homeConnections: 'home-connections', /** span */ homeConnectionsCount: 'home-connections-count', - /** div */ + /** span */ + homeConnectionsCountForDialect: (dialect: string | number) => `home-connections-count-${dialect}`, + /** button */ homeConnectionsGroup: (dialect: string | number) => `home-connections-group-${dialect}`, /** section */ homeContinue: 'home-continue', @@ -4465,15 +4479,21 @@ export type TestId = | `history-select-${string}` | 'history-select-all' | 'home-command-palette' - | `home-connection-${string}` | 'home-connections' | 'home-connections-count' + | `home-connections-count-${string}` | `home-connections-group-${string}` | 'home-continue' | 'home-continue-snapshots' | 'home-continue-sql' | 'home-continue-sync' | 'home-continue-utilities' + | 'home-drivers' + | 'home-drivers-action-error' + | 'home-drivers-check' + | 'home-drivers-error' + | 'home-drivers-loading' + | 'home-drivers-queue' | 'home-open-btn' | `home-recent-${string}` | 'home-recents' diff --git a/apps/web/src/frontend/app/shell/DriverInventoryCard.test.tsx b/apps/web/src/frontend/app/shell/DriverInventoryCard.test.tsx new file mode 100644 index 000000000..6f67985ad --- /dev/null +++ b/apps/web/src/frontend/app/shell/DriverInventoryCard.test.tsx @@ -0,0 +1,201 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + */ +import React from 'react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { DriverInventoryItem, DriverInventoryReport, Permission } from '@foxschema/shared'; + +const api = vi.hoisted(() => ({ + list: (_latest?: boolean): Promise => Promise.reject(new Error('not set')), + install: vi.fn(async (_packageName: string) => ({ success: true })), + update: vi.fn(async (_packageName: string) => ({ success: true })), + remove: vi.fn(async (_packageName: string) => ({ success: true })), +})); + +vi.mock('@/shared/api/servicesApi', () => ({ + fetchDriverInventory: (latest?: boolean) => api.list(latest), + installDriverPackage: (packageName: string) => api.install(packageName), + updateDriverPackage: (packageName: string) => api.update(packageName), + removeDriverPackage: (packageName: string) => api.remove(packageName), +})); + +vi.mock('@/shared/api/schemaApi', () => ({ + invalidateCache: () => {}, +})); + +import { useAuthStore } from '@/platform/authStore'; +import { DriverInventoryCard, driverRowTestId } from './DriverInventoryCard'; + +const row = (over: Partial & Pick): DriverInventoryItem => ({ + dialects: ['mongodb'], + labels: ['MongoDB'], + installed: false, + updateAvailable: false, + bundled: false, + removable: false, + ...over, +}); + +const report = (drivers: DriverInventoryItem[], latestChecked = false): DriverInventoryReport => ({ + drivers, + checkedAt: 1, + latestChecked, +}); + +function signIn(role: 'admin' | 'editor', permissions: Permission[] = []) { + useAuthStore.setState({ + user: { id: 'u1', email: 'u1@example.test', onboardingCompleted: true, role, permissions }, + status: 'ready', + }); +} + +const text = (id: string) => screen.getByTestId(id).textContent; + +describe('DriverInventoryCard', () => { + beforeEach(() => { + api.install.mockClear(); + api.update.mockClear(); + api.remove.mockClear(); + signIn('admin'); + api.list = async () => + report([ + row({ + packageName: 'pg', + dialects: ['postgres', 'cockroachdb'], + labels: ['PostgreSQL', 'CockroachDB'], + installed: true, + version: '8.23.1', + bundled: false, + removable: true, + }), + row({ packageName: 'mongodb', installCommand: 'npm install mongodb' }), + ]); + }); + + it('lists installed and missing drivers', async () => { + render(); + await waitFor(() => expect(text(`${driverRowTestId('pg')}-state`)).toContain('Installed · 8.23.1')); + expect(text(`${driverRowTestId('mongodb')}-state`)).toBe('Not installed'); + expect(screen.getByTestId(`${driverRowTestId('mongodb')}-install`)).toBeTruthy(); + expect(screen.getByTestId(`${driverRowTestId('pg')}-remove`)).toBeTruthy(); + }); + + it('offers no Install without admin.drivers', async () => { + signIn('editor'); + render(); + await waitFor(() => expect(screen.getByTestId(driverRowTestId('mongodb'))).toBeTruthy()); + expect(screen.queryByTestId(`${driverRowTestId('mongodb')}-install`)).toBeNull(); + }); + + it('installs a missing driver and refreshes the list', async () => { + let installed = false; + api.list = async () => + report([ + row({ + packageName: 'mongodb', + installed, + version: installed ? '6.9.0' : undefined, + removable: installed, + }), + ]); + api.install.mockImplementation(async () => { + installed = true; + return { success: true }; + }); + render(); + await waitFor(() => expect(screen.getByTestId(`${driverRowTestId('mongodb')}-install`)).toBeTruthy()); + fireEvent.click(screen.getByTestId(`${driverRowTestId('mongodb')}-install`)); + await waitFor(() => expect(text(`${driverRowTestId('mongodb')}-state`)).toBe('Installed · 6.9.0')); + expect(api.install).toHaveBeenCalledWith('mongodb'); + }); + + it('checks versions on first paint and offers Update when the registry is newer', async () => { + api.list = async (latest?: boolean) => + report( + [ + row({ + packageName: 'mongodb', + installed: true, + version: '6.0.0', + latest: latest ? '6.9.0' : undefined, + updateAvailable: !!latest, + removable: true, + }), + ], + !!latest + ); + render(); + await waitFor(() => expect(screen.getByTestId(`${driverRowTestId('mongodb')}-update`)).toBeTruthy()); + expect(text(`${driverRowTestId('mongodb')}-state`)).toContain('6.0.0 → 6.9.0'); + fireEvent.click(screen.getByTestId('home-drivers-check')); + await waitFor(() => expect(screen.getByTestId(`${driverRowTestId('mongodb')}-update`)).toBeTruthy()); + }); + + it('queues a second driver while the first npm is still running', async () => { + let releaseInstall: (() => void) | undefined; + let pgUpdated = false; + api.list = async () => + report([ + row({ packageName: 'mongodb', installCommand: 'npm install mongodb' }), + row({ + packageName: 'mssql', + dialects: ['sqlserver'], + labels: ['SQL Server'], + installed: true, + version: '12.7.13', + latest: '12.7.14', + updateAvailable: true, + removable: true, + }), + ]); + api.install.mockImplementation( + () => + new Promise((resolve) => { + releaseInstall = () => resolve({ success: true }); + }) + ); + api.update.mockImplementation(async () => { + pgUpdated = true; + return { success: true }; + }); + render(); + await waitFor(() => expect(screen.getByTestId(`${driverRowTestId('mongodb')}-install`)).toBeTruthy()); + fireEvent.click(screen.getByTestId(`${driverRowTestId('mongodb')}-install`)); + fireEvent.click(screen.getByTestId(`${driverRowTestId('mssql')}-update`)); + await waitFor(() => expect(text(`${driverRowTestId('mssql')}-update`)).toContain('Waiting')); + expect(screen.getByTestId('home-drivers-queue')).toBeTruthy(); + expect(api.update).not.toHaveBeenCalled(); + expect(releaseInstall).toBeTypeOf('function'); + releaseInstall!(); + await waitFor(() => expect(api.update).toHaveBeenCalledWith('mssql')); + expect(pgUpdated).toBe(true); + await waitFor(() => expect(screen.queryByTestId('home-drivers-queue')).toBeNull()); + }); + + it('asks once more before removing', async () => { + let gone = false; + api.list = async () => + report([ + row({ + packageName: 'mongodb', + installed: !gone, + version: gone ? undefined : '6.9.0', + removable: !gone, + }), + ]); + api.remove.mockImplementation(async () => { + gone = true; + return { success: true }; + }); + render(); + await waitFor(() => expect(screen.getByTestId(`${driverRowTestId('mongodb')}-remove`)).toBeTruthy()); + fireEvent.click(screen.getByTestId(`${driverRowTestId('mongodb')}-remove`)); + expect(api.remove).not.toHaveBeenCalled(); + fireEvent.click(screen.getByTestId(`${driverRowTestId('mongodb')}-remove-confirm`)); + await waitFor(() => expect(text(`${driverRowTestId('mongodb')}-state`)).toBe('Not installed')); + expect(api.remove).toHaveBeenCalledWith('mongodb'); + }); +}); diff --git a/apps/web/src/frontend/app/shell/DriverInventoryCard.tsx b/apps/web/src/frontend/app/shell/DriverInventoryCard.tsx new file mode 100644 index 000000000..37036987f --- /dev/null +++ b/apps/web/src/frontend/app/shell/DriverInventoryCard.tsx @@ -0,0 +1,291 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * Home → Drivers, under Services: which dialect packages are installed, a + * registry version check on first paint, and install / update / remove for + * accounts with admin.drivers. Only better-sqlite3 (the metadata store) + * cannot be removed. + * + * npm writes one node_modules tree, so two installs at once fail with + * ENOTEMPTY. Clicks queue here and run one after another. + */ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { Download, Loader2, Package, RefreshCw, Trash2 } from 'lucide-react'; +import type { DriverInventoryItem, DriverInventoryReport } from '@foxschema/shared'; +import { useAuthStore } from '@/platform/authStore'; +import { + fetchDriverInventory, + installDriverPackage, + removeDriverPackage, + updateDriverPackage, +} from '@/shared/api/servicesApi'; +import { invalidateCache } from '@/shared/api/schemaApi'; +import { SQL_ICON_STROKE } from '@/shared/lib/iconStyle'; + +type ActionKind = 'install' | 'update' | 'remove'; +type RowBusy = { kind: ActionKind; phase: 'running' | 'waiting' }; +type Job = { pkg: string; kind: ActionKind; fn: () => Promise }; + +export function driverRowTestId(packageName: string): string { + return `home-driver-${packageName.replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '')}`; +} + +function stateLabel(row: DriverInventoryItem): string { + if (!row.installed) return 'Not installed'; + const version = row.version ? ` · ${row.version}` : ''; + if (row.updateAvailable && row.latest) return `Update available · ${row.version} → ${row.latest}`; + if (row.latest) return `Installed${version} (latest ${row.latest})`; + return `Installed${version}`; +} + +function actionLabel(idle: string, busy: RowBusy | undefined, runningWord: string): string { + if (!busy) return idle; + if (busy.phase === 'waiting') return 'Waiting…'; + return runningWord; +} + +const Action: React.FC<{ + testId: string; + label: string; + busy: boolean; + tone: 'accent' | 'quiet' | 'danger'; + icon: React.ReactNode; + onClick: () => void; +}> = ({ testId, label, busy, tone, icon, onClick }) => { + const toneClass = + tone === 'accent' + ? 'bg-amber-400 on-accent-fg hover:bg-amber-300 disabled:bg-slate-700 disabled:text-slate-400' + : tone === 'danger' + ? 'border border-red-900/80 bg-slate-950 text-red-300 hover:border-red-700 hover:text-red-200 disabled:border-slate-800 disabled:text-slate-500' + : 'border border-slate-700 bg-slate-950 text-slate-300 hover:border-slate-500 hover:text-slate-100 disabled:border-slate-800 disabled:text-slate-500'; + return ( + + ); +}; + +export const DriverInventoryCard: React.FC = () => { + const canManage = useAuthStore((s) => s.can('admin.drivers')); + const [report, setReport] = useState(null); + const [error, setError] = useState(null); + const [checking, setChecking] = useState(false); + const [busyByPkg, setBusyByPkg] = useState>({}); + const [pendingRemove, setPendingRemove] = useState(null); + const [actionError, setActionError] = useState(null); + const queueRef = useRef([]); + const drainingRef = useRef(false); + const inflightRef = useRef(new Set()); + + const refresh = useCallback(async () => { + try { + setReport(await fetchDriverInventory(true)); + setError(null); + } catch (e: unknown) { + setError(e instanceof Error ? e.message : 'Could not read installed drivers'); + } + }, []); + + const load = useCallback(async () => { + setChecking(true); + try { + let lastError: unknown; + for (let attempt = 0; attempt < 4; attempt++) { + try { + setReport(await fetchDriverInventory(true)); + setError(null); + lastError = null; + break; + } catch (e: unknown) { + lastError = e; + if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, 400 * (attempt + 1))); + } + } + if (lastError) { + setError(lastError instanceof Error ? lastError.message : 'Could not read installed drivers'); + } + } finally { + setChecking(false); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + const drain = useCallback(async () => { + if (drainingRef.current) return; + drainingRef.current = true; + while (queueRef.current.length) { + const job = queueRef.current.shift()!; + setBusyByPkg((current) => ({ ...current, [job.pkg]: { kind: job.kind, phase: 'running' } })); + try { + await job.fn(); + invalidateCache('driver:'); + await refresh(); + } catch (e: unknown) { + setActionError((prev) => { + const message = e instanceof Error ? e.message : 'Driver action failed'; + return prev ? `${prev}\n${message}` : message; + }); + } finally { + inflightRef.current.delete(job.pkg); + setBusyByPkg((current) => { + const next = { ...current }; + delete next[job.pkg]; + return next; + }); + } + } + drainingRef.current = false; + }, [refresh]); + + const enqueue = (pkg: string, kind: ActionKind, fn: () => Promise) => { + if (inflightRef.current.has(pkg)) return; + inflightRef.current.add(pkg); + const idle = !drainingRef.current && queueRef.current.length === 0; + if (idle) setActionError(null); + setPendingRemove(null); + const phase: RowBusy['phase'] = drainingRef.current || queueRef.current.length > 0 ? 'waiting' : 'running'; + queueRef.current.push({ pkg, kind, fn }); + setBusyByPkg((current) => ({ ...current, [pkg]: { kind, phase } })); + void drain(); + }; + + const npmBusy = Object.keys(busyByPkg).length > 0; + const waiting = Object.values(busyByPkg).some((row) => row.phase === 'waiting'); + + return ( +
+

+ + Drivers + + +

+ {error && ( +

+ {error} +

+ )} + {actionError && ( +

+ {actionError} +

+ )} + {waiting && ( +

+ One npm at a time — the next driver starts when this one finishes. +

+ )} +
    + {!report && !error && ( +
  • + Checking… +
    • + )} + {report?.drivers.map((row) => { + const id = driverRowTestId(row.packageName); + const tone = !row.installed ? 'off' : row.updateAvailable ? 'warn' : 'running'; + const dot = tone === 'running' ? 'bg-sky-400' : tone === 'warn' ? 'bg-amber-400' : 'bg-slate-500'; + const rowBusy = busyByPkg[row.packageName]; + const thisBusy = !!rowBusy; + return ( +
  • +
    • + ); + })} +
+
+ ); +}; diff --git a/apps/web/src/frontend/app/shell/HomeView.test.tsx b/apps/web/src/frontend/app/shell/HomeView.test.tsx index 2f7913005..b44331519 100644 --- a/apps/web/src/frontend/app/shell/HomeView.test.tsx +++ b/apps/web/src/frontend/app/shell/HomeView.test.tsx @@ -61,11 +61,11 @@ describe('HomeView', () => { expect(ensureSchema).not.toHaveBeenCalled(); }); - it('opens a saved connection as a SQL destination without introspecting', () => { + it('opens Compare from a provider count without listing credentials', () => { render(); - fireEvent.click(screen.getByTestId('home-connection-c1')); - expect(ensureConnectionSelected).toHaveBeenCalledWith('c1'); - expect(setActiveView).toHaveBeenCalledWith('sqlEditor'); + fireEvent.click(screen.getByTestId('home-connections-group-sqlite')); + expect(setActiveView).toHaveBeenCalledWith('sync'); + expect(ensureConnectionSelected).not.toHaveBeenCalled(); expect(ensureSchema).not.toHaveBeenCalled(); }); @@ -82,13 +82,11 @@ describe('HomeView', () => { it('groups connections by provider on the home index', () => { render(); - expect(screen.getByTestId('home-connections').textContent).toMatch(/Connections by provider/); - expect(screen.getByTestId('home-connections-group-postgres')).toBeTruthy(); - expect(screen.getByTestId('home-connections-group-sqlite')).toBeTruthy(); - const sqliteGroup = screen.getByTestId('home-connections-group-sqlite'); - expect(sqliteGroup.textContent).toMatch(/Demo SQLite/); - expect(sqliteGroup.textContent).toMatch(/Lab SQLite/); - expect(screen.getByTestId('home-connections-group-postgres').textContent).toMatch(/Warehouse/); + const section = screen.getByTestId('home-connections'); + expect(section.textContent).toMatch(/Connections by provider/); + expect(screen.getByTestId('home-connections-count-postgres').textContent).toBe('1'); + expect(screen.getByTestId('home-connections-count-sqlite').textContent).toBe('2'); + expect(section.textContent).not.toMatch(/Demo SQLite|Lab SQLite|Warehouse|db\.internal/); }); it('opens the Utilities workspace from Home', () => { diff --git a/apps/web/src/frontend/app/shell/HomeView.tsx b/apps/web/src/frontend/app/shell/HomeView.tsx index 83ac2f605..21b264b6b 100644 --- a/apps/web/src/frontend/app/shell/HomeView.tsx +++ b/apps/web/src/frontend/app/shell/HomeView.tsx @@ -8,12 +8,13 @@ */ import React from 'react'; import { Camera, Database, GitCompareArrows, Search, Terminal, Wrench } from 'lucide-react'; -import { openRecentQuery, selectEditorConnection, useRecentQueries } from '@/features/sql-editor'; +import { openRecentQuery, useRecentQueries } from '@/features/sql-editor'; import { useSyncStore } from '@/features/compare'; import { useUiStore } from '@/platform/uiStore'; import { formatRelativeDay } from '@/shared/lib/relativeTime'; import { openCommandPalette } from './commandPaletteEvent'; import { ServiceStatusCard } from './ServiceStatusCard'; +import { DriverInventoryCard } from './DriverInventoryCard'; import { diffBriefing } from '@/features/schema-diff'; import { dialectLabel } from '@/shared/lib/dialectLabel'; @@ -22,6 +23,19 @@ function previewSql(sql: string): string { return line.length > 88 ? `${line.slice(0, 87)}…` : line; } +function connectionsByProvider( + connections: Array<{ dialect: string }> +): Array<{ dialect: string; label: string; count: number }> { + const counts = new Map(); + for (const connection of connections) { + const dialect = connection.dialect.toLowerCase(); + const row = counts.get(dialect); + if (row) row.count += 1; + else counts.set(dialect, { dialect, label: dialectLabel(connection.dialect), count: 1 }); + } + return [...counts.values()].sort((a, b) => a.label.localeCompare(b.label)); +} + export const HomeView: React.FC = () => { // Not the SQL editor store: that loads with the editor, not with Home. const recentQueries = useRecentQueries((s) => s.recentQueries); @@ -36,13 +50,9 @@ export const HomeView: React.FC = () => { setActiveView('sqlEditor'); }; - const openConnection = (id: string) => { - void selectEditorConnection(id); - setActiveView('sqlEditor'); - }; - const briefing = diffBriefing(compareResult?.tables); const lastQuery = recentQueries[0]; + const providerCounts = connectionsByProvider(connections); return (
@@ -137,6 +147,7 @@ export const HomeView: React.FC = () => { +

@@ -188,64 +199,28 @@ export const HomeView: React.FC = () => { {connections.length === 0 ? (

Save a connection from Compare to see it here.

) : ( -
- {[...connections] - .sort( - (a, b) => - dialectLabel(a.dialect).localeCompare(dialectLabel(b.dialect)) || - (a.name || '').localeCompare(b.name || '') - ) - .reduce>( - (groups, c) => { - const key = c.dialect.toLowerCase(); - const last = groups[groups.length - 1]; - if (last && last.dialect === key) { - last.items = [...last.items, c]; - } else { - groups.push({ dialect: key, label: dialectLabel(c.dialect), items: [c] }); - } - return groups; - }, - [] - ) - .map((group) => ( -
-

- {group.label} - - {group.items.length} - -

-
    - {group.items.map((c) => ( -
  • - -
    • - ))} -
-
- ))} -
+
    + {providerCounts.map((group) => ( +
  • + +
    • + ))} +
)}

diff --git a/apps/web/src/frontend/shared/api/servicesApi.ts b/apps/web/src/frontend/shared/api/servicesApi.ts index 65545be92..f8e3046ea 100644 --- a/apps/web/src/frontend/shared/api/servicesApi.ts +++ b/apps/web/src/frontend/shared/api/servicesApi.ts @@ -3,11 +3,28 @@ * Copyright 2024-2026 Huy Phan * SPDX-License-Identifier: Apache-2.0 * - * The state of the services the app runs, for the Services card on Home. + * The state of the services the app runs, for the Services card on Home, and + * the driver inventory under it. */ -import type { ServiceStatusReport } from '@foxschema/shared'; +import type { DriverInventoryReport, ServiceStatusReport } from '@foxschema/shared'; import { api } from './client'; export function fetchServiceStatus(): Promise { return api.get('/system/services'); } + +export function fetchDriverInventory(latest = false): Promise { + return api.get('/system/drivers', latest ? { query: { latest: 1 } } : undefined); +} + +export function installDriverPackage(packageName: string): Promise<{ success: boolean; hint?: string }> { + return api.post('/system/drivers/install', { packageName }); +} + +export function updateDriverPackage(packageName: string): Promise<{ success: boolean; hint?: string }> { + return api.post('/system/drivers/update', { packageName }); +} + +export function removeDriverPackage(packageName: string): Promise<{ success: boolean; hint?: string }> { + return api.post('/system/drivers/remove', { packageName }); +} diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index a9db88881..71fda0125 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -12,7 +12,19 @@ export default defineConfig({ react(), tailwindcss(), // .br and .gz beside every text file, sent by the server in place of it. - precompress() + precompress(), + { + name: 'skip-driver-npm-reload', + configureServer(server) { + // watch.ignored is not enough: Vite's package-data watcher still + // full-reloads when npm pkg set writes apps/web/package.json. + server.watcher.unwatch(['**/package.json', '**/package-lock.json']); + }, + handleHotUpdate({ file }) { + const n = file.replaceAll('\\', '/'); + if (/\/(package-lock\.json|package\.json)$/.test(n) && !n.includes('/src/')) return []; + }, + }, ], build: { rolldownOptions: { @@ -85,6 +97,17 @@ export default defineConfig({ dedupe: ['react', 'react-dom', 'ajv'], }, server: { + watch: { + // Driver install/remove rewrites workspace package.json and the lockfile. + // Watching those full-reloads the page, which aborts the in-flight npm + // and leaves node_modules half-renamed (ENOTEMPTY). + ignored: [ + '**/node_modules/**', + '**/.git/**', + '**/package.json', + '**/package-lock.json', + ], + }, // Explicit IPv4 bind — some Cursor/cloud port-forwards fail on // dual-stack `:::5173` with ERR_CONNECTION_REFUSED from the client. host: '0.0.0.0', @@ -99,6 +122,8 @@ export default defineConfig({ // Match DEFAULT_API_PORT (3210). Override with API_PORT when needed. target: `http://localhost:${process.env.API_PORT || 3210}`, changeOrigin: true, + // Driver install (ibm_db clidriver) exceeds the default 120s proxy idle. + timeout: 10 * 60 * 1000, // Say it was forwarded. The dev server listens on every interface, and // without these headers the API would take a browser elsewhere on the // LAN for someone at this machine, who may run first-run setup without diff --git a/docs/CODE_MAP.md b/docs/CODE_MAP.md index 9b0b8a3ef..60ae65414 100644 --- a/docs/CODE_MAP.md +++ b/docs/CODE_MAP.md @@ -27,6 +27,7 @@ packages/shared Contracts the frontend, server and CLI must agree on the COMMUNITY_NAV registry (`nav.ts`), the shared data-migrate cap (`DATA_MIGRATE_MAX_OPS`), GET `/api/system/services` (`service-status.ts`), + GET `/api/system/drivers` (`driver-inventory.ts`), and the connection-name rule (`connection-name.ts`: 120 characters, no control characters). Browser-safe. @@ -160,7 +161,7 @@ database/ The metadata store and its migrations. | Folder | What it covers | |---|---| | `access` | Database permission inspection and DBA utilities. Index-fragmentation custom SQL is gated read-only in `index-fragmentation.service.ts` (same tokenizer as `/sql/execute`). | -| `system` | Health and version, updates, app info, the metadata-database probe, activity, Home Services (`GET /api/system/services` → `readServiceStatus`: API always `running`; engine `running` / `not_configured` / `unreachable` from the workflow `/health` probe). Status only — start/stop is later. | +| `system` | Health and version, updates, app info, the metadata-database probe, activity, Home Services (`GET /api/system/services` → `readServiceStatus`: API always `running`; engine `running` / `not_configured` / `unreachable` from the workflow `/health` probe). Status only — start/stop is later. Home Drivers (`GET /api/system/drivers`, `POST .../install|update|remove` → `drivers.service`: one row per npm package; `admin.drivers` to change; only `better-sqlite3` cannot be removed). | | `admin` | Users and roles, secrets, cloud credentials, policy | | `backup` | Per-user backup defaults (`GET`/`PUT /api/backup-settings`). Commands themselves are built in `@foxschema/sql` (`modules/utilities/backup.ts` + `providers//*.backup.ts`); the panel runs only a server-side SQL backup, after confirmation, and lists the backups the server recorded (`backupHistoryQuery`); restores are never run. | | `auth` | Sign-in, setup, password and SSO routes (the services are `platform/identity`) | @@ -201,6 +202,8 @@ A feature uses the layers it needs; it does not create empty ones. app/ The application shell and settings screens. Home's Services card is `app/shell/ServiceStatusCard.tsx` (`GET /api/system/services` via `shared/api/servicesApi.ts`); it does not start or stop anything. + Drivers sit under it: `app/shell/DriverInventoryCard.tsx` + (`GET /api/system/drivers`). features/ The view registry: `featureRegistry.ts` (each view's rail button, who may see it, where someone goes who may not, what it adds to the top toolbar, @@ -290,6 +293,7 @@ the page-epoch guard, bookmarks and recents, SQL variables. | Backup / restore commands for an engine | `packages/sql/src/providers//*.backup.ts` registered in `modules/utilities/backup.registry.ts`. Saved folder/format defaults: `packages/server/src/features/backup/`. | | Safe column-change SQL for an engine | `packages/sql/src/providers//*.column-change.ts` registered in `modules/utilities/column-change.registry.ts`. The facade (`planColumnChange`) finds impact and blockers; omit the engine to answer "not yet" rather than guessed SQL. The panel is `apps/web/.../ColumnChangePanel.tsx`; no server route. | | Home Services / engine reachability | `packages/server/src/features/system/service-status.ts` (`GET /api/system/services`). Types in `packages/shared/src/service-status.ts`. The card is `apps/web/.../app/shell/ServiceStatusCard.tsx`. | +| Home Drivers inventory | `packages/server/src/features/system/drivers.service.ts` (`GET /api/system/drivers`, install/update/remove). Types in `packages/shared/src/driver-inventory.ts`. The card is `apps/web/.../app/shell/DriverInventoryCard.tsx`. | | A dialect capability the app must branch on | `packages/sql/src/modules/capabilities/` (`dialect-features.ts`) and `packages/sql/src/modules/dialect/` | | Guard or cross-cutting HTTP concern | `packages/server/src/platform/` | | How a dialect lexes SQL (permission gates, splitter) | `packages/sql/src/modules/sql-text/sql-readings.ts` + `sql-tokens.ts`. Do not add a second scanner. | diff --git a/docs/FEATURE_MAP.md b/docs/FEATURE_MAP.md index e035a9351..3c29e8def 100644 --- a/docs/FEATURE_MAP.md +++ b/docs/FEATURE_MAP.md @@ -147,9 +147,9 @@ in its folders and its e2e specs (`apps/e2e/src/tests`). | `platform.auth` | Sign-in & accounts — password sign-in, register, first-run setup code, launch session (use Fox before creating an account), email verification, forgot / reset password, SSO (Google, Microsoft, GitHub, broker), sign out other sessions, onboarding wizard | Sign-in page, account banners | `/api/auth` (17), `/api/signup` (3) | server `auth`, `users`, `platform/identity`; web `auth` | — (public routes) | `users`, `sessions`, `auth_codes`, `user_preferences` | server 9, web 3; e2e `auth-recovery`, `smoke` | | `platform.admin` | Administration — users (create, invite, role, activate, set password, issue code, hand over admin), role → permission matrix, one admin or several, sign-in settings (providers, mail, broker, public URL), every workspace (restore archived), Demo role and demo mode (banner, sign-in hint) | Profile → Access control | `/api/admin` (19) | server `admin`; web `admin` | `admin.users`, `admin.roles` | `users`, `role_permissions`, `app_settings` | server 4, web 5; e2e `admin-policy`, `demo-role` | | `platform.workspaces` | Workspaces — rail button and menu, create, switch, archive, restore, settings, members and workspace roles (a personal workspace lends no more than its account holds), invites (accept / decline, 14 days), public discover / join, shared workspace scripts | Rail button under the Fox mark | `/api/workspaces` (20) | server `workspaces`, `platform/workspaces`; web `workspaces` | `workspace.members`, `workspace.settings`, `workspace.create` | `workspaces`, `workspace_members`, `workspace_invites`, `workspace_scripts` | server 5, web 3; e2e `workspaces`, `workspaces-sharing-limits` | -| `platform.connections` | Connections & drivers — credential manager, connection form (password / Windows / LDAP / No password on Redis and MongoDB, SSL, pool, SQLite / DuckDB file picker), schema list, connection test, driver check and Install driver, cloud provider credentials | Creds (rail), connection modal | `/api/connections` (4), `/api/connection/test`, `/api/db/test`, `/api/driver` (2) | server `connections`, `platform/connections`, `internal/driver-install`; web `connections` | per workspace role; `admin.drivers` to install | `connections`, `cloud_provider_credentials` | server 1, web 1 | +| `platform.connections` | Connections & drivers — credential manager, connection form (password / Windows / LDAP / No password on Redis and MongoDB, SSL, pool, SQLite / DuckDB file picker), schema list, connection test, driver check and Install driver, Home Drivers inventory (list / version check / update / remove), cloud provider credentials | Creds (rail), connection modal, Home | `/api/connections` (4), `/api/connection/test`, `/api/db/test`, `/api/driver` (2), `/api/system/drivers` (4) | server `connections`, `system` (`drivers.service.ts`), `platform/connections`, `internal/driver-install`; web `connections`, `app/shell/DriverInventoryCard` | per workspace role; `admin.drivers` to install, update, or remove | `connections`, `cloud_provider_credentials` | server 1, web 1 | | `platform.secrets` | App secrets — vault entries, cloud secret providers, resolving a secret | Creds → secrets, editor Secrets panel | `/api/app-secrets` (9) | server `admin` (app-secrets mount) | `secrets.view/create/edit/delete` | `app_secrets`, `cloud_provider_credentials` | — | -| `platform.system` | Shell, health, updates — activity rail, top toolbar, command palette, home, **Services** card (API / Web UI / Workflow engine status; no start/stop yet), backend-offline banner, activity, app info, config, update check / apply, email and update settings, send feedback | App shell, Prefs, Home | `/api/health`, `/api/config`, `/api/app-info`, `/api/activity`, `/api/updates` (2), `/api/system/services` | server `system` (`service-status.ts`); web `app/shell` (`ServiceStatusCard`), `app/settings`, `feedback` | — | `app_settings` | server 2, web 3; e2e `smoke`, `lazy-loading`, `send-feedback`, `auto-error-catch` | +| `platform.system` | Shell, health, updates — activity rail, top toolbar, command palette, home, **Services** card (API / Web UI / Workflow engine status; no start/stop yet), **Drivers** card (installed packages, version check, install / update / remove), backend-offline banner, activity, app info, config, update check / apply, email and update settings, send feedback | App shell, Prefs, Home | `/api/health`, `/api/config`, `/api/app-info`, `/api/activity`, `/api/updates` (2), `/api/system/services`, `/api/system/drivers` (4) | server `system` (`service-status.ts`, `drivers.service.ts`); web `app/shell` (`ServiceStatusCard`, `DriverInventoryCard`), `app/settings`, `feedback` | `admin.drivers` to change drivers | `app_settings` | server 2, web 3; e2e `smoke`, `lazy-loading`, `send-feedback`, `auto-error-catch` | ### Schema diff --git a/docs/releases/UNRELEASED.md b/docs/releases/UNRELEASED.md index ef0c9f44f..121e8a6bc 100644 --- a/docs/releases/UNRELEASED.md +++ b/docs/releases/UNRELEASED.md @@ -76,6 +76,22 @@ on the server, so it needs the new **Install drivers** permission (admins by default, from the account role); without it the banner gives the command to run. Every Install button shares one status, and the server runs one install at a time. +Home → **Drivers** (under Services) lists each driver package this install can +load, with its version. Opening Home checks npm for newer releases. **Install**, +**Update**, and **Remove** use the same **Install drivers** permission; only +SQLite (`better-sqlite3`, the metadata store) cannot be removed. Install saves +the package on `@foxschema/db` so a missing peer (Redis, MongoDB) is actually +fetched, including nested dependencies. **Update** also pins every workspace +that already lists the package (so SQL Server is not stuck on the web app's +`12.7.13` while only `@foxschema/db` moves), and rewrites a caret peer that +would block a major (`^1.5` vs DuckDB 2). Two clicks queue rather than +running two npm processes in one `node_modules` (that was the `ENOTEMPTY` +failure). Remove/update retries leftover rename dirs and does not reload the +page when the lockfile changes. Load and npm errors keep a package-relative +file (`@clickhouse/client/dist/index.js`), not the install directory; action +responses no longer return `cwd` / raw npm output. A dropped TCP socket +(`ECONNRESET`) is explained as the database closing before login. + ## Workflow: steer a paused run; connections stay in their workspace **Runs** can cancel a live job and answer **Ask a person**. Saving or running @@ -94,7 +110,7 @@ catalog names. at the bottom of the rail is migration-run history. They are not the same list. - **Utils** is its own view (Index Management, **Backup & Restore**, Clone Table, insights, Query files, DB users & grants) — not a SQL Editor sidebar. -- Saved credentials are grouped by dialect; Home continues last compare / query. +- Saved credentials are counted by dialect on Home (no host, user, or name list); Home continues last compare / query. **⌘K** / the command palette list the same views the rail shows. ## Team workspaces diff --git a/docs/testing/TEST_IDS.md b/docs/testing/TEST_IDS.md index 39e0f7a4e..bc75ed9fe 100644 --- a/docs/testing/TEST_IDS.md +++ b/docs/testing/TEST_IDS.md @@ -8,7 +8,7 @@ or `TestIds` for the same tree with autocomplete. `{name}` marks a part filled i time; "in FilterPicker" marks an ID a shared component draws from the prop written here. How to write a test with them: [WRITING_E2E.md](WRITING_E2E.md). -Every control has a test ID: **1024** buttons, text boxes, selects and textareas. +Every control has a test ID: **1026** buttons, text boxes, selects and textareas. ## access @@ -399,6 +399,17 @@ Every control has a test ID: **1024** buttons, text boxes, selects and textareas - `command-palette-item-{id}` · button - **DemoBanner** · `app/shell/DemoBanner.tsx` - `demo-banner` · div +- **DriverInventoryCard** · `app/shell/DriverInventoryCard.tsx` + - `{driverRowTestId(row.packageName)}` · li · passed in by the parent + - `{id}-detail` · span · passed in by the parent + - `{id}-state` · span · passed in by the parent + - `{testId}` · button · passed in by the parent + - `home-drivers` · section · Drivers + - `home-drivers-action-error` · p + - `home-drivers-check` · button + - `home-drivers-error` · p + - `home-drivers-loading` · li + - `home-drivers-queue` · p - **featureRegistry** · `app/features/featureRegistry.ts` - `sync-pane-history-btn` · data · Snapshots - `view-access-btn` · data @@ -408,10 +419,10 @@ Every control has a test ID: **1024** buttons, text boxes, selects and textareas - `view-workflow-btn` · data - **HomeView** · `app/shell/HomeView.tsx` - `home-command-palette` · button · Search workspaces and recents ⌘K - - `home-connection-{id}` · button - `home-connections` · section · Connections by provider - `home-connections-count` · span - - `home-connections-group-{dialect}` · div + - `home-connections-count-{dialect}` · span + - `home-connections-group-{dialect}` · button - `home-continue` · section - `home-continue-snapshots` · button · Snapshots Schema history Timeline, briefing, and compare pane - `home-continue-sql` · button · Last query diff --git a/docs/user-guide.md b/docs/user-guide.md index a614b832e..b9773efd6 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -103,7 +103,12 @@ and saved connections (grouped by dialect). A **Services** card lists the **API* set but its health check did not answer). A running engine that is Disabled in Workflow settings still shows Running, with “Not taking new runs”. The card refreshes every 10 seconds while Home is open. Status only: there is no Start -or Stop from here yet. **⌘K** / **Ctrl+K** searches the same views the left rail +or Stop from here yet. Under it, **Drivers** lists each database driver package +this install can load (several dialects share one package, e.g. `pg`), whether +it is installed and which version. Opening Home checks npm for a newer +release; **Check versions** runs that again. **Install**, **Update**, and **Remove** +need the **Install drivers** permission (admins by default). Only SQLite +(`better-sqlite3`, the metadata store) cannot be removed. **⌘K** / **Ctrl+K** searches the same views the left rail shows, plus recents. The left rail is the Fox mark (Home), the **workspace** button (the one workspace this account is acting in), then **Compare**, **Editor**, **Utils**, **Access**, **Workflow**, **Snapshots**, and **Creds** and diff --git a/packages/db/src/cores/driver-detector.test.ts b/packages/db/src/cores/driver-detector.test.ts new file mode 100644 index 000000000..0b6a3c548 --- /dev/null +++ b/packages/db/src/cores/driver-detector.test.ts @@ -0,0 +1,32 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + */ +import { describe, expect, it } from 'vitest'; +import { DriverDetector, readInstalledPackageVersion } from './driver-detector.js'; + +describe('readInstalledPackageVersion', () => { + it('reads pg from its package.json, not from the module export', () => { + const version = readInstalledPackageVersion('pg'); + expect(version).toMatch(/^\d+\.\d+/); + expect(DriverDetector.checkDialect('postgres')).toMatchObject({ + packageName: 'pg', + installed: true, + version, + }); + }); + + it('returns nothing for a package that is not installed', () => { + expect(readInstalledPackageVersion('foxschema-no-such-driver-package')).toBeUndefined(); + }); +}); + +describe('DriverDetector.detectAll', () => { + it('reuses one check for dialects that share a package', () => { + const pg = DriverDetector.detectAll().filter((info) => info.packageName === 'pg'); + expect(pg.length).toBeGreaterThan(1); + expect(new Set(pg.map((info) => info.installed)).size).toBe(1); + expect(new Set(pg.map((info) => info.version)).size).toBe(1); + }); +}); diff --git a/packages/db/src/cores/driver-detector.ts b/packages/db/src/cores/driver-detector.ts index 6b3246f82..0dd1b542c 100644 --- a/packages/db/src/cores/driver-detector.ts +++ b/packages/db/src/cores/driver-detector.ts @@ -1,4 +1,6 @@ +import { readFileSync } from 'node:fs'; import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; import type { DriverInfo } from '@foxschema/sql'; import { getAdapter, ADAPTERS } from '../providers/adapter-registry.js'; import { hasDb2Clidriver, setupDb2ClientEnv } from '../providers/db2/db2.env.js'; @@ -7,6 +9,43 @@ import { driverLoadReason } from './driver-loader.js'; const nodeRequire = createRequire(import.meta.url); +/** + * Version printed on the package's own package.json, which most drivers do + * not also export as `mod.version`. Walks up from the resolved entry when + * `package.json` is not in the package's exports map. + */ +export function readInstalledPackageVersion(packageName: string): string | undefined { + try { + const pkgPath = nodeRequire.resolve(`${packageName}/package.json`); + // eslint-disable-next-line security/detect-non-literal-fs-filename -- path is require.resolve of an allow-listed driver package + const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { version?: string }; + return pkg.version; + } catch { + try { + const main = nodeRequire.resolve(packageName); + let dir = dirname(main); + for (let i = 0; i < 6; i++) { + try { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- walks up from the resolved driver entry looking for its manifest + const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { + name?: string; + version?: string; + }; + if (pkg.name === packageName && pkg.version) return pkg.version; + } catch { + /* keep walking */ + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } catch { + /* not installed */ + } + return undefined; + } +} + export type { DriverInfo }; /** @@ -49,7 +88,7 @@ export class DriverDetector { provider: dialect, packageName, installed: true, - version: mod?.version ?? mod?.default?.version, + version: readInstalledPackageVersion(packageName) ?? mod?.version ?? mod?.default?.version, }; } catch (error: unknown) { const message = driverLoadReason(error); @@ -85,6 +124,14 @@ export class DriverDetector { } static detectAll(): DriverInfo[] { - return Object.keys(ADAPTERS).map((dialect) => this.checkDialect(dialect)); + const byPackage = new Map(); + return Object.keys(ADAPTERS).map((dialect) => { + const packageName = getAdapter(dialect).packageName; + const prior = byPackage.get(packageName); + if (prior) return { ...prior, provider: dialect }; + const info = this.checkDialect(dialect); + byPackage.set(packageName, info); + return info; + }); } } diff --git a/packages/db/src/cores/driver-loader.test.ts b/packages/db/src/cores/driver-loader.test.ts index 44e9bcc43..4f38277fd 100644 --- a/packages/db/src/cores/driver-loader.test.ts +++ b/packages/db/src/cores/driver-loader.test.ts @@ -50,8 +50,19 @@ describe('driverLoadReason', () => { expect(driverLoadReason(error)).toBe("Cannot find module 'redis'"); }); - it('keeps any other error whole', () => { + it('keeps a native-load error without the install directory', () => { const error = new Error('dlopen(/opt/ibm_db/build/Release/odbc_bindings.node, 0x0001): image not found'); - expect(driverLoadReason(error)).toBe(error.message); + expect(driverLoadReason(error)).toBe('dlopen(odbc_bindings.node, 0x0001): image not found'); + }); + + it('does not quote the server home directory from an ENOENT after a driver update', () => { + const error = new Error( + "ENOENT: no such file or directory, open '/Users/hiphan/Documents/GitHub/foxSchema/node_modules/@clickhouse/client/dist/index.js'" + ); + expect(driverLoadReason(error)).toBe( + "ENOENT: no such file or directory, open '@clickhouse/client/dist/index.js'" + ); + expect(driverLoadReason(error)).not.toMatch(/\/Users\//); + expect(driverLoadReason(error)).not.toContain('foxSchema'); }); }); diff --git a/packages/db/src/cores/driver-loader.ts b/packages/db/src/cores/driver-loader.ts index 00f0e0a11..07635dd46 100644 --- a/packages/db/src/cores/driver-loader.ts +++ b/packages/db/src/cores/driver-loader.ts @@ -30,15 +30,39 @@ export interface RequireDriverOptions { unwrapDefault?: boolean; } +const QUOTED_ABS_PATH = /(['"])((?:\/|[A-Za-z]:[\\/])(?:(?!\1).)+)\1/g; +const UNQUOTED_ABS_PATH = /(?:^|[\s,(=])((?:\/|[A-Za-z]:[\\/])[^\s'")\]]+)/g; + +/** Keep `pkg/file.js` from a `node_modules` path; otherwise only the filename. */ +function publicInstallPath(abs: string): string { + const normalized = abs.replaceAll('\\', '/'); + const marker = '/node_modules/'; + const at = normalized.lastIndexOf(marker); + if (at >= 0) return normalized.slice(at + marker.length); + const base = normalized.split('/').pop(); + return base && base !== normalized ? base : ''; +} + +/** + * Drop install-layout paths from an error shown in the browser. An ENOENT + * after a driver update used to quote + * `/Users/…/foxSchema/node_modules/@clickhouse/client/dist/index.js`. + */ +export function redactInstallPaths(text: string): string { + const quoted = text.replace(QUOTED_ABS_PATH, (_m, q: string, p: string) => `${q}${publicInstallPath(p)}${q}`); + return quoted.replace(UNQUOTED_ABS_PATH, (full, p: string) => full.slice(0, full.length - p.length) + publicInstallPath(p)); +} + /** * Why a driver would not load, fit to show a user. Node's "Cannot find * module" error ends with a Require stack: the absolute paths of the server's - * own files. That says nothing about the fix and shows the server's folder - * layout, so it is cut. Any other error (a native build that will not load) - * is kept whole: its detail is what diagnoses it. + * own files. ENOENT / dlopen quote those paths in the message itself. Both + * leak the install layout, so they are stripped; the package-relative file + * (or the filename) is kept so the failure is still diagnosable. */ export function driverLoadReason(error: unknown): string { - return errorMessage(error).split(/\r?\n\s*Require stack:/)[0]!.trim(); + const head = errorMessage(error).split(/\r?\n\s*Require stack:/)[0]!.trim(); + return redactInstallPaths(head); } /** Load `packageName` for `label` (a dialect name), or throw an install hint. */ diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 035ecafc3..b794742cb 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -25,7 +25,8 @@ export { ConnectionFactory } from './cores/connection-factory.js'; // cores/open-database.ts for why this sits alongside the factory rather than // replacing it. export { openDatabase, queryOnce, type OpenDatabase } from './cores/open-database.js'; -export { DriverDetector } from './cores/driver-detector.js'; +export { DriverDetector, readInstalledPackageVersion } from './cores/driver-detector.js'; +export { driverLoadReason, redactInstallPaths } from './cores/driver-loader.js'; export { assertSafeIdentifier } from './cores/sql-identifier.js'; export { BoundedPoolCache, diff --git a/packages/server/src/api/http-contract.test.ts b/packages/server/src/api/http-contract.test.ts index 6b3b1244b..cfaff6659 100644 --- a/packages/server/src/api/http-contract.test.ts +++ b/packages/server/src/api/http-contract.test.ts @@ -164,6 +164,10 @@ const ROUTES: RouteExpectation[] = [ { method: 'POST', path: '/api/signup', status: 400 }, { method: 'POST', path: '/api/signup/skip', status: 200 }, { method: 'GET', path: '/api/signup/state', status: 200 }, + { method: 'GET', path: '/api/system/drivers', status: 200 }, + { method: 'POST', path: '/api/system/drivers/install', status: 400 }, + { method: 'POST', path: '/api/system/drivers/remove', status: 400 }, + { method: 'POST', path: '/api/system/drivers/update', status: 400 }, { method: 'GET', path: '/api/system/services', status: 200 }, { method: 'POST', path: '/api/sql/code-cell', status: 400 }, { method: 'POST', path: '/api/sql/code-cell/transpile', status: 400 }, diff --git a/packages/server/src/features/connections/connection-tools.routes.ts b/packages/server/src/features/connections/connection-tools.routes.ts index 21849c8f5..1bb1cc519 100644 --- a/packages/server/src/features/connections/connection-tools.routes.ts +++ b/packages/server/src/features/connections/connection-tools.routes.ts @@ -56,47 +56,59 @@ export function createConnectionToolRoutes(deps: { const { installAndVerifyDriver, driverInstallHints, + npmErrorExcerpt, + publicNpmCommand, } = await import('../../internal/driver-install'); const result = await installAndVerifyDriver(packageName, versionPin); + const tryCmd = publicNpmCommand(result.manualCommand); + const failed = { extra: { success: false } }; + if (result.code === null) { // npm never started (not on PATH, blocked by policy). The spawn error // is the only useful detail; without this it was reported as "install // finished but the driver failed to load", which sends the user off // debugging the driver instead of their PATH. - const detail = (result.stderr || result.stdout).trim().slice(-2000); - sendError(res, 'failed', `Could not run npm for ${packageName}${detail ? `: ${detail}` : ''}. ` + - `Try it yourself: ${result.manualCommand}. ${driverInstallHints(packageName)}`, { - extra: { success: false, stderr: result.stderr, cwd: result.cwd }, - }); + const detail = npmErrorExcerpt(result.stderr, result.stdout); + sendError( + res, + 'failed', + `Could not run npm for ${packageName}${detail ? `: ${detail}` : ''}. ` + + `Try it yourself: ${tryCmd}. ${driverInstallHints(packageName)}`, + failed + ); return; } if (result.code !== 0) { - const detail = (result.stderr || result.stdout).trim().slice(-2000); - sendError(res, 'failed', `npm install ${packageName} failed (exit ${result.code})${detail ? `: ${detail}` : ''}. ` + - `Try: ${result.manualCommand}. ${driverInstallHints(packageName)}`, { - extra: { success: false, stderr: result.stderr, cwd: result.cwd }, - }); + const detail = npmErrorExcerpt(result.stderr, result.stdout); + sendError( + res, + 'failed', + `npm install ${packageName} failed (exit ${result.code})${detail ? `: ${detail}` : ''}. ` + + `Try: ${tryCmd}. ${driverInstallHints(packageName)}`, + failed + ); return; } if (!result.ok) { // npm exited 0 but driver still does not load (scripts skipped / wrong arch). - sendError(res, 'failed', `Install finished but ${packageName} still failed to load` + + sendError( + res, + 'failed', + `Install finished but ${packageName} still failed to load` + (result.error ? `: ${result.error}` : '') + - `. Try: ${result.manualCommand}. ${driverInstallHints(packageName)}` + - ` Then restart Fox Schema (\`foxschema stop && foxschema\`).`, { - extra: { success: false, stderr: result.stderr, cwd: result.cwd }, - }); + `. Try: ${tryCmd}. ${driverInstallHints(packageName)}` + + ` Then restart Fox Schema (\`foxschema stop && foxschema\`).`, + failed + ); return; } res.send({ success: true, - stdout: result.stdout, - cwd: result.cwd, hint: 'If the driver still shows missing, restart Fox Schema so the process reloads native bindings.', }); } catch (error: unknown) { diff --git a/packages/server/src/features/system/drivers.service.test.ts b/packages/server/src/features/system/drivers.service.test.ts new file mode 100644 index 000000000..5d824bdcf --- /dev/null +++ b/packages/server/src/features/system/drivers.service.test.ts @@ -0,0 +1,145 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + */ +import { afterEach, describe, expect, it } from 'vitest'; +import type { DriverInfo } from '@foxschema/sql'; +import { + clearDriverLatestCache, + fetchNpmLatest, + groupDriverInventory, + isBundledDriverPackage, + knownDriverPackage, + listDriverInventory, + removeDriverPackage, +} from './drivers.service'; + +afterEach(() => clearDriverLatestCache()); + +const info = (over: Partial & Pick): DriverInfo => ({ + installed: true, + ...over, +}); + +describe('knownDriverPackage', () => { + it('accepts a package the adapter registry names, and nothing else', () => { + expect(knownDriverPackage('mongodb')).toBe('mongodb'); + expect(knownDriverPackage(' pg ')).toBe('pg'); + expect(knownDriverPackage('')).toBeNull(); + expect(knownDriverPackage('evil')).toBeNull(); + expect(knownDriverPackage({ packageName: 'pg' })).toBeNull(); + }); +}); + +describe('groupDriverInventory', () => { + it('collapses dialects that share one npm package into one row', () => { + const rows = groupDriverInventory([ + info({ provider: 'postgres', packageName: 'pg', version: '8.23.1' }), + info({ provider: 'cockroachdb', packageName: 'pg', version: '8.23.1' }), + info({ provider: 'yugabytedb', packageName: 'pg', version: '8.23.1' }), + info({ provider: 'redshift', packageName: 'pg', version: '8.23.1' }), + info({ provider: 'mongodb', packageName: 'mongodb', installed: false, installCommand: 'npm install mongodb' }), + ]); + const pg = rows.find((row) => row.packageName === 'pg'); + expect(pg?.dialects).toEqual(['postgres', 'cockroachdb', 'yugabytedb', 'redshift']); + expect(pg?.installed).toBe(true); + expect(pg?.version).toBe('8.23.1'); + expect(pg?.removable).toBe(true); + expect(pg?.latest).toBeUndefined(); + const mongo = rows.find((row) => row.packageName === 'mongodb'); + expect(mongo?.installed).toBe(false); + expect(mongo?.removable).toBe(false); + }); + + it('marks a newer registry version as an update, and bundled packages not removable', () => { + const rows = groupDriverInventory( + [ + info({ provider: 'postgres', packageName: 'pg', version: '8.0.0' }), + info({ provider: 'sqlite', packageName: 'better-sqlite3', version: '13.0.3' }), + ], + { + isBundled: (name) => name === 'better-sqlite3', + latestByPackage: { pg: '8.23.1', 'better-sqlite3': '13.0.3' }, + } + ); + const pg = rows.find((row) => row.packageName === 'pg')!; + expect(pg.updateAvailable).toBe(true); + expect(pg.latest).toBe('8.23.1'); + expect(pg.removable).toBe(true); + const sqlite = rows.find((row) => row.packageName === 'better-sqlite3')!; + expect(sqlite.updateAvailable).toBe(false); + expect(sqlite.bundled).toBe(true); + expect(sqlite.removable).toBe(false); + }); +}); + +describe('isBundledDriverPackage', () => { + it('never lets better-sqlite3 be removed — it is the metadata store', () => { + expect(isBundledDriverPackage('better-sqlite3')).toBe(true); + expect(isBundledDriverPackage('mongodb')).toBe(false); + expect(isBundledDriverPackage('pg')).toBe(false); + }); +}); + +describe('removeDriverPackage', () => { + it('refuses to uninstall the metadata SQLite driver without touching npm', async () => { + const result = await removeDriverPackage('better-sqlite3'); + expect(result).toMatchObject({ ok: false, code: 'invalid_input' }); + if (result.ok) return; + expect(result.message).toMatch(/cannot be removed/); + }); +}); + +describe('listDriverInventory', () => { + it('lists each adapter package once, with the real pg driver installed here', async () => { + const report = await listDriverInventory(); + expect(report.latestChecked).toBe(false); + const names = report.drivers.map((row) => row.packageName); + expect(new Set(names).size).toBe(names.length); + expect(names).toContain('pg'); + expect(names).toContain('mongodb'); + const pg = report.drivers.find((row) => row.packageName === 'pg'); + expect(pg?.installed).toBe(true); + expect(pg?.version).toMatch(/^\d+\.\d+/); + expect(pg?.latest).toBeUndefined(); + expect(pg?.dialects).toEqual(expect.arrayContaining(['postgres', 'cockroachdb'])); + }); + + it('coalesces overlapping registry fetches for the same package', async () => { + let calls = 0; + const fetchImpl = (async () => { + calls += 1; + await new Promise((resolve) => setTimeout(resolve, 30)); + return new Response(JSON.stringify({ version: '1.2.3' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof fetch; + const [a, b] = await Promise.all([fetchNpmLatest('pg', fetchImpl), fetchNpmLatest('pg', fetchImpl)]); + expect(a).toBe('1.2.3'); + expect(b).toBe('1.2.3'); + expect(calls).toBe(1); + }); + + it('fills latest only when asked, from the injected registry', async () => { + const report = await listDriverInventory( + { latest: true }, + { + detectAll: () => [ + info({ provider: 'mongodb', packageName: 'mongodb', version: '6.0.0' }), + info({ provider: 'redis', packageName: 'redis', installed: false }), + ], + isBundled: () => false, + fetchLatest: async (pkg) => (pkg === 'mongodb' ? '6.9.0' : null), + } + ); + expect(report.latestChecked).toBe(true); + const mongo = report.drivers.find((row) => row.packageName === 'mongodb')!; + expect(mongo.latest).toBe('6.9.0'); + expect(mongo.updateAvailable).toBe(true); + const redis = report.drivers.find((row) => row.packageName === 'redis')!; + expect(redis.latest).toBeNull(); + expect(redis.updateAvailable).toBe(false); + }); +}); diff --git a/packages/server/src/features/system/drivers.service.ts b/packages/server/src/features/system/drivers.service.ts new file mode 100644 index 000000000..335eeea3d --- /dev/null +++ b/packages/server/src/features/system/drivers.service.ts @@ -0,0 +1,265 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * Inventory of dialect driver packages for the Home Drivers card: one row per + * npm package (pg covers Postgres / Cockroach / Yugabyte / Redshift), optional + * registry version check, and install / update / remove through the same npm + * target as POST /driver/install. + */ +import { ADAPTERS, DriverDetector, IBM_DB_VERSION, getProviderSettings, type DriverInfo } from '@foxschema/db'; +import type { DriverInventoryItem, DriverInventoryReport, ErrorCode } from '@foxschema/shared'; +import { isNewer } from '../../internal/updates.service'; +import { + driverInstallHints, + installAndVerifyDriver, + npmErrorExcerpt, + publicNpmCommand, + removeAndVerifyDriver, + type NpmInstallResult, + type VerifyDriverResult, +} from '../../internal/driver-install'; + +const KNOWN_PACKAGES = new Set(Object.values(ADAPTERS).map((adapter) => adapter.packageName)); + +const LATEST_TTL_MS = 30 * 60 * 1000; +const LATEST_TIMEOUT_MS = 4_000; + +type LatestCacheEntry = { version: string | null; at: number }; +const latestCache = new Map(); +const latestInflight = new Map>(); + +export function clearDriverLatestCache(): void { + latestCache.clear(); + latestInflight.clear(); +} + +export function knownDriverPackage(raw: unknown): string | null { + if (typeof raw !== 'string') return null; + const name = raw.trim(); + return KNOWN_PACKAGES.has(name) ? name : null; +} + +function dialectLabel(dialect: string): string { + try { + return getProviderSettings(dialect).label; + } catch { + // Unknown dialect id — keep the wire value rather than dropping the row. + return dialect; + } +} + +export function isBundledDriverPackage(packageName: string): boolean { + return packageName === 'better-sqlite3'; +} + +function registryLatestUrl(packageName: string): string { + const encoded = packageName.startsWith('@') + ? `@${encodeURIComponent(packageName.slice(1))}` + : encodeURIComponent(packageName); + return `https://registry.npmjs.org/${encoded}/latest`; +} + +export async function fetchNpmLatest( + packageName: string, + fetchImpl: typeof fetch = fetch +): Promise { + const hit = latestCache.get(packageName); + if (hit && Date.now() - hit.at < LATEST_TTL_MS) return hit.version; + const pending = latestInflight.get(packageName); + if (pending) return pending; + const job = (async () => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), LATEST_TIMEOUT_MS); + try { + const res = await fetchImpl(registryLatestUrl(packageName), { + headers: { Accept: 'application/json', 'User-Agent': 'FoxSchema' }, + signal: controller.signal, + }); + if (!res.ok) return null; + const data = (await res.json()) as { version?: string }; + const version = data.version?.trim() ? data.version.trim() : null; + latestCache.set(packageName, { version, at: Date.now() }); + return version; + } catch { + // abort / network / non-JSON — leave uncached so the next Check can retry + return null; + } finally { + clearTimeout(timer); + } + })().finally(() => latestInflight.delete(packageName)); + latestInflight.set(packageName, job); + return job; +} + +export function groupDriverInventory( + infos: DriverInfo[], + opts: { + isBundled?: (packageName: string) => boolean; + latestByPackage?: Record; + } = {} +): DriverInventoryItem[] { + const bundledOf = opts.isBundled ?? isBundledDriverPackage; + const byPkg = new Map(); + for (const info of infos) { + const list = byPkg.get(info.packageName) ?? []; + list.push(info); + byPkg.set(info.packageName, list); + } + const items: DriverInventoryItem[] = []; + for (const [packageName, rows] of byPkg) { + const first = rows[0]; + if (!first) continue; + const dialects = [...new Set(rows.map((row) => row.provider))]; + const representative = rows.find((row) => row.installed) ?? first; + const installed = rows.some((row) => row.installed); + const version = rows.find((row) => row.version)?.version; + const latestChecked = opts.latestByPackage !== undefined && packageName in opts.latestByPackage; + const latest = latestChecked ? opts.latestByPackage?.[packageName] : undefined; + const updateAvailable = + installed && typeof latest === 'string' && typeof version === 'string' && isNewer(latest, version); + const bundled = bundledOf(packageName); + items.push({ + packageName, + dialects, + labels: dialects.map(dialectLabel), + installed, + version, + ...(latestChecked ? { latest } : {}), + updateAvailable, + bundled, + removable: installed && !bundled, + installCommand: representative.installCommand, + error: installed ? undefined : representative.error, + }); + } + return items.sort((a, b) => a.packageName.localeCompare(b.packageName)); +} + +export type DriverInventoryDeps = { + detectAll?: () => DriverInfo[]; + fetchLatest?: (packageName: string) => Promise; + isBundled?: (packageName: string) => boolean; +}; + +export async function listDriverInventory( + opts: { latest?: boolean } = {}, + deps: DriverInventoryDeps = {} +): Promise { + const infos = (deps.detectAll ?? (() => DriverDetector.detectAll()))(); + const packages = [...new Set(infos.map((info) => info.packageName))]; + let latestByPackage: Record | undefined; + if (opts.latest) { + const fetchLatest = deps.fetchLatest ?? fetchNpmLatest; + const pairs = await Promise.all( + packages.map(async (pkg) => { + // ibm_db is pinned; skip the registry — "latest" is the version we install. + if (pkg === 'ibm_db') return [pkg, IBM_DB_VERSION] as const; + return [pkg, await fetchLatest(pkg)] as const; + }) + ); + latestByPackage = Object.fromEntries(pairs); + } + return { + drivers: groupDriverInventory(infos, { + isBundled: deps.isBundled, + latestByPackage, + }), + checkedAt: Date.now(), + latestChecked: !!opts.latest, + }; +} + +export type DriverActionResult = + | { ok: true; hint?: string } + | { ok: false; code: ErrorCode; message: string; extra?: Record }; + +const FAILED_DRIVER_EXTRA = { success: false }; + +function fromNpmResult( + packageName: string, + result: NpmInstallResult & Partial & { stillInstalled?: boolean }, + verb: 'install' | 'update' | 'uninstall' +): DriverActionResult { + const hints = verb === 'uninstall' ? '' : ` ${driverInstallHints(packageName)}`; + const tryCmd = publicNpmCommand(result.manualCommand); + if (result.code === null) { + const detail = npmErrorExcerpt(result.stderr, result.stdout); + return { + ok: false, + code: 'failed', + message: + `Could not run npm for ${packageName}${detail ? `: ${detail}` : ''}. ` + + `Try it yourself: ${tryCmd}.${hints}`, + extra: FAILED_DRIVER_EXTRA, + }; + } + if (result.code !== 0) { + const detail = npmErrorExcerpt(result.stderr, result.stdout); + const npmVerb = verb === 'uninstall' ? 'uninstall' : verb; + return { + ok: false, + code: 'failed', + message: + `npm ${npmVerb} ${packageName} failed (exit ${result.code})${detail ? `: ${detail}` : ''}. ` + + `Try: ${tryCmd}.${hints}`, + extra: FAILED_DRIVER_EXTRA, + }; + } + if (verb === 'uninstall') { + if (result.stillInstalled) { + return { + ok: false, + code: 'failed', + message: + `${packageName} is still loadable after uninstall — another package in this install still lists it. ` + + `Restart Fox Schema after removing it from every workspace.`, + extra: FAILED_DRIVER_EXTRA, + }; + } + return { ok: true, hint: 'Restart Fox Schema if a connection still sees the old driver.' }; + } + if (!result.ok) { + return { + ok: false, + code: 'failed', + message: + `${verb === 'update' ? 'Update' : 'Install'} finished but ${packageName} still failed to load` + + (result.error ? `: ${result.error}` : '') + + `. Try: ${tryCmd}.${hints}` + + ` Then restart Fox Schema (\`foxschema stop && foxschema\`).`, + extra: FAILED_DRIVER_EXTRA, + }; + } + return { + ok: true, + hint: 'If the driver still shows missing, restart Fox Schema so the process reloads native bindings.', + }; +} + +async function versionPin(packageName: string, latest: boolean): Promise { + if (packageName === 'ibm_db') return IBM_DB_VERSION; + const resolved = await fetchNpmLatest(packageName); + if (resolved) return resolved; + return latest ? 'latest' : undefined; +} + +export async function installDriverPackage(packageName: string): Promise { + return fromNpmResult(packageName, await installAndVerifyDriver(packageName, await versionPin(packageName, false)), 'install'); +} + +export async function updateDriverPackage(packageName: string): Promise { + return fromNpmResult(packageName, await installAndVerifyDriver(packageName, await versionPin(packageName, true)), 'update'); +} + +export async function removeDriverPackage(packageName: string): Promise { + if (isBundledDriverPackage(packageName)) { + return { + ok: false, + code: 'invalid_input', + message: `The ${packageName} driver ships with Fox Schema and cannot be removed.`, + }; + } + return fromNpmResult(packageName, await removeAndVerifyDriver(packageName), 'uninstall'); +} diff --git a/packages/server/src/features/system/index.ts b/packages/server/src/features/system/index.ts index d88305757..6c8f8a95b 100644 --- a/packages/server/src/features/system/index.ts +++ b/packages/server/src/features/system/index.ts @@ -2,7 +2,8 @@ * Fox Schema (foxschema) * Copyright 2024-2026 Huy Phan * SPDX-License-Identifier: Apache-2.0 - * The system feature: liveness, version and updates, app info, activity. + * The system feature: liveness, version and updates, app info, activity, + * Home Services and Drivers. */ import type { ServerFeatureModule } from '../../platform/feature-module'; import { createSystemPublicRoutes, createSystemRoutes } from './system.routes'; diff --git a/packages/server/src/features/system/system.routes.test.ts b/packages/server/src/features/system/system.routes.test.ts index 38dcf5992..3b173ca6c 100644 --- a/packages/server/src/features/system/system.routes.test.ts +++ b/packages/server/src/features/system/system.routes.test.ts @@ -7,7 +7,8 @@ * and a failure to record never fails the answer. */ import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { AppRequest } from '../../platform/http/types'; +import type { Permission } from '@foxschema/shared'; +import type { AppRequest, AuthedRequest, Middleware } from '../../platform/http/types'; import { fakeReply, routeHandler } from '../../platform/http/route-doubles'; import type { AppSettingsStore } from '../../platform/settings/app-settings.service'; import { createSystemRoutes } from './system.routes'; @@ -44,3 +45,52 @@ describe('GET /app-info', () => { expect(call.body().db).toBeTruthy(); }); }); + +const systemRoutes = createSystemRoutes().flatten(); +const systemRoute = (method: string, path: string) => { + const found = systemRoutes.find((r) => r.method === method && r.path === path); + if (!found) throw new Error(`${method} ${path} is not registered`); + return found; +}; + +async function guardsAnswer(middlewares: unknown[], caller: Partial) { + const { reply, sent } = fakeReply(); + for (const middleware of middlewares as Middleware[]) { + let passed = false; + await middleware(caller as AuthedRequest, reply, () => (passed = true)); + if (!passed) return sent; + } + return 'passed' as const; +} + +const caller = (appRole: 'viewer' | 'editor' | 'owner' | 'admin', permissions: Permission[] = []) => ({ + userId: `${appRole}-1`, + appRole, + permissions: new Set(permissions), +}); + +describe('driver inventory routes', () => { + it('leaves listing drivers open to everyone signed in', async () => { + expect(await guardsAnswer(systemRoute('GET', '/system/drivers').middlewares, caller('viewer'))).toBe('passed'); + }); + + it.each(['/system/drivers/install', '/system/drivers/update', '/system/drivers/remove'] as const)( + 'refuses %s without admin.drivers', + async (path) => { + const sent = await guardsAnswer(systemRoute('POST', path).middlewares, caller('owner', ['editor.run'])); + expect(sent).not.toBe('passed'); + if (sent === 'passed') return; + expect(sent.status).toBe(403); + expect(sent.body.missing).toEqual(['admin.drivers']); + } + ); + + it('lets an admin install, update and remove', async () => { + for (const path of ['/system/drivers/install', '/system/drivers/update', '/system/drivers/remove']) { + expect(await guardsAnswer(systemRoute('POST', path).middlewares, caller('admin'))).toBe('passed'); + expect(await guardsAnswer(systemRoute('POST', path).middlewares, caller('editor', ['admin.drivers']))).toBe( + 'passed' + ); + } + }); +}); diff --git a/packages/server/src/features/system/system.routes.ts b/packages/server/src/features/system/system.routes.ts index 5ea1dfa28..76c7db868 100644 --- a/packages/server/src/features/system/system.routes.ts +++ b/packages/server/src/features/system/system.routes.ts @@ -14,8 +14,17 @@ import { AppSettingsStore } from '../../platform/settings/app-settings.service'; import { targetLocks } from '../../platform/guards/target-lock'; import { isLocalSingleUser } from '../../platform/runtime/deployment'; import { sendError } from '../../platform/http/respond'; +import { requirePermissions } from '../../platform/authorization/rbac.guard'; import { WorkflowSettingsService } from '../workflow'; import { readServiceStatus, type ServiceStatusDeps } from './service-status'; +import { + installDriverPackage, + knownDriverPackage, + listDriverInventory, + removeDriverPackage, + updateDriverPackage, + type DriverActionResult, +} from './drivers.service'; import { keySchemeInfo } from '../../platform/crypto/crypto'; import { getMetadataDbConfig, SUPPORTED_ENGINES, type DbEngine } from '../../database/config'; import { createMetadataStore } from '../../database/stores/registry'; @@ -59,6 +68,23 @@ export function createSystemRoutes( res.send(await readServiceStatus(serviceStatusDeps)); }); + // Home → Drivers: one row per npm package. `?latest=1` asks the registry. + router.get('/system/drivers', async (req: AppRequest, res: FastifyReply) => { + const raw = (req.query as { latest?: unknown } | undefined)?.latest; + const latest = raw === '1' || raw === 'true' || raw === true; + res.send(await listDriverInventory({ latest })); + }); + + router.post('/system/drivers/install', requirePermissions('admin.drivers'), async (req: AppRequest, res: FastifyReply) => { + await runDriverAction(req, res, installDriverPackage); + }); + router.post('/system/drivers/update', requirePermissions('admin.drivers'), async (req: AppRequest, res: FastifyReply) => { + await runDriverAction(req, res, updateDriverPackage); + }); + router.post('/system/drivers/remove', requirePermissions('admin.drivers'), async (req: AppRequest, res: FastifyReply) => { + await runDriverAction(req, res, removeDriverPackage); + }); + // In-app update check — compares the running version against npm (default). router.get('/updates/check', async (_req: AppRequest, res: FastifyReply) => { res.send(await checkForUpdate()); @@ -173,6 +199,31 @@ export function createSystemRoutes( return router; } +function sendDriverAction(res: FastifyReply, result: DriverActionResult): void { + if (!result.ok) { + sendError(res, result.code, result.message, { extra: result.extra }); + return; + } + res.send({ success: true, hint: result.hint }); +} + +/** ibm_db clidriver download + native build routinely exceeds the 120s default. */ +const DRIVER_NPM_REQUEST_TIMEOUT_MS = 10 * 60 * 1000; + +async function runDriverAction( + req: AppRequest, + res: FastifyReply, + run: (packageName: string) => Promise +): Promise { + req.raw.setTimeout(DRIVER_NPM_REQUEST_TIMEOUT_MS); + const pkg = knownDriverPackage((req.body as { packageName?: unknown } | null)?.packageName); + if (!pkg) { + sendError(res, 'invalid_input', 'A known driver package name is required.'); + return; + } + sendDriverAction(res, await run(pkg)); +} + function defaultServiceStatusDeps(): ServiceStatusDeps { const settings = new WorkflowSettingsService(); return { diff --git a/packages/server/src/internal/driver-install.test.ts b/packages/server/src/internal/driver-install.test.ts index bd2e3e477..5b05978dd 100644 --- a/packages/server/src/internal/driver-install.test.ts +++ b/packages/server/src/internal/driver-install.test.ts @@ -5,7 +5,17 @@ import { driverInstallHints, ibmDbPackageDir, isCacheKeyForPackage, + isNpmRenameStagingName, + isTransientNpmFailure, + npmErrorExcerpt, + npmUninstallArgs, purgePackageRequireCache, + resolveDriverInstallTarget, + DRIVER_WORKSPACE_PACKAGE, + driverInstallPkgSets, + formatDriverInstallCommand, + peerRangeAllowsVersion, + publicNpmCommand, } from './driver-install'; describe('driverInstallHints — advice must match the driver', () => { @@ -28,6 +38,121 @@ describe('driverInstallHints — advice must match the driver', () => { }); }); +describe('DRIVER_WORKSPACE_PACKAGE', () => { + it('installs every driver where @foxschema/db can require it', () => { + // Installing mongodb into @foxschema/web nested it under apps/web/node_modules, + // and the loader in packages/db still threw Cannot find module 'mongodb'. + expect(DRIVER_WORKSPACE_PACKAGE).toBe('@foxschema/db'); + const target = resolveDriverInstallTarget(import.meta.url); + expect(target.mode).toBe('workspace'); + if (target.mode === 'workspace') expect(target.workspacePkg).toBe('@foxschema/db'); + }); +}); + +describe('npmUninstallArgs', () => { + it('uninstalls from the same @foxschema/db workspace install uses', () => { + const { args, manualCommand } = npmUninstallArgs('mongodb'); + expect(args).toEqual(['uninstall', 'mongodb', '-w', '@foxschema/db']); + expect(manualCommand).toBe('npm uninstall mongodb -w @foxschema/db'); + }); + + it('can target another workspace without rebuilding argv by hand', () => { + expect(npmUninstallArgs('ibm_db', undefined, '@foxschema/web').args).toEqual([ + 'uninstall', + 'ibm_db', + '-w', + '@foxschema/web', + ]); + }); +}); + +describe('npm error text', () => { + it('recognizes npm rename staging dirs, not .vite or .bin', () => { + expect(isNpmRenameStagingName('.follow-redirects-4tvSnoXN')).toBe(true); + expect(isNpmRenameStagingName('.ibm_db-5V4mzFVg')).toBe(true); + expect(isNpmRenameStagingName('.bin')).toBe(false); + expect(isNpmRenameStagingName('.vite')).toBe(false); + expect(isNpmRenameStagingName('.vite-vwHNAlXN')).toBe(false); + expect(isNpmRenameStagingName('.package-lock.json')).toBe(false); + }); + + it('treats ENOTEMPTY as a retryable npm race', () => { + expect(isTransientNpmFailure('npm error code ENOTEMPTY\nnpm error syscall rename')).toBe(true); + expect(isTransientNpmFailure('npm error code ETARGET\nNo matching version')).toBe(false); + }); + + it('drops npm warn / EBADENGINE so the UI shows the real error', () => { + const excerpt = npmErrorExcerpt( + [ + 'npm warn Unknown env config "devdir".', + 'npm warn EBADENGINE Unsupported engine {', + 'npm error code ENOTEMPTY', + "npm error ENOTEMPTY: directory not empty, rename 'follow-redirects'", + ].join('\n') + ); + expect(excerpt).toContain('ENOTEMPTY'); + expect(excerpt).not.toMatch(/devdir/); + expect(excerpt).not.toMatch(/EBADENGINE/); + }); + + it('does not quote the install directory from an npm path error', () => { + const excerpt = npmErrorExcerpt( + "npm error path /Users/hiphan/Documents/GitHub/foxSchema/node_modules/@clickhouse/client" + ); + expect(excerpt).toContain('npm error path @clickhouse/client'); + expect(excerpt).not.toMatch(/\/Users\//); + }); + + it('replaces --prefix with a placeholder so the copy-paste command has no install directory', () => { + const cmd = publicNpmCommand( + 'npm uninstall ibm_db --prefix /Users/hiphan/Documents/GitHub/foxSchema' + ); + expect(cmd).toBe('npm uninstall ibm_db --prefix '); + expect(cmd).not.toContain(process.cwd()); + }); +}); + +describe('install args', () => { + it('writes the package onto @foxschema/db dependencies so a peer is actually fetched', () => { + // `npm install redis --save -w @foxschema/db` is a no-op when redis is + // already an optional peer: exit 0, nothing in node_modules. + const cmd = resolveDriverInstallTarget(import.meta.url).manualCommand('redis'); + expect(cmd).toContain('npm pkg set dependencies.redis=latest -w @foxschema/db'); + expect(cmd).toContain('npm install --foreground-scripts'); + expect(cmd).not.toMatch(/npm install --foreground-scripts -w /); + }); + + it('pins mssql on @foxschema/web as well so a web pin cannot keep the old version', () => { + const root = resolveDriverInstallTarget(import.meta.url).cwd; + const sets = driverInstallPkgSets(root, 'mssql', '12.7.14'); + const web = sets.find((row) => row.workspace === '@foxschema/web'); + expect(web?.assignments).toContain('dependencies.mssql=12.7.14'); + expect(formatDriverInstallCommand(sets)).toContain('-w @foxschema/web'); + }); + + it('pins a scoped DuckDB name with bracket keys so npm pkg set keeps the package', () => { + const root = resolveDriverInstallTarget(import.meta.url).cwd; + const sets = driverInstallPkgSets(root, '@duckdb/node-api', '2.0.0-r.1'); + expect(sets.flatMap((row) => row.assignments).join(' ')).toMatch( + /\[@duckdb\/node-api\]=2\.0\.0-r\.1/ + ); + }); +}); + +describe('peerRangeAllowsVersion', () => { + it('lets >=6 keep covering mongodb 7.8.0', () => { + expect(peerRangeAllowsVersion('>=6', '7.8.0')).toBe(true); + }); + + it('does not let ^1.5.4-r.1 cover DuckDB 2.0.0-r.1', () => { + expect(peerRangeAllowsVersion('^1.5.4-r.1', '2.0.0-r.1')).toBe(false); + }); + + it('does not let an exact ClickHouse pin cover a newer patch', () => { + expect(peerRangeAllowsVersion('1.24.1', '1.25.0')).toBe(false); + }); +}); + describe('DRIVER_NPM_INSTALL_ENV', () => { it('forces npm 11 allow-scripts on so ibm_db postinstall can run', () => { expect(DRIVER_NPM_INSTALL_ENV.npm_config_ignore_scripts).toBe('false'); @@ -36,32 +161,32 @@ describe('DRIVER_NPM_INSTALL_ENV', () => { }); describe('ibmDbPackageDir', () => { - it('resolves the installed ibm_db package', () => { + it('is the ibm_db package directory when that driver is installed', () => { const dir = ibmDbPackageDir(); - expect(dir).toBeTruthy(); + if (dir === null) return; expect(dir).toMatch(/ibm_db$/); }); }); describe('isCacheKeyForPackage — match a path segment, not a substring', () => { it('matches the package itself', () => { - expect(isCacheKeyForPackage('/repo/node_features/pg/lib/index.js', 'pg')).toBe(true); + expect(isCacheKeyForPackage('/repo/node_modules/pg/lib/index.js', 'pg')).toBe(true); }); it('does not match packages whose name merely starts the same', () => { // Purging `pg` by substring also evicted pg-pool and pg-protocol, // re-instantiating module state unrelated to the install. - expect(isCacheKeyForPackage('/repo/node_features/pg-pool/index.js', 'pg')).toBe(false); - expect(isCacheKeyForPackage('/repo/node_features/pg-protocol/dist/x.js', 'pg')).toBe(false); + expect(isCacheKeyForPackage('/repo/node_modules/pg-pool/index.js', 'pg')).toBe(false); + expect(isCacheKeyForPackage('/repo/node_modules/pg-protocol/dist/x.js', 'pg')).toBe(false); }); it('does not match a path that merely contains the name', () => { - expect(isCacheKeyForPackage('/home/pg/project/node_features/mssql/x.js', 'pg')).toBe(false); + expect(isCacheKeyForPackage('/home/pg/project/node_modules/mssql/x.js', 'pg')).toBe(false); }); it('matches a nested copy', () => { expect( - isCacheKeyForPackage('/repo/node_features/foo/node_features/pg/lib/x.js', 'pg') + isCacheKeyForPackage('/repo/node_modules/foo/node_modules/pg/lib/x.js', 'pg') ).toBe(true); }); diff --git a/packages/server/src/internal/driver-install.ts b/packages/server/src/internal/driver-install.ts index c526db95f..71871cac3 100644 --- a/packages/server/src/internal/driver-install.ts +++ b/packages/server/src/internal/driver-install.ts @@ -8,31 +8,48 @@ * resolves to `/` when routes are bundled into `ui-server.js` — installs then * fail intermittently or write into the wrong tree. * - * ibm_db is required from `@foxschema/db` (DriverDetector). Installing only - * into `@foxschema/web` can leave the module under apps/web/node_modules where - * packages/db cannot resolve it (sibling package node_modules are not searched). + * Drivers are required from `@foxschema/db` (DriverDetector / requireDriver). + * Installing only into `@foxschema/web` can leave the module under + * apps/web/node_modules where packages/db cannot resolve it (sibling package + * node_modules are not searched). That is why a MongoDB/Redis Install used to + * add the package to the web app and still throw "Cannot find module". */ import { spawn } from 'node:child_process'; -import { existsSync, readFileSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync, rmSync } from 'node:fs'; import { dirname, join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; -import { hasDb2Clidriver, setupDb2ClientEnv } from '@foxschema/db'; -import { errorMessage } from '@foxschema/sql'; +import { driverLoadReason, hasDb2Clidriver, redactInstallPaths, setupDb2ClientEnv } from '@foxschema/db'; const nodeRequire = createRequire(import.meta.url); +/** Same require site as `requireDriver` in `@foxschema/db`. */ +function dbDriverRequire() { + try { + return createRequire(nodeRequire.resolve('@foxschema/db')); + } catch { + return nodeRequire; + } +} + export type DriverInstallMode = 'workspace' | 'prefix'; -export type DriverInstallTarget = { - cwd: string; - mode: DriverInstallMode; - workspacePkg?: string; - /** Human-facing npm command for copy/paste. */ - manualCommand: (spec: string) => string; - /** argv for `npm` (no binary). */ - npmArgs: (spec: string) => string[]; -}; +/** Workspace that `@foxschema/db` can require optional drivers from. */ +export const DRIVER_WORKSPACE_PACKAGE = '@foxschema/db'; + +export type DriverInstallTarget = + | { + cwd: string; + mode: 'workspace'; + workspacePkg: string; + manualCommand: (spec: string) => string; + } + | { + cwd: string; + mode: 'prefix'; + manualCommand: (spec: string) => string; + npmArgs: (spec: string) => string[]; + }; function readPkgName(dir: string): string | null { try { @@ -51,16 +68,9 @@ function isFoxschemaMonorepo(dir: string): boolean { // eslint-disable-next-line security/detect-non-literal-fs-filename -- walks up from this module's own directory looking for the workspace manifest const raw = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { name?: string; - workspaces?: string[] | { packages?: string[] }; + workspaces?: unknown; }; if (raw.name !== 'foxschema' && raw.name !== undefined && !raw.workspaces) return false; - const ws = Array.isArray(raw.workspaces) - ? raw.workspaces - : (raw.workspaces?.packages ?? []); - if (!ws.some((p) => p.includes('apps/web') || p.includes('packages/*') || p === 'apps/*')) { - // eslint-disable-next-line security/detect-non-literal-fs-filename -- walks up from this module's own directory looking for the workspace manifest - return existsSync(join(dir, 'apps/web/package.json')); - } // eslint-disable-next-line security/detect-non-literal-fs-filename -- walks up from this module's own directory looking for the workspace manifest return existsSync(join(dir, 'apps/web/package.json')); } catch { @@ -68,11 +78,6 @@ function isFoxschemaMonorepo(dir: string): boolean { } } -/** Workspace package that owns the optional dep / require site for a driver. */ -export function workspacePackageForDriver(packageName: string): string { - return packageName === 'ibm_db' ? '@foxschema/db' : '@foxschema/web'; -} - function packageNameFromSpec(spec: string): string { // ibm_db@4.0.1 → ibm_db; @scope/pkg@1 → @scope/pkg if (spec.startsWith('@')) { @@ -82,14 +87,129 @@ function packageNameFromSpec(spec: string): string { return spec.split('@')[0] || spec; } -function workspaceTarget(cwd: string, workspacePkg: string): DriverInstallTarget { +function versionFromSpec(spec: string): string { + const name = packageNameFromSpec(spec); + if (spec === name) return 'latest'; + const rest = spec.slice(name.length); + return rest.startsWith('@') && rest.length > 1 ? rest.slice(1) : 'latest'; +} + +const DRIVER_WORKSPACE_DIRS: Record = { + '@foxschema/db': 'packages/db', + '@foxschema/web': 'apps/web', +}; + +type DepField = 'dependencies' | 'optionalDependencies' | 'peerDependencies'; + +type WorkspaceManifest = Partial>>; + +function readWorkspaceManifest(root: string, workspacePkg: string): WorkspaceManifest | null { + const rel = DRIVER_WORKSPACE_DIRS[workspacePkg]; + if (!rel) return null; + try { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- workspace path is one of two fixed package folders + return JSON.parse(readFileSync(join(root, rel, 'package.json'), 'utf8')) as WorkspaceManifest; + } catch { + return null; + } +} + +function majorOf(version: string): string { + return version.match(/\d+/)?.[0] ?? ''; +} + +/** + * A peer that already includes `version` is left alone (`>=6` vs 7.8.0). + * `^1.5.4-r.1` does not include DuckDB 2.0.0-r.1 — leaving it is why Update + * wrote 2.x onto `@foxschema/web` and npm still hoisted 1.5.4-r.1. ClickHouse + * moved because its peer was the exact pin `1.24.1`. + */ +export function peerRangeAllowsVersion(range: string, version: string): boolean { + const r = range.trim(); + if (!r || r === '*' || r === 'x' || r === 'latest') return true; + if (r === version) return true; + if (r.startsWith('>=') || r.startsWith('>')) return true; + if (r.startsWith('^')) return majorOf(r.slice(1)) === majorOf(version); + if (r.startsWith('~')) { + const a = r.slice(1).replace(/^[^\d]*/, '').split('.'); + const b = version.replace(/^[^\d]*/, '').split('.'); + return a[0] === b[0] && a[1] === b[1]; + } + if (/^\d/.test(r)) return r === version; + return false; +} + +/** Bracket form is what `npm pkg set` documents for scoped names. */ +function pkgSetAssignment(field: DepField, packageName: string, version: string): string { + if (packageName.startsWith('@')) return `${field}[${packageName}]=${version}`; + return `${field}.${packageName}=${version}`; +} + +export type DriverPkgAssignment = { workspace: string; assignments: string[] }; + +/** + * Which manifests to pin, and which fields. `mssql` is a hard dep of + * `@foxschema/web` and a peer of `@foxschema/db` — writing only `-w @foxschema/db` + * leaves the web pin at 12.7.13 and the hoisted copy never moves. + */ +export function driverInstallPkgSets( + root: string, + packageName: string, + version: string +): DriverPkgAssignment[] { + const byWs = new Map(); + const add = (workspace: string, field: DepField) => { + const assignment = pkgSetAssignment(field, packageName, version); + const list = byWs.get(workspace) ?? []; + if (!list.includes(assignment)) list.push(assignment); + byWs.set(workspace, list); + }; + + let dbFetches = false; + for (const workspace of Object.keys(DRIVER_WORKSPACE_DIRS)) { + const manifest = readWorkspaceManifest(root, workspace); + if (!manifest) continue; + for (const field of ['dependencies', 'optionalDependencies', 'peerDependencies'] as const) { + const current = manifest[field]?.[packageName]; + if (current === undefined) continue; + if (field === 'peerDependencies' && peerRangeAllowsVersion(current, version)) continue; + add(workspace, field); + if ( + workspace === DRIVER_WORKSPACE_PACKAGE && + (field === 'dependencies' || field === 'optionalDependencies') + ) { + dbFetches = true; + } + } + } + if (!dbFetches) add(DRIVER_WORKSPACE_PACKAGE, 'dependencies'); + return [...byWs.entries()].map(([workspace, assignments]) => ({ workspace, assignments })); +} + +export function formatDriverInstallCommand(sets: DriverPkgAssignment[]): string { + const pkgSets = sets.map((row) => { + const args = row.assignments.map((a) => (a.includes('[') ? `'${a}'` : a)).join(' '); + return `npm pkg set ${args} -w ${row.workspace}`; + }); + return [...pkgSets, 'npm install --foreground-scripts'].join(' && '); +} + +/** Copy-paste npm text without the install directory. Spawn still uses the real cwd. */ +export function publicNpmCommand(command: string): string { + return redactInstallPaths( + command.replace(/--prefix(?:\s+|=)(?:"[^"]+"|'[^']+'|\S+)/g, '--prefix ') + ); +} + +function workspaceTarget(cwd: string): DriverInstallTarget { return { cwd, mode: 'workspace', - workspacePkg, - npmArgs: (spec) => ['install', spec, '--foreground-scripts', '-w', workspacePkg], + workspacePkg: DRIVER_WORKSPACE_PACKAGE, manualCommand: (spec) => - `npm install ${spec} --foreground-scripts -w ${workspacePkg}`, + formatDriverInstallCommand( + driverInstallPkgSets(cwd, packageNameFromSpec(spec), versionFromSpec(spec)) + ), }; } @@ -97,9 +217,9 @@ function prefixTarget(cwd: string): DriverInstallTarget { return { cwd, mode: 'prefix', - npmArgs: (spec) => ['install', spec, '--foreground-scripts', '--prefix', cwd], + npmArgs: (spec) => ['install', spec, '--save', '--foreground-scripts', '--prefix', cwd], manualCommand: (spec) => - `npm install ${spec} --foreground-scripts --prefix "${cwd}"`, + `npm install ${spec} --save --foreground-scripts --prefix `, }; } @@ -107,12 +227,8 @@ function prefixTarget(cwd: string): DriverInstallTarget { * Locate install target from the calling module URL (routes.ts or bundled * ui-server.js) and optional process.cwd() fallback. */ -export function resolveDriverInstallTarget( - fromUrl: string = import.meta.url, - packageName = 'ibm_db' -): DriverInstallTarget { +export function resolveDriverInstallTarget(fromUrl: string = import.meta.url): DriverInstallTarget { const here = dirname(fileURLToPath(fromUrl)); - const wsPkg = workspacePackageForDriver(packageName); // Candidate roots by depth, each validated by isFoxschemaMonorepo before // use — which is why the backend moving from apps/web/src/backend to @@ -121,13 +237,13 @@ export function resolveDriverInstallTarget( // and invisible to tsc, so they are checked, never trusted. const fromApiSource = resolve(here, '../../../../..'); if (isFoxschemaMonorepo(fromApiSource)) { - return workspaceTarget(fromApiSource, wsPkg); + return workspaceTarget(fromApiSource); } // packages/server/src/ → ../../../.. = monorepo root const fromModules = resolve(here, '../../../..'); if (isFoxschemaMonorepo(fromModules)) { - return workspaceTarget(fromModules, wsPkg); + return workspaceTarget(fromModules); } // Bundled ui-server: …/foxschema/dist/ui-server.js → package root is … @@ -138,7 +254,7 @@ export function resolveDriverInstallTarget( } if (isFoxschemaMonorepo(process.cwd())) { - return workspaceTarget(process.cwd(), wsPkg); + return workspaceTarget(process.cwd()); } // Global npm / Homebrew: install next to the running foxschema package if found @@ -149,13 +265,11 @@ export function resolveDriverInstallTarget( /* not resolvable */ } - if (packageName === 'ibm_db') { - try { - const dbPkg = dirname(nodeRequire.resolve('@foxschema/db/package.json')); - return prefixTarget(dbPkg); - } catch { - /* ignore */ - } + try { + const dbPkg = dirname(nodeRequire.resolve('@foxschema/db/package.json')); + return prefixTarget(dbPkg); + } catch { + /* ignore */ } return prefixTarget(process.cwd()); @@ -189,18 +303,70 @@ export const DRIVER_NPM_INSTALL_ENV = { npm_config_dangerously_allow_all_scripts: 'true', } as const; -/** Spawn npm with install scripts forced on, resolving rather than throwing. */ -function runNpm( +/** npm rename/rm races (ENOTEMPTY) when Vite or a previous npm still holds node_modules. */ +export function isTransientNpmFailure(stderr: string, stdout = ''): boolean { + return /\b(ENOTEMPTY|EEXIST|EBUSY|EPERM)\b/.test(`${stderr}\n${stdout}`); +} + +/** + * npm's rename dest when swapping a package: `.follow-redirects-4tvSnoXN`. + * A failed uninstall leaves these behind; the next npm then hits ENOTEMPTY. + * Never matches `.bin`, `.vite`, `.vitest-cache`, `.package-lock.json`. + */ +export function isNpmRenameStagingName(name: string): boolean { + if (!name.startsWith('.') || name.startsWith('.bin') || name.startsWith('.vite') || name.startsWith('.vitest')) { + return false; + } + if (name === '.package-lock.json') return false; + return /^\.[^/]+-[A-Za-z0-9_-]{6,12}$/.test(name); +} + +export function clearNpmRenameStaging(nodeModulesDir: string): void { + let names: string[] = []; + try { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- workspace node_modules of the install we just ran npm in + names = readdirSync(nodeModulesDir); + } catch { + return; + } + for (const name of names) { + if (!isNpmRenameStagingName(name)) continue; + try { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- names are filtered to npm's rename-staging pattern + rmSync(join(nodeModulesDir, name), { recursive: true, force: true }); + } catch { + // still held open; the retry may still succeed + } + } +} + +/** Drop npm warn / EBADENGINE noise so the real `npm error` is what the UI shows. */ +export function npmErrorExcerpt(stderr: string, stdout = ''): string { + const lines = `${stderr}\n${stdout}` + .split('\n') + .map((line) => line.trimEnd()) + .filter(Boolean); + const useful = lines.filter((line) => !/^npm warn /i.test(line) && !/EBADENGINE/.test(line)); + return redactInstallPaths((useful.length ? useful : lines).join('\n').trim().slice(-2000)); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function spawnCaptured( + command: string, args: string[], cwd: string, - manualCommand: string + manualCommand: string, + extra: { shell?: boolean } = {} ): Promise { return new Promise((resolvePromise) => { - const proc = spawn('npm', args, { + const proc = spawn(command, args, { cwd, stdio: 'pipe', env: { ...process.env, ...DRIVER_NPM_INSTALL_ENV }, - shell: process.platform === 'win32', + ...(extra.shell ? { shell: true } : {}), }); let stdout = ''; let stderr = ''; @@ -211,8 +377,8 @@ function runNpm( stderr += d.toString(); }); proc.on('error', (err: Error) => { - // `code: null` distinguishes "npm never started" (not on PATH) from a - // non-zero exit; the caller reports these differently. + // `code: null` distinguishes "the binary never started" (not on PATH) + // from a non-zero exit; the caller reports these differently. resolvePromise({ ok: false, code: null, @@ -229,47 +395,35 @@ function runNpm( }); } -/** Spawn `node` in a package directory (ibm_db postinstall, ignoring npm allow-scripts). */ -function runNode( +function spawnNpmOnce(args: string[], cwd: string, manualCommand: string): Promise { + return spawnCaptured('npm', args, cwd, manualCommand, { shell: process.platform === 'win32' }); +} + +/** Spawn npm with install scripts forced on, resolving rather than throwing. */ +async function runNpm( args: string[], cwd: string, manualCommand: string ): Promise { - return new Promise((resolvePromise) => { - const proc = spawn(process.execPath, args, { - cwd, - stdio: 'pipe', - env: { ...process.env, ...DRIVER_NPM_INSTALL_ENV }, - }); - let stdout = ''; - let stderr = ''; - proc.stdout?.on('data', (d: Buffer) => { - stdout += d.toString(); - }); - proc.stderr?.on('data', (d: Buffer) => { - stderr += d.toString(); - }); - proc.on('error', (err: Error) => { - resolvePromise({ - ok: false, - code: null, - stdout, - stderr: stderr || err.message, - cwd, - args, - manualCommand, - }); - }); - proc.on('close', (code) => { - resolvePromise({ ok: code === 0, code, stdout, stderr, cwd, args, manualCommand }); - }); - }); + clearNpmRenameStaging(join(cwd, 'node_modules')); + let last = await spawnNpmOnce(args, cwd, manualCommand); + for (let attempt = 0; attempt < 3 && !last.ok && isTransientNpmFailure(last.stderr, last.stdout); attempt++) { + clearNpmRenameStaging(join(cwd, 'node_modules')); + await sleep(500 * 2 ** attempt); + last = await spawnNpmOnce(args, cwd, manualCommand); + } + return last; +} + +/** Spawn `node` in a package directory (ibm_db postinstall, ignoring npm allow-scripts). */ +function runNode(args: string[], cwd: string, manualCommand: string): Promise { + return spawnCaptured(process.execPath, args, cwd, manualCommand); } /** Directory of the resolved `ibm_db` package, or null if it is not installed. */ export function ibmDbPackageDir(): string | null { try { - return dirname(nodeRequire.resolve('ibm_db/package.json')); + return dirname(dbDriverRequire().resolve('ibm_db/package.json')); } catch { return null; } @@ -298,46 +452,104 @@ export function runIbmDbDriverInstallJs(): Promise { return runNode(args, dir, manualCommand); } -/** Run `npm install --foreground-scripts` into the resolved target. */ -export function npmInstallPackage( +/** + * Run `npm install` into the resolved target. + * + * Workspace installs cannot be `npm install mongodb --save -w @foxschema/db`: + * when the package is already an optional peer, npm 11 exits 0, writes + * nothing, and verify then throws Cannot find module. Writing it onto + * `@foxschema/db` dependencies first forces a real fetch (including nested + * deps like `@redis/client`). + */ +export async function npmInstallPackage( packageSpec: string, fromUrl?: string ): Promise { const pkg = packageNameFromSpec(packageSpec); - const target = resolveDriverInstallTarget(fromUrl, pkg); + const target = resolveDriverInstallTarget(fromUrl); + if (target.mode === 'workspace') { + const version = versionFromSpec(packageSpec); + const sets = driverInstallPkgSets(target.cwd, pkg, version); + const manualCommand = formatDriverInstallCommand(sets); + for (const { workspace, assignments } of sets) { + const setResult = await runNpm( + ['pkg', 'set', ...assignments, '-w', workspace], + target.cwd, + manualCommand + ); + if (!setResult.ok) return setResult; + } + // Root install, not `-w db`: a web pin (mssql 12.7.13) would otherwise + // keep the hoisted copy from moving. + return runNpm(['install', '--foreground-scripts'], target.cwd, manualCommand); + } const args = target.npmArgs(packageSpec); return runNpm(args, target.cwd, target.manualCommand(packageSpec)); } -/** Drop cached failed/successful requires so a fresh install can load. */ +/** argv / cwd / copy-paste command for `npm uninstall ` at the same target as install. */ +export function npmUninstallArgs( + packageName: string, + fromUrl?: string, + workspacePkg?: string +): { args: string[]; cwd: string; manualCommand: string } { + const pkg = packageNameFromSpec(packageName); + const target = resolveDriverInstallTarget(fromUrl); + const args = + target.mode === 'workspace' + ? ['uninstall', pkg, '-w', workspacePkg ?? target.workspacePkg] + : ['uninstall', pkg, '--prefix', target.cwd]; + return { args, cwd: target.cwd, manualCommand: publicNpmCommand(`npm ${args.join(' ')}`) }; +} + +/** Run `npm uninstall ` from the same target install uses. */ +export function npmUninstallPackage( + packageName: string, + fromUrl?: string +): Promise { + const { args, cwd, manualCommand } = npmUninstallArgs(packageName, fromUrl); + return runNpm(args, cwd, manualCommand); +} + /** * Does this require-cache key belong to `packageName`? * - * Matches a whole `node_features//` path segment, never a bare substring: + * Matches a whole `node_modules//` path segment, never a bare substring: * purging `pg` by substring also evicts pg-pool, pg-protocol and anything else * whose path merely contains "pg", re-instantiating unrelated module state. * Both separators are checked so a Windows cache key is handled too. */ export function isCacheKeyForPackage(key: string, packageName: string): boolean { return ( - key.includes(`/node_features/${packageName}/`) || + key.includes(`/node_modules/${packageName}/`) || key.includes(`\\node_modules\\${packageName}\\`) ); } export function purgePackageRequireCache(packageName: string): void { + const req = dbDriverRequire(); try { - const resolved = nodeRequire.resolve(packageName); - delete nodeRequire.cache[resolved]; + const resolved = req.resolve(packageName); + delete req.cache[resolved]; const root = dirname(resolved) + sep; - for (const key of Object.keys(nodeRequire.cache)) { + for (const key of Object.keys(req.cache)) { if (key.startsWith(root) || isCacheKeyForPackage(key, packageName)) { - delete nodeRequire.cache[key]; + delete req.cache[key]; } } } catch { - for (const key of Object.keys(nodeRequire.cache)) { - if (isCacheKeyForPackage(key, packageName)) delete nodeRequire.cache[key]; + for (const key of Object.keys(req.cache)) { + if (isCacheKeyForPackage(key, packageName)) delete req.cache[key]; + } + } + // Failed lookups are cached too. After `npm install redis`, a require that + // already threw MODULE_NOT_FOUND would keep failing until this is cleared. + const pathCache = (nodeRequire('node:module') as { _pathCache?: Record })._pathCache; + if (pathCache) { + for (const key of Object.keys(pathCache)) { + if (isCacheKeyForPackage(key, packageName) || key.endsWith(`\u0000${packageName}`)) { + delete pathCache[key]; + } } } } @@ -371,11 +583,11 @@ export function verifyInstalledDriver(packageName: string): VerifyDriverResult { setupDb2ClientEnv(); } try { - nodeRequire(packageName); + dbDriverRequire()(packageName); } catch (e) { return { ok: false, - error: errorMessage(e), + error: driverLoadReason(e), needsClidriverRebuild: packageName === 'ibm_db', }; } @@ -406,7 +618,8 @@ export function createInstallQueue(): (key: string, run: () => Promise) => }; } -const installQueue = createInstallQueue(); +type QueuedDriverResult = NpmInstallResult & VerifyDriverResult & { stillInstalled?: boolean }; +const installQueue = createInstallQueue(); /** * Install + verify. For ibm_db, retries once with a forced reinstall when the @@ -420,6 +633,35 @@ export function installAndVerifyDriver( return installQueue(packageName, () => runInstallAndVerify(packageName, versionPin)); } +export type RemoveDriverResult = NpmInstallResult & { stillInstalled: boolean }; + +/** + * Uninstall + drop the require cache. `stillInstalled` is true when the + * package still loads afterwards (another workspace still lists it). + * Queued with installs so two npm writes never share one tree. + */ +export function removeAndVerifyDriver(packageName: string): Promise { + return installQueue(packageName, async () => { + let result = await npmUninstallPackage(packageName); + if (!result.ok) return { ...result, stillInstalled: true }; + purgePackageRequireCache(packageName); + let stillInstalled = verifyInstalledDriver(packageName).ok; + // Optional peers often live on @foxschema/web as well. One workspace + // uninstall leaves the package loadable from the other. Do not run this + // when the db uninstall already failed — a second npm on a half-renamed + // tree is how ENOTEMPTY shows up as "try -w @foxschema/web". + const target = resolveDriverInstallTarget(); + if (stillInstalled && target.mode === 'workspace') { + const web = npmUninstallArgs(packageName, undefined, '@foxschema/web'); + result = await runNpm(web.args, web.cwd, web.manualCommand); + if (!result.ok) return { ...result, stillInstalled: true }; + purgePackageRequireCache(packageName); + stillInstalled = verifyInstalledDriver(packageName).ok; + } + return { ...result, stillInstalled }; + }) as Promise; +} + async function runInstallAndVerify( packageName: string, versionPin?: string @@ -434,12 +676,12 @@ async function runInstallAndVerify( // lifecycle script fires, so the retry was a no-op. `npm rebuild` exists // for exactly this. Verified on npm 11.6.2: re-install --force skips the // postinstall, rebuild runs it. - const target = resolveDriverInstallTarget(import.meta.url, 'ibm_db'); + const target = resolveDriverInstallTarget(); const rebuildArgs = target.mode === 'workspace' - ? ['rebuild', 'ibm_db', '--foreground-scripts', '-w', target.workspacePkg ?? '@foxschema/db'] + ? ['rebuild', 'ibm_db', '--foreground-scripts', '-w', target.workspacePkg] : ['rebuild', 'ibm_db', '--foreground-scripts', '--prefix', target.cwd]; - result = await runNpm(rebuildArgs, target.cwd, `npm ${rebuildArgs.join(' ')}`); + result = await runNpm(rebuildArgs, target.cwd, publicNpmCommand(`npm ${rebuildArgs.join(' ')}`)); verify = verifyInstalledDriver(packageName); } @@ -464,20 +706,18 @@ export function driverInstallHints(packageName: string): string { 'Also try: foxschema drivers install db2', 'Or use Docker (Db2 included, linux/amd64): docker pull 5nickels/foxschema:latest', ]; - { - if (process.platform === 'darwin') { - lines.unshift( - 'macOS: ensure Xcode CLT (`xcode-select --install`) and a native Node for your CPU (`node -p process.arch`).' - ); - } else if (process.platform === 'linux' && process.arch === 'arm64') { - lines.unshift( - 'linux/arm64: ibm_db has no official build — use Docker linux/amd64 (or an x64 host).' - ); - } else if (process.platform === 'win32') { - lines.unshift( - 'Windows: install scripts must run so clidriver downloads; avoid npm config set ignore-scripts true.' - ); - } + if (process.platform === 'darwin') { + lines.unshift( + 'macOS: ensure Xcode CLT (`xcode-select --install`) and a native Node for your CPU (`node -p process.arch`).' + ); + } else if (process.platform === 'linux' && process.arch === 'arm64') { + lines.unshift( + 'linux/arm64: ibm_db has no official build — use Docker linux/amd64 (or an x64 host).' + ); + } else if (process.platform === 'win32') { + lines.unshift( + 'Windows: install scripts must run so clidriver downloads; avoid npm config set ignore-scripts true.' + ); } return lines.join(' '); } diff --git a/packages/server/src/platform/contracts/actor.test.ts b/packages/server/src/platform/contracts/actor.test.ts new file mode 100644 index 000000000..ade3746f2 --- /dev/null +++ b/packages/server/src/platform/contracts/actor.test.ts @@ -0,0 +1,24 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + */ +import { describe, expect, it } from 'vitest'; +import { publicThrownMessage, toApiError } from './actor'; + +describe('publicThrownMessage', () => { + it('rewrites a pg ECONNRESET so the UI is not the syscall name', () => { + const error = Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }); + expect(publicThrownMessage(error, 'Failed to load schema')).toMatch(/closed the connection before login \(ECONNRESET\)/); + expect(publicThrownMessage(error, 'Failed to load schema')).not.toBe('read ECONNRESET'); + }); +}); + +describe('toApiError', () => { + it('keeps ECONNRESET as failed with the rewritten sentence', () => { + const body = toApiError(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }), 'Failed to load schema'); + expect(body.code).toBe('failed'); + expect(body.error).toMatch(/ECONNRESET/); + expect(body.error).not.toBe('read ECONNRESET'); + }); +}); diff --git a/packages/server/src/platform/contracts/actor.ts b/packages/server/src/platform/contracts/actor.ts index 8f7a0e968..bb8cf27cc 100644 --- a/packages/server/src/platform/contracts/actor.ts +++ b/packages/server/src/platform/contracts/actor.ts @@ -97,6 +97,23 @@ function asCircuitOpen(error: unknown): { retryAfterSec: number } | undefined { return { retryAfterSec: Math.max(1, Math.ceil(ms / 1000)) }; } +/** + * Node's connect errnos (`read ECONNRESET`) are true, and useless in the UI. + * Keep the code in the sentence so logs and e2e still match. + */ +export function publicThrownMessage(error: unknown, fallback: string): string { + const raw = error instanceof Error ? error.message : String(error ?? ''); + const code = + error && typeof error === 'object' && 'code' in error + ? String((error as { code?: unknown }).code ?? '') + : ''; + const hay = `${code} ${raw}`; + if (/ECONNRESET/i.test(hay)) { + return 'The database closed the connection before login (ECONNRESET). The host accepted TCP then dropped it — often a paused cluster, a firewall, TLS mismatch, or a local Docker database that is listening but not ready.'; + } + return raw.trim() || fallback; +} + /** * Map any thrown value to the wire error contract. * @@ -111,7 +128,7 @@ export function toApiError(error: unknown, fallback: string): ApiErrorBody { if (circuit) { return { ok: false, - error: error instanceof Error ? error.message : fallback, + error: publicThrownMessage(error, fallback), code: 'unavailable', retryAfterSec: circuit.retryAfterSec, }; @@ -119,7 +136,7 @@ export function toApiError(error: unknown, fallback: string): ApiErrorBody { return { ok: false, - error: error instanceof Error ? error.message : fallback, + error: publicThrownMessage(error, fallback), code: 'failed', }; } diff --git a/packages/shared/src/driver-inventory.ts b/packages/shared/src/driver-inventory.ts new file mode 100644 index 000000000..d4d32bbbc --- /dev/null +++ b/packages/shared/src/driver-inventory.ts @@ -0,0 +1,41 @@ +/** + * Fox Schema (foxschema) + * Copyright 2024-2026 Huy Phan + * SPDX-License-Identifier: Apache-2.0 + * + * What GET /api/system/drivers answers: one row per npm driver package the + * app can load (several dialects share pg / mysql2 / mssql). Used by the + * Drivers card on Home, under Services. + */ + +export interface DriverInventoryItem { + packageName: string; + /** Dialects that load through this package, in registry order. */ + dialects: string[]; + /** Written names for `dialects`, same order. */ + labels: string[]; + installed: boolean; + version?: string; + /** + * Present after a version check (`?latest=1`). `null` means the registry + * did not answer for this package. + */ + latest?: string | null; + updateAvailable: boolean; + /** + * Required for Fox Schema itself (`better-sqlite3` for the metadata store). + * Those cannot be removed. + */ + bundled: boolean; + /** Installed and not bundled — Remove is offered. */ + removable: boolean; + installCommand?: string; + error?: string; +} + +export interface DriverInventoryReport { + drivers: DriverInventoryItem[]; + checkedAt: number; + /** True when `latest` was asked for on this response. */ + latestChecked: boolean; +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index b90fb4bf0..188a6b603 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -18,6 +18,7 @@ export * from './lokee-wire'; export * from './nav'; export * from './password-policy'; export * from './service-status'; +export * from './driver-inventory'; export * from './connection-name'; export * from './migration-status'; export * from './data-migrate'; diff --git a/packages/shared/src/permissions.ts b/packages/shared/src/permissions.ts index 160ec63ae..91d728f2e 100644 --- a/packages/shared/src/permissions.ts +++ b/packages/shared/src/permissions.ts @@ -163,7 +163,7 @@ export const PERMISSION_META: PermissionMeta[] = [ { id: 'admin.roles', group: 'Admin', label: 'Configure roles', description: 'Edit which FoxSchema permissions each app role receives, including Grant privileges for database GRANT/REVOKE.' }, { id: 'app.write', group: 'Account', label: 'Save and change', description: 'Save connections, secrets, scripts and settings, and run anything that writes. Without it an account only reads: it browses, compares and runs queries, and every other route refuses it. The Demo role is the role without it. Install-wide: comes from the account role, not a workspace role.' }, { id: 'account.self', group: 'Account', label: 'Manage own account', description: 'Change its own profile and password, ask for a reset or a verification email, and sign its other sessions out. A shared login (the Demo role) has it withheld, so no visitor changes the account for everyone. Install-wide: comes from the account role.' }, - { id: 'admin.drivers', group: 'Admin', label: 'Install drivers', description: 'Install a database driver package on the server (npm install, and a native build for Db2). It downloads and runs code on the machine Fox Schema runs on, for every workspace. Install-wide: comes from the account role, not a workspace role.' }, + { id: 'admin.drivers', group: 'Admin', label: 'Install drivers', description: 'Install, update, or remove a database driver package on the server (npm, and a native build for Db2). It downloads and runs code on the machine Fox Schema runs on, for every workspace. Install-wide: comes from the account role, not a workspace role.' }, ]; const ALL = [...PERMISSIONS];