diff --git a/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.test.ts b/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.test.ts index 9c1870822..e7fae6691 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.test.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.test.ts @@ -263,6 +263,12 @@ describe('primary key alter', () => { expect(sql[0]).toMatch(/DROP PRIMARY KEY/i); expect(sql[1]).toBe('ALTER TABLE t ADD PRIMARY KEY (a, b);'); }); + + it('reviews a ClickHouse composite key change instead of ALTER ADD PRIMARY KEY', () => { + const sql = generatePkAlterSql('order_coupons', 'clickhouse', ['order_id'], ['order_id', 'coupon_id']); + expect(sql.join('\n')).toMatch(/-- review:/); + expect(sql.join('\n')).not.toMatch(/ALTER TABLE .* ADD .*PRIMARY KEY/i); + }); }); describe('create / drop table', () => { @@ -283,6 +289,22 @@ describe('create / drop table', () => { expect(sql[0]).not.toMatch(/INTEGER NOT NULL PRIMARY KEY/); }); + it('maps a ClickHouse composite PK to MergeTree ORDER BY, not a constraint', () => { + const sql = generateCreateTableSql( + 'order_coupons', + [ + col({ name: 'order_id', type: 'Int32', nullable: false, primaryKey: true }), + col({ name: 'coupon_id', type: 'Int32', nullable: false, primaryKey: true }), + col({ name: 'applied_at', type: 'DateTime', nullable: false }), + ], + ['order_id', 'coupon_id'], + 'clickhouse' + ); + expect(sql[0]).toMatch(/ENGINE = MergeTree ORDER BY \(order_id, coupon_id\)/); + expect(sql[0]).not.toMatch(/CONSTRAINT/); + expect(sql[0]).not.toContain('PRIMARY KEY (order_id, coupon_id)'); + }); + it('uses OBJECT_ID guard for sqlserver create', () => { const sql = generateCreateTableSql( 't', @@ -345,6 +367,10 @@ describe('foreign keys & triggers', () => { expect(quoteTableRef('My Schema.Odd-Table', 'postgres')).toBe('"My Schema"."Odd-Table"'); expect(qualifyTableName('test1', 'mcve', 'postgres')).toBe('mcve.test1'); expect(qualifyTableName('mcve.test1', 'other', 'postgres')).toBe('mcve.test1'); + expect(qualifyTableName('legacy_audit_log', 'demo_b', 'clickhouse')).toBe( + 'demo_b.legacy_audit_log' + ); + expect(qualifyTableName('t', 'demo', 'sqlite')).toBe('t'); const sql = generateAddForeignKeySql( 'test2', diff --git a/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.ts b/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.ts index d68a390d0..c32f02796 100644 --- a/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.ts +++ b/apps/web/src/frontend/features/sql-editor/lib/tableBlueprintSql.ts @@ -41,7 +41,9 @@ export function qualifyTableName( const sch = schema?.trim(); if (!sch) return name; const d = dialectName.toLowerCase(); - if (d === 'sqlite' || d === 'clickhouse') return name; + // SQLite is file-scoped and has no schema. ClickHouse databases *are* the + // schema — qualify so DROP/CREATE hit demo_b.t instead of the client default. + if (d === 'sqlite') return name; return `${sch}.${name}`; } @@ -727,12 +729,22 @@ export function generatePkAlterSql( stmts.push(...dialect.dropPrimaryKeyStatements(qTable, qPk)); } if (next.length > 0) { - const cols = next.map((c) => quoteIdent(c, dialectName)).join(', '); - const constraint = - pkName && pkName.toUpperCase() !== 'PRIMARY' - ? `CONSTRAINT ${quoteIdent(pkName, dialectName)} ` - : ''; - stmts.push(`ALTER TABLE ${qTable} ADD ${constraint}PRIMARY KEY (${cols});`); + const quoted = next.map((c) => quoteIdent(c, dialectName)); + if (dialect.addPrimaryKeyStatements) { + stmts.push( + ...dialect.addPrimaryKeyStatements( + qTable, + quoted, + pkName && pkName.toUpperCase() !== 'PRIMARY' ? quoteIdent(pkName, dialectName) : undefined + ) + ); + } else { + const constraint = + pkName && pkName.toUpperCase() !== 'PRIMARY' + ? `CONSTRAINT ${quoteIdent(pkName, dialectName)} ` + : ''; + stmts.push(`ALTER TABLE ${qTable} ADD ${constraint}PRIMARY KEY (${quoted.join(', ')});`); + } } return stmts; } @@ -845,7 +857,8 @@ function createTableBody( } return ` ${buildColumnDef(c, dialectName)}`; }); - if (pkColumns.length > 0 && !sqliteIdentityPkCol) { + const dialect = resolveDialect(dialectName); + if (pkColumns.length > 0 && !sqliteIdentityPkCol && dialect.inlinePrimaryKeyConstraint !== false) { const cols = pkColumns.map((c) => quoteIdent(c, dialectName)).join(', '); const constraint = pkName && pkName.toUpperCase() !== 'PRIMARY' @@ -860,6 +873,27 @@ function createTableBody( return `(\n${lines.join(',\n')}\n)`; } +/** ENGINE / ORDER BY (and similar) after the column list. Empty on most dialects. */ +function createTableSuffixSql( + tableName: string, + columns: ColumnInfo[], + pkColumns: string[], + dialectName: string +): string { + const dialect = resolveDialect(dialectName); + if (!dialect.createTableSuffix) return ''; + const table: TableSchema = { + name: tableName, + objectType: 'TABLE', + columns, + indices: [], + foreignKeys: [], + ...(pkColumns.length ? { primaryKey: { columns: pkColumns } } : {}), + }; + const quoted = pkColumns.map((c) => (/[()]/.test(c) ? c : quoteIdent(c, dialectName))); + return dialect.createTableSuffix(table, quoted); +} + /** * CREATE TABLE with existence guard (IF NOT EXISTS or dialect equivalent). * Optional `foreignKeys` are inlined when the dialect supports CREATE-table FKs @@ -881,31 +915,32 @@ export function generateCreateTableSql( const support = dialectFkConstraintSupport(dialectName); const inlineFks = support.createInline ? foreignKeys : undefined; const body = createTableBody(columns, pkColumns, dialectName, pkName, inlineFks, schema); + const suffix = createTableSuffixSql(name, columns, pkColumns, dialectName); const d = dialectName.toLowerCase(); const objectIdName = qualifyTableName(name, schema, dialectName); const ifNotExists = opts?.ifNotExists !== false; if (!ifNotExists) { - return [`CREATE TABLE ${qTable} ${body};`]; + return [`CREATE TABLE ${qTable} ${body}${suffix};`]; } if (dialectFamily(d) === 'sqlserver') { const escaped = objectIdName.replace(/'/g, "''"); return [ - `IF OBJECT_ID(N'${escaped}', N'U') IS NULL\nCREATE TABLE ${qTable} ${body};`, + `IF OBJECT_ID(N'${escaped}', N'U') IS NULL\nCREATE TABLE ${qTable} ${body}${suffix};`, ]; } if (d === 'oracle') { // Oracle 23c+ supports IF NOT EXISTS; older versions ignore and may error on re-run. - return [`CREATE TABLE IF NOT EXISTS ${qTable} ${body};`]; + return [`CREATE TABLE IF NOT EXISTS ${qTable} ${body}${suffix};`]; } if (d === 'db2') { // DB2 has no CREATE TABLE IF NOT EXISTS — emit plain CREATE (Apply fails if exists). return [ `-- review: DB2 has no CREATE TABLE IF NOT EXISTS — fails if ${name} already exists`, - `CREATE TABLE ${qTable} ${body};`, + `CREATE TABLE ${qTable} ${body}${suffix};`, ]; } - return [`CREATE TABLE IF NOT EXISTS ${qTable} ${body};`]; + return [`CREATE TABLE IF NOT EXISTS ${qTable} ${body}${suffix};`]; } /** diff --git a/docker/init/clickhouse/01_seed.sql b/docker/init/clickhouse/01_seed.sql index 67f55bd0f..f0678fa16 100644 --- a/docker/init/clickhouse/01_seed.sql +++ b/docker/init/clickhouse/01_seed.sql @@ -79,6 +79,14 @@ CREATE TABLE demo_a.coupons ( valid_until Nullable(Date) ) ENGINE = MergeTree ORDER BY id; +-- Composite sorting key. ClickHouse has no table-level PRIMARY KEY constraint; +-- ORDER BY (order_id, coupon_id) is the MergeTree equivalent of a composite PK. +CREATE TABLE demo_a.order_coupons ( + order_id Int32, + coupon_id Int32, + applied_at DateTime DEFAULT now() +) ENGINE = MergeTree ORDER BY (order_id, coupon_id); + CREATE VIEW demo_a.v_active_products AS SELECT id, name, price, sku FROM demo_a.products diff --git a/packages/db/src/providers/clickHouse/clickhouse.provider.ts b/packages/db/src/providers/clickHouse/clickhouse.provider.ts index 8cfdced63..9ea56994f 100644 --- a/packages/db/src/providers/clickHouse/clickhouse.provider.ts +++ b/packages/db/src/providers/clickHouse/clickhouse.provider.ts @@ -1,5 +1,5 @@ import { ConnectionFactory } from '../../cores/connection-factory.js'; -import { dbSchemaToTableSchemas, errorMessage } from '@foxschema/sql'; +import { dbSchemaToTableSchemas, errorMessage, parseClickHouseKeyList } from '@foxschema/sql'; import { type SchemaProvider, type ConnectionOptions, @@ -15,7 +15,7 @@ import { type TableSchema, } from '@foxschema/sql'; -interface ChTableRaw { name: string; engine: string; } +interface ChTableRaw { name: string; engine: string; sorting_key: string; primary_key: string; } interface ChViewRaw { name: string; definition: string; } interface ChColumnRaw { table: string; @@ -23,6 +23,7 @@ interface ChColumnRaw { position: number; type: string; is_in_primary_key: number | boolean; + is_in_sorting_key: number | boolean; default_kind: string; default_expression: string; } @@ -74,7 +75,7 @@ export class ClickHouseProvider implements SchemaProvider { const [rawTables, rawViews, rawColumns] = await Promise.all([ exec( - `SELECT name, engine FROM system.tables + `SELECT name, engine, sorting_key, primary_key FROM system.tables WHERE database = $1 AND engine NOT IN ('View', 'MaterializedView', 'LiveView', 'Window', 'Merge', 'Distributed') ORDER BY name`, [db] @@ -86,7 +87,7 @@ export class ClickHouseProvider implements SchemaProvider { [db] ), exec( - `SELECT table, name, position, type, is_in_primary_key, default_kind, default_expression + `SELECT table, name, position, type, is_in_primary_key, is_in_sorting_key, default_kind, default_expression FROM system.columns WHERE database = $1 ORDER BY table, position`, [db] @@ -102,11 +103,30 @@ export class ClickHouseProvider implements SchemaProvider { const indexes: Record = {}; const indexColumns: Record = {}; - // Init tables + // Catalog sorting_key / primary_key win: expressions like cityHash64(id) + // live only there. Column flags fill in when the catalog strings are empty. + const catalogPk = new Set(); for (const t of rawTables) { - tables[t.name] = { name: t.name, columns: {}, primaryKey: [], foreignKeys: [], uniqueConstraints: [], indexes: [] }; + const orderBy = parseClickHouseKeyList(t.sorting_key); + const fromPk = parseClickHouseKeyList(t.primary_key); + tables[t.name] = { + name: t.name, + columns: {}, + primaryKey: fromPk, + foreignKeys: [], + uniqueConstraints: [], + indexes: [], + ...(t.engine ? { engine: t.engine } : {}), + ...(orderBy.length ? { orderBy } : {}), + }; columns[t.name] = []; - primaryKeys[t.name] = []; + primaryKeys[t.name] = fromPk.map((column, i) => ({ + name: column, + constName: 'PRIMARY_KEY', + column, + colSeq: i + 1, + })); + if (fromPk.length) catalogPk.add(t.name); foreignKeys[t.name] = []; uniqueConstraints[t.name] = []; indexes[t.name] = []; @@ -125,12 +145,15 @@ export class ClickHouseProvider implements SchemaProvider { }; if (tables[c.table]) { tables[c.table].columns[c.name] = col; - if (isPk) tables[c.table].primaryKey.push(c.name); + if (isPk && !catalogPk.has(c.table)) tables[c.table].primaryKey.push(c.name); } (columns[c.table] ??= []).push(col); - if (isPk) { + if (isPk && !catalogPk.has(c.table)) { (primaryKeys[c.table] ??= []).push({ name: c.name, constName: 'PRIMARY_KEY', column: c.name, colSeq: c.position }); } + if (c.is_in_sorting_key && tables[c.table] && !tables[c.table].orderBy?.length) { + (tables[c.table].orderBy ??= []).push(c.name); + } } // Views diff --git a/packages/sql/src/cores/schema-to-tables.test.ts b/packages/sql/src/cores/schema-to-tables.test.ts index 2b1f5838f..a5cd8884e 100644 --- a/packages/sql/src/cores/schema-to-tables.test.ts +++ b/packages/sql/src/cores/schema-to-tables.test.ts @@ -236,3 +236,36 @@ describe('groupForeignKeyRows', () => { expect(out[0]!.fk.referencedColumns).toEqual([]); }); }); + +describe('dbSchemaToTableSchemas ClickHouse engine / ORDER BY', () => { + it('copies engine and sorting key onto TableSchema', () => { + const coupons: DbTable = { + name: 'coupons', + columns: { id: { name: 'id', type: 'Int32', nullable: false } }, + primaryKey: ['id'], + foreignKeys: [], + uniqueConstraints: [], + indexes: [], + engine: 'MergeTree', + orderBy: ['id'], + }; + const [t] = dbSchemaToTableSchemas(emptySchema({ coupons })); + expect(t?.engine).toBe('MergeTree'); + expect(t?.orderBy).toEqual(['id']); + expect(t?.primaryKey?.columns).toEqual(['id']); + }); + + it('omits engine / ORDER BY when the catalog did not send them', () => { + const orders: DbTable = { + name: 'orders', + columns: { id: { name: 'id', type: 'int', nullable: false } }, + primaryKey: ['id'], + foreignKeys: [], + uniqueConstraints: [], + indexes: [], + }; + const [t] = dbSchemaToTableSchemas(emptySchema({ orders })); + expect(t?.engine).toBeUndefined(); + expect(t?.orderBy).toBeUndefined(); + }); +}); diff --git a/packages/sql/src/cores/schema-to-tables.ts b/packages/sql/src/cores/schema-to-tables.ts index 06ddc98e8..810624e07 100644 --- a/packages/sql/src/cores/schema-to-tables.ts +++ b/packages/sql/src/cores/schema-to-tables.ts @@ -204,6 +204,8 @@ export function dbSchemaToTableSchemas(dbSchema: DbSchema): TableSchema[] { objectType: table.isMqt ? 'MQT' : 'TABLE', definition: table.definition, tablespace: table.tablespace, + ...(table.engine ? { engine: table.engine } : {}), + ...(table.orderBy?.length ? { orderBy: table.orderBy } : {}), ...(table.storeType ? { storeType: table.storeType } : {}), triggers: triggersByTable.get(table.name) ?? [], primaryKey: pkColumns.length > 0 diff --git a/packages/sql/src/index.ts b/packages/sql/src/index.ts index 302870972..c69055d2c 100644 --- a/packages/sql/src/index.ts +++ b/packages/sql/src/index.ts @@ -368,6 +368,7 @@ export type { } from './modules/sql-editor/code-cell-types.js'; export { isCodeCellLast, isCodeCellVars, CODE_CELL_KIND_LABEL } from './modules/sql-editor/code-cell-types.js'; export type { SqlDialect, CanonicalType, CanonicalBase, RenderedType } from './modules/dialect/sql-dialect.interface.js'; +export { parseClickHouseKeyList } from './providers/clickHouse/clickhouse.sql-dialect.js'; export { resolveDialect, tryResolveDialect, DIALECT_MAP } from './modules/dialect/registry.js'; export { dialectFeatures, diff --git a/packages/sql/src/interfaces/schema.interface.ts b/packages/sql/src/interfaces/schema.interface.ts index 396916b08..a01b88aca 100644 --- a/packages/sql/src/interfaces/schema.interface.ts +++ b/packages/sql/src/interfaces/schema.interface.ts @@ -111,6 +111,18 @@ export interface TableSchema { functionKind?: 'scalar' | 'table'; /** Storage tablespace (tables/MQTs). */ tablespace?: string; + /** + * Table engine, when the catalog has one. ClickHouse MergeTree family + * (`MergeTree`, `ReplacingMergeTree`, …). Absent for engines that do not + * name a storage engine on the table. + */ + engine?: string; + /** + * Sorting / ORDER BY expressions, in key order. ClickHouse MergeTree + * requires this; it is usually the logical primary key and may be a + * prefix-plus-extra of `primaryKey.columns`. Absent elsewhere. + */ + orderBy?: string[]; /** * What a non-SQL store keeps behind this "table" — for Redis `hash`, `json`, * `string`, `mixed`, `list`, `set`, `zset` or `stream`. Shown as a badge; @@ -150,6 +162,10 @@ export interface DbTable { isMqt?: boolean; /** Materialized view's backing query (Postgres matviews). Undefined for plain tables and for dialects that don't expose it (e.g. DB2 MQTs). */ definition?: string; + /** See `TableSchema.engine`. */ + engine?: string; + /** See `TableSchema.orderBy`. */ + orderBy?: string[]; /** See `TableSchema.storeType`. */ storeType?: string; } diff --git a/packages/sql/src/modules/dialect/sql-dialect.interface.ts b/packages/sql/src/modules/dialect/sql-dialect.interface.ts index 62e6dba89..cb6d1b78a 100644 --- a/packages/sql/src/modules/dialect/sql-dialect.interface.ts +++ b/packages/sql/src/modules/dialect/sql-dialect.interface.ts @@ -84,6 +84,34 @@ export interface SqlDialect { /** Statement(s) to drop an existing primary key before adding a new one. */ dropPrimaryKeyStatements(tableName: string, pkName: string | undefined): string[]; + /** + * Add a primary key. Default: `ALTER TABLE t ADD [CONSTRAINT name] PRIMARY KEY (cols)`. + * Override when that form is invalid (ClickHouse: `-- review:` recreate). + */ + addPrimaryKeyStatements?(tableName: string, quotedColumns: string[], pkName: string | undefined): string[]; + + /** + * Emit `PRIMARY KEY (cols)` inside CREATE TABLE. Default true. + * ClickHouse only allows CHECK/ASSUME there. + */ + inlinePrimaryKeyConstraint?: boolean; + + /** + * Clauses after the CREATE TABLE column list. ClickHouse: ` ENGINE = MergeTree ORDER BY …`. + * `quotedOrderBy` is identifier-quoted, or left raw for a function expression. + */ + createTableSuffix?(table: TableSchema, quotedOrderBy: string[]): string; + + /** + * ENGINE / ORDER BY changed. Omit when the dialect has no table engine. + * ClickHouse cannot ALTER those in place — returns `-- review:` to recreate. + */ + engineChangeStatements?( + tableName: string, + source: TableSchema, + target: TableSchema + ): string[]; + /** Parse a native type string (e.g. "VARCHAR(255)") into the canonical model. */ parseType(raw: string): CanonicalType; diff --git a/packages/sql/src/modules/migrations/sql-generator.module.test.ts b/packages/sql/src/modules/migrations/sql-generator.module.test.ts index 75d277f33..610900403 100644 --- a/packages/sql/src/modules/migrations/sql-generator.module.test.ts +++ b/packages/sql/src/modules/migrations/sql-generator.module.test.ts @@ -1424,3 +1424,152 @@ describe('role membership speaks each engine’s own syntax', () => { expect(sql).toContain('REVOKE ROLE foxrole FROM USER gone;'); }); }); + +describe('ClickHouse CREATE TABLE uses ENGINE / ORDER BY, not a PRIMARY KEY constraint', () => { + const coupons = tableSchema({ + name: 'coupons', + primaryKey: { name: 'PRIMARY_KEY', columns: ['id'] }, + columns: [ + { name: 'id', type: 'Int32', nullable: false, primaryKey: true }, + { name: 'code', type: 'String', nullable: false, primaryKey: false }, + { name: 'discount_pct', type: 'Decimal(5, 2)', nullable: false, primaryKey: false }, + { name: 'valid_until', type: 'Date', nullable: true, primaryKey: false }, + ], + }); + + it('maps a logical primary key to MergeTree ORDER BY', () => { + const ddl = gen.generateObjectDdl(coupons, 'clickhouse'); + expect(ddl).toContain('CREATE TABLE coupons ('); + expect(ddl).toContain('id Int32'); + expect(ddl).toMatch(/\) ENGINE = MergeTree ORDER BY id;/); + expect(ddl).not.toMatch(/PRIMARY KEY/); + expect(ddl).not.toMatch(/CONSTRAINT PRIMARY_KEY/); + }); + + it('maps a composite primary key to ORDER BY (cols)', () => { + const ddl = gen.generateObjectDdl( + tableSchema({ + name: 'order_coupons', + primaryKey: { name: 'PRIMARY_KEY', columns: ['order_id', 'coupon_id'] }, + columns: [ + { name: 'order_id', type: 'Int32', nullable: false, primaryKey: true }, + { name: 'coupon_id', type: 'Int32', nullable: false, primaryKey: true }, + { name: 'applied_at', type: 'DateTime', nullable: false, primaryKey: false }, + ], + }), + 'clickhouse' + ); + expect(ddl).toMatch(/\) ENGINE = MergeTree ORDER BY \(order_id, coupon_id\);/); + expect(ddl).not.toMatch(/CONSTRAINT/); + expect(ddl).not.toMatch(/PRIMARY KEY \(order_id, coupon_id\)/); + }); + + it('keeps a captured engine and sorting key', () => { + const ddl = gen.generateObjectDdl( + tableSchema({ + ...coupons, + engine: 'ReplacingMergeTree', + orderBy: ['id', 'code'], + }), + 'clickhouse' + ); + expect(ddl).toMatch(/\) ENGINE = ReplacingMergeTree PRIMARY KEY id ORDER BY \(id, code\);/); + }); + + it('does not change Postgres / DB2 CREATE TABLE shape', () => { + const pg = gen.generateObjectDdl(coupons, 'postgres'); + expect(pg).toContain('CONSTRAINT PRIMARY_KEY PRIMARY KEY (id)'); + expect(pg).not.toMatch(/ENGINE/); + const db2 = gen.generateObjectDdl(coupons, 'db2'); + expect(db2).toContain('CONSTRAINT PRIMARY_KEY PRIMARY KEY (id)'); + expect(db2).not.toMatch(/ENGINE/); + }); + + it('emits the same ENGINE / ORDER BY form from a migration plan', () => { + const diff: TableDiff = { + tableName: 'COUPONS', + objectType: 'TABLE', + status: 'ADDED', + columnDiffs: [], + indexDiffs: [], + foreignKeyDiffs: [], + sourceTable: coupons, + }; + const sql = gen + .generateMigrationPlan([diff], 'clickhouse', { targetSchema: 'demo_b' }) + .flatMap((s) => s.statements) + .join('\n'); + expect(sql).toMatch(/CREATE TABLE demo_b\.coupons \(/); + expect(sql).toMatch(/ENGINE = MergeTree ORDER BY id;/); + expect(sql).not.toMatch(/PRIMARY KEY \(id\)/); + }); + + it('reviews an in-place key change instead of ALTER TABLE ADD PRIMARY KEY', () => { + const diff: TableDiff = { + tableName: 'COUPONS', + objectType: 'TABLE', + status: 'MODIFIED', + columnDiffs: [], + indexDiffs: [], + foreignKeyDiffs: [], + sourceTable: tableSchema({ + name: 'coupons', + primaryKey: { name: 'PRIMARY_KEY', columns: ['id', 'code'] }, + columns: coupons.columns, + engine: 'MergeTree', + orderBy: ['id', 'code'], + }), + targetTable: tableSchema({ + name: 'coupons', + primaryKey: { name: 'PRIMARY_KEY', columns: ['id'] }, + columns: coupons.columns, + engine: 'MergeTree', + orderBy: ['id'], + }), + }; + const sql = gen.generateMigrationPlan([diff], 'clickhouse').flatMap((s) => s.statements).join('\n'); + expect(sql).toMatch(/-- review:.*PRIMARY KEY is part of the ENGINE clause/i); + expect(sql).not.toMatch(/ALTER TABLE .* ADD .*PRIMARY KEY/i); + }); + + it('reviews an ORDER BY change without emitting ALTER TABLE ADD PRIMARY KEY', () => { + const diff: TableDiff = { + tableName: 'COUPONS', + objectType: 'TABLE', + status: 'MODIFIED', + columnDiffs: [], + indexDiffs: [], + foreignKeyDiffs: [], + sourceTable: tableSchema({ + ...coupons, + engine: 'MergeTree', + orderBy: ['id', 'code'], + }), + targetTable: tableSchema({ + ...coupons, + engine: 'MergeTree', + orderBy: ['id'], + }), + }; + const sql = gen.generateMigrationPlan([diff], 'clickhouse').flatMap((s) => s.statements).join('\n'); + expect(sql).toMatch(/-- review:.*ENGINE \/ ORDER BY cannot be altered in place/i); + expect(sql).not.toMatch(/ALTER TABLE .* ADD .*PRIMARY KEY/i); + }); + + it('drops a ClickHouse table with DROP TABLE IF EXISTS', () => { + const diff: TableDiff = { + tableName: 'LEGACY_AUDIT_LOG', + objectType: 'TABLE', + status: 'REMOVED', + columnDiffs: [], + indexDiffs: [], + foreignKeyDiffs: [], + targetTable: tableSchema({ name: 'legacy_audit_log', columns: coupons.columns }), + }; + const sql = gen + .generateMigrationPlan([diff], 'clickhouse', { targetSchema: 'demo_b' }) + .flatMap((s) => s.statements) + .join('\n'); + expect(sql).toMatch(/DROP TABLE IF EXISTS demo_b\.legacy_audit_log;/); + }); +}); diff --git a/packages/sql/src/modules/migrations/sql-generator.module.ts b/packages/sql/src/modules/migrations/sql-generator.module.ts index ae61c3b99..2113fea9b 100644 --- a/packages/sql/src/modules/migrations/sql-generator.module.ts +++ b/packages/sql/src/modules/migrations/sql-generator.module.ts @@ -324,7 +324,7 @@ export class SqlGeneratorModule { const lines = table.columns.map((c) => ` ${this.renderColumn(c, dialect, mapping, warnings)}`); const pkCols = this.primaryKeyColumns(table); - if (pkCols.length > 0) { + if (pkCols.length > 0 && dialect.inlinePrimaryKeyConstraint !== false) { const pkName = table.primaryKey?.name; // MySQL names every PK constraint "PRIMARY" — emitting CONSTRAINT PRIMARY PRIMARY KEY // is redundant and confusing. Skip the CONSTRAINT clause for that reserved name. @@ -347,7 +347,17 @@ export class SqlGeneratorModule { } } - return `CREATE TABLE ${this.qualify(table.name, mapping)} (\n${lines.join(',\n')}\n);`; + let suffix = ''; + if (dialect.createTableSuffix) { + const orderSrc = table.orderBy?.length ? table.orderBy : pkCols; + // ClickHouse sorting_key can be `tuple()` or `cityHash64(id)` — those are + // expressions, not identifiers, and quoting them is a syntax error. + suffix = dialect.createTableSuffix( + table, + orderSrc.map((c) => (/[()]/.test(c) ? c : this.ident(c))) + ); + } + return `CREATE TABLE ${this.qualify(table.name, mapping)} (\n${lines.join(',\n')}\n)${suffix};`; } /** @@ -867,14 +877,28 @@ export class SqlGeneratorModule { } if (srcPk.length > 0) { const pkName = obj.sourceTable?.primaryKey?.name; - // MySQL/MariaDB always name the PK constraint "PRIMARY" — emitting - // CONSTRAINT PRIMARY PRIMARY KEY is invalid syntax. Skip the CONSTRAINT - // clause for that reserved name (mirrors renderCreateTable's guard). - const constraint = pkName && pkName.toUpperCase() !== 'PRIMARY' ? `CONSTRAINT ${this.bareName(pkName)} ` : ''; - statements.push(`ALTER TABLE ${tableName} ADD ${constraint}PRIMARY KEY (${srcPk.join(', ')});`); + if (dialect.addPrimaryKeyStatements) { + statements.push( + ...dialect.addPrimaryKeyStatements( + tableName, + srcPk.map((c) => this.ident(c)), + pkName ? this.bareName(pkName) : undefined + ) + ); + } else { + // MySQL/MariaDB always name the PK constraint "PRIMARY" — emitting + // CONSTRAINT PRIMARY PRIMARY KEY is invalid syntax. Skip the CONSTRAINT + // clause for that reserved name (mirrors renderCreateTable's guard). + const constraint = pkName && pkName.toUpperCase() !== 'PRIMARY' ? `CONSTRAINT ${this.bareName(pkName)} ` : ''; + statements.push(`ALTER TABLE ${tableName} ADD ${constraint}PRIMARY KEY (${srcPk.join(', ')});`); + } } } + if (obj.sourceTable && obj.targetTable && dialect.engineChangeStatements) { + statements.push(...dialect.engineChangeStatements(tableName, obj.sourceTable, obj.targetTable)); + } + for (const idx of obj.indexDiffs.filter((i) => i.status === 'ADDED' || i.status === 'MODIFIED')) { const srcIdx = idx.source; if (!srcIdx) continue; diff --git a/packages/sql/src/modules/schema-diff/compare.module.test.ts b/packages/sql/src/modules/schema-diff/compare.module.test.ts index 6a490adb7..2b07af560 100644 --- a/packages/sql/src/modules/schema-diff/compare.module.test.ts +++ b/packages/sql/src/modules/schema-diff/compare.module.test.ts @@ -90,6 +90,36 @@ describe('CompareModule.compare', () => { expect(r.tables[0].status).toBe('MODIFIED'); }); + it('detects a ClickHouse ORDER BY change while the logical PK stays the same', async () => { + const src = table({ + name: 'COUPONS', + primaryKey: { name: 'PRIMARY_KEY', columns: ['id'] }, + engine: 'MergeTree', + orderBy: ['id', 'code'], + }); + const tgt = table({ + name: 'COUPONS', + primaryKey: { name: 'PRIMARY_KEY', columns: ['id'] }, + engine: 'MergeTree', + orderBy: ['id'], + }); + const r = await cmp.compare([src], [tgt]); + expect(r.tables[0].status).toBe('MODIFIED'); + }); + + it('detects a ClickHouse table engine change', async () => { + const src = table({ name: 'COUPONS', engine: 'ReplacingMergeTree', orderBy: ['id'] }); + const tgt = table({ name: 'COUPONS', engine: 'MergeTree', orderBy: ['id'] }); + const r = await cmp.compare([src], [tgt]); + expect(r.tables[0].status).toBe('MODIFIED'); + }); + + it('does not mark other dialects modified for missing engine / ORDER BY', async () => { + const t = table({ name: 'A', columns: [col('ID')] }); + const r = await cmp.compare([t], [structuredClone(t)]); + expect(r.tables[0].status).toBe('UNCHANGED'); + }); + it('detects a sequence attribute change', async () => { const src = table({ name: 'S', objectType: 'SEQUENCE', sequence: { increment: '1' } }); const tgt = table({ name: 'S', objectType: 'SEQUENCE', sequence: { increment: '10' } }); diff --git a/packages/sql/src/modules/schema-diff/compare.module.ts b/packages/sql/src/modules/schema-diff/compare.module.ts index 1d090d7fc..c39cc9e20 100644 --- a/packages/sql/src/modules/schema-diff/compare.module.ts +++ b/packages/sql/src/modules/schema-diff/compare.module.ts @@ -107,6 +107,9 @@ export class CompareModule { const triggerDiffs = this.compareTriggers(source.triggers ?? [], target.triggers ?? []); const pkChanged = !this.sameColumns(source.primaryKey?.columns, target.primaryKey?.columns); + const engineChanged = + (source.engine ?? '').toLowerCase() !== (target.engine ?? '').toLowerCase(); + const orderByChanged = !this.sameColumns(source.orderBy, target.orderBy); // Sequences and user-defined types carry their state in dedicated fields const sequenceChanged = JSON.stringify(source.sequence ?? {}) !== JSON.stringify(target.sequence ?? {}); @@ -120,6 +123,8 @@ export class CompareModule { foreignKeyDiffs.some((d) => d.status !== 'UNCHANGED') || triggerDiffs.some((d) => d.status !== 'UNCHANGED') || pkChanged || + engineChanged || + orderByChanged || sequenceChanged || userTypeChanged || this.normalizeDefinition(source.definition) !== this.normalizeDefinition(target.definition); diff --git a/packages/sql/src/providers/DIALECTS.md b/packages/sql/src/providers/DIALECTS.md index 00b74b447..bfe7ddb9c 100644 --- a/packages/sql/src/providers/DIALECTS.md +++ b/packages/sql/src/providers/DIALECTS.md @@ -21,6 +21,10 @@ Each dialect lives in `providers//.sql-dialect.ts`. | `serialSequenceFromDefault` | no pre-created sequence | serial columns reference a backing sequence (Postgres) | | `dropDependentViewsBlock` / `recreateDependentViewsBlock` | none | ALTER/DROP COLUMN fails while a view depends on the column (Postgres) | | `nullableTypeWrapper` | append `NULL`/`NOT NULL` | nullability is encoded in the type (ClickHouse `Nullable(T)`) | +| `inlinePrimaryKeyConstraint` | emit `PRIMARY KEY (cols)` inside CREATE TABLE | engine rejects a table-level PK (ClickHouse: only CHECK/ASSUME there) | +| `createTableSuffix` | none | clauses after the column list — ClickHouse `ENGINE = MergeTree ORDER BY …` | +| `addPrimaryKeyStatements` | `ALTER TABLE t ADD [CONSTRAINT] PRIMARY KEY (cols)` | engine cannot ADD PRIMARY KEY (ClickHouse: `-- review:` recreate) | +| `engineChangeStatements` | none | ENGINE / ORDER BY changed and cannot ALTER in place (ClickHouse) | | `quoteIdentifier` | ANSI `"name"`, embedded `"` doubled | engine uses another wrapper — MySQL/MariaDB/TiDB backticks, SQL Server/Azure brackets | | `roleMemberStatement` | `GRANT role TO member;` / `REVOKE role FROM member;` | engine does not take the standard form — Db2 names the member kind, SQL Server uses `ALTER ROLE … ADD MEMBER` | | `postColumnChangeStatements` | none | table needs maintenance after column changes before its keys/indexes are rebuilt (DB2 `REORG`) | @@ -147,6 +151,23 @@ Each dialect lives in `providers//.sql-dialect.ts`. ### ClickHouse (`clickhouse.sql-dialect.ts`) - `nullableTypeWrapper` ⇒ `Nullable(T)`; generator skips `NULL`/`NOT NULL` keywords. +- **No table-level `PRIMARY KEY` constraint.** ClickHouse only allows `CHECK` / + `ASSUME` inside the column list; `CONSTRAINT PRIMARY_KEY PRIMARY KEY (id)` is + `Syntax error … Expected one of: CHECK, ASSUME`. `inlinePrimaryKeyConstraint` + is false. Logical PK columns (and captured `orderBy`) become + `ENGINE = MergeTree ORDER BY id` via `createTableSuffix`. A composite logical + PK (`order_id, coupon_id`) is `ORDER BY (order_id, coupon_id)` — ClickHouse + MergeTree supports multi-column keys that way, not as a table constraint. + When the logical PK is a prefix of a longer `orderBy`, ClickHouse also gets + `PRIMARY KEY id` on the engine clause (not inside the column list). A captured + `engine` (e.g. `ReplacingMergeTree`) is used as-is; no key → `ORDER BY tuple()`. +- Changing PK / ENGINE / ORDER BY in place is impossible — `dropPrimaryKeyStatements`, + `addPrimaryKeyStatements`, and `engineChangeStatements` emit `-- review:` to + recreate the table, never `ALTER TABLE … ADD PRIMARY KEY`. +- Provider reads `system.tables.engine / sorting_key / primary_key` plus + `system.columns.is_in_primary_key / is_in_sorting_key`. Compare treats engine + and ORDER BY as table-level changes; other dialects omit those fields and + stay UNCHANGED. - No traditional indexes/triggers/FKs — those hooks are absent by design. - No transactions: the adapter's begin/commit/rollback are no-ops, so `clickhouse.settings.ts` sets `transactions: false` (as Redis and MongoDB do). diff --git a/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.test.ts b/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.test.ts new file mode 100644 index 000000000..f9a702f38 --- /dev/null +++ b/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from 'vitest'; +import { clickHouseSqlDialect, parseClickHouseKeyList } from './clickhouse.sql-dialect.js'; +import type { TableSchema } from '../../interfaces/index.js'; + +describe('parseClickHouseKeyList', () => { + it('splits a comma-separated sorting key', () => { + expect(parseClickHouseKeyList('id, created_at')).toEqual(['id', 'created_at']); + }); + + it('keeps a nested function as one expression', () => { + expect(parseClickHouseKeyList('cityHash64(id, 1), created_at')).toEqual([ + 'cityHash64(id, 1)', + 'created_at', + ]); + }); + + it('unwraps a parenthesized catalog list', () => { + expect(parseClickHouseKeyList('(id, created_at)')).toEqual(['id', 'created_at']); + expect(parseClickHouseKeyList('(cityHash64(id), created_at)')).toEqual([ + 'cityHash64(id)', + 'created_at', + ]); + }); + + it('treats tuple() as a single expression', () => { + expect(parseClickHouseKeyList('tuple()')).toEqual(['tuple()']); + }); + + it('returns empty for a blank catalog value', () => { + expect(parseClickHouseKeyList('')).toEqual([]); + expect(parseClickHouseKeyList(undefined)).toEqual([]); + }); +}); + +describe('clickHouseSqlDialect CREATE TABLE suffix', () => { + const table = (over: Partial = {}): TableSchema => ({ + name: 'coupons', + objectType: 'TABLE', + columns: [], + indices: [], + foreignKeys: [], + ...over, + }); + + it('defaults to MergeTree ORDER BY the quoted key', () => { + expect(clickHouseSqlDialect.createTableSuffix?.(table(), ['id'])).toBe( + ' ENGINE = MergeTree ORDER BY id' + ); + }); + + it('wraps a composite ORDER BY', () => { + expect(clickHouseSqlDialect.createTableSuffix?.(table({ engine: 'MergeTree' }), ['id', 'code'])).toBe( + ' ENGINE = MergeTree ORDER BY (id, code)' + ); + }); + + it('uses tuple() when there is no key', () => { + expect(clickHouseSqlDialect.createTableSuffix?.(table(), [])).toBe( + ' ENGINE = MergeTree ORDER BY tuple()' + ); + }); + + it('does not inline a PRIMARY KEY constraint', () => { + expect(clickHouseSqlDialect.inlinePrimaryKeyConstraint).toBe(false); + }); + + it('emits an engine PRIMARY KEY only when it differs from ORDER BY', () => { + const t = table({ + primaryKey: { name: 'PRIMARY_KEY', columns: ['id'] }, + orderBy: ['id', 'code'], + }); + expect(clickHouseSqlDialect.createTableSuffix?.(t, ['id', 'code'])).toBe( + ' ENGINE = MergeTree PRIMARY KEY id ORDER BY (id, code)' + ); + }); + + it('treats ENGINE = MergeTree as the same engine as MergeTree', () => { + const src = table({ engine: 'ENGINE = MergeTree', orderBy: ['id'] }); + const tgt = table({ engine: 'MergeTree', orderBy: ['id'] }); + expect(clickHouseSqlDialect.engineChangeStatements?.('coupons', src, tgt)).toEqual([]); + }); +}); diff --git a/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.ts b/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.ts index 6a08f1f76..009e50afc 100644 --- a/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.ts +++ b/packages/sql/src/providers/clickHouse/clickhouse.sql-dialect.ts @@ -1,4 +1,5 @@ import type { SqlDialect, ColumnSpec } from '../../modules/dialect/sql-dialect.interface.js'; +import type { TableSchema } from '../../interfaces/index.js'; import { makeDialectTypeFns, plain, sized, decimalAs, tokenizeType } from '../../modules/dialect/type-mapping.js'; // ClickHouse type names are case-sensitive. Lowercase the token for parseMap lookup. @@ -69,6 +70,81 @@ const CH_DECIMAL_PRECISION: Record = { decimal256: 76, }; +function unwrapOuterParens(raw: string): string { + let s = raw.trim(); + while (s.startsWith('(') && s.endsWith(')') && s.length >= 2) { + let depth = 0; + let wrapsEntire = true; + for (let i = 0; i < s.length; i++) { + const ch = s[i]!; + if (ch === '(') depth += 1; + else if (ch === ')') { + depth -= 1; + if (depth === 0 && i !== s.length - 1) { + wrapsEntire = false; + break; + } + } + } + if (!wrapsEntire || depth !== 0) break; + s = s.slice(1, -1).trim(); + } + return s; +} + +/** + * Split a ClickHouse sorting_key / primary_key catalog string on top-level + * commas. `id, created_at` → two columns; `cityHash64(id, 1)` stays one + * expression so a nested comma does not become a fake column. + */ +export function parseClickHouseKeyList(raw: string | null | undefined): string[] { + if (!raw) return []; + const out: string[] = []; + let current = ''; + let depth = 0; + for (const ch of unwrapOuterParens(raw)) { + if (ch === '(') depth += 1; + else if (ch === ')') depth = Math.max(0, depth - 1); + if (ch === ',' && depth === 0) { + const piece = current.trim(); + if (piece) out.push(piece); + current = ''; + continue; + } + current += ch; + } + const last = current.trim(); + if (last) out.push(last); + return out; +} + +function clickHouseEngineName(engine: string | undefined): string { + const stripped = (engine ?? '').trim().replace(/^ENGINE\s*=\s*/i, '').trim(); + return stripped || 'MergeTree'; +} + +function clickHouseEngineSql(engine: string | undefined): string { + return `ENGINE = ${clickHouseEngineName(engine)}`; +} + +function clickHouseOrderBy(table: TableSchema): string[] { + return table.orderBy?.length ? table.orderBy : table.primaryKey?.columns ?? []; +} + +function clickHouseOrderBySql(quotedOrderBy: string[]): string { + if (quotedOrderBy.length === 0) return 'tuple()'; + if (quotedOrderBy.length === 1) return quotedOrderBy[0]!; + return `(${quotedOrderBy.join(', ')})`; +} + +function sameKeyList(a: string[] = [], b: string[] = []): boolean { + return a.length === b.length && a.every((c, i) => c.toUpperCase() === (b[i] ?? '').toUpperCase()); +} + +function clickHousePkReview(tableName: string): string[] { + return [`-- review: ${tableName}: ClickHouse PRIMARY KEY is part of the ENGINE clause and cannot be altered in place. Recreate the table to change the key.`]; +} + function parseClickHouseType(raw: string) { const parsed = types.parseType(raw); const tok = tokenizeType(raw); @@ -113,8 +189,38 @@ export const clickHouseSqlDialect: SqlDialect = { }, dropPrimaryKeyStatements(tableName: string, _pkName: string | undefined): string[] { - // ClickHouse PRIMARY KEY is embedded in the ENGINE clause — it cannot be dropped separately. - return [`-- review: ${tableName}: ClickHouse PRIMARY KEY is part of the ENGINE clause and cannot be altered in place. Recreate the table to change the key.`]; + return clickHousePkReview(tableName); + }, + + addPrimaryKeyStatements(tableName: string, _quotedColumns: string[], _pkName: string | undefined): string[] { + return clickHousePkReview(tableName); + }, + + // Table-level PRIMARY KEY (cols) is not valid inside the column list — only + // CHECK / ASSUME. The key is ENGINE … ORDER BY after the closing paren. + inlinePrimaryKeyConstraint: false, + + createTableSuffix(table: TableSchema, quotedOrderBy: string[]): string { + const pkCols = table.primaryKey?.columns ?? []; + // ClickHouse PRIMARY KEY defaults to ORDER BY; only emit it when they differ. + const pkClause = + pkCols.length > 0 && !sameKeyList(pkCols, clickHouseOrderBy(table)) + ? ` PRIMARY KEY ${clickHouseOrderBySql(pkCols)}` + : ''; + return ` ${clickHouseEngineSql(table.engine)}${pkClause} ORDER BY ${clickHouseOrderBySql(quotedOrderBy)}`; + }, + + engineChangeStatements(tableName: string, source: TableSchema, target: TableSchema): string[] { + const srcOrder = clickHouseOrderBy(source); + if ( + clickHouseEngineName(source.engine).toLowerCase() === clickHouseEngineName(target.engine).toLowerCase() && + sameKeyList(srcOrder, clickHouseOrderBy(target)) + ) { + return []; + } + return [ + `-- review: ${tableName}: ClickHouse ENGINE / ORDER BY cannot be altered in place. Recreate the table to apply ${clickHouseEngineSql(source.engine)} ORDER BY ${clickHouseOrderBySql(srcOrder)}.`, + ]; }, // ClickHouse uses Nullable(T) wrapper instead of NULL / NOT NULL keywords.