diff --git a/.changeset/fixedstring-implicit-hastoken.md b/.changeset/fixedstring-implicit-hastoken.md new file mode 100644 index 0000000000..3826b27edb --- /dev/null +++ b/.changeset/fixedstring-implicit-hastoken.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +fix: cast FixedString columns before implicit hasToken search + +A bare search term against a FixedString column such as OTel TraceId compiled to +hasToken(lower(TraceId), ...), which ClickHouse rejects. Those haystacks are now +CAST to String first. diff --git a/packages/common-utils/src/__tests__/clickhouse.test.ts b/packages/common-utils/src/__tests__/clickhouse.test.ts index f8e6551272..86c81c5d99 100644 --- a/packages/common-utils/src/__tests__/clickhouse.test.ts +++ b/packages/common-utils/src/__tests__/clickhouse.test.ts @@ -2,6 +2,7 @@ import { ChSql, chSqlToAliasMap, convertCHDataTypeToJSType, + isCHFixedStringType, JSDataType, } from '@/clickhouse'; import { ClickhouseClient } from '@/clickhouse/node'; @@ -141,6 +142,20 @@ describe('convertCHDataTypeToJSType - unit - type', () => { }); }); +describe('isCHFixedStringType', () => { + it('matches FixedString and wrappers, and ignores String', () => { + expect(isCHFixedStringType('FixedString(32)')).toBe(true); + expect(isCHFixedStringType('Nullable(FixedString(32))')).toBe(true); + expect(isCHFixedStringType('LowCardinality(FixedString(16))')).toBe(true); + expect( + isCHFixedStringType('LowCardinality(Nullable(FixedString(32)))'), + ).toBe(true); + expect(isCHFixedStringType('String')).toBe(false); + expect(isCHFixedStringType('Nullable(String)')).toBe(false); + expect(isCHFixedStringType('LowCardinality(String)')).toBe(false); + }); +}); + describe('chSqlToAliasMap - alias unit test', () => { it('No alias', () => { const chSqlInput: ChSql = { diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index c3f897b3db..85b7b50971 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -2039,6 +2039,269 @@ describe('CustomSchemaSQLSerializerV2 - text indices', () => { }); }); +describe('CustomSchemaSQLSerializerV2 - FixedString implicit search', () => { + const metadata = getMetadata( + new ClickhouseClient({ host: 'http://localhost:8123' }), + ); + metadata.getSkipIndices = jest.fn().mockResolvedValue([]); + metadata.getSetting = jest.fn().mockResolvedValue(undefined); + metadata.getColumns = jest.fn().mockResolvedValue([]); + metadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockResolvedValue(new Map()); + metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => { + if (column === 'TraceId') { + return { name: 'TraceId', type: 'FixedString(32)' }; + } + if (column === 'NullableTraceId') { + return { + name: 'NullableTraceId', + type: 'Nullable(FixedString(32))', + }; + } + if (column === 'Body') { + return { name: 'Body', type: 'String' }; + } + if (column === 'LcTraceId') { + return { + name: 'LcTraceId', + type: 'LowCardinality(FixedString(32))', + }; + } + return undefined; + }); + + const databaseName = 'default'; + const tableName = 'otel_traces'; + const connectionId = 'test'; + + it('casts a FixedString implicit column before hasToken', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'TraceId', + }); + const sql = await new SearchQueryBuilder( + '917b9785d3414031a4790e263753e43b', + serializer, + ).build(); + expect(sql).toBe( + "((hasToken(lower(CAST(TraceId AS String)), lower('917b9785d3414031a4790e263753e43b'))))", + ); + }); + + it('casts a double-quoted FixedString identifier', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: '"TraceId"', + }); + const sql = await new SearchQueryBuilder('abc', serializer).build(); + expect(sql).toBe( + '((hasToken(lower(CAST("TraceId" AS String)), lower(\'abc\'))))', + ); + }); + + it('casts a quoted FixedString identifier without dropping the quotes', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: '`TraceId`', + }); + const sql = await new SearchQueryBuilder('abc', serializer).build(); + expect(sql).toBe( + "((hasToken(lower(CAST(`TraceId` AS String)), lower('abc'))))", + ); + }); + + it('casts Nullable(FixedString) implicit columns', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'NullableTraceId', + }); + const sql = await new SearchQueryBuilder('abc', serializer).build(); + expect(sql).toBe( + "((hasToken(lower(CAST(NullableTraceId AS String)), lower('abc'))))", + ); + }); + + it('casts a FixedString bodyExpression when implicitColumnExpression is unset', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + bodyExpression: 'TraceId', + }); + const sql = await new SearchQueryBuilder('abc', serializer).build(); + expect(sql).toBe( + "((hasToken(lower(CAST(TraceId AS String)), lower('abc'))))", + ); + }); + + it('leaves String implicit columns on hasToken(lower(column))', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'Body', + }); + const sql = await new SearchQueryBuilder('foo', serializer).build(); + expect(sql).toBe("((hasToken(lower(Body), lower('foo'))))"); + }); + + it('does not cast a multi-column implicit expression', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'Body, TraceId', + }); + const sql = await new SearchQueryBuilder('foo', serializer).build(); + expect(sql).toBe( + "((hasToken(lower(concatWithSeparator(';',Body,TraceId)), lower('foo'))))", + ); + }); + + it('keeps an explicit FixedString field search on ILIKE', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'Body', + }); + const sql = await new SearchQueryBuilder( + 'TraceId:917b9785d3414031a4790e263753e43b', + serializer, + ).build(); + expect(sql).toBe("((TraceId ILIKE '%917b9785d3414031a4790e263753e43b%'))"); + expect(sql).not.toContain('hasToken'); + }); + + it('casts each hasToken of a multi-token term and leaves LIKE on the raw column', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'TraceId', + }); + const sql = await new SearchQueryBuilder('"foo bar"', serializer).build(); + expect(sql).toBe( + "((hasToken(lower(CAST(TraceId AS String)), lower('foo')) AND hasToken(lower(CAST(TraceId AS String)), lower('bar')) AND (lower(TraceId) LIKE lower('%foo bar%'))))", + ); + }); + + it('keeps the raw column when text-index search is forced on', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'TraceId', + useTextIndexForImplicitColumn: UseTextIndex.Enabled, + }); + const sql = await new SearchQueryBuilder('foo', serializer).build(); + expect(sql).toBe("((hasAllTokens(TraceId, 'foo')))"); + }); + + it('keeps a FixedString text index on the raw column', async () => { + const getSkipIndices = metadata.getSkipIndices; + const getSetting = metadata.getSetting; + metadata.getSkipIndices = jest.fn().mockResolvedValue([ + { + name: 'idx_trace_text', + type: 'text', + typeFull: 'text(tokenizer=splitByNonAlpha)', + expression: 'TraceId', + granularity: '1', + }, + ]); + metadata.getSetting = jest + .fn() + .mockImplementation(async ({ settingName }) => { + if (settingName === 'enable_full_text_index') { + return '1'; + } + return undefined; + }); + try { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'TraceId', + }); + const sql = await new SearchQueryBuilder('foo', serializer).build(); + expect(sql).toBe("((hasAllTokens(TraceId, 'foo')))"); + } finally { + metadata.getSkipIndices = getSkipIndices; + metadata.getSetting = getSetting; + } + }); + + it('casts a LowCardinality(FixedString) implicit column before hasToken', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'LcTraceId', + }); + const sql = await new SearchQueryBuilder('abc', serializer).build(); + expect(sql).toBe( + "((hasToken(lower(CAST(LcTraceId AS String)), lower('abc'))))", + ); + }); + + it('casts inside a negated implicit FixedString search', async () => { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'TraceId', + }); + const sql = await new SearchQueryBuilder('-abc', serializer).build(); + expect(sql).toBe( + "((NOT hasToken(lower(CAST(TraceId AS String)), lower('abc'))))", + ); + }); + + it('falls back to uncast hasToken when the column lookup fails', async () => { + const getColumn = metadata.getColumn; + metadata.getColumn = jest + .fn() + .mockRejectedValue(new Error('describe failed')); + try { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'TraceId', + }); + const sql = await new SearchQueryBuilder('abc', serializer).build(); + expect(sql).toBe("((hasToken(lower(TraceId), lower('abc'))))"); + } finally { + metadata.getColumn = getColumn; + } + }); +}); + describe('CustomSchemaSQLSerializerV2 - indexCoversColumn', () => { const metadata = getMetadata( new ClickhouseClient({ host: 'http://localhost:8123' }), diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 460c997c62..3059e74da6 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -11,6 +11,7 @@ import type { import type { ClickHouseClient as WebClickHouseClient } from '@clickhouse/client-web'; import * as SQLParser from 'node-sql-parser'; +import { stripTypeWrappers } from '@/core/eventDeltas'; import { getMetadata, Metadata, @@ -142,6 +143,19 @@ export const convertCHDataTypeToJSType = ( return null; }; +/** + * True when the ClickHouse type is FixedString, including Nullable and + * LowCardinality wrappers. + * + * convertCHDataTypeToJSType maps FixedString to JSDataType.String, which is + * right for search semantics (ILIKE, equality). hasToken rejects a FixedString + * haystack, so that fallback has to CAST the column to String. hasAllTokens + * accepts FixedString and must keep the original column so a text index matches. + */ +export const isCHFixedStringType = (dataType: string): boolean => { + return stripTypeWrappers(dataType).startsWith('FixedString'); +}; + export const isJSDataTypeJSONStringifiable = ( dataType: JSDataType | null | undefined, ) => { diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 61326a599a..e6a34a8e92 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -7,6 +7,7 @@ import { convertCHDataTypeToJSType, convertCHTypeToLuceneSearchType, extractInnerCHArrayJSType, + isCHFixedStringType, JSDataType, } from '@/clickhouse'; import { @@ -19,6 +20,7 @@ import { parseKeyPath, SkipIndexMetadata, TableConnection, + unquoteIdentifier, } from '@/core/metadata'; import { parseTokenizerFromTextIndex, @@ -483,6 +485,11 @@ export abstract class SQLSerializer implements Serializer { column?: string; columnJSON?: { string: string; number: string }; propertyType?: JSDataType; + /** + * The resolved column is FixedString. Token functions (hasToken, + * hasAllTokens) reject that haystack and need CAST(... AS String). + */ + isFixedString?: boolean; isArray?: boolean; found: boolean; mapKey?: string; @@ -1466,10 +1473,18 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { mapKey, mapKeyIndexExpression, arrayMapKeyExpression, + isFixedString, } = await this.getColumnForField(field, context); if (!found) { return this.NOT_FOUND_QUERY; } + // hasToken rejects FixedString; lower(FixedString) stays FixedString, so a + // bare search against TraceId (FixedString(32) in the OTel schema) errors. + // hasAllTokens accepts FixedString and is how a text index is used, so that + // path keeps the raw column. LIKE accepts FixedString too. Bloom-filter + // index expressions are used as stored. + const tokenHaystack = + isFixedString && column ? `CAST(${column} AS String)` : column; const expressionPostfix = mapKeyIndexExpression && !isNegatedField && @@ -1574,6 +1589,8 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { // When the text index is on lower(column), we must pass lower(column) // as the first argument and wrap the tokens in lower() to match. + // Do not CAST here: hasAllTokens accepts FixedString, and CAST would + // no longer match an index built on the column. const hasAllTokensColumn = textIndexHasLower ? `lower(${column})` : column; @@ -1643,7 +1660,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { return `(${isNegatedField ? 'NOT (' : ''}${[ ...tokens.map(token => SqlString.format(`hasToken(lower(?), lower(?))`, [ - SqlString.raw(column), + SqlString.raw(tokenHaystack), token, ]), ), @@ -1656,7 +1673,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { } else { return SqlString.format( `(${isNegatedField ? 'NOT ' : ''}hasToken(lower(?), lower(?)))`, - [SqlString.raw(column), term], + [SqlString.raw(tokenHaystack), term], ); } } @@ -1931,6 +1948,34 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { return false; } + /** + * Whether a single implicit-column expression is a FixedString column. + * Only a bare identifier is looked up. A function wrapper such as + * lower(TraceId) is left as written: concatWithSeparator returns String, + * and guessing the result type of an arbitrary expression is out of scope. + * A failed lookup keeps the previous hasToken SQL. + */ + private async columnExpressionIsFixedString( + expression: string, + ): Promise { + const columnName = unquoteIdentifier(expression.trim()); + if (!columnName || columnName.includes('(')) { + return false; + } + try { + const meta = await this.metadata.getColumn({ + databaseName: this.databaseName, + tableName: this.tableName, + column: columnName, + connectionId: this.connectionId, + }); + return meta != null && isCHFixedStringType(meta.type); + } catch (error) { + console.debug('Error resolving implicit column type', error); + return false; + } + } + async getColumnForField(field: string, context: SerializerContext) { // Fall back to bodyExpression for implicit column expression. // values can be empty if previously configured then removed. @@ -1967,6 +2012,11 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { : fieldFinal, columnJSON: undefined, propertyType: JSDataType.String, + // concatWithSeparator returns String even when an input is FixedString. + isFixedString: + expressions.length === 1 + ? await this.columnExpressionIsFixedString(expressions[0]) + : false, found: true, }; }