diff --git a/.changeset/map-keys-text-index-fallthrough.md b/.changeset/map-keys-text-index-fallthrough.md new file mode 100644 index 0000000000..ed38a7ea22 --- /dev/null +++ b/.changeset/map-keys-text-index-fallthrough.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +fix: fall back to the Map key scan when the text index read fails + +`getMapKeys` returned an empty list whenever its `mergeTreeTextIndex` query +failed, so the rollup and bounded `mapKeys` scan paths never ran. ClickHouse +refuses `mergeTreeTextIndex` with `ACCESS_DENIED` on any table with a row +policy, which left search autocomplete with no Map keys on those tables. A +failed text index read now falls through to the next strategy. After a row +policy denial it skips the key rollup table, which the policy doesn't cover, +and reads keys only from the policy-filtered source table. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 354d1e4c6b..f3050f662e 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2216,6 +2216,242 @@ describe('Metadata', () => { expect(keysA).toEqual(['a']); expect(keysB).toEqual(['b']); }); + + describe('when the mergeTreeTextIndex query fails', () => { + // Shaped like ClickhouseClient.query's wrapped error: message plus the + // client's ClickHouseError as `cause`. + const accessDenied = new Error( + 'Cannot read from `mergeTreeTextIndex` because a row policy is applied on table default.otel_logs', + { cause: { type: 'ACCESS_DENIED', code: '497' } }, + ); + const distributedError = new Error( + 'Table function mergeTreeTextIndex expects a MergeTree table, got Distributed', + { cause: { type: 'BAD_ARGUMENTS', code: '36' } }, + ); + const metadataMVs = { + keyRollupTable: 'otel_logs_key_rollup_15m', + kvRollupTable: 'otel_logs_kv_rollup_15m', + granularity: '15 minute' as const, + }; + const keyIndexLookup: TextIndexInfoLookup = new Map([ + [ + 'LogAttributes', + { + key: { indexName: 'idx_log_attr_key', mapColumn: 'LogAttributes' }, + }, + ], + ]); + const mockScanResponses = (mock: jest.Mock) => + mock + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), + }) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + const args = { + databaseName: 'default', + tableName: 'otel_logs', + column: 'LogAttributes', + connectionId: 'conn-1', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2026-05-11T16:00:00Z'), + new Date('2026-05-11T17:00:00Z'), + ] as [Date, Date], + }; + + const buildWithTextIndex = (lookup: TextIndexInfoLookup) => { + const md = new Metadata(mockClickhouseClient, new MetadataCache()); + jest.spyOn(md, 'getServerVersion').mockResolvedValue([26, 3, 0, 0]); + jest.spyOn(md, 'getMapColumnTextIndexes').mockResolvedValue(lookup); + return md; + }; + + const queries = () => + (mockClickhouseClient.query as jest.Mock).mock.calls.map( + ([opts]) => opts.query as string, + ); + + let warn: jest.SpyInstance; + beforeEach(() => { + warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { + warn.mockRestore(); + }); + + it('falls through to the bounded scan when the key text index read fails', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + + const keys = await md.getMapKeys(args); + + expect(keys).toEqual(['http.method']); + expect(queries()[0]).toContain('mergeTreeTextIndex('); + expect(queries().at(-1)).toContain('sampledKeys'); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('key text index query failed'), + accessDenied, + ); + }); + + it('falls through to the bounded scan when the kv text index read fails', async () => { + const md = buildWithTextIndex( + new Map([ + [ + 'LogAttributes', + { + kv: { + columnName: 'LogAttributes', + indexName: 'idx_log_attr_kv', + mapColumn: 'LogAttributes', + separator: '=', + useHasAny: true, + }, + }, + ], + ]), + ); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + + const keys = await md.getMapKeys({ ...args, metadataMVs }); + + expect(keys).toEqual(['http.method']); + expect(queries()[0]).toContain('mergeTreeTextIndex('); + expect(queries().some(q => q.includes('ColumnIdentifier'))).toBe(false); + expect(queries().at(-1)).toContain('sampledKeys'); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('kv text index query failed'), + accessDenied, + ); + }); + + it('skips the rollup table after a row-policy denial and scans the source table', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + + const keys = await md.getMapKeys({ ...args, metadataMVs }); + + expect(keys).toEqual(['http.method']); + expect(queries().some(q => q.includes('ColumnIdentifier'))).toBe(false); + expect(queries().at(-1)).toContain('sampledKeys'); + }); + + it('detects a row-policy denial from the message when the error has no cause', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + new Error(accessDenied.message), + ), + ); + + await md.getMapKeys({ ...args, metadataMVs }); + + expect(queries().some(q => q.includes('ColumnIdentifier'))).toBe(false); + }); + + it('still tries the rollup table for failures that are not access denials', async () => { + const md = buildWithTextIndex(keyIndexLookup); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce(distributedError) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ Key: 'user.id' }] }), + }); + + const keys = await md.getMapKeys({ ...args, metadataMVs }); + + expect(keys).toEqual(['user.id']); + expect(queries()).toHaveLength(2); + expect(queries()[1]).toContain('ColumnIdentifier = '); + }); + + it('bounds the text index read and forwards the abort signal', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + const signal = new AbortController().signal; + + await md.getMapKeys({ ...args, signal }); + + const [first] = (mockClickhouseClient.query as jest.Mock).mock.calls[0]; + expect(first.clickhouse_settings).toMatchObject({ + max_execution_time: 15, + timeout_overflow_mode: 'throw', + }); + expect(first.abort_signal).toBe(signal); + }); + + it('falls through to the rollup when the text index read times out', async () => { + const md = buildWithTextIndex(keyIndexLookup); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce( + new Error('Timeout exceeded: elapsed 15 seconds', { + cause: { type: 'TIMEOUT_EXCEEDED', code: '159' }, + }), + ) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ Key: 'user.id' }] }), + }); + + const keys = await md.getMapKeys({ ...args, metadataMVs }); + + expect(keys).toEqual(['user.id']); + expect(queries()[1]).toContain('ColumnIdentifier = '); + }); + + it('rethrows an aborted text index read without running or caching the fallback', async () => { + const md = buildWithTextIndex(keyIndexLookup); + const controller = new AbortController(); + const abortError = new Error('The user aborted a request.'); + (mockClickhouseClient.query as jest.Mock).mockImplementationOnce(() => { + controller.abort(); + return Promise.reject(abortError); + }); + + await expect( + md.getMapKeys({ ...args, metadataMVs, signal: controller.signal }), + ).rejects.toBe(abortError); + expect(queries()).toHaveLength(1); + + (mockClickhouseClient.query as jest.Mock).mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + expect(await md.getMapKeys({ ...args, metadataMVs })).toEqual([ + 'http.method', + ]); + expect(queries()[1]).toContain('mergeTreeTextIndex('); + }); + + it('caches the fallback result so the failing index read is not retried', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + + await md.getMapKeys(args); + const callsAfterFirst = queries().length; + expect(await md.getMapKeys(args)).toEqual(['http.method']); + expect(queries()).toHaveLength(callsAfterFirst); + }); + }); }); // Regression guard for commit 612bb2f9a: the `keyRollupTable` MV is no diff --git a/packages/common-utils/src/clickhouse/__tests__/index.test.ts b/packages/common-utils/src/clickhouse/__tests__/index.test.ts index 02f5030fb8..69e70a306d 100644 --- a/packages/common-utils/src/clickhouse/__tests__/index.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/index.test.ts @@ -2,11 +2,44 @@ import { ClickHouseQueryError, convertCHDataTypeToJSType, extractColumnReferencesFromKey, + isAccessDeniedError, isMissingColumnError, JSDataType, } from '@/clickhouse'; import { ClickhouseClient } from '@/clickhouse/node'; +describe('isAccessDeniedError', () => { + const withCause = (cause: object) => + Object.assign(new ClickHouseQueryError('query failed', 'SELECT 1'), { + cause, + }); + + it('detects ACCESS_DENIED from the client error type', () => { + expect(isAccessDeniedError(withCause({ type: 'ACCESS_DENIED' }))).toBe( + true, + ); + }); + + it('detects ACCESS_DENIED from the numeric code', () => { + expect(isAccessDeniedError(withCause({ code: '497' }))).toBe(true); + }); + + it.each([ + 'Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)', + 'Cannot read from `mergeTreeTextIndex` because a row policy is applied on table default.otel_logs', + ])('detects ACCESS_DENIED from the message: %s', msg => { + expect(isAccessDeniedError(new Error(msg))).toBe(true); + }); + + it('returns false for other errors', () => { + expect( + isAccessDeniedError(withCause({ type: 'BAD_ARGUMENTS', code: '36' })), + ).toBe(false); + expect(isAccessDeniedError(new Error('Code: 4970. Timeout'))).toBe(false); + expect(isAccessDeniedError(undefined)).toBe(false); + }); +}); + describe('isMissingColumnError', () => { it.each([ 'Code: 47. DB::Exception: Unknown identifier: foo', diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index fd13c86d16..b533941d03 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -335,6 +335,18 @@ export function isMissingColumnError(error: unknown): boolean { ); } +/** ClickHouse ACCESS_DENIED (497), e.g. a row policy blocking mergeTreeTextIndex. */ +export function isAccessDeniedError(error: unknown): boolean { + const cause = error instanceof Error ? error.cause : undefined; + if (cause != null && typeof cause === 'object') { + const type = 'type' in cause ? cause.type : undefined; + const code = 'code' in cause ? cause.code : undefined; + if (type === 'ACCESS_DENIED' || String(code) === '497') return true; + } + const msg = error instanceof Error ? error.message : String(error ?? ''); + return /ACCESS_DENIED|Code: 497\b|row policy is applied/i.test(msg); +} + /** * Returns columns referenced in given expression, where the expression is a comma-separated list of SQL expressions * E.g. "id, toStartOfInterval(timestamp, toIntervalDay(3)), user_id, json.a.b". diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 06ef78269d..dcfa524535 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -11,6 +11,7 @@ import { convertCHDataTypeToJSType, extractColumnReferencesFromKey, filterColumnMetaByType, + isAccessDeniedError, JSDataType, Row, streamToAsyncIterator, @@ -994,6 +995,15 @@ export class Metadata { supportsMergeTreeTextIndex(clickhouseVersion); // Text Index path: query the key rollup index const textIndexInfo = textIndexInfoLookup.get(column); + const textIndexQuerySettings: ClickHouseSettings = { + ...this.getClickHouseSettings(), + // 'throw', not 'break': a partial key list would be cached without a + // TTL. On timeout the rollup/scan paths below take over. + max_execution_time: 15, + timeout_overflow_mode: 'throw', + }; + // Set when the text index read hits a row policy; see the rollup guard. + let rowPolicyDenied = false; // Without timestampValueExpression, partsFilter can't bound this either. #3037 if ( textIndexInfo?.key?.indexName && @@ -1019,7 +1029,8 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: this.getClickHouseSettings(), + clickhouse_settings: textIndexQuerySettings, + abort_signal: signal, }) .then(r => r.json<{ key: string }>()) .then(d => d.data.map(r => r.key).filter(Boolean)); @@ -1028,11 +1039,16 @@ export class Metadata { return keys; } } catch (e) { + // The caller gave up; don't start the fallback with a dead signal and + // cache its instant rejection under the rollup key. + if (signal?.aborted) throw e; + // e.g. ACCESS_DENIED under a row policy or BAD_ARGUMENTS on a + // Distributed table; the paths below still work. + rowPolicyDenied = isAccessDeniedError(e); console.warn( - 'getMapKeys rollup query failed for key text index query', + 'getMapKeys key text index query failed; falling through to the next strategy', e, ); - return []; } } else if ( textIndexInfo?.kv?.indexName && @@ -1059,7 +1075,8 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: this.getClickHouseSettings(), + clickhouse_settings: textIndexQuerySettings, + abort_signal: signal, }) .then(r => r.json<{ key: string }>()) .then(d => d.data.map(r => r.key).filter(Boolean)); @@ -1068,16 +1085,20 @@ export class Metadata { return keys; } } catch (e) { + // See the key text index catch above. + if (signal?.aborted) throw e; + rowPolicyDenied = isAccessDeniedError(e); console.warn( - 'getMapKeys rollup query failed for kv text index query', + 'getMapKeys kv text index query failed; falling through to the next strategy', e, ); - return []; } } - // Rollup path: query the key rollup table filtered by ColumnIdentifier and date range - if (metadataMVs && alignedDateRange) { + // Rollup path: query the key rollup table filtered by ColumnIdentifier and date range. + // Rollups are separate tables the source's row policy doesn't cover, so + // after a policy denial only the source-table scan below is safe. + if (metadataMVs && alignedDateRange && !rowPolicyDenied) { // Own cache key: the shipped OTel rollups only index NativeColumn, so // Map columns come back empty here and must fall through to the bounded // scan below. Caching [] under cacheKey would block that fallback.