From 951c9c5190044deb4d3e7aeb84aca24df2cbcd12 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:14:31 -0700 Subject: [PATCH 1/4] fix(common-utils): fall through to mapKeys scan when text index read fails getMapKeys returned [] whenever its mergeTreeTextIndex query threw, so the rollup and bounded mapKeys scan paths never ran. ClickHouse refuses mergeTreeTextIndex with ACCESS_DENIED on any table with a row policy (ClickHouse/ClickHouse#119032), which left Map key autocomplete empty on those tables. Log and fall through to the next strategy instead. Fixes #3265 Refs HDX-5598 --- .changeset/map-keys-text-index-fallthrough.md | 13 ++ .../src/__tests__/metadata.test.ts | 171 ++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 9 +- 3 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 .changeset/map-keys-text-index-fallthrough.md diff --git a/.changeset/map-keys-text-index-fallthrough.md b/.changeset/map-keys-text-index-fallthrough.md new file mode 100644 index 0000000000..92544150db --- /dev/null +++ b/.changeset/map-keys-text-index-fallthrough.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +fix: fall back to the Map key scan when the text index read fails + +`getMapKeys` returned an empty list whenever its `mergeTreeTextIndex` query +failed, so the rollup and bounded `mapKeys` scan paths never ran. ClickHouse +refuses `mergeTreeTextIndex` with `ACCESS_DENIED` on any table with a row +policy, which left search autocomplete with no Map keys on those tables. A +failed text index read now falls through to the next strategy. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 354d1e4c6b..ee838cd5ce 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2216,6 +2216,177 @@ describe('Metadata', () => { expect(keysA).toEqual(['a']); expect(keysB).toEqual(['b']); }); + + describe('when the mergeTreeTextIndex query fails', () => { + const accessDenied = new Error( + 'Cannot read from `mergeTreeTextIndex` because a row policy is applied on table default.otel_logs', + ); + const args = { + databaseName: 'default', + tableName: 'otel_logs', + column: 'LogAttributes', + connectionId: 'conn-1', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2026-05-11T16:00:00Z'), + new Date('2026-05-11T17:00:00Z'), + ] as [Date, Date], + }; + + const buildWithTextIndex = (lookup: TextIndexInfoLookup) => { + const md = new Metadata(mockClickhouseClient, new MetadataCache()); + jest.spyOn(md, 'getServerVersion').mockResolvedValue([26, 3, 0, 0]); + jest.spyOn(md, 'getMapColumnTextIndexes').mockResolvedValue(lookup); + return md; + }; + + const queries = () => + (mockClickhouseClient.query as jest.Mock).mock.calls.map( + ([opts]) => opts.query as string, + ); + + let warn: jest.SpyInstance; + beforeEach(() => { + warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { + warn.mockRestore(); + }); + + it('falls through to the bounded scan when the key text index read fails', async () => { + const md = buildWithTextIndex( + new Map([ + [ + 'LogAttributes', + { + key: { + indexName: 'idx_log_attr_key', + mapColumn: 'LogAttributes', + }, + }, + ], + ]), + ); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce(accessDenied) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), + }) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + + const keys = await md.getMapKeys(args); + + expect(keys).toEqual(['http.method']); + expect(queries()[0]).toContain('mergeTreeTextIndex('); + expect(queries().at(-1)).toContain('sampledKeys'); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('key text index query failed'), + accessDenied, + ); + }); + + it('falls through to the bounded scan when the kv text index read fails', async () => { + const md = buildWithTextIndex( + new Map([ + [ + 'LogAttributes', + { + kv: { + columnName: 'LogAttributes', + indexName: 'idx_log_attr_kv', + mapColumn: 'LogAttributes', + separator: '=', + useHasAny: true, + }, + }, + ], + ]), + ); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce(accessDenied) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), + }) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + + const keys = await md.getMapKeys(args); + + expect(keys).toEqual(['http.method']); + expect(queries()[0]).toContain('mergeTreeTextIndex('); + expect(queries().at(-1)).toContain('sampledKeys'); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('kv text index query failed'), + accessDenied, + ); + }); + + it('tries the key rollup table before the scan when metadataMVs is configured', async () => { + const md = buildWithTextIndex( + new Map([ + [ + 'LogAttributes', + { + key: { + indexName: 'idx_log_attr_key', + mapColumn: 'LogAttributes', + }, + }, + ], + ]), + ); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce(accessDenied) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ Key: 'user.id' }] }), + }); + + const keys = await md.getMapKeys({ + ...args, + metadataMVs: { + keyRollupTable: 'otel_logs_key_rollup_15m', + kvRollupTable: 'otel_logs_kv_rollup_15m', + granularity: '15 minute', + }, + }); + + expect(keys).toEqual(['user.id']); + expect(queries()).toHaveLength(2); + expect(queries()[1]).toContain('ColumnIdentifier = '); + }); + + it('caches the fallback result so the failing index read is not retried', async () => { + const md = buildWithTextIndex( + new Map([ + [ + 'LogAttributes', + { + key: { + indexName: 'idx_log_attr_key', + mapColumn: 'LogAttributes', + }, + }, + ], + ]), + ); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce(accessDenied) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), + }) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + + await md.getMapKeys(args); + const callsAfterFirst = queries().length; + expect(await md.getMapKeys(args)).toEqual(['http.method']); + expect(queries()).toHaveLength(callsAfterFirst); + }); + }); }); // Regression guard for commit 612bb2f9a: the `keyRollupTable` MV is no diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 06ef78269d..eade3ce9b9 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1028,11 +1028,12 @@ export class Metadata { return keys; } } catch (e) { + // e.g. ACCESS_DENIED under a row policy or BAD_ARGUMENTS on a + // Distributed table; the rollup and scan paths below still work. console.warn( - 'getMapKeys rollup query failed for key text index query', + 'getMapKeys key text index query failed; falling through to the next strategy', e, ); - return []; } } else if ( textIndexInfo?.kv?.indexName && @@ -1068,11 +1069,11 @@ export class Metadata { return keys; } } catch (e) { + // See the key text index catch above. console.warn( - 'getMapKeys rollup query failed for kv text index query', + 'getMapKeys kv text index query failed; falling through to the next strategy', e, ); - return []; } } From 0b85f691865b44e2ceaf7aaa96b66166fb66a125 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:37:02 -0700 Subject: [PATCH 2/4] fix(common-utils): skip key rollup after a row-policy denial and bound text index reads Rollup tables are separate tables that a row policy on the source table does not cover. Once mergeTreeTextIndex fails with ACCESS_DENIED we know a policy applies, so skip the rollup and read keys only from the policy-filtered source-table scan. Other text index failures still try the rollup first. Also cap the text index read at 15s (break on overflow, matching the other getMapKeys strategies) and forward the abort signal, so a slow index read plus the scan fallback can't stack up. Refs HDX-5598 --- .changeset/map-keys-text-index-fallthrough.md | 4 +- .../src/__tests__/metadata.test.ts | 166 ++++++++++-------- .../src/clickhouse/__tests__/index.test.ts | 33 ++++ packages/common-utils/src/clickhouse/index.ts | 12 ++ packages/common-utils/src/core/metadata.ts | 25 ++- 5 files changed, 163 insertions(+), 77 deletions(-) diff --git a/.changeset/map-keys-text-index-fallthrough.md b/.changeset/map-keys-text-index-fallthrough.md index 92544150db..ed38a7ea22 100644 --- a/.changeset/map-keys-text-index-fallthrough.md +++ b/.changeset/map-keys-text-index-fallthrough.md @@ -10,4 +10,6 @@ fix: fall back to the Map key scan when the text index read fails failed, so the rollup and bounded `mapKeys` scan paths never ran. ClickHouse refuses `mergeTreeTextIndex` with `ACCESS_DENIED` on any table with a row policy, which left search autocomplete with no Map keys on those tables. A -failed text index read now falls through to the next strategy. +failed text index read now falls through to the next strategy. After a row +policy denial it skips the key rollup table, which the policy doesn't cover, +and reads keys only from the policy-filtered source table. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index ee838cd5ce..90f29ec567 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2218,9 +2218,37 @@ describe('Metadata', () => { }); describe('when the mergeTreeTextIndex query fails', () => { + // Shaped like ClickhouseClient.query's wrapped error: message plus the + // client's ClickHouseError as `cause`. const accessDenied = new Error( 'Cannot read from `mergeTreeTextIndex` because a row policy is applied on table default.otel_logs', + { cause: { type: 'ACCESS_DENIED', code: '497' } }, ); + const distributedError = new Error( + 'Table function mergeTreeTextIndex expects a MergeTree table, got Distributed', + { cause: { type: 'BAD_ARGUMENTS', code: '36' } }, + ); + const metadataMVs = { + keyRollupTable: 'otel_logs_key_rollup_15m', + kvRollupTable: 'otel_logs_kv_rollup_15m', + granularity: '15 minute' as const, + }; + const keyIndexLookup: TextIndexInfoLookup = new Map([ + [ + 'LogAttributes', + { + key: { indexName: 'idx_log_attr_key', mapColumn: 'LogAttributes' }, + }, + ], + ]); + const mockScanResponses = (mock: jest.Mock) => + mock + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), + }) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); const args = { databaseName: 'default', tableName: 'otel_logs', @@ -2254,27 +2282,12 @@ describe('Metadata', () => { }); it('falls through to the bounded scan when the key text index read fails', async () => { - const md = buildWithTextIndex( - new Map([ - [ - 'LogAttributes', - { - key: { - indexName: 'idx_log_attr_key', - mapColumn: 'LogAttributes', - }, - }, - ], - ]), + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), ); - (mockClickhouseClient.query as jest.Mock) - .mockRejectedValueOnce(accessDenied) - .mockResolvedValueOnce({ - json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), - }) - .mockResolvedValueOnce({ - json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), - }); const keys = await md.getMapKeys(args); @@ -2304,19 +2317,17 @@ describe('Metadata', () => { ], ]), ); - (mockClickhouseClient.query as jest.Mock) - .mockRejectedValueOnce(accessDenied) - .mockResolvedValueOnce({ - json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), - }) - .mockResolvedValueOnce({ - json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), - }); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); - const keys = await md.getMapKeys(args); + const keys = await md.getMapKeys({ ...args, metadataMVs }); expect(keys).toEqual(['http.method']); expect(queries()[0]).toContain('mergeTreeTextIndex('); + expect(queries().some(q => q.includes('ColumnIdentifier'))).toBe(false); expect(queries().at(-1)).toContain('sampledKeys'); expect(warn).toHaveBeenCalledWith( expect.stringContaining('kv text index query failed'), @@ -2324,62 +2335,75 @@ describe('Metadata', () => { ); }); - it('tries the key rollup table before the scan when metadataMVs is configured', async () => { - const md = buildWithTextIndex( - new Map([ - [ - 'LogAttributes', - { - key: { - indexName: 'idx_log_attr_key', - mapColumn: 'LogAttributes', - }, - }, - ], - ]), + it('skips the rollup table after a row-policy denial and scans the source table', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + + const keys = await md.getMapKeys({ ...args, metadataMVs }); + + expect(keys).toEqual(['http.method']); + expect(queries().some(q => q.includes('ColumnIdentifier'))).toBe(false); + expect(queries().at(-1)).toContain('sampledKeys'); + }); + + it('detects a row-policy denial from the message when the error has no cause', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + new Error(accessDenied.message), + ), ); + + await md.getMapKeys({ ...args, metadataMVs }); + + expect(queries().some(q => q.includes('ColumnIdentifier'))).toBe(false); + }); + + it('still tries the rollup table for failures that are not access denials', async () => { + const md = buildWithTextIndex(keyIndexLookup); (mockClickhouseClient.query as jest.Mock) - .mockRejectedValueOnce(accessDenied) + .mockRejectedValueOnce(distributedError) .mockResolvedValueOnce({ json: () => Promise.resolve({ data: [{ Key: 'user.id' }] }), }); - const keys = await md.getMapKeys({ - ...args, - metadataMVs: { - keyRollupTable: 'otel_logs_key_rollup_15m', - kvRollupTable: 'otel_logs_kv_rollup_15m', - granularity: '15 minute', - }, - }); + const keys = await md.getMapKeys({ ...args, metadataMVs }); expect(keys).toEqual(['user.id']); expect(queries()).toHaveLength(2); expect(queries()[1]).toContain('ColumnIdentifier = '); }); + it('bounds the text index read and forwards the abort signal', async () => { + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), + ); + const signal = new AbortController().signal; + + await md.getMapKeys({ ...args, signal }); + + const [first] = (mockClickhouseClient.query as jest.Mock).mock.calls[0]; + expect(first.clickhouse_settings).toMatchObject({ + max_execution_time: 15, + timeout_overflow_mode: 'break', + }); + expect(first.abort_signal).toBe(signal); + }); + it('caches the fallback result so the failing index read is not retried', async () => { - const md = buildWithTextIndex( - new Map([ - [ - 'LogAttributes', - { - key: { - indexName: 'idx_log_attr_key', - mapColumn: 'LogAttributes', - }, - }, - ], - ]), + const md = buildWithTextIndex(keyIndexLookup); + mockScanResponses( + (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( + accessDenied, + ), ); - (mockClickhouseClient.query as jest.Mock) - .mockRejectedValueOnce(accessDenied) - .mockResolvedValueOnce({ - json: () => Promise.resolve({ data: [lowCardinalityMapColumn] }), - }) - .mockResolvedValueOnce({ - json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), - }); await md.getMapKeys(args); const callsAfterFirst = queries().length; diff --git a/packages/common-utils/src/clickhouse/__tests__/index.test.ts b/packages/common-utils/src/clickhouse/__tests__/index.test.ts index 02f5030fb8..69e70a306d 100644 --- a/packages/common-utils/src/clickhouse/__tests__/index.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/index.test.ts @@ -2,11 +2,44 @@ import { ClickHouseQueryError, convertCHDataTypeToJSType, extractColumnReferencesFromKey, + isAccessDeniedError, isMissingColumnError, JSDataType, } from '@/clickhouse'; import { ClickhouseClient } from '@/clickhouse/node'; +describe('isAccessDeniedError', () => { + const withCause = (cause: object) => + Object.assign(new ClickHouseQueryError('query failed', 'SELECT 1'), { + cause, + }); + + it('detects ACCESS_DENIED from the client error type', () => { + expect(isAccessDeniedError(withCause({ type: 'ACCESS_DENIED' }))).toBe( + true, + ); + }); + + it('detects ACCESS_DENIED from the numeric code', () => { + expect(isAccessDeniedError(withCause({ code: '497' }))).toBe(true); + }); + + it.each([ + 'Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)', + 'Cannot read from `mergeTreeTextIndex` because a row policy is applied on table default.otel_logs', + ])('detects ACCESS_DENIED from the message: %s', msg => { + expect(isAccessDeniedError(new Error(msg))).toBe(true); + }); + + it('returns false for other errors', () => { + expect( + isAccessDeniedError(withCause({ type: 'BAD_ARGUMENTS', code: '36' })), + ).toBe(false); + expect(isAccessDeniedError(new Error('Code: 4970. Timeout'))).toBe(false); + expect(isAccessDeniedError(undefined)).toBe(false); + }); +}); + describe('isMissingColumnError', () => { it.each([ 'Code: 47. DB::Exception: Unknown identifier: foo', diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index fd13c86d16..b533941d03 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -335,6 +335,18 @@ export function isMissingColumnError(error: unknown): boolean { ); } +/** ClickHouse ACCESS_DENIED (497), e.g. a row policy blocking mergeTreeTextIndex. */ +export function isAccessDeniedError(error: unknown): boolean { + const cause = error instanceof Error ? error.cause : undefined; + if (cause != null && typeof cause === 'object') { + const type = 'type' in cause ? cause.type : undefined; + const code = 'code' in cause ? cause.code : undefined; + if (type === 'ACCESS_DENIED' || String(code) === '497') return true; + } + const msg = error instanceof Error ? error.message : String(error ?? ''); + return /ACCESS_DENIED|Code: 497\b|row policy is applied/i.test(msg); +} + /** * Returns columns referenced in given expression, where the expression is a comma-separated list of SQL expressions * E.g. "id, toStartOfInterval(timestamp, toIntervalDay(3)), user_id, json.a.b". diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index eade3ce9b9..bb508a74c7 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -11,6 +11,7 @@ import { convertCHDataTypeToJSType, extractColumnReferencesFromKey, filterColumnMetaByType, + isAccessDeniedError, JSDataType, Row, streamToAsyncIterator, @@ -994,6 +995,14 @@ export class Metadata { supportsMergeTreeTextIndex(clickhouseVersion); // Text Index path: query the key rollup index const textIndexInfo = textIndexInfoLookup.get(column); + // Bounded so a slow index read plus the scan fallback can't stack up. + const textIndexQuerySettings: ClickHouseSettings = { + ...this.getClickHouseSettings(), + timeout_overflow_mode: 'break', + max_execution_time: 15, + }; + // Set when the text index read hits a row policy; see the rollup guard. + let rowPolicyDenied = false; // Without timestampValueExpression, partsFilter can't bound this either. #3037 if ( textIndexInfo?.key?.indexName && @@ -1019,7 +1028,8 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: this.getClickHouseSettings(), + clickhouse_settings: textIndexQuerySettings, + abort_signal: signal, }) .then(r => r.json<{ key: string }>()) .then(d => d.data.map(r => r.key).filter(Boolean)); @@ -1029,7 +1039,8 @@ export class Metadata { } } catch (e) { // e.g. ACCESS_DENIED under a row policy or BAD_ARGUMENTS on a - // Distributed table; the rollup and scan paths below still work. + // Distributed table; the paths below still work. + rowPolicyDenied = isAccessDeniedError(e); console.warn( 'getMapKeys key text index query failed; falling through to the next strategy', e, @@ -1060,7 +1071,8 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: this.getClickHouseSettings(), + clickhouse_settings: textIndexQuerySettings, + abort_signal: signal, }) .then(r => r.json<{ key: string }>()) .then(d => d.data.map(r => r.key).filter(Boolean)); @@ -1070,6 +1082,7 @@ export class Metadata { } } catch (e) { // See the key text index catch above. + rowPolicyDenied = isAccessDeniedError(e); console.warn( 'getMapKeys kv text index query failed; falling through to the next strategy', e, @@ -1077,8 +1090,10 @@ export class Metadata { } } - // Rollup path: query the key rollup table filtered by ColumnIdentifier and date range - if (metadataMVs && alignedDateRange) { + // Rollup path: query the key rollup table filtered by ColumnIdentifier and date range. + // Rollups are separate tables the source's row policy doesn't cover, so + // after a policy denial only the source-table scan below is safe. + if (metadataMVs && alignedDateRange && !rowPolicyDenied) { // Own cache key: the shipped OTel rollups only index NativeColumn, so // Map columns come back empty here and must fall through to the bounded // scan below. Caching [] under cacheKey would block that fallback. From 573eb888bdebb6137608f978a4c27c447e36053f Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:43:25 -0700 Subject: [PATCH 3/4] fix(common-utils): drop the text index read timeout timeout_overflow_mode 'break' would return a partial key list as a success and write it to the TTL-less metadata cache, and the pinned max_execution_time overrode the deployment's own setting. Planning-time failures like ACCESS_DENIED don't need it. Keep forwarding the abort signal. Refs HDX-5598 --- packages/common-utils/src/__tests__/metadata.test.ts | 6 +----- packages/common-utils/src/core/metadata.ts | 10 ++-------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 90f29ec567..4a3725d3f5 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2378,7 +2378,7 @@ describe('Metadata', () => { expect(queries()[1]).toContain('ColumnIdentifier = '); }); - it('bounds the text index read and forwards the abort signal', async () => { + it('forwards the abort signal to the text index read', async () => { const md = buildWithTextIndex(keyIndexLookup); mockScanResponses( (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( @@ -2390,10 +2390,6 @@ describe('Metadata', () => { await md.getMapKeys({ ...args, signal }); const [first] = (mockClickhouseClient.query as jest.Mock).mock.calls[0]; - expect(first.clickhouse_settings).toMatchObject({ - max_execution_time: 15, - timeout_overflow_mode: 'break', - }); expect(first.abort_signal).toBe(signal); }); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index bb508a74c7..1062f90f5c 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -995,12 +995,6 @@ export class Metadata { supportsMergeTreeTextIndex(clickhouseVersion); // Text Index path: query the key rollup index const textIndexInfo = textIndexInfoLookup.get(column); - // Bounded so a slow index read plus the scan fallback can't stack up. - const textIndexQuerySettings: ClickHouseSettings = { - ...this.getClickHouseSettings(), - timeout_overflow_mode: 'break', - max_execution_time: 15, - }; // Set when the text index read hits a row policy; see the rollup guard. let rowPolicyDenied = false; // Without timestampValueExpression, partsFilter can't bound this either. #3037 @@ -1028,7 +1022,7 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: textIndexQuerySettings, + clickhouse_settings: this.getClickHouseSettings(), abort_signal: signal, }) .then(r => r.json<{ key: string }>()) @@ -1071,7 +1065,7 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: textIndexQuerySettings, + clickhouse_settings: this.getClickHouseSettings(), abort_signal: signal, }) .then(r => r.json<{ key: string }>()) From 33c371ee4d6b82f9410ae4e1e05d83e37b211b71 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:35:25 -0700 Subject: [PATCH 4/4] fix(common-utils): rethrow aborted text index reads and bound them with throw-on-timeout An aborted text index read used to fall through to the rollup with a dead signal; the rollup rejected at once, returned [], and getOrFetch cached that under the rollup key with no TTL, disabling the rollup for that column and range for the life of the process (reachable from the MCP describe_source timeout). Rethrow when the signal is aborted. Cap the index read at 15s with timeout_overflow_mode 'throw', matching the rollup and scan caps, so a slow read falls through promptly. 'throw' rather than 'break' so a timeout never caches a partial key list. Refs HDX-5598 --- .../src/__tests__/metadata.test.ts | 47 ++++++++++++++++++- packages/common-utils/src/core/metadata.ts | 15 +++++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 4a3725d3f5..f3050f662e 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2378,7 +2378,7 @@ describe('Metadata', () => { expect(queries()[1]).toContain('ColumnIdentifier = '); }); - it('forwards the abort signal to the text index read', async () => { + it('bounds the text index read and forwards the abort signal', async () => { const md = buildWithTextIndex(keyIndexLookup); mockScanResponses( (mockClickhouseClient.query as jest.Mock).mockRejectedValueOnce( @@ -2390,9 +2390,54 @@ describe('Metadata', () => { await md.getMapKeys({ ...args, signal }); const [first] = (mockClickhouseClient.query as jest.Mock).mock.calls[0]; + expect(first.clickhouse_settings).toMatchObject({ + max_execution_time: 15, + timeout_overflow_mode: 'throw', + }); expect(first.abort_signal).toBe(signal); }); + it('falls through to the rollup when the text index read times out', async () => { + const md = buildWithTextIndex(keyIndexLookup); + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce( + new Error('Timeout exceeded: elapsed 15 seconds', { + cause: { type: 'TIMEOUT_EXCEEDED', code: '159' }, + }), + ) + .mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ Key: 'user.id' }] }), + }); + + const keys = await md.getMapKeys({ ...args, metadataMVs }); + + expect(keys).toEqual(['user.id']); + expect(queries()[1]).toContain('ColumnIdentifier = '); + }); + + it('rethrows an aborted text index read without running or caching the fallback', async () => { + const md = buildWithTextIndex(keyIndexLookup); + const controller = new AbortController(); + const abortError = new Error('The user aborted a request.'); + (mockClickhouseClient.query as jest.Mock).mockImplementationOnce(() => { + controller.abort(); + return Promise.reject(abortError); + }); + + await expect( + md.getMapKeys({ ...args, metadataMVs, signal: controller.signal }), + ).rejects.toBe(abortError); + expect(queries()).toHaveLength(1); + + (mockClickhouseClient.query as jest.Mock).mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + expect(await md.getMapKeys({ ...args, metadataMVs })).toEqual([ + 'http.method', + ]); + expect(queries()[1]).toContain('mergeTreeTextIndex('); + }); + it('caches the fallback result so the failing index read is not retried', async () => { const md = buildWithTextIndex(keyIndexLookup); mockScanResponses( diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 1062f90f5c..dcfa524535 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -995,6 +995,13 @@ export class Metadata { supportsMergeTreeTextIndex(clickhouseVersion); // Text Index path: query the key rollup index const textIndexInfo = textIndexInfoLookup.get(column); + const textIndexQuerySettings: ClickHouseSettings = { + ...this.getClickHouseSettings(), + // 'throw', not 'break': a partial key list would be cached without a + // TTL. On timeout the rollup/scan paths below take over. + max_execution_time: 15, + timeout_overflow_mode: 'throw', + }; // Set when the text index read hits a row policy; see the rollup guard. let rowPolicyDenied = false; // Without timestampValueExpression, partsFilter can't bound this either. #3037 @@ -1022,7 +1029,7 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: this.getClickHouseSettings(), + clickhouse_settings: textIndexQuerySettings, abort_signal: signal, }) .then(r => r.json<{ key: string }>()) @@ -1032,6 +1039,9 @@ export class Metadata { return keys; } } catch (e) { + // The caller gave up; don't start the fallback with a dead signal and + // cache its instant rejection under the rollup key. + if (signal?.aborted) throw e; // e.g. ACCESS_DENIED under a row policy or BAD_ARGUMENTS on a // Distributed table; the paths below still work. rowPolicyDenied = isAccessDeniedError(e); @@ -1065,7 +1075,7 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: this.getClickHouseSettings(), + clickhouse_settings: textIndexQuerySettings, abort_signal: signal, }) .then(r => r.json<{ key: string }>()) @@ -1076,6 +1086,7 @@ export class Metadata { } } catch (e) { // See the key text index catch above. + if (signal?.aborted) throw e; rowPolicyDenied = isAccessDeniedError(e); console.warn( 'getMapKeys kv text index query failed; falling through to the next strategy',