From f929c0ec8f34c1e013f3954fbafc9ad9c132baee Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:46:56 -0400 Subject: [PATCH] fix: improve test index filter loading --- .../DBSearchPageFilters/hooks.test.tsx | 1 + .../components/DBSearchPageFilters/hooks.ts | 3 + .../src/__tests__/metadata.int.test.ts | 137 +++++- .../src/__tests__/metadata.test.ts | 256 +++++++++++ packages/common-utils/src/core/metadata.ts | 405 +++++++++++++++--- packages/common-utils/src/core/utils.ts | 13 + 6 files changed, 744 insertions(+), 71 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index a3c567233e..430061b240 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -1101,6 +1101,7 @@ describe('useFetchFacets', () => { tableName: 'other_table', connectionId: 'conn2', keyExpressions: ['ServiceName'], + mapTextIndexPartsPerKey: 'all', }), ); expect(result.current.data.keyValues).toEqual([ diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index be73a26336..7b8c361d75 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -256,6 +256,9 @@ function useFacets({ metadataMVs: tableConnection.metadataMVs, keyExpressions: [sqlKey], maxValuesPerKey: LOAD_MORE_LOAD_LIMIT, + // One key, so reading every part that holds it stays cheap, and + // the best part alone may hold far fewer values than requested. + mapTextIndexPartsPerKey: 'all', dateRange, timestampValueExpression: source?.timestampValueExpression ?? diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index e27a2ccc0e..a5ef168aa7 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -3,7 +3,8 @@ import { ClickHouseClient } from '@clickhouse/client'; import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node'; import { supportsMergeTreeTextIndex } from '@/core/clickhouseVersion'; -import { Metadata, MetadataCache } from '@/core/metadata'; +import { KeyValues, Metadata, MetadataCache } from '@/core/metadata'; +import { MapTextIndexPartsPerKey } from '@/core/utils'; import { parseKvItemsCastExpression, parseKvItemsExpression, @@ -1554,4 +1555,138 @@ describe('Metadata Integration Tests', () => { }); }, ); + + describe('getAllKeyValues - Map values from the parts a keys index lists', () => { + const tableName = 'test_map_values_best_parts'; + const keyExpressions = [ + "LogAttributes['shared']", + "LogAttributes['rare.key']", + "LogAttributes['old.only']", + ]; + let metadata: Metadata; + let textIndexSupported = false; + + beforeAll(async () => { + const probe = new Metadata(hdxClient, new MetadataCache()); + textIndexSupported = supportsMergeTreeTextIndex( + await probe.getServerVersion({ connectionId: 'test_connection' }), + ); + if (!textIndexSupported) return; + + await client.command({ + query: `CREATE OR REPLACE TABLE default.${tableName} ( + Timestamp DateTime64(9), + LogAttributes Map(LowCardinality(String), String), + LogAttributeItems Array(String) MATERIALIZED + arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')), + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + ) + ENGINE = MergeTree() + PARTITION BY toDate(Timestamp) + ORDER BY Timestamp + `, + }); + // Every insert below must stay its own part. + await client.command({ + query: `SYSTEM STOP MERGES default.${tableName}`, + }); + const insert = (minutesAgo: number, attributes: string, rows: number) => + client.command({ + query: `INSERT INTO default.${tableName} (Timestamp, LogAttributes) + SELECT now64(9) - INTERVAL ${minutesAgo} MINUTE, ${attributes} + FROM numbers(${rows})`, + }); + // Written before the keys index exists, so the index never lists it. + await insert(30, `map('shared', 'from-old', 'old.only', 'x')`, 5); + await client.command({ + query: `ALTER TABLE default.${tableName} + ADD INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`, + }); + // The part holding `shared` in the most rows, and the only one with + // `rare.key`. + await insert( + 20, + `if(number = 0, map('shared', 'from-big', 'rare.key', 'r1'), map('shared', 'from-big'))`, + 50, + ); + await insert(10, `map('shared', 'from-new')`, 1); + }); + + afterAll(async () => { + if (!textIndexSupported) return; + await client.command({ + query: `DROP TABLE IF EXISTS default.${tableName}`, + }); + }); + + beforeEach(() => { + metadata = new Metadata(hdxClient, new MetadataCache()); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + const getValues = (mapTextIndexPartsPerKey?: MapTextIndexPartsPerKey) => + metadata.getAllKeyValues({ + databaseName: 'default', + tableName, + connectionId: 'test_connection', + keyExpressions, + mapTextIndexPartsPerKey, + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date(Date.now() - 60 * 60 * 1000), + new Date(Date.now() + 60 * 1000), + ], + }); + + const byKey = (result: KeyValues[]) => + Object.fromEntries(result.map(({ key, value }) => [key, value])); + + it('reads the best and newest part per key, and reports keys the index misses', async () => { + if (!textIndexSupported) { + console.warn( + 'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()', + ); + return; + } + const warnSpy = jest + .spyOn(console, 'warn') + .mockImplementation(() => undefined); + + const values = byKey(await getValues()); + + expect(values["LogAttributes['shared']"]?.sort()).toEqual([ + 'from-big', + 'from-new', + ]); + expect(values["LogAttributes['rare.key']"]).toEqual(['r1']); + expect(values).not.toHaveProperty("LogAttributes['old.only']"); + const warnings = warnSpy.mock.calls.map(([message]) => String(message)); + expect(warnings).toEqual([ + expect.stringContaining( + 'LogAttributes keys [old.only]: keys index idx_log_attr_key lists the requested keys in 2 of 3 parts', + ), + ]); + }); + + it("reads every part the keys index lists when parts per key is 'all'", async () => { + if (!textIndexSupported) { + console.warn( + 'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()', + ); + return; + } + jest.spyOn(console, 'warn').mockImplementation(() => undefined); + + const values = byKey(await getValues('all')); + + expect(values["LogAttributes['shared']"]?.sort()).toEqual([ + 'from-big', + 'from-new', + ]); + expect(values["LogAttributes['rare.key']"]).toEqual(['r1']); + }); + }); }); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 207cc3abcb..4e78a2fc10 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1709,6 +1709,262 @@ describe('Metadata', () => { consoleWarnSpy.mockRestore(); }); }); + + describe('Map columns with a keys-only text index', () => { + const setupWithKeysIndex = () => { + setupDefaultLogsSchema(); + jest.spyOn(metadata, 'getMapColumnTextIndexes').mockResolvedValue( + new Map([ + [ + 'LogAttributes', + { + kv: { + columnName: 'LogAttributeItems', + mapColumn: 'LogAttributes', + indexName: 'idx_log_attr_items', + separator: '=', + useHasAny: false, + }, + key: { + indexName: 'idx_log_attr_key', + mapColumn: 'LogAttributes', + }, + }, + ], + [ + 'ResourceAttributes', + { + kv: { + columnName: 'ResourceAttributeItems', + mapColumn: 'ResourceAttributes', + indexName: 'idx_res_attr_items', + separator: '=', + useHasAny: false, + }, + }, + ], + ]), + ); + }; + + const mockRows = (rows: Record[]) => + (mockClickhouseClient.query as jest.Mock).mockImplementation( + ({ query }: { query: string }) => + Promise.resolve({ + json: () => + Promise.resolve({ + data: query.includes('keys_catalog_') ? rows : [], + }), + }), + ); + + const bestPartsCall = () => + (mockClickhouseClient.query as jest.Mock).mock.calls.find((c: any[]) => + (c[0].query as string).includes('keys_catalog_'), + )?.[0]; + + const row = ( + key: string, + value: string[], + { indexed = 1, covered = 4, total = 4 } = {}, + ) => ({ + column: 'LogAttributes', + key, + value, + indexed, + covered_parts: covered, + total_parts: total, + }); + + it('reads each key only from the parts the keys index lists for it', async () => { + setupWithKeysIndex(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "LogAttributes['user.id']", + ], + }); + + const call = bestPartsCall(); + expect(call).toBeDefined(); + const sql = (call.query as string).replace(/\s+/g, ' '); + const params = Object.values(call.query_params); + expect(params).toEqual( + expect.arrayContaining(['idx_log_attr_key', 'idx_log_attr_items']), + ); + // Catalog: one pass over the keys index, restricted to the requested + // keys, top part by cardinality plus the newest part holding the key. + expect(sql).toContain( + "WHERE has(matching_parts, part_name) AND token IN ('requestId', 'user.id')", + ); + expect(sql).toContain( + 'arraySlice(arrayReverseSort(groupArray((cardinality, part_name))), 1, 1).2', + ); + expect(sql).toContain( + 'argMax(part_name, indexOf(matching_parts, part_name))', + ); + // One LIMITed branch per key, filtered on the raw token by a + // constant prefix and on parts by the WITH scalar. + expect(sql.match(/UNION ALL/g)).toHaveLength(1); + for (const key of ['requestId', 'user.id']) { + expect(sql).toContain( + `WHERE has(keys_catalog_0.1['${key}'], part_name) AND startsWith(token, '${key}=') AND token != '${key}='`, + ); + } + expect(sql).toMatch(/LIMIT \{HYPERDX_PARAM_\d+:Int32\} \)/); + expect(params).toContain(20); + expect(sql).not.toMatch(/JOIN|substringIndex|splitByChar/); + expect(call.clickhouse_settings).toMatchObject({ max_block_size: '1' }); + expect(call.clickhouse_settings).not.toHaveProperty( + 'enable_filesystem_cache', + ); + }); + + it.each([ + [2, ', 1, 2).2'], + [5, ', 1, 5).2'], + ])( + 'reads the top %p parts per key when asked', + async (partsPerKey, expected) => { + setupWithKeysIndex(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['requestId']"], + mapTextIndexPartsPerKey: partsPerKey, + }); + + expect(bestPartsCall().query).toContain(expected); + }, + ); + + it("reads every part holding the key when parts per key is 'all'", async () => { + setupWithKeysIndex(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['requestId']"], + mapTextIndexPartsPerKey: 'all', + }); + + const sql = bestPartsCall().query as string; + expect(sql).toContain('groupArray(part_name) AS parts'); + expect(sql).not.toContain('arraySlice('); + }); + + it.each([0, 1.5, -1])('rejects %p parts per key', async partsPerKey => { + setupWithKeysIndex(); + + await expect( + metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['requestId']"], + mapTextIndexPartsPerKey: partsPerKey, + }), + ).rejects.toThrow('mapTextIndexPartsPerKey must be'); + }); + + it('SQL-escapes keys inlined into the branch', async () => { + setupWithKeysIndex(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['it's']"], + }); + + const sql = bestPartsCall().query as string; + expect(sql).toContain("startsWith(token, 'it\\'s=')"); + expect(sql).toContain("keys_catalog_0.1['it\\'s']"); + expect(sql).not.toContain("'it's"); + }); + + it('keeps reading every part for Map columns without a keys index', async () => { + setupWithKeysIndex(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "ResourceAttributes['service.name']", + ], + }); + + const queries = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.map((c: any[]) => c[0].query as string); + const scan = queries.find(q => q.includes('GROUP BY column, key')); + expect(scan).toContain("startsWith(token, 'service.name=')"); + expect(scan).not.toContain('requestId'); + expect(bestPartsCall().query).not.toContain('service.name'); + }); + + it('returns the values per key and drops keys without any', async () => { + setupWithKeysIndex(); + mockRows([row('requestId', ['a', 'b']), row('user.id', [])]); + + const result = await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "LogAttributes['user.id']", + ], + }); + + expect(result).toEqual([ + { key: "LogAttributes['requestId']", value: ['a', 'b'] }, + ]); + }); + + it('warns about unlisted keys while the keys index misses parts of the range', async () => { + setupWithKeysIndex(); + const consoleWarnSpy = jest + .spyOn(console, 'warn') + .mockImplementation(() => undefined); + mockRows([ + row('requestId', ['a'], { covered: 3, total: 4 }), + row('old.key', [], { indexed: 0, covered: 3, total: 4 }), + ]); + + const result = await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "LogAttributes['old.key']", + ], + }); + + expect(result).toEqual([ + { key: "LogAttributes['requestId']", value: ['a'] }, + ]); + expect(consoleWarnSpy).toHaveBeenCalledTimes(1); + const message = consoleWarnSpy.mock.calls[0][0] as string; + expect(message).toContain('LogAttributes keys [old.key]'); + expect(message).toContain( + 'idx_log_attr_key lists the requested keys in 3 of 4 parts in the date range', + ); + expect(message).toContain( + 'ALTER TABLE default.otel_logs MATERIALIZE INDEX idx_log_attr_key', + ); + }); + + it('does not warn about unlisted keys when the keys index lists every part', async () => { + setupWithKeysIndex(); + const consoleWarnSpy = jest + .spyOn(console, 'warn') + .mockImplementation(() => undefined); + mockRows([row('gone.key', [], { indexed: 0 })]); + + const result = await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['gone.key']"], + }); + + expect(result).toEqual([]); + expect(consoleWarnSpy).not.toHaveBeenCalled(); + }); + }); }); describe('getValuesDistribution', () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 52429d6e4f..26e51892a4 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -50,6 +50,7 @@ import { import { getAlignedDateRange, getDistributedTableArgs, + MapTextIndexPartsPerKey, MetadataMVQueryOptions, objectHash, splitAndTrimWithBracket, @@ -65,6 +66,8 @@ const DEFAULT_MAX_KEYS = 1000; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; +export const DEFAULT_MAP_TEXT_INDEX_PARTS_PER_KEY = 1; + /** * The index read could not be scoped to the requested date range, so it was * refused in favour of a path that can. Not a failure — an expected property @@ -92,6 +95,66 @@ export type KeyValues = { value: string[] | number[]; }; +type MapTextIndexValuesQuery = TableConnection & { + queryOptions: TextIndexMapColumnQueryOptions; + dateRange: [Date, Date]; + timestampValueExpression: string; + signal?: AbortSignal; +}; + +type MapTextIndexBestPartsRow = { + column: string; + key: string; + value: string[]; + /** 1 when the keys index lists the key in some part of the date range. */ + indexed: number; + /** Parts of the date range the keys index lists any requested key in. */ + covered_parts: number; + total_parts: number; +}; + +/** + * Keys the keys index doesn't list return no values. When some parts of the + * date range are missing from the index too, those keys may exist there, so + * say so instead of reporting nothing or reading every part. + */ +const warnOnKeysIndexGaps = ({ + databaseName, + tableName, + queryOptions, + rows, +}: { + databaseName: string; + tableName: string; + queryOptions: TextIndexMapColumnQueryOptions; + rows: MapTextIndexBestPartsRow[]; +}) => { + const gaps = new Map< + string, + { keys: string[]; coveredParts: number; totalParts: number } + >(); + for (const row of rows) { + if (row.indexed || row.covered_parts >= row.total_parts) continue; + const gap = gaps.get(row.column) ?? { + keys: [], + coveredParts: row.covered_parts, + totalParts: row.total_parts, + }; + gap.keys.push(row.key); + gaps.set(row.column, gap); + } + for (const [column, { keys, coveredParts, totalParts }] of gaps) { + const index = queryOptions.get(column)?.keysIndexName; + console.warn( + `getMapTextIndexKeyValues: no values read for ${column} keys [${keys.join(', ')}]: ` + + `keys index ${index} lists the requested keys in ${coveredParts} of ${totalParts} parts in the date range. ` + + `Parts written before the index was added are not read until they merge or ` + + `ALTER TABLE ${databaseName}.${tableName} MATERIALIZE INDEX ${index} runs. ` + + `Parts where ${column} is always empty are also unlisted.`, + ); + } +}; + export type MetricNames = { names: string[]; /** True when more names matched than `limit`, so the page is incomplete. */ @@ -813,17 +876,30 @@ export class Metadata { timestampValueExpression?: string; }): Promise { if (!dateRange || !timestampValueExpression) return chSql`1`; - const startTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})`; - const endTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }})`; return chSql`part_name IN ( SELECT name FROM system.parts - WHERE database = ${{ String: databaseName }} AND table = ${{ String: tableName }} - AND active=1 - AND ((min_time >= ${startTime} AND min_time <= ${endTime}) OR (max_time <= ${endTime} AND max_time >= ${startTime}) OR (min_time <= ${startTime} AND max_time >= ${endTime})) + WHERE ${this.overlappingPartsCondition({ databaseName, tableName, dateRange })} )`; } + /** `system.parts` filter for the active parts overlapping `dateRange`. */ + private overlappingPartsCondition({ + databaseName, + tableName, + dateRange, + }: { + databaseName: string; + tableName: string; + dateRange: [Date, Date]; + }): ChSql { + const startTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})`; + const endTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }})`; + return chSql`database = ${{ String: databaseName }} AND table = ${{ String: tableName }} + AND active=1 + AND ((min_time >= ${startTime} AND min_time <= ${endTime}) OR (max_time <= ${endTime} AND max_time >= ${startTime}) OR (min_time <= ${startTime} AND max_time >= ${endTime}))`; + } + /** Rejects what `mergeTreeIndex` cannot read; callers fall back to a scan. */ private async assertIndexReadable({ databaseName, @@ -1560,77 +1636,42 @@ export class Metadata { dateRange, timestampValueExpression, signal, - }: TableConnection & { - queryOptions: TextIndexMapColumnQueryOptions; - dateRange: [Date, Date]; - timestampValueExpression: string; - signal?: AbortSignal; - }): Promise { + }: MapTextIndexValuesQuery): Promise { const queryOptionsHash = objectHash(queryOptions); const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getMapTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { - const sqlBranches: Array = []; - for (const [columnName, info] of queryOptions.entries()) { - const orChain = concatChSql( - ' OR ', - // Inline keys as SQL-escaped literals, not bind params: ~100 - // per-key params exceed the web client's URL param budget and - // silently switch the request to a multipart body that proxies - // may reject. Keys are ingest-controlled, hence SqlString.escape. - info.keys.map( - k => - chSql`startsWith(token, ${{ - UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`), - }})`, - ), + const withKeysIndex: TextIndexMapColumnQueryOptions = new Map(); + const withoutKeysIndex: TextIndexMapColumnQueryOptions = new Map(); + for (const [columnName, info] of queryOptions) { + (info.keysIndexName ? withKeysIndex : withoutKeysIndex).set( + columnName, + info, ); - const partsFilter = await this.partsOverlapFilter({ - databaseName, - tableName, - dateRange, - timestampValueExpression, - }); - const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; - const sql = chSql` - SELECT * FROM ( - SELECT ${{ String: columnName }} as column, - substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, - groupUniqArray(${{ Int32: info.limit }})(${valueSql}) AS value - FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) - WHERE ${partsFilter} - AND (${orChain}) - AND ${valueSql} != '' - GROUP BY column, key - )`; - sqlBranches.push(sql); } - const sql = concatChSql(' UNION ALL ', sqlBranches); - - return await this.clickhouseClient - .query<'JSON'>({ - query: sql.sql, - query_params: sql.params, - connectionId, - clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', - ...this.getClickHouseSettings(), - }, - abort_signal: signal, - }) - .then(res => - res.json<{ column: string; key: string; value: string[] }>(), - ) - .then(d => - d.data.map(row => ({ - key: `${row.column}['${row.key}']`, - value: row.value, - })), - ); + const args = { + databaseName, + tableName, + connectionId, + dateRange, + timestampValueExpression, + signal, + }; + const results = await Promise.all([ + withKeysIndex.size > 0 + ? this.readMapTextIndexValuesFromBestParts({ + ...args, + queryOptions: withKeysIndex, + }) + : [], + withoutKeysIndex.size > 0 + ? this.scanMapTextIndexValues({ + ...args, + queryOptions: withoutKeysIndex, + }) + : [], + ]); + return results.flat(); } catch (error) { // Text-index queries can fail transiently (part merged mid-read, // unsupported server, etc.). Isolate the failure so sibling @@ -1644,6 +1685,207 @@ export class Metadata { }); } + /** + * Reads each key's values only from the parts the Map's keys-only text + * index says hold it in the most rows, plus the newest part holding it. On + * object storage, every part a key is read from costs a request or more per + * index file, so reading every part for every key is slow however few bytes + * it reads. + * + * Parts written before the keys index existed are missing from it, and keys + * found only in those parts return nothing (see `warnOnKeysIndexGaps`). + */ + private async readMapTextIndexValuesFromBestParts({ + databaseName, + tableName, + connectionId, + queryOptions, + dateRange, + signal, + }: MapTextIndexValuesQuery): Promise { + const textIndex = (indexName: string) => + chSql`mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: indexName }})`; + const catalogs: ChSql[] = []; + const branches: ChSql[] = []; + for (const [i, [columnName, info]] of [...queryOptions].entries()) { + if (!info.keysIndexName) continue; + const catalog = { UNSAFE_RAW_SQL: `keys_catalog_${i}` }; + const partsSql = + info.partsPerKey === 'all' + ? chSql`groupArray(part_name)` + : chSql`arrayDistinct(arrayPushBack( + arraySlice(arrayReverseSort(groupArray((cardinality, part_name))), 1, ${{ + UNSAFE_RAW_SQL: inlineNonNegativeInt( + info.partsPerKey, + 'partsPerKey', + ), + }}).2, + argMax(part_name, indexOf(matching_parts, part_name)) + ))`; + // Only the requested keys: a Map whose key names carry IDs can list + // millions of keys per part, and reading them all costs minutes of CPU + // and overflows the scalar. A constant `token IN` on the bare column + // reads just the dictionary blocks holding these keys. + const keysListSql = concatChSql( + ', ', + // Escaped literals rather than params; see `scanMapTextIndexValues`. + info.keys.map(k => chSql`${{ UNSAFE_RAW_SQL: SqlString.escape(k) }}`), + ); + // A WITH scalar is evaluated once, before the branches are planned, so + // they can prune parts with it. Picking parts with a join instead + // disables pruning and every branch reads every part. + catalogs.push(chSql`( + SELECT ( + CAST(groupArray((key, parts)), 'Map(String, Array(String))'), + toUInt32(length(groupUniqArrayArray(key_parts))) + ) + FROM ( + SELECT token AS key, ${partsSql} AS parts, groupArray(part_name) AS key_parts + FROM ${textIndex(info.keysIndexName)} + WHERE has(matching_parts, part_name) + AND token IN (${keysListSql}) + GROUP BY key + ) + ) AS ${catalog}`); + for (const key of info.keys) { + // Escaped literals rather than params; see `scanMapTextIndexValues`. + // The token filter must be a constant prefix on the bare `token` + // column (including the separator, or `k1` matches `k10=`): any + // function around `token` disables the dictionary lookup. + const keySql = { UNSAFE_RAW_SQL: SqlString.escape(key) }; + const prefixSql = { + UNSAFE_RAW_SQL: SqlString.escape(`${key}${info.separator}`), + }; + branches.push(chSql` + SELECT ${{ String: columnName }} AS column, ${keySql} AS key, + groupArray(substring(token, length(${prefixSql}) + 1)) AS value, + toUInt8(mapContains(${catalog}.1, ${keySql})) AS indexed, + ${catalog}.2 AS covered_parts, + toUInt32(length(matching_parts)) AS total_parts + FROM ( + SELECT DISTINCT token + FROM ${textIndex(info.indexName)} + WHERE has(${catalog}.1[${keySql}], part_name) + AND startsWith(token, ${prefixSql}) + AND token != ${prefixSql} + LIMIT ${{ Int32: info.limit }} + )`); + } + } + if (branches.length === 0) return []; + + // Oldest first, so `indexOf(matching_parts, part_name)` ranks recency. + const sql = chSql` + WITH ( + SELECT arraySort(groupArray((max_time, name))).2 + FROM system.parts + WHERE ${this.overlappingPartsCondition({ databaseName, tableName, dateRange })} + ) AS matching_parts, + ${concatChSql(',\n', catalogs)} + ${concatChSql('\nUNION ALL\n', branches)}`; + + const { data } = await this.clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', + ...this.getClickHouseSettings(), + // The reader emits whole dictionary blocks until it has + // max_block_size rows; at 1, each branch's LIMIT stops it after the + // first block instead of ~128. + max_block_size: '1', + // Makes the WITH aliases visible to every UNION ALL branch. + enable_global_with_statement: 1, + enable_filesystem_cache: 0, + }, + abort_signal: signal, + }) + .then(res => res.json()); + + warnOnKeysIndexGaps({ databaseName, tableName, queryOptions, rows: data }); + // Without GROUP BY, a branch that found no tokens still yields a row. + return data + .filter(row => row.value.length > 0) + .map(row => ({ key: `${row.column}['${row.key}']`, value: row.value })); + } + + /** Every key's values from every part in the date range, in one pass. */ + private async scanMapTextIndexValues({ + databaseName, + tableName, + connectionId, + queryOptions, + dateRange, + timestampValueExpression, + signal, + }: MapTextIndexValuesQuery): Promise { + const sqlBranches: Array = []; + for (const [columnName, info] of queryOptions.entries()) { + const orChain = concatChSql( + ' OR ', + // Inline keys as SQL-escaped literals, not bind params: ~100 + // per-key params exceed the web client's URL param budget and + // silently switch the request to a multipart body that proxies + // may reject. Keys are ingest-controlled, hence SqlString.escape. + info.keys.map( + k => + chSql`startsWith(token, ${{ + UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`), + }})`, + ), + ); + const partsFilter = await this.partsOverlapFilter({ + databaseName, + tableName, + dateRange, + timestampValueExpression, + }); + const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; + const sql = chSql` + SELECT * FROM ( + SELECT ${{ String: columnName }} as column, + substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, + groupUniqArray(${{ Int32: info.limit }})(${valueSql}) AS value + FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) + WHERE ${partsFilter} + AND (${orChain}) + AND ${valueSql} != '' + GROUP BY column, key + )`; + sqlBranches.push(sql); + } + const sql = concatChSql(' UNION ALL ', sqlBranches); + + return this.clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', + ...this.getClickHouseSettings(), + }, + abort_signal: signal, + }) + .then(res => res.json<{ column: string; key: string; value: string[] }>()) + .then(d => + d.data.map(row => ({ + key: `${row.column}['${row.key}']`, + value: row.value, + })), + ); + } + private async getTextIndexKeyValues({ databaseName, tableName, @@ -2608,6 +2850,7 @@ export class Metadata { tableName, keyExpressions, maxValuesPerKey = 20, + mapTextIndexPartsPerKey = DEFAULT_MAP_TEXT_INDEX_PARTS_PER_KEY, connectionId, metadataMVs, dateRange, @@ -2618,12 +2861,31 @@ export class Metadata { tableName: string; keyExpressions: string[]; maxValuesPerKey?: number; + /** + * For Map columns with both a `key=value` and a keys-only text index: how + * many parts each key's values are read from. A number N reads the N parts + * holding the key in the most rows plus the newest part holding it; 'all' + * reads every part holding it. + */ + mapTextIndexPartsPerKey?: MapTextIndexPartsPerKey; connectionId: string; metadataMVs?: MetadataMaterializedViews; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { + if ( + mapTextIndexPartsPerKey !== 'all' && + !( + Number.isInteger(mapTextIndexPartsPerKey) && + mapTextIndexPartsPerKey >= 1 + ) + ) { + throw new Error( + `mapTextIndexPartsPerKey must be a positive integer or 'all', got: ${String(mapTextIndexPartsPerKey)}`, + ); + } + if (keyExpressions.length === 0) return []; if (keyExpressions.length > GET_ALL_KEY_VALUES_CHUNK_SIZE) { @@ -2634,6 +2896,7 @@ export class Metadata { tableName, keyExpressions: batch, maxValuesPerKey, + mapTextIndexPartsPerKey, connectionId, metadataMVs, dateRange, @@ -2688,6 +2951,8 @@ export class Metadata { if (!entry) { entry = { indexName: mapTextIndex.kv.indexName, + keysIndexName: mapTextIndex.key?.indexName, + partsPerKey: mapTextIndexPartsPerKey, limit: maxValuesPerKey, separator: mapTextIndex.kv.separator, keys: [], diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index e0a05c46ef..ba5c41dcce 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1825,12 +1825,25 @@ export type TextIndexColumnQueryOptions = Map< } >; +/** + * How many parts of the date range to read each Map key's values from: the N + * parts holding the key in the most rows (plus the newest part holding it), or + * every part holding it. + */ +export type MapTextIndexPartsPerKey = number | 'all'; + // This type serves as options to fetch values from map text indices. // This is a record of Map Column Name to required query parameters. export type TextIndexMapColumnQueryOptions = Map< string, { indexName: string; + /** + * Keys-only text index over the same Map. Without it, every key's values + * are read from every part in the date range. + */ + keysIndexName?: string; + partsPerKey: MapTextIndexPartsPerKey; limit: number; separator: string; keys: string[];