Skip to content

Commit 001b112

Browse files
committed
fix: scale the text index token cap by the parts read
1 parent 3f46469 commit 001b112

3 files changed

Lines changed: 138 additions & 7 deletions

File tree

‎packages/common-utils/src/__tests__/metadata.int.test.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1757,4 +1757,99 @@ describe('Metadata Integration Tests', () => {
17571757
});
17581758
},
17591759
);
1760+
1761+
describe('getAllKeyValues - values repeated across parts', () => {
1762+
// mergeTreeTextIndex returns one row per part and token, so a token cap
1763+
// that ignores the part count stops before the later parts' values. Each
1764+
// part holds the same SHARED values plus OWN values no other part has.
1765+
const tableName = 'test_text_index_values_across_parts';
1766+
const PARTS = 25;
1767+
const SHARED = 1000;
1768+
const OWN = 100;
1769+
const dateRange = (): [Date, Date] => [
1770+
new Date(Date.now() - 60 * 60 * 1000),
1771+
new Date(Date.now() + 60 * 1000),
1772+
];
1773+
let textIndexSupported = false;
1774+
1775+
beforeAll(async () => {
1776+
const probe = new Metadata(hdxClient, new MetadataCache());
1777+
textIndexSupported = supportsMergeTreeTextIndex(
1778+
await probe.getServerVersion({ connectionId: 'test_connection' }),
1779+
);
1780+
if (!textIndexSupported) return;
1781+
1782+
await client.command({
1783+
query: `CREATE OR REPLACE TABLE default.${tableName} (
1784+
Timestamp DateTime64(9),
1785+
ResourceAttributes Map(LowCardinality(String), String),
1786+
ResourceAttributeItems Array(String) MATERIALIZED
1787+
arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')),
1788+
Tags Array(String),
1789+
INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1,
1790+
INDEX idx_tags Tags TYPE text(tokenizer = 'array') GRANULARITY 1
1791+
)
1792+
ENGINE = MergeTree()
1793+
PARTITION BY toDate(Timestamp)
1794+
ORDER BY Timestamp
1795+
SETTINGS text_index_dictionary_block_size = 8192
1796+
`,
1797+
});
1798+
await client.command({
1799+
query: `SYSTEM STOP MERGES default.${tableName}`,
1800+
});
1801+
for (let i = 0; i < PARTS; i++) {
1802+
await client.command({
1803+
query: `INSERT INTO default.${tableName} (Timestamp, ResourceAttributes, Tags)
1804+
SELECT now64(9) - INTERVAL 10 MINUTE, map('pod', v), [v]
1805+
FROM (
1806+
SELECT if(number < ${SHARED}, concat('shared-', toString(number)), concat('own-${i}-', toString(number))) AS v
1807+
FROM numbers(${SHARED + OWN})
1808+
)`,
1809+
});
1810+
}
1811+
});
1812+
1813+
afterAll(async () => {
1814+
await client.command({
1815+
query: `DROP TABLE IF EXISTS default.${tableName}`,
1816+
});
1817+
});
1818+
1819+
it.each([
1820+
['Map key=value', "ResourceAttributes['pod']"],
1821+
['plain-column', 'Tags'],
1822+
])(
1823+
'returns every distinct value from the %s text index, with and without a part limit',
1824+
async (_, keyExpression) => {
1825+
if (!textIndexSupported) {
1826+
console.warn(
1827+
'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
1828+
);
1829+
return;
1830+
}
1831+
1832+
// PARTS is above the default limit of 20, so unset reads 20 parts and
1833+
// null reads all of them.
1834+
for (const [mapKeyIndexMaxParts, partsRead] of [
1835+
[undefined, 20],
1836+
[null, PARTS],
1837+
] as const) {
1838+
const metadata = new Metadata(hdxClient, new MetadataCache());
1839+
const result = await metadata.getAllKeyValues({
1840+
databaseName: 'default',
1841+
tableName,
1842+
keyExpressions: [keyExpression],
1843+
maxValuesPerKey: 10000,
1844+
mapKeyIndexMaxParts,
1845+
connectionId: 'test_connection',
1846+
dateRange: dateRange(),
1847+
timestampValueExpression: 'Timestamp',
1848+
});
1849+
expect(result).toHaveLength(1);
1850+
expect(result[0].value).toHaveLength(SHARED + OWN * partsRead);
1851+
}
1852+
},
1853+
);
1854+
});
17601855
});

‎packages/common-utils/src/__tests__/metadata.test.ts‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1382,7 +1382,8 @@ describe('Metadata', () => {
13821382

13831383
expect(sql).toContain('mergeTreeTextIndex(');
13841384
expect(sql).toContain('topKWeighted(');
1385-
expect(sql).toContain('LIMIT 20000');
1385+
// 20,000 tokens for each of the default 20 parts.
1386+
expect(sql).toContain('LIMIT 400000');
13861387
expect(sql).not.toContain('startsWith(token,');
13871388
});
13881389

@@ -1436,12 +1437,38 @@ describe('Metadata', () => {
14361437
expect(branches[0]).toContain("'requestId' AS key");
14371438
expect(branches[1]).toContain("startsWith(token, 'region=')");
14381439
for (const branch of branches) {
1439-
expect(branch).toContain('LIMIT 20000');
1440+
// 20,000 tokens for each of the default 20 parts.
1441+
expect(branch).toContain('LIMIT 400000');
14401442
}
14411443
expect(call.clickhouse_settings.max_rows_to_read).toBe('0');
14421444
expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined();
14431445
});
14441446

1447+
it.each([
1448+
['map KV', "LogAttributes['requestId']"],
1449+
['native', 'TraceId'],
1450+
])(
1451+
'scales the %s text index token cap by the parts read, and drops it when every part is read',
1452+
async (_, keyExpression) => {
1453+
setupDefaultLogsSchema();
1454+
withDictionaryBlockSize(8192);
1455+
1456+
for (const mapKeyIndexMaxParts of [5, null]) {
1457+
await metadata.getAllKeyValues({
1458+
...baseArgs,
1459+
keyExpressions: [keyExpression],
1460+
mapKeyIndexMaxParts,
1461+
});
1462+
}
1463+
1464+
const tokenLimits = (mockClickhouseClient.query as jest.Mock).mock.calls
1465+
.map(([opts]) => opts.query as string)
1466+
.filter(q => q.includes('mergeTreeTextIndex('))
1467+
.map(q => q.match(/LIMIT (\d+)/)?.[1]);
1468+
expect(tokenLimits).toEqual(['100000', undefined]);
1469+
},
1470+
);
1471+
14451472
it('reads all Map keys in one pass when the index has small dictionary blocks', async () => {
14461473
setupDefaultLogsSchema();
14471474
withDictionaryBlockSize(512);

‎packages/common-utils/src/core/metadata.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,19 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20;
7979
// Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle.
8080
export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100;
8181

82-
// Tokens a text index values lookup reads per key. The dictionary is sorted,
83-
// so a key with more distinct values than this gets the most frequent among
84-
// its alphabetically first ones. Keep it above "Load more"'s 10,000 values.
82+
// Distinct values a text index values lookup reads per key. The dictionary is
83+
// sorted, so a key with more than this gets the most frequent among its
84+
// alphabetically first ones. Keep it above "Load more"'s 10,000 values.
8585
const TEXT_INDEX_TOKENS_PER_KEY = 20_000;
86+
87+
// mergeTreeTextIndex returns one row per part and token, so a value in every
88+
// part read takes maxParts rows; scaling the LIMIT by maxParts keeps room for
89+
// TEXT_INDEX_TOKENS_PER_KEY distinct values. A read of every part ("Load
90+
// more") isn't capped.
91+
const textIndexTokenLimit = (maxParts: number | null): ChSql =>
92+
maxParts == null
93+
? chSql``
94+
: chSql`LIMIT ${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(TEXT_INDEX_TOKENS_PER_KEY * maxParts, 'maxParts') }}`;
8695
// Keys or columns per text index values query; see dispatchKeyValueQueries.
8796
const TEXT_INDEX_KEYS_PER_QUERY = 10;
8897
// Smallest dictionary block size that gets one read per key; see
@@ -1865,7 +1874,7 @@ export class Metadata {
18651874
WHERE ${partsFilter}
18661875
AND startsWith(token, ${keyPrefix(k)})
18671876
AND v != ''
1868-
LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
1877+
${textIndexTokenLimit(maxParts)}
18691878
)`);
18701879
}
18711880
}
@@ -1961,7 +1970,7 @@ export class Metadata {
19611970
FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }})
19621971
WHERE ${partsFilter}
19631972
AND token != ''
1964-
LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}
1973+
${textIndexTokenLimit(maxParts)}
19651974
)`;
19661975
sqlBranches.push(sql);
19671976
}

0 commit comments

Comments
 (0)