perf: look up Map text index block sizes in parallel

Author: knudtty

packages/common-utils/src/__tests__/metadata.test.ts

@@ -2217,6 +2217,52 @@ describe('Metadata', () => {
         value: ['r1'],
       };
 
+      it("looks up every Map column's dictionary block size at once", async () => {
+        setupDefaultLogsSchema();
+        const kvIndex = (mapColumn: string, indexName: string) => ({
+          kv: {
+            columnName: mapColumn,
+            mapColumn,
+            indexName,
+            separator: '=',
+            useHasAny: false,
+          },
+        });
+        jest.spyOn(metadata, 'getMapColumnTextIndexes').mockResolvedValue(
+          new Map([
+            ['LogAttributes', kvIndex('LogAttributes', 'idx_log_attr_items')],
+            [
+              'ResourceAttributes',
+              kvIndex('ResourceAttributes', 'idx_res_attr_items'),
+            ],
+          ]),
+        );
+        let releaseSettings: () => void = () => {};
+        const settings = jest
+          .spyOn(metadata, 'getMergeTreeSettings')
+          .mockReturnValue(
+            new Promise(resolve => {
+              releaseSettings = () => resolve(undefined);
+            }),
+          );
+
+        const result = metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: [
+            "LogAttributes['requestId']",
+            "ResourceAttributes['service.name']",
+          ],
+        });
+        // Both lookups start before either finishes.
+        for (let i = 0; i < 50 && settings.mock.calls.length < 2; i++) {
+          await new Promise(resolve => setImmediate(resolve));
+        }
+        expect(settings).toHaveBeenCalledTimes(2);
+
+        releaseSettings();
+        await result;
+      });
+
       it('yields each query as it finishes, without waiting for the slowest', async () => {
         setupDefaultLogsSchema();
         let releaseMapTextIndex: () => void = () => {};

packages/common-utils/src/core/metadata.ts

@@ -3141,14 +3141,21 @@ export class Metadata {
     // high-cardinality key holds up only its own batch's values. Within a
     // batch, large dictionary blocks get one read per key; see
     // getMapTextIndexKeyValues.
-    for (const [column, options] of mapTextIndexQueryOptions) {
-      const perKey =
-        (await this.getTextIndexDictionaryBlockSize({
-          databaseName,
-          tableName,
-          connectionId,
-          indexName: options.indexName,
-        })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE;
+    // Every Map column's block size is looked up at once, not one at a time.
+    const mapColumns = await Promise.all(
+      [...mapTextIndexQueryOptions].map(async ([column, options]) => ({
+        column,
+        options,
+        perKey:
+          (await this.getTextIndexDictionaryBlockSize({
+            databaseName,
+            tableName,
+            connectionId,
+            indexName: options.indexName,
+          })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE,
+      })),
+    );
+    for (const { column, options, perKey } of mapColumns) {
       for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) {
         promises.push(
           this.getMapTextIndexKeyValues({