Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1101,6 +1101,7 @@ describe('useFetchFacets', () => {
tableName: 'other_table',
connectionId: 'conn2',
keyExpressions: ['ServiceName'],
mapTextIndexPartsPerKey: 'all',
}),
);
expect(result.current.data.keyValues).toEqual([
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/DBSearchPageFilters/hooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,9 @@ function useFacets({
metadataMVs: tableConnection.metadataMVs,
keyExpressions: [sqlKey],
maxValuesPerKey: LOAD_MORE_LOAD_LIMIT,
// One key, so reading every part that holds it stays cheap, and
// the best part alone may hold far fewer values than requested.
mapTextIndexPartsPerKey: 'all',

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Filter fix lacks release entry

This changes how users load filter values, but the PR has no changeset. AGENTS.md requires a changeset for behavior changes to @hyperdx/app. Add a patch changeset before merging so the fix is included in the release.

Context Used: AGENTS.md (source)

Fix in Claude Code Fix in Conductor Fix in Cursor Fix in Codex

dateRange,
timestampValueExpression:
source?.timestampValueExpression ??
Expand Down
137 changes: 136 additions & 1 deletion packages/common-utils/src/__tests__/metadata.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { ClickHouseClient } from '@clickhouse/client';

import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node';
import { supportsMergeTreeTextIndex } from '@/core/clickhouseVersion';
import { Metadata, MetadataCache } from '@/core/metadata';
import { KeyValues, Metadata, MetadataCache } from '@/core/metadata';
import { MapTextIndexPartsPerKey } from '@/core/utils';
import {
parseKvItemsCastExpression,
parseKvItemsExpression,
Expand Down Expand Up @@ -1554,4 +1555,138 @@ describe('Metadata Integration Tests', () => {
});
},
);

describe('getAllKeyValues - Map values from the parts a keys index lists', () => {
const tableName = 'test_map_values_best_parts';
const keyExpressions = [
"LogAttributes['shared']",
"LogAttributes['rare.key']",
"LogAttributes['old.only']",
];
let metadata: Metadata;
let textIndexSupported = false;

beforeAll(async () => {
const probe = new Metadata(hdxClient, new MetadataCache());
textIndexSupported = supportsMergeTreeTextIndex(
await probe.getServerVersion({ connectionId: 'test_connection' }),
);
if (!textIndexSupported) return;

await client.command({
query: `CREATE OR REPLACE TABLE default.${tableName} (
Timestamp DateTime64(9),
LogAttributes Map(LowCardinality(String), String),
LogAttributeItems Array(String) MATERIALIZED
arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')),
INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
)
ENGINE = MergeTree()
PARTITION BY toDate(Timestamp)
ORDER BY Timestamp
`,
});
// Every insert below must stay its own part.
await client.command({
query: `SYSTEM STOP MERGES default.${tableName}`,
});
const insert = (minutesAgo: number, attributes: string, rows: number) =>
client.command({
query: `INSERT INTO default.${tableName} (Timestamp, LogAttributes)
SELECT now64(9) - INTERVAL ${minutesAgo} MINUTE, ${attributes}
FROM numbers(${rows})`,
});
// Written before the keys index exists, so the index never lists it.
await insert(30, `map('shared', 'from-old', 'old.only', 'x')`, 5);
await client.command({
query: `ALTER TABLE default.${tableName}
ADD INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`,
});
// The part holding `shared` in the most rows, and the only one with
// `rare.key`.
await insert(
20,
`if(number = 0, map('shared', 'from-big', 'rare.key', 'r1'), map('shared', 'from-big'))`,
50,
);
await insert(10, `map('shared', 'from-new')`, 1);
});

afterAll(async () => {
if (!textIndexSupported) return;
await client.command({
query: `DROP TABLE IF EXISTS default.${tableName}`,
});
});

beforeEach(() => {
metadata = new Metadata(hdxClient, new MetadataCache());
});

afterEach(() => {
jest.restoreAllMocks();
});

const getValues = (mapTextIndexPartsPerKey?: MapTextIndexPartsPerKey) =>
metadata.getAllKeyValues({
databaseName: 'default',
tableName,
connectionId: 'test_connection',
keyExpressions,
mapTextIndexPartsPerKey,
timestampValueExpression: 'Timestamp',
dateRange: [
new Date(Date.now() - 60 * 60 * 1000),
new Date(Date.now() + 60 * 1000),
],
});

const byKey = (result: KeyValues[]) =>
Object.fromEntries(result.map(({ key, value }) => [key, value]));

it('reads the best and newest part per key, and reports keys the index misses', async () => {
if (!textIndexSupported) {
console.warn(
'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
);
return;
}
const warnSpy = jest
.spyOn(console, 'warn')
.mockImplementation(() => undefined);

const values = byKey(await getValues());

expect(values["LogAttributes['shared']"]?.sort()).toEqual([
'from-big',
'from-new',
]);
expect(values["LogAttributes['rare.key']"]).toEqual(['r1']);
expect(values).not.toHaveProperty("LogAttributes['old.only']");
const warnings = warnSpy.mock.calls.map(([message]) => String(message));
expect(warnings).toEqual([
expect.stringContaining(
'LogAttributes keys [old.only]: keys index idx_log_attr_key lists the requested keys in 2 of 3 parts',
),
]);
});

it("reads every part the keys index lists when parts per key is 'all'", async () => {
if (!textIndexSupported) {
console.warn(
'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
);
return;
}
jest.spyOn(console, 'warn').mockImplementation(() => undefined);

const values = byKey(await getValues('all'));

expect(values["LogAttributes['shared']"]?.sort()).toEqual([
'from-big',
'from-new',
]);
Comment on lines +1683 to +1688

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 All-mode test misses the difference

The 'all' integration test uses only two indexed parts. The default path already selects both: the largest and the newest. Both tests therefore expect the same values, so this test would still pass if 'all' accidentally used the default path. Add a third indexed part with a unique value, and check that only 'all' returns it.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Claude Code Fix in Conductor Fix in Cursor Fix in Codex

expect(values["LogAttributes['rare.key']"]).toEqual(['r1']);
});
});
});
Loading
Loading