Skip to content

Commit f5b4b4e

Browse files
authored
Merge branch 'main' into claude/db-statement-index-prefilter
2 parents 474f80a + d341ba0 commit f5b4b4e

3 files changed

Lines changed: 209 additions & 12 deletions

File tree

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
'@hyperdx/app': patch
3+
---
4+
5+
fix: resolve row details with the primary key plus whichever of `_block_number`
6+
/ `_block_offset` the table persists, instead of falling back to the slow
7+
full-row lookup when only one of `enable_block_number_column` /
8+
`enable_block_offset_column` is set

‎packages/app/src/components/DBRowTable.tsx‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1482,6 +1482,23 @@ export function appendSelectWithAdditionalKeys(
14821482
}
14831483
}
14841484

1485+
// Block columns are only stable row identifiers when the table persists them
1486+
// (the virtual versions are renumbered on merge). Either one combined with the
1487+
// primary key is unique enough: two rows would need identical key values
1488+
// (incl. a nanosecond timestamp) plus the same insert block or the same offset
1489+
// within their blocks. So a table that persists only one of them still skips
1490+
// the slow full-row lookup.
1491+
export function getPersistedBlockColumns(engineFull: string): string[] {
1492+
const columns: string[] = [];
1493+
if (engineFull.includes('enable_block_number_column = 1')) {
1494+
columns.push('_block_number');
1495+
}
1496+
if (engineFull.includes('enable_block_offset_column = 1')) {
1497+
columns.push('_block_offset');
1498+
}
1499+
return columns;
1500+
}
1501+
14851502
function getSelectLength(select: SelectList): number {
14861503
if (typeof select === 'string') {
14871504
return select.split(',').filter(s => s.trim().length > 0).length;
@@ -1520,16 +1537,16 @@ export function useConfigWithAdditionalSelect(
15201537
}
15211538

15221539
let extraKeys: string[] = [];
1540+
let hasBlockColumns = false;
15231541

15241542
if (sourceId) {
1525-
const engineFull = tableMetadata?.engine_full ?? '';
1526-
1527-
const hasBlockColumns =
1528-
engineFull.includes('enable_block_number_column = 1') &&
1529-
engineFull.includes('enable_block_offset_column = 1');
1543+
const blockColumns = getPersistedBlockColumns(
1544+
tableMetadata?.engine_full ?? '',
1545+
);
15301546

1531-
if (hasBlockColumns) {
1532-
extraKeys = ['_block_number', '_block_offset'];
1547+
if (blockColumns.length > 0) {
1548+
extraKeys = blockColumns;
1549+
hasBlockColumns = true;
15331550
} else if (columns?.some(c => c.name === '__hdx_id')) {
15341551
extraKeys = ['__hdx_id'];
15351552
}
@@ -1548,10 +1565,6 @@ export function useConfigWithAdditionalSelect(
15481565
// column references like Timestamp, ServiceName) so the row WHERE clause
15491566
// can use only these, avoiding expensive index loading on large columns
15501567
// like Body.
1551-
const hasBlockColumns =
1552-
extraKeys.includes('_block_number') &&
1553-
extraKeys.includes('_block_offset');
1554-
15551568
const rowKeyColumns = hasBlockColumns
15561569
? new Set([
15571570
...splitAndTrimWithBracket(partitionKey),

‎packages/app/src/components/__tests__/DBRowTable.test.tsx‎

Lines changed: 177 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,37 @@
11
import { Provider } from 'jotai';
22
import type { ReactElement } from 'react';
3-
import { screen, waitFor, within } from '@testing-library/react';
3+
import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
4+
import { renderHook, screen, waitFor, within } from '@testing-library/react';
45
import userEvent from '@testing-library/user-event';
56

67
import {
78
appendSelectWithAdditionalKeys,
9+
getPersistedBlockColumns,
810
RawLogTable,
11+
useConfigWithAdditionalSelect,
912
} from '@/components/DBRowTable';
1013
import * as useChartConfigModule from '@/hooks/useChartConfig';
1114
import { RowWhereResult } from '@/hooks/useRowWhere';
1215
import { RowClickAction, UserPreferences } from '@/useUserPreferences';
1316

1417
const mockSetQueryState = jest.fn();
1518

19+
// useConfigWithAdditionalSelect reads table metadata through react-query
20+
// hooks that return a large UseQueryResult; the hook only reads `data`.
21+
// Reading the handles back through `requireMock` keeps them typed as mocks
22+
// without asserting a complete UseQueryResult.
23+
jest.mock('@/hooks/useMetadata', () => ({
24+
...jest.requireActual('@/hooks/useMetadata'),
25+
useTableMetadata: jest.fn(),
26+
useColumns: jest.fn(),
27+
}));
28+
29+
const { useTableMetadata: mockUseTableMetadata, useColumns: mockUseColumns } =
30+
jest.requireMock<{
31+
useTableMetadata: jest.Mock;
32+
useColumns: jest.Mock;
33+
}>('@/hooks/useMetadata');
34+
1635
// The inline expanded row reads the row query params through nuqs, which needs
1736
// a mounted Next router. Routing every setter through one mock lets tests assert
1837
// which param the expanded row's side panel button writes.
@@ -774,3 +793,160 @@ describe('appendSelectWithAdditionalKeys', () => {
774793
});
775794
});
776795
});
796+
797+
describe('getPersistedBlockColumns', () => {
798+
const engine = (settings: string) =>
799+
`MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, Timestamp) SETTINGS index_granularity = 8192${settings}`;
800+
801+
it('returns both block columns when both are persisted', () => {
802+
expect(
803+
getPersistedBlockColumns(
804+
engine(
805+
', enable_block_number_column = 1, enable_block_offset_column = 1',
806+
),
807+
),
808+
).toEqual(['_block_number', '_block_offset']);
809+
});
810+
811+
it('returns only _block_offset when only enable_block_offset_column is set', () => {
812+
expect(
813+
getPersistedBlockColumns(engine(', enable_block_offset_column = 1')),
814+
).toEqual(['_block_offset']);
815+
});
816+
817+
it('returns only _block_number when only enable_block_number_column is set', () => {
818+
expect(
819+
getPersistedBlockColumns(engine(', enable_block_number_column = 1')),
820+
).toEqual(['_block_number']);
821+
});
822+
823+
it('returns nothing when neither setting is present', () => {
824+
expect(getPersistedBlockColumns(engine(''))).toEqual([]);
825+
expect(getPersistedBlockColumns('')).toEqual([]);
826+
});
827+
});
828+
829+
describe('useConfigWithAdditionalSelect', () => {
830+
const config: BuilderChartConfigWithDateRange = {
831+
connection: 'conn',
832+
from: { databaseName: 'default', tableName: 'otel_logs' },
833+
select: 'Timestamp, ServiceName, Body',
834+
where: '',
835+
whereLanguage: 'sql',
836+
timestampValueExpression: 'Timestamp',
837+
dateRange: [new Date(0), new Date(1)],
838+
};
839+
840+
const mockMetadata = ({
841+
engineFull,
842+
columns = [],
843+
}: {
844+
engineFull: string;
845+
columns?: { name: string; type: string }[];
846+
}) => {
847+
mockUseTableMetadata.mockReturnValue({
848+
data: {
849+
primary_key: 'toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp',
850+
partition_key: 'toDate(Timestamp)',
851+
engine_full: engineFull,
852+
},
853+
});
854+
mockUseColumns.mockReturnValue({ data: columns });
855+
};
856+
857+
afterEach(() => {
858+
mockUseTableMetadata.mockReset();
859+
mockUseColumns.mockReset();
860+
});
861+
862+
it('restricts the row lookup to PK + _block_offset when only the offset column is persisted', () => {
863+
mockMetadata({
864+
engineFull:
865+
'MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) SETTINGS index_granularity = 8192, enable_block_offset_column = 1',
866+
});
867+
868+
const { result } = renderHook(() =>
869+
useConfigWithAdditionalSelect(config, 'source-id'),
870+
);
871+
872+
expect(result.current?.select).toBe(
873+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_offset',
874+
);
875+
expect(result.current?.additionalKeysLength).toBe(3);
876+
expect(result.current?.rowKeyColumns).toEqual(
877+
new Set([
878+
'toDate(Timestamp)',
879+
'toStartOfFiveMinutes(Timestamp)',
880+
'ServiceName',
881+
'Timestamp',
882+
'_block_offset',
883+
]),
884+
);
885+
expect(result.current?.rowKeyColumns?.has('Body')).toBe(false);
886+
});
887+
888+
it('restricts the row lookup to PK + _block_number when only the number column is persisted', () => {
889+
mockMetadata({
890+
engineFull:
891+
'MergeTree ORDER BY Timestamp SETTINGS enable_block_number_column = 1',
892+
});
893+
894+
const { result } = renderHook(() =>
895+
useConfigWithAdditionalSelect(config, 'source-id'),
896+
);
897+
898+
expect(result.current?.select).toBe(
899+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number',
900+
);
901+
expect(result.current?.rowKeyColumns?.has('_block_number')).toBe(true);
902+
expect(result.current?.rowKeyColumns?.has('Body')).toBe(false);
903+
});
904+
905+
it('uses both block columns when both are persisted', () => {
906+
mockMetadata({
907+
engineFull:
908+
'MergeTree ORDER BY Timestamp SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1',
909+
});
910+
911+
const { result } = renderHook(() =>
912+
useConfigWithAdditionalSelect(config, 'source-id'),
913+
);
914+
915+
expect(result.current?.select).toBe(
916+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number,_block_offset',
917+
);
918+
expect(result.current?.rowKeyColumns?.has('_block_number')).toBe(true);
919+
expect(result.current?.rowKeyColumns?.has('_block_offset')).toBe(true);
920+
});
921+
922+
it('falls back to __hdx_id and the full row when no block column is persisted', () => {
923+
mockMetadata({
924+
engineFull:
925+
'MergeTree ORDER BY Timestamp SETTINGS index_granularity = 8192',
926+
columns: [{ name: '__hdx_id', type: 'String' }],
927+
});
928+
929+
const { result } = renderHook(() =>
930+
useConfigWithAdditionalSelect(config, 'source-id'),
931+
);
932+
933+
expect(result.current?.select).toBe(
934+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),__hdx_id',
935+
);
936+
expect(result.current?.rowKeyColumns).toBeUndefined();
937+
});
938+
939+
it('ignores block columns for aggregate queries without a sourceId', () => {
940+
mockMetadata({
941+
engineFull:
942+
'MergeTree ORDER BY Timestamp SETTINGS enable_block_offset_column = 1',
943+
});
944+
945+
const { result } = renderHook(() => useConfigWithAdditionalSelect(config));
946+
947+
expect(result.current?.select).toBe(
948+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp)',
949+
);
950+
expect(result.current?.rowKeyColumns).toBeUndefined();
951+
});
952+
});

0 commit comments

Comments
 (0)