Skip to content

Commit 4696fae

Browse files
committed
feat(app): fall back to server MergeTree settings for block columns
1 parent 9df8b8d commit 4696fae

7 files changed

Lines changed: 386 additions & 18 deletions

File tree

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@hyperdx/app': patch
3+
'@hyperdx/common-utils': patch
4+
---
5+
6+
feat: fall back to server MergeTree settings to detect persisted `_block_number` and `_block_offset` columns when the table's CREATE query omits them

‎packages/app/src/components/DBRowTable.tsx‎

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,11 @@ import {
7878
useRenderedSqlChartConfig,
7979
} from '@/hooks/useChartConfig';
8080
import { CsvColumn, useCsvExport } from '@/hooks/useCsvExport';
81-
import { useColumns, useTableMetadata } from '@/hooks/useMetadata';
81+
import {
82+
useColumns,
83+
useMergeTreeSettings,
84+
useTableMetadata,
85+
} from '@/hooks/useMetadata';
8286
import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
8387
import { useGroupedPatterns } from '@/hooks/usePatterns';
8488
import { useRowSelection } from '@/hooks/useRowSelection';
@@ -1535,15 +1539,36 @@ export function appendSelectWithAdditionalKeys(
15351539
// (incl. a nanosecond timestamp) plus the same insert block or the same offset
15361540
// within their blocks. So a table that persists only one of them still skips
15371541
// the slow full-row lookup.
1538-
export function getPersistedBlockColumns(engineFull: string): string[] {
1539-
const columns: string[] = [];
1540-
if (engineFull.includes('enable_block_number_column = 1')) {
1541-
columns.push('_block_number');
1542-
}
1543-
if (engineFull.includes('enable_block_offset_column = 1')) {
1544-
columns.push('_block_offset');
1545-
}
1546-
return columns;
1542+
const BLOCK_COLUMN_SETTINGS = [
1543+
{ column: '_block_number', setting: 'enable_block_number_column' },
1544+
{ column: '_block_offset', setting: 'enable_block_offset_column' },
1545+
];
1546+
1547+
// engine_full ends with the SETTINGS clause.
1548+
function getEngineSetting(engineFull: string, name: string) {
1549+
const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? '';
1550+
return splitAndTrimWithBracket(settingsClause)
1551+
.map(setting => setting.split(' = '))
1552+
.find(([key]) => key === name)?.[1];
1553+
}
1554+
1555+
function omitsBlockColumnSetting(engineFull: string) {
1556+
return BLOCK_COLUMN_SETTINGS.some(
1557+
({ setting }) => getEngineSetting(engineFull, setting) == null,
1558+
);
1559+
}
1560+
1561+
// Falls back to server values for omitted settings; pass them for MergeTree.
1562+
export function getPersistedBlockColumns(
1563+
engineFull: string,
1564+
serverMergeTreeSettings?: Map<string, string> | null,
1565+
): string[] {
1566+
return BLOCK_COLUMN_SETTINGS.filter(({ setting }) => {
1567+
const value =
1568+
getEngineSetting(engineFull, setting) ??
1569+
serverMergeTreeSettings?.get(setting);
1570+
return value === '1' || value === 'true';
1571+
}).map(({ column }) => column);
15471572
}
15481573

15491574
function getSelectLength(select: SelectList): number {
@@ -1575,11 +1600,27 @@ export function useConfigWithAdditionalSelect(
15751600
{ enabled: !!sourceId },
15761601
);
15771602

1603+
const engineFull = tableMetadata?.engine_full ?? '';
1604+
const needsMergeTreeSettings =
1605+
!!sourceId &&
1606+
!!tableMetadata?.engine?.includes('MergeTree') &&
1607+
omitsBlockColumnSetting(engineFull);
1608+
1609+
const { data: mergeTreeSettings, isLoading: isMergeTreeSettingsLoading } =
1610+
useMergeTreeSettings(config.connection, {
1611+
enabled: needsMergeTreeSettings && !!config.connection,
1612+
});
1613+
15781614
const primaryKey = tableMetadata?.primary_key;
15791615
const partitionKey = tableMetadata?.partition_key;
15801616

15811617
return useMemo(() => {
1582-
if (primaryKey == null || partitionKey == null) {
1618+
// Wait for settings to avoid querying twice.
1619+
if (
1620+
primaryKey == null ||
1621+
partitionKey == null ||
1622+
isMergeTreeSettingsLoading
1623+
) {
15831624
return undefined;
15841625
}
15851626

@@ -1588,7 +1629,8 @@ export function useConfigWithAdditionalSelect(
15881629

15891630
if (sourceId) {
15901631
const blockColumns = getPersistedBlockColumns(
1591-
tableMetadata?.engine_full ?? '',
1632+
engineFull,
1633+
needsMergeTreeSettings ? mergeTreeSettings : undefined,
15921634
);
15931635

15941636
if (blockColumns.length > 0) {
@@ -1623,7 +1665,17 @@ export function useConfigWithAdditionalSelect(
16231665
: undefined;
16241666

16251667
return { ...config, select, additionalKeysLength, rowKeyColumns };
1626-
}, [primaryKey, partitionKey, config, tableMetadata, columns, sourceId]);
1668+
}, [
1669+
primaryKey,
1670+
partitionKey,
1671+
isMergeTreeSettingsLoading,
1672+
config,
1673+
engineFull,
1674+
needsMergeTreeSettings,
1675+
columns,
1676+
mergeTreeSettings,
1677+
sourceId,
1678+
]);
16271679
}
16281680

16291681
function selectColumnMapWithoutAdditionalKeys(

‎packages/app/src/components/__tests__/DBRowTable.test.tsx‎

Lines changed: 151 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,18 @@ jest.mock('@/hooks/useMetadata', () => ({
2424
...jest.requireActual('@/hooks/useMetadata'),
2525
useTableMetadata: jest.fn(),
2626
useColumns: jest.fn(),
27+
useMergeTreeSettings: jest.fn(),
2728
}));
2829

29-
const { useTableMetadata: mockUseTableMetadata, useColumns: mockUseColumns } =
30-
jest.requireMock<{
31-
useTableMetadata: jest.Mock;
32-
useColumns: jest.Mock;
33-
}>('@/hooks/useMetadata');
30+
const {
31+
useTableMetadata: mockUseTableMetadata,
32+
useColumns: mockUseColumns,
33+
useMergeTreeSettings: mockUseMergeTreeSettings,
34+
} = jest.requireMock<{
35+
useTableMetadata: jest.Mock;
36+
useColumns: jest.Mock;
37+
useMergeTreeSettings: jest.Mock;
38+
}>('@/hooks/useMetadata');
3439

3540
// The inline expanded row reads the row query params through nuqs, which needs
3641
// a mounted Next router. Routing every setter through one mock lets tests assert
@@ -859,6 +864,41 @@ describe('getPersistedBlockColumns', () => {
859864
expect(getPersistedBlockColumns(engine(''))).toEqual([]);
860865
expect(getPersistedBlockColumns('')).toEqual([]);
861866
});
867+
868+
it('accepts true as an enabled value', () => {
869+
expect(
870+
getPersistedBlockColumns(engine(', enable_block_offset_column = true')),
871+
).toEqual(['_block_offset']);
872+
});
873+
874+
it('falls back to the server setting when the table does not set it', () => {
875+
const serverSettings = new Map([
876+
['enable_block_number_column', '1'],
877+
['enable_block_offset_column', '1'],
878+
]);
879+
expect(getPersistedBlockColumns(engine(''), serverSettings)).toEqual([
880+
'_block_number',
881+
'_block_offset',
882+
]);
883+
expect(
884+
getPersistedBlockColumns(
885+
engine(', enable_block_number_column = 1'),
886+
new Map([['enable_block_offset_column', '1']]),
887+
),
888+
).toEqual(['_block_number', '_block_offset']);
889+
});
890+
891+
it('prefers the table setting over the server setting', () => {
892+
expect(
893+
getPersistedBlockColumns(
894+
engine(', enable_block_number_column = 0'),
895+
new Map([
896+
['enable_block_number_column', '1'],
897+
['enable_block_offset_column', '0'],
898+
]),
899+
),
900+
).toEqual([]);
901+
});
862902
});
863903

864904
describe('useConfigWithAdditionalSelect', () => {
@@ -873,25 +913,40 @@ describe('useConfigWithAdditionalSelect', () => {
873913
};
874914

875915
const mockMetadata = ({
916+
engine = 'MergeTree',
876917
engineFull,
877918
columns = [],
919+
mergeTreeSettings = new Map(),
920+
isMergeTreeSettingsLoading = false,
878921
}: {
922+
engine?: string;
879923
engineFull: string;
880924
columns?: { name: string; type: string }[];
925+
mergeTreeSettings?: Map<string, string>;
926+
isMergeTreeSettingsLoading?: boolean;
881927
}) => {
882928
mockUseTableMetadata.mockReturnValue({
883929
data: {
884930
primary_key: 'toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp',
885931
partition_key: 'toDate(Timestamp)',
932+
engine,
886933
engine_full: engineFull,
887934
},
888935
});
889936
mockUseColumns.mockReturnValue({ data: columns });
937+
// react-query never reports a disabled query as loading
938+
mockUseMergeTreeSettings.mockImplementation(
939+
(_connectionId: string, { enabled }: { enabled: boolean }) => ({
940+
data: isMergeTreeSettingsLoading ? undefined : mergeTreeSettings,
941+
isLoading: enabled && isMergeTreeSettingsLoading,
942+
}),
943+
);
890944
};
891945

892946
afterEach(() => {
893947
mockUseTableMetadata.mockReset();
894948
mockUseColumns.mockReset();
949+
mockUseMergeTreeSettings.mockReset();
895950
});
896951

897952
it('restricts the row lookup to PK + _block_offset when only the offset column is persisted', () => {
@@ -984,4 +1039,95 @@ describe('useConfigWithAdditionalSelect', () => {
9841039
);
9851040
expect(result.current?.rowKeyColumns).toBeUndefined();
9861041
});
1042+
1043+
it('uses the server MergeTree settings when the table does not set them', () => {
1044+
mockMetadata({
1045+
engine: 'SharedMergeTree',
1046+
engineFull:
1047+
'SharedMergeTree ORDER BY Timestamp SETTINGS index_granularity = 8192',
1048+
mergeTreeSettings: new Map([
1049+
['enable_block_number_column', '1'],
1050+
['enable_block_offset_column', '1'],
1051+
]),
1052+
});
1053+
1054+
const { result } = renderHook(() =>
1055+
useConfigWithAdditionalSelect(config, 'source-id'),
1056+
);
1057+
1058+
expect(result.current?.select).toBe(
1059+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number,_block_offset',
1060+
);
1061+
expect(result.current?.rowKeyColumns?.has('_block_number')).toBe(true);
1062+
expect(result.current?.rowKeyColumns?.has('_block_offset')).toBe(true);
1063+
});
1064+
1065+
it('ignores the server MergeTree settings for non-MergeTree engines', () => {
1066+
mockMetadata({
1067+
engine: 'Merge',
1068+
engineFull: "Merge('default', '^otel_logs')",
1069+
mergeTreeSettings: new Map([['enable_block_number_column', '1']]),
1070+
});
1071+
1072+
const { result } = renderHook(() =>
1073+
useConfigWithAdditionalSelect(config, 'source-id'),
1074+
);
1075+
1076+
expect(mockUseMergeTreeSettings).toHaveBeenCalledWith('conn', {
1077+
enabled: false,
1078+
});
1079+
expect(result.current?.select).toBe(
1080+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp)',
1081+
);
1082+
expect(result.current?.rowKeyColumns).toBeUndefined();
1083+
});
1084+
1085+
it('skips the server MergeTree settings when the table sets both', () => {
1086+
mockMetadata({
1087+
engineFull:
1088+
'MergeTree ORDER BY Timestamp SETTINGS enable_block_number_column = 0, enable_block_offset_column = 0',
1089+
mergeTreeSettings: new Map([
1090+
['enable_block_number_column', '1'],
1091+
['enable_block_offset_column', '1'],
1092+
]),
1093+
isMergeTreeSettingsLoading: true,
1094+
});
1095+
1096+
const { result } = renderHook(() =>
1097+
useConfigWithAdditionalSelect(config, 'source-id'),
1098+
);
1099+
1100+
expect(mockUseMergeTreeSettings).toHaveBeenCalledWith('conn', {
1101+
enabled: false,
1102+
});
1103+
expect(result.current?.select).toBe(
1104+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp)',
1105+
);
1106+
});
1107+
1108+
it('waits for the server MergeTree settings before returning a config', () => {
1109+
mockMetadata({
1110+
engineFull: 'MergeTree ORDER BY Timestamp',
1111+
isMergeTreeSettingsLoading: true,
1112+
});
1113+
1114+
const { result, rerender } = renderHook(() =>
1115+
useConfigWithAdditionalSelect(config, 'source-id'),
1116+
);
1117+
1118+
expect(mockUseMergeTreeSettings).toHaveBeenCalledWith('conn', {
1119+
enabled: true,
1120+
});
1121+
expect(result.current).toBeUndefined();
1122+
1123+
mockMetadata({
1124+
engineFull: 'MergeTree ORDER BY Timestamp',
1125+
mergeTreeSettings: new Map([['enable_block_offset_column', '1']]),
1126+
});
1127+
rerender();
1128+
1129+
expect(result.current?.select).toBe(
1130+
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_offset',
1131+
);
1132+
});
9871133
});

‎packages/app/src/hooks/__tests__/useMetadata.test.tsx‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
deduplicate2dArray,
1919
intersect2dArray,
2020
useGetKeyValues,
21+
useMergeTreeSettings,
2122
useMultipleAllFields,
2223
useMultipleGetKeyValues,
2324
} from '@/hooks/useMetadata';
@@ -948,6 +949,62 @@ describe('useMultipleAllFields', () => {
948949
});
949950
});
950951

952+
describe('useMergeTreeSettings', () => {
953+
let wrapper: React.ComponentType<{ children: any }>;
954+
let mockMetadata: Metadata;
955+
const settings = new Map([['enable_block_number_column', '1']]);
956+
957+
beforeEach(() => {
958+
jest.clearAllMocks();
959+
mockMetadata = createMockMetadata();
960+
jest.spyOn(metadataModule, 'getMetadata').mockReturnValue(mockMetadata);
961+
const queryClient = new QueryClient({
962+
defaultOptions: { queries: { retry: false } },
963+
});
964+
wrapper = ({ children }) => (
965+
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
966+
);
967+
});
968+
969+
it('keeps a successful result fresh across mounts', async () => {
970+
const spy = jest
971+
.spyOn(mockMetadata, 'getMergeTreeSettings')
972+
.mockResolvedValue(settings);
973+
974+
const first = renderHook(() => useMergeTreeSettings('conn1'), { wrapper });
975+
await waitFor(() => expect(first.result.current.data).toEqual(settings));
976+
first.unmount();
977+
978+
const second = renderHook(() => useMergeTreeSettings('conn1'), { wrapper });
979+
expect(second.result.current.data).toEqual(settings);
980+
expect(spy).toHaveBeenCalledTimes(1);
981+
});
982+
983+
it('returns null on failure and refetches on the next mount', async () => {
984+
const spy = jest
985+
.spyOn(mockMetadata, 'getMergeTreeSettings')
986+
.mockResolvedValueOnce(undefined)
987+
.mockResolvedValueOnce(settings);
988+
989+
const first = renderHook(() => useMergeTreeSettings('conn1'), { wrapper });
990+
await waitFor(() => expect(first.result.current.data).toBeNull());
991+
first.unmount();
992+
993+
const second = renderHook(() => useMergeTreeSettings('conn1'), { wrapper });
994+
await waitFor(() => expect(second.result.current.data).toEqual(settings));
995+
expect(spy).toHaveBeenCalledTimes(2);
996+
});
997+
998+
it('does not query without a connection', () => {
999+
const spy = jest.spyOn(mockMetadata, 'getMergeTreeSettings');
1000+
1001+
const { result } = renderHook(() => useMergeTreeSettings(''), { wrapper });
1002+
1003+
expect(result.current.isLoading).toBe(false);
1004+
expect(spy).not.toHaveBeenCalled();
1005+
});
1006+
});
1007+
9511008
describe('deduplicate2dArray', () => {
9521009
// Test basic deduplication
9531010
it('should remove duplicate objects across 2D array', () => {

0 commit comments

Comments
 (0)