Skip to content

Commit ab7645d

Browse files
authored
feat: Add a minimum date to MV configuration (#1549)
Closes HDX-3129 # Summary This PR adds a "minimum date" setting to Materialized view configurations. The minimum date is optional. When provided, the materialized view will not be used for any query that includes date ranges prior to the minimum date. ## Demo https://github.com/user-attachments/assets/de0efb45-786e-4b45-b053-f1dc7c322f2e
1 parent ac1a2f7 commit ab7645d

6 files changed

Lines changed: 204 additions & 8 deletions

File tree

‎.changeset/calm-dragons-lie.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@hyperdx/common-utils": patch
3+
"@hyperdx/app": patch
4+
---
5+
6+
feat: Add a minimum date to MV configuration

‎packages/app/src/components/MaterializedViews/MVConfigSummary.tsx‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import { useMemo } from 'react';
22
import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
33
import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types';
4-
import { Group, Pill, Stack, Table, Text } from '@mantine/core';
4+
import { Grid, Group, Pill, Stack, Table, Text } from '@mantine/core';
5+
6+
import { FormatTime } from '@/useFormatTime';
57

68
export default function MVConfigSummary({
79
config,
@@ -29,12 +31,24 @@ export default function MVConfigSummary({
2931

3032
return (
3133
<Stack gap="md">
32-
<div>
33-
<Text size="sm" fw={500} mb="xs">
34-
Minimum Granularity
35-
</Text>
36-
<Pill>{config.minGranularity}</Pill>
37-
</div>
34+
<Grid columns={2}>
35+
<Grid.Col span={1}>
36+
<Text size="sm" fw={500} mb="xs">
37+
Granularity
38+
</Text>
39+
<Pill>{config.minGranularity}</Pill>
40+
</Grid.Col>
41+
{config.minDate && (
42+
<Grid.Col span={1}>
43+
<Text size="sm" fw={500} mb="xs">
44+
Minimum Date
45+
</Text>
46+
<Pill>
47+
<FormatTime value={config.minDate} format="withYear" />
48+
</Pill>
49+
</Grid.Col>
50+
)}
51+
</Grid>
3852

3953
<div>
4054
<Text size="sm" fw={500} mb="xs">

‎packages/app/src/components/SourceForm.tsx‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
Text,
3434
Tooltip,
3535
} from '@mantine/core';
36+
import { DateInput } from '@mantine/dates';
3637
import { notifications } from '@mantine/notifications';
3738
import {
3839
IconCirclePlus,
@@ -402,7 +403,7 @@ function MaterializedViewFormSection({
402403
</Group>
403404
</Grid.Col>
404405

405-
<Grid.Col span={1}>
406+
<Grid.Col span={2}>
406407
<Text size="xs" fw={500} mb={4}>
407408
Timestamp Column
408409
</Text>
@@ -445,6 +446,38 @@ function MaterializedViewFormSection({
445446
)}
446447
/>
447448
</Grid.Col>
449+
450+
<Grid.Col span={1}>
451+
<Text size="xs" fw={500} mb={4}>
452+
Minimum Date
453+
<Tooltip
454+
label="(Optional) The earliest date and time (in the local timezone) for which the materialized view contains data. If not provided, then HyperDX will assume that the materialized view contains data for all dates for which the source table contains data."
455+
color="dark"
456+
c="white"
457+
multiline
458+
maw={600}
459+
>
460+
<IconHelpCircle size={14} className="cursor-pointer ms-1" />
461+
</Tooltip>
462+
</Text>
463+
<Controller
464+
control={control}
465+
name={`materializedViews.${mvIndex}.minDate`}
466+
render={({ field }) => (
467+
<DateInput
468+
{...field}
469+
value={field.value ? new Date(field.value) : undefined}
470+
onChange={dateStr =>
471+
field.onChange(dateStr ? dateStr.toISOString() : null)
472+
}
473+
clearable
474+
highlightToday
475+
placeholder="YYYY-MM-DD HH:mm:ss"
476+
valueFormat="YYYY-MM-DD HH:mm:ss"
477+
/>
478+
)}
479+
/>
480+
</Grid.Col>
448481
</Grid>
449482

450483
<Box>

‎packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,122 @@ describe('materializedViews', () => {
781781
'Custom count() expressions are not supported with materialized views.',
782782
]);
783783
});
784+
785+
it("should not use the materialized view when the chart config references a date range prior to the MV's date range", async () => {
786+
const chartConfig: ChartConfigWithOptDateRange = {
787+
from: {
788+
databaseName: 'default',
789+
tableName: 'otel_spans',
790+
},
791+
select: [
792+
{
793+
valueExpression: '',
794+
aggFn: 'count',
795+
},
796+
],
797+
where: '',
798+
connection: 'test-connection',
799+
dateRange: [
800+
new Date('2022-12-01T00:00:00Z'),
801+
new Date('2022-12-31T23:59:59Z'),
802+
],
803+
};
804+
805+
const result = await tryConvertConfigToMaterializedViewSelect(
806+
chartConfig,
807+
{
808+
...MV_CONFIG_METRIC_ROLLUP_1M,
809+
minDate: '2023-01-01T00:00:00Z',
810+
},
811+
metadata,
812+
);
813+
814+
expect(result.optimizedConfig).toBeUndefined();
815+
expect(result.errors).toEqual([
816+
'The selected date range includes dates for which this view does not contain data.',
817+
]);
818+
});
819+
820+
it('should not use the materialized view when the chart config has no date range and the MV has a minimum date', async () => {
821+
const chartConfig: ChartConfigWithOptDateRange = {
822+
from: {
823+
databaseName: 'default',
824+
tableName: 'otel_spans',
825+
},
826+
select: [
827+
{
828+
valueExpression: '',
829+
aggFn: 'count',
830+
},
831+
],
832+
where: '',
833+
connection: 'test-connection',
834+
};
835+
836+
const result = await tryConvertConfigToMaterializedViewSelect(
837+
chartConfig,
838+
{
839+
...MV_CONFIG_METRIC_ROLLUP_1M,
840+
minDate: '2023-01-01T00:00:00Z',
841+
},
842+
metadata,
843+
);
844+
845+
expect(result.optimizedConfig).toBeUndefined();
846+
expect(result.errors).toEqual([
847+
'The selected date range includes dates for which this view does not contain data.',
848+
]);
849+
});
850+
851+
it("should use the materialized view when the chart config's date range is after the MV's minimum date", async () => {
852+
const chartConfig: ChartConfigWithOptDateRange = {
853+
from: {
854+
databaseName: 'default',
855+
tableName: 'otel_spans',
856+
},
857+
select: [
858+
{
859+
valueExpression: '',
860+
aggFn: 'count',
861+
},
862+
],
863+
where: '',
864+
connection: 'test-connection',
865+
dateRange: [
866+
new Date('2023-12-01T00:00:00Z'),
867+
new Date('2023-12-31T23:59:59Z'),
868+
],
869+
};
870+
871+
const result = await tryConvertConfigToMaterializedViewSelect(
872+
chartConfig,
873+
{
874+
...MV_CONFIG_METRIC_ROLLUP_1M,
875+
minDate: '2023-01-01T00:00:00Z',
876+
},
877+
metadata,
878+
);
879+
880+
expect(result.optimizedConfig).toEqual({
881+
from: {
882+
databaseName: 'default',
883+
tableName: 'metrics_rollup_1m',
884+
},
885+
dateRange: [
886+
new Date('2023-12-01T00:00:00Z'),
887+
new Date('2023-12-31T23:59:59Z'),
888+
],
889+
select: [
890+
{
891+
valueExpression: 'count',
892+
aggFn: 'sum',
893+
},
894+
],
895+
where: '',
896+
connection: 'test-connection',
897+
});
898+
expect(result.errors).toBeUndefined();
899+
});
784900
});
785901

786902
describe('isUnsupportedCountFunction', () => {

‎packages/common-utils/src/core/materializedViews.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,24 @@ function mvConfigSupportsGranularity(
153153
return chartGranularitySeconds >= mvGranularitySeconds;
154154
}
155155

156+
function mvConfigSupportsDateRange(
157+
mvConfig: MaterializedViewConfiguration,
158+
chartConfig: ChartConfigWithOptDateRange,
159+
) {
160+
if (mvConfig.minDate && !chartConfig.dateRange) {
161+
return false;
162+
}
163+
164+
if (!mvConfig.minDate || !chartConfig.dateRange) {
165+
return true;
166+
}
167+
168+
const [startDate] = chartConfig.dateRange;
169+
const minDate = new Date(mvConfig.minDate);
170+
171+
return startDate >= minDate;
172+
}
173+
156174
const COUNT_FUNCTION_PATTERN = /\bcount(If)?\s*\(/i;
157175
export function isUnsupportedCountFunction(selectItem: SelectItem): boolean {
158176
return COUNT_FUNCTION_PATTERN.test(selectItem.valueExpression);
@@ -268,6 +286,14 @@ export async function tryConvertConfigToMaterializedViewSelect<
268286
};
269287
}
270288

289+
if (mvConfig.minDate && !mvConfigSupportsDateRange(mvConfig, chartConfig)) {
290+
return {
291+
errors: [
292+
'The selected date range includes dates for which this view does not contain data.',
293+
],
294+
};
295+
}
296+
271297
if (!mvConfigSupportsGranularity(mvConfig, chartConfig)) {
272298
const error = chartConfig.granularity
273299
? `Granularity must be at least ${mvConfig.minGranularity}.`

‎packages/common-utils/src/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,7 @@ export const MaterializedViewConfigurationSchema = z.object({
646646
tableName: z.string().min(1, 'Materialized View Table is required'),
647647
dimensionColumns: z.string(),
648648
minGranularity: SQLIntervalSchema,
649+
minDate: z.string().datetime().nullish(),
649650
timestampColumn: z
650651
.string()
651652
.min(1, 'Materialized View Timestamp column is required'),

0 commit comments

Comments
 (0)