From a31aea56dd332fa70ee07f4eda881095d5eef8a7 Mon Sep 17 00:00:00 2001 From: yanglongwei Date: Mon, 2 Nov 2026 10:56:50 +0000 Subject: [PATCH] feat: add ClickHouse LIMIT AFTER/UNTIL as ChartConfig clauses for Table and Search. --- .../api/src/mcp/tools/dashboards/schemas.ts | 16 +++ .../src/mcp/tools/savedSearches/schemas.ts | 10 ++ packages/api/src/models/savedSearch.ts | 6 + .../routers/external-api/v2/savedSearches.ts | 2 + .../external-api/v2/utils/dashboards.ts | 23 +++ packages/api/src/utils/zod.ts | 4 + packages/app/src/DBSearchPage.tsx | 135 +++++++++++++++++- .../ChartEditor/__tests__/utils.test.ts | 31 ++++ .../app/src/components/ChartEditor/utils.ts | 21 +++ .../DBEditTimeChartForm/ChartActionBar.tsx | 36 +++++ packages/app/src/components/DBRowTable.tsx | 10 +- .../components/SQLEditor/SQLInlineEditor.tsx | 8 +- .../app/src/hooks/useOffsetPaginatedQuery.tsx | 19 ++- packages/app/src/types.ts | 2 + .../src/__tests__/renderChartConfig.test.ts | 117 +++++++++++++++ .../src/__tests__/variables.test.ts | 18 +++ .../core/__tests__/searchChartConfig.test.ts | 23 +++ .../src/core/renderChartConfig.ts | 37 ++++- .../src/core/searchChartConfig.ts | 15 ++ packages/common-utils/src/types.ts | 37 ++++- packages/common-utils/src/variables.ts | 14 ++ 21 files changed, 561 insertions(+), 23 deletions(-) diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index bd678a768e..486053511d 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -841,6 +841,22 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ 'When ordering by an alias that contains spaces or special characters, ' + `wrap the alias in quotes: e.g. '"P95 Latency" DESC'.`, ), + limitAfter: z + .string() + .max(10000) + .optional() + .describe( + 'ClickHouse LIMIT AFTER start expression. Mutually exclusive with OFFSET. ' + + 'Example: "service_id >= 3". Requires ORDER BY.', + ), + limitUntil: z + .string() + .max(10000) + .optional() + .describe( + 'ClickHouse LIMIT UNTIL end expression. Used with LIMIT AFTER. ' + + 'Example: "service_id >= 8".', + ), asRatio: z.boolean().optional().describe(asRatioDescription), groupByColumnsOnLeft: z .boolean() diff --git a/packages/api/src/mcp/tools/savedSearches/schemas.ts b/packages/api/src/mcp/tools/savedSearches/schemas.ts index 92e8d3ca80..b865db3078 100644 --- a/packages/api/src/mcp/tools/savedSearches/schemas.ts +++ b/packages/api/src/mcp/tools/savedSearches/schemas.ts @@ -78,6 +78,16 @@ export const mcpSaveSavedSearchSchema = z.object({ .string() .optional() .describe('Sort expression. Example: "Timestamp DESC"'), + limitAfter: z + .string() + .optional() + .describe( + 'ClickHouse LIMIT AFTER start expression. Mutually exclusive with OFFSET pagination.', + ), + limitUntil: z + .string() + .optional() + .describe('ClickHouse LIMIT UNTIL end expression. Used with LIMIT AFTER.'), sourceId: objectIdSchema.describe( 'Source ID — call clickstack_list_sources to find available sources.', ), diff --git a/packages/api/src/models/savedSearch.ts b/packages/api/src/models/savedSearch.ts index afda2efcb8..94fc0f8c62 100644 --- a/packages/api/src/models/savedSearch.ts +++ b/packages/api/src/models/savedSearch.ts @@ -11,6 +11,8 @@ type ExternalSavedSearch = { where?: string; whereLanguage?: string; orderBy?: string; + limitAfter?: string; + limitUntil?: string; sourceId?: string; tags?: string[]; filters?: unknown[]; @@ -52,6 +54,8 @@ const savedSearchSchema = new Schema< where: String, whereLanguage: String, orderBy: String, + limitAfter: String, + limitUntil: String, source: { type: mongoose.Schema.Types.ObjectId, required: true, @@ -88,6 +92,8 @@ savedSearchSchema.methods.toExternalJSON = function (): ExternalSavedSearch { where: this.where, whereLanguage: this.whereLanguage, orderBy: this.orderBy, + limitAfter: this.limitAfter, + limitUntil: this.limitUntil, sourceId: this.source ? String(this.source) : undefined, tags: this.tags, filters: this.filters, diff --git a/packages/api/src/routers/external-api/v2/savedSearches.ts b/packages/api/src/routers/external-api/v2/savedSearches.ts index 63b6959a5c..3786f7a4c6 100644 --- a/packages/api/src/routers/external-api/v2/savedSearches.ts +++ b/packages/api/src/routers/external-api/v2/savedSearches.ts @@ -98,6 +98,8 @@ const savedSearchRequestSchema = z.object({ .default(''), whereLanguage: z.enum(['lucene', 'sql']).optional().default('lucene'), orderBy: z.string().max(1024).optional().default(''), + limitAfter: z.string().max(4096).optional().default(''), + limitUntil: z.string().max(4096).optional().default(''), tags: tagsSchema.default([]), filters: z .array(savedSearchFilterSchema) diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 8ffa9564b3..565aa57e0d 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -587,6 +587,8 @@ export const convertToExternalTileChartConfig = ( ? config.select.map(convertToExternalSelectItem) : [DEFAULT_SELECT_ITEM], orderBy: stringValueOrDefault(config.orderBy, undefined), + limitAfter: config.limit?.after || undefined, + limitUntil: config.limit?.until || undefined, ...externalFormulaFields, ...externalShowOperandSeriesField, }; @@ -597,6 +599,8 @@ export const convertToExternalTileChartConfig = ( select: stringValueOrDefault(config.select, ''), where: config.where, whereLanguage: config.whereLanguage ?? 'lucene', + limitAfter: config.limit?.after || undefined, + limitUntil: config.limit?.until || undefined, }; case DisplayType.Markdown: return { @@ -917,6 +921,23 @@ const convertExternalPromqlConfigToInternal = ( } }; +function limitAfterUntilFields(externalConfig: { + limitAfter?: string; + limitUntil?: string; +}): { limit?: { after?: string; until?: string } } { + const after = externalConfig.limitAfter?.trim(); + const until = externalConfig.limitUntil?.trim(); + if (!after && !until) { + return {}; + } + return { + limit: { + ...(after ? { after } : {}), + ...(until ? { until } : {}), + }, + }; +} + export function convertToInternalTileConfig( externalTile: ConfigTile, ): DashboardDocument['tiles'][number] { @@ -1049,6 +1070,7 @@ export function convertToInternalTileConfig( source: externalConfig.sourceId, where: '', seriesReturnType: externalConfig.asRatio ? 'ratio' : undefined, + ...limitAfterUntilFields(externalConfig), name, description, } satisfies BuilderSavedChartConfig; @@ -1190,6 +1212,7 @@ export function convertToInternalTileConfig( name, description, whereLanguage: externalConfig.whereLanguage ?? 'lucene', + ...limitAfterUntilFields(externalConfig), } satisfies BuilderSavedChartConfig; break; case 'event_patterns': diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 8ec394afdf..f841d04a8e 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -422,6 +422,8 @@ const externalDashboardTableChartConfigSchema = z.object({ groupBy: z.string().max(10000).optional(), having: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), + limitAfter: z.string().max(10000).optional(), + limitUntil: z.string().max(10000).optional(), asRatio: z.boolean().optional(), numberFormat: NumberFormatSchema.optional(), groupByColumnsOnLeft: z.boolean().optional(), @@ -571,6 +573,8 @@ const externalDashboardSearchChartConfigSchema = z.object({ select: z.string().max(10000), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, + limitAfter: z.string().max(10000).optional(), + limitUntil: z.string().max(10000).optional(), }); // Extended schema for the /api/v2/search endpoint — adds orderBy which is not diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 9f5f62522c..953e675c8e 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -200,6 +200,8 @@ const SearchConfigSchema = z.object({ where: z.string(), whereLanguage: z.enum(['sql', 'lucene']), orderBy: z.string(), + limitAfter: z.string(), + limitUntil: z.string(), filters: z.array( z.union([ z.object({ @@ -543,6 +545,8 @@ function SaveSearchModalComponent({ searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig.source ?? '', orderBy: searchedConfig.orderBy ?? '', + limitAfter: searchedConfig.limitAfter ?? '', + limitUntil: searchedConfig.limitUntil ?? '', filters: searchedConfig.filters ?? [], tags: tags, }, @@ -571,6 +575,8 @@ function SaveSearchModalComponent({ searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig.source ?? '', orderBy: searchedConfig.orderBy ?? '', + limitAfter: searchedConfig.limitAfter ?? '', + limitUntil: searchedConfig.limitUntil ?? '', filters: searchedConfig.filters ?? [], tags: tags, }); @@ -805,7 +811,16 @@ function useLiveUpdate({ * and returns a chart config. */ function useSearchedConfigToChartConfig( - { select, source, whereLanguage, where, filters, orderBy }: SearchConfig, + { + select, + source, + whereLanguage, + where, + filters, + orderBy, + limitAfter, + limitUntil, + }: SearchConfig, defaultSearchConfig?: Partial, ) { const { data: sourceObj, isLoading } = useSource({ @@ -826,6 +841,8 @@ function useSearchedConfigToChartConfig( select: select || defaultSearchConfig?.select || null, displayType: DisplayType.Search, ...(resolvedOrderBy != null ? { orderBy: resolvedOrderBy } : {}), + limitAfter: limitAfter || defaultSearchConfig?.limitAfter, + limitUntil: limitUntil || defaultSearchConfig?.limitUntil, }); return { @@ -844,6 +861,8 @@ function useSearchedConfigToChartConfig( whereLanguage, defaultOrderBy, orderBy, + limitAfter, + limitUntil, ]); } @@ -923,6 +942,8 @@ const queryStateMap = { whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), filters: parseAsJsonEncoded(), orderBy: parseAsStringEncoded, + limitAfter: parseAsStringEncoded, + limitUntil: parseAsStringEncoded, }; export function buildSavedSearchNavigationUrl( @@ -1202,6 +1223,8 @@ function DBSearchPageContent() { : defaultSourceId), filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', + limitAfter: searchedConfig.limitAfter ?? '', + limitUntil: searchedConfig.limitUntil ?? '', }, resetOptions: { keepDirtyValues: true, @@ -1237,6 +1260,8 @@ function DBSearchPageContent() { source: _savedSearch?.source, filters: _savedSearch?.filters ?? [], orderBy: _savedSearch?.orderBy || defaultOrderBy, + limitAfter: _savedSearch?.limitAfter ?? '', + limitUntil: _savedSearch?.limitUntil ?? '', }; }, [searchedSource, inputSource, savedSearch, defaultOrderBy, savedSearchId]); @@ -1270,6 +1295,8 @@ function DBSearchPageContent() { source: searchedConfig?.source ?? undefined, filters: searchedConfig?.filters ?? [], orderBy: searchedConfig?.orderBy ?? '', + limitAfter: searchedConfig?.limitAfter ?? '', + limitUntil: searchedConfig?.limitUntil ?? '', }); } }, [searchedConfig, reset, prevSearched]); @@ -1300,6 +1327,8 @@ function DBSearchPageContent() { whereLanguage: savedSearch.whereLanguage as 'sql' | 'lucene', filters: savedSearch.filters ?? [], orderBy: savedSearch.orderBy ?? '', + limitAfter: savedSearch.limitAfter ?? '', + limitUntil: savedSearch.limitUntil ?? '', }); return; } @@ -1317,6 +1346,8 @@ function DBSearchPageContent() { whereLanguage: getStoredLanguage() ?? 'lucene', filters: [], orderBy: '', + limitAfter: '', + limitUntil: '', }); return; } @@ -1365,7 +1396,16 @@ function DBSearchPageContent() { ({ recordExploration = true }: { recordExploration?: boolean } = {}) => { onSearch(displayedTimeInputValue); handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { + ({ + select, + where, + whereLanguage, + source, + filters, + orderBy, + limitAfter, + limitUntil, + }) => { setSearchedConfig({ select, where, @@ -1373,6 +1413,8 @@ function DBSearchPageContent() { source, filters, orderBy, + limitAfter, + limitUntil, }); if ( recordExploration && @@ -1607,11 +1649,15 @@ function DBSearchPageContent() { searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', + limitAfter: searchedConfig.limitAfter ?? '', + limitUntil: searchedConfig.limitUntil ?? '', }), [ chartSourceId, searchedConfig.filters, searchedConfig.orderBy, + searchedConfig.limitAfter, + searchedConfig.limitUntil, searchedConfig.select, searchedConfig.where, searchedConfig.whereLanguage, @@ -1678,6 +1724,8 @@ function DBSearchPageContent() { searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig.source ?? '', orderBy: searchedConfig.orderBy ?? '', + limitAfter: searchedConfig.limitAfter ?? '', + limitUntil: searchedConfig.limitUntil ?? '', filters: searchedConfig.filters ?? [], tags: newTags, }, @@ -2323,19 +2371,67 @@ function DBSearchPageContent() { sourceId={inputSource} /> ); + const limitAfter = ( + + ); + const limitUntil = ( + + ); + const limitClause = ( + + {limitAfter} + {limitUntil} + + ); if (stacked) { return ( <> {select} {orderBy} + {limitClause} ); } return ( - <> - {select} - {orderBy} - + + + {select} + {orderBy} + + {limitClause} + ); }; @@ -2480,6 +2576,7 @@ function DBSearchPageContent() { px="sm" pt={isCompact ? 4 : 'sm'} wrap={isCompact ? 'wrap' : 'nowrap'} + align="flex-start" > + {chartConfig.limit?.after ? ( + <> + + AFTER + + + + + + ) : null} + {chartConfig.limit?.until ? ( + <> + + UNTIL + + + + + + ) : null} {chartConfig.whereLanguage === 'lucene' diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index 63f4e2ded3..7928bc4a57 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -400,6 +400,37 @@ describe('convertFormStateToSavedChartConfig', () => { expect(lineResult.orderBy).toBeUndefined(); }); + it('preserves LIMIT AFTER / UNTIL only for Table and Search displayTypes', () => { + const limit = { limit: 10, after: 'service_id >= 3', until: 'service_id >= 8' }; + const form: ChartEditorFormState = { + displayType: DisplayType.Table, + series: [seriesItem], + limit, + }; + const tableResult = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(tableResult.limit).toEqual(limit); + + const searchResult = convertFormStateToSavedChartConfig( + { + ...form, + displayType: DisplayType.Search, + select: 'Timestamp, Body', + series: [seriesItem], + }, + logSource, + ) as BuilderSavedChartConfig; + expect(searchResult.limit).toEqual(limit); + + const lineResult = convertFormStateToSavedChartConfig( + { ...form, displayType: DisplayType.Line }, + logSource, + ) as BuilderSavedChartConfig; + expect(lineResult.limit).toEqual({ limit: 10 }); + }); + it('defaults where to empty string when undefined', () => { const form: ChartEditorFormState = { displayType: DisplayType.Line, diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 5607dcdaac..d23f359786 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -89,6 +89,7 @@ function normalizeChartConfig< | 'select' | 'having' | 'orderBy' + | 'limit' | 'displayType' | 'metricTables' | 'onClick' @@ -133,6 +134,11 @@ function normalizeChartConfig< orderBy: isCustomOrderByDisplayType(config.displayType) ? config.orderBy : undefined, + // LIMIT AFTER / UNTIL are first-class ChartConfig clauses, but the + // editor only exposes them on Table and Search (row) charts. + limit: isLimitAfterUntilDisplayType(config.displayType) + ? config.limit + : stripLimitAfterUntil(config.limit), onClick: config.onClick && config.displayType === DisplayType.Table ? config.onClick @@ -250,6 +256,21 @@ const isCustomOrderByDisplayType = ( displayType === DisplayType.Bar || displayType === DisplayType.Pie; +const isLimitAfterUntilDisplayType = ( + displayType: DisplayType | undefined, +): displayType is DisplayType.Table | DisplayType.Search => + displayType === DisplayType.Table || displayType === DisplayType.Search; + +function stripLimitAfterUntil< + T extends { after?: string; until?: string; all?: boolean } | undefined, +>(limit: T): T { + if (limit == null) { + return limit; + } + const { after: _after, until: _until, all: _all, ...rest } = limit; + return rest as T; +} + // Re-exported from common-utils (single source of truth shared with the // external API / MCP tile validation) so the "Add Formula" gating in // ChartEditorControls and the server-side surfaces cannot drift. diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartActionBar.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartActionBar.tsx index aa25ac16cb..c0d706f346 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartActionBar.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartActionBar.tsx @@ -149,6 +149,42 @@ export function ChartActionBar({ /> )} + {(activeTab === 'table' || activeTab === 'search') && + !isRawSqlInput && + configType !== 'promql' && ( + +
+ +
+
+ +
+
+ )} {tabQueriesData(activeTab) && setDisplayedTimeInputValue != null && displayedTimeInputValue != null && diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 69e9a15d80..8a4cf9d66f 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1755,9 +1755,17 @@ function DBSqlRowTableComponent({ }, [initialSortBy, prevInitialSortBy]); const mergedConfigObj = useMemo(() => { + const defaults = searchChartConfigDefaults(me?.team); const base = { - ...searchChartConfigDefaults(me?.team), + ...defaults, ...config, + // Keep the search row-limit cap and user LIMIT AFTER/UNTIL together. + // Spreading `config` alone would replace `{ limit: N }` with + // `{ after, until }` and drop the cap. + limit: { + ...defaults.limit, + ...config.limit, + }, }; if (orderByArray.length) { base.orderBy = orderByArray.map(o => { diff --git a/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx b/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx index 6d902c766c..be33f43cc4 100644 --- a/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx +++ b/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx @@ -177,6 +177,10 @@ export default function SQLInlineEditor({ const compartmentRef = useRef(new Compartment()); const hasNonEmptyValue = value.trim().length > 0; + // Empty WHERE uses query history; empty SELECT/ORDER BY/AFTER/UNTIL + // (no queryHistoryType) still need column completions, same as SELECT. + const showColumnAutocomplete = + hasNonEmptyValue || queryHistoryType == null; const updateAutocompleteColumns = useCallback( (viewRef: EditorView) => { @@ -205,7 +209,7 @@ export default function SQLInlineEditor({ }); viewRef.dispatch({ effects: compartmentRef.current.reconfigure( - hasNonEmptyValue ? auto : queryHistoryList, + showColumnAutocomplete ? auto : queryHistoryList, ), }); }, @@ -215,7 +219,7 @@ export default function SQLInlineEditor({ variableCompletions, disableKeywordAutocomplete, createHistoryList, - hasNonEmptyValue, + showColumnAutocomplete, ], ); diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index e8b5b573f6..3995d7032d 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -25,6 +25,7 @@ import { } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp, + hasLimitAfterUntil, TSource, } from '@hyperdx/common-utils/dist/types'; import { @@ -139,7 +140,8 @@ function getNextPageParam( if ( lastPage == null || isRawSqlChartConfig(config) || - isPromqlChartConfig(config) + isPromqlChartConfig(config) || + (isBuilderChartConfig(config) && hasLimitAfterUntil(config.limit)) ) { return undefined; } @@ -247,10 +249,17 @@ const queryFn: QueryFunction = async ({ ? { ...config, dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date], - limit: { - limit: config.limit?.limit, - offset: pageParam.offset, - }, + limit: hasLimitAfterUntil(config.limit) + ? { + limit: config.limit?.limit, + after: config.limit?.after, + until: config.limit?.until, + all: config.limit?.all, + } + : { + limit: config.limit?.limit, + offset: pageParam.offset, + }, } : config; diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index 9c549f77cc..7d697b528f 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -52,6 +52,8 @@ export type SearchConfig = { whereLanguage?: BuilderChartConfig['whereLanguage'] | null; filters?: Filter[] | null; orderBy?: string | null; + limitAfter?: string | null; + limitUntil?: string | null; }; export type StacktraceFrame = { diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 0a6761b2c0..2ecaace514 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -5694,4 +5694,121 @@ describe('renderChartConfig', () => { ); }); }); + + describe('LIMIT AFTER / UNTIL', () => { + const tableConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'http_request_logs', + }, + select: 'timestamp, service_id', + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'timestamp', + dateRange: [ + new Date('2025-02-12 00:12:34Z'), + new Date('2025-02-14 00:12:34Z'), + ], + orderBy: 'service_id ASC', + }; + + it('renders LIMIT n AFTER start_expr', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...tableConfig, + limit: { limit: 10, after: 'service_id >= 3' }, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('LIMIT 10 AFTER service_id >= 3'); + expect(sql).not.toContain('OFFSET'); + }); + + it('renders LIMIT n AFTER start_expr UNTIL end_expr', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...tableConfig, + limit: { + limit: 10, + after: 'service_id >= 3', + until: 'service_id >= 8', + }, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + 'LIMIT 10 AFTER service_id >= 3 UNTIL service_id >= 8', + ); + }); + + it('renders LIMIT AFTER without n', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...tableConfig, + limit: { after: 'service_id >= 3' }, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toMatch(/LIMIT\s+AFTER service_id >= 3/); + }); + + it('renders LIMIT n AFTER start_expr ALL UNTIL end_expr', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...tableConfig, + limit: { + limit: 5, + after: 'service_id >= 3', + until: 'service_id >= 8', + all: true, + }, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + 'LIMIT 5 AFTER service_id >= 3 ALL UNTIL service_id >= 8', + ); + }); + + it('still renders LIMIT n OFFSET m when after/until are absent', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...tableConfig, + limit: { limit: 5, offset: 10 }, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('LIMIT 5 OFFSET 10'); + }); + + it('throws when OFFSET is combined with AFTER', async () => { + await expect( + renderChartConfig( + { + ...tableConfig, + limit: { limit: 10, offset: 5, after: 'service_id >= 3' }, + }, + mockMetadata, + querySettings, + ), + ).rejects.toThrow('LIMIT OFFSET cannot be combined with LIMIT AFTER/UNTIL'); + }); + }); }); diff --git a/packages/common-utils/src/__tests__/variables.test.ts b/packages/common-utils/src/__tests__/variables.test.ts index 9614f0310d..16174afe7f 100644 --- a/packages/common-utils/src/__tests__/variables.test.ts +++ b/packages/common-utils/src/__tests__/variables.test.ts @@ -1644,6 +1644,24 @@ describe('substituteChartConfigVariables', () => { }); }); + it('expands references in LIMIT AFTER / UNTIL expressions', () => { + expect( + substituteChartConfigVariables( + builderConfig({ + where: '', + limit: { after: 'Env = $env', until: 'Env != $env' }, + variables: [variable('env', ['prod'])], + }), + ), + ).toMatchObject({ + limit: { + after: "Env = 'prod'", + until: "Env != 'prod'", + }, + variables: undefined, + }); + }); + it('expands a lucene where clause using the lucene format', () => { expect( substituteChartConfigVariables( diff --git a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts index 99d28a5c53..6110f44d25 100644 --- a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts +++ b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts @@ -424,6 +424,29 @@ describe('buildSearchChartConfig', () => { expect(config.displayType).toBe(DisplayType.Search); }); + + it('maps limitAfter/limitUntil onto ChartConfig.limit', () => { + const config = buildSearchChartConfig(makeLogSource(), { + where: '', + limitAfter: 'Timestamp >= now() - INTERVAL 1 HOUR', + limitUntil: 'Timestamp >= now()', + }); + + expect(config.limit).toEqual({ + after: 'Timestamp >= now() - INTERVAL 1 HOUR', + until: 'Timestamp >= now()', + }); + }); + + it('omits limit when after/until are empty', () => { + const config = buildSearchChartConfig(makeLogSource(), { + where: '', + limitAfter: ' ', + limitUntil: '', + }); + + expect(config.limit).toBeUndefined(); + }); }); describe('end-to-end shape parity with DBSearchPage', () => { diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index f0c5af8fbd..cd959504df 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -1717,16 +1717,45 @@ function renderOrderBy( function renderLimit( chartConfig: BuilderChartConfigWithOptDateRange, ): ChSql | undefined { - if (chartConfig.limit == null || chartConfig.limit.limit == null) { + const limitCfg = chartConfig.limit; + if (limitCfg == null) { + return undefined; + } + + const after = + limitCfg.after != null && limitCfg.after.trim() !== '' + ? limitCfg.after.trim() + : undefined; + const until = + limitCfg.until != null && limitCfg.until.trim() !== '' + ? limitCfg.until.trim() + : undefined; + + if (limitCfg.offset != null && (after != null || until != null)) { + throw new Error('LIMIT OFFSET cannot be combined with LIMIT AFTER/UNTIL'); + } + + if (after != null || until != null) { + const n = + limitCfg.limit != null ? chSql`${{ Int32: limitCfg.limit }}` : []; + const afterSql = + after != null ? chSql` AFTER ${{ UNSAFE_RAW_SQL: after }}` : []; + const allSql = limitCfg.all === true ? chSql` ALL` : []; + const untilSql = + until != null ? chSql` UNTIL ${{ UNSAFE_RAW_SQL: until }}` : []; + return chSql`${n}${afterSql}${allSql}${untilSql}`; + } + + if (limitCfg.limit == null) { return undefined; } const offset = - chartConfig.limit.offset != null - ? chSql` OFFSET ${{ Int32: chartConfig.limit.offset }}` + limitCfg.offset != null + ? chSql` OFFSET ${{ Int32: limitCfg.offset }}` : []; - return chSql`${{ Int32: chartConfig.limit.limit }}${offset}`; + return chSql`${{ Int32: limitCfg.limit }}${offset}`; } function renderSettings( diff --git a/packages/common-utils/src/core/searchChartConfig.ts b/packages/common-utils/src/core/searchChartConfig.ts index 37144db8ab..b9a9668986 100644 --- a/packages/common-utils/src/core/searchChartConfig.ts +++ b/packages/common-utils/src/core/searchChartConfig.ts @@ -71,6 +71,10 @@ export type SearchChartConfigInput = { select?: SelectList | null; orderBy?: SortSpecificationList | null; groupBy?: SelectList | null; + /** ClickHouse `LIMIT AFTER` start expression. */ + limitAfter?: string | null; + /** ClickHouse `LIMIT UNTIL` end expression. */ + limitUntil?: string | null; // Display / time-range knobs displayType?: DisplayType; @@ -173,6 +177,17 @@ export function buildSearchChartConfig( ...(mergedFilters.length > 0 ? { filters: mergedFilters } : {}), ...(input.groupBy != null ? { groupBy: input.groupBy } : {}), ...(input.orderBy != null ? { orderBy: input.orderBy } : {}), + ...(() => { + const after = input.limitAfter?.trim(); + const until = input.limitUntil?.trim(); + if (!after && !until) return {}; + return { + limit: { + ...(after ? { after } : {}), + ...(until ? { until } : {}), + }, + }; + })(), ...(input.dateRange != null ? { dateRange: input.dateRange } : {}), ...(input.dateRangeStartInclusive != null ? { dateRangeStartInclusive: input.dateRangeStartInclusive } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index b8c3bc1525..9c1165dc19 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -508,10 +508,35 @@ export const SortSpecificationSchema = z.intersection( export const SortSpecificationListSchema = z .array(SortSpecificationSchema) .or(z.string()); -export const LimitSchema = z.object({ - limit: z.number().optional(), - offset: z.number().optional(), -}); +export function hasLimitAfterUntil(limit?: { + after?: string; + until?: string; +} | null): boolean { + return ( + (limit?.after != null && limit.after.trim() !== '') || + (limit?.until != null && limit.until.trim() !== '') + ); +} + +export const LimitSchema = z + .object({ + limit: z.number().optional(), + offset: z.number().optional(), + // ClickHouse `LIMIT [n] AFTER start_expr [ALL] [UNTIL end_expr]`. + // Mutually exclusive with `offset` — OFFSET and AFTER/UNTIL cannot be + // combined in one LIMIT clause. + after: z.string().optional(), + until: z.string().optional(), + all: z.boolean().optional(), + }) + .superRefine((val, ctx) => { + if (val.offset != null && hasLimitAfterUntil(val)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'offset cannot be combined with after or until', + }); + } + }); export const ChSqlSchema = z.object({ sql: z.string(), @@ -561,7 +586,7 @@ export type SelectList = z.infer; export type SortSpecificationList = z.infer; -type Limit = { limit?: number; offset?: number }; +export type Limit = z.infer; export type SelectSQLStatement = { select: SelectList; @@ -1392,6 +1417,8 @@ export const SavedSearchSchema = z.object({ source: z.string(), tags: z.array(z.string()), orderBy: z.string().optional(), + limitAfter: z.string().optional(), + limitUntil: z.string().optional(), filters: z.array(FilterSchema).optional(), alerts: z.array(AlertSchema).optional(), }); diff --git a/packages/common-utils/src/variables.ts b/packages/common-utils/src/variables.ts index 9abe461e7c..d92f0e5236 100644 --- a/packages/common-utils/src/variables.ts +++ b/packages/common-utils/src/variables.ts @@ -949,6 +949,7 @@ export type BuilderVariableFields = { havingLanguage?: SearchConditionLanguage; groupBy?: SelectList; orderBy?: SortSpecificationList; + limit?: { after?: string; until?: string }; }; /** @@ -1016,6 +1017,19 @@ export function mapBuilderVariableTemplates( ...(config.orderBy != null ? { orderBy: mapSortList(config.orderBy, map) } : {}), + ...(config.limit != null + ? { + limit: { + ...config.limit, + ...(config.limit.after + ? { after: map(config.limit.after, 'sql') } + : {}), + ...(config.limit.until + ? { until: map(config.limit.until, 'sql') } + : {}), + }, + } + : {}), }; }