From 8e52cef41d37f76f948024d9c33ae911a3b936e0 Mon Sep 17 00:00:00 2001 From: Mike Shi Date: Fri, 29 May 2026 16:12:13 -0700 Subject: [PATCH 001/728] feat(dashboard): auto-resize font in number tiles to fit container (#2373) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Dashboard "Number" tiles previously rendered their value at a hard-coded `4rem` font size. On smaller dashboard tiles — especially with long formatted values like large counts, currencies, or percentages — the value would overflow the tile or get clipped by the surrounding chart container. This PR introduces an `AutoSizeNumber` component inside `DBNumberChart` that dynamically picks the largest font size at which the formatted value fits its container, with a min of 10px and a max of 72px (close to the previous `4rem`). The picked size is recomputed: - on initial layout, - whenever the tile's container is resized (via `ResizeObserver`, debounced through a single `requestAnimationFrame` per resize burst), - whenever the formatted value changes. The text is forced onto a single line (`white-space: nowrap`) so long values shrink rather than wrap awkwardly, and the container has 12px of internal padding so the value never visually touches the tile edges. Initial render (before the first measurement) uses a sensible mid-range default of 36px rather than starting at the maximum. The sizing logic is a small binary search that probes the actual rendered DOM (`scrollWidth` / `scrollHeight`), so it correctly reflects the user's font, locale, and number formatting without needing to model glyph widths. A `react-error-boundary` wraps the auto-sizing component so that any runtime failure in the measurement / `ResizeObserver` pipeline falls back to a `SimpleNumber` component that mirrors the pre-PR fixed-size rendering — dashboards never go blank because of a measurement bug. ### Screenshots or video [dashboard_number_tile_auto_resize_demo.mp4](https://cursor.com/agents/bc-16e87253-5bdb-4ab0-962f-59d0b093f4b0/artifacts?path=%2Fopt%2Fcursor%2Fartifacts%2Fdashboard_number_tile_auto_resize_demo.mp4) The recording shows the dashboard being created with a Number tile rendering `1234567`, and the same tile opened in fullscreen. The value visibly grows from a medium font in the dashboard tile to a much larger font in the fullscreen view, demonstrating that the font size is driven by the container size. | State | Screenshot | | :---- | :--------- | | Default tile size | [Number tile at default dashboard size](https://cursor.com/agents/bc-16e87253-5bdb-4ab0-962f-59d0b093f4b0/artifacts?path=%2Fopt%2Fcursor%2Fartifacts%2Fnumber_tile_default_size.webp) | | Fullscreen / large container | [Number tile fullscreen with large auto-sized font](https://cursor.com/agents/bc-16e87253-5bdb-4ab0-962f-59d0b093f4b0/artifacts?path=%2Fopt%2Fcursor%2Fartifacts%2Fnumber_tile_fullscreen_large.webp) | | Browser zoomed out (smaller pixel container) | [Number tile at smaller pixel container](https://cursor.com/agents/bc-16e87253-5bdb-4ab0-962f-59d0b093f4b0/artifacts?path=%2Fopt%2Fcursor%2Fartifacts%2Fnumber_tile_small_size.webp) | ### How to test on Vercel preview **Preview routes:** `/dashboards` **Steps:** 1. Open `/dashboards`, create a new dashboard, and add a Number tile that renders a long value (e.g. raw SQL `SELECT 1234567890123 AS value`). 2. Drag the tile's bottom-right resize handle to the smallest grid size and confirm the rendered number shrinks to fit without overflowing or being clipped. 3. Drag the same handle to enlarge the tile and confirm the rendered number grows to fill the new space (capped at ~72px). 4. Open the tile's fullscreen / detail view and verify the value renders at a noticeably larger font than in the in-grid tile. ### References - Linear Issue: - Related PRs: --- **Testing performed by the agent** - `make ci-lint` — lint + TypeScript typecheck across all packages — passed. - `yarn ci:unit src/components/__tests__/DBNumberChart.test.tsx` — 20/20 tests passed, including five new tests covering the auto-sizing path and error-boundary fallback: - renders a `px`-based font size (vs the old `4rem`), - keeps the value on a single line, - shrinks toward the configured minimum on a narrow container with a long value (driven by mocked `clientWidth` / `scrollWidth`), - grows toward the configured maximum on a wide container, - falls back to the simpler `SimpleNumber` rendering when the auto-sizer's measurement pipeline throws. - Manual GUI testing of the dashboard Number tile (recording and screenshots above) confirming the auto-sizing behavior end-to-end. To show artifacts inline, enable in settings.
Open in Web Open in Cursor 
Co-authored-by: Cursor Agent <199161495+cursoragent@users.noreply.github.com> Co-authored-by: Brandon Pereira <7552738+brandon-pereira@users.noreply.github.com> --- .../dashboard-number-tile-font-resize.md | 10 + packages/app/src/components/DBNumberChart.tsx | 176 +++++++++++++++++- .../__tests__/DBNumberChart.test.tsx | 159 ++++++++++++++++ 3 files changed, 339 insertions(+), 6 deletions(-) create mode 100644 .changeset/dashboard-number-tile-font-resize.md diff --git a/.changeset/dashboard-number-tile-font-resize.md b/.changeset/dashboard-number-tile-font-resize.md new file mode 100644 index 0000000000..1bdcaa5297 --- /dev/null +++ b/.changeset/dashboard-number-tile-font-resize.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/app': patch +--- + +feat(dashboard): auto-resize font in number tiles to fit container + +Number tiles now automatically scale their font size to fit the available +width, preventing text overflow on narrow tiles and making better use of +space on wide ones. Includes an error boundary so a single broken tile +does not crash the entire dashboard. diff --git a/packages/app/src/components/DBNumberChart.tsx b/packages/app/src/components/DBNumberChart.tsx index c17b508911..fe38c26a9a 100644 --- a/packages/app/src/components/DBNumberChart.tsx +++ b/packages/app/src/components/DBNumberChart.tsx @@ -1,4 +1,5 @@ -import { useMemo } from 'react'; +import { useLayoutEffect, useMemo, useRef, useState } from 'react'; +import { ErrorBoundary } from 'react-error-boundary'; import { filterColumnMetaByType, JSDataType, @@ -28,6 +29,171 @@ import ChartErrorState, { } from './charts/ChartErrorState'; import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator'; +const NUMBER_TILE_MIN_FONT_SIZE = 10; +const NUMBER_TILE_MAX_FONT_SIZE = 72; +// Initial / fallback font size used before the first measurement runs +// (and as a sensible mid-range size if measurement is unavailable). +// Tuned to look reasonable in a typical "default" tile (~6 columns wide +// on a 24-col grid) without overflowing. +const NUMBER_TILE_DEFAULT_FONT_SIZE = 36; +const NUMBER_TILE_PADDING = 12; + +function fitFontSize( + textEl: HTMLElement, + availableWidth: number, + availableHeight: number, + minFontSize: number, + maxFontSize: number, +): number { + if (availableWidth <= 0 || availableHeight <= 0) { + return minFontSize; + } + + let lo = minFontSize; + let hi = maxFontSize; + let best = minFontSize; + + while (lo <= hi) { + const mid = Math.floor((lo + hi) / 2); + textEl.style.fontSize = `${mid}px`; + if ( + textEl.scrollWidth <= availableWidth && + textEl.scrollHeight <= availableHeight + ) { + best = mid; + lo = mid + 1; + } else { + hi = mid - 1; + } + } + + return best; +} + +// Plain centered rendering used as a fallback when AutoSizeNumber's +// measurement / ResizeObserver pipeline throws unexpectedly. Mirrors the +// pre-auto-size implementation so dashboards keep showing the value even +// if the resize logic encounters a runtime error. +function SimpleNumber({ + children, + color, +}: { + children: React.ReactNode; + color?: string; +}) { + return ( + + + {children} + + + ); +} + +// Renders the formatted number at the largest font size (between +// NUMBER_TILE_MIN_FONT_SIZE and NUMBER_TILE_MAX_FONT_SIZE) that fits the +// surrounding tile, recomputing whenever the tile resizes or the value +// changes. Prevents long values (large counts, currency, percentages) +// from overflowing small dashboard tiles. +function AutoSizeNumber({ + children, + color, +}: { + children: React.ReactNode; + color?: string; +}) { + const containerRef = useRef(null); + const textRef = useRef(null); + const [fontSize, setFontSize] = useState( + NUMBER_TILE_DEFAULT_FONT_SIZE, + ); + + useLayoutEffect(() => { + const container = containerRef.current; + const textEl = textRef.current; + if (!container || !textEl) return; + + let raf: number | null = null; + const recalc = () => { + raf = null; + if (!container || !textEl) return; + const availW = container.clientWidth - NUMBER_TILE_PADDING * 2; + const availH = container.clientHeight - NUMBER_TILE_PADDING * 2; + const next = fitFontSize( + textEl, + availW, + availH, + NUMBER_TILE_MIN_FONT_SIZE, + NUMBER_TILE_MAX_FONT_SIZE, + ); + setFontSize(next); + }; + + recalc(); + + if (typeof ResizeObserver === 'undefined') { + return; + } + + const ro = new ResizeObserver(() => { + if (raf != null) return; + raf = requestAnimationFrame(recalc); + }); + ro.observe(container); + + return () => { + ro.disconnect(); + if (raf != null) cancelAnimationFrame(raf); + }; + }, [children]); + + return ( + + + {children} + + + ); +} + +// Wraps AutoSizeNumber in an error boundary so a runtime failure in the +// measurement / ResizeObserver pipeline never blanks out the tile — +// instead the dashboard falls back to the original fixed-size rendering. +function SafeAutoSizeNumber({ + children, + color, +}: { + children: React.ReactNode; + color?: string; +}) { + return ( + {children}} + > + {children} + + ); +} + export default function DBNumberChart({ config, enabled = true, @@ -157,11 +323,9 @@ export default function DBNumberChart({ No data found within time range. ) : ( - - - {formattedValue ?? 'N/A'} - - + + {formattedValue ?? 'N/A'} + )} ); diff --git a/packages/app/src/components/__tests__/DBNumberChart.test.tsx b/packages/app/src/components/__tests__/DBNumberChart.test.tsx index 8bdc539217..7a727e7bcd 100644 --- a/packages/app/src/components/__tests__/DBNumberChart.test.tsx +++ b/packages/app/src/components/__tests__/DBNumberChart.test.tsx @@ -349,4 +349,163 @@ describe('DBNumberChart', () => { expect(screen.getByText('1234')).toBeInTheDocument(); }); }); + + describe('auto-sized font', () => { + it('renders the value with a pixel-based font size so the tile can resize it', () => { + renderWithMantine(); + + const textEl = screen.getByText('1234'); + const fontSize = textEl.style.fontSize; + + // The value should be sized in px (set by AutoSizeNumber's + // ResizeObserver-driven calculation) rather than the previous + // hard-coded "4rem". Anything non-empty ending in "px" indicates + // the auto-sizing path is active. + expect(fontSize).toMatch(/px$/); + }); + + it('does not let the value wrap to multiple lines', () => { + renderWithMantine(); + const textEl = screen.getByText('1234'); + // Long numbers should shrink to fit on a single line rather + // than wrap and visually break the tile. + expect(textEl.style.whiteSpace).toBe('nowrap'); + }); + + /** + * Drives the binary-search measurement pass by faking the DOM + * geometry that the AutoSizeNumber component reads: + * + * - container.clientWidth / clientHeight -> available tile size + * - text.scrollWidth / scrollHeight -> rendered text size at + * the current font-size, modeled as proportional to the assigned + * font-size in px and the length of the text (a reasonable + * approximation of how a real browser would lay it out). + * + * With those mocks in place we can assert that the picked font size + * actually shrinks for narrow containers and grows for wide ones. + */ + const installGeometryMocks = ( + containerWidth: number, + containerHeight: number, + // px of width per character at 1px font; tweak so realistic numbers + // produce realistic measurements during the binary search. + widthPerChar = 0.6, + ) => { + const containerSpy = jest + .spyOn(HTMLElement.prototype, 'clientWidth', 'get') + .mockImplementation(function (this: HTMLElement) { + return this.tagName === 'DIV' ? containerWidth : 0; + }); + const containerHeightSpy = jest + .spyOn(HTMLElement.prototype, 'clientHeight', 'get') + .mockImplementation(function (this: HTMLElement) { + return this.tagName === 'DIV' ? containerHeight : 0; + }); + const scrollWidthSpy = jest + .spyOn(HTMLElement.prototype, 'scrollWidth', 'get') + .mockImplementation(function (this: HTMLElement) { + if (this.tagName !== 'P') return 0; + const fs = parseFloat(this.style.fontSize) || 0; + return Math.ceil((this.textContent ?? '').length * widthPerChar * fs); + }); + const scrollHeightSpy = jest + .spyOn(HTMLElement.prototype, 'scrollHeight', 'get') + .mockImplementation(function (this: HTMLElement) { + if (this.tagName !== 'P') return 0; + const fs = parseFloat(this.style.fontSize) || 0; + // line-height: 1.1 + return Math.ceil(fs * 1.1); + }); + + return () => { + containerSpy.mockRestore(); + containerHeightSpy.mockRestore(); + scrollWidthSpy.mockRestore(); + scrollHeightSpy.mockRestore(); + }; + }; + + it('shrinks the font for a narrow tile so the value fits without overflow', () => { + const restore = installGeometryMocks(120, 80); + try { + mockUseQueriedChartConfig.mockReturnValue({ + data: { data: [{ value: 1234567890 }] }, + isLoading: false, + isError: false, + }); + + renderWithMantine(); + + const textEl = screen.getByText('1234567890'); + const fontSize = parseFloat(textEl.style.fontSize); + + // Available width after padding (12 px each side) is 96 px. + // 10 chars * 0.6 width-per-char = 6 px/char, so the largest + // font size that fits is floor(96 / (10 * 0.6)) = 16 px. Picked + // size must be <= 16 and >= the configured min of 10 px. + expect(fontSize).toBeLessThanOrEqual(16); + expect(fontSize).toBeGreaterThanOrEqual(10); + } finally { + restore(); + } + }); + + it('grows the font for a wide tile so the value fills the available space', () => { + const restore = installGeometryMocks(800, 400); + try { + renderWithMantine(); + + const textEl = screen.getByText('1234'); + const fontSize = parseFloat(textEl.style.fontSize); + + // 4 chars * 0.6 = 2.4 px per font-size px, so a 800-px-wide + // container could theoretically fit very large fonts; the + // auto-sizer should clamp to the configured max (72 px). + expect(fontSize).toBeGreaterThanOrEqual(60); + expect(fontSize).toBeLessThanOrEqual(72); + } finally { + restore(); + } + }); + }); + + describe('error boundary fallback', () => { + /** + * Force the auto-sizer's measurement pipeline to throw by making + * `clientWidth` raise an error during the layout effect. The + * surrounding ErrorBoundary should catch the failure and fall back + * to the simpler fixed-size rendering, which still shows the value + * (so dashboards never go blank because of a measurement bug). + */ + it('falls back to a simpler text rendering if AutoSizeNumber throws', () => { + const containerSpy = jest + .spyOn(HTMLElement.prototype, 'clientWidth', 'get') + .mockImplementation(function (this: HTMLElement) { + if (this.tagName === 'DIV') { + throw new Error('boom: simulated layout failure'); + } + return 0; + }); + + // ErrorBoundary logs the caught error to console.error; silence + // it so the test output stays focused on the assertions. + const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); + + try { + renderWithMantine(); + + const textEl = screen.getByText('1234'); + expect(textEl).toBeInTheDocument(); + + // The fallback uses Mantine's `size="4rem"` rather than an + // inline px font-size, so the auto-sized inline style should + // not be present on the rendered element. + expect(textEl.style.fontSize).toBe(''); + } finally { + containerSpy.mockRestore(); + errSpy.mockRestore(); + } + }); + }); }); From ef18bcedb585e56b7089174e63458c658656032d Mon Sep 17 00:00:00 2001 From: Elizabet Oliveira Date: Mon, 1 Jun 2026 12:44:05 +0100 Subject: [PATCH 002/728] style(search): unify xs input height to 30px across search header (#2391) Co-authored-by: Cursor --- packages/app/src/DBSearchPage.tsx | 5 ++++- .../app/src/components/SQLEditor/SQLInlineEditor.tsx | 2 +- .../app/src/components/SearchInput/AutocompleteInput.tsx | 2 +- .../components/SearchInput/SearchWhereInput.module.scss | 9 ++++++--- packages/app/src/components/TimePicker/TimePicker.tsx | 4 +++- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 92fe91474c..4e7a6a21de 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -277,6 +277,7 @@ function SearchSubmitButton({ type="submit" leftSection={} style={{ flexShrink: 0 }} + size="xs" > Run @@ -1982,6 +1983,7 @@ export function DBSearchPage() { minWidth="min(600px, 100%)" dateRange={searchedTimeRange} sourceId={inputSource} + size="xs" /> {isLive && ( } + rightSection={} maxDropdownHeight={280} data={data} disabled={isTablesLoading} @@ -68,7 +68,17 @@ function DBTableSelect({ ref={inputRef} size={size} className="flex-grow-1" - {...rightSectionProps} + /> + setIsSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled(previewSource)} + /> + setIsSchemaPreviewOpen(false)} /> ); diff --git a/packages/app/src/components/DBTracePanel.tsx b/packages/app/src/components/DBTracePanel.tsx index f1ffcb00e2..b6deae7923 100644 --- a/packages/app/src/components/DBTracePanel.tsx +++ b/packages/app/src/components/DBTracePanel.tsx @@ -1,4 +1,4 @@ -import { ReactNode, useEffect, useMemo, useState } from 'react'; +import { ReactNode, useEffect, useState } from 'react'; import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; @@ -28,7 +28,9 @@ import { parseAsJsonEncoded } from '@/utils/queryParsers'; import { RowDataPanel } from './DBRowDataPanel'; import { RowOverviewPanel } from './DBRowOverviewPanel'; -import SourceSchemaPreview from './SourceSchemaPreview'; +import SourceSchemaPreview, { + isSourceSchemaPreviewEnabled, +} from './SourceSchemaPreview'; import { SourceSelectControlled } from './SourceSelect'; const eventRowWhereParser = parseAsJsonEncoded<{ @@ -141,9 +143,8 @@ export default function DBTracePanel({ }; }, [traceId, setEventRowWhere]); - const sourceSchemaPreview = useMemo(() => { - return ; - }, [childSourceData]); + const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = + useState(false); const [displayedTab, setDisplayedTab] = useState(Tab.Overview); return ( @@ -177,7 +178,16 @@ export default function DBTracePanel({ control={control} name="source" size="xs" - sourceSchemaPreview={sourceSchemaPreview} + onSchemaPreview={() => setIsSourceSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( + childSourceData, + )} + /> + setIsSourceSchemaPreviewOpen(false)} /> diff --git a/packages/app/src/components/SelectControlled.tsx b/packages/app/src/components/SelectControlled.tsx index b4653198a1..439a263f10 100644 --- a/packages/app/src/components/SelectControlled.tsx +++ b/packages/app/src/components/SelectControlled.tsx @@ -2,7 +2,7 @@ import { useCallback } from 'react'; import { useController, UseControllerProps } from 'react-hook-form'; import { Select, SelectProps } from '@mantine/core'; -export enum SelectControlledSpecialValues { +enum SelectControlledSpecialValues { CreateNewValue = '_create_new_value', EditValue = '_edit_value', } diff --git a/packages/app/src/components/SourceSchemaPreview.tsx b/packages/app/src/components/SourceSchemaPreview.tsx index 3dc10b5ef5..0b62799182 100644 --- a/packages/app/src/components/SourceSchemaPreview.tsx +++ b/packages/app/src/components/SourceSchemaPreview.tsx @@ -44,7 +44,7 @@ const SourceSchemaInfoIcon = ({ Schema @@ -128,6 +128,15 @@ interface SourceSchemaPreviewProps { source?: SourceSchemaPreviewSource; iconStyles?: React.CSSProperties; variant?: 'icon' | 'text'; + /** + * When true, the trigger element (icon or text button) is NOT rendered; + * the parent owns open state and drives the modal via `open` / `onClose`. + * Useful when the trigger lives outside this component (e.g. a kebab menu + * adjacent to a source picker). + */ + controlled?: boolean; + open?: boolean; + onClose?: () => void; } const METRIC_TYPE_NAMES: Record = { @@ -138,16 +147,21 @@ const METRIC_TYPE_NAMES: Record = { [MetricsDataType.ExponentialHistogram]: 'Exponential Histogram', }; -const SourceSchemaPreview = ({ - source, - iconStyles, - variant = 'icon', -}: SourceSchemaPreviewProps) => { - const [isModalOpen, setIsModalOpen] = useState(false); - - const isMetricSource = source?.kind === 'metric'; +/** + * Build the list of tables (and their titles) to show in the schema preview + * modal for a given source. Internal helper that powers both + * `` rendering and `isSourceSchemaPreviewEnabled`, + * so callers can decide whether the preview has anything to show before + * rendering a trigger. + */ +function getSourceSchemaTables( + source?: SourceSchemaPreviewSource, +): (TableSchemaPreviewProps & { title: string })[] { const tables: (TableSchemaPreviewProps & { title: string })[] = []; - if (source && isMetricSource) { + if (!source) return tables; + + const isMetricSource = source.kind === 'metric'; + if (isMetricSource) { tables.push( ...Object.values(MetricsDataType) .map(metricType => ({ @@ -162,7 +176,7 @@ const SourceSchemaPreview = ({ title: METRIC_TYPE_NAMES[metricType], })), ); - } else if (source && source.from.tableName) { + } else if (source.from.tableName) { tables.push({ databaseName: source.from.databaseName, tableName: source.from.tableName, @@ -171,31 +185,61 @@ const SourceSchemaPreview = ({ }); } - const mvConfigs = source?.materializedViews ?? []; + const mvConfigs = source.materializedViews ?? []; tables.push( ...mvConfigs.map(({ tableName, databaseName }) => ({ databaseName, tableName, - connectionId: source!.connection, + connectionId: source.connection, title: `${tableName} (MV)`, })), ); - const isEnabled = !!source && tables.length > 0; + return tables; +} + +export function isSourceSchemaPreviewEnabled( + source?: SourceSchemaPreviewSource, +): boolean { + return !!source && getSourceSchemaTables(source).length > 0; +} + +const SourceSchemaPreview = ({ + source, + iconStyles, + variant = 'icon', + controlled = false, + open, + onClose, +}: SourceSchemaPreviewProps) => { + const [internalOpen, setInternalOpen] = useState(false); + const isModalOpen = controlled ? !!open : internalOpen; + const handleClose = () => { + if (controlled) { + onClose?.(); + } else { + setInternalOpen(false); + } + }; + + const tables = getSourceSchemaTables(source); + const isEnabled = isSourceSchemaPreviewEnabled(source); return ( <> - setIsModalOpen(true)} - iconStyles={iconStyles} - tableCount={tables.length} - variant={variant} - /> + {!controlled && ( + setInternalOpen(true)} + iconStyles={iconStyles} + tableCount={tables.length} + variant={variant} + /> + )} {isEnabled && ( setIsModalOpen(false)} + onClose={handleClose} size="auto" title={tables.length > 1 ? `Table Schemas` : `Table Schema`} > diff --git a/packages/app/src/components/SourceSelect.tsx b/packages/app/src/components/SourceSelect.tsx index ef015b9597..d00e23e656 100644 --- a/packages/app/src/components/SourceSelect.tsx +++ b/packages/app/src/components/SourceSelect.tsx @@ -2,17 +2,24 @@ import { memo, useCallback, useMemo } from 'react'; import { UseControllerProps, useWatch } from 'react-hook-form'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { + ActionIcon, ComboboxChevron, ComboboxItem, Group, + Menu, SelectProps, - UnstyledButton, + Tooltip, } from '@mantine/core'; -import { IconPlus, IconSettings, IconStack } from '@tabler/icons-react'; - -import SelectControlled, { - SelectControlledSpecialValues, -} from '@/components/SelectControlled'; +import { + IconCode, + IconDotsVertical, + IconPencil, + IconPlus, + IconSettings, + IconStack, +} from '@tabler/icons-react'; + +import SelectControlled from '@/components/SelectControlled'; import { SOURCE_KIND_ICONS, useFilteredSortedSourceItems, @@ -22,59 +29,144 @@ import { useSources } from '@/source'; import styles from '../../styles/SourceSelectControlled.module.scss'; -interface SourceSelectRightSectionProps { - sourceSchemaPreview?: React.ReactNode; +interface SourceManagementMenuProps { + hasSelection: boolean; + onSchemaPreview?: () => void; + isSchemaPreviewEnabled?: boolean; + /** + * Open the edit form for the currently selected source. Should act + * on `hasSelection ? currentSourceId : null`. Hidden from the menu + * when no handler is passed. + */ + onEdit?: () => void; + /** + * Open the list view of all sources (e.g. `/team#sources`). Hidden + * from the menu when no handler is passed; typically the case in + * local mode where no list-view surface exists. + */ + onManageSources?: () => void; + onCreate?: () => void; } -export const SourceSelectRightSection = ({ - sourceSchemaPreview, -}: SourceSelectRightSectionProps) => { - if (!sourceSchemaPreview) { - return { - rightSection: , - }; +/** + * Adjacent kebab menu that consolidates source-management actions: + * View schema, Edit source, Manage sources, Create new source. + * + * `Edit source` operates on the current selection; `Manage sources` + * opens the all-sources list view. Each item hides if its handler + * isn't wired, so callers can pick the right combination for their + * surface (local mode, for example, has no list view and should leave + * `onManageSources` unset). + * + * Exposed so non-`SourceSelectControlled` callers (e.g. `DBTableSelect`) + * can attach the same surface if needed. + */ +export const SourceManagementMenu = ({ + hasSelection, + onSchemaPreview, + isSchemaPreviewEnabled = true, + onEdit, + onManageSources, + onCreate, +}: SourceManagementMenuProps) => { + const items: React.ReactNode[] = []; + + if (onSchemaPreview) { + items.push( + } + onClick={onSchemaPreview} + disabled={!hasSelection || !isSchemaPreviewEnabled} + > + View schema + , + ); + } + + if (onEdit) { + items.push( + } + onClick={onEdit} + disabled={!hasSelection} + > + Edit source + , + ); } - return { - rightSection: ( - <> - ) => { - e.stopPropagation(); - e.preventDefault(); - }} - className={styles.sourceSchemaPreviewButton} - > - {sourceSchemaPreview} - - - - ), - rightSectionWidth: 70, - }; -}; + if (onManageSources) { + items.push( + } + onClick={onManageSources} + > + Manage sources + , + ); + } -const OPTION_ICONS: Record = { - [SelectControlledSpecialValues.CreateNewValue]: , - [SelectControlledSpecialValues.EditValue]: , + if (onCreate) { + if (items.length > 0) { + items.push(); + } + items.push( + } + onClick={onCreate} + > + Create new source + , + ); + } + + if (items.length === 0) return null; + + return ( + + + + + + + + + {items} + + ); }; function SourceSelectControlledComponent({ size, onCreate, onEdit, + onManageSources, + onSchemaPreview, + isSchemaPreviewEnabled, allowedSourceKinds, connectionId, comboboxProps, - sourceSchemaPreview, ...props }: { size?: string; onCreate?: () => void; onEdit?: () => void; + onManageSources?: () => void; + onSchemaPreview?: () => void; + isSchemaPreviewEnabled?: boolean; allowedSourceKinds?: SourceKind[]; connectionId?: string; - sourceSchemaPreview?: React.ReactNode; } & UseControllerProps & SelectProps) { const { data } = useSources(); @@ -96,9 +188,7 @@ function SourceSelectControlledComponent({ const renderOption = useCallback( ({ option }: { option: ComboboxItem }) => { - const icon = - OPTION_ICONS[option.value] ?? - SOURCE_KIND_ICONS[sourceKindMap.get(option.value) ?? '']; + const icon = SOURCE_KIND_ICONS[sourceKindMap.get(option.value) ?? '']; if (!icon) return option.label; return ( @@ -110,54 +200,50 @@ function SourceSelectControlledComponent({ [sourceKindMap], ); - const hasActions = !!onCreate || !!onEdit; - const sourceItems = useFilteredSortedSourceItems({ sources: data, allowedSourceKinds, connectionId, }); - const values = useMemo(() => { - if (!hasActions) { - return sourceItems; - } - - const actionItems: { value: string; label: string }[] = []; - if (onCreate) { - actionItems.push({ - value: SelectControlledSpecialValues.CreateNewValue, - label: 'Create New Source', - }); - } - if (onEdit) { - actionItems.push({ - value: SelectControlledSpecialValues.EditValue, - label: 'Edit Sources', - }); - } - - return [...sourceItems, { group: 'Actions', items: actionItems }]; - }, [sourceItems, onCreate, onEdit, hasActions]); - - const rightSectionProps = SourceSelectRightSection({ sourceSchemaPreview }); + const hasSelection = !!selectedSourceId; + const hasMenu = + !!onCreate || !!onEdit || !!onManageSources || !!onSchemaPreview; return ( - + + } + /> + {hasMenu && ( + + )} + ); } diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 7544b14bc1..a73ed1ca40 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -1521,6 +1521,59 @@ function TraceTableModelForm(props: TableModelProps) { ]} value={value} onChange={onChange} + // Mantine 9's Slider styles use the pattern + // `:where([data-orientation="vertical"]) .`, + // which matches when ANY ancestor has + // `data-orientation="vertical"`. Mantine Card sets + // `data-orientation="vertical"` by default, and the + // SourceForm renders inside a Card, so the slider's + // trackContainer/track/bar/thumb/markWrapper/ + // markLabel all pick up the vertical-orientation + // styling: the track collapses to 8px wide and the + // four marks stack on top of each other. Override + // every affected part back to its horizontal + // default so the slider renders correctly inside + // the Card. + styles={{ + trackContainer: { + width: '100%', + flexDirection: 'row', + height: 'calc(var(--slider-size) * 2)', + }, + track: { + width: '100%', + height: 'var(--slider-size)', + }, + bar: { + top: 0, + bottom: 0, + height: '100%', + insetInlineStart: 'var(--slider-bar-offset)', + width: 'var(--slider-bar-width)', + }, + thumb: { + left: 'var(--slider-thumb-offset)', + top: '50%', + right: 'auto', + bottom: 'auto', + transform: 'translate(-50%, -50%)', + }, + markWrapper: { + insetInlineStart: + 'calc(var(--mark-offset) - var(--slider-size) / 2)', + top: 0, + bottom: 'auto', + width: 'auto', + }, + markLabel: { + transform: + 'translate(calc(-50% + var(--slider-size) / 2), calc(var(--mantine-spacing-xs) / 2))', + }, + label: { + top: '-36px', + insetInlineStart: 'auto', + }, + }} /> )} @@ -1913,7 +1966,7 @@ function PromqlTableModelForm({ }, [setValue]); // PromQL sources use the standard database + table fields from BaseSourceSchema. - // No additional fields needed — the table should point to the TimeSeries engine table. + // No additional fields needed; the table should point to the TimeSeries engine table. return null; } diff --git a/packages/app/src/components/Sources/SourcesList.tsx b/packages/app/src/components/Sources/SourcesList.tsx index 54961ca122..56732610c9 100644 --- a/packages/app/src/components/Sources/SourcesList.tsx +++ b/packages/app/src/components/Sources/SourcesList.tsx @@ -1,4 +1,5 @@ -import React, { useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; +import { useRouter } from 'next/router'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, @@ -73,6 +74,29 @@ export function SourcesList({ const [editedSourceId, setEditedSourceId] = useState(null); const [isCreatingSource, setIsCreatingSource] = useState(false); + // Honor `?source=` from the URL so kebab "Edit source" deep-links + // expand the right entry. We auto-expand once per param value, then + // strip it from the URL so subsequent in-page interactions don't + // re-fight the user. `expandedFromQueryRef` prevents the effect from + // toggling back on every keystroke that ticks `router.query`. + const router = useRouter(); + const expandedFromQueryRef = useRef(null); + useEffect(() => { + if (!router.isReady) return; + const sourceParam = router.query.source; + const sourceId = Array.isArray(sourceParam) ? sourceParam[0] : sourceParam; + if (!sourceId || expandedFromQueryRef.current === sourceId) return; + if (!sources?.some(s => s.id === sourceId)) return; + expandedFromQueryRef.current = sourceId; + setEditedSourceId(sourceId); + const { source: _omit, ...rest } = router.query; + void router.replace( + { pathname: router.pathname, query: rest, hash: 'sources' }, + undefined, + { shallow: true }, + ); + }, [router, sources]); + const isLoading = isLoadingConnections || isLoadingSources; const error = connectionsError || sourcesError; diff --git a/packages/app/src/components/__tests__/DBTracePanel.test.tsx b/packages/app/src/components/__tests__/DBTracePanel.test.tsx index 8594db58da..ca95483eb4 100644 --- a/packages/app/src/components/__tests__/DBTracePanel.test.tsx +++ b/packages/app/src/components/__tests__/DBTracePanel.test.tsx @@ -39,6 +39,8 @@ jest.mock('../SourceSelect', () => ({ jest.mock('../SourceSchemaPreview', () => ({ __esModule: true, default: () =>
, + isSourceSchemaPreviewEnabled: () => false, + getSourceSchemaTables: () => [], })); describe('DBTracePanel', () => { diff --git a/packages/app/src/components/__tests__/SourceSchemaPreview.test.tsx b/packages/app/src/components/__tests__/SourceSchemaPreview.test.tsx new file mode 100644 index 0000000000..d033a5a3fb --- /dev/null +++ b/packages/app/src/components/__tests__/SourceSchemaPreview.test.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import SourceSchemaPreview from '../SourceSchemaPreview'; + +jest.mock('@/hooks/useMetadata', () => ({ + useTableMetadata: () => ({ data: undefined, isLoading: false }), +})); + +jest.mock('../ChartSQLPreview', () => ({ + SQLPreview: () =>
SQL Preview
, +})); + +const mockLogSource = { + connection: 'conn-1', + from: { databaseName: 'default', tableName: 'logs' }, +}; + +describe('SourceSchemaPreview — controlled mode', () => { + it('does not render the uncontrolled trigger when controlled is true', () => { + renderWithMantine( + , + ); + // The non-controlled mode would render a clickable icon/text trigger; + // in controlled mode nothing should render when the modal is closed. + expect(screen.queryByText('Schema')).not.toBeInTheDocument(); + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); + + it('opens the modal when open is true', () => { + renderWithMantine( + , + ); + expect(screen.getByRole('dialog')).toBeInTheDocument(); + }); + + it('does not show the modal when open is false', () => { + renderWithMantine( + , + ); + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); + + it('calls onClose when the modal close button is clicked', async () => { + const onClose = jest.fn(); + renderWithMantine( + , + ); + // Mantine 9 CloseButton renders without aria-label; find by its class. + const closeBtn = document.querySelector( + '.mantine-Modal-close', + ) as HTMLElement; + expect(closeBtn).toBeTruthy(); + await userEvent.click(closeBtn); + expect(onClose).toHaveBeenCalledTimes(1); + }); + + it('calls onClose on Escape key press', async () => { + const onClose = jest.fn(); + renderWithMantine( + , + ); + await userEvent.keyboard('{Escape}'); + expect(onClose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/app/src/components/__tests__/SourceSelect.test.tsx b/packages/app/src/components/__tests__/SourceSelect.test.tsx new file mode 100644 index 0000000000..08b1e23b37 --- /dev/null +++ b/packages/app/src/components/__tests__/SourceSelect.test.tsx @@ -0,0 +1,137 @@ +import React from 'react'; +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import { SourceManagementMenu } from '../SourceSelect'; + +describe('SourceManagementMenu', () => { + it('does not render the kebab trigger when no actions are wired', () => { + renderWithMantine(); + expect(screen.queryByTestId('source-actions-menu')).not.toBeInTheDocument(); + }); + + it('renders the kebab trigger when onSchemaPreview is wired', () => { + renderWithMantine( + , + ); + expect(screen.getByTestId('source-actions-menu')).toBeInTheDocument(); + }); + + it('renders the kebab trigger when only onEdit is wired', () => { + renderWithMantine( + , + ); + expect(screen.getByTestId('source-actions-menu')).toBeInTheDocument(); + }); + + it('renders the kebab trigger when only onManageSources is wired', () => { + renderWithMantine( + , + ); + expect(screen.getByTestId('source-actions-menu')).toBeInTheDocument(); + }); + + it('renders the kebab trigger when only onCreate is wired', () => { + renderWithMantine( + , + ); + expect(screen.getByTestId('source-actions-menu')).toBeInTheDocument(); + }); + + describe('View schema item', () => { + it('is disabled when hasSelection is false', async () => { + renderWithMantine( + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + const viewSchemaBtn = await screen.findByText('View schema'); + expect(viewSchemaBtn.closest('button')).toHaveAttribute( + 'data-disabled', + 'true', + ); + }); + + it('is disabled when isSchemaPreviewEnabled is false', async () => { + renderWithMantine( + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + const viewSchemaBtn = await screen.findByText('View schema'); + expect(viewSchemaBtn.closest('button')).toHaveAttribute( + 'data-disabled', + 'true', + ); + }); + + it('is enabled when hasSelection and isSchemaPreviewEnabled are both true', async () => { + renderWithMantine( + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + const viewSchemaBtn = await screen.findByText('View schema'); + expect(viewSchemaBtn.closest('button')).not.toHaveAttribute( + 'data-disabled', + ); + }); + + it('calls onSchemaPreview when clicked', async () => { + const onSchemaPreview = jest.fn(); + renderWithMantine( + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + await userEvent.click(await screen.findByText('View schema')); + expect(onSchemaPreview).toHaveBeenCalledTimes(1); + }); + }); + + it('calls onEdit when Edit source is clicked', async () => { + const onEdit = jest.fn(); + renderWithMantine( + // hasSelection required: Edit source is disabled without a selection. + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + await userEvent.click(await screen.findByText('Edit source')); + expect(onEdit).toHaveBeenCalledTimes(1); + }); + + it('calls onManageSources when Manage sources is clicked', async () => { + const onManageSources = jest.fn(); + renderWithMantine( + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + await userEvent.click(await screen.findByText('Manage sources')); + expect(onManageSources).toHaveBeenCalledTimes(1); + }); + + it('calls onCreate when Create new source is clicked', async () => { + const onCreate = jest.fn(); + renderWithMantine( + , + ); + await userEvent.click(screen.getByTestId('source-actions-menu')); + await userEvent.click(await screen.findByText('Create new source')); + expect(onCreate).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/app/styles/SourceSelectControlled.module.scss b/packages/app/styles/SourceSelectControlled.module.scss index f586b5d2a7..6f648df5be 100644 --- a/packages/app/styles/SourceSelectControlled.module.scss +++ b/packages/app/styles/SourceSelectControlled.module.scss @@ -1,3 +1,6 @@ +/* Legacy class. Still used by DBTableSelect for its inline schema-preview + * text button. New code uses SourceManagementMenu (kebab) instead. + */ .sourceSchemaPreviewButton { pointer-events: all; font-size: var(--mantine-font-size-xxs); @@ -16,3 +19,74 @@ border-color: var(--color-border); } } + +/* Chip + kebab wrapper. Holds the Mantine Select and the SourceManagementMenu + * ActionIcon as a single inline-flex unit. When the kebab is present + * (`[data-with-menu]`), the two surfaces are "glued" into a split-button chip: + * the adjoining corners are squared, the kebab overlaps the input's right + * border by 1px so they share a single vertical seam, and the kebab stretches + * to match the input's height so the chip reads as one unit. + * + * Override Mantine via its own CSS variables (`--input-radius`, `--ai-radius`) + * rather than fighting class-level `border-radius`: Mantine v7 reads these + * vars to compute the rounded corners, so setting them is the canonical + * override path. + */ +.sourceSelectGroup { + display: inline-flex; + align-items: stretch; + gap: 0; + + &[data-with-menu] { + /* Square only the right edge of the input. */ + .sourceSelectInput { + --input-radius: 0; + + border-top-left-radius: var(--mantine-radius-default); + border-bottom-left-radius: var(--mantine-radius-default); + } + + /* Square only the left edge of the kebab. Fixed 30px height matches the + * input chip; -1px margin overlaps the input's right edge so the shared + * border is one pixel wide, not two. */ + .sourceMenuButton { + --ai-radius: 0; + + width: 30px; + height: 30px; + margin-left: -1px; + background-color: var(--color-bg-field-addon); + border: 1px solid var(--input-bd, var(--color-border)); + border-top-right-radius: var(--mantine-radius-default); + border-bottom-right-radius: var(--mantine-radius-default); + } + } +} + +/* Chip hover state. The Mantine Select input is the "chip" surface; + * we tint its background and force a pointer cursor on hover so the + * chevron + label feel like a single clickable affordance even though + * the input itself is searchable. + */ +.sourceSelectInput { + cursor: pointer; + transition: + background-color 0.15s ease, + border-color 0.15s ease; + + &:hover:not(:focus-within) { + background-color: var(--mantine-color-default-hover); + } + + /* Once the user actually starts typing, keep text cursor semantics. */ + &:focus-within { + cursor: text; + } +} + +/* Kebab button. Mantine's subtle variant gives the hover tint; layout shared + * with the input is set up in the `[data-with-menu]` block above. + */ +.sourceMenuButton { + flex-shrink: 0; +} diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index b56bad6fbf..3490a016d3 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -102,13 +102,20 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { await searchPage.goto(); }); - test('should show source actions in dropdown', async () => { - // Open source selector dropdown - await searchPage.sourceDropdown.click(); + test('should show source actions in kebab menu', async () => { + // Open the source-actions kebab menu next to the source picker. + await searchPage.sourceActionsMenu.click(); - // Verify action items are visible in the dropdown + // Per-source and create actions are always wired. + await expect(searchPage.editSourceItem).toBeVisible(); await expect(searchPage.createNewSourceItem).toBeVisible(); - await expect(searchPage.editSourcesItem).toBeVisible(); + + // `Manage sources` is wired only in non-local (full-stack) mode. + if (process.env.E2E_FULLSTACK === 'true') { + await expect(searchPage.manageSourcesItem).toBeVisible(); + } else { + await expect(searchPage.manageSourcesItem).toHaveCount(0); + } }); test( @@ -142,7 +149,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { ); test('should show proper fields when creating a new source', async () => { - await searchPage.sourceDropdown.click(); + await searchPage.sourceActionsMenu.click(); await searchPage.createNewSourceItem.click(); // for each source type (log, trace, session, metric), verify the correct fields are shown for (const sourceData of allSourcesData) { diff --git a/packages/app/tests/e2e/page-objects/SearchPage.ts b/packages/app/tests/e2e/page-objects/SearchPage.ts index 6734ab72b9..9682c977f1 100644 --- a/packages/app/tests/e2e/page-objects/SearchPage.ts +++ b/packages/app/tests/e2e/page-objects/SearchPage.ts @@ -73,12 +73,24 @@ export class SearchPage { this.sourceSelector = page.getByTestId('source-selector'); } + get sourceActionsMenu() { + return this.page.getByTestId('source-actions-menu'); + } + get createNewSourceItem() { - return this.page.getByRole('option', { name: 'Create New Source' }); + return this.page.getByRole('menuitem', { name: 'Create new source' }); + } + + get editSourceItem() { + return this.page.getByRole('menuitem', { name: 'Edit source' }); } - get editSourcesItem() { - return this.page.getByRole('option', { name: 'Edit Sources' }); + get manageSourcesItem() { + return this.page.getByRole('menuitem', { name: 'Manage sources' }); + } + + get viewSchemaItem() { + return this.page.getByRole('menuitem', { name: 'View schema' }); } /** @@ -98,8 +110,8 @@ export class SearchPage { } async openEditSourceModal() { - await this.sourceSelector.click(); - await this.editSourcesItem.click(); + await this.sourceActionsMenu.click(); + await this.editSourceItem.click(); } async sourceModalShowOptionalFields() { From 947d72241207a0d2fb015e3a22cfb75489552bbc Mon Sep 17 00:00:00 2001 From: Elizabet Oliveira Date: Mon, 1 Jun 2026 16:47:57 +0100 Subject: [PATCH 005/728] feat(theme): add viz-range and drawer-elevation design tokens (HDX-3950) (#2392) --- .../src/theme/themes/clickstack/_tokens.scss | 22 +++++++++++++++++++ .../app/src/theme/themes/hyperdx/_tokens.scss | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/packages/app/src/theme/themes/clickstack/_tokens.scss b/packages/app/src/theme/themes/clickstack/_tokens.scss index 0c77f5498c..0651f8ef9d 100644 --- a/packages/app/src/theme/themes/clickstack/_tokens.scss +++ b/packages/app/src/theme/themes/clickstack/_tokens.scss @@ -200,6 +200,17 @@ --color-chart-warning-highlight: #f5c94d; --color-chart-error-highlight: #ffa090; + /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ + --color-viz-range-handle: var(--palette-brand-300); + --color-viz-range-handle-grip: var(--color-text-inverted); + --color-viz-range-border: var(--palette-brand-700); + --color-viz-range-overlay: rgb(0 0 0 / 20%); + --color-viz-range-brush: rgb(250 255 105 / 20%); + --color-viz-range-brush-border: rgb(250 255 105 / 50%); + + /* Shadows */ + --shadow-drawer: -4px 0 24px rgb(0 0 0 / 25%); + /* Mantine Overrides */ --mantine-color-body: var(--color-bg-body) !important; --mantine-color-text: var(--color-text) !important; @@ -401,6 +412,17 @@ --color-chart-warning-highlight: #f5c94d; --color-chart-error-highlight: #ffa090; + /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ + --color-viz-range-handle: var(--mantine-color-indigo-8); + --color-viz-range-handle-grip: var(--mantine-color-indigo-1); + --color-viz-range-border: var(--mantine-color-indigo-8); + --color-viz-range-overlay: rgb(0 0 0 / 4%); + --color-viz-range-brush: rgb(67 126 239 / 20%); + --color-viz-range-brush-border: var(--mantine-color-indigo-8); + + /* Shadows */ + --shadow-drawer: -4px 0 24px rgb(0 0 0 / 4%); + /* Mantine Overrides */ --mantine-color-body: var(--color-bg-body) !important; --mantine-color-text: var(--color-text) !important; diff --git a/packages/app/src/theme/themes/hyperdx/_tokens.scss b/packages/app/src/theme/themes/hyperdx/_tokens.scss index 1503bbc4ae..88f8e6ff8b 100644 --- a/packages/app/src/theme/themes/hyperdx/_tokens.scss +++ b/packages/app/src/theme/themes/hyperdx/_tokens.scss @@ -116,6 +116,17 @@ --color-chart-warning-highlight: #f5c94d; --color-chart-error-highlight: #ffa090; + /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ + --color-viz-range-handle: var(--mantine-color-dark-2); + --color-viz-range-handle-grip: rgb(255 255 255 / 45%); + --color-viz-range-border: var(--mantine-color-dark-3); + --color-viz-range-overlay: rgb(0 0 0 / 20%); + --color-viz-range-brush: rgb(100 149 237 / 25%); + --color-viz-range-brush-border: rgb(100 149 237 / 60%); + + /* Shadows */ + --shadow-drawer: -4px 0 24px rgb(0 0 0 / 25%); + /* Mantine Overrides */ --mantine-color-body: var(--color-bg-body) !important; --mantine-color-text: var(--color-text); @@ -228,6 +239,17 @@ --color-chart-warning-highlight: #f5c94d; --color-chart-error-highlight: #ffa090; + /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ + --color-viz-range-handle: var(--mantine-color-gray-5); + --color-viz-range-handle-grip: rgb(255 255 255 / 80%); + --color-viz-range-border: var(--mantine-color-gray-4); + --color-viz-range-overlay: rgb(0 0 0 / 10%); + --color-viz-range-brush: rgb(66 105 208 / 20%); + --color-viz-range-brush-border: rgb(66 105 208 / 50%); + + /* Shadows */ + --shadow-drawer: -4px 0 24px rgb(0 0 0 / 4%); + /* Mantine Overrides */ --mantine-color-body: var(--color-bg-body); --mantine-color-text: var(--color-text); From 9d713999af4d8bd20493cfff02b1a2eff631cce0 Mon Sep 17 00:00:00 2001 From: Elizabet Oliveira Date: Mon, 1 Jun 2026 18:44:08 +0100 Subject: [PATCH 006/728] fix(PageHeader): keep sticky header below drawer overlays (#2394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The sticky `PageHeader` (and its `stickyRow`) were pinned at `z-index: 100`, which floated them above app drawers. Drawers in this codebase don't use Mantine's default `modal` elevation (200) — they explicitly opt into a much lower stack via `ZIndexContext` (`contextZIndex + 10`, see `DBRowSidePanel`, `NodeDetailsSidePanel`, `PodDetailsSidePanel`, etc.), so a top-level drawer overlay ends up at `--overlay-z-index: 10`. The header would therefore poke through the drawer overlay and remain interactive while a drawer was open. This drops both `.header` and `.stickyRow` to `z-index: 2` — 8 below the top-level drawer z-index — so: - Drawer overlays (and the drawer body itself) reliably cover the sticky page chrome. - The header still wins against normal scrolling page content, which uses `auto`/no z-index. - Nested drawers (which keep stacking via `ZIndexContext`) continue to layer above their parents unchanged. Comments in the SCSS document the relationship so the next person tweaking these values has the context. ## Test plan - [ ] Open a page that uses `PageHeader` (e.g. Alerts, DB Service Map, Kubernetes Dashboard) and open a side panel / drawer; confirm the drawer overlay fully covers the sticky header and the header is no longer clickable through the overlay. - [ ] Scroll a long page and confirm the sticky header still sits above normal content (charts, tables, lists). - [ ] Open a nested drawer (e.g. row side panel from within another side panel) and confirm stacking still works as before. Made with [Cursor](https://cursor.com) --- .../page-header-z-index-below-drawers.md | 20 +++++++++++ .../src/components/FullscreenPanelModal.tsx | 34 +++++++++++++------ .../app/src/components/PageHeader.module.scss | 12 +++++-- 3 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 .changeset/page-header-z-index-below-drawers.md diff --git a/.changeset/page-header-z-index-below-drawers.md b/.changeset/page-header-z-index-below-drawers.md new file mode 100644 index 0000000000..31d4dd178e --- /dev/null +++ b/.changeset/page-header-z-index-below-drawers.md @@ -0,0 +1,20 @@ +--- +'@hyperdx/app': patch +--- + +fix(z-index): keep sticky header below drawers and drawers above the fullscreen tile modal + +Two related z-index regressions: + +- `PageHeader` was pinned at `z-index: 100`, but app drawers opt into a + much lower stack via `ZIndexContext` (`contextZIndex + 10`, so a + top-level drawer renders at `z-index: 10`). The sticky header therefore + floated above the drawer overlay. The header now sits at `z-index: 2` so + drawer overlays reliably cover the page chrome while the header still + wins against normal scrolling content. +- `FullscreenPanelModal` used Mantine's default modal z-index (`200`) and + didn't propagate it through `ZIndexContext`. Clicking a row in a + fullscreen search tile opened a `DBRowSidePanel` drawer at `z-index: 10` + that was hidden behind the modal. The modal now follows the existing + `contextZIndex + 10` pattern and wraps its children in a + `ZIndexContext.Provider`, so child drawers stack on top of it. diff --git a/packages/app/src/components/FullscreenPanelModal.tsx b/packages/app/src/components/FullscreenPanelModal.tsx index e7f9c61c4a..1b3df9d793 100644 --- a/packages/app/src/components/FullscreenPanelModal.tsx +++ b/packages/app/src/components/FullscreenPanelModal.tsx @@ -1,6 +1,8 @@ import { Box, Modal } from '@mantine/core'; import { useHotkeys } from '@mantine/hooks'; +import { useZIndex, ZIndexContext } from '@/zIndex'; + export default function FullscreenPanelModal({ opened, onClose, @@ -15,6 +17,13 @@ export default function FullscreenPanelModal({ // YouTube-style 'f' key to toggle fullscreen useHotkeys([['f', () => opened && onClose()]]); + // Stack on top of any ancestor drawer/modal (`contextZIndex + 10`) and + // expose this value via `ZIndexContext` so any drawer/popover opened from + // inside the fullscreen view (e.g. clicking a row in a search tile) lands + // above the modal instead of being hidden behind it. + const contextZIndex = useZIndex(); + const modalZIndex = contextZIndex + 10; + return ( - - {children} - + + + {children} + + ); } diff --git a/packages/app/src/components/PageHeader.module.scss b/packages/app/src/components/PageHeader.module.scss index 3408b57ead..1bde82d825 100644 --- a/packages/app/src/components/PageHeader.module.scss +++ b/packages/app/src/components/PageHeader.module.scss @@ -14,7 +14,12 @@ padding: 0 var(--mantine-spacing-sm); position: sticky; top: 0; - z-index: 100; + + /* Top-level drawers in this app are rendered at z-index 10 + (`contextZIndex + 10` from `ZIndexContext`, see e.g. `DBRowSidePanel`). + Sit 8 below that so the drawer overlay always covers the page chrome, + while the header still floats above normal scrolling content. */ + z-index: 2; } /* Applied automatically when a `stickyRow` is provided: the chrome scrolls @@ -40,7 +45,10 @@ padding: var(--mantine-spacing-sm); position: sticky; top: 0; - z-index: 100; + + /* Match `.header` — sit 8 below the top-level drawer z-index (10) so + drawer overlays always cover this row. */ + z-index: 2; } /* Applied when the sticky row sits directly under chrome AND is not From 6bd4c9e525e15bf6baff3d27c666e09bfe83d32c Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:58:28 -0400 Subject: [PATCH 007/728] fix: map alias with filters (#2393) ## Summary If you try and add a filter for an existing column that is a map access column, (ex: `ResourceAttributes['service.name']`), ClickHouse will return the column as `arrayElement(ResourceAttributes, 'service.name')`. Previously filters were defined in SQL, but now they are lucene, which means it doesn't automatically translate anymore. We could either make the filters aware of how to parse and translate that statement, or add an alias to the SELECT if a map is accessed. This PR adds both. ### Screenshots or video https://github.com/user-attachments/assets/d278962c-c33c-4c5e-a00e-90f8bee16f1b ### How to test on Vercel preview **Preview routes:** **Steps:** 1. Add a `ResourceAttributes['service.name'] 2. Hover over the column titled `ResourceAttributes['service.name']` on any row 3. Click "toggle filter" 4. Observe that the behavior is handled correctly ### References - Linear Issue: Closes HDX-4411 --- .changeset/thin-pets-relate.md | 6 ++ packages/app/src/components/DBRowTable.tsx | 16 ++++ .../components/__tests__/DBRowTable.test.tsx | 88 +++++++++++++++++++ .../src/__tests__/metadata.test.ts | 24 +++++ packages/common-utils/src/core/metadata.ts | 23 ++++- 5 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 .changeset/thin-pets-relate.md diff --git a/.changeset/thin-pets-relate.md b/.changeset/thin-pets-relate.md new file mode 100644 index 0000000000..71fe2ca5b1 --- /dev/null +++ b/.changeset/thin-pets-relate.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +--- + +fix: make filters aware of map column return arrayElement diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 9d2c8c31e6..53779e8d1d 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -29,6 +29,7 @@ import { isJSDataTypeJSONStringifiable, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; +import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, @@ -1508,6 +1509,18 @@ function selectColumnMapWithoutAdditionalKeys( ); } +export function addMapAliasesToSelect(select: string): string { + const selects = splitAndTrimWithBracket(select); + for (let i = 0; i < selects.length; i++) { + const key = selects[i]; + const path = parseKeyPath(key); + if (path.length === 2) { + selects[i] = `${key} as "${path[0]}['${path[1]}']"`; + } + } + return selects.join(','); +} + export type DBRowTableVariant = 'default' | 'muted'; function DBSqlRowTableComponent({ @@ -1602,6 +1615,9 @@ function DBSqlRowTableComponent({ ...searchChartConfigDefaults(me?.team), ...config, }; + if (typeof base.select === 'string') { + base.select = addMapAliasesToSelect(base.select); + } if (orderByArray.length) { base.orderBy = orderByArray.map(o => { return { diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index ffb459b2f5..03e8a80db6 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -2,6 +2,7 @@ import { screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { + addMapAliasesToSelect, appendSelectWithAdditionalKeys, RawLogTable, } from '@/components/DBRowTable'; @@ -416,3 +417,90 @@ describe('appendSelectWithAdditionalKeys', () => { }); }); }); + +describe('addMapAliasesToSelect', () => { + it('leaves plain columns unchanged and collapses ", " to ","', () => { + expect(addMapAliasesToSelect('Timestamp, ServiceName, SeverityText')).toBe( + 'Timestamp,ServiceName,SeverityText', + ); + }); + + it('aliases a single-quoted bracket-form map key', () => { + expect(addMapAliasesToSelect("ResourceAttributes['service.name']")).toBe( + `ResourceAttributes['service.name'] as "ResourceAttributes['service.name']"`, + ); + }); + + it('aliases only the map keys in a mixed multi-column input', () => { + expect( + addMapAliasesToSelect( + "Timestamp, ResourceAttributes['service.name'], SeverityText", + ), + ).toBe( + `Timestamp,ResourceAttributes['service.name'] as "ResourceAttributes['service.name']",SeverityText`, + ); + }); + + it('preserves function-call expressions whose arguments contain commas', () => { + expect( + addMapAliasesToSelect( + "Timestamp, toDateTime(now64(3, 'UTC')), ServiceName", + ), + ).toBe(`Timestamp,toDateTime(now64(3, 'UTC')),ServiceName`); + }); + + it('aliases an arrayElement(Col, key) Map subscript', () => { + expect( + addMapAliasesToSelect("arrayElement(ResourceAttributes, 'service.name')"), + ).toBe( + `arrayElement(ResourceAttributes, 'service.name') as "ResourceAttributes['service.name']"`, + ); + }); + + it('passes already-aliased plain columns through unchanged', () => { + expect(addMapAliasesToSelect('Timestamp as ts, ServiceName as svc')).toBe( + 'Timestamp as ts,ServiceName as svc', + ); + }); + + it('does NOT re-alias a bracket-form map subscript that already carries an "as " clause', () => { + expect( + addMapAliasesToSelect("ResourceAttributes['service.name'] as svc"), + ).toBe("ResourceAttributes['service.name'] as svc"); + }); + + it('does NOT re-alias an already-aliased map subscript within a multi-column input', () => { + expect( + addMapAliasesToSelect( + "Timestamp, ResourceAttributes['service.name'] as svc, SeverityText", + ), + ).toBe("Timestamp,ResourceAttributes['service.name'] as svc,SeverityText"); + }); + + it('does NOT alias unbracketed dot-notation tokens', () => { + expect(addMapAliasesToSelect('user_data.name')).toBe('user_data.name'); + expect(addMapAliasesToSelect('user_data.address.city')).toBe( + 'user_data.address.city', + ); + }); + + it('does NOT alias multi-segment dotted native columns', () => { + expect(addMapAliasesToSelect('SpanAttributes.k8s.resource.name')).toBe( + 'SpanAttributes.k8s.resource.name', + ); + }); + + it('does NOT alias toString(x).y attribute-access expressions', () => { + expect(addMapAliasesToSelect('toString(SpanAttributes).k8s')).toBe( + 'toString(SpanAttributes).k8s', + ); + }); + + it('does NOT alias backtick-quoted dotted JSON paths', () => { + expect(addMapAliasesToSelect('`host`.`arch`')).toBe('`host`.`arch`'); + }); + + it('returns an empty string for empty input', () => { + expect(addMapAliasesToSelect('')).toBe(''); + }); +}); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index bbd40781ef..083c9005a2 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1471,4 +1471,28 @@ describe('parseKeyPath', () => { "ResourceAttributes['service.name", ]); }); + + it('parses single-quoted arrayElement function-call form', () => { + expect( + parseKeyPath("arrayElement(ResourceAttributes, 'service.name')"), + ).toEqual(['ResourceAttributes', 'service.name']); + }); + + it('parses arrayElement keys with dots in the map key', () => { + expect( + parseKeyPath("arrayElement(SpanAttributes, 'http.request.method')"), + ).toEqual(['SpanAttributes', 'http.request.method']); + }); + + it('tolerates whitespace inside arrayElement', () => { + expect( + parseKeyPath("arrayElement( ResourceAttributes , 'service.name' )"), + ).toEqual(['ResourceAttributes', 'service.name']); + }); + + it('does not parse arrayElement with a missing closing paren', () => { + expect( + parseKeyPath("arrayElement(ResourceAttributes, 'service.name'"), + ).toEqual(["arrayElement(ResourceAttributes, 'service.name'"]); + }); }); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index a80feb1fad..59688cef15 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1768,9 +1768,17 @@ export type Field = { }; /** - * Parses a bracket-notation key string into a path array. - * e.g. `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` - * `ServiceName` → `['ServiceName']` + * Parses a Map-access key string into a path array. + * + * Handles both ClickHouse Map access forms: + * - Bracket form (canonical input): + * `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` + * `ResourceAttributes["service.name"]` → `['ResourceAttributes', 'service.name']` + * - Function-call form (what ClickHouse renders in result column names): + * `arrayElement(ResourceAttributes, 'service.name')` → `['ResourceAttributes', 'service.name']` + * + * Native columns return a single-element path: + * `ServiceName` → `['ServiceName']` */ export function parseKeyPath(key: string): string[] { const singleIdx = key.indexOf("['"); @@ -1781,6 +1789,15 @@ export function parseKeyPath(key: string): string[] { if (doubleIdx !== -1 && key.endsWith('"]')) { return [key.slice(0, doubleIdx), key.slice(doubleIdx + 2, -2)]; } + // ClickHouse Map access function-call form. ClickHouse renders Map + // subscript expressions (e.g. `SpanAttributes['http.route']`) as + // `arrayElement(SpanAttributes, 'http.route')` in result column names. + const ARRAY_ELEMENT_PREFIX = 'arrayElement('; + if (key.startsWith(ARRAY_ELEMENT_PREFIX) && key.endsWith(')')) { + const inner = key.slice(ARRAY_ELEMENT_PREFIX.length, -1); + const parts = inner.split(',').map(e => e.trim()); + return [parts[0], parts[1].replaceAll("'", '')]; + } return [key]; } From 9357642d4f0c95757385ee7b74598a1cde9afe8d Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:48:20 +0200 Subject: [PATCH 008/728] testing(e2e): backport e2e action split from EE (#2400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Backports the reusable **E2E composite-action split** from the EE PR [DeploySentinel/hyperdx-ee#2178](https://github.com/DeploySentinel/hyperdx-ee/pull/2178) into OSS. **Why:** `e2e-tests.yml` carried ~100 lines of inlined setup/teardown (port env vars, Node/yarn/build, Playwright install, Docker Compose with health checks, artifact upload, container teardown). Over in EE that block was extracted into two composite actions so the pipeline lives in one place. Backporting the same extraction keeps the OSS and EE E2E pipelines structurally aligned, removes the duplicated block, and reduces future merge conflicts whenever the setup pipeline changes on either side. **Behavior is unchanged** — same slot-0 ports, same `[1,2,3,4]` sharding, same artifacts, same health checks and teardown. The moved shell pipeline is byte-identical to the original (the only additions are the `shell: bash` lines that composite-action `run` steps require). The `main.yml` call site is untouched, since `e2e-tests.yml` keeps the same `workflow_call` interface. ### What changed - **New `.github/actions/e2e-setup/action.yml`** — composite action that sets the `HDX_E2E_*` port env vars and common Playwright env vars via `$GITHUB_ENV`, then runs Node setup, `yarn install`, `nx run-many -t ci:build`, Playwright browser install, and the E2E Docker Compose stack (MongoDB + ClickHouse) with health checks. - **New `.github/actions/e2e-teardown/action.yml`** — composite action that uploads `playwright-report-` / `test-results-` and stops the Docker stack. All steps run with `if: always()`. Takes an `artifact-suffix` input (`${{ matrix.shard }}`). - **Modified `.github/workflows/e2e-tests.yml`** — the inlined setup/teardown blocks are replaced by the two composite actions (94 deletions → 5 insertions). Trigger, job structure, `runs-on`, sharding, and the `main.yml` call site are unchanged. ### How to test on Vercel preview **Preview routes:** N/A — non-UI change (CI configuration only). ### References - Linear Issue: Closes [HDX-4428](https://linear.app/clickhouse/issue/HDX-4428/backport-e2e-ci-split-to-oss) - Related PRs: https://github.com/DeploySentinel/hyperdx-ee/pull/2178 --- .github/actions/e2e-setup/action.yml | 91 +++++++++++++++++++++++ .github/actions/e2e-teardown/action.yml | 35 +++++++++ .github/workflows/e2e-tests.yml | 99 ++----------------------- 3 files changed, 131 insertions(+), 94 deletions(-) create mode 100644 .github/actions/e2e-setup/action.yml create mode 100644 .github/actions/e2e-teardown/action.yml diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml new file mode 100644 index 0000000000..c2d2c80c5b --- /dev/null +++ b/.github/actions/e2e-setup/action.yml @@ -0,0 +1,91 @@ +name: E2E Setup +description: | + Shared setup for E2E test workflows: sets the HDX_E2E_* port env vars and + common Playwright env vars, then runs Node.js setup, dependency install, + build, Playwright browser install, and starts the E2E Docker Compose stack + (MongoDB + ClickHouse) with health checks. The caller is responsible for + `actions/checkout`, running Playwright, uploading artifacts, and tearing + down the Docker stack. +runs: + using: composite + steps: + # E2E port configuration (slot 0 defaults — must match scripts/test-e2e.sh) + - name: Set E2E environment variables + shell: bash + run: | + MONGO_PORT=21100 + echo "HDX_E2E_SLOT=0" >> $GITHUB_ENV + echo "HDX_E2E_OPAMP_PORT=20320" >> $GITHUB_ENV + echo "HDX_E2E_CH_PORT=20500" >> $GITHUB_ENV + echo "HDX_E2E_CH_NATIVE_PORT=20600" >> $GITHUB_ENV + echo "HDX_E2E_API_PORT=21000" >> $GITHUB_ENV + echo "HDX_E2E_MONGO_PORT=$MONGO_PORT" >> $GITHUB_ENV + echo "HDX_E2E_APP_LOCAL_PORT=21200" >> $GITHUB_ENV + echo "HDX_E2E_APP_PORT=21300" >> $GITHUB_ENV + echo "MONGO_URI=mongodb://localhost:$MONGO_PORT/hyperdx-e2e" >> $GITHUB_ENV + echo "E2E_FULLSTACK=true" >> $GITHUB_ENV + echo "E2E_UNIQUE_USER=true" >> $GITHUB_ENV + echo "E2E_API_HEALTH_CHECK_MAX_RETRIES=60" >> $GITHUB_ENV + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + cache-dependency-path: yarn.lock + cache: yarn + + - name: Install dependencies + shell: bash + run: yarn install + + - name: Build dependencies + shell: bash + run: npx nx run-many -t ci:build + + - name: Install Playwright browsers + shell: bash + run: cd packages/app && npx playwright install --with-deps chromium + + - name: Start E2E Docker Compose + shell: bash + run: | + # Pre-pull images with retries to handle transient Docker Hub timeouts + for attempt in 1 2 3; do + if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull; then + echo "Docker images pulled successfully on attempt $attempt" + break + fi + if [ "$attempt" -eq 3 ]; then + echo "Failed to pull Docker images after 3 attempts" + exit 1 + fi + echo "Docker pull failed (attempt $attempt/3), retrying in 10s..." + sleep 10 + done + docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d + echo "Waiting for MongoDB..." + for i in $(seq 1 30); do + if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml exec -T db mongosh --quiet --eval "db.adminCommand({ping:1})" >/dev/null 2>&1; then + echo "MongoDB is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "MongoDB failed to become ready after 30 seconds" + exit 1 + fi + echo "Waiting for MongoDB... ($i/30)" + sleep 1 + done + echo "Waiting for ClickHouse..." + for i in $(seq 1 60); do + if curl -sf http://localhost:${HDX_E2E_CH_PORT}/ping >/dev/null 2>&1; then + echo "ClickHouse is ready" + break + fi + if [ "$i" -eq 60 ]; then + echo "ClickHouse failed to become ready after 60 seconds" + exit 1 + fi + echo "Waiting for ClickHouse... ($i/60)" + sleep 1 + done diff --git a/.github/actions/e2e-teardown/action.yml b/.github/actions/e2e-teardown/action.yml new file mode 100644 index 0000000000..b9e78e3140 --- /dev/null +++ b/.github/actions/e2e-teardown/action.yml @@ -0,0 +1,35 @@ +name: E2E Teardown +description: | + Upload Playwright artifacts and stop the E2E Docker Compose stack. + Always runs (even on failure) so containers are cleaned up and reports + are available for failed runs. +inputs: + artifact-suffix: + description: > + Suffix appended to artifact names, e.g. "1" → playwright-report-1. + required: true +runs: + using: composite + steps: + - name: Upload Playwright report + uses: actions/upload-artifact@v7 + if: always() + with: + name: playwright-report-${{ inputs.artifact-suffix }} + path: packages/app/playwright-report/ + retention-days: 30 + + - name: Upload test results + uses: actions/upload-artifact@v7 + if: always() + with: + name: test-results-${{ inputs.artifact-suffix }} + path: packages/app/test-results/ + retention-days: 30 + + - name: Stop E2E containers + if: always() + shell: bash + run: + docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml + down -v diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 5eb9f24904..9cd34cb1fd 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -14,109 +14,20 @@ jobs: matrix: shard: [1, 2, 3, 4] - # E2E port configuration (slot 0 defaults — must match scripts/test-e2e.sh) - env: - HDX_E2E_SLOT: '0' - HDX_E2E_OPAMP_PORT: '20320' - HDX_E2E_CH_PORT: '20500' - HDX_E2E_CH_NATIVE_PORT: '20600' - HDX_E2E_API_PORT: '21000' - HDX_E2E_MONGO_PORT: '21100' - HDX_E2E_APP_LOCAL_PORT: '21200' - HDX_E2E_APP_PORT: '21300' - steps: - name: Checkout uses: actions/checkout@v6 - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache-dependency-path: 'yarn.lock' - cache: 'yarn' - - - name: Install dependencies - run: yarn install - - - name: Build dependencies - run: npx nx run-many -t ci:build - - - name: Install Playwright browsers - run: cd packages/app && npx playwright install --with-deps chromium - - - name: Start E2E Docker Compose - run: | - # Pre-pull images with retries to handle transient Docker Hub timeouts - for attempt in 1 2 3; do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull; then - echo "Docker images pulled successfully on attempt $attempt" - break - fi - if [ "$attempt" -eq 3 ]; then - echo "Failed to pull Docker images after 3 attempts" - exit 1 - fi - echo "Docker pull failed (attempt $attempt/3), retrying in 10s..." - sleep 10 - done - docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d - echo "Waiting for MongoDB..." - for i in $(seq 1 30); do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml exec -T db mongosh --quiet --eval "db.adminCommand({ping:1})" >/dev/null 2>&1; then - echo "MongoDB is ready" - break - fi - if [ "$i" -eq 30 ]; then - echo "MongoDB failed to become ready after 30 seconds" - exit 1 - fi - echo "Waiting for MongoDB... ($i/30)" - sleep 1 - done - echo "Waiting for ClickHouse..." - for i in $(seq 1 60); do - if curl -sf http://localhost:${HDX_E2E_CH_PORT}/ping >/dev/null 2>&1; then - echo "ClickHouse is ready" - break - fi - if [ "$i" -eq 60 ]; then - echo "ClickHouse failed to become ready after 60 seconds" - exit 1 - fi - echo "Waiting for ClickHouse... ($i/60)" - sleep 1 - done + - name: E2E setup + uses: ./.github/actions/e2e-setup - name: Run Playwright tests (full-stack mode) - env: - E2E_FULLSTACK: 'true' - E2E_UNIQUE_USER: 'true' - E2E_API_HEALTH_CHECK_MAX_RETRIES: '60' - MONGO_URI: - mongodb://localhost:${{ env.HDX_E2E_MONGO_PORT }}/hyperdx-e2e run: | cd packages/app yarn test:e2e --shard=${{ matrix.shard }}/4 - - name: Upload Playwright report - uses: actions/upload-artifact@v7 - if: always() - with: - name: playwright-report-${{ matrix.shard }} - path: packages/app/playwright-report/ - retention-days: 30 - - - name: Upload test results - uses: actions/upload-artifact@v7 + - name: E2E teardown if: always() + uses: ./.github/actions/e2e-teardown with: - name: test-results-${{ matrix.shard }} - path: packages/app/test-results/ - retention-days: 30 - - - name: Stop E2E containers - if: always() - run: - docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml - down -v + artifact-suffix: ${{ matrix.shard }} From adac913defc6d9f10196a356a177aab0b9a4acfc Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 2 Jun 2026 08:24:10 -0600 Subject: [PATCH 009/728] refactor(mcp): rename tool prefixes from hyperdx_ to clickstack_ (#2396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename all MCP references from `hyperdx_` to `clickstack_` as part of the general MCP server marketing rename. ## Changes - **MCP server name**: `hyperdx` → `clickstack` - **OTel tracer name**: `hyperdx-mcp` → `clickstack-mcp` - **All 19 tool names**: `hyperdx_*` → `clickstack_*` (e.g. `hyperdx_search` → `clickstack_search`) - **Tool descriptions, error messages, and prompts**: updated all cross-references - **MCP.md**: title, config keys, and tool table updated - **Tests**: all tool name assertions updated CHANGELOGs left unchanged (historical accuracy). --- .changeset/rename-mcp-tools-to-clickstack.md | 10 + MCP.md | 46 ++--- packages/api/src/mcp/__tests__/alerts.test.ts | 64 +++--- .../api/src/mcp/__tests__/dashboards.test.ts | 186 +++++++++--------- .../api/src/mcp/__tests__/deltasTool.test.ts | 10 +- .../api/src/mcp/__tests__/queryTool.test.ts | 94 ++++----- .../src/mcp/__tests__/savedSearches.test.ts | 48 +++-- .../api/src/mcp/__tests__/sources.test.ts | 32 +-- packages/api/src/mcp/__tests__/trace.test.ts | 66 +++---- packages/api/src/mcp/mcpServer.ts | 2 +- .../api/src/mcp/prompts/dashboards/content.ts | 80 ++++---- .../api/src/mcp/prompts/dashboards/helpers.ts | 2 +- .../api/src/mcp/prompts/dashboards/index.ts | 6 +- packages/api/src/mcp/tools/alerts/getAlert.ts | 4 +- .../api/src/mcp/tools/alerts/getWebhook.ts | 6 +- .../api/src/mcp/tools/alerts/saveAlert.ts | 4 +- packages/api/src/mcp/tools/alerts/schemas.ts | 2 +- .../mcp/tools/dashboards/deleteDashboard.ts | 6 +- .../src/mcp/tools/dashboards/getDashboard.ts | 4 +- .../api/src/mcp/tools/dashboards/queryTile.ts | 8 +- .../src/mcp/tools/dashboards/saveDashboard.ts | 12 +- .../api/src/mcp/tools/dashboards/schemas.ts | 32 +-- .../api/src/mcp/tools/query/eventDeltas.ts | 18 +- .../api/src/mcp/tools/query/eventPatterns.ts | 14 +- packages/api/src/mcp/tools/query/helpers.ts | 4 +- .../src/mcp/tools/query/runEventPatterns.ts | 8 +- packages/api/src/mcp/tools/query/schemas.ts | 2 +- packages/api/src/mcp/tools/query/search.ts | 10 +- packages/api/src/mcp/tools/query/sql.ts | 16 +- packages/api/src/mcp/tools/query/table.ts | 6 +- .../api/src/mcp/tools/query/timeseries.ts | 6 +- .../mcp/tools/savedSearches/getSavedSearch.ts | 4 +- .../tools/savedSearches/saveSavedSearch.ts | 6 +- .../src/mcp/tools/savedSearches/schemas.ts | 2 +- .../src/mcp/tools/sources/describeSource.ts | 20 +- .../api/src/mcp/tools/sources/listSources.ts | 16 +- packages/api/src/mcp/tools/trace/breakdown.ts | 18 +- packages/api/src/mcp/tools/trace/waterfall.ts | 14 +- packages/api/src/mcp/utils/tracing.ts | 2 +- packages/common-utils/src/core/eventDeltas.ts | 4 +- 40 files changed, 461 insertions(+), 433 deletions(-) create mode 100644 .changeset/rename-mcp-tools-to-clickstack.md diff --git a/.changeset/rename-mcp-tools-to-clickstack.md b/.changeset/rename-mcp-tools-to-clickstack.md new file mode 100644 index 0000000000..70347fa33c --- /dev/null +++ b/.changeset/rename-mcp-tools-to-clickstack.md @@ -0,0 +1,10 @@ +--- +"@hyperdx/api": patch +"@hyperdx/common-utils": patch +--- + +refactor(mcp): rename all MCP tool prefixes from `hyperdx_` to `clickstack_` + +Rename the MCP server name from `hyperdx` to `clickstack` and update all 19 +tool names (e.g. `hyperdx_search` → `clickstack_search`), along with +descriptions, prompts, error messages, and test references. diff --git a/MCP.md b/MCP.md index 7f8e89af8c..3616e5c9a6 100644 --- a/MCP.md +++ b/MCP.md @@ -1,4 +1,4 @@ -# HyperDX MCP Server +# ClickStack MCP Server HyperDX exposes a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) server that @@ -41,7 +41,7 @@ instance URL (e.g. `http://localhost:8080`). ### Claude Code ```bash -claude mcp add --transport http hyperdx /api/mcp \ +claude mcp add --transport http clickstack /api/mcp \ --header "Authorization: Bearer " ``` @@ -51,7 +51,7 @@ Add the following to your [OpenCode config](https://opencode.ai/docs/config): ```json "mcp": { - "hyperdx": { + "clickstack": { "enabled": true, "type": "remote", "url": "/api/mcp", @@ -70,7 +70,7 @@ settings): ```json { "mcpServers": { - "hyperdx": { + "clickstack": { "url": "/api/mcp", "headers": { "Authorization": "Bearer " @@ -108,22 +108,22 @@ with: | Tool | Description | | ----------------------------- | -------------------------------------------------------------------------------------------- | -| `hyperdx_list_sources` | List all data sources and connections as a lightweight catalog (IDs, names, kinds) | -| `hyperdx_describe_source` | Full column schema, attribute keys, and sampled low-cardinality values for a single source | -| `hyperdx_timeseries` | Plot metrics over time as a line or stacked bar chart | -| `hyperdx_table` | Compute aggregated metrics as a table, single number, or pie chart | -| `hyperdx_search` | Browse individual log, event, or trace rows | -| `hyperdx_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | -| `hyperdx_event_deltas` | Compare two row groups and rank properties by how their value distributions differ | -| `hyperdx_sql` | Execute raw ClickHouse SQL for advanced queries (JOINs, CTEs, sub-queries) | -| `hyperdx_get_dashboard` | List all dashboards or get full detail for a specific dashboard | -| `hyperdx_save_dashboard` | Create or update a dashboard with tiles (charts, tables, numbers, search, markdown) | -| `hyperdx_delete_dashboard` | Permanently delete a dashboard and its attached alerts | -| `hyperdx_query_tile` | Execute the query for a specific dashboard tile to validate results | -| `hyperdx_get_saved_search` | List all saved searches or get full detail for a specific saved search | -| `hyperdx_save_saved_search` | Create or update a saved search (reusable query against a data source) | -| `hyperdx_get_alert` | List alerts (summary) or get full detail with evaluation history; filter by state | -| `hyperdx_save_alert` | Create a new alert or update an existing one | -| `hyperdx_trace_waterfall` | Fetch all spans in a single trace as a parent/child waterfall tree with optional correlated logs | -| `hyperdx_trace_top_time_consuming_operations` | Aggregate breakdown of child operations by cumulative time across matching parent traces | -| `hyperdx_get_webhook` | List available webhook destinations for use as alert notification channels | +| `clickstack_list_sources` | List all data sources and connections as a lightweight catalog (IDs, names, kinds) | +| `clickstack_describe_source` | Full column schema, attribute keys, and sampled low-cardinality values for a single source | +| `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart | +| `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart | +| `clickstack_search` | Browse individual log, event, or trace rows | +| `clickstack_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | +| `clickstack_event_deltas` | Compare two row groups and rank properties by how their value distributions differ | +| `clickstack_sql` | Execute raw ClickHouse SQL for advanced queries (JOINs, CTEs, sub-queries) | +| `clickstack_get_dashboard` | List all dashboards or get full detail for a specific dashboard | +| `clickstack_save_dashboard` | Create or update a dashboard with tiles (charts, tables, numbers, search, markdown) | +| `clickstack_delete_dashboard` | Permanently delete a dashboard and its attached alerts | +| `clickstack_query_tile` | Execute the query for a specific dashboard tile to validate results | +| `clickstack_get_saved_search` | List all saved searches or get full detail for a specific saved search | +| `clickstack_save_saved_search` | Create or update a saved search (reusable query against a data source) | +| `clickstack_get_alert` | List alerts (summary) or get full detail with evaluation history; filter by state | +| `clickstack_save_alert` | Create a new alert or update an existing one | +| `clickstack_trace_waterfall` | Fetch all spans in a single trace as a parent/child waterfall tree with optional correlated logs | +| `clickstack_trace_top_time_consuming_operations` | Aggregate breakdown of child operations by cumulative time across matching parent traces | +| `clickstack_get_webhook` | List available webhook destinations for use as alert notification channels | diff --git a/packages/api/src/mcp/__tests__/alerts.test.ts b/packages/api/src/mcp/__tests__/alerts.test.ts index 05c39a3daf..d31d851258 100644 --- a/packages/api/src/mcp/__tests__/alerts.test.ts +++ b/packages/api/src/mcp/__tests__/alerts.test.ts @@ -124,15 +124,15 @@ describe('MCP Alert Tools', () => { }).save(); } - // ─── hyperdx_get_alert ──────────────────────────────────────────────────── + // ─── clickstack_get_alert ──────────────────────────────────────────────────── - describe('hyperdx_get_alert', () => { + describe('clickstack_get_alert', () => { describe('list (no id)', () => { it('should list all alerts with slim summary fields', async () => { await createTestAlert({ name: 'Alert 1' }); await createTestAlert({ name: 'Alert 2' }); - const result = await callTool(client, 'hyperdx_get_alert', {}); + const result = await callTool(client, 'clickstack_get_alert', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -153,7 +153,7 @@ describe('MCP Alert Tools', () => { }); it('should return empty array when no alerts exist', async () => { - const result = await callTool(client, 'hyperdx_get_alert', {}); + const result = await callTool(client, 'clickstack_get_alert', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -165,7 +165,7 @@ describe('MCP Alert Tools', () => { await createTestAlert({ name: 'OK', state: AlertState.OK }); await createTestAlert({ name: 'Disabled', state: AlertState.DISABLED }); - const result = await callTool(client, 'hyperdx_get_alert', { + const result = await callTool(client, 'clickstack_get_alert', { state: 'ALERT', }); @@ -179,7 +179,7 @@ describe('MCP Alert Tools', () => { it('should return empty array when no alerts match state filter', async () => { await createTestAlert({ name: 'All Good', state: AlertState.OK }); - const result = await callTool(client, 'hyperdx_get_alert', { + const result = await callTool(client, 'clickstack_get_alert', { state: 'ALERT', }); @@ -198,7 +198,7 @@ describe('MCP Alert Tools', () => { const client2 = await createTestClient(otherTeamContext); // List should be empty for the other team - const listResult = await callTool(client2, 'hyperdx_get_alert', {}); + const listResult = await callTool(client2, 'clickstack_get_alert', {}); const output = JSON.parse(getFirstText(listResult)); expect(output).toHaveLength(0); @@ -220,7 +220,7 @@ describe('MCP Alert Tools', () => { // no name set }).save(); - const result = await callTool(client, 'hyperdx_get_alert', {}); + const result = await callTool(client, 'clickstack_get_alert', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -244,7 +244,7 @@ describe('MCP Alert Tools', () => { // no name set }).save(); - const result = await callTool(client, 'hyperdx_get_alert', {}); + const result = await callTool(client, 'clickstack_get_alert', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -257,7 +257,7 @@ describe('MCP Alert Tools', () => { it('should get full alert detail with history when valid id is provided', async () => { const alert = await createTestAlert({ name: 'Detail Test' }); - const result = await callTool(client, 'hyperdx_get_alert', { + const result = await callTool(client, 'clickstack_get_alert', { id: alert._id.toString(), }); @@ -274,7 +274,7 @@ describe('MCP Alert Tools', () => { }); it('should return error for invalid ObjectId format', async () => { - const result = await callTool(client, 'hyperdx_get_alert', { + const result = await callTool(client, 'clickstack_get_alert', { id: 'not-a-valid-id', }); @@ -284,7 +284,7 @@ describe('MCP Alert Tools', () => { it('should return error for non-existent alert id', async () => { const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_get_alert', { + const result = await callTool(client, 'clickstack_get_alert', { id: fakeId, }); @@ -301,7 +301,7 @@ describe('MCP Alert Tools', () => { }; const client2 = await createTestClient(otherTeamContext); - const getResult = await callTool(client2, 'hyperdx_get_alert', { + const getResult = await callTool(client2, 'clickstack_get_alert', { id: alert._id.toString(), }); expect(getResult.isError).toBe(true); @@ -312,15 +312,15 @@ describe('MCP Alert Tools', () => { }); }); - // ─── hyperdx_save_alert ─────────────────────────────────────────────────── + // ─── clickstack_save_alert ─────────────────────────────────────────────────── - describe('hyperdx_save_alert', () => { + describe('clickstack_save_alert', () => { describe('create', () => { it('should create a saved-search alert', async () => { const savedSearch = await createTestSavedSearch(); const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'saved_search', savedSearchId: savedSearch._id.toString(), threshold: 50, @@ -346,7 +346,7 @@ describe('MCP Alert Tools', () => { const dashboard = await createTestDashboardWithTile(); const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'tile', dashboardId: dashboard._id.toString(), tileId: 'tile-1', @@ -371,7 +371,7 @@ describe('MCP Alert Tools', () => { it('should reject tile source without dashboardId', async () => { const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'tile', tileId: 'tile-1', threshold: 100, @@ -391,7 +391,7 @@ describe('MCP Alert Tools', () => { const dashboard = await createTestDashboardWithTile(); const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'tile', dashboardId: dashboard._id.toString(), threshold: 100, @@ -410,7 +410,7 @@ describe('MCP Alert Tools', () => { it('should reject saved_search source without savedSearchId', async () => { const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'saved_search', threshold: 100, thresholdType: 'above', @@ -429,7 +429,7 @@ describe('MCP Alert Tools', () => { const webhook = await createTestWebhook(); const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'saved_search', savedSearchId: fakeId, threshold: 100, @@ -448,7 +448,7 @@ describe('MCP Alert Tools', () => { it('should reject webhook channel without webhookId', async () => { const savedSearch = await createTestSavedSearch(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'saved_search', savedSearchId: savedSearch._id.toString(), threshold: 100, @@ -467,7 +467,7 @@ describe('MCP Alert Tools', () => { const savedSearch = await createTestSavedSearch(); const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'saved_search', savedSearchId: savedSearch._id.toString(), threshold: 100, @@ -487,7 +487,7 @@ describe('MCP Alert Tools', () => { const savedSearch = await createTestSavedSearch(); const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { source: 'saved_search', savedSearchId: savedSearch._id.toString(), threshold: 75, @@ -532,7 +532,7 @@ describe('MCP Alert Tools', () => { name: 'Original Name', }).save(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { id: alert._id.toString(), source: 'saved_search', savedSearchId: savedSearch._id.toString(), @@ -559,7 +559,7 @@ describe('MCP Alert Tools', () => { const webhook = await createTestWebhook(); const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { id: fakeId, source: 'saved_search', savedSearchId: savedSearch._id.toString(), @@ -580,7 +580,7 @@ describe('MCP Alert Tools', () => { const savedSearch = await createTestSavedSearch(); const webhook = await createTestWebhook(); - const result = await callTool(client, 'hyperdx_save_alert', { + const result = await callTool(client, 'clickstack_save_alert', { id: '!!!', source: 'saved_search', savedSearchId: savedSearch._id.toString(), @@ -599,9 +599,9 @@ describe('MCP Alert Tools', () => { }); }); - // ─── hyperdx_get_webhook ────────────────────────────────────────────────── + // ─── clickstack_get_webhook ────────────────────────────────────────────────── - describe('hyperdx_get_webhook', () => { + describe('clickstack_get_webhook', () => { it('should list all webhooks with slim fields', async () => { await Webhook.create({ team: team._id, @@ -616,7 +616,7 @@ describe('MCP Alert Tools', () => { url: 'https://example.com/hook2', }); - const result = await callTool(client, 'hyperdx_get_webhook', {}); + const result = await callTool(client, 'clickstack_get_webhook', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -635,7 +635,7 @@ describe('MCP Alert Tools', () => { }); it('should return empty array when no webhooks exist', async () => { - const result = await callTool(client, 'hyperdx_get_webhook', {}); + const result = await callTool(client, 'clickstack_get_webhook', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -656,7 +656,7 @@ describe('MCP Alert Tools', () => { }; const client2 = await createTestClient(otherTeamContext); - const listResult = await callTool(client2, 'hyperdx_get_webhook', {}); + const listResult = await callTool(client2, 'clickstack_get_webhook', {}); const output = JSON.parse(getFirstText(listResult)); expect(output).toHaveLength(0); diff --git a/packages/api/src/mcp/__tests__/dashboards.test.ts b/packages/api/src/mcp/__tests__/dashboards.test.ts index 6ed04e2ff9..5c63a326f8 100644 --- a/packages/api/src/mcp/__tests__/dashboards.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards.test.ts @@ -76,7 +76,7 @@ describe('MCP Dashboard Tools', () => { await server.stop(); }); - describe('hyperdx_get_dashboard', () => { + describe('clickstack_get_dashboard', () => { it('should list all dashboards when no id provided', async () => { await new Dashboard({ name: 'Dashboard 1', @@ -91,7 +91,7 @@ describe('MCP Dashboard Tools', () => { tags: ['tag2'], }).save(); - const result = await callTool(client, 'hyperdx_get_dashboard', {}); + const result = await callTool(client, 'clickstack_get_dashboard', {}); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -109,7 +109,7 @@ describe('MCP Dashboard Tools', () => { tags: ['test'], }).save(); - const result = await callTool(client, 'hyperdx_get_dashboard', { + const result = await callTool(client, 'clickstack_get_dashboard', { id: dashboard._id.toString(), }); @@ -123,7 +123,7 @@ describe('MCP Dashboard Tools', () => { it('should return error for non-existent dashboard id', async () => { const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_get_dashboard', { + const result = await callTool(client, 'clickstack_get_dashboard', { id: fakeId, }); @@ -132,10 +132,10 @@ describe('MCP Dashboard Tools', () => { }); }); - describe('hyperdx_save_dashboard', () => { + describe('clickstack_save_dashboard', () => { it('should create a new dashboard with tiles', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'New MCP Dashboard', tiles: [ { @@ -169,7 +169,7 @@ describe('MCP Dashboard Tools', () => { }); it('should create a dashboard with a markdown tile', async () => { - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Markdown Dashboard', tiles: [ { @@ -192,7 +192,7 @@ describe('MCP Dashboard Tools', () => { const sourceId = traceSource._id.toString(); // Create first - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Original Name', tiles: [ { @@ -208,7 +208,7 @@ describe('MCP Dashboard Tools', () => { const created = JSON.parse(getFirstText(createResult)); // Update - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'Updated Name', tiles: [ @@ -235,7 +235,7 @@ describe('MCP Dashboard Tools', () => { it('should return error for missing source ID', async () => { const fakeSourceId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Bad Dashboard', tiles: [ { @@ -255,7 +255,7 @@ describe('MCP Dashboard Tools', () => { it('should return error when updating non-existent dashboard', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { id: '000000000000000000000000', name: 'Ghost Dashboard', tiles: [ @@ -276,7 +276,7 @@ describe('MCP Dashboard Tools', () => { it('should create a dashboard with multiple tile types', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Multi-tile Dashboard', tiles: [ { @@ -354,7 +354,7 @@ describe('MCP Dashboard Tools', () => { it('should create a dashboard with a raw SQL tile', async () => { const connectionId = connection._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'SQL Dashboard', tiles: [ { @@ -376,7 +376,7 @@ describe('MCP Dashboard Tools', () => { it('should create a dashboard with a heatmap tile on a Trace source', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Heatmap Dashboard', tiles: [ { @@ -439,7 +439,7 @@ describe('MCP Dashboard Tools', () => { where: 'level:error AND service:checkout', }; - const saveResult = await callTool(client, 'hyperdx_save_dashboard', { + const saveResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Heatmap Full Round-Trip', tiles: [ { @@ -457,14 +457,14 @@ describe('MCP Dashboard Tools', () => { expect(saved.tiles).toHaveLength(1); expect(saved.tiles[0].config).toMatchObject(createConfig); - const getResult = await callTool(client, 'hyperdx_get_dashboard', { + const getResult = await callTool(client, 'clickstack_get_dashboard', { id: saved.id, }); expect(getResult.isError).toBeFalsy(); const fetched = JSON.parse(getFirstText(getResult)); expect(fetched.tiles[0].config).toMatchObject(createConfig); - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: saved.id, name: 'Heatmap Full Round-Trip', tiles: [ @@ -478,9 +478,13 @@ describe('MCP Dashboard Tools', () => { const updated = JSON.parse(getFirstText(updateResult)); expect(updated.tiles[0].config).toMatchObject(updatedConfig); - const getAfterUpdate = await callTool(client, 'hyperdx_get_dashboard', { - id: saved.id, - }); + const getAfterUpdate = await callTool( + client, + 'clickstack_get_dashboard', + { + id: saved.id, + }, + ); expect(getAfterUpdate.isError).toBeFalsy(); const refetched = JSON.parse(getFirstText(getAfterUpdate)); expect(refetched.tiles[0].config).toMatchObject(updatedConfig); @@ -488,7 +492,7 @@ describe('MCP Dashboard Tools', () => { it('should reject heatmap tile with empty valueExpression at the schema layer', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Bad Heatmap', tiles: [ { @@ -514,7 +518,7 @@ describe('MCP Dashboard Tools', () => { async numberFormat => { const sourceId = traceSource._id.toString(); - const saveResult = await callTool(client, 'hyperdx_save_dashboard', { + const saveResult = await callTool(client, 'clickstack_save_dashboard', { name: `NumberFormat ${numberFormat.output}`, tiles: [ { @@ -547,7 +551,7 @@ describe('MCP Dashboard Tools', () => { expect(saveResult.isError).toBeFalsy(); const saved = JSON.parse(getFirstText(saveResult)); - const getResult = await callTool(client, 'hyperdx_get_dashboard', { + const getResult = await callTool(client, 'clickstack_get_dashboard', { id: saved.id, }); expect(getResult.isError).toBeFalsy(); @@ -567,7 +571,7 @@ describe('MCP Dashboard Tools', () => { it('should reject numberFormat with an unknown output value', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Bad NumberFormat', tiles: [ { @@ -602,7 +606,7 @@ describe('MCP Dashboard Tools', () => { name: 'Logs', }); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Heatmap on Log Source', tiles: [ { @@ -634,7 +638,7 @@ describe('MCP Dashboard Tools', () => { name: 'Logs', }); - const created = await callTool(client, 'hyperdx_save_dashboard', { + const created = await callTool(client, 'clickstack_save_dashboard', { name: 'Line on Log Source', tiles: [ { @@ -650,7 +654,7 @@ describe('MCP Dashboard Tools', () => { expect(created.isError).toBeFalsy(); const saved = JSON.parse(getFirstText(created)); - const update = await callTool(client, 'hyperdx_save_dashboard', { + const update = await callTool(client, 'clickstack_save_dashboard', { id: saved.id, name: 'Line on Log Source', tiles: [ @@ -682,7 +686,7 @@ describe('MCP Dashboard Tools', () => { name: 'Logs', }); - const created = await callTool(client, 'hyperdx_save_dashboard', { + const created = await callTool(client, 'clickstack_save_dashboard', { name: 'Heatmap re-pointed at Log', tiles: [ { @@ -698,7 +702,7 @@ describe('MCP Dashboard Tools', () => { expect(created.isError).toBeFalsy(); const saved = JSON.parse(getFirstText(created)); - const update = await callTool(client, 'hyperdx_save_dashboard', { + const update = await callTool(client, 'clickstack_save_dashboard', { id: saved.id, name: 'Heatmap re-pointed at Log', tiles: [ @@ -742,7 +746,7 @@ describe('MCP Dashboard Tools', () => { numberFormat: { output: 'number' as const, average: true }, }; - const save = await callTool(client, 'hyperdx_save_dashboard', { + const save = await callTool(client, 'clickstack_save_dashboard', { name: 'Mixed Tile Round-Trip', tiles: [ { name: 'Heatmap Tile', config: heatmapConfig }, @@ -764,7 +768,7 @@ describe('MCP Dashboard Tools', () => { const fetched = JSON.parse( getFirstText( - await callTool(client, 'hyperdx_get_dashboard', { id: saved.id }), + await callTool(client, 'clickstack_get_dashboard', { id: saved.id }), ), ); const fetchedByName: Record = @@ -777,7 +781,7 @@ describe('MCP Dashboard Tools', () => { }); }); - describe('hyperdx_save_dashboard - containers and tabs', () => { + describe('clickstack_save_dashboard - containers and tabs', () => { // Mirrors the v2 external-API "Containers and tabs" describe block so // the MCP path enforces the same 5 cross-field rules: container id // unique, tab id unique within a container, tile.containerId @@ -821,7 +825,7 @@ describe('MCP Dashboard Tools', () => { }, ]; - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Containers MCP Round-Trip', tiles: [ buildTile(sourceId, { @@ -884,7 +888,7 @@ describe('MCP Dashboard Tools', () => { expect(createdTilesByName.Ungrouped.tabId).toBeUndefined(); // Verify a fresh GET via the get tool also returns the structure. - const getResult = await callTool(client, 'hyperdx_get_dashboard', { + const getResult = await callTool(client, 'clickstack_get_dashboard', { id: created.id, }); const fetched = JSON.parse(getFirstText(getResult)); @@ -913,7 +917,7 @@ describe('MCP Dashboard Tools', () => { tabId: undefined, }; - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'Containers MCP Round-Trip', tiles: [ @@ -949,7 +953,7 @@ describe('MCP Dashboard Tools', () => { it('should reject duplicate container ids', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Duplicate Container Ids', tiles: [buildTile(sourceId, {})], containers: [ @@ -968,7 +972,7 @@ describe('MCP Dashboard Tools', () => { // array. Cross-tile-ref rules (covered below) fire through // collectTileContainerRefIssues and produce plain prose. const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Duplicate Tab Ids', tiles: [buildTile(sourceId, {})], containers: [ @@ -992,7 +996,7 @@ describe('MCP Dashboard Tools', () => { it('should reject a tile that supplies tabId without containerId', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Tab Without Container', tiles: [buildTile(sourceId, { tabId: 'errors' })], containers: [ @@ -1013,7 +1017,7 @@ describe('MCP Dashboard Tools', () => { it('should reject a tile that references an unknown containerId', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Unknown Container', tiles: [buildTile(sourceId, { containerId: 'does-not-exist' })], containers: [{ id: 'real', title: 'Real', collapsed: false }], @@ -1027,7 +1031,7 @@ describe('MCP Dashboard Tools', () => { it('should reject a tile that references an unknown tabId within a container', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Unknown Tab', tiles: [ buildTile(sourceId, { @@ -1063,7 +1067,7 @@ describe('MCP Dashboard Tools', () => { tabs: [{ id: 'errors', title: 'Errors' }], }, ]; - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'PUT-without-containers fallback', tiles: [ buildTile(sourceId, { @@ -1077,7 +1081,7 @@ describe('MCP Dashboard Tools', () => { expect(createResult.isError).toBeFalsy(); const created = JSON.parse(getFirstText(createResult)); - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'PUT-without-containers fallback', tiles: created.tiles, @@ -1097,7 +1101,7 @@ describe('MCP Dashboard Tools', () => { // persisted containers, and the response normalizes [] back to // absent on read. const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Wipe containers', tiles: [buildTile(sourceId, { name: 'Tile' })], containers: [{ id: 'overview', title: 'Overview', collapsed: false }], @@ -1111,7 +1115,7 @@ describe('MCP Dashboard Tools', () => { containerId: undefined, tabId: undefined, }; - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'Wipe containers', tiles: [wipedTile], @@ -1121,7 +1125,7 @@ describe('MCP Dashboard Tools', () => { const updated = JSON.parse(getFirstText(updateResult)); expect(updated.containers).toBeUndefined(); - const getResult = await callTool(client, 'hyperdx_get_dashboard', { + const getResult = await callTool(client, 'clickstack_get_dashboard', { id: created.id, }); const fetched = JSON.parse(getFirstText(getResult)); @@ -1135,7 +1139,7 @@ describe('MCP Dashboard Tools', () => { it('should accept a 256-char tile.containerId and reject 257', async () => { const sourceId = traceSource._id.toString(); const idAtMax = 'c'.repeat(256); - const okResult = await callTool(client, 'hyperdx_save_dashboard', { + const okResult = await callTool(client, 'clickstack_save_dashboard', { name: '256-char containerId', tiles: [buildTile(sourceId, { containerId: idAtMax })], containers: [{ id: idAtMax, title: 'Max', collapsed: false }], @@ -1143,11 +1147,15 @@ describe('MCP Dashboard Tools', () => { expect(okResult.isError).toBeFalsy(); const idTooLong = 'c'.repeat(257); - const tooLongResult = await callTool(client, 'hyperdx_save_dashboard', { - name: '257-char containerId', - tiles: [buildTile(sourceId, { containerId: idTooLong })], - containers: [{ id: idTooLong, title: 'Too long', collapsed: false }], - }); + const tooLongResult = await callTool( + client, + 'clickstack_save_dashboard', + { + name: '257-char containerId', + tiles: [buildTile(sourceId, { containerId: idTooLong })], + containers: [{ id: idTooLong, title: 'Too long', collapsed: false }], + }, + ); expect(tooLongResult.isError).toBe(true); expect(getFirstText(tooLongResult)).toContain( 'String must contain at most 256 character(s)', @@ -1156,7 +1164,7 @@ describe('MCP Dashboard Tools', () => { }); }); - describe('hyperdx_save_dashboard - dashboard filters', () => { + describe('clickstack_save_dashboard - dashboard filters', () => { // The MCP input schema delegates to createDashboardBodySchema / // updateDashboardBodySchema for the filter shape. These tests guard // that the MCP path lights up the same filter round-trip the v2 REST @@ -1179,7 +1187,7 @@ describe('MCP Dashboard Tools', () => { it('should round-trip filters on create, get, and update', async () => { const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Service Detail (MCP filter round-trip)', tiles: [traceTile(sourceId)], filters: [ @@ -1227,7 +1235,7 @@ describe('MCP Dashboard Tools', () => { }); // Fetch and assert the same shape. - const getResult = await callTool(client, 'hyperdx_get_dashboard', { + const getResult = await callTool(client, 'clickstack_get_dashboard', { id: created.id, }); const fetched = JSON.parse(getFirstText(getResult)); @@ -1235,7 +1243,7 @@ describe('MCP Dashboard Tools', () => { // Update: rename the first filter and drop the second. The first // filter keeps its id, the second is dropped (not preserved). - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'Service Detail (MCP filter round-trip)', tiles: [traceTile(sourceId)], @@ -1263,15 +1271,15 @@ describe('MCP Dashboard Tools', () => { }); it('should accept a create payload with stray filter ids (copy-paste round-trip)', async () => { - // An LLM that copies a filter out of hyperdx_get_dashboard and - // back into hyperdx_save_dashboard for a NEW dashboard ships an + // An LLM that copies a filter out of clickstack_get_dashboard and + // back into clickstack_save_dashboard for a NEW dashboard ships an // `id` on the filter. The body schema for create is // `.strict()` and rejects `id`, so saveDashboard normalizes by // stripping ids before validation. Without that normalization // this payload would 4xx with a confusing strict-validation // error. const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Create with stray filter id', tiles: [traceTile(sourceId)], filters: [ @@ -1303,7 +1311,7 @@ describe('MCP Dashboard Tools', () => { // id, so saveDashboard fills one in. Without that normalization // this payload would 4xx complaining that id is missing. const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Update adds new filter', tiles: [traceTile(sourceId)], filters: [ @@ -1318,7 +1326,7 @@ describe('MCP Dashboard Tools', () => { const created = JSON.parse(getFirstText(createResult)); const existingFilterId = created.filters[0].id; - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'Update adds new filter', tiles: [traceTile(sourceId)], @@ -1360,7 +1368,7 @@ describe('MCP Dashboard Tools', () => { // saved tile would include empty-message rows instead of the // filtered set the prompt promises. const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'Top Error Messages', tiles: [ { @@ -1387,14 +1395,14 @@ describe('MCP Dashboard Tools', () => { }); }); - describe('hyperdx_save_dashboard - table onClick linking', () => { + describe('clickstack_save_dashboard - table onClick linking', () => { // Mirrors the v2 external-API onClick tests so the MCP path // enforces the same drill-down rules: row-click can target /search // for a log/trace source or another dashboard, by concrete ID or // by templated name, with optional whereTemplate/filters. it('should round-trip a table tile with a search onClick by ID', async () => { const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick search by id', tiles: [ { @@ -1440,7 +1448,7 @@ describe('MCP Dashboard Tools', () => { ], }); - const getResult = await callTool(client, 'hyperdx_get_dashboard', { + const getResult = await callTool(client, 'clickstack_get_dashboard', { id: created.id, }); const fetched = JSON.parse(getFirstText(getResult)); @@ -1459,7 +1467,7 @@ describe('MCP Dashboard Tools', () => { team: team._id, }).save(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick dashboard by id', tiles: [ { @@ -1511,7 +1519,7 @@ describe('MCP Dashboard Tools', () => { it('should round-trip a templated dashboard onClick (mode=template)', async () => { const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick dashboard by template', tiles: [ { @@ -1553,7 +1561,7 @@ describe('MCP Dashboard Tools', () => { it('should round-trip onClick on a raw SQL table tile', async () => { const connectionId = connection._id.toString(); const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'SQL onClick', tiles: [ { @@ -1604,7 +1612,7 @@ describe('MCP Dashboard Tools', () => { it('should reject a table tile onClick referencing a non-existent dashboard', async () => { const sourceId = traceSource._id.toString(); const ghostDashboardId = new mongoose.Types.ObjectId().toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick missing dashboard', tiles: [ { @@ -1652,7 +1660,7 @@ describe('MCP Dashboard Tools', () => { }); const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick metric source', tiles: [ { @@ -1685,7 +1693,7 @@ describe('MCP Dashboard Tools', () => { // input schema and surfaces as the SDK's "Input validation error" // envelope rather than a custom 400 message. const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick bad object id', tiles: [ { @@ -1713,7 +1721,7 @@ describe('MCP Dashboard Tools', () => { // future tightening of the schema doesn't quietly change it // without updating the docs that callers rely on. const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick no whereLanguage', tiles: [ { @@ -1744,7 +1752,7 @@ describe('MCP Dashboard Tools', () => { // The schema is a discriminated union on `type`; a bogus type // must be rejected up front. const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'hyperdx_save_dashboard', { + const result = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick bad type', tiles: [ { @@ -1772,7 +1780,7 @@ describe('MCP Dashboard Tools', () => { // mirrors POST: missing dashboards are rejected with the same // message. const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'OnClick update validation', tiles: [ { @@ -1789,7 +1797,7 @@ describe('MCP Dashboard Tools', () => { const created = JSON.parse(getFirstText(createResult)); const ghostDashboardId = new mongoose.Types.ObjectId().toString(); - const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + const updateResult = await callTool(client, 'clickstack_save_dashboard', { id: created.id, name: 'OnClick update validation', tiles: [ @@ -1814,7 +1822,7 @@ describe('MCP Dashboard Tools', () => { }); }); - describe('hyperdx_delete_dashboard', () => { + describe('clickstack_delete_dashboard', () => { it('should delete an existing dashboard', async () => { const dashboard = await new Dashboard({ name: 'To Delete', @@ -1822,7 +1830,7 @@ describe('MCP Dashboard Tools', () => { team: team._id, }).save(); - const result = await callTool(client, 'hyperdx_delete_dashboard', { + const result = await callTool(client, 'clickstack_delete_dashboard', { id: dashboard._id.toString(), }); @@ -1837,7 +1845,7 @@ describe('MCP Dashboard Tools', () => { }); it('should return error for non-existent dashboard', async () => { - const result = await callTool(client, 'hyperdx_delete_dashboard', { + const result = await callTool(client, 'clickstack_delete_dashboard', { id: '000000000000000000000000', }); @@ -1846,9 +1854,9 @@ describe('MCP Dashboard Tools', () => { }); }); - describe('hyperdx_query_tile', () => { + describe('clickstack_query_tile', () => { it('should return error for non-existent dashboard', async () => { - const result = await callTool(client, 'hyperdx_query_tile', { + const result = await callTool(client, 'clickstack_query_tile', { dashboardId: '000000000000000000000000', tileId: 'some-tile-id', }); @@ -1859,7 +1867,7 @@ describe('MCP Dashboard Tools', () => { it('should return error for non-existent tile', async () => { const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Tile Query Test', tiles: [ { @@ -1874,7 +1882,7 @@ describe('MCP Dashboard Tools', () => { }); const dashboard = JSON.parse(getFirstText(createResult)); - const result = await callTool(client, 'hyperdx_query_tile', { + const result = await callTool(client, 'clickstack_query_tile', { dashboardId: dashboard.id, tileId: 'non-existent-tile-id', }); @@ -1885,7 +1893,7 @@ describe('MCP Dashboard Tools', () => { it('should return error for invalid time range', async () => { const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Time Range Test', tiles: [ { @@ -1900,7 +1908,7 @@ describe('MCP Dashboard Tools', () => { }); const dashboard = JSON.parse(getFirstText(createResult)); - const result = await callTool(client, 'hyperdx_query_tile', { + const result = await callTool(client, 'clickstack_query_tile', { dashboardId: dashboard.id, tileId: dashboard.tiles[0].id, startTime: 'not-a-date', @@ -1912,7 +1920,7 @@ describe('MCP Dashboard Tools', () => { it('should execute query for a valid tile', async () => { const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'hyperdx_save_dashboard', { + const createResult = await callTool(client, 'clickstack_save_dashboard', { name: 'Query Tile Test', tiles: [ { @@ -1927,7 +1935,7 @@ describe('MCP Dashboard Tools', () => { }); const dashboard = JSON.parse(getFirstText(createResult)); - const result = await callTool(client, 'hyperdx_query_tile', { + const result = await callTool(client, 'clickstack_query_tile', { dashboardId: dashboard.id, tileId: dashboard.tiles[0].id, startTime: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), @@ -1945,7 +1953,7 @@ describe('MCP Dashboard Tools', () => { const prompt = buildQueryGuidePrompt(); // Ensure each section the bot called out has a heatmap entry so // the LLM cannot skip the constraints when producing a heatmap - // tile through hyperdx_save_dashboard. + // tile through clickstack_save_dashboard. const sections = [ '== AGGREGATION FUNCTIONS (aggFn) ==', '== PER-TILE TYPE CONSTRAINTS ==', @@ -2156,7 +2164,7 @@ describe('MCP Dashboard Tools', () => { const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); const section = prompt.slice(mistakesIdx); expect(section).toMatch( - /on EVERY tile after hyperdx_save_dashboard, not just one/, + /on EVERY tile after clickstack_save_dashboard, not just one/, ); }); @@ -2295,8 +2303,8 @@ describe('MCP Dashboard Tools', () => { for (let i = 1; i <= 6; i++) { expect(wf).toMatch(new RegExp(`^${i}\\. `, 'm')); } - // Step 2 must point at hyperdx_get_dashboard with no id. - expect(wf).toMatch(/hyperdx_get_dashboard \(no id\)/); + // Step 2 must point at clickstack_get_dashboard with no id. + expect(wf).toMatch(/clickstack_get_dashboard \(no id\)/); // Step 4 must call out grouping tiles into containers before save. expect(wf).toMatch(/group them into 2-4 containers/); // Step 6 must call out EVERY tile, not just one. diff --git a/packages/api/src/mcp/__tests__/deltasTool.test.ts b/packages/api/src/mcp/__tests__/deltasTool.test.ts index 30c6868608..1d9beb5d87 100644 --- a/packages/api/src/mcp/__tests__/deltasTool.test.ts +++ b/packages/api/src/mcp/__tests__/deltasTool.test.ts @@ -71,9 +71,9 @@ describe('MCP Event Deltas Tool', () => { }); describe('schema serialization', () => { - it('exposes hyperdx_event_deltas via tools/list with target + baseline groups', async () => { + it('exposes clickstack_event_deltas via tools/list with target + baseline groups', async () => { const { tools } = await client.listTools(); - const t = tools.find(t => t.name === 'hyperdx_event_deltas'); + const t = tools.find(t => t.name === 'clickstack_event_deltas'); expect(t).toBeDefined(); const schema = t!.inputSchema; const props = Object.keys(schema.properties ?? {}); @@ -90,7 +90,7 @@ describe('MCP Event Deltas Tool', () => { describe('validation', () => { it('rejects target where endTime <= startTime', async () => { - const result = await callTool(client, 'hyperdx_event_deltas', { + const result = await callTool(client, 'clickstack_event_deltas', { sourceId: logSource._id.toString(), target: { startTime: '2026-05-10T01:00:00Z', @@ -147,7 +147,7 @@ describe('MCP Event Deltas Tool', () => { }); it('ranks SeverityText and ServiceName as top differentiating properties', async () => { - const result = await callTool(client, 'hyperdx_event_deltas', { + const result = await callTool(client, 'clickstack_event_deltas', { sourceId: logSource._id.toString(), target: { startTime: new Date(now - 5 * 60_000).toISOString(), @@ -208,7 +208,7 @@ describe('MCP Event Deltas Tool', () => { }); it('returns includeHidden:true with a hidden array containing high-cardinality / id fields', async () => { - const result = await callTool(client, 'hyperdx_event_deltas', { + const result = await callTool(client, 'clickstack_event_deltas', { sourceId: logSource._id.toString(), target: { startTime: new Date(now - 5 * 60_000).toISOString(), diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index d51a6f92a0..b5264580b5 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -87,9 +87,9 @@ describe('MCP Query Tools', () => { // ─── Schema serialization ──────────────────────────────────────────────────── describe('schema serialization', () => { - it('should expose hyperdx_timeseries with expected properties', async () => { + it('should expose clickstack_timeseries with expected properties', async () => { const { tools } = await client.listTools(); - const tool = tools.find(t => t.name === 'hyperdx_timeseries'); + const tool = tools.find(t => t.name === 'clickstack_timeseries'); expect(tool).toBeDefined(); const props = Object.keys(tool!.inputSchema.properties ?? {}); @@ -102,9 +102,9 @@ describe('MCP Query Tools', () => { expect(tool!.inputSchema.required).toContain('select'); }); - it('should expose hyperdx_table with expected properties', async () => { + it('should expose clickstack_table with expected properties', async () => { const { tools } = await client.listTools(); - const tool = tools.find(t => t.name === 'hyperdx_table'); + const tool = tools.find(t => t.name === 'clickstack_table'); expect(tool).toBeDefined(); const props = Object.keys(tool!.inputSchema.properties ?? {}); @@ -116,9 +116,9 @@ describe('MCP Query Tools', () => { expect(tool!.inputSchema.required).toContain('select'); }); - it('should expose hyperdx_search with expected properties', async () => { + it('should expose clickstack_search with expected properties', async () => { const { tools } = await client.listTools(); - const tool = tools.find(t => t.name === 'hyperdx_search'); + const tool = tools.find(t => t.name === 'clickstack_search'); expect(tool).toBeDefined(); const props = Object.keys(tool!.inputSchema.properties ?? {}); @@ -129,9 +129,9 @@ describe('MCP Query Tools', () => { expect(tool!.inputSchema.required).toContain('sourceId'); }); - it('should expose hyperdx_event_patterns with expected properties', async () => { + it('should expose clickstack_event_patterns with expected properties', async () => { const { tools } = await client.listTools(); - const tool = tools.find(t => t.name === 'hyperdx_event_patterns'); + const tool = tools.find(t => t.name === 'clickstack_event_patterns'); expect(tool).toBeDefined(); const props = Object.keys(tool!.inputSchema.properties ?? {}); @@ -143,9 +143,9 @@ describe('MCP Query Tools', () => { expect(tool!.inputSchema.required).toContain('sourceId'); }); - it('should expose hyperdx_sql with expected properties', async () => { + it('should expose clickstack_sql with expected properties', async () => { const { tools } = await client.listTools(); - const tool = tools.find(t => t.name === 'hyperdx_sql'); + const tool = tools.find(t => t.name === 'clickstack_sql'); expect(tool).toBeDefined(); const props = Object.keys(tool!.inputSchema.properties ?? {}); @@ -158,11 +158,11 @@ describe('MCP Query Tools', () => { }); }); - // ─── hyperdx_timeseries ───────────────────────────────────────────────────── + // ─── clickstack_timeseries ───────────────────────────────────────────────────── - describe('hyperdx_timeseries', () => { + describe('clickstack_timeseries', () => { it('should execute a line chart query', async () => { - const result = await callTool(client, 'hyperdx_timeseries', { + const result = await callTool(client, 'clickstack_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -176,7 +176,7 @@ describe('MCP Query Tools', () => { }); it('should execute a stacked bar chart query', async () => { - const result = await callTool(client, 'hyperdx_timeseries', { + const result = await callTool(client, 'clickstack_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], shape: 'stacked_bar', @@ -189,7 +189,7 @@ describe('MCP Query Tools', () => { }); it('should default to line shape when shape is omitted', async () => { - const result = await callTool(client, 'hyperdx_timeseries', { + const result = await callTool(client, 'clickstack_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], }); @@ -199,7 +199,7 @@ describe('MCP Query Tools', () => { }); it('should accept granularity in correct format', async () => { - const result = await callTool(client, 'hyperdx_timeseries', { + const result = await callTool(client, 'clickstack_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], granularity: '1 minute', @@ -212,7 +212,7 @@ describe('MCP Query Tools', () => { }); it('should return error for invalid time range', async () => { - const result = await callTool(client, 'hyperdx_timeseries', { + const result = await callTool(client, 'clickstack_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], startTime: 'invalid-date', @@ -223,11 +223,11 @@ describe('MCP Query Tools', () => { }); }); - // ─── hyperdx_table ────────────────────────────────────────────────────────── + // ─── clickstack_table ────────────────────────────────────────────────────────── - describe('hyperdx_table', () => { + describe('clickstack_table', () => { it('should execute a table query', async () => { - const result = await callTool(client, 'hyperdx_table', { + const result = await callTool(client, 'clickstack_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], groupBy: 'SpanName', @@ -242,7 +242,7 @@ describe('MCP Query Tools', () => { }); it('should execute a number query', async () => { - const result = await callTool(client, 'hyperdx_table', { + const result = await callTool(client, 'clickstack_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], shape: 'number', @@ -255,7 +255,7 @@ describe('MCP Query Tools', () => { }); it('should execute a pie query', async () => { - const result = await callTool(client, 'hyperdx_table', { + const result = await callTool(client, 'clickstack_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], shape: 'pie', @@ -270,7 +270,7 @@ describe('MCP Query Tools', () => { it('should auto-upgrade shape:"number" to "table" when select has multiple items', async () => { // This should NOT error — it should silently upgrade to table - const result = await callTool(client, 'hyperdx_table', { + const result = await callTool(client, 'clickstack_table', { sourceId: traceSource._id.toString(), select: [ { aggFn: 'count' }, @@ -286,7 +286,7 @@ describe('MCP Query Tools', () => { }); it('should auto-upgrade shape:"pie" to "table" when select has multiple items', async () => { - const result = await callTool(client, 'hyperdx_table', { + const result = await callTool(client, 'clickstack_table', { sourceId: traceSource._id.toString(), select: [ { aggFn: 'count' }, @@ -303,7 +303,7 @@ describe('MCP Query Tools', () => { }); it('should use default time range when not provided', async () => { - const result = await callTool(client, 'hyperdx_table', { + const result = await callTool(client, 'clickstack_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], }); @@ -313,11 +313,11 @@ describe('MCP Query Tools', () => { }); }); - // ─── hyperdx_search ───────────────────────────────────────────────────────── + // ─── clickstack_search ───────────────────────────────────────────────────────── - describe('hyperdx_search', () => { + describe('clickstack_search', () => { it('should execute a search query', async () => { - const result = await callTool(client, 'hyperdx_search', { + const result = await callTool(client, 'clickstack_search', { sourceId: traceSource._id.toString(), where: '', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -329,7 +329,7 @@ describe('MCP Query Tools', () => { }); it('should respect maxResults parameter', async () => { - const result = await callTool(client, 'hyperdx_search', { + const result = await callTool(client, 'clickstack_search', { sourceId: traceSource._id.toString(), maxResults: 10, startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -340,7 +340,7 @@ describe('MCP Query Tools', () => { }); it('should use default time range when not provided', async () => { - const result = await callTool(client, 'hyperdx_search', { + const result = await callTool(client, 'clickstack_search', { sourceId: traceSource._id.toString(), }); @@ -349,7 +349,7 @@ describe('MCP Query Tools', () => { }); it('should reject calls missing sourceId', async () => { - const result = await callTool(client, 'hyperdx_search', { + const result = await callTool(client, 'clickstack_search', { where: 'level:error', }); @@ -358,11 +358,11 @@ describe('MCP Query Tools', () => { }); }); - // ─── hyperdx_event_patterns ───────────────────────────────────────────────── + // ─── clickstack_event_patterns ───────────────────────────────────────────────── - describe('hyperdx_event_patterns', () => { + describe('clickstack_event_patterns', () => { it('should execute an event_patterns query on a log source', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: logSource._id.toString(), startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), @@ -382,7 +382,7 @@ describe('MCP Query Tools', () => { }); it('should execute with explicit bodyExpression on trace source', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: traceSource._id.toString(), bodyExpression: 'SpanName', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -398,7 +398,7 @@ describe('MCP Query Tools', () => { }); it('should accept custom bodyExpression on log source', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: logSource._id.toString(), bodyExpression: 'SeverityText', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -412,7 +412,7 @@ describe('MCP Query Tools', () => { }); it('should respect sampleSize parameter', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: logSource._id.toString(), sampleSize: 100, startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -424,7 +424,7 @@ describe('MCP Query Tools', () => { }); it('should respect where filter', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: logSource._id.toString(), where: "SeverityText = 'ERROR'", whereLanguage: 'sql', @@ -437,7 +437,7 @@ describe('MCP Query Tools', () => { }); it('should reject calls missing sourceId', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -447,7 +447,7 @@ describe('MCP Query Tools', () => { }); it('should return error for non-existent source', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: '000000000000000000000000', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), @@ -496,7 +496,7 @@ describe('MCP Query Tools', () => { }); it('should mine patterns from seeded data and return non-empty results', async () => { - const result = await callTool(client, 'hyperdx_event_patterns', { + const result = await callTool(client, 'clickstack_event_patterns', { sourceId: logSource._id.toString(), startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), endTime: new Date(now.getTime() + 60 * 1000).toISOString(), @@ -535,11 +535,11 @@ describe('MCP Query Tools', () => { }); }); - // ─── hyperdx_sql ──────────────────────────────────────────────────────────── + // ─── clickstack_sql ──────────────────────────────────────────────────────────── - describe('hyperdx_sql', () => { + describe('clickstack_sql', () => { it('should execute a raw SQL query', async () => { - const result = await callTool(client, 'hyperdx_sql', { + const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), sql: 'SELECT 1 AS value', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -551,7 +551,7 @@ describe('MCP Query Tools', () => { }); it('should execute SQL with time macros', async () => { - const result = await callTool(client, 'hyperdx_sql', { + const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), sql: `SELECT count() AS cnt FROM ${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE} WHERE $__timeFilter(Timestamp) LIMIT 10`, startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -563,7 +563,7 @@ describe('MCP Query Tools', () => { }); it('should use default time range when not provided', async () => { - const result = await callTool(client, 'hyperdx_sql', { + const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), sql: 'SELECT 1 AS value', }); @@ -573,7 +573,7 @@ describe('MCP Query Tools', () => { }); it('should return error for invalid time range', async () => { - const result = await callTool(client, 'hyperdx_sql', { + const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), sql: 'SELECT 1', startTime: 'not-a-date', diff --git a/packages/api/src/mcp/__tests__/savedSearches.test.ts b/packages/api/src/mcp/__tests__/savedSearches.test.ts index b63b752d95..81756fb156 100644 --- a/packages/api/src/mcp/__tests__/savedSearches.test.ts +++ b/packages/api/src/mcp/__tests__/savedSearches.test.ts @@ -87,15 +87,19 @@ describe('MCP Saved Search Tools', () => { }); } - // ─── hyperdx_get_saved_search ───────────────────────────────────────────── + // ─── clickstack_get_saved_search ───────────────────────────────────────────── - describe('hyperdx_get_saved_search', () => { + describe('clickstack_get_saved_search', () => { describe('list (no id)', () => { it('should list all saved searches with slim summary fields', async () => { await createTestSavedSearch({ name: 'Search 1' }); await createTestSavedSearch({ name: 'Search 2' }); - const result = await callTool(client, 'hyperdx_get_saved_search', {}); + const result = await callTool( + client, + 'clickstack_get_saved_search', + {}, + ); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -115,7 +119,11 @@ describe('MCP Saved Search Tools', () => { }); it('should return empty array when no saved searches exist', async () => { - const result = await callTool(client, 'hyperdx_get_saved_search', {}); + const result = await callTool( + client, + 'clickstack_get_saved_search', + {}, + ); expect(result.isError).toBeFalsy(); const output = JSON.parse(getFirstText(result)); @@ -133,7 +141,7 @@ describe('MCP Saved Search Tools', () => { const listResult = await callTool( client2, - 'hyperdx_get_saved_search', + 'clickstack_get_saved_search', {}, ); const output = JSON.parse(getFirstText(listResult)); @@ -150,7 +158,7 @@ describe('MCP Saved Search Tools', () => { where: 'level:error', }); - const result = await callTool(client, 'hyperdx_get_saved_search', { + const result = await callTool(client, 'clickstack_get_saved_search', { id: savedSearch._id.toString(), }); @@ -170,7 +178,7 @@ describe('MCP Saved Search Tools', () => { }); it('should return error for invalid ObjectId format', async () => { - const result = await callTool(client, 'hyperdx_get_saved_search', { + const result = await callTool(client, 'clickstack_get_saved_search', { id: 'not-a-valid-id', }); @@ -180,7 +188,7 @@ describe('MCP Saved Search Tools', () => { it('should return error for non-existent saved search id', async () => { const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_get_saved_search', { + const result = await callTool(client, 'clickstack_get_saved_search', { id: fakeId, }); @@ -190,12 +198,12 @@ describe('MCP Saved Search Tools', () => { }); }); - // ─── hyperdx_save_saved_search ──────────────────────────────────────────── + // ─── clickstack_save_saved_search ──────────────────────────────────────────── - describe('hyperdx_save_saved_search', () => { + describe('clickstack_save_saved_search', () => { describe('create', () => { it('should create a new saved search', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'Error Traces', sourceId: traceSource._id.toString(), where: 'StatusCode:Error', @@ -220,7 +228,7 @@ describe('MCP Saved Search Tools', () => { }); it('should create a saved search with minimal fields', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'Minimal Search', sourceId: traceSource._id.toString(), }); @@ -234,7 +242,7 @@ describe('MCP Saved Search Tools', () => { }); it('should create a saved search with SQL where language', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'SQL Search', sourceId: traceSource._id.toString(), where: "StatusCode = 'Error'", @@ -248,7 +256,7 @@ describe('MCP Saved Search Tools', () => { }); it('should create a saved search with select and orderBy', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'Full Search', sourceId: traceSource._id.toString(), select: 'body,service.name,duration', @@ -263,7 +271,7 @@ describe('MCP Saved Search Tools', () => { }); it('should create a saved search with filters', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'Filtered Search', sourceId: traceSource._id.toString(), filters: [ @@ -285,7 +293,7 @@ describe('MCP Saved Search Tools', () => { }); it('should reject invalid sourceId', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'Bad Source', sourceId: 'not-a-valid-id', }); @@ -295,7 +303,7 @@ describe('MCP Saved Search Tools', () => { }); it('should include url in response when FRONTEND_URL is set', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { name: 'URL Test', sourceId: traceSource._id.toString(), }); @@ -314,7 +322,7 @@ describe('MCP Saved Search Tools', () => { name: 'Original Name', }); - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { id: savedSearch._id.toString(), name: 'Updated Name', sourceId: traceSource._id.toString(), @@ -338,7 +346,7 @@ describe('MCP Saved Search Tools', () => { it('should return error for non-existent saved search on update', async () => { const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { id: fakeId, name: 'Ghost Search', sourceId: traceSource._id.toString(), @@ -349,7 +357,7 @@ describe('MCP Saved Search Tools', () => { }); it('should return error for invalid ObjectId format on update', async () => { - const result = await callTool(client, 'hyperdx_save_saved_search', { + const result = await callTool(client, 'clickstack_save_saved_search', { id: '!!!', name: 'Bad ID', sourceId: traceSource._id.toString(), diff --git a/packages/api/src/mcp/__tests__/sources.test.ts b/packages/api/src/mcp/__tests__/sources.test.ts index aea5ead6d7..008e4fe852 100644 --- a/packages/api/src/mcp/__tests__/sources.test.ts +++ b/packages/api/src/mcp/__tests__/sources.test.ts @@ -96,11 +96,11 @@ describe('MCP Source Tools', () => { await server.stop(); }); - // ── hyperdx_list_sources ────────────────────────────────────────────────── + // ── clickstack_list_sources ────────────────────────────────────────────────── - describe('hyperdx_list_sources', () => { + describe('clickstack_list_sources', () => { it('should list sources and connections as a lightweight catalog', async () => { - const result = await callTool(client, 'hyperdx_list_sources'); + const result = await callTool(client, 'clickstack_list_sources'); expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); @@ -129,11 +129,11 @@ describe('MCP Source Tools', () => { // Should include nextStep guidance pointing to describe_source expect(output.nextStep).toBeDefined(); - expect(output.nextStep).toContain('hyperdx_describe_source'); + expect(output.nextStep).toContain('clickstack_describe_source'); }); it('should include keyColumns from source config without ClickHouse queries', async () => { - const result = await callTool(client, 'hyperdx_list_sources'); + const result = await callTool(client, 'clickstack_list_sources'); const output = JSON.parse(getFirstText(result)); const trace = output.sources.find( @@ -159,7 +159,7 @@ describe('MCP Source Tools', () => { }; const client2 = await createTestClient(context2); - const result = await callTool(client2, 'hyperdx_list_sources'); + const result = await callTool(client2, 'clickstack_list_sources'); const output = JSON.parse(getFirstText(result)); expect(output.sources).toHaveLength(0); @@ -169,11 +169,11 @@ describe('MCP Source Tools', () => { }); }); - // ── hyperdx_describe_source ─────────────────────────────────────────────── + // ── clickstack_describe_source ─────────────────────────────────────────────── - describe('hyperdx_describe_source', () => { + describe('clickstack_describe_source', () => { it('should return full column schema for a trace source', async () => { - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: traceSource._id.toString(), }); @@ -197,7 +197,7 @@ describe('MCP Source Tools', () => { }); it('should return full column schema for a log source', async () => { - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: logSource._id.toString(), }); @@ -223,7 +223,7 @@ describe('MCP Source Tools', () => { }); it('should include map attribute keys', async () => { - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: traceSource._id.toString(), }); const output = JSON.parse(getFirstText(result)); @@ -237,7 +237,7 @@ describe('MCP Source Tools', () => { }); it('should include keyColumns for trace source', async () => { - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: traceSource._id.toString(), }); const output = JSON.parse(getFirstText(result)); @@ -267,7 +267,7 @@ describe('MCP Source Tools', () => { name: 'Metrics', }); - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: metricSource._id.toString(), }); @@ -290,7 +290,7 @@ describe('MCP Source Tools', () => { }); it('should include usage guidance and nextSteps', async () => { - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: traceSource._id.toString(), }); const output = JSON.parse(getFirstText(result)); @@ -301,7 +301,7 @@ describe('MCP Source Tools', () => { }); it('should return error for non-existent source', async () => { - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: '000000000000000000000000', }); @@ -330,7 +330,7 @@ describe('MCP Source Tools', () => { name: 'Other Traces', }); - const result = await callTool(client, 'hyperdx_describe_source', { + const result = await callTool(client, 'clickstack_describe_source', { sourceId: otherSource._id.toString(), }); diff --git a/packages/api/src/mcp/__tests__/trace.test.ts b/packages/api/src/mcp/__tests__/trace.test.ts index 691b55a590..adc5dd30f5 100644 --- a/packages/api/src/mcp/__tests__/trace.test.ts +++ b/packages/api/src/mcp/__tests__/trace.test.ts @@ -147,9 +147,9 @@ describe('MCP Trace Tools', () => { // ─── Schema serialization ────────────────────────────────────────────────── describe('schema serialization', () => { - it('should expose hyperdx_trace_waterfall with expected properties', async () => { + it('should expose clickstack_trace_waterfall with expected properties', async () => { const { tools } = await client.listTools(); - const tool = tools.find(t => t.name === 'hyperdx_trace_waterfall'); + const tool = tools.find(t => t.name === 'clickstack_trace_waterfall'); expect(tool).toBeDefined(); const props = Object.keys(tool!.inputSchema.properties ?? {}); @@ -162,10 +162,10 @@ describe('MCP Trace Tools', () => { expect(tool!.inputSchema.required).toContain('sourceId'); }); - it('should expose hyperdx_trace_top_time_consuming_operations with expected properties', async () => { + it('should expose clickstack_trace_top_time_consuming_operations with expected properties', async () => { const { tools } = await client.listTools(); const tool = tools.find( - t => t.name === 'hyperdx_trace_top_time_consuming_operations', + t => t.name === 'clickstack_trace_top_time_consuming_operations', ); expect(tool).toBeDefined(); @@ -183,11 +183,11 @@ describe('MCP Trace Tools', () => { }); }); - // ─── hyperdx_trace_waterfall ─────────────────────────────────────────────── + // ─── clickstack_trace_waterfall ─────────────────────────────────────────────── - describe('hyperdx_trace_waterfall', () => { + describe('clickstack_trace_waterfall', () => { it('should return error for non-existent source', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: '000000000000000000000000', }); expect(result.isError).toBe(true); @@ -195,7 +195,7 @@ describe('MCP Trace Tools', () => { }); it('should return error for non-trace source', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: logSource._id.toString(), }); expect(result.isError).toBe(true); @@ -203,7 +203,7 @@ describe('MCP Trace Tools', () => { }); it('should return error for invalid time range', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), startTime: 'not-a-date', }); @@ -212,7 +212,7 @@ describe('MCP Trace Tools', () => { }); it('should return no-match hint when no traces exist', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), @@ -276,7 +276,7 @@ describe('MCP Trace Tools', () => { }); it('should fetch a trace by traceId and return a waterfall tree', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), traceId: TRACE_ID, startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), @@ -306,7 +306,7 @@ describe('MCP Trace Tools', () => { }); it('should auto-pick the slowest trace when traceId is omitted', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), pickFilter: `ServiceName:${WF_SVC}`, pickBy: 'slowest', @@ -321,7 +321,7 @@ describe('MCP Trace Tools', () => { }); it('should auto-pick the most recent trace', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), pickFilter: `ServiceName:${WF_SVC}`, pickBy: 'most_recent', @@ -335,7 +335,7 @@ describe('MCP Trace Tools', () => { }); it('should respect maxSpans and note truncation', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), traceId: TRACE_ID, maxSpans: 2, @@ -371,7 +371,7 @@ describe('MCP Trace Tools', () => { }, ]); - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), traceId: TRACE_ID, includeLogs: true, @@ -388,7 +388,7 @@ describe('MCP Trace Tools', () => { }); it('should omit logs when includeLogs is false', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), traceId: TRACE_ID, includeLogs: false, @@ -402,7 +402,7 @@ describe('MCP Trace Tools', () => { }); it('should apply pickFilter to narrow which trace is picked', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), pickFilter: `ServiceName:${WF_SVC}`, pickBy: 'slowest', @@ -416,7 +416,7 @@ describe('MCP Trace Tools', () => { }); it('should apply pickFilter with sql language', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), pickFilter: `ServiceName = '${WF_SVC}'`, pickFilterLanguage: 'sql', @@ -455,7 +455,7 @@ describe('MCP Trace Tools', () => { }); it('should auto-pick trace with an error span', async () => { - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: traceSource._id.toString(), pickFilter: 'ServiceName:wf-test-err-svc', pickBy: 'first_error', @@ -498,7 +498,7 @@ describe('MCP Trace Tools', () => { }); const TRACE_ID = 'aaaabbbbccccddddeeeeffffgggghhhh'; - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: noLogTraceSource._id.toString(), traceId: TRACE_ID, includeLogs: true, @@ -538,7 +538,7 @@ describe('MCP Trace Tools', () => { }); const TRACE_ID = 'aaaabbbbccccddddeeeeffffgggghhhh'; - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: badLogTraceSource._id.toString(), traceId: TRACE_ID, includeLogs: true, @@ -579,7 +579,7 @@ describe('MCP Trace Tools', () => { }); const TRACE_ID = 'aaaabbbbccccddddeeeeffffgggghhhh'; - const result = await callTool(client, 'hyperdx_trace_waterfall', { + const result = await callTool(client, 'clickstack_trace_waterfall', { sourceId: wrongKindTraceSource._id.toString(), traceId: TRACE_ID, includeLogs: true, @@ -595,13 +595,13 @@ describe('MCP Trace Tools', () => { }); }); - // ─── hyperdx_trace_top_time_consuming_operations ─────────────────────────── + // ─── clickstack_trace_top_time_consuming_operations ─────────────────────────── - describe('hyperdx_trace_top_time_consuming_operations', () => { + describe('clickstack_trace_top_time_consuming_operations', () => { it('should return error for non-existent source', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: '000000000000000000000000', parentFilter: "ServiceName = 'api-gateway'", @@ -616,7 +616,7 @@ describe('MCP Trace Tools', () => { it('should return error for non-trace source', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: logSource._id.toString(), parentFilter: "ServiceName = 'api-gateway'", @@ -631,7 +631,7 @@ describe('MCP Trace Tools', () => { it('should return error for invalid time range', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: "ServiceName = 'api-gateway'", @@ -646,7 +646,7 @@ describe('MCP Trace Tools', () => { it('should return empty operations when no traces match', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: "ServiceName = 'nonexistent-service'", @@ -753,7 +753,7 @@ describe('MCP Trace Tools', () => { it('should break down child operations ranked by total time', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, @@ -790,7 +790,7 @@ describe('MCP Trace Tools', () => { // Only trace 2 has a parent >= 850ms (900ms) const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, @@ -815,7 +815,7 @@ describe('MCP Trace Tools', () => { it('should respect topN parameter', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, @@ -840,7 +840,7 @@ describe('MCP Trace Tools', () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, @@ -860,7 +860,7 @@ describe('MCP Trace Tools', () => { it('should return error for invalid parentFilter SQL', async () => { const result = await callTool( client, - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { sourceId: traceSource._id.toString(), parentFilter: 'INVALID SQL @@@ SYNTAX', diff --git a/packages/api/src/mcp/mcpServer.ts b/packages/api/src/mcp/mcpServer.ts index 0635db892b..885cc245e2 100644 --- a/packages/api/src/mcp/mcpServer.ts +++ b/packages/api/src/mcp/mcpServer.ts @@ -13,7 +13,7 @@ import { McpContext } from './tools/types'; export function createServer(context: McpContext) { const server = new McpServer({ - name: 'hyperdx', + name: 'clickstack', version: `${CODE_VERSION}-beta`, }); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 6ba6f2c437..cb60705ffa 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -17,25 +17,25 @@ export function buildCreateDashboardPrompt( ${userContext} ${sourceSummary} -IMPORTANT: Call hyperdx_list_sources first to get source IDs, then hyperdx_describe_source for each source you plan to query. This gives you the full column schema, attribute keys, and sampled values (e.g. SeverityText, StatusCode). The source IDs above are correct, but you need the schema details to write accurate queries. +IMPORTANT: Call clickstack_list_sources first to get source IDs, then clickstack_describe_source for each source you plan to query. This gives you the full column schema, attribute keys, and sampled values (e.g. SeverityText, StatusCode). The source IDs above are correct, but you need the schema details to write accurate queries. == WORKFLOW == -1. Call hyperdx_list_sources to discover source IDs and connection IDs. -2. Call hyperdx_describe_source for each source you plan to query to get column schema, attribute keys, and sampled low-cardinality values (SeverityText, StatusCode, ServiceName, etc.). -3. Call hyperdx_get_dashboard (no id) to list existing dashboards. If any exist, fetch one or two with their id and skim them to pick up local idioms (naming style, time ranges, common groupings) before adding new ones. Skip when the workspace is empty. +1. Call clickstack_list_sources to discover source IDs and connection IDs. +2. Call clickstack_describe_source for each source you plan to query to get column schema, attribute keys, and sampled low-cardinality values (SeverityText, StatusCode, ServiceName, etc.). +3. Call clickstack_get_dashboard (no id) to list existing dashboards. If any exist, fetch one or two with their id and skim them to pick up local idioms (naming style, time ranges, common groupings) before adding new ones. Skip when the workspace is empty. 4. Pick the right source kind for the question. Trace data for request volume / latency / errors. Log data for severity / messages. Metric data for system telemetry. 5. Sketch tiles, THEN group them into 2-4 containers (Overview / Trends / Errors is a sane default for a service-health pattern) before assembling the save payload. Ungrouped tiles render as a flat sprawl that fails the readability test at five or more tiles. -6. Call hyperdx_save_dashboard with the tiles, containers, and dashboard-level filters. -7. Call hyperdx_query_tile on EVERY tile (not just one) to confirm queries return data and the dashboard is not silently degraded. +6. Call clickstack_save_dashboard with the tiles, containers, and dashboard-level filters. +7. Call clickstack_query_tile on EVERY tile (not just one) to confirm queries return data and the dashboard is not silently degraded. == UPDATING AN EXISTING DASHBOARD == -When updating a dashboard (passing the top-level \`id\` to hyperdx_save_dashboard), +When updating a dashboard (passing the top-level \`id\` to clickstack_save_dashboard), the \`filters\` array must be fully self-describing: every filter needs an \`id\`: - Existing filter you are KEEPING: copy its \`id\` verbatim from the - hyperdx_get_dashboard response. The same applies if you are renaming + clickstack_get_dashboard response. The same applies if you are renaming or tweaking it. This preserves any savedFilterValues bound to that id. - New filter you are ADDING in this update: generate a fresh random 24-character hex string (a Mongo-style ObjectId) and set it as \`id\`. @@ -46,11 +46,11 @@ the \`filters\` array must be fully self-describing: every filter needs an \`id\ Recommended pattern: - 1. \`hyperdx_get_dashboard({ id })\` to capture the current \`filters[]\` + 1. \`clickstack_get_dashboard({ id })\` to capture the current \`filters[]\` and their ids. 2. Build the new array: spread the kept filters as-is, append new filters with freshly-minted ids, omit removed ones. - 3. \`hyperdx_save_dashboard({ id, ..., filters: })\`. + 3. \`clickstack_save_dashboard({ id, ..., filters: })\`. == TILE TYPE GUIDE == @@ -72,15 +72,15 @@ Use RAW SQL tiles (with connectionId) only for queries the builder cannot expres - Top-level columns use PascalCase by default: Duration, StatusCode, SpanName, Body, SeverityText, ServiceName, SpanKind. NOTE: These are defaults for the standard HyperDX schema. Custom sources may use different names. - Always call hyperdx_describe_source to get the real column names, keyColumns, and sampled values for each source. + Always call clickstack_describe_source to get the real column names, keyColumns, and sampled values for each source. - Map-type columns use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']. NEVER use dot notation for Map columns. Always use brackets. - JSON-type columns use dot notation: JsonColumn.key.subkey. - Check the jsType returned by hyperdx_describe_source to determine whether a column is Map or JSON. + Check the jsType returned by clickstack_describe_source to determine whether a column is Map or JSON. == DESIGN CHECKLIST == -Apply these before calling hyperdx_save_dashboard. Each rule is enforced by the renderer; ignoring one is a runtime error you will then need to fix. +Apply these before calling clickstack_save_dashboard. Each rule is enforced by the renderer; ignoring one is a runtime error you will then need to fix. 1. ONE QUESTION PER DASHBOARD. A dashboard answers a single observability question well. Split if the request mixes traces, logs, and metrics into unrelated views. @@ -104,7 +104,7 @@ Apply these before calling hyperdx_save_dashboard. Each rule is enforced by the 9. FOR FOCUSED PER-DIMENSION DASHBOARDS, DECLARE A DASHBOARD-LEVEL FILTER. Pass filters: [{ type: "QUERY_EXPRESSION", name, expression, sourceId }] at the top level. The user gets a dropdown in the dashboard header; by default every tile is re-scoped when a value is picked. On mixed-source dashboards add appliesToSourceIds: ["", ...] to restrict the filter to only the tiles whose source carries that column. Omit the field to keep the broadcast-to-all-tiles default. Do NOT hardcode the dimension into each tile's where clause. -10. UPDATE IS REPLACE, NOT MERGE. hyperdx_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call hyperdx_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. +10. UPDATE IS REPLACE, NOT MERGE. clickstack_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call clickstack_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. 11. GROUP RELATED TILES INTO CONTAINERS. REQUIRED at five or more tiles, no exceptions. An ungrouped wall of nine or ten tiles is a readability failure even when each tile is correct in isolation. Containers are the right way to introduce structure; markdown tiles for section labels are not. @@ -121,17 +121,17 @@ Apply these before calling hyperdx_save_dashboard. Each rule is enforced by the ] Use tabs when one container needs to show different views of the same data (Throughput / Latency / Errors over time, for instance); the tab bar appears when a container has two or more tabs declared. -12. VALIDATE EVERY TILE AFTER SAVE. After hyperdx_save_dashboard, call hyperdx_query_tile on EVERY tile (not just one). Save validates input shape; it does NOT validate query semantics. Some queries pass save and fail at render time (known gaps: Lucene comparison/wildcard on map attributes, metric tiles with multiple metricTables, malformed having clauses). If query_tile returns an error, fix the tile and re-save before declaring the dashboard ready. +12. VALIDATE EVERY TILE AFTER SAVE. After clickstack_save_dashboard, call clickstack_query_tile on EVERY tile (not just one). Save validates input shape; it does NOT validate query semantics. Some queries pass save and fail at render time (known gaps: Lucene comparison/wildcard on map attributes, metric tiles with multiple metricTables, malformed having clauses). If query_tile returns an error, fix the tile and re-save before declaring the dashboard ready. 13. NO TITLE-RECAP MARKDOWN TILE. The dashboard's name shows in the title bar. Adding a markdown tile with the dashboard name (or a "About this dashboard" header) doubles the title and eats a row of vertical space because markdown heading styles render at title-bar scale. Skip the markdown tile entirely on starter dashboards. == ADAPT, DO NOT COPY == -The dashboard_examples prompt returns concrete example shapes. Read them as patterns to adapt to the user's actual request and schema, not as literal templates to substitute names into. Specifically: the literal "ServiceName", "StatusCode:STATUS_CODE_ERROR", "Duration" come from the standard HyperDX schema. Real source schemas may use different column names and status values; always verify with hyperdx_list_sources before reusing literals. +The dashboard_examples prompt returns concrete example shapes. Read them as patterns to adapt to the user's actual request and schema, not as literal templates to substitute names into. Specifically: the literal "ServiceName", "StatusCode:STATUS_CODE_ERROR", "Duration" come from the standard HyperDX schema. Real source schemas may use different column names and status values; always verify with clickstack_list_sources before reusing literals. IMPORTANT: The exact values for StatusCode and SeverityText vary by deployment. Do NOT assume values like "STATUS_CODE_ERROR", "Ok", "error", or "fatal". -Always call hyperdx_describe_source first and inspect the lowCardinalityValues +Always call clickstack_describe_source first and inspect the lowCardinalityValues returned for each source to discover the real values used in your data. == DEFAULT TIME WINDOW == @@ -143,12 +143,12 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie - Using valueExpression with aggFn "count" (count takes no valueExpression). - Forgetting valueExpression for non-count aggFns (avg, sum, min, max, quantile all require it). - Using dot notation for Map-type attributes (always use SpanAttributes['key'] bracket syntax). -- Skipping hyperdx_list_sources + hyperdx_describe_source (you need real source IDs, column names, and values). -- Skipping hyperdx_query_tile after save (tiles can silently fail on syntax or attribute mismatches). +- Skipping clickstack_list_sources + clickstack_describe_source (you need real source IDs, column names, and values). +- Skipping clickstack_query_tile after save (tiles can silently fail on syntax or attribute mismatches). - Setting chart-level numberFormat on a table that mixes counts and durations (counts render as 0:00:00). - Multiple select items on number / pie / heatmap tiles (each takes exactly one). - Missing level on aggFn "quantile" (must specify 0.5, 0.9, 0.95, or 0.99). -- Assuming StatusCode or SeverityText values (always inspect lowCardinalityValues from hyperdx_describe_source). +- Assuming StatusCode or SeverityText values (always inspect lowCardinalityValues from clickstack_describe_source). - Heatmap on a non-Trace source (heatmap is Trace-only today). - Hardcoding a focus dimension into every tile's where clause (use a dashboard-level filter instead).`; } @@ -614,7 +614,7 @@ or a more detailed dashboard. Useful when the dashboard answers inside that service?". Replace with the ID of an existing service-detail -dashboard. Use hyperdx_get_dashboard (no id) to list dashboards and +dashboard. Use clickstack_get_dashboard (no id) to list dashboards and copy the right ID and discover which filters are available. { @@ -741,8 +741,8 @@ SQL TEMPLATE REFERENCE: 'Concrete dashboard examples for common observability patterns. ' + 'These are PATTERNS to adapt, not literal templates. Adapt the structure ' + "(table shape, container layout, filter expression) to the user's request; " + - 'always verify column names and status literals via hyperdx_list_sources before reusing them.\n' + - 'Replace ${sourceId} / ${connectionId} placeholders with real IDs from hyperdx_list_sources.\n\n'; + 'always verify column names and status literals via clickstack_list_sources before reusing them.\n' + + 'Replace ${sourceId} / ${connectionId} placeholders with real IDs from clickstack_list_sources.\n\n'; if (pattern) { const key = pattern.toLowerCase().replace(/[\s-]+/g, '_'); @@ -765,7 +765,7 @@ SQL TEMPLATE REFERENCE: } export function buildQueryGuidePrompt(): string { - return `Reference guide for writing queries with HyperDX MCP tools (hyperdx_timeseries, hyperdx_table, hyperdx_search, hyperdx_event_patterns, hyperdx_sql, and hyperdx_save_dashboard). + return `Reference guide for writing queries with ClickStack MCP tools (clickstack_timeseries, clickstack_table, clickstack_search, clickstack_event_patterns, clickstack_sql, and clickstack_save_dashboard). == AGGREGATION FUNCTIONS (aggFn) == @@ -794,7 +794,7 @@ Top-level columns (PascalCase defaults, use directly in valueExpression and grou Duration, StatusCode, SpanName, ServiceName, SpanKind, Body, SeverityText, Timestamp, TraceId, SpanId, ParentSpanId NOTE: These are defaults for the standard HyperDX schema. Custom sources may - use different column names. Always verify with hyperdx_describe_source, which returns + use different column names. Always verify with clickstack_describe_source, which returns the real column names and keyColumns expressions for each source. Map-type columns (bracket syntax, access keys via ['key']): @@ -810,7 +810,7 @@ IMPORTANT: Always use bracket syntax for Map-type columns. Never use dot notatio JSON-type columns (dot notation, access nested keys via dot path): JsonColumn.key.subkey - NOTE: Check the jsType field returned by hyperdx_describe_source to determine + NOTE: Check the jsType field returned by clickstack_describe_source to determine whether a column is Map (use brackets) or JSON (use dots). == LUCENE FILTER SYNTAX == @@ -831,7 +831,7 @@ Used in the "where" field of select items and search tiles. NOTE: In Lucene filters, use dot notation for attribute keys (service.name, http.method). This is different from valueExpression and groupBy which require bracket syntax (SpanAttributes['http.method']). -GOTCHA: Lucene queries that reference map-attribute keys via dotted paths (http.status_code, db.system, deployment.environment, anything under SpanAttributes / ResourceAttributes / LogAttributes) are NOT reliably translated to bracket access. Operators (>=, >, <=, <), wildcards (*), AND simple equality all hit translator gaps depending on the column. Symptoms: tile saves fine; hyperdx_query_tile returns an error or returns zero rows when there should be matches; the dashboard renders "Error loading chart" on the tile. Do NOT trust dotted-path Lucene; use SQL with bracket access for ALL map-attribute filtering. +GOTCHA: Lucene queries that reference map-attribute keys via dotted paths (http.status_code, db.system, deployment.environment, anything under SpanAttributes / ResourceAttributes / LogAttributes) are NOT reliably translated to bracket access. Operators (>=, >, <=, <), wildcards (*), AND simple equality all hit translator gaps depending on the column. Symptoms: tile saves fine; clickstack_query_tile returns an error or returns zero rows when there should be matches; the dashboard renders "Error loading chart" on the tile. Do NOT trust dotted-path Lucene; use SQL with bracket access for ALL map-attribute filtering. Unreliable in Lucene (any operation, any operator): http.status_code:>=500 (operator) @@ -927,7 +927,7 @@ For configType: "sql" tiles, write ClickHouse SQL with template macros: search No select items (select is a column list string). where is the filter. markdown No select items. Set markdown field with content. -NOTE: Authoring builder tiles on a metric source is not reliable today. The MCP select-item shape does not carry the metricName / metricType fields the metric query path needs, and a save with a metric sourceId may render in the UI as "Both table name and UUID are empty" even though the save itself succeeded. For metrics, use a raw SQL tile (configType: "sql") with explicit table reference. The standard tables that back a metric source are otel_metrics_gauge, otel_metrics_sum, and otel_metrics_histogram; hyperdx_list_sources returns the metric source's metricTables map so you know which table holds which metric kind. Discovery: metric source schemas today do NOT publish mapAttributeKeys for ResourceAttributes / Attributes the way log and trace sources do, so attribute keys must be discovered by sampling (SELECT DISTINCT mapKeys(Attributes) FROM ...). +NOTE: Authoring builder tiles on a metric source is not reliable today. The MCP select-item shape does not carry the metricName / metricType fields the metric query path needs, and a save with a metric sourceId may render in the UI as "Both table name and UUID are empty" even though the save itself succeeded. For metrics, use a raw SQL tile (configType: "sql") with explicit table reference. The standard tables that back a metric source are otel_metrics_gauge, otel_metrics_sum, and otel_metrics_histogram; clickstack_list_sources returns the metric source's metricTables map so you know which table holds which metric kind. Discovery: metric source schemas today do NOT publish mapAttributeKeys for ResourceAttributes / Attributes the way log and trace sources do, so attribute keys must be discovered by sampling (SELECT DISTINCT mapKeys(Attributes) FROM ...). == NUMBER FORMAT == @@ -997,11 +997,11 @@ Destination types: { type: "search", target, whereLanguage, whereTemplate?, filters? } Opens the /search page for a log or trace source. Metric and session sources are rejected by the server (the /search page does not render those kinds). { type: "dashboard", target, whereLanguage, whereTemplate?, filters? } - Opens another HyperDX dashboard owned by the same team. + Opens another ClickStack dashboard owned by the same team. Target shape, how the destination is identified: target: { mode: "id", id: "" } - Pins a specific source (for type=search) or dashboard (for type=dashboard) by its ID. Prefer this when the target is known. Find source IDs with hyperdx_list_sources; find dashboard IDs with hyperdx_get_dashboard (no id arg returns the list of all dashboards). + Pins a specific source (for type=search) or dashboard (for type=dashboard) by its ID. Prefer this when the target is known. Find source IDs with clickstack_list_sources; find dashboard IDs with clickstack_get_dashboard (no id arg returns the list of all dashboards). target: { mode: "template", template: "" } Renders the template against the clicked row's columns, then resolves the result by NAME on the destination team. Example template "{{Service}}" reads the row's Service value and looks up a source/dashboard with that name. Use this when the destination depends on which row was clicked. A constant template like "Service Detail" (no Handlebars vars) simply resolves to the dashboard or source with that exact name. @@ -1088,7 +1088,7 @@ For onClick drill-down on a plain top-level column (ServiceName), no workaround == CONTAINERS AND TABS == -Optional dashboard organization layer. Group related tiles visually with optional tab bars. Pass on hyperdx_save_dashboard as a top-level "containers" array; reference containers from tiles via "containerId" (and "tabId" when the container has tabs). +Optional dashboard organization layer. Group related tiles visually with optional tab bars. Pass on clickstack_save_dashboard as a top-level "containers" array; reference containers from tiles via "containerId" (and "tabId" when the container has tabs). Container shape: { id, title, collapsed, collapsible?, bordered?, tabs? } @@ -1112,7 +1112,7 @@ Rules enforced by the API: - A tile's tabId requires containerId to be set. Example: - hyperdx_save_dashboard({ + clickstack_save_dashboard({ name: "Service Overview", containers: [ { @@ -1149,7 +1149,7 @@ containers[i].tabs[j].id). Read the path to know which input to fix. == EVENT PATTERN MINING == -Use hyperdx_event_patterns when asked to find common log messages, recurring +Use clickstack_event_patterns when asked to find common log messages, recurring patterns, noisy services, or top event types. This is the right choice whenever the question is about "most common logs", "top patterns", "what is generating the most log volume", or similar pattern-discovery questions. @@ -1161,7 +1161,7 @@ It samples random events and clusters them with the Drain algorithm, returning: - a whereSnippet for each pattern to drill into matching raw events Required parameters: - - sourceId (call hyperdx_list_sources first) + - sourceId (call clickstack_list_sources first) - startTime / endTime for the time window Optional parameters: @@ -1170,7 +1170,7 @@ Optional parameters: - bodyExpression: column to mine (auto-detected: Body for logs, SpanName for traces) Example: find top patterns for production services over the last 4 hours: - hyperdx_event_patterns({ + clickstack_event_patterns({ sourceId: "", startTime: "<4 hours ago ISO>", endTime: "", @@ -1190,12 +1190,12 @@ Example: find top patterns for production services over the last 4 hours: 3. Using dot notation for Map-type attributes in valueExpression or groupBy Wrong: groupBy: "SpanAttributes.http.method" Correct: groupBy: "SpanAttributes['http.method']" - NOTE: JSON-type columns DO use dot notation. Check jsType from hyperdx_describe_source. + NOTE: JSON-type columns DO use dot notation. Check jsType from clickstack_describe_source. 4. Multiple select items on number / pie / heatmap tiles Wrong: displayType: "number", select: [{ aggFn: "count" }, { aggFn: "avg", ... }] Correct: displayType: "number", select: [{ aggFn: "count" }] - Note: hyperdx_table (the query tool) auto-upgrades shape:"number" to "table" when select has >1 item; dashboard tiles do not. + Note: clickstack_table (the query tool) auto-upgrades shape:"number" to "table" when select has >1 item; dashboard tiles do not. 5. Missing level for quantile Wrong: { aggFn: "quantile", valueExpression: "Duration" } @@ -1212,7 +1212,7 @@ Example: find top patterns for production services over the last 4 hours: The dashboard filter applies globally; tiles do not need the literal. On mixed-source dashboards where only some tiles carry the column, add appliesToSourceIds: ["", ...] to scope the filter to just those sources instead of breaking the unrelated tiles. 8. Forgetting to validate tiles after saving - Always call hyperdx_query_tile on EVERY tile after hyperdx_save_dashboard, not just one. Save validates input shape but not query semantics. Several known gaps (Lucene comparison/wildcard on map attributes, builder tiles on metric sources, malformed having) pass save and fail at render time. A dashboard with one bad tile renders the whole page in a degraded state; the user sees "Error loading chart" with no way to know which tile broke unless you validated. If query_tile returns an error, fix the where / SQL and re-save before declaring the dashboard ready. + Always call clickstack_query_tile on EVERY tile after clickstack_save_dashboard, not just one. Save validates input shape but not query semantics. Several known gaps (Lucene comparison/wildcard on map attributes, builder tiles on metric sources, malformed having) pass save and fail at render time. A dashboard with one bad tile renders the whole page in a degraded state; the user sees "Error loading chart" with no way to know which tile broke unless you validated. If query_tile returns an error, fix the where / SQL and re-save before declaring the dashboard ready. 9. Using sourceId with SQL tiles or connectionId with builder tiles Builder tiles (line, table, etc.) use sourceId. @@ -1220,13 +1220,13 @@ Example: find top patterns for production services over the last 4 hours: 10. Assuming StatusCode or SeverityText values Values like STATUS_CODE_ERROR, Ok, error, fatal vary by deployment. - Always call hyperdx_describe_source and inspect the lowCardinalityValues + Always call clickstack_describe_source and inspect the lowCardinalityValues before writing filters that depend on these columns. 11. Heatmap tile on a non-Trace source Wrong: { displayType: "heatmap", sourceId: "", select: [...] } Correct: { displayType: "heatmap", sourceId: "", select: [...] } - Heatmap is currently restricted to Trace sources. Pick a source whose kind is "trace" from hyperdx_list_sources. + Heatmap is currently restricted to Trace sources. Pick a source whose kind is "trace" from clickstack_list_sources. 12. Heatmap tile with empty or missing valueExpression Wrong: select: [{}] diff --git a/packages/api/src/mcp/prompts/dashboards/helpers.ts b/packages/api/src/mcp/prompts/dashboards/helpers.ts index 364f2a9407..f0c4e3d470 100644 --- a/packages/api/src/mcp/prompts/dashboards/helpers.ts +++ b/packages/api/src/mcp/prompts/dashboards/helpers.ts @@ -5,7 +5,7 @@ export function buildSourceSummary( connections: { _id: unknown; name: string }[], ): string { if (sources.length === 0 && connections.length === 0) { - return 'No sources or connections found. Call hyperdx_list_sources to discover available data.'; + return 'No sources or connections found. Call clickstack_list_sources to discover available data.'; } const lines: string[] = []; diff --git a/packages/api/src/mcp/prompts/dashboards/index.ts b/packages/api/src/mcp/prompts/dashboards/index.ts index e98e9f312d..a8735ee33e 100644 --- a/packages/api/src/mcp/prompts/dashboards/index.ts +++ b/packages/api/src/mcp/prompts/dashboards/index.ts @@ -26,7 +26,7 @@ const dashboardPrompts: PromptDefinition = (server, context) => { { title: 'Create a Dashboard', description: - 'Create a HyperDX dashboard with the MCP tools. ' + + 'Create a ClickStack dashboard with the MCP tools. ' + 'Follow the recommended workflow, pick tile types, write queries, ' + 'and validate results against your real data sources.', argsSchema: { @@ -72,7 +72,7 @@ const dashboardPrompts: PromptDefinition = (server, context) => { 'Failed to fetch sources for create_dashboard prompt', ); sourceSummary = - 'Could not fetch sources. Call hyperdx_list_sources to discover available data.'; + 'Could not fetch sources. Call clickstack_list_sources to discover available data.'; traceSourceId = ''; logSourceId = ''; } @@ -106,7 +106,7 @@ const dashboardPrompts: PromptDefinition = (server, context) => { 'Concrete dashboard example shapes for common observability patterns: ' + 'service_inventory, service_detail, log_analytics, backend_dependencies, drilldown_links, infrastructure_sql. ' + 'Each example carries a "when to use" header. Adapt the structure to the user\'s request; ' + - 'do not copy literal column names or status values without verifying via hyperdx_list_sources first.', + 'do not copy literal column names or status values without verifying via clickstack_list_sources first.', argsSchema: { pattern: z .string() diff --git a/packages/api/src/mcp/tools/alerts/getAlert.ts b/packages/api/src/mcp/tools/alerts/getAlert.ts index b6059ebd7b..6b39655d93 100644 --- a/packages/api/src/mcp/tools/alerts/getAlert.ts +++ b/packages/api/src/mcp/tools/alerts/getAlert.ts @@ -50,7 +50,7 @@ export function registerGetAlert(server: McpServer, context: McpContext): void { const frontendUrl = config.FRONTEND_URL; server.registerTool( - 'hyperdx_get_alert', + 'clickstack_get_alert', { title: 'Get Alert(s)', description: @@ -75,7 +75,7 @@ export function registerGetAlert(server: McpServer, context: McpContext): void { ), }), }, - withToolTracing('hyperdx_get_alert', context, async ({ id, state }) => { + withToolTracing('clickstack_get_alert', context, async ({ id, state }) => { // ── List all alerts (slim summary) ── if (!id) { const query: Record = { diff --git a/packages/api/src/mcp/tools/alerts/getWebhook.ts b/packages/api/src/mcp/tools/alerts/getWebhook.ts index e87db90fb7..0707b0c1ab 100644 --- a/packages/api/src/mcp/tools/alerts/getWebhook.ts +++ b/packages/api/src/mcp/tools/alerts/getWebhook.ts @@ -13,16 +13,16 @@ export function registerGetWebhook( const { teamId } = context; server.registerTool( - 'hyperdx_get_webhook', + 'clickstack_get_webhook', { title: 'List Webhooks', description: 'List available webhook destinations (id, name, service type). ' + 'Use the returned id as the webhookId when creating alerts with ' + - 'hyperdx_save_alert.', + 'clickstack_save_alert.', inputSchema: z.object({}), }, - withToolTracing('hyperdx_get_webhook', context, async () => { + withToolTracing('clickstack_get_webhook', context, async () => { const webhooks = await Webhook.find({ team: teamId }); const output = webhooks.map(wh => ({ diff --git a/packages/api/src/mcp/tools/alerts/saveAlert.ts b/packages/api/src/mcp/tools/alerts/saveAlert.ts index c693858b8e..88e9b7635c 100644 --- a/packages/api/src/mcp/tools/alerts/saveAlert.ts +++ b/packages/api/src/mcp/tools/alerts/saveAlert.ts @@ -40,7 +40,7 @@ export function registerSaveAlert( const frontendUrl = config.FRONTEND_URL; server.registerTool( - 'hyperdx_save_alert', + 'clickstack_save_alert', { title: 'Create or Update Alert', description: @@ -49,7 +49,7 @@ export function registerSaveAlert( 'metric crosses a threshold. A webhook notification channel is required.', inputSchema: mcpSaveAlertSchema, }, - withToolTracing('hyperdx_save_alert', context, async input => { + withToolTracing('clickstack_save_alert', context, async input => { // ── Runtime cross-field validation ── const validationError = validateSaveAlertInput(input); if (validationError) { diff --git a/packages/api/src/mcp/tools/alerts/schemas.ts b/packages/api/src/mcp/tools/alerts/schemas.ts index d5878e9847..c678863430 100644 --- a/packages/api/src/mcp/tools/alerts/schemas.ts +++ b/packages/api/src/mcp/tools/alerts/schemas.ts @@ -6,7 +6,7 @@ import { import { z } from 'zod'; // --------------------------------------------------------------------------- -// MCP-compatible flat Zod schema for hyperdx_save_alert. +// MCP-compatible flat Zod schema for clickstack_save_alert. // // The MCP SDK's normalizeObjectSchema() cannot serialize ZodEffects // (superRefine) or discriminatedUnion. We keep the inputSchema as a plain diff --git a/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts b/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts index 5d9fb48b78..0b00f6a4b1 100644 --- a/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts @@ -15,18 +15,18 @@ export function registerDeleteDashboard( const { teamId } = context; server.registerTool( - 'hyperdx_delete_dashboard', + 'clickstack_delete_dashboard', { title: 'Delete Dashboard', description: 'Permanently delete a dashboard by ID. Also removes any alerts attached to its tiles. ' + - 'Use hyperdx_get_dashboard (without an ID) to list available dashboard IDs.', + 'Use clickstack_get_dashboard (without an ID) to list available dashboard IDs.', inputSchema: z.object({ id: z.string().describe('Dashboard ID to delete.'), }), }, withToolTracing( - 'hyperdx_delete_dashboard', + 'clickstack_delete_dashboard', context, async ({ id: dashboardId }) => { if (!mongoose.Types.ObjectId.isValid(dashboardId)) { diff --git a/packages/api/src/mcp/tools/dashboards/getDashboard.ts b/packages/api/src/mcp/tools/dashboards/getDashboard.ts index ee9b2d73a8..412b9e17b3 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboard.ts @@ -18,7 +18,7 @@ export function registerGetDashboard( const frontendUrl = config.FRONTEND_URL; server.registerTool( - 'hyperdx_get_dashboard', + 'clickstack_get_dashboard', { title: 'Get Dashboard(s)', description: @@ -33,7 +33,7 @@ export function registerGetDashboard( ), }), }, - withToolTracing('hyperdx_get_dashboard', context, async ({ id }) => { + withToolTracing('clickstack_get_dashboard', context, async ({ id }) => { if (!id) { const dashboards = await getDashboards( new mongoose.Types.ObjectId(teamId), diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index 5604bee8df..77b12df45b 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -16,21 +16,21 @@ export function registerQueryTile( const { teamId } = context; server.registerTool( - 'hyperdx_query_tile', + 'clickstack_query_tile', { title: 'Query a Dashboard Tile', description: 'Execute the query for a specific tile on an existing dashboard. ' + 'Useful for validating that a tile returns data or for spot-checking results ' + 'without rebuilding the query from scratch. ' + - 'Use hyperdx_get_dashboard with an ID to find tile IDs.', + 'Use clickstack_get_dashboard with an ID to find tile IDs.', inputSchema: z.object({ dashboardId: z.string().describe('Dashboard ID.'), tileId: z .string() .describe( 'Tile ID within the dashboard. ' + - 'Obtain from hyperdx_get_dashboard.', + 'Obtain from clickstack_get_dashboard.', ), startTime: z .string() @@ -46,7 +46,7 @@ export function registerQueryTile( }), }, withToolTracing( - 'hyperdx_query_tile', + 'clickstack_query_tile', context, async ({ dashboardId, tileId, startTime, endTime }) => { const timeRange = parseTimeRange(startTime, endTime); diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index d6bee8eb5c..4e118c6212 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -41,13 +41,13 @@ export function registerSaveDashboard( const frontendUrl = config.FRONTEND_URL; server.registerTool( - 'hyperdx_save_dashboard', + 'clickstack_save_dashboard', { title: 'Create or Update Dashboard', description: 'Create a new dashboard (omit id) or update an existing one (provide id). ' + - 'Call hyperdx_list_sources first to obtain sourceId and connectionId values. ' + - 'IMPORTANT: After saving a dashboard, always run hyperdx_query_tile on each tile ' + + 'Call clickstack_list_sources first to obtain sourceId and connectionId values. ' + + 'IMPORTANT: After saving a dashboard, always run clickstack_query_tile on each tile ' + 'to confirm the queries work and return expected data. Tiles can silently fail ' + 'due to incorrect filter syntax, missing attributes, or wrong column names.', inputSchema: z.object({ @@ -65,7 +65,7 @@ export function registerSaveDashboard( }), }, withToolTracing( - 'hyperdx_save_dashboard', + 'clickstack_save_dashboard', context, async ({ id: dashboardId, @@ -311,7 +311,7 @@ async function createDashboard({ ...(frontendUrl ? { url: `${frontendUrl}/dashboards/${newDashboard._id}` } : {}), - hint: 'Use hyperdx_query_tile to test individual tile queries before viewing the dashboard.', + hint: 'Use clickstack_query_tile to test individual tile queries before viewing the dashboard.', }, null, 2, @@ -570,7 +570,7 @@ async function updateDashboard({ ...(frontendUrl ? { url: `${frontendUrl}/dashboards/${updatedDashboard._id}` } : {}), - hint: 'Use hyperdx_query_tile to test individual tile queries before viewing the dashboard.', + hint: 'Use clickstack_query_tile to test individual tile queries before viewing the dashboard.', }, null, 2, diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 8a0fc5e8d1..fc692acbb7 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1,6 +1,6 @@ // prose-lint: allow-file // MCP tool descriptions intentionally use the established en-dash -// separator (e.g. "Source ID – call hyperdx_list_sources") for LLM +// separator (e.g. "Source ID – call clickstack_list_sources") for LLM // readability. Reformatting all separators is out of scope here. import { AggregateFunctionSchema, @@ -174,8 +174,8 @@ const mcpOnClickTargetSchema = z .min(1) .describe( 'Concrete source ID (for type=search) or dashboard ID (for type=dashboard). ' + - 'Get source IDs from hyperdx_list_sources; get dashboard IDs from ' + - 'hyperdx_get_dashboard (no id arg returns the list). ' + + 'Get source IDs from clickstack_list_sources; get dashboard IDs from ' + + 'clickstack_get_dashboard (no id arg returns the list). ' + 'For type=search the source kind must be "log" or "trace"; the /search ' + 'page does not render metric/session sources.', ), @@ -245,7 +245,9 @@ const mcpOnClickSearchSchema = z const mcpOnClickDashboardSchema = z .object({ - type: z.literal('dashboard').describe('Link to another HyperDX dashboard.'), + type: z + .literal('dashboard') + .describe('Link to another ClickStack dashboard.'), target: mcpOnClickTargetSchema, whereTemplate: z .string() @@ -366,7 +368,7 @@ const mcpTileLayoutSchema = z.object({ const mcpLineTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z.literal('line').describe('Line chart over time'), - sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + sourceId: z.string().describe('Source ID – call clickstack_list_sources'), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -397,7 +399,7 @@ const mcpBarTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('stacked_bar') .describe('Stacked bar chart over time'), - sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + sourceId: z.string().describe('Source ID – call clickstack_list_sources'), select: z.array(mcpTileSelectItemSchema).min(1).max(20), groupBy: z.string().optional(), fillNulls: z.boolean().optional().default(true), @@ -409,7 +411,7 @@ const mcpBarTileSchema = mcpTileLayoutSchema.extend({ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z.literal('table').describe('Tabular aggregated data'), - sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + sourceId: z.string().describe('Source ID – call clickstack_list_sources'), select: z.array(mcpTileSelectItemSchema).min(1).max(20), groupBy: z .string() @@ -444,7 +446,7 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z.literal('number').describe('Single aggregate scalar value'), - sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + sourceId: z.string().describe('Source ID – call clickstack_list_sources'), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -461,7 +463,7 @@ const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ const mcpPieTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z.literal('pie').describe('Pie chart'), - sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + sourceId: z.string().describe('Source ID – call clickstack_list_sources'), select: z.array(mcpTileSelectItemSchema).length(1), groupBy: z .string() @@ -512,7 +514,7 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'Source ID. Must be a Trace source today; use hyperdx_list_sources and ' + + 'Source ID. Must be a Trace source today; use clickstack_list_sources and ' + 'pick one whose kind is "trace".', ), select: z @@ -539,7 +541,7 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ const mcpSearchTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z.literal('search').describe('Log/event search results list'), - sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + sourceId: z.string().describe('Source ID – call clickstack_list_sources'), where: z .string() .optional() @@ -577,7 +579,7 @@ const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ connectionId: z .string() .describe( - 'Connection ID (not sourceId) – call hyperdx_list_sources to find available connections', + 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections', ), sqlTemplate: z .string() @@ -614,7 +616,7 @@ export const mcpTilesParam = z .array(mcpTileSchema) .describe( 'Array of dashboard tiles. Each tile needs a name, optional layout (x/y/w/h), and a config block. ' + - 'The config block varies by displayType – use hyperdx_list_sources for sourceId and connectionId values.\n\n' + + 'The config block varies by displayType – use clickstack_list_sources for sourceId and connectionId values.\n\n' + 'Example tiles:\n' + '1. Line chart: { "name": "Error Rate", "config": { "displayType": "line", "sourceId": "", ' + '"groupBy": "ResourceAttributes[\'service.name\']", "select": [{ "aggFn": "count", "where": "StatusCode:STATUS_CODE_ERROR" }] } }\n' + @@ -640,7 +642,7 @@ const mcpDashboardFilterSchema = z .describe( 'Filter identity. ' + 'On UPDATE of an existing dashboard, every filter in the array MUST carry ' + - 'an id: pass the exact id returned by hyperdx_get_dashboard for any filter ' + + 'an id: pass the exact id returned by clickstack_get_dashboard for any filter ' + 'you are keeping (so saved values bound to it stay attached), and generate ' + 'a fresh random hex/ObjectId string for any filter you are adding in this ' + 'update. Omitting `id` on an existing filter would orphan its saved values; ' + @@ -670,7 +672,7 @@ const mcpDashboardFilterSchema = z ), sourceId: objectIdSchema.describe( 'Source the filter values are pulled from (for the dropdown). ' + - 'Get IDs from hyperdx_list_sources.', + 'Get IDs from clickstack_list_sources.', ), sourceMetricType: z .nativeEnum(MetricsDataType) diff --git a/packages/api/src/mcp/tools/query/eventDeltas.ts b/packages/api/src/mcp/tools/query/eventDeltas.ts index f5ded14674..7ffdca1570 100644 --- a/packages/api/src/mcp/tools/query/eventDeltas.ts +++ b/packages/api/src/mcp/tools/query/eventDeltas.ts @@ -47,7 +47,7 @@ const deltasSchema = z.object({ .string() .describe( 'Source ID. Works for trace and log sources. ' + - 'Call hyperdx_list_sources for available sources.', + 'Call clickstack_list_sources for available sources.', ), target: groupSchema.describe( 'Target ("outlier") group — the rows you suspect are different (e.g. the ' + @@ -140,7 +140,7 @@ export function registerEventDeltas(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_event_deltas', + 'clickstack_event_deltas', { title: 'Compare Events: Target vs Baseline', description: @@ -177,9 +177,9 @@ export function registerEventDeltas(server: McpServer, context: McpContext) { ' Any pair of row sets over the same source works — the tool just ' + ' asks "what is statistically different about target vs baseline".\n\n' + 'WHEN NOT TO USE: when the question is already known to be about a ' + - 'specific attribute (use hyperdx_table groupBy), when you want raw ' + - 'rows (use hyperdx_search), or when you need a time-series shape ' + - '(use hyperdx_timeseries).\n\n' + + 'specific attribute (use clickstack_table groupBy), when you want raw ' + + 'rows (use clickstack_search), or when you need a time-series shape ' + + '(use clickstack_timeseries).\n\n' + 'OUTPUT SHAPE: an array of properties, each with rank, key, score, ' + 'semanticBoost (true for well-known OTel attrs like service.name / ' + 'http.method / error.type / status), targetCount and baselineCount ' + @@ -197,7 +197,7 @@ export function registerEventDeltas(server: McpServer, context: McpContext) { inputSchema: deltasSchema, }, withToolTracing( - 'hyperdx_event_deltas', + 'clickstack_event_deltas', context, async (input: DeltasInput) => { const targetRange = parseTimeRange( @@ -272,7 +272,7 @@ export function registerEventDeltas(server: McpServer, context: McpContext) { content: [ { type: 'text' as const, - text: `Source not found: ${input.sourceId}. Call hyperdx_list_sources to find available source IDs.`, + text: `Source not found: ${input.sourceId}. Call clickstack_list_sources to find available source IDs.`, }, ], }; @@ -286,7 +286,7 @@ export function registerEventDeltas(server: McpServer, context: McpContext) { content: [ { type: 'text' as const, - text: `Source ${input.sourceId} is kind="${source.kind}". hyperdx_event_deltas requires a trace or log source.`, + text: `Source ${input.sourceId} is kind="${source.kind}". clickstack_event_deltas requires a trace or log source.`, }, ], }; @@ -511,7 +511,7 @@ export function registerEventDeltas(server: McpServer, context: McpContext) { ? { warning: 'One or both groups returned 0 rows — verify the time ' + - 'window and where filter via hyperdx_search.', + 'window and where filter via clickstack_search.', } : {}), }, diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts index c3da47cca7..a2a85f3d0f 100644 --- a/packages/api/src/mcp/tools/query/eventPatterns.ts +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -66,7 +66,7 @@ export function registerEventPatterns(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_event_patterns', + 'clickstack_event_patterns', { title: 'Event Pattern Mining', description: @@ -76,15 +76,15 @@ export function registerEventPatterns(server: McpServer, context: McpContext) { 'Use this when asked about "top patterns", "common logs", "noisy services", ' + '"recurring messages", or log noise analysis.\n\n' + 'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' + - 'a follow-up hyperdx_search call to browse matching raw events.\n\n' + - 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + + 'a follow-up clickstack_search call to browse matching raw events.\n\n' + + 'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' + 'When to use which tool:\n' + - ' - hyperdx_event_patterns: clustering / recurring shapes / noise analysis\n' + - ' - hyperdx_search: raw individual rows\n' + - ' - hyperdx_table: aggregated metrics / counts / top-N', + ' - clickstack_event_patterns: clustering / recurring shapes / noise analysis\n' + + ' - clickstack_search: raw individual rows\n' + + ' - clickstack_table: aggregated metrics / counts / top-N', inputSchema: eventPatternsSchema, }, - withToolTracing('hyperdx_event_patterns', context, async input => { + withToolTracing('clickstack_event_patterns', context, async input => { const timeRange = parseTimeRange(input.startTime, input.endTime); if ('error' in timeRange) { return { diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 51f83f2d80..7fbd3d200a 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -216,7 +216,7 @@ export async function runConfigTile( content: [ { type: 'text' as const, - text: `Source not found: ${builderConfig.source}. Call hyperdx_list_sources to discover available source IDs.`, + text: `Source not found: ${builderConfig.source}. Call clickstack_list_sources to discover available source IDs.`, }, ], }; @@ -339,7 +339,7 @@ export async function runConfigTile( content: [ { type: 'text' as const, - text: `Connection not found: ${savedConfig.connection}. Call hyperdx_list_sources to discover available connection IDs.`, + text: `Connection not found: ${savedConfig.connection}. Call clickstack_list_sources to discover available connection IDs.`, }, ], }; diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index 1e11df8a37..47f757cea6 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -71,7 +71,7 @@ export async function runEventPatterns( content: [ { type: 'text' as const, - text: `Source not found: ${sourceId}. Call hyperdx_list_sources to discover available source IDs.`, + text: `Source not found: ${sourceId}. Call clickstack_list_sources to discover available source IDs.`, }, ], }; @@ -88,7 +88,7 @@ export async function runEventPatterns( content: [ { type: 'text' as const, - text: `Connection not found for source: ${sourceId}. Call hyperdx_list_sources to discover available source IDs.`, + text: `Connection not found for source: ${sourceId}. Call clickstack_list_sources to discover available source IDs.`, }, ], }; @@ -286,7 +286,7 @@ export async function runEventPatterns( // ── Format response ── // Convert trend timestamps to ISO strings, extract sample body texts, // and build a whereSnippet per pattern so the agent can drill into - // matching events via a follow-up hyperdx_search query. + // matching events via a follow-up clickstack_search query. const sampledCount = sampleRows.length; const slicedPatterns = rawPatterns.slice(0, topN); @@ -349,7 +349,7 @@ export async function runEventPatterns( (trendBuckets > 0 ? 'trend.count is similarly extrapolated from sample bucket counts. ' : '') + - 'Use whereSnippet as the "where" parameter in a hyperdx_search call to browse matching raw events.', + 'Use whereSnippet as the "where" parameter in a clickstack_search call to browse matching raw events.', }; const { data: trimmedOutput, isTrimmed } = trimToolResponse(output); diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 217a84c30d..5fe673da7f 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -125,7 +125,7 @@ export const endTimeSchema = z export const sourceIdSchema = z .string() .describe( - 'Source ID (required). Call hyperdx_list_sources to find available sources.', + 'Source ID (required). Call clickstack_list_sources to find available sources.', ); export const whereSchema = z diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 272a757a54..624639ffd7 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -46,21 +46,21 @@ export function registerSearch(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_search', + 'clickstack_search', { title: 'Search Events', description: 'Browse individual log/event/trace rows. ' + 'Use this when you need to see raw events, investigate specific log lines, ' + 'or drill into individual records matching a filter.\n\n' + - 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + - 'For aggregated metrics, use hyperdx_table instead. ' + - 'For pattern discovery, use hyperdx_event_patterns instead.\n\n' + + 'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' + + 'For aggregated metrics, use clickstack_table instead. ' + + 'For pattern discovery, use clickstack_event_patterns instead.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + "Map attributes use bracket syntax: SpanAttributes['http.method'].", inputSchema: searchSchema, }, - withToolTracing('hyperdx_search', context, async input => { + withToolTracing('clickstack_search', context, async input => { const timeRange = parseTimeRange(input.startTime, input.endTime); if ('error' in timeRange) { return { diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts index c6a096675c..4e7fede713 100644 --- a/packages/api/src/mcp/tools/query/sql.ts +++ b/packages/api/src/mcp/tools/query/sql.ts @@ -13,7 +13,7 @@ const sqlSchema = z.object({ .string() .describe( 'Connection ID (required). This is the only split tool that needs a connectionId ' + - 'instead of sourceId. Call hyperdx_list_sources to find available connections.', + 'instead of sourceId. Call clickstack_list_sources to find available connections.', ), sql: z .string() @@ -56,23 +56,23 @@ export function registerSql(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_sql', + 'clickstack_sql', { title: 'Raw SQL Query', description: 'Execute raw ClickHouse SQL. ' + 'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' + 'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' + - 'Requires connectionId (not sourceId) — call hyperdx_list_sources to find connections. ' + - 'Call hyperdx_describe_source to discover column names before writing SQL.\n\n' + + 'Requires connectionId (not sourceId) — call clickstack_list_sources to find connections. ' + + 'Call clickstack_describe_source to discover column names before writing SQL.\n\n' + 'Results are always returned as table rows — for time-series semantics, ' + 'include a time column and ORDER BY it in your SQL.\n\n' + - 'For standard aggregations use hyperdx_table. ' + - 'For time-series charts use hyperdx_timeseries. ' + - 'For browsing rows use hyperdx_search.', + 'For standard aggregations use clickstack_table. ' + + 'For time-series charts use clickstack_timeseries. ' + + 'For browsing rows use clickstack_search.', inputSchema: sqlSchema, }, - withToolTracing('hyperdx_sql', context, async input => { + withToolTracing('clickstack_sql', context, async input => { const timeRange = parseTimeRange(input.startTime, input.endTime); if ('error' in timeRange) { return { diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index cf1e15cb03..41b7e44f0b 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -154,14 +154,14 @@ export function registerTable(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_table', + 'clickstack_table', { title: 'Aggregation Table', description: 'Compute aggregated metrics as a table, single number, or pie chart. ' + 'Use this for grouped aggregations, top-N queries, single-value KPIs, ' + 'or proportional breakdowns.\n\n' + - 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + + 'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' + 'Use the top-level "where" to scope the entire query (e.g. filter by service). ' + 'Each select item can also have its own "where" for per-metric cohort ' + 'comparisons (compiles to If(...)). Both can be used together.\n\n' + @@ -173,7 +173,7 @@ export function registerTable(server: McpServer, context: McpContext) { 'it is transparently upgraded to "table".', inputSchema: tableSchema, }, - withToolTracing('hyperdx_table', context, async input => { + withToolTracing('clickstack_table', context, async input => { const timeRange = parseTimeRange(input.startTime, input.endTime); if ('error' in timeRange) { return { diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index 190ae2f58c..283effb48a 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -69,20 +69,20 @@ export function registerTimeseries(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_timeseries', + 'clickstack_timeseries', { title: 'Time-Series Chart', description: 'Plot metrics over time as a line or stacked bar chart. ' + 'Use this when you need to visualize trends, compare time-series, ' + 'or monitor metric changes over a time window.\n\n' + - 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first. ' + + 'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first. ' + 'Each select item defines one plotted series.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + "Map attributes use bracket syntax: SpanAttributes['http.method'].", inputSchema: timeseriesSchema, }, - withToolTracing('hyperdx_timeseries', context, async input => { + withToolTracing('clickstack_timeseries', context, async input => { const timeRange = parseTimeRange(input.startTime, input.endTime); if ('error' in timeRange) { return { diff --git a/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts b/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts index 90d33bb18b..663566c691 100644 --- a/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts +++ b/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts @@ -17,7 +17,7 @@ export function registerGetSavedSearch( const frontendUrl = config.FRONTEND_URL; server.registerTool( - 'hyperdx_get_saved_search', + 'clickstack_get_saved_search', { title: 'Get Saved Search(es)', description: @@ -34,7 +34,7 @@ export function registerGetSavedSearch( ), }), }, - withToolTracing('hyperdx_get_saved_search', context, async ({ id }) => { + withToolTracing('clickstack_get_saved_search', context, async ({ id }) => { // ── List all saved searches (slim query — only fetch the fields we need) ── if (!id) { const savedSearches = await SavedSearch.find( diff --git a/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts b/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts index e348eb21ff..b8f81020c2 100644 --- a/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts +++ b/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts @@ -21,16 +21,16 @@ export function registerSaveSavedSearch( const frontendUrl = config.FRONTEND_URL; server.registerTool( - 'hyperdx_save_saved_search', + 'clickstack_save_saved_search', { title: 'Create or Update Saved Search', description: 'Create a new saved search (omit id) or update an existing one (provide id). ' + 'A saved search stores a reusable query against a data source. ' + - 'Use hyperdx_list_sources to find the sourceId.', + 'Use clickstack_list_sources to find the sourceId.', inputSchema: mcpSaveSavedSearchSchema, }, - withToolTracing('hyperdx_save_saved_search', context, async input => { + withToolTracing('clickstack_save_saved_search', context, async input => { const isUpdate = !!input.id; // ── Validate ID for updates ── diff --git a/packages/api/src/mcp/tools/savedSearches/schemas.ts b/packages/api/src/mcp/tools/savedSearches/schemas.ts index d3348be3d9..bd5bbccbb3 100644 --- a/packages/api/src/mcp/tools/savedSearches/schemas.ts +++ b/packages/api/src/mcp/tools/savedSearches/schemas.ts @@ -60,7 +60,7 @@ export const mcpSaveSavedSearchSchema = z.object({ sourceId: z .string() .describe( - 'Source ID — call hyperdx_list_sources to find available sources.', + 'Source ID — call clickstack_list_sources to find available sources.', ), tags: z .array(z.string()) diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 8f2fefd5ee..a8249ae329 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -44,7 +44,7 @@ async function describeSourceSchema( content: [ { type: 'text' as const, - text: `Source "${sourceId}" not found. Call hyperdx_list_sources to see available source IDs.`, + text: `Source "${sourceId}" not found. Call clickstack_list_sources to see available source IDs.`, }, ], }; @@ -103,7 +103,7 @@ async function describeSourceSchema( { source: meta, nextSteps: { - query: `Use hyperdx_timeseries, hyperdx_table, or hyperdx_search with sourceId "${sourceId}" and the metric tables above.`, + query: `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}" and the metric tables above.`, }, }, null, @@ -291,7 +291,7 @@ async function describeSourceSchema( const lcValuesHint = skippedStages.length > 0 ? 'Value sampling was partially skipped due to timeout. ' + - 'The values shown may be incomplete — verify with a hyperdx_search query if needed.' + 'The values shown may be incomplete — verify with a clickstack_search query if needed.' : 'These are the REAL values in your data — use them in filters instead of guessing. ' + 'Example: where: "SeverityText:error" (if \'error\' appears in the sampled values above).'; @@ -305,7 +305,7 @@ async function describeSourceSchema( lowCardinalityValues: lcValuesHint, }, nextSteps: { - query: `Use hyperdx_timeseries, hyperdx_table, or hyperdx_search with sourceId "${sourceId}" and the columns/attributes above.`, + query: `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}" and the columns/attributes above.`, mapAttributeAccess: "Use bracket syntax for map columns: ResourceAttributes['service.name'], SpanAttributes['http.method']", }, @@ -335,14 +335,14 @@ export function registerDescribeSource( const { teamId } = context; server.registerTool( - 'hyperdx_describe_source', + 'clickstack_describe_source', { title: 'Describe Source Schema', description: 'CALL THIS BEFORE WRITING QUERIES — prevents unknown-column errors.\n\n' + 'Returns the full column schema, map-attribute keys, and sampled low-cardinality ' + 'values (e.g. SeverityText, StatusCode, ServiceName) for a single data source.\n\n' + - 'Workflow: call hyperdx_list_sources first to get source IDs, then call this tool ' + + 'Workflow: call clickstack_list_sources first to get source IDs, then call this tool ' + 'for each source you plan to query.\n\n' + 'Returns:\n' + '- columns[]: column name, ClickHouse type, and JS type\n' + @@ -356,12 +356,12 @@ export function registerDescribeSource( sourceId: z .string() .describe( - 'The source ID to describe. Get this from hyperdx_list_sources.', + 'The source ID to describe. Get this from clickstack_list_sources.', ), }), }, withToolTracing( - 'hyperdx_describe_source', + 'clickstack_describe_source', context, async ({ sourceId }) => { const controller = new AbortController(); @@ -388,7 +388,7 @@ export function registerDescribeSource( if (e instanceof Error && e.message === 'DESCRIBE_TIMEOUT') { logger.warn( { teamId, sourceId }, - 'hyperdx_describe_source timed out', + 'clickstack_describe_source timed out', ); return { isError: true, @@ -397,7 +397,7 @@ export function registerDescribeSource( type: 'text' as const, text: 'Schema discovery timed out. The ClickHouse server may be under load. ' + - 'Try again, or use hyperdx_list_sources for basic source info without schema details.', + 'Try again, or use clickstack_list_sources for basic source info without schema details.', }, ], }; diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index 7efe719d06..0c1e614689 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -15,21 +15,21 @@ export function registerListSources( const { teamId } = context; server.registerTool( - 'hyperdx_list_sources', + 'clickstack_list_sources', { title: 'List Sources & Connections', description: 'List all data sources (logs, metrics, traces) and database connections available to this team. ' + 'Returns source IDs, names, kinds, and connection IDs as a lightweight catalog.\n\n' + - 'NEXT STEP: After identifying the source(s) you need, call hyperdx_describe_source with the ' + + 'NEXT STEP: After identifying the source(s) you need, call clickstack_describe_source with the ' + 'sourceId to get the full column schema, attribute keys, and sampled values. ' + 'This two-step approach avoids fetching expensive schema details for sources you do not need.\n\n' + - 'NOTE: For most queries, use source IDs with hyperdx_timeseries, hyperdx_table, ' + - 'hyperdx_search, or hyperdx_event_patterns. ' + - 'Connection IDs are only needed for hyperdx_sql (raw ClickHouse SQL).', + 'NOTE: For most queries, use source IDs with clickstack_timeseries, clickstack_table, ' + + 'clickstack_search, or clickstack_event_patterns. ' + + 'Connection IDs are only needed for clickstack_sql (raw ClickHouse SQL).', inputSchema: z.object({}), }, - withToolTracing('hyperdx_list_sources', context, async () => { + withToolTracing('clickstack_list_sources', context, async () => { const [sources, connections] = await Promise.all([ getSources(teamId.toString()), getConnectionsByTeam(teamId.toString()), @@ -85,9 +85,9 @@ export function registerListSources( name: c.name, })), nextStep: - 'Call hyperdx_describe_source with a sourceId above to get the full column schema, ' + + 'Call clickstack_describe_source with a sourceId above to get the full column schema, ' + 'attribute keys, and sampled low-cardinality values before writing queries. ' + - 'connectionId is only needed for hyperdx_sql.', + 'connectionId is only needed for clickstack_sql.', }; return { content: [ diff --git a/packages/api/src/mcp/tools/trace/breakdown.ts b/packages/api/src/mcp/tools/trace/breakdown.ts index 7d4887ff01..0fd1a2f0d2 100644 --- a/packages/api/src/mcp/tools/trace/breakdown.ts +++ b/packages/api/src/mcp/tools/trace/breakdown.ts @@ -1,7 +1,7 @@ /** - * hyperdx_trace_top_time_consuming_operations + * clickstack_trace_top_time_consuming_operations * - * Aggregate counterpart to hyperdx_trace_waterfall. Given a parent-span + * Aggregate counterpart to clickstack_trace_waterfall. Given a parent-span * filter (a service + operation, optionally constrained to slow durations), * return the child operations consuming the most cumulative time across all * traces matching the parent filter — ranked by total_time_ms DESC. @@ -34,7 +34,7 @@ const traceBreakdownSchema = z.object({ .string() .describe( 'Trace source ID. Must be a source of kind="trace". ' + - 'Call hyperdx_list_sources to find available sources.', + 'Call clickstack_list_sources to find available sources.', ), parentFilter: z .string() @@ -102,7 +102,7 @@ export function registerTraceBreakdown(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', { title: 'Top Time-Consuming Operations Across Matching Traces', description: @@ -138,7 +138,7 @@ export function registerTraceBreakdown(server: McpServer, context: McpContext) { 'contained at least one such span), `p50_ms`, `p99_ms`. Plus a ' + '`summary` block with the matched-parent count.\n\n' + 'NEXT STEP after this tool: once a dominant slow child operation is ' + - 'identified, the canonical follow-up is hyperdx_event_deltas with ' + + 'identified, the canonical follow-up is clickstack_event_deltas with ' + 'slow-vs-fast spans of THAT child operation as target/baseline ' + "(target = {where: SpanName='' AND Duration > X}, " + "baseline = {where: SpanName='' AND Duration <= Y}). " + @@ -147,13 +147,13 @@ export function registerTraceBreakdown(server: McpServer, context: McpContext) { 'CROSS-SERVICE BREAKDOWN: this tool does NOT scope children to the ' + "parent's service. Slow cross-service calls (database, cache, " + 'upstream HTTP) surface naturally — useful for triage.\n\n' + - 'PAIR TOOL: hyperdx_trace_waterfall returns ONE concrete trace as a ' + + 'PAIR TOOL: clickstack_trace_waterfall returns ONE concrete trace as a ' + "parent/child tree. Use it for an example after this tool's " + 'aggregate breakdown has pointed you at the slow downstream operation.', inputSchema: traceBreakdownSchema, }, withToolTracing( - 'hyperdx_trace_top_time_consuming_operations', + 'clickstack_trace_top_time_consuming_operations', context, async (rawInput: TraceBreakdownInput) => { const input = traceBreakdownSchema.parse(rawInput); @@ -174,7 +174,7 @@ export function registerTraceBreakdown(server: McpServer, context: McpContext) { content: [ { type: 'text' as const, - text: `Source not found: ${input.sourceId}. Call hyperdx_list_sources to find available source IDs.`, + text: `Source not found: ${input.sourceId}. Call clickstack_list_sources to find available source IDs.`, }, ], }; @@ -185,7 +185,7 @@ export function registerTraceBreakdown(server: McpServer, context: McpContext) { content: [ { type: 'text' as const, - text: `Source ${input.sourceId} is kind="${source.kind}". hyperdx_trace_top_time_consuming_operations requires a source of kind="trace".`, + text: `Source ${input.sourceId} is kind="${source.kind}". clickstack_trace_top_time_consuming_operations requires a source of kind="trace".`, }, ], }; diff --git a/packages/api/src/mcp/tools/trace/waterfall.ts b/packages/api/src/mcp/tools/trace/waterfall.ts index b37354cac6..ba7293e2bc 100644 --- a/packages/api/src/mcp/tools/trace/waterfall.ts +++ b/packages/api/src/mcp/tools/trace/waterfall.ts @@ -23,7 +23,7 @@ const traceSchema = z.object({ .string() .describe( 'Trace source ID. Must be a source of kind "trace". ' + - 'Call hyperdx_list_sources to find available sources.', + 'Call clickstack_list_sources to find available sources.', ), traceId: z .string() @@ -169,7 +169,7 @@ export function registerTraceWaterfall(server: McpServer, context: McpContext) { const { teamId } = context; server.registerTool( - 'hyperdx_trace_waterfall', + 'clickstack_trace_waterfall', { title: 'Trace Waterfall (single trace)', description: @@ -179,7 +179,7 @@ export function registerTraceWaterfall(server: McpServer, context: McpContext) { 'the cascade for you instead of forcing the model to write self-JOINs in raw SQL.\n\n' + 'NOT THE RIGHT TOOL when the question is "where does the time go across MANY ' + 'slow traces of operation X" — that is the aggregate question, and the answer ' + - 'is hyperdx_trace_top_time_consuming_operations (called per affected ' + + 'is clickstack_trace_top_time_consuming_operations (called per affected ' + '(service, operation) with `minParentDurationMs`). Use this tool only when you ' + 'want a single concrete example to inspect, or after the aggregate breakdown ' + 'has already identified a suspicious operation.\n\n' + @@ -199,7 +199,7 @@ export function registerTraceWaterfall(server: McpServer, context: McpContext) { "Prefer this over running raw SQL with JOINs on TraceId — it uses the source's " + 'configured traceIdExpression / parentSpanIdExpression / spanIdExpression so ' + 'attribute extraction stays consistent with the rest of the platform.\n\n' + - 'PAIR TOOL: hyperdx_trace_top_time_consuming_operations is the aggregate ' + + 'PAIR TOOL: clickstack_trace_top_time_consuming_operations is the aggregate ' + 'counterpart — given a parent-span filter, it ranks child operations by total ' + 'time across MANY matching traces. Use that when the question is "where does ' + 'time go in slow X" (aggregate), and use THIS tool when the question is "show ' + @@ -207,7 +207,7 @@ export function registerTraceWaterfall(server: McpServer, context: McpContext) { inputSchema: traceSchema, }, withToolTracing( - 'hyperdx_trace_waterfall', + 'clickstack_trace_waterfall', context, async (rawInput: TraceInput) => { const input = traceSchema.parse(rawInput); @@ -227,7 +227,7 @@ export function registerTraceWaterfall(server: McpServer, context: McpContext) { content: [ { type: 'text' as const, - text: `Source not found: ${input.sourceId}. Call hyperdx_list_sources to find available source IDs.`, + text: `Source not found: ${input.sourceId}. Call clickstack_list_sources to find available source IDs.`, }, ], }; @@ -238,7 +238,7 @@ export function registerTraceWaterfall(server: McpServer, context: McpContext) { content: [ { type: 'text' as const, - text: `Source ${input.sourceId} is kind="${source.kind}". hyperdx_trace_waterfall requires a source of kind="trace".`, + text: `Source ${input.sourceId} is kind="${source.kind}". clickstack_trace_waterfall requires a source of kind="trace".`, }, ], }; diff --git a/packages/api/src/mcp/utils/tracing.ts b/packages/api/src/mcp/utils/tracing.ts index 38809bcd81..37f12870ec 100644 --- a/packages/api/src/mcp/utils/tracing.ts +++ b/packages/api/src/mcp/utils/tracing.ts @@ -5,7 +5,7 @@ import logger from '@/utils/logger'; import type { McpContext } from '../tools/types'; -const mcpTracer = opentelemetry.trace.getTracer('hyperdx-mcp', CODE_VERSION); +const mcpTracer = opentelemetry.trace.getTracer('clickstack-mcp', CODE_VERSION); type ToolResult = { content: { type: 'text'; text: string }[]; diff --git a/packages/common-utils/src/core/eventDeltas.ts b/packages/common-utils/src/core/eventDeltas.ts index 060b40d3bc..e969e4be57 100644 --- a/packages/common-utils/src/core/eventDeltas.ts +++ b/packages/common-utils/src/core/eventDeltas.ts @@ -1,6 +1,6 @@ /** * Event-deltas algorithm — pure functions used by both the HyperDX UI - * (DBDeltaChart) and the MCP server (hyperdx_event_deltas tool). + * (DBDeltaChart) and the MCP server (clickstack_event_deltas tool). * * Given two row sets — a target group and a baseline group — the algorithm * ranks each property by how differently its values are distributed between @@ -22,7 +22,7 @@ */ export function flattenData(data: Record): Record { const result: Record = {}; - // eslint-disable-next-line security/detect-object-injection -- prop is built from known object keys via recursion, not user input + function recurse(cur: Record, prop: string) { if (Object(cur) !== cur) { result[prop] = cur; // eslint-disable-line security/detect-object-injection From e3eeb504b21a816630b547d10e89d8dcabbbb059 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:45:03 -0400 Subject: [PATCH 010/728] fix: rollback lucene sql filter change (#2401) --- .changeset/thin-pets-relate.md | 6 - AGENTS.md | 13 - packages/api/src/utils/zod.ts | 2 +- packages/app/src/ChartUtils.tsx | 20 +- .../app/src/__tests__/searchFilters.test.ts | 445 +---------------- .../src/components/DBRowJsonViewer.test.tsx | 19 +- .../app/src/components/DBRowJsonViewer.tsx | 19 +- packages/app/src/components/DBRowTable.tsx | 16 - packages/app/src/components/EventTag.tsx | 6 +- .../components/__tests__/DBRowTable.test.tsx | 88 ---- .../__tests__/useDashboardFilters.test.tsx | 81 +--- .../app/src/hooks/useDashboardFilters.tsx | 56 +-- packages/app/src/searchFilters.tsx | 96 +--- .../features/dashboard-table-linking.spec.ts | 14 +- .../src/__tests__/colon-in-mapkey.test.ts | 112 ----- .../src/__tests__/filters.test.ts | 451 +++--------------- .../src/__tests__/metadata.test.ts | 24 - .../src/__tests__/queryParser.test.ts | 13 - .../src/core/__tests__/linkUrlBuilder.test.ts | 36 +- packages/common-utils/src/core/metadata.ts | 23 +- packages/common-utils/src/filters.ts | 163 +------ packages/common-utils/src/queryParser.ts | 25 +- 22 files changed, 196 insertions(+), 1532 deletions(-) delete mode 100644 .changeset/thin-pets-relate.md delete mode 100644 packages/common-utils/src/__tests__/colon-in-mapkey.test.ts diff --git a/.changeset/thin-pets-relate.md b/.changeset/thin-pets-relate.md deleted file mode 100644 index 71fe2ca5b1..0000000000 --- a/.changeset/thin-pets-relate.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch ---- - -fix: make filters aware of map column return arrayElement diff --git a/AGENTS.md b/AGENTS.md index 6455078d16..70dbe826a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,19 +71,6 @@ and lint issues across all packages. Pre-commit hooks handle this when committing, but if you finish edits without committing, run `yarn lint:fix` before stopping. -## Before Writing New Utility Functions - -Before implementing a new parsing, formatting, or transformation function, -search for existing functions that already do the same thing or something close. -Common places to check: - -- `packages/common-utils/src/core/metadata.ts` — key/column parsing utilities - (e.g. `parseKeyPath`) -- `packages/common-utils/src/queryParser.ts` — Lucene parsing (`parse`, - `decodeSpecialTokens`), AST type guards, serialization helpers -- `packages/common-utils/src/filters.ts` — filter state serialization and - parsing (`filtersToQuery`, `parseLuceneFilter`) - ## Key Principles 1. **Multi-tenancy**: All data is scoped to `Team` - ensure proper filtering diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index bb6906e674..b2d3e53ebb 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -146,7 +146,7 @@ export type ExternalDashboardFilter = z.infer< >; export const externalDashboardSavedFilterValueSchema = z.object({ - type: z.enum(['sql', 'lucene']).optional().default('sql'), + type: z.literal('sql').optional().default('sql'), condition: z.string().max(10000), }); diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index a2a9a05c4b..544c6762d5 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -17,10 +17,6 @@ import { getAlignedDateRange, Granularity, } from '@hyperdx/common-utils/dist/core/utils'; -import { - type FilterState, - filtersToQuery, -} from '@hyperdx/common-utils/dist/filters'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunction as AggFnV2, @@ -927,19 +923,13 @@ export function buildEventsSearchUrl({ // Add group-by column filters if (groupFilters && groupFilters.length > 0) { - const filterState: FilterState = {}; - for (const { column, value } of groupFilters) { + groupFilters.forEach(({ column, value }) => { if (column && value != null) { - if (!filterState[column]) { - filterState[column] = { - included: new Set(), - excluded: new Set(), - }; - } - filterState[column].included.add(String(value)); + // Can't use SQLString.escape here because the search endpoint relies on exist match for UI + const condition = `${column} IN (${SqlString.escape(value)})`; + additionalFilters.push({ type: 'sql', condition }); } - } - additionalFilters.push(...filtersToQuery(filterState)); + }); } // Add Y-axis value range filter (±threshold) for charts diff --git a/packages/app/src/__tests__/searchFilters.test.ts b/packages/app/src/__tests__/searchFilters.test.ts index 1d30fc7314..ec91631c03 100644 --- a/packages/app/src/__tests__/searchFilters.test.ts +++ b/packages/app/src/__tests__/searchFilters.test.ts @@ -1,11 +1,5 @@ import { enableMapSet } from 'immer'; -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { filtersToQuery } from '@hyperdx/common-utils/dist/filters'; -import { - CustomSchemaSQLSerializerV2, - SearchQueryBuilder, -} from '@hyperdx/common-utils/dist/queryParser'; import { act, renderHook } from '@testing-library/react'; import { @@ -636,222 +630,6 @@ describe('searchFilters', () => { }); }); - describe('round-trip: Lucene filters', () => { - it('round-trips Map filter with single included value', () => { - const filters = { - "LogAttributes['service.name']": { - included: new Set(['my-app']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - expect(query).toEqual([ - { - type: 'lucene', - condition: 'LogAttributes.service.name:"my-app"', - }, - ]); - // After round-trip, bracket-notation key becomes dot notation - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.service.name': { - included: new Set(['my-app']), - excluded: new Set(), - }, - }); - }); - - it('round-trips Map filter with multiple included values', () => { - const filters = { - "ResourceAttributes['env']": { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'ResourceAttributes.env': { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, - }); - }); - - it('round-trips Map filter with excluded values', () => { - const filters = { - "LogAttributes['level']": { - included: new Set(), - excluded: new Set(['debug', 'trace']), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.level': { - included: new Set(), - excluded: new Set(['debug', 'trace']), - }, - }); - }); - - it('round-trips values containing double quotes', () => { - const filters = { - "LogAttributes['msg']": { - included: new Set(['say "hello"']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.msg': { - included: new Set(['say "hello"']), - excluded: new Set(), - }, - }); - }); - - it('round-trips values containing backslashes', () => { - const filters = { - "LogAttributes['path']": { - included: new Set(['C:\\path\\to\\file']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.path': { - included: new Set(['C:\\path\\to\\file']), - excluded: new Set(), - }, - }); - }); - - it('round-trips plain key filters', () => { - const filters = { - service: { - included: new Set(['app']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - service: { - included: new Set(['app']), - excluded: new Set(), - }, - }); - }); - - it('round-trips boolean filter values', () => { - const filters = { - isRootSpan: { - included: new Set([true]), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - isRootSpan: { - included: new Set([true]), - excluded: new Set(), - }, - }); - // Verify .has(true) works (boolean, not string) - expect(parsed.filters.isRootSpan.included.has(true)).toBe(true); - expect(parsed.filters.isRootSpan.included.has('true')).toBe(false); - }); - - it('round-trips empty string values', () => { - const filters = { - tag: { - included: new Set(['']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters.tag.included).toEqual(new Set([''])); - }); - - it('round-trips values with Lucene reserved characters', () => { - const filters = { - query: { - included: new Set(['(foo) AND [bar]']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters.query.included).toEqual( - new Set(['(foo) AND [bar]']), - ); - }); - - it('round-trips range filters through Lucene', () => { - const filters = { - duration: { - included: new Set(), - excluded: new Set(), - range: { min: 10, max: 500 }, - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters.duration.range).toEqual({ min: 10, max: 500 }); - }); - - it('merges mixed sql and lucene filters for the same key', () => { - const parsed = parseQuery([ - { type: 'sql', condition: `service IN ('app')` }, - { type: 'lucene', condition: 'service:"web"' }, - ]); - expect(parsed.filters.service.included).toEqual(new Set(['app', 'web'])); - }); - - it('parses existing lucene filter from URL/API', () => { - const parsed = parseQuery([ - { - type: 'lucene', - condition: - 'LogAttributes.service:"app-1" OR LogAttributes.service:"app-2"', - }, - ]); - expect(parsed.filters).toEqual({ - 'LogAttributes.service': { - included: new Set(['app-1', 'app-2']), - excluded: new Set(), - }, - }); - }); - - it('preserves unparseable Lucene filters as passthrough', () => { - const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); - const parsed = parseQuery([ - { type: 'lucene', condition: '((((' }, - { type: 'lucene', condition: 'service:"app"' }, - ]); - // Valid filter is parsed - expect(parsed.filters.service.included).toEqual(new Set(['app'])); - // Invalid filter is preserved in passthroughFilters - expect(parsed.passthroughFilters).toEqual([ - { type: 'lucene', condition: '((((' }, - ]); - warnSpy.mockRestore(); - }); - - it('returns empty array for valid but non-quoted Lucene (no passthrough)', () => { - const parsed = parseQuery([{ type: 'lucene', condition: 'service:foo' }]); - // Unquoted term — valid Lucene but not a filter, nothing extracted - expect(Object.keys(parsed.filters)).toHaveLength(0); - expect(parsed.passthroughFilters).toHaveLength(0); - }); - }); - describe('useSearchPageFilterState', () => { const onFilterChange = jest.fn(); @@ -869,15 +647,20 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'service:"app"' }, - { type: 'lucene', condition: 'level:"error"' }, + { + type: 'sql', + condition: "service IN ('app')", + }, + { + type: 'sql', + condition: "level IN ('error')", + }, ]); }); - it('updating filter query from legacy sql input', () => { + it('updating filter query', () => { const { result } = renderHook(() => useSearchPageFilterState({ - // Legacy sql filters are still parsed correctly searchQuery: [ { type: 'sql', condition: `service IN ('hdx-oss-dev-app')` }, { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, @@ -892,14 +675,10 @@ describe('searchFilters', () => { result.current.setFilterValue('another_facet', 'some_value'); }); - // Output is now lucene after round-trip expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'hyperdx_event_type:"span"' }, - { - type: 'lucene', - condition: '(level:"info" OR level:"ok")', - }, - { type: 'lucene', condition: 'another_facet:"some_value"' }, + { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, + { type: 'sql', condition: `level IN ('info', 'ok')` }, + { type: 'sql', condition: `another_facet IN ('some_value')` }, ]); }); @@ -920,12 +699,12 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'service:"hdx-oss-dev-app"' }, - { type: 'lucene', condition: 'hyperdx_event_type:"span"' }, + { type: 'sql', condition: `service IN ('hdx-oss-dev-app')` }, + { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, ]); }); - it('correctly hydrates filter state from sql query', () => { + it('correctly hydrates filter state from query', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ @@ -966,43 +745,11 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenCalledWith([ - { type: 'lucene', condition: 'service:"app"' }, - { type: 'lucene', condition: 'level:"info"' }, - ]); - }); - - it('migrates legacy SQL filters to Lucene on load', () => { - const onFilterChangeMigrate = jest.fn(); - renderHook(() => - useSearchPageFilterState({ - searchQuery: [ - { type: 'sql', condition: `service IN ('app')` }, - { type: 'sql', condition: `level IN ('info', 'ok')` }, - ], - onFilterChange: onFilterChangeMigrate, - }), - ); - - // Migration fires on mount, converting SQL to Lucene - expect(onFilterChangeMigrate).toHaveBeenCalledWith([ - { type: 'lucene', condition: 'service:"app"' }, - { type: 'lucene', condition: '(level:"info" OR level:"ok")' }, + { type: 'sql', condition: `service IN ('app')` }, + { type: 'sql', condition: `level IN ('info')` }, // Should only have the included value, no excluded values ]); }); - it('does not migrate when filters are already Lucene', () => { - const onFilterChangeNoMigrate = jest.fn(); - renderHook(() => - useSearchPageFilterState({ - searchQuery: [{ type: 'lucene', condition: 'service:"app"' }], - onFilterChange: onFilterChangeNoMigrate, - }), - ); - - // No migration needed — onFilterChange should not be called - expect(onFilterChangeNoMigrate).not.toHaveBeenCalled(); - }); - describe('retainFiltersByColumns', () => { it('returns [] and does not touch URL when filter state is empty', () => { const onFilterChangeLocal = jest.fn(); @@ -1078,8 +825,8 @@ describe('searchFilters', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ - { type: 'lucene', condition: 'OldColumn:"x"' }, - { type: 'lucene', condition: 'AnotherGone:"y"' }, + { type: 'sql', condition: `OldColumn IN ('x')` }, + { type: 'sql', condition: `AnotherGone IN ('y')` }, ], onFilterChange: onFilterChangeLocal, }), @@ -1101,8 +848,8 @@ describe('searchFilters', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ - { type: 'lucene', condition: 'ServiceName:"app"' }, - { type: 'lucene', condition: 'Body:"oops"' }, + { type: 'sql', condition: `ServiceName IN ('app')` }, + { type: 'sql', condition: `Body IN ('oops')` }, ], onFilterChange: onFilterChangeLocal, }), @@ -1117,157 +864,9 @@ describe('searchFilters', () => { expect(dropped).toEqual(['Body']); expect(onFilterChangeLocal).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'ServiceName:"app"' }, - ]); - }); - - it('preserves passthrough filters when reconciling', () => { - const onFilterChangeLocal = jest.fn(); - const warnSpy = jest - .spyOn(console, 'warn') - .mockImplementation(() => {}); - // Unparseable Lucene (mismatched parens) is preserved by parseQuery - // as a passthrough filter. - const { result } = renderHook(() => - useSearchPageFilterState({ - searchQuery: [ - { type: 'lucene', condition: 'ServiceName:"app"' }, - { type: 'lucene', condition: 'Body:"oops"' }, - { type: 'lucene', condition: '((((' }, - ], - onFilterChange: onFilterChangeLocal, - }), - ); - - let dropped: string[] = []; - act(() => { - dropped = result.current.retainFiltersByColumns( - new Set(['ServiceName']), - ); - }); - - expect(dropped).toEqual(['Body']); - // Passthrough filter rides along after the kept filters. - expect(onFilterChangeLocal).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'ServiceName:"app"' }, - { type: 'lucene', condition: '((((' }, + { type: 'sql', condition: `ServiceName IN ('app')` }, ]); - warnSpy.mockRestore(); - }); - }); - }); - - describe('filters use direct_read optimization', () => { - const metadata = getMetadata( - new ClickhouseClient({ host: 'http://localhost:8123' }), - ); - - metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => { - if (column === 'LogAttributes') { - return { name: 'LogAttributes', type: 'Map(String, String)' }; - } - return undefined; - }); - metadata.getMaterializedColumnsLookupTable = jest - .fn() - .mockResolvedValue(new Map()); - metadata.getColumns = jest.fn().mockResolvedValue([ - { - name: 'LogAttributes', - type: 'Map(String, String)', - default_type: '', - default_expression: '', - }, - { - name: 'LogAttributeItems', - type: 'Array(String)', - default_type: 'ALIAS', - default_expression: - "arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String)))", - }, - ]); - metadata.getSkipIndices = jest.fn().mockResolvedValue([ - { - name: 'idx_log_attr_items', - type: 'text', - typeFull: 'text(tokenizer=array)', - expression: 'LogAttributeItems', - granularity: 1, - }, - ]); - metadata.getSetting = jest.fn().mockResolvedValue('0'); - metadata.getServerVersion = jest - .fn() - .mockResolvedValue([26, 5, 0, 0] as const); - - const serializer = new CustomSchemaSQLSerializerV2({ - metadata, - databaseName: 'default', - tableName: 'otel_logs', - connectionId: 'test', - }); - - it('single included filter value uses has() direct_read', async () => { - const filters = filtersToQuery({ - "LogAttributes['service.name']": { - included: new Set(['my-app']), - excluded: new Set(), - }, - }); - - const condition = filters[0]; - expect(condition.type).toBe('lucene'); - const builder = new SearchQueryBuilder( - (condition as { condition: string }).condition, - serializer, - ); - const sql = await builder.build(); - expect(sql).toContain( - "has(`LogAttributeItems`, concat('service.name', '=', 'my-app'))", - ); - }); - - it('multiple included filter values each use has() direct_read', async () => { - const filters = filtersToQuery({ - "LogAttributes['env']": { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, }); - - const condition = filters[0]; - expect(condition.type).toBe('lucene'); - const builder = new SearchQueryBuilder( - (condition as { condition: string }).condition, - serializer, - ); - const sql = await builder.build(); - expect(sql).toContain( - "has(`LogAttributeItems`, concat('env', '=', 'prod'))", - ); - expect(sql).toContain( - "has(`LogAttributeItems`, concat('env', '=', 'staging'))", - ); - }); - - it('excluded filter value uses NOT has() direct_read', async () => { - const filters = filtersToQuery({ - "LogAttributes['level']": { - included: new Set(), - excluded: new Set(['debug']), - }, - }); - - const condition = filters[0]; - expect(condition.type).toBe('lucene'); - const builder = new SearchQueryBuilder( - (condition as { condition: string }).condition, - serializer, - ); - const sql = await builder.build(); - expect(sql).toContain( - "NOT has(`LogAttributeItems`, concat('level', '=', 'debug'))", - ); }); }); }); diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index e889a6df69..161a5abfe6 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -53,10 +53,10 @@ describe('DBRowJsonViewer', () => { }); // Helper to render component - const renderComponent = (data: any, jsonColumns?: string[]) => { + const renderComponent = (data: any) => { return renderWithMantine( - + , ); }; @@ -133,21 +133,6 @@ describe('DBRowJsonViewer', () => { ); }); - it('adds filters on JSON-typed column as plain dot notation (no toString, no backticks)', () => { - const jsonData = { - Metadata: { - endpoint: 'grpc://thingy:8888/Verify', - }, - }; - renderComponent(jsonData, ['Metadata']); - clickLineButton('endpoint', 'Add to Filters'); - - expect(mockOnPropertyAddClick).toHaveBeenCalledWith( - 'Metadata.endpoint', - 'grpc://thingy:8888/Verify', - ); - }); - it('toggles columns with correct path formatting', () => { renderComponent(logData); clickLineButton('field1', 'Column'); diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index e6aea39c5f..65abd56115 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -27,7 +27,6 @@ import { IconTextWrap, } from '@tabler/icons-react'; -import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; import HyperJson, { GetLineActions, LineAction } from '@/components/HyperJson'; import { mergePath } from '@/utils'; import { @@ -371,6 +370,8 @@ export function DBRowJsonViewer({ ({ keyPath, value, isInParsedJson, parsedJsonRootPath }) => { const actions: LineAction[] = []; const fieldPath = mergePath(keyPath, jsonColumns); + const isJsonColumn = + keyPath.length > 0 && jsonColumns?.includes(keyPath[0]); // Add to Filters action (strings only) // FIXME: TOTAL HACK To disallow adding timestamp to filters @@ -393,6 +394,7 @@ export function DBRowJsonViewer({ onClick: () => { let filterFieldPath = fieldPath; + // Handle parsed JSON from string columns using JSONExtractString if (isInParsedJson && parsedJsonRootPath) { const jsonQuery = buildJSONExtractQuery( keyPath, @@ -401,14 +403,23 @@ export function DBRowJsonViewer({ ); if (jsonQuery) { filterFieldPath = jsonQuery; + } else { + // We're at the root of the parsed JSON, treat as string + filterFieldPath = isJsonColumn + ? `toString(${fieldPath})` + : fieldPath; } + } else { + // Regular JSON column or non-JSON field + filterFieldPath = isJsonColumn + ? `toString(${fieldPath})` + : fieldPath; } - const cleanedPath = cleanClickHouseExpression(filterFieldPath); - onPropertyAddClick(cleanedPath, value); + onPropertyAddClick(filterFieldPath, value); notifications.show({ color: 'green', - message: `Added "${cleanedPath} = ${value}" to filters`, + message: `Added "${fieldPath} = ${value}" to filters`, }); }, }); diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 53779e8d1d..9d2c8c31e6 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -29,7 +29,6 @@ import { isJSDataTypeJSONStringifiable, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, @@ -1509,18 +1508,6 @@ function selectColumnMapWithoutAdditionalKeys( ); } -export function addMapAliasesToSelect(select: string): string { - const selects = splitAndTrimWithBracket(select); - for (let i = 0; i < selects.length; i++) { - const key = selects[i]; - const path = parseKeyPath(key); - if (path.length === 2) { - selects[i] = `${key} as "${path[0]}['${path[1]}']"`; - } - } - return selects.join(','); -} - export type DBRowTableVariant = 'default' | 'muted'; function DBSqlRowTableComponent({ @@ -1615,9 +1602,6 @@ function DBSqlRowTableComponent({ ...searchChartConfigDefaults(me?.team), ...config, }; - if (typeof base.select === 'string') { - base.select = addMapAliasesToSelect(base.select); - } if (orderByArray.length) { base.orderBy = orderByArray.map(o => { return { diff --git a/packages/app/src/components/EventTag.tsx b/packages/app/src/components/EventTag.tsx index d2ae465604..9cb6177fb0 100644 --- a/packages/app/src/components/EventTag.tsx +++ b/packages/app/src/components/EventTag.tsx @@ -5,7 +5,6 @@ import { SearchConditionLanguage } from '@hyperdx/common-utils/dist/types'; import { Button, Group, Popover, Stack, Text, Tooltip } from '@mantine/core'; import { IconCirclePlus, IconLink, IconSearch } from '@tabler/icons-react'; -import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; import { isLinkableUrl } from '@/utils/highlightedAttributes'; export default function EventTag({ @@ -106,10 +105,7 @@ export default function EventTag({ size="xs" rightSection={} onClick={() => { - onPropertyAddClick( - cleanClickHouseExpression(sqlExpression), - value, - ); + onPropertyAddClick(sqlExpression, value); setOpened(false); }} > diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index 03e8a80db6..ffb459b2f5 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -2,7 +2,6 @@ import { screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { - addMapAliasesToSelect, appendSelectWithAdditionalKeys, RawLogTable, } from '@/components/DBRowTable'; @@ -417,90 +416,3 @@ describe('appendSelectWithAdditionalKeys', () => { }); }); }); - -describe('addMapAliasesToSelect', () => { - it('leaves plain columns unchanged and collapses ", " to ","', () => { - expect(addMapAliasesToSelect('Timestamp, ServiceName, SeverityText')).toBe( - 'Timestamp,ServiceName,SeverityText', - ); - }); - - it('aliases a single-quoted bracket-form map key', () => { - expect(addMapAliasesToSelect("ResourceAttributes['service.name']")).toBe( - `ResourceAttributes['service.name'] as "ResourceAttributes['service.name']"`, - ); - }); - - it('aliases only the map keys in a mixed multi-column input', () => { - expect( - addMapAliasesToSelect( - "Timestamp, ResourceAttributes['service.name'], SeverityText", - ), - ).toBe( - `Timestamp,ResourceAttributes['service.name'] as "ResourceAttributes['service.name']",SeverityText`, - ); - }); - - it('preserves function-call expressions whose arguments contain commas', () => { - expect( - addMapAliasesToSelect( - "Timestamp, toDateTime(now64(3, 'UTC')), ServiceName", - ), - ).toBe(`Timestamp,toDateTime(now64(3, 'UTC')),ServiceName`); - }); - - it('aliases an arrayElement(Col, key) Map subscript', () => { - expect( - addMapAliasesToSelect("arrayElement(ResourceAttributes, 'service.name')"), - ).toBe( - `arrayElement(ResourceAttributes, 'service.name') as "ResourceAttributes['service.name']"`, - ); - }); - - it('passes already-aliased plain columns through unchanged', () => { - expect(addMapAliasesToSelect('Timestamp as ts, ServiceName as svc')).toBe( - 'Timestamp as ts,ServiceName as svc', - ); - }); - - it('does NOT re-alias a bracket-form map subscript that already carries an "as " clause', () => { - expect( - addMapAliasesToSelect("ResourceAttributes['service.name'] as svc"), - ).toBe("ResourceAttributes['service.name'] as svc"); - }); - - it('does NOT re-alias an already-aliased map subscript within a multi-column input', () => { - expect( - addMapAliasesToSelect( - "Timestamp, ResourceAttributes['service.name'] as svc, SeverityText", - ), - ).toBe("Timestamp,ResourceAttributes['service.name'] as svc,SeverityText"); - }); - - it('does NOT alias unbracketed dot-notation tokens', () => { - expect(addMapAliasesToSelect('user_data.name')).toBe('user_data.name'); - expect(addMapAliasesToSelect('user_data.address.city')).toBe( - 'user_data.address.city', - ); - }); - - it('does NOT alias multi-segment dotted native columns', () => { - expect(addMapAliasesToSelect('SpanAttributes.k8s.resource.name')).toBe( - 'SpanAttributes.k8s.resource.name', - ); - }); - - it('does NOT alias toString(x).y attribute-access expressions', () => { - expect(addMapAliasesToSelect('toString(SpanAttributes).k8s')).toBe( - 'toString(SpanAttributes).k8s', - ); - }); - - it('does NOT alias backtick-quoted dotted JSON paths', () => { - expect(addMapAliasesToSelect('`host`.`arch`')).toBe('`host`.`arch`'); - }); - - it('returns an empty string for empty input', () => { - expect(addMapAliasesToSelect('')).toBe(''); - }); -}); diff --git a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx index 2aa490a135..baa8904b3b 100644 --- a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx @@ -90,7 +90,7 @@ describe('useDashboardFilters', () => { ); }); - it('should generate lucene condition for multi-select values', () => { + it('should generate IN clause for multi-select values', () => { const { result } = renderHook(() => useDashboardFilters(mockFilters)); act(() => { @@ -105,7 +105,7 @@ describe('useDashboardFilters', () => { const query = result2.current.filterQueries[0]; const condition = 'condition' in query ? query.condition : ''; expect(condition).toEqual( - '(environment:"production" OR environment:"staging")', + "toString(environment) IN ('production', 'staging')", ); }); @@ -272,81 +272,4 @@ describe('useDashboardFilters', () => { expect(result.current.ignoredFilterExpressions).toEqual([]); }); }); - - it('should match bracket-notation expressions after Lucene round-trip', () => { - const bracketFilters: DashboardFilter[] = [ - { - id: 'filter-bracket', - type: 'QUERY_EXPRESSION', - name: 'Pod', - expression: "SpanAttributes['k8s.pod.name']", - source: 'traces', - }, - ]; - - const { result } = renderHook(() => useDashboardFilters(bracketFilters)); - - act(() => { - result.current.setFilterValue("SpanAttributes['k8s.pod.name']", [ - 'pod-1', - ]); - }); - - const { result: result2 } = renderHook(() => - useDashboardFilters(bracketFilters), - ); - - // The bracket-notation expression should still match after the Lucene - // round-trip converts the key to dot notation internally. - expect( - result2.current.filterValues["SpanAttributes['k8s.pod.name']"]?.included, - ).toEqual(new Set(['pod-1'])); - expect(result2.current.ignoredFilterExpressions).toEqual([]); - }); - - it('should match dot-notation URL key to bracket-notation expression', () => { - const bracketFilters: DashboardFilter[] = [ - { - id: 'filter-bracket', - type: 'QUERY_EXPRESSION', - name: 'Pod', - expression: "SpanAttributes['k8s.pod.name']", - source: 'traces', - }, - ]; - - // Pre-seed URL state with a dot-notation Lucene filter (as would be - // stored after a round-trip through filtersToQuery → parseQuery). - mockState = [ - { - type: 'lucene', - condition: 'SpanAttributes.k8s.pod.name:"pod-1"', - }, - ]; - - const { result } = renderHook(() => useDashboardFilters(bracketFilters)); - - expect( - result.current.filterValues["SpanAttributes['k8s.pod.name']"]?.included, - ).toEqual(new Set(['pod-1'])); - expect(result.current.ignoredFilterExpressions).toEqual([]); - }); - - it('should migrate legacy SQL filters to Lucene on load', () => { - // Pre-seed URL with old-format SQL filters - mockState = [{ type: 'sql', condition: "environment IN ('production')" }]; - - renderHook(() => useDashboardFilters(mockFilters)); - - // Migration should have called setFilterQueries with Lucene format - expect(mockSetState).toHaveBeenCalled(); - const lastCall = - mockSetState.mock.calls[mockSetState.mock.calls.length - 1]; - // setFilterQueries receives an updater or a value; resolve it - const result = - typeof lastCall[0] === 'function' ? lastCall[0](mockState) : lastCall[0]; - expect(result).toEqual([ - { type: 'lucene', condition: 'environment:"production"' }, - ]); - }); }); diff --git a/packages/app/src/hooks/useDashboardFilters.tsx b/packages/app/src/hooks/useDashboardFilters.tsx index 093cf84254..53211b0000 100644 --- a/packages/app/src/hooks/useDashboardFilters.tsx +++ b/packages/app/src/hooks/useDashboardFilters.tsx @@ -1,6 +1,5 @@ -import { useCallback, useEffect, useMemo, useRef } from 'react'; +import { useCallback, useMemo } from 'react'; import { useQueryState } from 'nuqs'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, filtersToQuery, @@ -22,19 +21,19 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { (expression: string, values: string[]) => { setFilterQueries(prev => { const { filters: filterValues } = parseQuery(prev ?? []); - // Normalize the expression to dot notation so it matches the keys - // returned by parseQuery (which converts bracket notation to dots). - const key = parseKeyPath(expression).join('.'); if (values.length === 0) { - delete filterValues[key]; + delete filterValues[expression]; } else { - filterValues[key] = { + filterValues[expression] = { included: new Set(values), excluded: new Set(), }; } - return filtersToQuery(filterValues); + return filtersToQuery( + filterValues, + { stringifyKeys: false }, // Don't wrap keys with toString(), to preserve exact key names in URL query parameters + ); }); }, [setFilterQueries], @@ -48,28 +47,16 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { } = useMemo(() => { const { filters: parsedFilters } = parseQuery(filterQueries ?? []); const valuesForExistingFilters: FilterState = {}; + const knownExpressions = new Set(filters.map(f => f.expression)); const ignored: string[] = []; - // Build a normalized lookup so bracket-notation expressions - // (e.g. SpanAttributes['k8s.pod.name']) match the dot-notation keys - // returned by parseLuceneFilter (e.g. SpanAttributes.k8s.pod.name). - const normalizeKey = (k: string) => parseKeyPath(k).join('.'); - const normalizedParsed = new Map( - Object.entries(parsedFilters).map(([k, v]) => [normalizeKey(k), v]), - ); - const knownNormalized = new Set( - filters.map(f => normalizeKey(f.expression)), - ); - for (const { expression } of filters) { - const norm = normalizeKey(expression); - const match = normalizedParsed.get(norm); - if (match) { - valuesForExistingFilters[expression] = match; + if (expression in parsedFilters) { + valuesForExistingFilters[expression] = parsedFilters[expression]; } } for (const key of Object.keys(parsedFilters)) { - if (!knownNormalized.has(normalizeKey(key))) { + if (!knownExpressions.has(key)) { ignored.push(key); } } @@ -88,7 +75,12 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { return { valuesForExistingFilters, - queriesForExistingFilters: filtersToQuery(valuesForExistingFilters), + queriesForExistingFilters: filtersToQuery( + valuesForExistingFilters, + // Wrap keys in `toString()` to support JSON/Dynamic-type columns. + // All keys can be stringified, since filter select values are stringified as well. + { stringifyKeys: true }, + ), ignoredExpressions: ignored, filtersByExpression, }; @@ -123,20 +115,6 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { [valuesForExistingFilters, filtersByExpression], ); - // Migrate legacy SQL filters in the URL to Lucene on load - const hasMigratedRef = useRef(false); - useEffect(() => { - if (hasMigratedRef.current || !filterQueries) return; - const hasSqlFilters = filterQueries.some( - f => 'condition' in f && f.type === 'sql', - ); - if (hasSqlFilters) { - hasMigratedRef.current = true; - const { filters: parsed, passthroughFilters } = parseQuery(filterQueries); - setFilterQueries([...filtersToQuery(parsed), ...passthroughFilters]); - } - }, [filterQueries, setFilterQueries]); - return { filterValues: valuesForExistingFilters, filterQueries: queriesForExistingFilters, diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index 23ec18ba42..28d8fa50ae 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -1,11 +1,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import produce from 'immer'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { - coerceBooleanValue, type FilterState, filtersToQuery, - parseLuceneFilter, } from '@hyperdx/common-utils/dist/filters'; import type { Filter } from '@hyperdx/common-utils/dist/types'; @@ -300,8 +297,6 @@ export const parseQuery = ( q: Filter[], ): { filters: FilterState; - /** Filters that could not be parsed into FilterState (preserved for re-emission) */ - passthroughFilters: Filter[]; } => { const state = new Map< string, @@ -311,29 +306,7 @@ export const parseQuery = ( range?: { min: number; max: number }; } >(); - const passthroughFilters: Filter[] = []; for (const filter of q) { - if (filter.type === 'lucene') { - const parsedFields = parseLuceneFilter(filter.condition); - if (parsedFields === undefined) { - // Parse failure — preserve so it isn't silently lost - if (filter.condition.trim()) { - passthroughFilters.push(filter); - } - } else { - for (const { key, included, excluded, range } of parsedFields) { - if (!state.has(key)) { - state.set(key, { included: new Set(), excluded: new Set() }); - } - const sets = state.get(key)!; - for (const v of included) sets.included.add(v); - for (const v of excluded) sets.excluded.add(v); - if (range) sets.range = range; - } - } - continue; - } - if (filter.type !== 'sql') continue; // Check for BETWEEN condition (only when BETWEEN appears outside quotes) @@ -379,7 +352,7 @@ export const parseQuery = ( }); } } - return { filters: Object.fromEntries(state), passthroughFilters }; + return { filters: Object.fromEntries(state) }; }; export const useSearchPageFilterState = ({ @@ -394,7 +367,7 @@ export const useSearchPageFilterState = ({ return parseQuery(searchQuery); } catch (e) { console.error(e); - return { filters: {}, passthroughFilters: [] }; + return { filters: {} }; } }, [searchQuery]); @@ -408,32 +381,11 @@ export const useSearchPageFilterState = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [parsedQuery.filters]); - // Migrate legacy SQL filters to Lucene on load so URLs become canonical - const hasMigratedRef = useRef(false); - useEffect(() => { - if (hasMigratedRef.current) return; - const hasSqlFilters = searchQuery?.some( - f => 'condition' in f && f.type === 'sql', - ); - if (hasSqlFilters && Object.keys(parsedQuery.filters).length > 0) { - hasMigratedRef.current = true; - onFilterChange([ - ...filtersToQuery(parsedQuery.filters), - ...parsedQuery.passthroughFilters, - ]); - } - }, [searchQuery, parsedQuery, onFilterChange]); - const updateFilterQuery = useCallback( (newFilters: FilterState) => { - // Preserve unparseable filters so they aren't silently lost when - // the user toggles a sidebar facet. - onFilterChange([ - ...filtersToQuery(newFilters), - ...parsedQuery.passthroughFilters, - ]); + onFilterChange(filtersToQuery(newFilters)); }, - [onFilterChange, parsedQuery.passthroughFilters], + [onFilterChange], ); const setFilterValue = useCallback( @@ -442,21 +394,15 @@ export const useSearchPageFilterState = ({ value: string | boolean, action?: 'only' | 'exclude' | 'include', ) => { - // Normalize bracket notation to dot notation so the key matches - // what parseQuery returns after a Lucene round-trip. - const key = parseKeyPath(property).join('.'); - // Normalize "true"/"false" strings to booleans so the value matches - // what parseLuceneFilter returns after a Lucene round-trip. - const normalizedValue = coerceBooleanValue(value); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - if (!draft[key]) { - draft[key] = { included: new Set(), excluded: new Set() }; + if (!draft[property]) { + draft[property] = { included: new Set(), excluded: new Set() }; } if (action === 'only') { - draft[key] = { - included: new Set([normalizedValue]), + draft[property] = { + included: new Set([value]), excluded: new Set(), }; return; @@ -464,22 +410,22 @@ export const useSearchPageFilterState = ({ if (action === 'exclude') { // Remove from included if it was there - draft[key].included.delete(normalizedValue); + draft[property].included.delete(value); // Toggle in excluded - if (draft[key].excluded.has(normalizedValue)) { - draft[key].excluded.delete(normalizedValue); + if (draft[property].excluded.has(value)) { + draft[property].excluded.delete(value); } else { - draft[key].excluded.add(normalizedValue); + draft[property].excluded.add(value); } return; } // Regular toggle (include) - draft[key].excluded.delete(normalizedValue); - if (draft[key].included.has(normalizedValue)) { - draft[key].included.delete(normalizedValue); + draft[property].excluded.delete(value); + if (draft[property].included.has(value)) { + draft[property].included.delete(value); } else { - draft[key].included.add(normalizedValue); + draft[property].included.add(value); } }); updateFilterQuery(newFilters); @@ -491,13 +437,12 @@ export const useSearchPageFilterState = ({ const setFilterRange = useCallback( (property: string, range: { min: number; max: number }) => { - const key = parseKeyPath(property).join('.'); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - if (!draft[key]) { - draft[key] = { included: new Set(), excluded: new Set() }; + if (!draft[property]) { + draft[property] = { included: new Set(), excluded: new Set() }; } - draft[key].range = range; + draft[property].range = range; }); updateFilterQuery(newFilters); return newFilters; @@ -508,10 +453,9 @@ export const useSearchPageFilterState = ({ const clearFilter = useCallback( (property: string) => { - const key = parseKeyPath(property).join('.'); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - delete draft[key]; + delete draft[property]; }); updateFilterQuery(newFilters); return newFilters; diff --git a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts index d4f127b2aa..88c4d43d0e 100644 --- a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts @@ -205,10 +205,10 @@ test.describe( const url = new URL(page.url()); expect(url.searchParams.get('source')).toBe(logsSourceId); expect(url.searchParams.get('isLive')).toBe('false'); - // Default row click encodes the group-by filter as a Lucene condition - // in the JSON-encoded `filters` param. + // Default row click encodes the group-by filter as a SQL IN clause in + // the JSON-encoded `filters` param. const filters = url.searchParams.get('filters') ?? '[]'; - expect(filters).toContain(`ServiceName:\\"${serviceName}\\"`); + expect(filters).toContain(`ServiceName IN ('${serviceName}')`); const from = Number(url.searchParams.get('from')); const to = Number(url.searchParams.get('to')); expect(from).toBeGreaterThan(0); @@ -563,8 +563,8 @@ test.describe( const filters = JSON.parse(decodeURIComponent(filtersRaw!)); expect(filters).toEqual([ { - type: 'lucene', - condition: `ServiceName:"${serviceName}"`, + type: 'sql', + condition: `ServiceName IN ('${serviceName}')`, }, ]); }); @@ -649,8 +649,8 @@ test.describe( const filters = JSON.parse(decodeURIComponent(filtersRaw!)); expect(filters).toEqual([ { - type: 'lucene', - condition: `ServiceName:"${serviceName}"`, + type: 'sql', + condition: `ServiceName IN ('${serviceName}')`, }, ]); }); diff --git a/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts b/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts deleted file mode 100644 index 59cc207949..0000000000 --- a/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { filtersToQuery, parseLuceneFilter } from '@/filters'; -import { parse } from '@/queryParser'; - -describe('Colon-in-MAP-KEY scenario (LogAttributes with key containing colon)', () => { - it('parses Lucene when the colon in a field name is properly escaped', () => { - expect(() => - parse(String.raw`LogAttributes.foo\:bar:"value"`), - ).not.toThrow(); - }); - - it('filtersToQuery emits parseable Lucene for bracket-form Map key containing colon', () => { - const filters = { - "LogAttributes['foo:bar']": { - included: new Set(['value1']), - excluded: new Set(), - }, - }; - const result = filtersToQuery(filters); - const condition = (result[0] as { condition: string }).condition; - expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); - expect(() => parse(condition)).not.toThrow(); - }); - - it('filtersToQuery emits parseable Lucene for dot-form Map key containing colon', () => { - const filters = { - 'LogAttributes.foo:bar': { - included: new Set(['value1']), - excluded: new Set(), - }, - }; - const result = filtersToQuery(filters); - const condition = (result[0] as { condition: string }).condition; - expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); - expect(() => parse(condition)).not.toThrow(); - }); - - it('round-trips Map keys containing colons through filtersToQuery + parseLuceneFilter', () => { - const filters = { - "LogAttributes['foo:bar']": { - included: new Set(['value1']), - excluded: new Set(), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - const parsed = parseLuceneFilter(cond); - expect(parsed).toEqual([ - { - key: 'LogAttributes.foo:bar', - included: ['value1'], - excluded: [], - }, - ]); - }); - - it('round-trips Map keys with colons and excluded values', () => { - const filters = { - 'LogAttributes.foo:bar': { - included: new Set(), - excluded: new Set(['unwanted:value']), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - expect(cond).toBe(String.raw`-LogAttributes.foo\:bar:"unwanted:value"`); - expect(parseLuceneFilter(cond)).toEqual([ - { - key: 'LogAttributes.foo:bar', - included: [], - excluded: ['unwanted:value'], - }, - ]); - }); - - it('round-trips Map keys with multiple colons in the key segment', () => { - const filters = { - "LogAttributes['k8s.io:annotation:foo']": { - included: new Set(['v1']), - excluded: new Set(), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - expect(() => parse(cond)).not.toThrow(); - expect(parseLuceneFilter(cond)).toEqual([ - { - key: 'LogAttributes.k8s.io:annotation:foo', - included: ['v1'], - excluded: [], - }, - ]); - }); - - it('round-trips Map key colon-in-key + colon-in-value simultaneously', () => { - const filters = { - "LogAttributes['foo:bar']": { - included: new Set(['https://example.com:8080']), - excluded: new Set(), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - expect(() => parse(cond)).not.toThrow(); - expect(parseLuceneFilter(cond)).toEqual([ - { - key: 'LogAttributes.foo:bar', - included: ['https://example.com:8080'], - excluded: [], - }, - ]); - }); -}); diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index 9ba03e7c30..20253b4c8b 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -1,38 +1,36 @@ -import { type FilterState, filtersToQuery, parseLuceneFilter } from '@/filters'; -import type { Filter } from '@/types'; - -/** Extract condition string from a Filter (filtersToQuery never emits sql_ast) */ -const getCondition = (f: Filter): string => - 'condition' in f ? f.condition : ''; +import { filtersToQuery } from '@/filters'; describe('filters', () => { describe('filtersToQuery', () => { - it('should return empty array when no filters', () => { - expect(filtersToQuery({})).toEqual([]); + it('should return empty string when no filters', () => { + const filters = {}; + expect(filtersToQuery(filters)).toEqual([]); }); - it('should emit lucene for a single included value', () => { + it('should return query for one filter', () => { const filters = { a: { included: new Set(['b']), excluded: new Set() }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'a:"b"' }, + { type: 'sql', condition: "a IN ('b')" }, ]); }); - it('should emit lucene with parens for multiple included values', () => { + it('should return query for multiple filters', () => { const filters = { + a: { included: new Set(['b']), excluded: new Set() }, c: { included: new Set(['d', 'x']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: '(c:"d" OR c:"x")' }, + { type: 'sql', condition: "a IN ('b')" }, + { type: 'sql', condition: "c IN ('d', 'x')" }, ]); }); - it('should handle excluded values without parens', () => { + it('should handle excluded values', () => { const filters = { a: { included: new Set(['b']), @@ -40,458 +38,129 @@ describe('filters', () => { }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'a:"b"' }, - { type: 'lucene', condition: '-a:"c"' }, + { type: 'sql', condition: "a IN ('b')" }, + { type: 'sql', condition: "a NOT IN ('c')" }, ]); }); - it('should handle multiple excluded values without parens', () => { + it('should wrap keys with toString() when specified', () => { const filters = { - a: { - included: new Set(), - excluded: new Set([true, false]), + 'json.key': { + included: new Set(['value']), + excluded: new Set(['other value']), }, }; - expect(filtersToQuery(filters)).toEqual([ - { - type: 'lucene', - condition: '-a:"true" AND -a:"false"', - }, + expect(filtersToQuery(filters, { stringifyKeys: true })).toEqual([ + { type: 'sql', condition: "toString(json.key) IN ('value')" }, + { type: 'sql', condition: "toString(json.key) NOT IN ('other value')" }, ]); }); - it('should handle boolean filter values', () => { + it('should should handle boolean filter values', () => { const filters = { isRootSpan: { included: new Set([true]), excluded: new Set([]), }, - }; - expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'isRootSpan:"true"' }, - ]); - }); - - it('should escape double quotes in values', () => { - const filters = { - message: { - included: new Set(['say "hello"']), - excluded: new Set(), + another_column: { + included: new Set([]), + excluded: new Set([true, false]), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'message:"say \\"hello\\""' }, + { type: 'sql', condition: 'isRootSpan IN (true)' }, + { type: 'sql', condition: 'another_column NOT IN (true, false)' }, ]); }); - it('should pass through single quotes in values', () => { + it('should escape single quotes in filter values', () => { const filters = { message: { - included: new Set(["it's a test"]), + included: new Set(["my 'filter' key"]), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'message:"it\'s a test"' }, + { + type: 'sql', + condition: "message IN ('my ''filter'' key')", + }, ]); }); - it('should escape backslashes in values', () => { + it('should escape single quotes in excluded filter values', () => { const filters = { - FilePath: { - included: new Set(['C:\\path\\to\\file']), - excluded: new Set(), + message: { + included: new Set(), + excluded: new Set(["it's a test"]), }, }; expect(filtersToQuery(filters)).toEqual([ { - type: 'lucene', - condition: 'FilePath:"C:\\\\path\\\\to\\\\file"', + type: 'sql', + condition: "message NOT IN ('it''s a test')", }, ]); }); - it('should convert Map bracket-notation keys to dot notation', () => { + it('should escape single quotes with stringifyKeys', () => { const filters = { - "LogAttributes['service.name']": { - included: new Set(['my-app']), + 'json.key': { + included: new Set(["value with 'quotes'"]), excluded: new Set(), }, }; - expect(filtersToQuery(filters)).toEqual([ + expect(filtersToQuery(filters, { stringifyKeys: true })).toEqual([ { - type: 'lucene', - condition: 'LogAttributes.service.name:"my-app"', + type: 'sql', + condition: "toString(json.key) IN ('value with ''quotes''')", }, ]); }); - it('should handle Map bracket-notation with multiple values', () => { + it('should escape backslashes in filter values', () => { const filters = { - "LogAttributes['env']": { - included: new Set(['prod', 'staging']), + FilePath: { + included: new Set(['C:\\path\\to\\file']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ { - type: 'lucene', - condition: - '(LogAttributes.env:"prod" OR LogAttributes.env:"staging")', + type: 'sql', + condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')", }, ]); }); - it('should handle Map bracket-notation excluded values', () => { + it('should escape backslashes in excluded filter values', () => { const filters = { - "LogAttributes['level']": { + FilePath: { included: new Set(), - excluded: new Set(['debug']), - }, - }; - expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: '-LogAttributes.level:"debug"' }, - ]); - }); - - it('should handle Map bracket-notation with both included and excluded', () => { - const filters = { - "ResourceAttributes['host']": { - included: new Set(['web-1']), - excluded: new Set(['web-2']), + excluded: new Set(['C:\\path\\to\\file']), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'ResourceAttributes.host:"web-1"' }, - { type: 'lucene', condition: '-ResourceAttributes.host:"web-2"' }, - ]); - }); - - it('should emit lucene range syntax', () => { - const filters = { - duration: { - included: new Set(), - excluded: new Set(), - range: { min: 10, max: 500 }, + { + type: 'sql', + condition: "FilePath NOT IN ('C:\\\\path\\\\to\\\\file')", }, - }; - expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'duration:[10 TO 500]' }, ]); }); - it('should emit lucene range with dot notation for Map bracket-notation keys', () => { + it('should escape backslashes before single quotes so quotes stay escaped', () => { const filters = { - "LogAttributes['latency']": { - included: new Set(), + message: { + included: new Set(["a\\'b"]), excluded: new Set(), - range: { min: 0, max: 100 }, }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'LogAttributes.latency:[0 TO 100]' }, - ]); - }); - }); - - describe('parseLuceneFilter', () => { - it('should return empty array for empty string', () => { - expect(parseLuceneFilter('')).toEqual([]); - }); - - it('should return empty array for unquoted terms', () => { - expect(parseLuceneFilter('service:foo')).toEqual([]); - }); - - it('should parse a single included term', () => { - expect(parseLuceneFilter('service:"app"')).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - ]); - }); - - it('should parse a single excluded term', () => { - expect(parseLuceneFilter('-service:"app"')).toEqual([ - { key: 'service', included: [], excluded: ['app'] }, - ]); - }); - - it('should parse multiple included values with OR', () => { - expect(parseLuceneFilter('(service:"app" OR service:"web")')).toEqual([ - { key: 'service', included: ['app', 'web'], excluded: [] }, - ]); - }); - - it('should parse multiple excluded values with AND', () => { - expect( - parseLuceneFilter('-service:"debug" AND -service:"trace"'), - ).toEqual([ - { key: 'service', included: [], excluded: ['debug', 'trace'] }, - ]); - }); - - it('should parse mixed included and excluded for the same field', () => { - expect( - parseLuceneFilter('-service:"bingo" (service:"foo" OR service:"bar")'), - ).toEqual([ - { key: 'service', included: ['foo', 'bar'], excluded: ['bingo'] }, - ]); - }); - - it('should parse multiple distinct fields', () => { - expect(parseLuceneFilter('service:"app" level:"info"')).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - { key: 'level', included: ['info'], excluded: [] }, - ]); - }); - - it('should parse multiple fields with mixed negation', () => { - expect( - parseLuceneFilter('service:"app" -level:"debug" level:"info"'), - ).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - { key: 'level', included: ['info'], excluded: ['debug'] }, - ]); - }); - - it('should parse dot-notation fields (Map columns)', () => { - expect(parseLuceneFilter('LogAttributes.env:"prod"')).toEqual([ - { key: 'LogAttributes.env', included: ['prod'], excluded: [] }, - ]); - }); - - it('should handle values with escaped double quotes', () => { - expect(parseLuceneFilter('message:"say \\"hello\\""')).toEqual([ - { key: 'message', included: ['say "hello"'], excluded: [] }, - ]); - }); - - it('should handle values with backslashes', () => { - expect(parseLuceneFilter('path:"C:\\\\dir\\\\file"')).toEqual([ - { key: 'path', included: ['C:\\dir\\file'], excluded: [] }, - ]); - }); - - it('should coerce "true"/"false" back to booleans', () => { - expect(parseLuceneFilter('isRootSpan:"true"')).toEqual([ - { key: 'isRootSpan', included: [true], excluded: [] }, - ]); - expect(parseLuceneFilter('-col:"true" AND -col:"false"')).toEqual([ - { key: 'col', included: [], excluded: [true, false] }, - ]); - }); - - it('should parse range terms', () => { - expect(parseLuceneFilter('duration:[10 TO 500]')).toEqual([ - { - key: 'duration', - included: [], - excluded: [], - range: { min: 10, max: 500 }, - }, - ]); - }); - - it('should parse range with dot-notation field', () => { - expect(parseLuceneFilter('LogAttributes.latency:[0 TO 100]')).toEqual([ { - key: 'LogAttributes.latency', - included: [], - excluded: [], - range: { min: 0, max: 100 }, + type: 'sql', + condition: "message IN ('a\\\\''b')", }, ]); }); - - it('should return undefined for invalid lucene syntax', () => { - expect(parseLuceneFilter('((((')).toBeUndefined(); - }); - }); - - describe('filtersToQuery -> parseLuceneFilter round-trip', () => { - it('round-trips a single included value', () => { - const state: FilterState = { - service: { - included: new Set(['app']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - ]); - }); - - it('round-trips multiple included values', () => { - const state: FilterState = { - env: { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].key).toBe('env'); - expect(parsed?.[0].included).toEqual( - expect.arrayContaining(['prod', 'staging']), - ); - }); - - it('round-trips excluded values', () => { - const state: FilterState = { - level: { - included: new Set(), - excluded: new Set(['debug', 'trace']), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].key).toBe('level'); - expect(parsed?.[0].excluded).toEqual( - expect.arrayContaining(['debug', 'trace']), - ); - }); - - it('round-trips boolean values preserving type', () => { - const state: FilterState = { - isRootSpan: { - included: new Set([true]), - excluded: new Set([false]), - }, - }; - const filters = filtersToQuery(state); - const includedParsed = parseLuceneFilter(getCondition(filters[0])); - const excludedParsed = parseLuceneFilter(getCondition(filters[1])); - expect(includedParsed?.[0].included).toEqual([true]); - expect(excludedParsed?.[0].excluded).toEqual([false]); - }); - - it('round-trips the literal string "true" without boolean coercion', () => { - // The string "true" is indistinguishable from boolean true after - // Lucene round-trip due to coerceBooleanValue. This test documents - // that known limitation. - const state: FilterState = { - status: { - included: new Set(['true']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - // Coerced to boolean — this is the expected (lossy) behavior - expect(parsed?.[0].included).toEqual([true]); - }); - - it('round-trips values with double quotes', () => { - const state: FilterState = { - msg: { - included: new Set(['say "hello"']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['say "hello"']); - }); - - it('round-trips values with backslashes', () => { - const state: FilterState = { - path: { - included: new Set(['C:\\dir\\file']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['C:\\dir\\file']); - }); - - it('round-trips empty string values', () => { - const state: FilterState = { - tag: { - included: new Set(['']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['']); - }); - - it('round-trips values containing Lucene reserved characters', () => { - const state: FilterState = { - query: { - included: new Set(['(foo) AND [bar]']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['(foo) AND [bar]']); - }); - - it('round-trips range filters', () => { - const state: FilterState = { - duration: { - included: new Set(), - excluded: new Set(), - range: { min: 10, max: 500 }, - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].range).toEqual({ min: 10, max: 500 }); - }); - - it('round-trips Map bracket-notation key', () => { - const state: FilterState = { - "LogAttributes['service.name']": { - included: new Set(['my-app']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - // Bracket notation normalizes to dot notation - expect(parsed?.[0].key).toBe('LogAttributes.service.name'); - expect(parsed?.[0].included).toEqual(['my-app']); - }); - - it('round-trips a JSON column dot-path with colons and slashes in value', () => { - const state: FilterState = { - 'Metadata.endpoint': { - included: new Set(['grpc://thingy:8888/Verify']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const condition = getCondition(filters[0]); - expect(condition).not.toMatch(/^toString\(/); - expect(condition).not.toMatch(/`/); - const parsed = parseLuceneFilter(condition); - expect(parsed?.[0].key).toBe('Metadata.endpoint'); - expect(parsed?.[0].included).toEqual(['grpc://thingy:8888/Verify']); - }); - - it('round-trips mixed included and excluded for same field', () => { - const state: FilterState = { - env: { - included: new Set(['prod', 'staging']), - excluded: new Set(['dev']), - }, - }; - const filters = filtersToQuery(state); - // filtersToQuery emits separate Filter entries for included/excluded - expect(filters).toHaveLength(2); - // Parse both and merge - const allParsed = filters.flatMap( - f => parseLuceneFilter(getCondition(f)) ?? [], - ); - const envEntries = allParsed.filter(p => p.key === 'env'); - const included = envEntries.flatMap(e => e.included); - const excluded = envEntries.flatMap(e => e.excluded); - expect(included).toEqual(expect.arrayContaining(['prod', 'staging'])); - expect(excluded).toEqual(['dev']); - }); }); }); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 083c9005a2..bbd40781ef 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1471,28 +1471,4 @@ describe('parseKeyPath', () => { "ResourceAttributes['service.name", ]); }); - - it('parses single-quoted arrayElement function-call form', () => { - expect( - parseKeyPath("arrayElement(ResourceAttributes, 'service.name')"), - ).toEqual(['ResourceAttributes', 'service.name']); - }); - - it('parses arrayElement keys with dots in the map key', () => { - expect( - parseKeyPath("arrayElement(SpanAttributes, 'http.request.method')"), - ).toEqual(['SpanAttributes', 'http.request.method']); - }); - - it('tolerates whitespace inside arrayElement', () => { - expect( - parseKeyPath("arrayElement( ResourceAttributes , 'service.name' )"), - ).toEqual(['ResourceAttributes', 'service.name']); - }); - - it('does not parse arrayElement with a missing closing paren', () => { - expect( - parseKeyPath("arrayElement(ResourceAttributes, 'service.name'"), - ).toEqual(["arrayElement(ResourceAttributes, 'service.name'"]); - }); }); diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 343b9b9f2e..a2afc2118f 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -494,19 +494,6 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { expect(actualSql).toBe(expectedSql); }); - it('"Add to Filters" repro: JSON column path + value with colons and slashes emits valid SQL', async () => { - const lucene = - 'ResourceAttributesJSON.endpoint:"grpc://auth-svc:50051/Verify"'; - const builder = new SearchQueryBuilder(lucene, serializer); - const actualSql = await builder.build(); - expect(actualSql).toContain( - 'toString(`ResourceAttributesJSON`.`endpoint`)', - ); - expect(actualSql).toContain('grpc://auth-svc:50051/Verify'); - expect(actualSql).not.toContain('http_COLON_'); - expect(actualSql).not.toContain('HDX_COLON'); - }); - describe('CustomSchemaSQLSerializerV2: implicit column falls back to bodyExpression', () => { // Symmetric counterpart to the one-way fallback in // `getEventBody` (packages/app/src/source.ts): a log source admin who diff --git a/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts b/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts index ee7b64b7f5..067e3dbbce 100644 --- a/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts +++ b/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts @@ -244,7 +244,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"MyService"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('MyService')" }]); } }); @@ -278,9 +278,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([ - { type: 'lucene', condition: '(ServiceName:"A" OR ServiceName:"B")' }, - ]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('A', 'B')" }]); } }); @@ -315,8 +313,8 @@ describe('renderOnClickSearch', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'ServiceName:"MyService"' }, - { type: 'lucene', condition: 'SeverityText:"error"' }, + { type: 'sql', condition: "ServiceName IN ('MyService')" }, + { type: 'sql', condition: "SeverityText IN ('error')" }, ]); } }); @@ -346,7 +344,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"O\'Malley"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('O''Malley')" }]); } }); @@ -376,7 +374,7 @@ describe('renderOnClickSearch', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'FilePath:"C:\\\\path\\\\to\\\\file"' }, + { type: 'sql', condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')" }, ]); } }); @@ -408,8 +406,8 @@ describe('renderOnClickSearch', () => { JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ { - type: 'lucene', - condition: 'SpanAttributes.url:"/users%2F42"', + type: 'sql', + condition: "SpanAttributes['url'] IN ('/users%2F42')", }, ]); } @@ -678,7 +676,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"MyService"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('MyService')" }]); } }); @@ -712,9 +710,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([ - { type: 'lucene', condition: '(ServiceName:"A" OR ServiceName:"B")' }, - ]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('A', 'B')" }]); } }); @@ -749,8 +745,8 @@ describe('renderOnClickDashboard', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'ServiceName:"MyService"' }, - { type: 'lucene', condition: 'SeverityText:"error"' }, + { type: 'sql', condition: "ServiceName IN ('MyService')" }, + { type: 'sql', condition: "SeverityText IN ('error')" }, ]); } }); @@ -780,7 +776,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"O\'Malley"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('O''Malley')" }]); } }); @@ -810,7 +806,7 @@ describe('renderOnClickDashboard', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'FilePath:"C:\\\\path\\\\to\\\\file"' }, + { type: 'sql', condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')" }, ]); } }); @@ -842,8 +838,8 @@ describe('renderOnClickDashboard', () => { JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ { - type: 'lucene', - condition: 'SpanAttributes.url:"/users%2F42"', + type: 'sql', + condition: "SpanAttributes['url'] IN ('/users%2F42')", }, ]); } diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 59688cef15..a80feb1fad 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1768,17 +1768,9 @@ export type Field = { }; /** - * Parses a Map-access key string into a path array. - * - * Handles both ClickHouse Map access forms: - * - Bracket form (canonical input): - * `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` - * `ResourceAttributes["service.name"]` → `['ResourceAttributes', 'service.name']` - * - Function-call form (what ClickHouse renders in result column names): - * `arrayElement(ResourceAttributes, 'service.name')` → `['ResourceAttributes', 'service.name']` - * - * Native columns return a single-element path: - * `ServiceName` → `['ServiceName']` + * Parses a bracket-notation key string into a path array. + * e.g. `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` + * `ServiceName` → `['ServiceName']` */ export function parseKeyPath(key: string): string[] { const singleIdx = key.indexOf("['"); @@ -1789,15 +1781,6 @@ export function parseKeyPath(key: string): string[] { if (doubleIdx !== -1 && key.endsWith('"]')) { return [key.slice(0, doubleIdx), key.slice(doubleIdx + 2, -2)]; } - // ClickHouse Map access function-call form. ClickHouse renders Map - // subscript expressions (e.g. `SpanAttributes['http.route']`) as - // `arrayElement(SpanAttributes, 'http.route')` in result column names. - const ARRAY_ELEMENT_PREFIX = 'arrayElement('; - if (key.startsWith(ARRAY_ELEMENT_PREFIX) && key.endsWith(')')) { - const inner = key.slice(ARRAY_ELEMENT_PREFIX.length, -1); - const parts = inner.split(',').map(e => e.trim()); - return [parts[0], parts[1].replaceAll("'", '')]; - } return [key]; } diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index e5ded218a4..9a7ac150b1 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1,14 +1,3 @@ -import lucene from '@hyperdx/lucene'; - -import { parseKeyPath } from '@/core/metadata'; -import { - decodeSpecialTokens, - isBinaryAST, - isLeftOnlyAST, - isNodeRangedTerm, - isNodeTerm, - parse, -} from '@/queryParser'; import { Filter } from '@/types'; export type FilterState = { @@ -19,25 +8,14 @@ export type FilterState = { }; }; -/** Escape a value for use inside a Lucene quoted term ("...") */ -const escapeLuceneQuotedTerm = (s: string) => { - return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +const escapeString = (s: string) => { + return s.replace(/\\/g, '\\\\').replace(/'/g, "''"); }; -/** - * Escape backslashes and colons so the field name survives Lucene parsing. - * Map sub-keys can legitimately contain `:` (e.g. `LogAttributes['foo:bar']` - * normalizes to `LogAttributes.foo:bar` via parseKeyPath().join('.')), and - * `:` is the Lucene field/value separator. Backslashes are escaped first so - * the inserted `\:` survives `encodeSpecialTokens`' `\\` → backslash-literal - * substitution; the encoder's matching `\:` → HDX_COLON rule then makes the - * colon opaque to the parser, and `decodeSpecialTokens` restores the - * original key on the consumer side. - */ -const escapeLuceneFieldName = (key: string): string => - key.replace(/\\/g, '\\\\').replace(/:/g, '\\:'); - -export const filtersToQuery = (filters: FilterState): Filter[] => { +export const filtersToQuery = ( + filters: FilterState, + { stringifyKeys = false }: { stringifyKeys?: boolean } = {}, +): Filter[] => { return Object.entries(filters) .filter( ([_, values]) => @@ -47,137 +25,30 @@ export const filtersToQuery = (filters: FilterState): Filter[] => { ) .flatMap(([key, values]) => { const conditions: Filter[] = []; - const luceneField = escapeLuceneFieldName(parseKeyPath(key).join('.')); + const actualKey = stringifyKeys ? `toString(${key})` : key; if (values.included.size > 0) { - const terms = Array.from(values.included).map( - v => `${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`, - ); conditions.push({ - type: 'lucene' as const, - condition: terms.length > 1 ? `(${terms.join(' OR ')})` : terms[0], + type: 'sql' as const, + condition: `${actualKey} IN (${Array.from(values.included) + .map(v => (typeof v === 'string' ? `'${escapeString(v)}'` : v)) + .join(', ')})`, }); } if (values.excluded.size > 0) { conditions.push({ - type: 'lucene' as const, - condition: Array.from(values.excluded) - .map(v => `-${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`) - .join(' AND '), + type: 'sql' as const, + condition: `${actualKey} NOT IN (${Array.from(values.excluded) + .map(v => (typeof v === 'string' ? `'${escapeString(v)}'` : v)) + .join(', ')})`, }); } - if (values.range != null) { - // Lucene range syntax: field:[min TO max] conditions.push({ - type: 'lucene' as const, - condition: `${luceneField}:[${values.range.min} TO ${values.range.max}]`, + type: 'sql' as const, + condition: `${actualKey} BETWEEN ${values.range.min} AND ${values.range.max}`, }); } return conditions; }); }; - -type CollectedTerm = { field: string; value: string; negated: boolean }; -type CollectedRange = { field: string; min: number; max: number }; - -/** - * Collect quoted terms and range terms from a Lucene AST. - */ -function collectFromAst( - ast: lucene.AST | lucene.Node, - terms: CollectedTerm[], - ranges: CollectedRange[], -): void { - if (isNodeTerm(ast)) { - if (!ast.quoted) return; - const negated = ast.field.startsWith('-'); - const field = decodeSpecialTokens(negated ? ast.field.slice(1) : ast.field); - terms.push({ field, value: decodeSpecialTokens(ast.term), negated }); - } else if (isNodeRangedTerm(ast)) { - const field = decodeSpecialTokens(ast.field); - const min = parseFloat(ast.term_min); - const max = parseFloat(ast.term_max); - if (!isNaN(min) && !isNaN(max)) { - ranges.push({ field, min, max }); - } - } else if (isBinaryAST(ast)) { - collectFromAst(ast.left, terms, ranges); - collectFromAst(ast.right, terms, ranges); - } else if (isLeftOnlyAST(ast)) { - collectFromAst(ast.left, terms, ranges); - } -} - -/** Coerce "true"/"false" strings back to booleans, pass through otherwise */ -export function coerceBooleanValue(v: string | boolean): string | boolean { - if (typeof v === 'boolean') return v; - if (v === 'true') return true; - if (v === 'false') return false; - return v; -} - -export type ParsedLuceneFilter = { - key: string; - included: (string | boolean)[]; - excluded: (string | boolean)[]; - range?: { min: number; max: number }; -}; - -/** - * Parse a Lucene filter condition back into per-field included/excluded values - * and optional ranges. - * - * Returns an array with one entry per distinct field (empty array if the input - * is valid Lucene but contains no quoted terms or ranges), or undefined if - * parsing fails entirely. - */ -export function parseLuceneFilter( - condition: string, -): ParsedLuceneFilter[] | undefined { - try { - const ast = parse(condition); - const terms: CollectedTerm[] = []; - const ranges: CollectedRange[] = []; - collectFromAst(ast, terms, ranges); - - // Group by field, coercing "true"/"false" back to booleans - const byField = new Map< - string, - { - included: (string | boolean)[]; - excluded: (string | boolean)[]; - range?: { min: number; max: number }; - } - >(); - - const getEntry = (field: string) => { - if (!byField.has(field)) { - byField.set(field, { included: [], excluded: [] }); - } - return byField.get(field)!; - }; - - for (const t of terms) { - const entry = getEntry(t.field); - const value = coerceBooleanValue(t.value); - if (t.negated) { - entry.excluded.push(value); - } else { - entry.included.push(value); - } - } - - for (const r of ranges) { - const entry = getEntry(r.field); - entry.range = { min: r.min, max: r.max }; - } - - return Array.from(byField.entries()).map(([key, vals]) => ({ - key, - ...vals, - })); - } catch { - return undefined; - } -} diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index fb195d016b..f96ffb635b 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -33,7 +33,7 @@ function encodeSpecialTokens(query: string): string { .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1') .replace(/\\:/g, 'HDX_COLON'); } -export function decodeSpecialTokens(query: string): string { +function decodeSpecialTokens(query: string): string { return query .replace(/\\"/g, '"') .replace(/HDX_BACKSLASH_LITERAL/g, '\\') @@ -61,21 +61,17 @@ function normalizeChExpression(expr: string): string { const IMPLICIT_FIELD = ''; // Type guards for lucene AST types -export function isNodeTerm( - node: lucene.Node | lucene.AST, -): node is lucene.NodeTerm { +function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { return 'term' in node && node.term != null; } -export function isNodeRangedTerm( +function isNodeRangedTerm( node: lucene.Node | lucene.AST, ): node is lucene.NodeRangedTerm { return 'inclusive' in node && node.inclusive != null; } -export function isBinaryAST( - ast: lucene.AST | lucene.Node, -): ast is lucene.BinaryAST { +function isBinaryAST(ast: lucene.AST | lucene.Node): ast is lucene.BinaryAST { return 'right' in ast && ast.right != null; } @@ -85,7 +81,7 @@ function hasStart( return 'start' in ast && !!ast.start; } -export function isLeftOnlyAST( +function isLeftOnlyAST( ast: lucene.AST | lucene.Node, ): ast is lucene.LeftOnlyAST { return ( @@ -1766,14 +1762,9 @@ async function nodeTerm( serializer: Serializer, context: SerializerContext, ): Promise { - const isImplicitField = node.field === IMPLICIT_FIELD; - const rawField = node.field[0] === '-' ? node.field.slice(1) : node.field; - // Decode special-token placeholders the emitter inserted in the field name - // (e.g. HDX_COLON for an escaped `:` in a Map sub-key). Leave the implicit - // field sentinel untouched so downstream comparisons against IMPLICIT_FIELD - // still match. - const field = isImplicitField ? rawField : decodeSpecialTokens(rawField); + const field = node.field[0] === '-' ? node.field.slice(1) : node.field; let isNegatedField = node.field[0] === '-'; + const isImplicitField = node.field === IMPLICIT_FIELD; // NodeTerm if (isNodeTerm(node)) { @@ -1875,7 +1866,7 @@ function createSerializerContext( return { ...currentContext, - implicitColumnExpression: decodeSpecialTokens(fieldWithoutNegation), + implicitColumnExpression: fieldWithoutNegation, ...(isNegatedAndParenthesized(ast) ? { isNegatedAndParenthesized: true } : {}), From a19ba549495e214c8264a3bc37e1b81dac3a6736 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 2 Jun 2026 12:04:38 -0600 Subject: [PATCH 011/728] feat(mcp): add patch_dashboard, get_dashboard_tile, search_dashboards tools (HDX-4139) (#2343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Add three new MCP dashboard tools for more granular, efficient operations alongside the existing full create/update path: - **`hyperdx_get_dashboard_tile`** — Retrieve a single tile by `tileId` without loading the full dashboard - **`hyperdx_patch_dashboard`** — Update dashboard name/tags and/or replace a single tile by `tileId` in one call. Unmentioned tiles and fields are preserved. Layout fields fall back to existing values when omitted. - **`hyperdx_search_dashboards`** — Search dashboards by name (case-insensitive partial match) and/or tags ## Why The current MCP dashboard tooling requires passing the full dashboard object on every update — making it slow and token-heavy for targeted edits like changing one tile's query or renaming a dashboard. These tools let the LLM do targeted edits without resubmitting everything. Closes HDX-4139 ## Additional improvements - **Add cross-references** — `save_dashboard` description now mentions `patch_dashboard` as the preferred tool for single-tile updates. - **Document Lucene substring matching** — Prominently documented that `field:value` is `ilike` (substring match, not equality) and `field:val*` is prefix-within-substring (not true prefix). Added to tool-level `WHERE_DESCRIPTION`, query guide prompt, and common mistakes section. - **Split test file** — Broke `dashboards.test.ts` (3040 lines) into 8 focused files under `__tests__/dashboards/` with a shared setup helper. ## Testing - 96 integration tests across 8 test files, all passing - TypeScript type check clean - ESLint 0 errors --- .changeset/mcp-dashboard-improvements.md | 21 + packages/api/jest.config.js | 2 +- .../api/src/mcp/__tests__/dashboards.test.ts | 2517 ----------------- .../dashboards/deleteDashboard.test.ts | 38 + .../__tests__/dashboards/getDashboard.test.ts | 65 + .../dashboards/getDashboardTile.test.ts | 103 + .../dashboards/patchDashboard.test.ts | 552 ++++ .../mcp/__tests__/dashboards/prompts.test.ts | 576 ++++ .../__tests__/dashboards/queryTile.test.ts | 110 + .../dashboards/saveDashboard.test.ts | 1806 ++++++++++++ .../dashboards/searchDashboards.test.ts | 261 ++ .../api/src/mcp/__tests__/dashboards/setup.ts | 81 + .../api/src/mcp/prompts/dashboards/content.ts | 46 +- .../mcp/tools/dashboards/getDashboardTile.ts | 83 + .../api/src/mcp/tools/dashboards/index.ts | 6 + .../mcp/tools/dashboards/patchDashboard.ts | 286 ++ .../src/mcp/tools/dashboards/saveDashboard.ts | 237 +- .../api/src/mcp/tools/dashboards/schemas.ts | 85 +- .../mcp/tools/dashboards/searchDashboards.ts | 115 + packages/api/src/mcp/tools/query/schemas.ts | 11 +- .../src/routers/external-api/v2/dashboards.ts | 252 +- .../external-api/v2/utils/dashboards.ts | 160 +- 22 files changed, 4422 insertions(+), 2991 deletions(-) create mode 100644 .changeset/mcp-dashboard-improvements.md delete mode 100644 packages/api/src/mcp/__tests__/dashboards.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/deleteDashboard.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/getDashboard.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/prompts.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/queryTile.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/searchDashboards.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/setup.ts create mode 100644 packages/api/src/mcp/tools/dashboards/getDashboardTile.ts create mode 100644 packages/api/src/mcp/tools/dashboards/patchDashboard.ts create mode 100644 packages/api/src/mcp/tools/dashboards/searchDashboards.ts diff --git a/.changeset/mcp-dashboard-improvements.md b/.changeset/mcp-dashboard-improvements.md new file mode 100644 index 0000000000..375224e649 --- /dev/null +++ b/.changeset/mcp-dashboard-improvements.md @@ -0,0 +1,21 @@ +--- +'@hyperdx/api': patch +--- + +feat(mcp): add patch_dashboard, get_dashboard_tile, search_dashboards tools + +Add three new MCP dashboard tools for granular operations: + +- `hyperdx_get_dashboard_tile` — retrieve a single tile by tileId +- `hyperdx_patch_dashboard` — update name/tags and/or replace one tile + without resubmitting the full dashboard +- `hyperdx_search_dashboards` — search by name and/or tags + +Fix empty parameter schema on patch/search tools caused by Zod +`.refine()` wrapping. Document Lucene substring matching limitations +prominently in tool descriptions and query guide prompt. + +**Breaking (minor):** Tile `name` on `hyperdx_save_dashboard` now requires +at least 1 character (`.min(1)`). Previously empty string `""` was accepted +and silently persisted as a blank title. Callers sending `name: ""` will +now receive a validation error. diff --git a/packages/api/jest.config.js b/packages/api/jest.config.js index 8fc45f0f9d..bfb2f9c832 100644 --- a/packages/api/jest.config.js +++ b/packages/api/jest.config.js @@ -10,7 +10,7 @@ module.exports = { testEnvironment: 'node', verbose: true, rootDir: './src', - testMatch: ['**/__tests__/*.test.ts?(x)'], + testMatch: ['**/__tests__/**/*.test.ts?(x)'], testTimeout: 30000, moduleNameMapper: { '@/(.*)$': '/$1', diff --git a/packages/api/src/mcp/__tests__/dashboards.test.ts b/packages/api/src/mcp/__tests__/dashboards.test.ts deleted file mode 100644 index 5c63a326f8..0000000000 --- a/packages/api/src/mcp/__tests__/dashboards.test.ts +++ /dev/null @@ -1,2517 +0,0 @@ -import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; -import { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import mongoose from 'mongoose'; - -import * as config from '@/config'; -import { - DEFAULT_DATABASE, - DEFAULT_TRACES_TABLE, - getLoggedInAgent, - getServer, -} from '@/fixtures'; -import Connection from '@/models/connection'; -import Dashboard from '@/models/dashboard'; -import { Source } from '@/models/source'; -import type { ExternalDashboardTileWithId } from '@/utils/zod'; - -import { - buildCreateDashboardPrompt, - buildDashboardExamplesPrompt, - buildQueryGuidePrompt, -} from '../prompts/dashboards/content'; -import { buildSourceSummary } from '../prompts/dashboards/helpers'; -import { McpContext } from '../tools/types'; -import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; - -describe('MCP Dashboard Tools', () => { - const server = getServer(); - let team: any; - let user: any; - let traceSource: any; - let connection: any; - let client: Client; - - beforeAll(async () => { - await server.start(); - }); - - beforeEach(async () => { - const result = await getLoggedInAgent(server); - team = result.team; - user = result.user; - - connection = await Connection.create({ - team: team._id, - name: 'Default', - host: config.CLICKHOUSE_HOST, - username: config.CLICKHOUSE_USER, - password: config.CLICKHOUSE_PASSWORD, - }); - - traceSource = await Source.create({ - kind: SourceKind.Trace, - team: team._id, - from: { - databaseName: DEFAULT_DATABASE, - tableName: DEFAULT_TRACES_TABLE, - }, - timestampValueExpression: 'Timestamp', - connection: connection._id, - name: 'Traces', - }); - - const context: McpContext = { - teamId: team._id.toString(), - userId: user._id.toString(), - }; - client = await createTestClient(context); - }); - - afterEach(async () => { - await client?.close(); - await server.clearDBs(); - }); - - afterAll(async () => { - await server.stop(); - }); - - describe('clickstack_get_dashboard', () => { - it('should list all dashboards when no id provided', async () => { - await new Dashboard({ - name: 'Dashboard 1', - tiles: [], - team: team._id, - tags: ['tag1'], - }).save(); - await new Dashboard({ - name: 'Dashboard 2', - tiles: [], - team: team._id, - tags: ['tag2'], - }).save(); - - const result = await callTool(client, 'clickstack_get_dashboard', {}); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output).toHaveLength(2); - expect(output[0]).toHaveProperty('id'); - expect(output[0]).toHaveProperty('name'); - expect(output[0]).toHaveProperty('tags'); - }); - - it('should get dashboard detail when id is provided', async () => { - const dashboard = await new Dashboard({ - name: 'My Dashboard', - tiles: [], - team: team._id, - tags: ['test'], - }).save(); - - const result = await callTool(client, 'clickstack_get_dashboard', { - id: dashboard._id.toString(), - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.id).toBe(dashboard._id.toString()); - expect(output.name).toBe('My Dashboard'); - expect(output.tags).toEqual(['test']); - expect(output.tiles).toEqual([]); - }); - - it('should return error for non-existent dashboard id', async () => { - const fakeId = '000000000000000000000000'; - const result = await callTool(client, 'clickstack_get_dashboard', { - id: fakeId, - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('not found'); - }); - }); - - describe('clickstack_save_dashboard', () => { - it('should create a new dashboard with tiles', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'New MCP Dashboard', - tiles: [ - { - name: 'Line Chart', - x: 0, - y: 0, - w: 12, - h: 4, - config: { - displayType: 'line', - sourceId, - select: [{ aggFn: 'count', where: '' }], - }, - }, - ], - tags: ['mcp-test'], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.id).toBeDefined(); - expect(output.name).toBe('New MCP Dashboard'); - expect(output.tiles).toHaveLength(1); - expect(output.tiles[0].config.displayType).toBe('line'); - expect(output.tags).toEqual(['mcp-test']); - - // Verify in database - const dashboard = await Dashboard.findById(output.id); - expect(dashboard).not.toBeNull(); - expect(dashboard?.name).toBe('New MCP Dashboard'); - }); - - it('should create a dashboard with a markdown tile', async () => { - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Markdown Dashboard', - tiles: [ - { - name: 'Notes', - config: { - displayType: 'markdown', - markdown: '# Hello World', - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles).toHaveLength(1); - expect(output.tiles[0].config.displayType).toBe('markdown'); - }); - - it('should update an existing dashboard', async () => { - const sourceId = traceSource._id.toString(); - - // Create first - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Original Name', - tiles: [ - { - name: 'Tile 1', - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - const created = JSON.parse(getFirstText(createResult)); - - // Update - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'Updated Name', - tiles: [ - { - name: 'Updated Tile', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - tags: ['updated'], - }); - - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.id).toBe(created.id); - expect(updated.name).toBe('Updated Name'); - expect(updated.tiles).toHaveLength(1); - expect(updated.tiles[0].name).toBe('Updated Tile'); - expect(updated.tiles[0].config.displayType).toBe('table'); - }); - - it('should return error for missing source ID', async () => { - const fakeSourceId = '000000000000000000000000'; - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Bad Dashboard', - tiles: [ - { - name: 'Bad Tile', - config: { - displayType: 'line', - sourceId: fakeSourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('source'); - }); - - it('should return error when updating non-existent dashboard', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - id: '000000000000000000000000', - name: 'Ghost Dashboard', - tiles: [ - { - name: 'Tile', - config: { - displayType: 'line', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('not found'); - }); - - it('should create a dashboard with multiple tile types', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Multi-tile Dashboard', - tiles: [ - { - name: 'Line', - x: 0, - y: 0, - w: 12, - h: 4, - config: { - displayType: 'line', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - { - name: 'Table', - x: 0, - y: 4, - w: 12, - h: 4, - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - groupBy: 'SpanName', - groupByColumnsOnLeft: true, - }, - }, - { - name: 'Number', - x: 0, - y: 8, - w: 6, - h: 3, - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - { - name: 'Pie', - x: 6, - y: 8, - w: 6, - h: 3, - config: { - displayType: 'pie', - sourceId, - select: [{ aggFn: 'count' }], - groupBy: 'SpanName', - }, - }, - { - name: 'Notes', - x: 0, - y: 11, - w: 12, - h: 2, - config: { displayType: 'markdown', markdown: '# Dashboard Notes' }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles).toHaveLength(5); - - const tableTile = output.tiles.find( - (t: { name: string }) => t.name === 'Table', - ); - expect(tableTile).toBeDefined(); - expect(tableTile.config.groupByColumnsOnLeft).toBe(true); - }); - - it('should create a dashboard with a raw SQL tile', async () => { - const connectionId = connection._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'SQL Dashboard', - tiles: [ - { - name: 'Raw SQL', - config: { - configType: 'sql', - displayType: 'table', - connectionId, - sqlTemplate: 'SELECT 1 AS value LIMIT 1', - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles).toHaveLength(1); - }); - - it('should create a dashboard with a heatmap tile on a Trace source', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Heatmap Dashboard', - tiles: [ - { - name: 'Latency Heatmap', - x: 0, - y: 0, - w: 12, - h: 4, - config: { - displayType: 'heatmap', - sourceId, - select: [ - { - valueExpression: 'Duration', - }, - ], - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles).toHaveLength(1); - expect(output.tiles[0].config.displayType).toBe('heatmap'); - expect(output.tiles[0].config.select[0]).toMatchObject({ - valueExpression: 'Duration', - }); - }); - - it('should round-trip every MCP-specific heatmap field through save, get, update, and re-get', async () => { - const sourceId = traceSource._id.toString(); - - const createConfig = { - displayType: 'heatmap' as const, - sourceId, - select: [ - { - valueExpression: 'Duration', - countExpression: 'count()', - heatmapScaleType: 'log' as const, - }, - ], - where: 'level:error', - whereLanguage: 'lucene' as const, - numberFormat: { output: 'duration' as const, factor: 1e-9 }, - }; - // Mutate select + where on update; numberFormat, whereLanguage, - // and sourceId carry forward via the spread. Re-asserting against - // updatedConfig catches a regression where PUT silently drops any - // carried-forward field. - const updatedConfig = { - ...createConfig, - select: [ - { - valueExpression: "SpanAttributes['http.duration']", - heatmapScaleType: 'linear' as const, - }, - ], - where: 'level:error AND service:checkout', - }; - - const saveResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Heatmap Full Round-Trip', - tiles: [ - { - name: 'Latency Heatmap', - x: 0, - y: 0, - w: 12, - h: 4, - config: createConfig, - }, - ], - }); - expect(saveResult.isError).toBeFalsy(); - const saved = JSON.parse(getFirstText(saveResult)); - expect(saved.tiles).toHaveLength(1); - expect(saved.tiles[0].config).toMatchObject(createConfig); - - const getResult = await callTool(client, 'clickstack_get_dashboard', { - id: saved.id, - }); - expect(getResult.isError).toBeFalsy(); - const fetched = JSON.parse(getFirstText(getResult)); - expect(fetched.tiles[0].config).toMatchObject(createConfig); - - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: saved.id, - name: 'Heatmap Full Round-Trip', - tiles: [ - { - ...fetched.tiles[0], - config: { ...fetched.tiles[0].config, ...updatedConfig }, - }, - ], - }); - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.tiles[0].config).toMatchObject(updatedConfig); - - const getAfterUpdate = await callTool( - client, - 'clickstack_get_dashboard', - { - id: saved.id, - }, - ); - expect(getAfterUpdate.isError).toBeFalsy(); - const refetched = JSON.parse(getFirstText(getAfterUpdate)); - expect(refetched.tiles[0].config).toMatchObject(updatedConfig); - }); - - it('should reject heatmap tile with empty valueExpression at the schema layer', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Bad Heatmap', - tiles: [ - { - name: 'Heatmap', - config: { - displayType: 'heatmap', - sourceId, - select: [{ valueExpression: '' }], - }, - }, - ], - }); - - expect(result.isError).toBe(true); - }); - - it.each([ - { output: 'duration', factor: 1e-9 }, - { output: 'data_rate' }, - { output: 'throughput' }, - ] as const)( - 'should round-trip numberFormat output "$output" through save and get', - async numberFormat => { - const sourceId = traceSource._id.toString(); - - const saveResult = await callTool(client, 'clickstack_save_dashboard', { - name: `NumberFormat ${numberFormat.output}`, - tiles: [ - { - name: 'Number Tile', - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - numberFormat, - }, - }, - { - name: 'Line Tile', - config: { - displayType: 'line', - sourceId, - select: [ - { aggFn: 'count' }, - { - aggFn: 'avg', - valueExpression: 'Duration', - numberFormat, - }, - ], - }, - }, - ], - }); - - expect(saveResult.isError).toBeFalsy(); - const saved = JSON.parse(getFirstText(saveResult)); - - const getResult = await callTool(client, 'clickstack_get_dashboard', { - id: saved.id, - }); - expect(getResult.isError).toBeFalsy(); - const fetched = JSON.parse(getFirstText(getResult)); - - const numberTile = fetched.tiles.find( - (t: { name: string }) => t.name === 'Number Tile', - ); - expect(numberTile.config.numberFormat).toEqual(numberFormat); - - const lineTile = fetched.tiles.find( - (t: { name: string }) => t.name === 'Line Tile', - ); - expect(lineTile.config.select[1].numberFormat).toEqual(numberFormat); - }, - ); - - it('should reject numberFormat with an unknown output value', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Bad NumberFormat', - tiles: [ - { - name: 'Number Tile', - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - numberFormat: { output: 'not_a_real_output' }, - }, - }, - ], - }); - - expect(result.isError).toBe(true); - }); - - it('should reject heatmap tile on a non-Trace source', async () => { - // Create a Log source so the heatmap source-kind gate has - // something to reject. The schema accepts the tile shape (the - // sourceId is valid), so this exercises the runtime check that - // the REST POST path also runs. - const logSource = await Source.create({ - kind: SourceKind.Log, - team: team._id, - from: { - databaseName: DEFAULT_DATABASE, - tableName: 'otel_logs', - }, - timestampValueExpression: 'Timestamp', - connection: connection._id, - name: 'Logs', - }); - - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Heatmap on Log Source', - tiles: [ - { - name: 'Heatmap', - config: { - displayType: 'heatmap', - sourceId: logSource._id.toString(), - select: [{ valueExpression: 'Duration' }], - }, - }, - ], - }); - - expect(result.isError).toBe(true); - const text = getFirstText(result); - expect(text).toContain('Trace source'); - expect(text).toContain(logSource._id.toString()); - }); - - // Exercises the update-side source-kind gate via filterChangedHeatmapTiles - // (displayType changed to heatmap on an existing tile). - it('should reject update that changes a tile to heatmap on a non-Trace source', async () => { - const logSource = await Source.create({ - kind: SourceKind.Log, - team: team._id, - from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, - timestampValueExpression: 'Timestamp', - connection: connection._id, - name: 'Logs', - }); - - const created = await callTool(client, 'clickstack_save_dashboard', { - name: 'Line on Log Source', - tiles: [ - { - name: 'Line', - config: { - displayType: 'line', - sourceId: logSource._id.toString(), - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - expect(created.isError).toBeFalsy(); - const saved = JSON.parse(getFirstText(created)); - - const update = await callTool(client, 'clickstack_save_dashboard', { - id: saved.id, - name: 'Line on Log Source', - tiles: [ - { - ...saved.tiles[0], - config: { - ...saved.tiles[0].config, - displayType: 'heatmap', - select: [{ valueExpression: 'Duration' }], - }, - }, - ], - }); - expect(update.isError).toBe(true); - const text = getFirstText(update); - expect(text).toContain('Trace source'); - expect(text).toContain(logSource._id.toString()); - }); - - // Exercises the update-side source-kind gate via filterChangedHeatmapTiles - // (sourceId changed on an existing heatmap tile). - it('should reject update that changes a heatmap tile source to a non-Trace source', async () => { - const logSource = await Source.create({ - kind: SourceKind.Log, - team: team._id, - from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, - timestampValueExpression: 'Timestamp', - connection: connection._id, - name: 'Logs', - }); - - const created = await callTool(client, 'clickstack_save_dashboard', { - name: 'Heatmap re-pointed at Log', - tiles: [ - { - name: 'Heatmap', - config: { - displayType: 'heatmap', - sourceId: traceSource._id.toString(), - select: [{ valueExpression: 'Duration' }], - }, - }, - ], - }); - expect(created.isError).toBeFalsy(); - const saved = JSON.parse(getFirstText(created)); - - const update = await callTool(client, 'clickstack_save_dashboard', { - id: saved.id, - name: 'Heatmap re-pointed at Log', - tiles: [ - { - ...saved.tiles[0], - config: { - ...saved.tiles[0].config, - sourceId: logSource._id.toString(), - }, - }, - ], - }); - expect(update.isError).toBe(true); - const text = getFirstText(update); - expect(text).toContain('Trace source'); - expect(text).toContain(logSource._id.toString()); - }); - - // Asserts each tile's config survives the serializer/deserializer cycle - // independently when mixed with other displayTypes on the same dashboard. - it('should round-trip a heatmap alongside line and number tiles in one dashboard', async () => { - const sourceId = traceSource._id.toString(); - - const heatmapConfig = { - displayType: 'heatmap' as const, - sourceId, - select: [ - { valueExpression: 'Duration', heatmapScaleType: 'log' as const }, - ], - }; - const lineConfig = { - displayType: 'line' as const, - sourceId, - select: [{ aggFn: 'count' as const }], - groupBy: "SpanAttributes['service.name']", - }; - const numberConfig = { - displayType: 'number' as const, - sourceId, - select: [{ aggFn: 'count' as const }], - numberFormat: { output: 'number' as const, average: true }, - }; - - const save = await callTool(client, 'clickstack_save_dashboard', { - name: 'Mixed Tile Round-Trip', - tiles: [ - { name: 'Heatmap Tile', config: heatmapConfig }, - { name: 'Line Tile', config: lineConfig }, - { name: 'Number Tile', config: numberConfig }, - ], - }); - expect(save.isError).toBeFalsy(); - const saved = JSON.parse(getFirstText(save)); - expect(saved.tiles).toHaveLength(3); - - const byName: Record = - Object.fromEntries( - saved.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), - ); - expect(byName['Heatmap Tile'].config).toMatchObject(heatmapConfig); - expect(byName['Line Tile'].config).toMatchObject(lineConfig); - expect(byName['Number Tile'].config).toMatchObject(numberConfig); - - const fetched = JSON.parse( - getFirstText( - await callTool(client, 'clickstack_get_dashboard', { id: saved.id }), - ), - ); - const fetchedByName: Record = - Object.fromEntries( - fetched.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), - ); - expect(fetchedByName['Heatmap Tile'].config).toMatchObject(heatmapConfig); - expect(fetchedByName['Line Tile'].config).toMatchObject(lineConfig); - expect(fetchedByName['Number Tile'].config).toMatchObject(numberConfig); - }); - }); - - describe('clickstack_save_dashboard - containers and tabs', () => { - // Mirrors the v2 external-API "Containers and tabs" describe block so - // the MCP path enforces the same 5 cross-field rules: container id - // unique, tab id unique within a container, tile.containerId - // resolves, tile.tabId resolves to a tab on that container, and - // tile.tabId requires tile.containerId. - const buildTile = ( - sourceId: string, - overrides: Record, - ) => ({ - name: 'Tile', - x: 0, - y: 0, - w: 6, - h: 3, - config: { - displayType: 'line' as const, - sourceId, - select: [{ aggFn: 'count' as const, where: '' }], - }, - ...overrides, - }); - - it('should round-trip containers, tabs, and tile containerId/tabId on create and update', async () => { - const sourceId = traceSource._id.toString(); - const containers = [ - { - id: 'service-health', - title: 'Service Health', - collapsed: false, - collapsible: true, - bordered: true, - tabs: [ - { id: 'errors', title: 'Errors' }, - { id: 'latency', title: 'Latency' }, - ], - }, - { - id: 'overview', - title: 'Overview', - collapsed: true, - }, - ]; - - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Containers MCP Round-Trip', - tiles: [ - buildTile(sourceId, { - name: 'In Group, Tab A', - containerId: 'service-health', - tabId: 'errors', - }), - buildTile(sourceId, { - name: 'In Group, Tab B', - containerId: 'service-health', - tabId: 'latency', - }), - // Tile inside a tabbed container without a tabId renders in - // the container shell rather than under a tab. Guards that the - // schema does not accidentally require tabId for every tile in - // a tabbed container. - buildTile(sourceId, { - name: 'In Tabbed Group, No Tab', - containerId: 'service-health', - }), - buildTile(sourceId, { - name: 'In Plain Group', - containerId: 'overview', - }), - buildTile(sourceId, { name: 'Ungrouped' }), - ], - tags: ['containers-mcp'], - containers, - }); - - expect(createResult.isError).toBeFalsy(); - const created = JSON.parse(getFirstText(createResult)); - expect(created.containers).toEqual(containers); - - // Typed as ExternalDashboardTileWithId so a typo on .containerId - // would fail typecheck instead of silently asserting undefined. - const createdTilesByName: Record = - Object.fromEntries( - created.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), - ); - expect(createdTilesByName['In Group, Tab A']).toMatchObject({ - containerId: 'service-health', - tabId: 'errors', - }); - expect(createdTilesByName['In Group, Tab B']).toMatchObject({ - containerId: 'service-health', - tabId: 'latency', - }); - expect(createdTilesByName['In Tabbed Group, No Tab']).toMatchObject({ - containerId: 'service-health', - }); - expect( - createdTilesByName['In Tabbed Group, No Tab'].tabId, - ).toBeUndefined(); - expect(createdTilesByName['In Plain Group']).toMatchObject({ - containerId: 'overview', - }); - expect(createdTilesByName['In Plain Group'].tabId).toBeUndefined(); - expect(createdTilesByName.Ungrouped.containerId).toBeUndefined(); - expect(createdTilesByName.Ungrouped.tabId).toBeUndefined(); - - // Verify a fresh GET via the get tool also returns the structure. - const getResult = await callTool(client, 'clickstack_get_dashboard', { - id: created.id, - }); - const fetched = JSON.parse(getFirstText(getResult)); - expect(fetched.containers).toEqual(containers); - - // Update: rename a tab, drop the second container, re-home tiles. - const updatedContainers = [ - { - id: 'service-health', - title: 'Service Health', - collapsed: true, - tabs: [ - { id: 'errors', title: 'Error Rate' }, - { id: 'latency', title: 'Latency' }, - ], - }, - ]; - const reHomedUngrouped = { - ...createdTilesByName.Ungrouped, - containerId: 'service-health', - tabId: 'errors', - }; - const droppedContainerTile = { - ...createdTilesByName['In Plain Group'], - containerId: undefined, - tabId: undefined, - }; - - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'Containers MCP Round-Trip', - tiles: [ - createdTilesByName['In Group, Tab A'], - createdTilesByName['In Group, Tab B'], - createdTilesByName['In Tabbed Group, No Tab'], - droppedContainerTile, - reHomedUngrouped, - ], - tags: ['containers-mcp'], - containers: updatedContainers, - }); - - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.containers).toEqual(updatedContainers); - const updatedTilesByName: Record = - Object.fromEntries( - updated.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), - ); - expect(updatedTilesByName['In Plain Group'].containerId).toBeUndefined(); - expect(updatedTilesByName.Ungrouped).toMatchObject({ - containerId: 'service-health', - tabId: 'errors', - }); - expect(updatedTilesByName['In Tabbed Group, No Tab'].containerId).toBe( - 'service-health', - ); - expect( - updatedTilesByName['In Tabbed Group, No Tab'].tabId, - ).toBeUndefined(); - }); - - it('should reject duplicate container ids', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Duplicate Container Ids', - tiles: [buildTile(sourceId, {})], - containers: [ - { id: 'dupe', title: 'A', collapsed: false }, - { id: 'dupe', title: 'B', collapsed: false }, - ], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Container IDs must be unique'); - }); - - it('should reject duplicate tab ids within a container', async () => { - // Structural rules (duplicate container ids, duplicate tab ids) fire - // through the body schema, which is reported as a stringified issue - // array. Cross-tile-ref rules (covered below) fire through - // collectTileContainerRefIssues and produce plain prose. - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Duplicate Tab Ids', - tiles: [buildTile(sourceId, {})], - containers: [ - { - id: 'service-health', - title: 'Service Health', - collapsed: false, - tabs: [ - { id: 'errors', title: 'Errors' }, - { id: 'errors', title: 'Errors Two' }, - ], - }, - ], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Duplicate tab id'); - expect(getFirstText(result)).toContain('errors'); - expect(getFirstText(result)).toContain('service-health'); - }); - - it('should reject a tile that supplies tabId without containerId', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Tab Without Container', - tiles: [buildTile(sourceId, { tabId: 'errors' })], - containers: [ - { - id: 'service-health', - title: 'Service Health', - collapsed: false, - tabs: [{ id: 'errors', title: 'Errors' }], - }, - ], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain( - 'tabId requires containerId to be set', - ); - }); - - it('should reject a tile that references an unknown containerId', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Unknown Container', - tiles: [buildTile(sourceId, { containerId: 'does-not-exist' })], - containers: [{ id: 'real', title: 'Real', collapsed: false }], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain( - 'unknown containerId "does-not-exist"', - ); - }); - - it('should reject a tile that references an unknown tabId within a container', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Unknown Tab', - tiles: [ - buildTile(sourceId, { - containerId: 'service-health', - tabId: 'ghost', - }), - ], - containers: [ - { - id: 'service-health', - title: 'Service Health', - collapsed: false, - tabs: [{ id: 'errors', title: 'Errors' }], - }, - ], - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('unknown tabId "ghost"'); - }); - - it('should preserve existing containers on update when body omits the field', async () => { - // Exercises the `effectiveContainers = parsedContainers ?? existingDashboard.containers ?? []` - // fallback in the MCP update handler. Without it, a PUT that - // updates only `tiles` would reject because the tile's - // containerId reference would resolve against an empty array. - const sourceId = traceSource._id.toString(); - const containers = [ - { - id: 'service-health', - title: 'Service Health', - collapsed: false, - tabs: [{ id: 'errors', title: 'Errors' }], - }, - ]; - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'PUT-without-containers fallback', - tiles: [ - buildTile(sourceId, { - name: 'In Group', - containerId: 'service-health', - tabId: 'errors', - }), - ], - containers, - }); - expect(createResult.isError).toBeFalsy(); - const created = JSON.parse(getFirstText(createResult)); - - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'PUT-without-containers fallback', - tiles: created.tiles, - // containers intentionally omitted - }); - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.containers).toEqual(containers); - expect(updated.tiles[0]).toMatchObject({ - containerId: 'service-health', - tabId: 'errors', - }); - }); - - it('should wipe persisted containers when update body sets containers: []', async () => { - // Mirror the v2 behavior: an explicit empty array clears the - // persisted containers, and the response normalizes [] back to - // absent on read. - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Wipe containers', - tiles: [buildTile(sourceId, { name: 'Tile' })], - containers: [{ id: 'overview', title: 'Overview', collapsed: false }], - }); - expect(createResult.isError).toBeFalsy(); - const created = JSON.parse(getFirstText(createResult)); - expect(created.containers).toHaveLength(1); - - const wipedTile = { - ...created.tiles[0], - containerId: undefined, - tabId: undefined, - }; - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'Wipe containers', - tiles: [wipedTile], - containers: [], - }); - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.containers).toBeUndefined(); - - const getResult = await callTool(client, 'clickstack_get_dashboard', { - id: created.id, - }); - const fetched = JSON.parse(getFirstText(getResult)); - expect(fetched.containers).toBeUndefined(); - }); - - // Bounds mirror DASHBOARD_CONTAINER_ID_MAX (256) on the tile-level - // containerId / tabId. 256 chars must accept; 257 must reject. - // 257 trips the inputSchema's `.max(256)` and surfaces back as the - // MCP SDK's "Input validation error" envelope. - it('should accept a 256-char tile.containerId and reject 257', async () => { - const sourceId = traceSource._id.toString(); - const idAtMax = 'c'.repeat(256); - const okResult = await callTool(client, 'clickstack_save_dashboard', { - name: '256-char containerId', - tiles: [buildTile(sourceId, { containerId: idAtMax })], - containers: [{ id: idAtMax, title: 'Max', collapsed: false }], - }); - expect(okResult.isError).toBeFalsy(); - - const idTooLong = 'c'.repeat(257); - const tooLongResult = await callTool( - client, - 'clickstack_save_dashboard', - { - name: '257-char containerId', - tiles: [buildTile(sourceId, { containerId: idTooLong })], - containers: [{ id: idTooLong, title: 'Too long', collapsed: false }], - }, - ); - expect(tooLongResult.isError).toBe(true); - expect(getFirstText(tooLongResult)).toContain( - 'String must contain at most 256 character(s)', - ); - expect(getFirstText(tooLongResult)).toContain('containerId'); - }); - }); - - describe('clickstack_save_dashboard - dashboard filters', () => { - // The MCP input schema delegates to createDashboardBodySchema / - // updateDashboardBodySchema for the filter shape. These tests guard - // that the MCP path lights up the same filter round-trip the v2 REST - // path already covers: filters are saved verbatim, get back with an - // assigned id, survive an update, and reject obvious bad inputs. - - const traceTile = (sourceId: string) => ({ - name: 'Volume', - x: 0, - y: 0, - w: 6, - h: 3, - config: { - displayType: 'line' as const, - sourceId, - select: [{ aggFn: 'count' as const, where: '' }], - }, - }); - - it('should round-trip filters on create, get, and update', async () => { - const sourceId = traceSource._id.toString(); - - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Service Detail (MCP filter round-trip)', - tiles: [traceTile(sourceId)], - filters: [ - { - type: 'QUERY_EXPRESSION', - name: 'Service', - expression: 'ServiceName', - sourceId, - }, - { - type: 'QUERY_EXPRESSION', - name: 'Environment', - expression: 'deployment.environment', - sourceId, - where: "deployment.environment = 'production'", - whereLanguage: 'sql', - }, - ], - }); - - expect(createResult.isError).toBeFalsy(); - const created = JSON.parse(getFirstText(createResult)); - expect(Array.isArray(created.filters)).toBe(true); - expect(created.filters).toHaveLength(2); - // The body schema assigns an id to each filter on create. Capture - // them so the update payload can include the same ids and the - // filter array round-trips identically (instead of being treated - // as wholesale replacement with new ids). - const [serviceFilter, envFilter] = created.filters; - expect(serviceFilter).toMatchObject({ - type: 'QUERY_EXPRESSION', - name: 'Service', - expression: 'ServiceName', - sourceId, - }); - expect(typeof serviceFilter.id).toBe('string'); - expect(serviceFilter.id.length).toBeGreaterThan(0); - expect(envFilter).toMatchObject({ - type: 'QUERY_EXPRESSION', - name: 'Environment', - expression: 'deployment.environment', - sourceId, - where: "deployment.environment = 'production'", - whereLanguage: 'sql', - }); - - // Fetch and assert the same shape. - const getResult = await callTool(client, 'clickstack_get_dashboard', { - id: created.id, - }); - const fetched = JSON.parse(getFirstText(getResult)); - expect(fetched.filters).toEqual(created.filters); - - // Update: rename the first filter and drop the second. The first - // filter keeps its id, the second is dropped (not preserved). - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'Service Detail (MCP filter round-trip)', - tiles: [traceTile(sourceId)], - filters: [ - { - id: serviceFilter.id, - type: 'QUERY_EXPRESSION', - name: 'Service (renamed)', - expression: 'ServiceName', - sourceId, - }, - ], - }); - - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.filters).toHaveLength(1); - expect(updated.filters[0]).toMatchObject({ - id: serviceFilter.id, - type: 'QUERY_EXPRESSION', - name: 'Service (renamed)', - expression: 'ServiceName', - sourceId, - }); - }); - - it('should accept a create payload with stray filter ids (copy-paste round-trip)', async () => { - // An LLM that copies a filter out of clickstack_get_dashboard and - // back into clickstack_save_dashboard for a NEW dashboard ships an - // `id` on the filter. The body schema for create is - // `.strict()` and rejects `id`, so saveDashboard normalizes by - // stripping ids before validation. Without that normalization - // this payload would 4xx with a confusing strict-validation - // error. - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Create with stray filter id', - tiles: [traceTile(sourceId)], - filters: [ - { - id: '999999999999999999999999', // simulates a copied id - type: 'QUERY_EXPRESSION', - name: 'Service', - expression: 'ServiceName', - sourceId, - }, - ], - }); - expect(result.isError).toBeFalsy(); - const dashboard = JSON.parse(getFirstText(result)); - expect(dashboard.filters).toHaveLength(1); - // The new dashboard gets a fresh id; the copied one is discarded. - expect(dashboard.filters[0].id).not.toBe('999999999999999999999999'); - expect(dashboard.filters[0]).toMatchObject({ - type: 'QUERY_EXPRESSION', - name: 'Service', - expression: 'ServiceName', - sourceId, - }); - }); - - it('should assign ids to new filters added during update (no-id round-trip)', async () => { - // An LLM updating a dashboard to ADD a brand-new filter ships - // a filter without an `id`. The body schema for update requires - // id, so saveDashboard fills one in. Without that normalization - // this payload would 4xx complaining that id is missing. - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Update adds new filter', - tiles: [traceTile(sourceId)], - filters: [ - { - type: 'QUERY_EXPRESSION', - name: 'Service', - expression: 'ServiceName', - sourceId, - }, - ], - }); - const created = JSON.parse(getFirstText(createResult)); - const existingFilterId = created.filters[0].id; - - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'Update adds new filter', - tiles: [traceTile(sourceId)], - filters: [ - // Existing filter, preserved by id. - { - id: existingFilterId, - type: 'QUERY_EXPRESSION', - name: 'Service', - expression: 'ServiceName', - sourceId, - }, - // Brand-new filter, no id. saveDashboard assigns one. - { - type: 'QUERY_EXPRESSION', - name: 'Environment', - expression: 'deployment.environment', - sourceId, - }, - ], - }); - expect(updateResult.isError).toBeFalsy(); - const updated = JSON.parse(getFirstText(updateResult)); - expect(updated.filters).toHaveLength(2); - const envFilter = updated.filters.find( - (f: { name: string }) => f.name === 'Environment', - ); - expect(envFilter).toBeDefined(); - expect(typeof envFilter.id).toBe('string'); - expect(envFilter.id.length).toBeGreaterThan(0); - expect(envFilter.id).not.toBe(existingFilterId); - }); - - it('should round-trip a table tile that uses a having clause', async () => { - // mcpTableTileSchema exposes `having` so the service_detail - // example's "Top Error Messages" pattern (groupBy StatusMessage - // with having: "StatusMessage != ''") survives MCP authoring. - // Without `having` on the schema, Zod would strip it and the - // saved tile would include empty-message rows instead of the - // filtered set the prompt promises. - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'Top Error Messages', - tiles: [ - { - name: 'Top Error Messages', - x: 0, - y: 0, - w: 12, - h: 6, - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count', alias: 'Count' }], - groupBy: 'StatusMessage', - having: "StatusMessage != ''", - orderBy: 'Count DESC', - groupByColumnsOnLeft: true, - }, - }, - ], - }); - expect(result.isError).toBeFalsy(); - const dashboard = JSON.parse(getFirstText(result)); - expect(dashboard.tiles[0].config.having).toBe("StatusMessage != ''"); - }); - }); - - describe('clickstack_save_dashboard - table onClick linking', () => { - // Mirrors the v2 external-API onClick tests so the MCP path - // enforces the same drill-down rules: row-click can target /search - // for a log/trace source or another dashboard, by concrete ID or - // by templated name, with optional whereTemplate/filters. - it('should round-trip a table tile with a search onClick by ID', async () => { - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick search by id', - tiles: [ - { - name: 'Errors by Service', - x: 0, - y: 0, - w: 12, - h: 6, - config: { - displayType: 'table', - sourceId, - groupBy: "ResourceAttributes['service.name']", - select: [{ aggFn: 'count' }], - onClick: { - type: 'search', - target: { mode: 'id', id: sourceId }, - whereLanguage: 'sql', - filters: [ - { - kind: 'expressionTemplate', - expression: 'ServiceName', - template: "{{ResourceAttributes['service.name']}}", - }, - ], - }, - }, - }, - ], - }); - - expect(createResult.isError).toBeFalsy(); - const created = JSON.parse(getFirstText(createResult)); - expect(created.tiles[0].config.onClick).toEqual({ - type: 'search', - target: { mode: 'id', id: sourceId }, - whereLanguage: 'sql', - filters: [ - { - kind: 'expressionTemplate', - expression: 'ServiceName', - template: "{{ResourceAttributes['service.name']}}", - }, - ], - }); - - const getResult = await callTool(client, 'clickstack_get_dashboard', { - id: created.id, - }); - const fetched = JSON.parse(getFirstText(getResult)); - expect(fetched.tiles[0].config.onClick).toEqual( - created.tiles[0].config.onClick, - ); - }); - - it('should round-trip a table tile with a dashboard onClick by ID', async () => { - const sourceId = traceSource._id.toString(); - // Create the target dashboard the onClick will link to so the - // server-side `getMissingOnClickDashboards` check resolves. - const targetDashboard = await new Dashboard({ - name: 'Service Detail', - tiles: [], - team: team._id, - }).save(); - - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick dashboard by id', - tiles: [ - { - name: 'Top Services', - x: 0, - y: 0, - w: 12, - h: 6, - config: { - displayType: 'table', - sourceId, - groupBy: "ResourceAttributes['service.name']", - select: [{ aggFn: 'count' }], - onClick: { - type: 'dashboard', - target: { - mode: 'id', - id: targetDashboard._id.toString(), - }, - whereLanguage: 'sql', - filters: [ - { - kind: 'expressionTemplate', - expression: 'ServiceName', - template: "{{ResourceAttributes['service.name']}}", - }, - ], - }, - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles[0].config.onClick).toEqual({ - type: 'dashboard', - target: { mode: 'id', id: targetDashboard._id.toString() }, - whereLanguage: 'sql', - filters: [ - { - kind: 'expressionTemplate', - expression: 'ServiceName', - template: "{{ResourceAttributes['service.name']}}", - }, - ], - }); - }); - - it('should round-trip a templated dashboard onClick (mode=template)', async () => { - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick dashboard by template', - tiles: [ - { - name: 'Service Picker', - x: 0, - y: 0, - w: 12, - h: 6, - config: { - displayType: 'table', - sourceId, - groupBy: 'SpanName', - select: [{ aggFn: 'count' }], - onClick: { - type: 'dashboard', - target: { - mode: 'template', - template: '{{TargetDashboard}}', - }, - whereLanguage: 'lucene', - whereTemplate: - "service.name:{{ResourceAttributes['service.name']}}", - }, - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles[0].config.onClick).toEqual({ - type: 'dashboard', - target: { mode: 'template', template: '{{TargetDashboard}}' }, - whereLanguage: 'lucene', - whereTemplate: "service.name:{{ResourceAttributes['service.name']}}", - }); - }); - - it('should round-trip onClick on a raw SQL table tile', async () => { - const connectionId = connection._id.toString(); - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'SQL onClick', - tiles: [ - { - name: 'Raw SQL Table', - x: 0, - y: 0, - w: 12, - h: 6, - config: { - configType: 'sql', - displayType: 'table', - connectionId, - sqlTemplate: - 'SELECT ServiceName, count() AS c FROM otel_traces GROUP BY ServiceName LIMIT 50', - onClick: { - type: 'search', - target: { mode: 'id', id: sourceId }, - whereLanguage: 'sql', - filters: [ - { - kind: 'expressionTemplate', - expression: 'ServiceName', - template: '{{ServiceName}}', - }, - ], - }, - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles[0].config.onClick).toEqual({ - type: 'search', - target: { mode: 'id', id: sourceId }, - whereLanguage: 'sql', - filters: [ - { - kind: 'expressionTemplate', - expression: 'ServiceName', - template: '{{ServiceName}}', - }, - ], - }); - }); - - it('should reject a table tile onClick referencing a non-existent dashboard', async () => { - const sourceId = traceSource._id.toString(); - const ghostDashboardId = new mongoose.Types.ObjectId().toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick missing dashboard', - tiles: [ - { - name: 'Top Services', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - onClick: { - type: 'dashboard', - target: { mode: 'id', id: ghostDashboardId }, - whereLanguage: 'sql', - }, - }, - }, - ], - }); - - expect(result.isError).toBe(true); - const text = getFirstText(result); - expect(text).toContain('onClick dashboard'); - expect(text).toContain(ghostDashboardId); - }); - - it('should reject a table tile onClick search target with a non-log/trace source', async () => { - // /search only renders log/trace sources; targeting a metric - // source must be rejected at save time, matching the REST POST - // path. Create the metric source inline so this test is - // self-contained. - const metricSource = await Source.create({ - kind: SourceKind.Metric, - team: team._id, - from: { - databaseName: DEFAULT_DATABASE, - tableName: '', - }, - metricTables: { - [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', - [MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum', - [MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram', - }, - timestampValueExpression: 'TimeUnix', - connection: connection._id, - name: 'Metrics', - }); - const sourceId = traceSource._id.toString(); - - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick metric source', - tiles: [ - { - name: 'Top Services', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - onClick: { - type: 'search', - target: { - mode: 'id', - id: metricSource._id.toString(), - }, - whereLanguage: 'sql', - }, - }, - }, - ], - }); - - expect(result.isError).toBe(true); - const text = getFirstText(result); - expect(text).toContain('log or trace'); - expect(text).toContain(metricSource._id.toString()); - }); - - it('should reject a table tile onClick with an invalid target.id', async () => { - // target.id must be a valid ObjectId. Bad shape is caught by the - // input schema and surfaces as the SDK's "Input validation error" - // envelope rather than a custom 400 message. - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick bad object id', - tiles: [ - { - name: 'Top Services', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - onClick: { - type: 'dashboard', - target: { mode: 'id', id: 'not-a-valid-object-id' }, - whereLanguage: 'sql', - }, - }, - }, - ], - }); - - expect(result.isError).toBe(true); - }); - - it('should accept onClick without whereLanguage (it is optional at the schema layer)', async () => { - // SearchConditionLanguageSchema is `.optional()` in common-utils, - // so omitting whereLanguage is allowed. Pin that behavior so a - // future tightening of the schema doesn't quietly change it - // without updating the docs that callers rely on. - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick no whereLanguage', - tiles: [ - { - name: 'Top Services', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - onClick: { - type: 'search', - target: { mode: 'id', id: sourceId }, - // whereLanguage intentionally omitted - }, - }, - }, - ], - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.tiles[0].config.onClick).toMatchObject({ - type: 'search', - target: { mode: 'id', id: sourceId }, - }); - }); - - it('should reject an unknown onClick.type', async () => { - // The schema is a discriminated union on `type`; a bogus type - // must be rejected up front. - const sourceId = traceSource._id.toString(); - const result = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick bad type', - tiles: [ - { - name: 'Top Services', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - onClick: { - type: 'navigate-to-mars', - target: { mode: 'id', id: sourceId }, - whereLanguage: 'sql', - }, - }, - }, - ], - }); - - expect(result.isError).toBe(true); - }); - - it('should validate onClick on update too', async () => { - // Round-trip a tile with no onClick, then update it to add an - // onClick pointing at a non-existent dashboard. The PUT path - // mirrors POST: missing dashboards are rejected with the same - // message. - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'OnClick update validation', - tiles: [ - { - name: 'Top Services', - config: { - displayType: 'table', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - expect(createResult.isError).toBeFalsy(); - const created = JSON.parse(getFirstText(createResult)); - - const ghostDashboardId = new mongoose.Types.ObjectId().toString(); - const updateResult = await callTool(client, 'clickstack_save_dashboard', { - id: created.id, - name: 'OnClick update validation', - tiles: [ - { - ...created.tiles[0], - config: { - ...created.tiles[0].config, - onClick: { - type: 'dashboard', - target: { mode: 'id', id: ghostDashboardId }, - whereLanguage: 'sql', - }, - }, - }, - ], - }); - - expect(updateResult.isError).toBe(true); - const text = getFirstText(updateResult); - expect(text).toContain('onClick dashboard'); - expect(text).toContain(ghostDashboardId); - }); - }); - - describe('clickstack_delete_dashboard', () => { - it('should delete an existing dashboard', async () => { - const dashboard = await new Dashboard({ - name: 'To Delete', - tiles: [], - team: team._id, - }).save(); - - const result = await callTool(client, 'clickstack_delete_dashboard', { - id: dashboard._id.toString(), - }); - - expect(result.isError).toBeFalsy(); - const output = JSON.parse(getFirstText(result)); - expect(output.deleted).toBe(true); - expect(output.id).toBe(dashboard._id.toString()); - - // Verify deleted from database - const found = await Dashboard.findById(dashboard._id); - expect(found).toBeNull(); - }); - - it('should return error for non-existent dashboard', async () => { - const result = await callTool(client, 'clickstack_delete_dashboard', { - id: '000000000000000000000000', - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('not found'); - }); - }); - - describe('clickstack_query_tile', () => { - it('should return error for non-existent dashboard', async () => { - const result = await callTool(client, 'clickstack_query_tile', { - dashboardId: '000000000000000000000000', - tileId: 'some-tile-id', - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('not found'); - }); - - it('should return error for non-existent tile', async () => { - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Tile Query Test', - tiles: [ - { - name: 'My Tile', - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - const dashboard = JSON.parse(getFirstText(createResult)); - - const result = await callTool(client, 'clickstack_query_tile', { - dashboardId: dashboard.id, - tileId: 'non-existent-tile-id', - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Tile not found'); - }); - - it('should return error for invalid time range', async () => { - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Time Range Test', - tiles: [ - { - name: 'Tile', - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - const dashboard = JSON.parse(getFirstText(createResult)); - - const result = await callTool(client, 'clickstack_query_tile', { - dashboardId: dashboard.id, - tileId: dashboard.tiles[0].id, - startTime: 'not-a-date', - }); - - expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Invalid'); - }); - - it('should execute query for a valid tile', async () => { - const sourceId = traceSource._id.toString(); - const createResult = await callTool(client, 'clickstack_save_dashboard', { - name: 'Query Tile Test', - tiles: [ - { - name: 'Count Tile', - config: { - displayType: 'number', - sourceId, - select: [{ aggFn: 'count' }], - }, - }, - ], - }); - const dashboard = JSON.parse(getFirstText(createResult)); - - const result = await callTool(client, 'clickstack_query_tile', { - dashboardId: dashboard.id, - tileId: dashboard.tiles[0].id, - startTime: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), - endTime: new Date().toISOString(), - }); - - // Should succeed (may have empty results since no data inserted) - expect(result.isError).toBeFalsy(); - expect(result.content).toHaveLength(1); - }); - }); - - describe('buildQueryGuidePrompt', () => { - it('documents heatmap in tile-type, constraints, mistakes, and aggFn sections', () => { - const prompt = buildQueryGuidePrompt(); - // Ensure each section the bot called out has a heatmap entry so - // the LLM cannot skip the constraints when producing a heatmap - // tile through clickstack_save_dashboard. - const sections = [ - '== AGGREGATION FUNCTIONS (aggFn) ==', - '== PER-TILE TYPE CONSTRAINTS ==', - '== COMMON MISTAKES ==', - ]; - for (const heading of sections) { - const idx = prompt.indexOf(heading); - expect(idx).toBeGreaterThan(-1); - // Each section's body extends until the next == heading or the - // end of the string. Checking for the substring "heatmap" in - // that slice is enough to assert the heatmap entry exists. - const next = prompt.indexOf('\n== ', idx + heading.length); - const body = prompt.slice(idx, next === -1 ? prompt.length : next); - expect(body.toLowerCase()).toContain('heatmap'); - } - }); - - it('documents the dashboard filter and per-series numberFormat sections', () => { - const prompt = buildQueryGuidePrompt(); - // Sections must exist AND carry the substantive content. Heading- - // only assertions let a future contributor empty out a section - // and silently pass review. - const sliceSection = (heading: string) => { - const idx = prompt.indexOf(heading); - if (idx === -1) return ''; - const next = prompt.indexOf('\n== ', idx + heading.length); - return prompt.slice(idx, next === -1 ? prompt.length : next); - }; - const filters = sliceSection('== DASHBOARD FILTERS =='); - expect(filters).toContain('QUERY_EXPRESSION'); - expect(filters).toContain('expression'); - expect(filters).toContain('sourceId'); - const numberFormat = sliceSection('== NUMBER FORMAT =='); - expect(numberFormat).toContain('factor: 0.000000001'); - expect(numberFormat).toContain('duration'); - expect(numberFormat.toLowerCase()).toContain('per-series'); - }); - - it('declares the MCP metric-authoring gap rather than referencing fields that do not exist', () => { - // The MCP select-item schema does not carry metricName / metricType, - // so any prompt that tells the model to author a metric tile via - // the builder path is teaching it to ship JSON that gets silently - // stripped. Guard with an explicit assertion so a future diff that - // restores metric-tile guidance fails this test loudly. - const prompt = buildQueryGuidePrompt(); - expect(prompt).not.toMatch(/metricName \+ metricType/); - expect(prompt).not.toMatch(/exactly 1 select item with metricName/); - const constraintsIdx = prompt.indexOf('== PER-TILE TYPE CONSTRAINTS =='); - const constraintsBody = prompt.slice(constraintsIdx); - // The constraints section closes with an explicit note that builder - // tiles on a metric source are not reliable today, with a fallback - // recipe to raw SQL. Anchor on the phrase so a future diff that - // drops the gap-acknowledgement fails loudly. - expect(constraintsBody).toMatch( - /Authoring builder tiles on a metric source is not reliable/, - ); - expect(constraintsBody).toMatch(/MCP select-item shape does not carry/); - }); - - it('documents table-tile onClick linking features', () => { - // Lock down the documentation for row-click drill-downs so a - // future refactor can't quietly drop the section the LLM relies - // on to wire up onClick correctly. - const prompt = buildQueryGuidePrompt(); - const idx = prompt.indexOf('== TABLE TILE LINKING (config.onClick) =='); - expect(idx).toBeGreaterThan(-1); - const next = prompt.indexOf('\n== ', idx + 1); - const section = prompt.slice(idx, next === -1 ? prompt.length : next); - - // Both destination types are mentioned. - expect(section).toContain('type: "search"'); - expect(section).toContain('type: "dashboard"'); - // Both target modes are mentioned. - expect(section).toContain('mode: "id"'); - expect(section).toContain('mode: "template"'); - // Templating fields are mentioned. - expect(section).toContain('whereTemplate'); - expect(section).toContain('filters'); - expect(section).toContain('expressionTemplate'); - // The two server-side validation error messages are quoted so - // the LLM can recognize and recover from them. - expect(section).toContain('onClick search source IDs'); - expect(section).toContain('onClick dashboard IDs'); - }); - - it('documents onClick pitfalls under common mistakes', () => { - const prompt = buildQueryGuidePrompt(); - const idx = prompt.indexOf('== COMMON MISTAKES =='); - expect(idx).toBeGreaterThan(-1); - const section = prompt.slice(idx); - // Mention the four failure modes we hit during validation. - expect(section.toLowerCase()).toContain('onclick on a non-table tile'); - expect(section.toLowerCase()).toContain('non-log/trace source'); - expect(section.toLowerCase()).toContain('missing wherelanguage'); - expect(section.toLowerCase()).toContain("isn't in the table"); - }); - - it('documents the missing-alias mistake explicitly', () => { - // Claude built three number tiles without alias on the quantile() - // select item, even though rule 2 of the design checklist says - // ALIAS EVERY SELECT ITEM. Spelling out the mistake under COMMON - // MISTAKES gives the model a second touchpoint to catch it. - const prompt = buildQueryGuidePrompt(); - const idx = prompt.indexOf('== COMMON MISTAKES =='); - const section = prompt.slice(idx); - expect(section).toMatch(/Missing alias on a select item/); - // Must call out that number tiles need the alias too; this is the - // exact case Claude got wrong. - expect(section).toMatch(/number tiles too/); - }); - - it('documents the Lucene map-attribute gotcha across all operators including simple equality', () => { - // The original rule covered only comparison/wildcard. Claude's - // second pass showed simple equality (db.system:mongodb) also - // fails to translate to SpanAttributes['db.system']. Broaden the - // gotcha to "ALL Lucene operations on map-attribute paths". - const prompt = buildQueryGuidePrompt(); - const luceneIdx = prompt.indexOf('== LUCENE FILTER SYNTAX =='); - const luceneBody = prompt.slice(luceneIdx); - expect(luceneBody).toMatch(/NOT reliably translated to bracket access/); - expect(luceneBody).toMatch(/http\.status_code:>=500/); // operator - expect(luceneBody).toMatch(/http\.route:\*/); // wildcard - expect(luceneBody).toMatch(/db\.system:mongodb/); // simple equality - expect(luceneBody).toMatch(/use SQL with bracket access/); - - // Also a top-level entry in COMMON MISTAKES for discoverability. - const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); - const mistakesBody = prompt.slice(mistakesIdx); - expect(mistakesBody).toMatch( - /Lucene on a map-attribute path \(any operation\)/, - ); - }); - - it('documents the Lucene fuzzy-substring behavior on top-level columns', () => { - // Lucene field:value translates to ilike(field, '%value%'), not - // equality. Claude's second pass hit this on SpanKind:Server, - // which matched broader strings than intended. The note has to - // be in the Lucene section so a model reading top-down sees it - // before writing a SpanKind filter. - const prompt = buildQueryGuidePrompt(); - const luceneIdx = prompt.indexOf('== LUCENE FILTER SYNTAX =='); - const luceneBody = prompt.slice(luceneIdx); - expect(luceneBody).toMatch(/FUZZY-MATCH NOTE/); - expect(luceneBody).toMatch(/ilike\(field, '%value%'\)/); - // The canonical fix is SQL with =. - expect(luceneBody).toMatch(/SpanKind = 'Server'/); - - // And a COMMON MISTAKES entry covering enum-like columns. - const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); - const mistakesBody = prompt.slice(mistakesIdx); - expect(mistakesBody).toMatch( - /Lucene field:value on enum-like top-level columns/, - ); - }); - - it('documents the empty-string trap for map-attribute groupBy', () => { - // Map columns return '' (not NULL) when a key is unset on a row. - // A groupBy on such a key produces an empty-string bucket alongside - // the real values, which is visual noise on tables and pies. The - // standard fix is where: " != ''" alongside the groupBy. - const prompt = buildQueryGuidePrompt(); - const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); - const mistakesBody = prompt.slice(mistakesIdx); - expect(mistakesBody).toMatch( - /Forgetting that map-attribute values are often empty strings/, - ); - expect(mistakesBody).toMatch(/db\.collection\.name'\] != ''/); - }); - - it('documents the groupBy alias workaround for map-attribute drill-downs', () => { - // Builder tiles don't expose a groupBy alias today; map-attribute - // groupBys produce result columns named like the raw expression, - // which onClick template lookups can't reference. The workaround - // (author the tile as raw SQL with AS) needs to be findable. - const prompt = buildQueryGuidePrompt(); - const idx = prompt.indexOf( - '== GROUPBY ALIASES AND ROW-CLICK TEMPLATES ==', - ); - expect(idx).toBeGreaterThan(-1); - const section = prompt.slice(idx); - expect(section).toMatch(/do NOT currently expose a groupBy alias/); - expect(section).toMatch(/configType: "sql", displayType: "table"/); - expect(section).toMatch(/AS Route/); - }); - - it('points at ClickStack and ClickHouse docs in REFERENCES', () => { - // Without external doc links, the LLM has to guess at Lucene - // operators and ClickHouse function names. Anchor the canonical - // URLs in the prompt so it can cite them and so a future doc - // restructure can be caught here. - const prompt = buildQueryGuidePrompt(); - const idx = prompt.indexOf('== REFERENCES =='); - expect(idx).toBeGreaterThan(-1); - const section = prompt.slice(idx); - expect(section).toContain( - 'https://clickhouse.com/docs/use-cases/observability/clickstack/search', - ); - expect(section).toContain('https://clickhouse.com/docs/sql-reference'); - expect(section).toContain( - 'https://clickhouse.com/docs/use-cases/observability/clickstack/dashboards/sql-visualizations', - ); - }); - - it('strengthens the validate-every-tile rule under common mistakes', () => { - // Saving validates input shape, not query semantics. The prompt - // has to push for query_tile on every tile, not "at least one". - const prompt = buildQueryGuidePrompt(); - const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); - const section = prompt.slice(mistakesIdx); - expect(section).toMatch( - /on EVERY tile after clickstack_save_dashboard, not just one/, - ); - }); - - it('flags the metric source builder gap', () => { - // Builder tiles on metric sources currently save but render with - // "Both table name and UUID are empty". Claude went 100% raw SQL - // across 21 metric tiles for this reason; the prompt has to make - // the workaround obvious so the model doesn't try the builder - // first and fail silently. - const prompt = buildQueryGuidePrompt(); - expect(prompt).toMatch(/Both table name and UUID are empty/); - expect(prompt).toMatch(/use a raw SQL tile/); - expect(prompt).toMatch(/otel_metrics_gauge/); - }); - - it('contains no em-dashes or en-dashes used as em-dashes', () => { - const prompt = buildQueryGuidePrompt(); - expect(prompt).not.toMatch(/—/); // [prose-lint: allow] - expect(prompt).not.toMatch(/ – /); // [prose-lint: allow] - }); - }); - - describe('buildSourceSummary', () => { - it('contains no em-dashes or en-dashes used as em-dashes', () => { - // buildSourceSummary is the source-list block that gets prepended - // to create_dashboard. The first iteration of this PR shipped with - // em-dashes here that the content.ts snapshot test missed because - // it calls the builders directly. Guard the helper output too. - const summary = buildSourceSummary( - [ - { - _id: '000000000000000000000001', - name: 'Traces', - kind: 'trace', - connection: '000000000000000000000002', - }, - { - _id: '000000000000000000000003', - name: 'Logs', - kind: 'log', - connection: '000000000000000000000002', - }, - ], - [{ _id: '000000000000000000000002', name: 'Default' }], - ); - expect(summary).not.toMatch(/—/); // [prose-lint: allow] - expect(summary).not.toMatch(/ – /); // [prose-lint: allow] - // Sanity: the helper still emits the source list, so the assertion - // above isn't trivially passing on an empty string. - expect(summary).toContain('Traces'); - expect(summary).toContain('AVAILABLE SOURCES'); - }); - }); - - describe('buildCreateDashboardPrompt', () => { - it('includes the design checklist and adapt-do-not-copy note', () => { - const prompt = buildCreateDashboardPrompt( - 'sources summary', - '000000000000000000000001', - '000000000000000000000002', - ); - expect(prompt).toContain('== DESIGN CHECKLIST =='); - // Each rule on the checklist exists at the same numbered position - // so a future contributor cannot silently drop one. Anchor on - // line start (with a trailing space) so digit-and-dot substrings - // like "0.000000001" elsewhere in the prompt cannot satisfy the - // assertion accidentally. - const checklistIdx = prompt.indexOf('== DESIGN CHECKLIST =='); - const adaptIdx = prompt.indexOf('== ADAPT, DO NOT COPY =='); - const checklistBody = prompt.slice(checklistIdx, adaptIdx); - // Thirteen rules: the original ten plus GROUP BY HAS NO ALIAS HOOK - // (rule 3), VALIDATE EVERY TILE AFTER SAVE (rule 12), and NO - // TITLE-RECAP MARKDOWN TILE (rule 13). Each came out of a live - // verification pass after watching Claude reliably ignore the soft - // "should" formulations or hit a schema gap the earlier checklist - // did not call out. - for (let i = 1; i <= 13; i++) { - expect(checklistBody).toMatch(new RegExp(`^${i}\\. `, 'm')); - } - expect(prompt).toContain('ADAPT, DO NOT COPY'); - // Rule 9 documents the replace-not-merge semantic on update; an - // agent that ignores this can silently drop tiles / filters / containers - // on a subsequent save call. - expect(checklistBody).toMatch(/UPDATE IS REPLACE, NOT MERGE/); - // Rule 2 must cover EVERY select item (including number tiles), not - // just aggregations on tables. The phrasing matters: Claude's first - // pass at this dropped aliases on number tiles because it read rule 2 - // as table-specific. The stronger phrasing pulls them in. - expect(checklistBody).toMatch(/ALIAS EVERY SELECT ITEM/); - // Rule 3 calls out the schema gap on groupBy: the chart config takes - // a single expression string with no alias field, so a table grouped - // by SpanAttributes['http.route'] renders arrayElement(...) as the - // column header. Naming the limit explicitly lets Claude pick a - // top-level column (SpanName, ServiceName) instead of grouping on - // the raw Map expression. - expect(checklistBody).toMatch(/GROUP BY HAS NO ALIAS HOOK/); - // Rule 10 must be a hard requirement at 5+ tiles, not a soft hint. - // Without the imperative, Claude built five dashboards averaging ten - // tiles each with zero containers. - expect(checklistBody).toMatch(/REQUIRED at five or more tiles/); - // Rule 11 + 12 close the post-save loop and the title-recap markdown - // habit Claude landed on every starter dashboard. - expect(checklistBody).toMatch(/VALIDATE EVERY TILE AFTER SAVE/); - expect(checklistBody).toMatch(/NO TITLE-RECAP MARKDOWN TILE/); - // Rule 2 now carries a concrete number-tile example. Claude's - // second pass still dropped aliases on every builder number tile, - // because the text rule was abstract; a copy-pasteable shape next - // to the rule is what makes it stick. - expect(checklistBody).toMatch( - /Number tile, correct:.*alias: "Server Requests"/, - ); - // Rule 10 now carries a concrete containers shape. Claude's - // second pass built five dashboards with 9-10 tiles each and zero - // containers despite the REQUIRED phrasing; a copy-pasteable shape - // inside the rule itself is the next-level push. - expect(checklistBody).toMatch(/Concrete shape \(copy this directly/); - expect(checklistBody).toMatch(/id: "kpis"/); - expect(checklistBody).toMatch(/id: "trends"/); - expect(checklistBody).toMatch(/id: "errors"/); - }); - - it('walks the workflow through six steps including read-existing and group-into-containers', () => { - // The workflow is the first thing a model anchors on. Claude - // skipped both "read existing dashboards before designing" and - // "group tiles into containers before saving" because neither was - // a numbered workflow step; both lived only in the checklist. The - // workflow now lists six steps with those two explicitly. - const prompt = buildCreateDashboardPrompt( - 'summary', - 'trace_src', - 'log_src', - ); - const wfIdx = prompt.indexOf('== WORKFLOW =='); - const wfEnd = prompt.indexOf('\n== ', wfIdx + 1); - const wf = prompt.slice(wfIdx, wfEnd); - for (let i = 1; i <= 6; i++) { - expect(wf).toMatch(new RegExp(`^${i}\\. `, 'm')); - } - // Step 2 must point at clickstack_get_dashboard with no id. - expect(wf).toMatch(/clickstack_get_dashboard \(no id\)/); - // Step 4 must call out grouping tiles into containers before save. - expect(wf).toMatch(/group them into 2-4 containers/); - // Step 6 must call out EVERY tile, not just one. - expect(wf).toMatch(/EVERY tile \(not just one\)/); - }); - - it('warns explicitly against title-recap markdown tiles in the tile type guide', () => { - // The TILE TYPE GUIDE entry for markdown is where a model reading - // top-down first decides whether to add an "About this dashboard" - // tile. The entry has to actively discourage it, not just describe - // the tile type, because every model trained on dashboard examples - // reaches for a title tile by default. - const prompt = buildCreateDashboardPrompt( - 'summary', - 'trace_src', - 'log_src', - ); - const guideIdx = prompt.indexOf('== TILE TYPE GUIDE =='); - const guideEnd = prompt.indexOf('\n== ', guideIdx + 1); - const guideBody = prompt.slice(guideIdx, guideEnd); - // The markdown line must say "skip" / "sparingly" / "do not" / - // "no headings", not just "use for notes". Look for the most - // load-bearing phrase. - expect(guideBody).toMatch(/Skip markdown tiles for starter dashboards/); - }); - - it('explains the 15-minute default time window UX trap', () => { - // Empty tiles on a fresh dashboard are usually a time-window problem, - // not a query problem. The prompt has to surface this so Claude - // tells the user (rather than silently padding individual tile time - // ranges to make a 24-hour view fit a 15-minute dashboard frame). - const prompt = buildCreateDashboardPrompt( - 'summary', - 'trace_src', - 'log_src', - ); - expect(prompt).toMatch(/DEFAULT TIME WINDOW/); - expect(prompt).toMatch(/15-minute default window/); - }); - - it('contains no em-dashes or en-dashes used as em-dashes', () => { - const prompt = buildCreateDashboardPrompt('sources summary', '', ''); - expect(prompt).not.toMatch(/—/); // [prose-lint: allow] - // En-dash flanked by spaces reads as an em-dash substitute and is - // disallowed by the voice rules. Numeric ranges (e.g. "1-20") are - // not in the checklist anyway, but guard against them just in case. - expect(prompt).not.toMatch(/ – /); // [prose-lint: allow] - }); - - it("mentions row-click linking in the table tile's description", () => { - // The create prompt is the LLM's primary entry point for new - // dashboards. Surface onClick on the TILE TYPE GUIDE so the - // model considers wiring it up on overview tables without the - // user having to ask. - const prompt = buildCreateDashboardPrompt( - 'summary', - 'trace_src', - 'log_src', - ); - const guideIdx = prompt.indexOf('== TILE TYPE GUIDE =='); - expect(guideIdx).toBeGreaterThan(-1); - const guideEnd = prompt.indexOf('\n== ', guideIdx + 1); - const guideBody = prompt.slice(guideIdx, guideEnd); - // The table line must hint at onClick and point at the - // consolidated TABLE TILE LINKING section in the query guide. - expect(guideBody).toContain('onClick'); - expect(guideBody).toContain('TABLE TILE LINKING'); - }); - }); - - describe('buildDashboardExamplesPrompt', () => { - it('exposes exactly the four verified trace/log examples plus infrastructure_sql', () => { - const all = buildDashboardExamplesPrompt( - '000000000000000000000001', - '000000000000000000000002', - '000000000000000000000003', - ); - // Order matters here only to keep the LLM\'s navigation stable - // across releases. If the order changes intentionally, update the - // snapshot. - const patternLineMatch = all.match(/Available patterns: ([^\n]+)/); - expect(patternLineMatch).toBeTruthy(); - const patternList = patternLineMatch?.[1].split(', '); - expect(patternList).toEqual([ - 'service_inventory', - 'service_detail', - 'log_analytics', - 'backend_dependencies', - 'drilldown_links', - 'infrastructure_sql', - ]); - }); - - it('renders each example with a leading "When to use" header', () => { - const all = buildDashboardExamplesPrompt( - '000000000000000000000001', - '000000000000000000000002', - '000000000000000000000003', - ); - for (const heading of [ - '== SERVICE INVENTORY ==', - '== SERVICE DETAIL ==', - '== LOG ANALYTICS ==', - '== BACKEND DEPENDENCIES ==', - '== INFRASTRUCTURE (Raw SQL) ==', - ]) { - expect(all).toContain(heading); - } - // Each non-SQL example should explain WHEN to reach for it. The - // SQL example does too, but the heading differs slightly. - const nonSqlSections = all - .split('== ') - .filter(s => - [ - 'SERVICE INVENTORY', - 'SERVICE DETAIL', - 'LOG ANALYTICS', - 'BACKEND DEPENDENCIES', - ].some(h => s.startsWith(h)), - ); - for (const section of nonSqlSections) { - expect(section.toLowerCase()).toContain('when to use'); - } - }); - - it('returns a single example when filtered by pattern', () => { - const single = buildDashboardExamplesPrompt( - '000000000000000000000001', - '000000000000000000000002', - '000000000000000000000003', - 'service_inventory', - ); - expect(single).toContain('== SERVICE INVENTORY =='); - expect(single).not.toContain('== SERVICE DETAIL =='); - expect(single).not.toContain('== LOG ANALYTICS =='); - }); - - it('wires service_inventory row-click into the service_detail dashboard', () => { - // The service_inventory pattern's main RED table should carry an - // onClick that drills into the partner "Service Detail" dashboard - // by name template, with a ServiceName filter rendered from the - // clicked row. Without this wiring, an agent following the canonical - // workflow would have to invent the drill-down each time and - // sometimes get the field shape wrong. - const single = buildDashboardExamplesPrompt( - '000000000000000000000001', - '000000000000000000000002', - '000000000000000000000003', - 'service_inventory', - ); - // The onClick target points at the partner pattern by name. - expect(single).toContain( - 'target: { mode: "template", template: "Service Detail" }', - ); - // The onClick filter expression matches service_detail's filter expression, - // so the destination dashboard's "Service" dropdown auto-populates. - expect(single).toContain('expression: "ServiceName"'); - // The template references the row's groupBy column (ServiceName). - expect(single).toContain('template: "{{ServiceName}}"'); - }); - - it('exposes a drilldown_links example pattern with onClick wiring', () => { - // Drew's drilldown_links example carries a self-contained - // onClick walkthrough (search drill-down + dashboard drill-down). - // Keep it discoverable so an agent asked to wire up drill-downs - // can opt into this example by pattern name. - const all = buildDashboardExamplesPrompt( - 'trace_src', - 'log_src', - 'conn_id', - ); - expect(all).toContain('drilldown_links'); - - const filtered = buildDashboardExamplesPrompt( - 'trace_src', - 'log_src', - 'conn_id', - 'drilldown_links', - ); - expect(filtered).toContain('ROW-CLICK DRILL-DOWN LINKS'); - expect(filtered).toContain('onClick'); - expect(filtered).toContain('type: "search"'); - expect(filtered).toContain('type: "dashboard"'); - expect(filtered).toContain('expressionTemplate'); - }); - - it('falls back to showing all examples when pattern is unknown', () => { - const fallback = buildDashboardExamplesPrompt( - '000000000000000000000001', - '000000000000000000000002', - '000000000000000000000003', - 'made_up_pattern', - ); - expect(fallback).toContain('No example found for pattern'); - // Listing every example name in the fallback gives the LLM a way - // to recover by re-requesting one of the known patterns. - expect(fallback).toContain('service_inventory'); - expect(fallback).toContain('service_detail'); - }); - - it('contains no em-dashes in any example body', () => { - const all = buildDashboardExamplesPrompt( - '000000000000000000000001', - '000000000000000000000002', - '000000000000000000000003', - ); - expect(all).not.toMatch(/—/); // [prose-lint: allow] - }); - }); -}); diff --git a/packages/api/src/mcp/__tests__/dashboards/deleteDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/deleteDashboard.test.ts new file mode 100644 index 0000000000..055887db15 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/deleteDashboard.test.ts @@ -0,0 +1,38 @@ +import Dashboard from '@/models/dashboard'; + +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_delete_dashboard', () => { + const ctx = setupDashboardTests(); + + it('should delete an existing dashboard', async () => { + const dashboard = await new Dashboard({ + name: 'To Delete', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_delete_dashboard', { + id: dashboard._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.deleted).toBe(true); + expect(output.id).toBe(dashboard._id.toString()); + + // Verify deleted from database + const found = await Dashboard.findById(dashboard._id); + expect(found).toBeNull(); + }); + + it('should return error for non-existent dashboard', async () => { + const result = await callTool(ctx.client!, 'clickstack_delete_dashboard', { + id: '000000000000000000000000', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/getDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/getDashboard.test.ts new file mode 100644 index 0000000000..64b9f9cd04 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/getDashboard.test.ts @@ -0,0 +1,65 @@ +import Dashboard from '@/models/dashboard'; + +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_get_dashboard', () => { + const ctx = setupDashboardTests(); + + // Inside each test, use ctx.team, ctx.traceSource, ctx.connection, ctx.client + // Replace bare `team` with `ctx.team`, `client` with `ctx.client!`, etc. + + it('should list all dashboards when no id provided', async () => { + await new Dashboard({ + name: 'Dashboard 1', + tiles: [], + team: ctx.team._id, + tags: ['tag1'], + }).save(); + await new Dashboard({ + name: 'Dashboard 2', + tiles: [], + team: ctx.team._id, + tags: ['tag2'], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_get_dashboard', {}); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(2); + expect(output[0]).toHaveProperty('id'); + expect(output[0]).toHaveProperty('name'); + expect(output[0]).toHaveProperty('tags'); + }); + + it('should get dashboard detail when id is provided', async () => { + const dashboard = await new Dashboard({ + name: 'My Dashboard', + tiles: [], + team: ctx.team._id, + tags: ['test'], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_get_dashboard', { + id: dashboard._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.id).toBe(dashboard._id.toString()); + expect(output.name).toBe('My Dashboard'); + expect(output.tags).toEqual(['test']); + expect(output.tiles).toEqual([]); + }); + + it('should return error for non-existent dashboard id', async () => { + const fakeId = '000000000000000000000000'; + const result = await callTool(ctx.client!, 'clickstack_get_dashboard', { + id: fakeId, + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts b/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts new file mode 100644 index 0000000000..8cd5a655ec --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts @@ -0,0 +1,103 @@ +import Dashboard from '@/models/dashboard'; + +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_get_dashboard_tile', () => { + const ctx = setupDashboardTests(); + + it('should return a single tile by tileId', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Tile Test Dashboard', + tiles: [ + { + name: 'Line Chart', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Number Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const dashboardId = created.id; + const tileId = created.tiles[0].id; + + const result = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { + dashboardId, + tileId, + }, + ); + + expect(result.isError).toBeFalsy(); + const tile = JSON.parse(getFirstText(result)); + expect(tile.id).toBe(tileId); + expect(tile.name).toBe('Line Chart'); + expect(tile.config.displayType).toBe('line'); + }); + + it('should return error for non-existent tileId', async () => { + const dashboard = await new Dashboard({ + name: 'Empty Dashboard', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { + dashboardId: dashboard._id.toString(), + tileId: 'nonexistent-tile', + }, + ); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Tile not found'); + }); + + it('should return error for non-existent dashboard', async () => { + const result = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { + dashboardId: '000000000000000000000000', + tileId: 'some-tile', + }, + ); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should return error for invalid dashboard ID', async () => { + const result = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { + dashboardId: 'not-valid', + tileId: 'some-tile', + }, + ); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Invalid dashboard ID'); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts new file mode 100644 index 0000000000..666f82edaf --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts @@ -0,0 +1,552 @@ +import Dashboard from '@/models/dashboard'; +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { + const ctx = setupDashboardTests(); + + it('should patch a tile config without affecting other tiles', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Patch Test Dashboard', + tiles: [ + { + name: 'Tile A', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Tile B', + x: 0, + y: 4, + w: 12, + h: 4, + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const dashboardId = created.id; + const tileAId = created.tiles[0].id; + + // Patch only tile A: change its name and displayType + const patchResult = await callTool( + ctx.client!, + 'clickstack_patch_dashboard', + { + dashboardId, + tileId: tileAId, + tile: { + name: 'Patched Tile A', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + }, + ); + + expect(patchResult.isError).toBeFalsy(); + const patchOutput = JSON.parse(getFirstText(patchResult)); + expect(patchOutput.patchedTile.name).toBe('Patched Tile A'); + expect(patchOutput.patchedTile.config.displayType).toBe('table'); + + // Verify the full dashboard: tile B should be untouched + const getResult = await callTool(ctx.client!, 'clickstack_get_dashboard', { + id: dashboardId, + }); + const dashboard = JSON.parse(getFirstText(getResult)); + expect(dashboard.tiles).toHaveLength(2); + + const tileA = dashboard.tiles.find( + (t: ExternalDashboardTileWithId) => t.id === tileAId, + ); + expect(tileA.name).toBe('Patched Tile A'); + expect(tileA.config.displayType).toBe('table'); + + const tileB = dashboard.tiles.find( + (t: ExternalDashboardTileWithId) => t.id !== tileAId, + ); + expect(tileB.name).toBe('Tile B'); + expect(tileB.config.displayType).toBe('number'); + }); + + it('should preserve existing layout when not specified in patch', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Layout Test', + tiles: [ + { + name: 'Positioned Tile', + x: 6, + y: 10, + w: 8, + h: 5, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const tileId = created.tiles[0].id; + + // Patch without specifying layout — layout should be preserved + const patchResult = await callTool( + ctx.client!, + 'clickstack_patch_dashboard', + { + dashboardId: created.id, + tileId, + tile: { + name: 'Renamed Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'avg', valueExpression: 'Duration' }], + }, + }, + }, + ); + + expect(patchResult.isError).toBeFalsy(); + const patched = JSON.parse(getFirstText(patchResult)); + expect(patched.patchedTile.x).toBe(6); + expect(patched.patchedTile.y).toBe(10); + expect(patched.patchedTile.w).toBe(8); + expect(patched.patchedTile.h).toBe(5); + expect(patched.patchedTile.name).toBe('Renamed Tile'); + }); + + it('should update dashboard name only (no tile patch)', async () => { + const dashboard = await new Dashboard({ + name: 'Original Name', + tiles: [], + team: ctx.team._id, + tags: ['tag1'], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: dashboard._id.toString(), + name: 'New Name', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.name).toBe('New Name'); + expect(output.tags).toEqual(['tag1']); + expect(output.patchedTile).toBeUndefined(); + + // Verify in DB + const updated = await Dashboard.findById(dashboard._id); + expect(updated?.name).toBe('New Name'); + }); + + it('should update tags only', async () => { + const dashboard = await new Dashboard({ + name: 'Tag Test', + tiles: [], + team: ctx.team._id, + tags: ['old'], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: dashboard._id.toString(), + tags: ['new1', 'new2'], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tags).toEqual(['new1', 'new2']); + expect(output.name).toBe('Tag Test'); + }); + + it('should update name and patch tile in one call', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Combo Test', + tiles: [ + { + name: 'A Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: created.id, + name: 'Combo Updated', + tileId: created.tiles[0].id, + tile: { + name: 'Updated Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'avg', valueExpression: 'Duration' }], + }, + }, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.name).toBe('Combo Updated'); + expect(output.patchedTile.name).toBe('Updated Tile'); + }); + + it('should return error for non-existent tileId', async () => { + const dashboard = await new Dashboard({ + name: 'Tile Not Found', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: dashboard._id.toString(), + tileId: 'nonexistent', + tile: { + name: 'Ghost', + config: { displayType: 'markdown', markdown: 'hello' }, + }, + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Tile not found'); + }); + + it('should return error for non-existent dashboard', async () => { + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: '000000000000000000000000', + name: 'Ghost', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should return error for missing source ID on patched tile', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Source Validation Test', + tiles: [ + { + name: 'Valid Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: created.id, + tileId: created.tiles[0].id, + tile: { + name: 'Bad Source', + config: { + displayType: 'line', + sourceId: '000000000000000000000000', + select: [{ aggFn: 'count' }], + }, + }, + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('source'); + }); + + it('should not modify other tiles in the database (positional update)', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Positional Update Test', + tiles: [ + { + name: 'Tile A', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Tile B', + x: 6, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Tile C', + x: 0, + y: 3, + w: 12, + h: 4, + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const dashboardId = created.id; + + // Snapshot the raw tiles from the DB before patching + const beforePatch = await Dashboard.findById(dashboardId).lean(); + const tilesBeforePatch = (beforePatch as any).tiles; + const tileBBefore = tilesBeforePatch[1]; + const tileCBefore = tilesBeforePatch[2]; + + // Patch only tile A (index 0) + const patchResult = await callTool( + ctx.client!, + 'clickstack_patch_dashboard', + { + dashboardId, + tileId: created.tiles[0].id, + tile: { + name: 'Tile A Patched', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'avg', valueExpression: 'Duration' }], + }, + }, + }, + ); + expect(patchResult.isError).toBeFalsy(); + + // Read raw DB tiles again — tiles B and C should be byte-identical + const afterPatch = await Dashboard.findById(dashboardId).lean(); + const tilesAfterPatch = (afterPatch as any).tiles; + + expect(tilesAfterPatch).toHaveLength(3); + // Tile B (index 1) and Tile C (index 2) should be untouched + expect(tilesAfterPatch[1]).toEqual(tileBBefore); + expect(tilesAfterPatch[2]).toEqual(tileCBefore); + // Tile A (index 0) should have changed + expect(tilesAfterPatch[0].config.name).toBe('Tile A Patched'); + }); + + it('should preserve tile name when omitted from patch (config-only patch)', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Name Preservation Test', + tiles: [ + { + name: 'Original Title', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const tileId = created.tiles[0].id; + + // Patch config only — no name field at all + const patchResult = await callTool( + ctx.client!, + 'clickstack_patch_dashboard', + { + dashboardId: created.id, + tileId, + tile: { + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'avg', valueExpression: 'Duration' }], + }, + }, + }, + ); + + expect(patchResult.isError).toBeFalsy(); + const patched = JSON.parse(getFirstText(patchResult)); + expect(patched.patchedTile.name).toBe('Original Title'); + expect(patched.patchedTile.config.displayType).toBe('table'); + + // Verify via get + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { dashboardId: created.id, tileId }, + ); + const tile = JSON.parse(getFirstText(getResult)); + expect(tile.name).toBe('Original Title'); + }); + + it('should return error when tile is removed between read and write', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Concurrent Delete Test', + tiles: [ + { + name: 'Tile A', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Tile B', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const tileAId = created.tiles[0].id; + + // Simulate a concurrent save_dashboard that removes tile A by + // directly updating the DB to drop it from the array. + await Dashboard.findByIdAndUpdate(created.id, { + $pull: { tiles: { id: tileAId } }, + }); + + // Now try to patch tile A — it no longer exists in the array. + const patchResult = await callTool( + ctx.client!, + 'clickstack_patch_dashboard', + { + dashboardId: created.id, + tileId: tileAId, + tile: { + name: 'Patched A', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + }, + ); + + expect(patchResult.isError).toBe(true); + // Could be "Tile not found" (caught at read) or "was not found at + // write time" (caught by positional $ miss). Either is acceptable. + const text = getFirstText(patchResult); + expect(text).toMatch(/not found|was not found/i); + }); + + it('should round-trip: patch then get_dashboard_tile', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Round Trip Test', + tiles: [ + { + name: 'Original', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const tileId = created.tiles[0].id; + + // Patch + await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: created.id, + tileId, + tile: { + name: 'Patched', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'avg', valueExpression: 'Duration' }], + }, + }, + }); + + // Get the tile back + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { + dashboardId: created.id, + tileId, + }, + ); + + expect(getResult.isError).toBeFalsy(); + const tile = JSON.parse(getFirstText(getResult)); + expect(tile.id).toBe(tileId); + expect(tile.name).toBe('Patched'); + expect(tile.config.displayType).toBe('number'); + expect(tile.config.select[0].aggFn).toBe('avg'); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts new file mode 100644 index 0000000000..5bb16e6113 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts @@ -0,0 +1,576 @@ +import { + buildCreateDashboardPrompt, + buildDashboardExamplesPrompt, + buildQueryGuidePrompt, +} from '../../prompts/dashboards/content'; +import { buildSourceSummary } from '../../prompts/dashboards/helpers'; + +describe('MCP Dashboard Prompts', () => { + describe('buildQueryGuidePrompt', () => { + it('documents heatmap in tile-type, constraints, mistakes, and aggFn sections', () => { + const prompt = buildQueryGuidePrompt(); + // Ensure each section the bot called out has a heatmap entry so + // the LLM cannot skip the constraints when producing a heatmap + // tile through clickstack_save_dashboard. + const sections = [ + '== AGGREGATION FUNCTIONS (aggFn) ==', + '== PER-TILE TYPE CONSTRAINTS ==', + '== COMMON MISTAKES ==', + ]; + for (const heading of sections) { + const idx = prompt.indexOf(heading); + expect(idx).toBeGreaterThan(-1); + // Each section's body extends until the next == heading or the + // end of the string. Checking for the substring "heatmap" in + // that slice is enough to assert the heatmap entry exists. + const next = prompt.indexOf('\n== ', idx + heading.length); + const body = prompt.slice(idx, next === -1 ? prompt.length : next); + expect(body.toLowerCase()).toContain('heatmap'); + } + }); + + it('documents the dashboard filter and per-series numberFormat sections', () => { + const prompt = buildQueryGuidePrompt(); + // Sections must exist AND carry the substantive content. Heading- + // only assertions let a future contributor empty out a section + // and silently pass review. + const sliceSection = (heading: string) => { + const idx = prompt.indexOf(heading); + if (idx === -1) return ''; + const next = prompt.indexOf('\n== ', idx + heading.length); + return prompt.slice(idx, next === -1 ? prompt.length : next); + }; + const filters = sliceSection('== DASHBOARD FILTERS =='); + expect(filters).toContain('QUERY_EXPRESSION'); + expect(filters).toContain('expression'); + expect(filters).toContain('sourceId'); + const numberFormat = sliceSection('== NUMBER FORMAT =='); + expect(numberFormat).toContain('factor: 0.000000001'); + expect(numberFormat).toContain('duration'); + expect(numberFormat.toLowerCase()).toContain('per-series'); + }); + + it('declares the MCP metric-authoring gap rather than referencing fields that do not exist', () => { + // The MCP select-item schema does not carry metricName / metricType, + // so any prompt that tells the model to author a metric tile via + // the builder path is teaching it to ship JSON that gets silently + // stripped. Guard with an explicit assertion so a future diff that + // restores metric-tile guidance fails this test loudly. + const prompt = buildQueryGuidePrompt(); + expect(prompt).not.toMatch(/metricName \+ metricType/); + expect(prompt).not.toMatch(/exactly 1 select item with metricName/); + const constraintsIdx = prompt.indexOf('== PER-TILE TYPE CONSTRAINTS =='); + const constraintsBody = prompt.slice(constraintsIdx); + // The constraints section closes with an explicit note that builder + // tiles on a metric source are not reliable today, with a fallback + // recipe to raw SQL. Anchor on the phrase so a future diff that + // drops the gap-acknowledgement fails loudly. + expect(constraintsBody).toMatch( + /Authoring builder tiles on a metric source is not reliable/, + ); + expect(constraintsBody).toMatch(/MCP select-item shape does not carry/); + }); + + it('documents table-tile onClick linking features', () => { + // Lock down the documentation for row-click drill-downs so a + // future refactor can't quietly drop the section the LLM relies + // on to wire up onClick correctly. + const prompt = buildQueryGuidePrompt(); + const idx = prompt.indexOf('== TABLE TILE LINKING (config.onClick) =='); + expect(idx).toBeGreaterThan(-1); + const next = prompt.indexOf('\n== ', idx + 1); + const section = prompt.slice(idx, next === -1 ? prompt.length : next); + + // Both destination types are mentioned. + expect(section).toContain('type: "search"'); + expect(section).toContain('type: "dashboard"'); + // Both target modes are mentioned. + expect(section).toContain('mode: "id"'); + expect(section).toContain('mode: "template"'); + // Templating fields are mentioned. + expect(section).toContain('whereTemplate'); + expect(section).toContain('filters'); + expect(section).toContain('expressionTemplate'); + // The two server-side validation error messages are quoted so + // the LLM can recognize and recover from them. + expect(section).toContain('onClick search source IDs'); + expect(section).toContain('onClick dashboard IDs'); + }); + + it('documents onClick pitfalls under common mistakes', () => { + const prompt = buildQueryGuidePrompt(); + const idx = prompt.indexOf('== COMMON MISTAKES =='); + expect(idx).toBeGreaterThan(-1); + const section = prompt.slice(idx); + // Mention the four failure modes we hit during validation. + expect(section.toLowerCase()).toContain('onclick on a non-table tile'); + expect(section.toLowerCase()).toContain('non-log/trace source'); + expect(section.toLowerCase()).toContain('missing wherelanguage'); + expect(section.toLowerCase()).toContain("isn't in the table"); + }); + + it('documents the missing-alias mistake explicitly', () => { + // Claude built three number tiles without alias on the quantile() + // select item, even though rule 2 of the design checklist says + // ALIAS EVERY SELECT ITEM. Spelling out the mistake under COMMON + // MISTAKES gives the model a second touchpoint to catch it. + const prompt = buildQueryGuidePrompt(); + const idx = prompt.indexOf('== COMMON MISTAKES =='); + const section = prompt.slice(idx); + expect(section).toMatch(/Missing alias on a select item/); + // Must call out that number tiles need the alias too; this is the + // exact case Claude got wrong. + expect(section).toMatch(/number tiles too/); + }); + + it('documents the Lucene map-attribute gotcha across all operators including simple equality', () => { + // The original rule covered only comparison/wildcard. Claude's + // second pass showed simple equality (db.system:mongodb) also + // fails to translate to SpanAttributes['db.system']. Broaden the + // gotcha to "ALL Lucene operations on map-attribute paths". + const prompt = buildQueryGuidePrompt(); + const luceneIdx = prompt.indexOf('== LUCENE FILTER SYNTAX =='); + const luceneBody = prompt.slice(luceneIdx); + expect(luceneBody).toMatch(/NOT reliably translated to bracket access/); + expect(luceneBody).toMatch(/http\.status_code:>=500/); // operator + expect(luceneBody).toMatch(/http\.route:\*/); // wildcard + expect(luceneBody).toMatch(/db\.system:mongodb/); // simple equality + expect(luceneBody).toMatch(/use SQL with bracket access/); + + // Also a top-level entry in COMMON MISTAKES for discoverability. + const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); + const mistakesBody = prompt.slice(mistakesIdx); + expect(mistakesBody).toMatch( + /Lucene on a map-attribute path \(any operation\)/, + ); + }); + + it('documents the Lucene fuzzy-substring behavior on top-level columns', () => { + // Lucene field:value translates to ilike(field, '%value%'), not + // equality. Claude's second pass hit this on SpanKind:Server, + // which matched broader strings than intended. The note has to + // be in the Lucene section so a model reading top-down sees it + // before writing a SpanKind filter. + const prompt = buildQueryGuidePrompt(); + const luceneIdx = prompt.indexOf('== LUCENE FILTER SYNTAX =='); + const luceneBody = prompt.slice(luceneIdx); + expect(luceneBody).toMatch(/SUBSTRING MATCHING/); + expect(luceneBody).toMatch(/ilike\(field, '%value%'\)/); + // The canonical fix is SQL with =. + expect(luceneBody).toMatch(/SpanKind = 'Server'/); + // Wildcard limitation is documented alongside substring matching. + expect(luceneBody).toMatch(/prefix-within-substring/); + expect(luceneBody).toMatch(/LIKE 'api%'/); + + // And a COMMON MISTAKES entry covering enum-like columns and wildcards. + const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); + const mistakesBody = prompt.slice(mistakesIdx); + expect(mistakesBody).toMatch(/Lucene field:value and field:value\*/); + }); + + it('documents the empty-string trap for map-attribute groupBy', () => { + // Map columns return '' (not NULL) when a key is unset on a row. + // A groupBy on such a key produces an empty-string bucket alongside + // the real values, which is visual noise on tables and pies. The + // standard fix is where: " != ''" alongside the groupBy. + const prompt = buildQueryGuidePrompt(); + const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); + const mistakesBody = prompt.slice(mistakesIdx); + expect(mistakesBody).toMatch( + /Forgetting that map-attribute values are often empty strings/, + ); + expect(mistakesBody).toMatch(/db\.collection\.name'\] != ''/); + }); + + it('documents the groupBy alias workaround for map-attribute drill-downs', () => { + // Builder tiles don't expose a groupBy alias today; map-attribute + // groupBys produce result columns named like the raw expression, + // which onClick template lookups can't reference. The workaround + // (author the tile as raw SQL with AS) needs to be findable. + const prompt = buildQueryGuidePrompt(); + const idx = prompt.indexOf( + '== GROUPBY ALIASES AND ROW-CLICK TEMPLATES ==', + ); + expect(idx).toBeGreaterThan(-1); + const section = prompt.slice(idx); + expect(section).toMatch(/do NOT currently expose a groupBy alias/); + expect(section).toMatch(/configType: "sql", displayType: "table"/); + expect(section).toMatch(/AS Route/); + }); + + it('points at ClickStack and ClickHouse docs in REFERENCES', () => { + // Without external doc links, the LLM has to guess at Lucene + // operators and ClickHouse function names. Anchor the canonical + // URLs in the prompt so it can cite them and so a future doc + // restructure can be caught here. + const prompt = buildQueryGuidePrompt(); + const idx = prompt.indexOf('== REFERENCES =='); + expect(idx).toBeGreaterThan(-1); + const section = prompt.slice(idx); + expect(section).toContain( + 'https://clickhouse.com/docs/use-cases/observability/clickstack/search', + ); + expect(section).toContain('https://clickhouse.com/docs/sql-reference'); + expect(section).toContain( + 'https://clickhouse.com/docs/use-cases/observability/clickstack/dashboards/sql-visualizations', + ); + }); + + it('strengthens the validate-every-tile rule under common mistakes', () => { + // Saving validates input shape, not query semantics. The prompt + // has to push for query_tile on every tile, not "at least one". + const prompt = buildQueryGuidePrompt(); + const mistakesIdx = prompt.indexOf('== COMMON MISTAKES =='); + const section = prompt.slice(mistakesIdx); + expect(section).toMatch( + /on EVERY tile after clickstack_save_dashboard, not just one/, + ); + }); + + it('flags the metric source builder gap', () => { + // Builder tiles on metric sources currently save but render with + // "Both table name and UUID are empty". Claude went 100% raw SQL + // across 21 metric tiles for this reason; the prompt has to make + // the workaround obvious so the model doesn't try the builder + // first and fail silently. + const prompt = buildQueryGuidePrompt(); + expect(prompt).toMatch(/Both table name and UUID are empty/); + expect(prompt).toMatch(/use a raw SQL tile/); + expect(prompt).toMatch(/otel_metrics_gauge/); + }); + + it('contains no em-dashes or en-dashes used as em-dashes', () => { + const prompt = buildQueryGuidePrompt(); + expect(prompt).not.toMatch(/\u2014/); // [prose-lint: allow] + expect(prompt).not.toMatch(/ \u2013 /); // [prose-lint: allow] + }); + }); + + describe('buildSourceSummary', () => { + it('contains no em-dashes or en-dashes used as em-dashes', () => { + // buildSourceSummary is the source-list block that gets prepended + // to create_dashboard. The first iteration of this PR shipped with + // em-dashes here that the content.ts snapshot test missed because + // it calls the builders directly. Guard the helper output too. + const summary = buildSourceSummary( + [ + { + _id: '000000000000000000000001', + name: 'Traces', + kind: 'trace', + connection: '000000000000000000000002', + }, + { + _id: '000000000000000000000003', + name: 'Logs', + kind: 'log', + connection: '000000000000000000000002', + }, + ], + [{ _id: '000000000000000000000002', name: 'Default' }], + ); + expect(summary).not.toMatch(/\u2014/); // [prose-lint: allow] + expect(summary).not.toMatch(/ \u2013 /); // [prose-lint: allow] + // Sanity: the helper still emits the source list, so the assertion + // above isn't trivially passing on an empty string. + expect(summary).toContain('Traces'); + expect(summary).toContain('AVAILABLE SOURCES'); + }); + }); + + describe('buildCreateDashboardPrompt', () => { + it('includes the design checklist and adapt-do-not-copy note', () => { + const prompt = buildCreateDashboardPrompt( + 'sources summary', + '000000000000000000000001', + '000000000000000000000002', + ); + expect(prompt).toContain('== DESIGN CHECKLIST =='); + // Each rule on the checklist exists at the same numbered position + // so a future contributor cannot silently drop one. Anchor on + // line start (with a trailing space) so digit-and-dot substrings + // like "0.000000001" elsewhere in the prompt cannot satisfy the + // assertion accidentally. + const checklistIdx = prompt.indexOf('== DESIGN CHECKLIST =='); + const adaptIdx = prompt.indexOf('== ADAPT, DO NOT COPY =='); + const checklistBody = prompt.slice(checklistIdx, adaptIdx); + // Thirteen rules: the original ten plus GROUP BY HAS NO ALIAS HOOK + // (rule 3), VALIDATE EVERY TILE AFTER SAVE (rule 12), and NO + // TITLE-RECAP MARKDOWN TILE (rule 13). Each came out of a live + // verification pass after watching Claude reliably ignore the soft + // "should" formulations or hit a schema gap the earlier checklist + // did not call out. + for (let i = 1; i <= 13; i++) { + expect(checklistBody).toMatch(new RegExp(`^${i}\\. `, 'm')); + } + expect(prompt).toContain('ADAPT, DO NOT COPY'); + // Rule 9 documents the replace-not-merge semantic on update; an + // agent that ignores this can silently drop tiles / filters / containers + // on a subsequent save call. + expect(checklistBody).toMatch(/UPDATE IS REPLACE, NOT MERGE/); + // Rule 2 must cover EVERY select item (including number tiles), not + // just aggregations on tables. The phrasing matters: Claude's first + // pass at this dropped aliases on number tiles because it read rule 2 + // as table-specific. The stronger phrasing pulls them in. + expect(checklistBody).toMatch(/ALIAS EVERY SELECT ITEM/); + // Rule 3 calls out the schema gap on groupBy: the chart config takes + // a single expression string with no alias field, so a table grouped + // by SpanAttributes['http.route'] renders arrayElement(...) as the + // column header. Naming the limit explicitly lets Claude pick a + // top-level column (SpanName, ServiceName) instead of grouping on + // the raw Map expression. + expect(checklistBody).toMatch(/GROUP BY HAS NO ALIAS HOOK/); + // Rule 10 must be a hard requirement at 5+ tiles, not a soft hint. + // Without the imperative, Claude built five dashboards averaging ten + // tiles each with zero containers. + expect(checklistBody).toMatch(/REQUIRED at five or more tiles/); + // Rule 11 + 12 close the post-save loop and the title-recap markdown + // habit Claude landed on every starter dashboard. + expect(checklistBody).toMatch(/VALIDATE EVERY TILE AFTER SAVE/); + expect(checklistBody).toMatch(/NO TITLE-RECAP MARKDOWN TILE/); + // Rule 2 now carries a concrete number-tile example. Claude's + // second pass still dropped aliases on every builder number tile, + // because the text rule was abstract; a copy-pasteable shape next + // to the rule is what makes it stick. + expect(checklistBody).toMatch( + /Number tile, correct:.*alias: "Server Requests"/, + ); + // Rule 10 now carries a concrete containers shape. Claude's + // second pass built five dashboards with 9-10 tiles each and zero + // containers despite the REQUIRED phrasing; a copy-pasteable shape + // inside the rule itself is the next-level push. + expect(checklistBody).toMatch(/Concrete shape \(copy this directly/); + expect(checklistBody).toMatch(/id: "kpis"/); + expect(checklistBody).toMatch(/id: "trends"/); + expect(checklistBody).toMatch(/id: "errors"/); + }); + + it('walks the workflow through six steps including read-existing and group-into-containers', () => { + // The workflow is the first thing a model anchors on. Claude + // skipped both "read existing dashboards before designing" and + // "group tiles into containers before saving" because neither was + // a numbered workflow step; both lived only in the checklist. The + // workflow now lists six steps with those two explicitly. + const prompt = buildCreateDashboardPrompt( + 'summary', + 'trace_src', + 'log_src', + ); + const wfIdx = prompt.indexOf('== WORKFLOW =='); + const wfEnd = prompt.indexOf('\n== ', wfIdx + 1); + const wf = prompt.slice(wfIdx, wfEnd); + for (let i = 1; i <= 6; i++) { + expect(wf).toMatch(new RegExp(`^${i}\\. `, 'm')); + } + // Step 2 must point at clickstack_get_dashboard with no id. + expect(wf).toMatch(/clickstack_get_dashboard \(no id\)/); + // Step 4 must call out grouping tiles into containers before save. + expect(wf).toMatch(/group them into 2-4 containers/); + // Step 6 must call out EVERY tile, not just one. + expect(wf).toMatch(/EVERY tile \(not just one\)/); + }); + + it('warns explicitly against title-recap markdown tiles in the tile type guide', () => { + // The TILE TYPE GUIDE entry for markdown is where a model reading + // top-down first decides whether to add an "About this dashboard" + // tile. The entry has to actively discourage it, not just describe + // the tile type, because every model trained on dashboard examples + // reaches for a title tile by default. + const prompt = buildCreateDashboardPrompt( + 'summary', + 'trace_src', + 'log_src', + ); + const guideIdx = prompt.indexOf('== TILE TYPE GUIDE =='); + const guideEnd = prompt.indexOf('\n== ', guideIdx + 1); + const guideBody = prompt.slice(guideIdx, guideEnd); + // The markdown line must say "skip" / "sparingly" / "do not" / + // "no headings", not just "use for notes". Look for the most + // load-bearing phrase. + expect(guideBody).toMatch(/Skip markdown tiles for starter dashboards/); + }); + + it('explains the 15-minute default time window UX trap', () => { + // Empty tiles on a fresh dashboard are usually a time-window problem, + // not a query problem. The prompt has to surface this so Claude + // tells the user (rather than silently padding individual tile time + // ranges to make a 24-hour view fit a 15-minute dashboard frame). + const prompt = buildCreateDashboardPrompt( + 'summary', + 'trace_src', + 'log_src', + ); + expect(prompt).toMatch(/DEFAULT TIME WINDOW/); + expect(prompt).toMatch(/15-minute default window/); + }); + + it('contains no em-dashes or en-dashes used as em-dashes', () => { + const prompt = buildCreateDashboardPrompt('sources summary', '', ''); + expect(prompt).not.toMatch(/\u2014/); // [prose-lint: allow] + // En-dash flanked by spaces reads as an em-dash substitute and is + // disallowed by the voice rules. Numeric ranges (e.g. "1-20") are + // not in the checklist anyway, but guard against them just in case. + expect(prompt).not.toMatch(/ \u2013 /); // [prose-lint: allow] + }); + + it("mentions row-click linking in the table tile's description", () => { + // The create prompt is the LLM's primary entry point for new + // dashboards. Surface onClick on the TILE TYPE GUIDE so the + // model considers wiring it up on overview tables without the + // user having to ask. + const prompt = buildCreateDashboardPrompt( + 'summary', + 'trace_src', + 'log_src', + ); + const guideIdx = prompt.indexOf('== TILE TYPE GUIDE =='); + expect(guideIdx).toBeGreaterThan(-1); + const guideEnd = prompt.indexOf('\n== ', guideIdx + 1); + const guideBody = prompt.slice(guideIdx, guideEnd); + // The table line must hint at onClick and point at the + // consolidated TABLE TILE LINKING section in the query guide. + expect(guideBody).toContain('onClick'); + expect(guideBody).toContain('TABLE TILE LINKING'); + }); + }); + + describe('buildDashboardExamplesPrompt', () => { + it('exposes exactly the four verified trace/log examples plus infrastructure_sql', () => { + const all = buildDashboardExamplesPrompt( + '000000000000000000000001', + '000000000000000000000002', + '000000000000000000000003', + ); + // Order matters here only to keep the LLM's navigation stable + // across releases. If the order changes intentionally, update the + // snapshot. + const patternLineMatch = all.match(/Available patterns: ([^\n]+)/); + expect(patternLineMatch).toBeTruthy(); + const patternList = patternLineMatch?.[1].split(', '); + expect(patternList).toEqual([ + 'service_inventory', + 'service_detail', + 'log_analytics', + 'backend_dependencies', + 'drilldown_links', + 'infrastructure_sql', + ]); + }); + + it('renders each example with a leading "When to use" header', () => { + const all = buildDashboardExamplesPrompt( + '000000000000000000000001', + '000000000000000000000002', + '000000000000000000000003', + ); + for (const heading of [ + '== SERVICE INVENTORY ==', + '== SERVICE DETAIL ==', + '== LOG ANALYTICS ==', + '== BACKEND DEPENDENCIES ==', + '== INFRASTRUCTURE (Raw SQL) ==', + ]) { + expect(all).toContain(heading); + } + // Each non-SQL example should explain WHEN to reach for it. The + // SQL example does too, but the heading differs slightly. + const nonSqlSections = all + .split('== ') + .filter(s => + [ + 'SERVICE INVENTORY', + 'SERVICE DETAIL', + 'LOG ANALYTICS', + 'BACKEND DEPENDENCIES', + ].some(h => s.startsWith(h)), + ); + for (const section of nonSqlSections) { + expect(section.toLowerCase()).toContain('when to use'); + } + }); + + it('returns a single example when filtered by pattern', () => { + const single = buildDashboardExamplesPrompt( + '000000000000000000000001', + '000000000000000000000002', + '000000000000000000000003', + 'service_inventory', + ); + expect(single).toContain('== SERVICE INVENTORY =='); + expect(single).not.toContain('== SERVICE DETAIL =='); + expect(single).not.toContain('== LOG ANALYTICS =='); + }); + + it('wires service_inventory row-click into the service_detail dashboard', () => { + // The service_inventory pattern's main RED table should carry an + // onClick that drills into the partner "Service Detail" dashboard + // by name template, with a ServiceName filter rendered from the + // clicked row. Without this wiring, an agent following the canonical + // workflow would have to invent the drill-down each time and + // sometimes get the field shape wrong. + const single = buildDashboardExamplesPrompt( + '000000000000000000000001', + '000000000000000000000002', + '000000000000000000000003', + 'service_inventory', + ); + // The onClick target points at the partner pattern by name. + expect(single).toContain( + 'target: { mode: "template", template: "Service Detail" }', + ); + // The onClick filter expression matches service_detail's filter expression, + // so the destination dashboard's "Service" dropdown auto-populates. + expect(single).toContain('expression: "ServiceName"'); + // The template references the row's groupBy column (ServiceName). + expect(single).toContain('template: "{{ServiceName}}"'); + }); + + it('exposes a drilldown_links example pattern with onClick wiring', () => { + // Drew's drilldown_links example carries a self-contained + // onClick walkthrough (search drill-down + dashboard drill-down). + // Keep it discoverable so an agent asked to wire up drill-downs + // can opt into this example by pattern name. + const all = buildDashboardExamplesPrompt( + 'trace_src', + 'log_src', + 'conn_id', + ); + expect(all).toContain('drilldown_links'); + + const filtered = buildDashboardExamplesPrompt( + 'trace_src', + 'log_src', + 'conn_id', + 'drilldown_links', + ); + expect(filtered).toContain('ROW-CLICK DRILL-DOWN LINKS'); + expect(filtered).toContain('onClick'); + expect(filtered).toContain('type: "search"'); + expect(filtered).toContain('type: "dashboard"'); + expect(filtered).toContain('expressionTemplate'); + }); + + it('falls back to showing all examples when pattern is unknown', () => { + const fallback = buildDashboardExamplesPrompt( + '000000000000000000000001', + '000000000000000000000002', + '000000000000000000000003', + 'made_up_pattern', + ); + expect(fallback).toContain('No example found for pattern'); + // Listing every example name in the fallback gives the LLM a way + // to recover by re-requesting one of the known patterns. + expect(fallback).toContain('service_inventory'); + expect(fallback).toContain('service_detail'); + }); + + it('contains no em-dashes in any example body', () => { + const all = buildDashboardExamplesPrompt( + '000000000000000000000001', + '000000000000000000000002', + '000000000000000000000003', + ); + expect(all).not.toMatch(/\u2014/); // [prose-lint: allow] + }); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTile.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTile.test.ts new file mode 100644 index 0000000000..3eb44a8676 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/queryTile.test.ts @@ -0,0 +1,110 @@ +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_query_tile', () => { + const ctx = setupDashboardTests(); + + it('should return error for non-existent dashboard', async () => { + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: '000000000000000000000000', + tileId: 'some-tile-id', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should return error for non-existent tile', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Tile Query Test', + tiles: [ + { + name: 'My Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: 'non-existent-tile-id', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Tile not found'); + }); + + it('should return error for invalid time range', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Time Range Test', + tiles: [ + { + name: 'Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: 'not-a-date', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Invalid'); + }); + + it('should execute query for a valid tile', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Query Tile Test', + tiles: [ + { + name: 'Count Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + // Should succeed (may have empty results since no data inserted) + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts new file mode 100644 index 0000000000..d0cfe53521 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts @@ -0,0 +1,1806 @@ +import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import mongoose from 'mongoose'; + +import { DEFAULT_DATABASE } from '@/fixtures'; +import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { + const ctx = setupDashboardTests(); + + describe('basic CRUD', () => { + it('should create a new dashboard with tiles', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'New MCP Dashboard', + tiles: [ + { + name: 'Line Chart', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count', where: '' }], + }, + }, + ], + tags: ['mcp-test'], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.id).toBeDefined(); + expect(output.name).toBe('New MCP Dashboard'); + expect(output.tiles).toHaveLength(1); + expect(output.tiles[0].config.displayType).toBe('line'); + expect(output.tags).toEqual(['mcp-test']); + + // Verify in database + const dashboard = await Dashboard.findById(output.id); + expect(dashboard).not.toBeNull(); + expect(dashboard?.name).toBe('New MCP Dashboard'); + }); + + it('should create a dashboard with a markdown tile', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Markdown Dashboard', + tiles: [ + { + name: 'Notes', + config: { + displayType: 'markdown', + markdown: '# Hello World', + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + expect(output.tiles[0].config.displayType).toBe('markdown'); + }); + + it('should update an existing dashboard', async () => { + const sourceId = ctx.traceSource._id.toString(); + + // Create first + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Original Name', + tiles: [ + { + name: 'Tile 1', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + + // Update + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'Updated Name', + tiles: [ + { + name: 'Updated Tile', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + tags: ['updated'], + }, + ); + + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.id).toBe(created.id); + expect(updated.name).toBe('Updated Name'); + expect(updated.tiles).toHaveLength(1); + expect(updated.tiles[0].name).toBe('Updated Tile'); + expect(updated.tiles[0].config.displayType).toBe('table'); + }); + + it('should return error for missing source ID', async () => { + const fakeSourceId = '000000000000000000000000'; + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Bad Dashboard', + tiles: [ + { + name: 'Bad Tile', + config: { + displayType: 'line', + sourceId: fakeSourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('source'); + }); + + it('should return error when updating non-existent dashboard', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + id: '000000000000000000000000', + name: 'Ghost Dashboard', + tiles: [ + { + name: 'Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should create a dashboard with multiple tile types', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Multi-tile Dashboard', + tiles: [ + { + name: 'Line', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Table', + x: 0, + y: 4, + w: 12, + h: 4, + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + groupBy: 'SpanName', + groupByColumnsOnLeft: true, + }, + }, + { + name: 'Number', + x: 0, + y: 8, + w: 6, + h: 3, + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Pie', + x: 6, + y: 8, + w: 6, + h: 3, + config: { + displayType: 'pie', + sourceId, + select: [{ aggFn: 'count' }], + groupBy: 'SpanName', + }, + }, + { + name: 'Notes', + x: 0, + y: 11, + w: 12, + h: 2, + config: { displayType: 'markdown', markdown: '# Dashboard Notes' }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(5); + + const tableTile = output.tiles.find( + (t: { name: string }) => t.name === 'Table', + ); + expect(tableTile).toBeDefined(); + expect(tableTile.config.groupByColumnsOnLeft).toBe(true); + }); + + it('should create a dashboard with a raw SQL tile', async () => { + const connectionId = ctx.connection._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'SQL Dashboard', + tiles: [ + { + name: 'Raw SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: 'SELECT 1 AS value LIMIT 1', + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + }); + + it('should create a dashboard with a heatmap tile on a Trace source', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Heatmap Dashboard', + tiles: [ + { + name: 'Latency Heatmap', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: 'heatmap', + sourceId, + select: [ + { + valueExpression: 'Duration', + }, + ], + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + expect(output.tiles[0].config.displayType).toBe('heatmap'); + expect(output.tiles[0].config.select[0]).toMatchObject({ + valueExpression: 'Duration', + }); + }); + + it('should round-trip every MCP-specific heatmap field through save, get, update, and re-get', async () => { + const sourceId = ctx.traceSource._id.toString(); + + const createConfig = { + displayType: 'heatmap' as const, + sourceId, + select: [ + { + valueExpression: 'Duration', + countExpression: 'count()', + heatmapScaleType: 'log' as const, + }, + ], + where: 'level:error', + whereLanguage: 'lucene' as const, + numberFormat: { output: 'duration' as const, factor: 1e-9 }, + }; + // Mutate select + where on update; numberFormat, whereLanguage, + // and sourceId carry forward via the spread. Re-asserting against + // updatedConfig catches a regression where PUT silently drops any + // carried-forward field. + const updatedConfig = { + ...createConfig, + select: [ + { + valueExpression: "SpanAttributes['http.duration']", + heatmapScaleType: 'linear' as const, + }, + ], + where: 'level:error AND service:checkout', + }; + + const saveResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Heatmap Full Round-Trip', + tiles: [ + { + name: 'Latency Heatmap', + x: 0, + y: 0, + w: 12, + h: 4, + config: createConfig, + }, + ], + }, + ); + expect(saveResult.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(saveResult)); + expect(saved.tiles).toHaveLength(1); + expect(saved.tiles[0].config).toMatchObject(createConfig); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: saved.id, + }, + ); + expect(getResult.isError).toBeFalsy(); + const fetched = JSON.parse(getFirstText(getResult)); + expect(fetched.tiles[0].config).toMatchObject(createConfig); + + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: saved.id, + name: 'Heatmap Full Round-Trip', + tiles: [ + { + ...fetched.tiles[0], + config: { ...fetched.tiles[0].config, ...updatedConfig }, + }, + ], + }, + ); + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.tiles[0].config).toMatchObject(updatedConfig); + + const getAfterUpdate = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: saved.id, + }, + ); + expect(getAfterUpdate.isError).toBeFalsy(); + const refetched = JSON.parse(getFirstText(getAfterUpdate)); + expect(refetched.tiles[0].config).toMatchObject(updatedConfig); + }); + + it('should reject heatmap tile with empty valueExpression at the schema layer', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Bad Heatmap', + tiles: [ + { + name: 'Heatmap', + config: { + displayType: 'heatmap', + sourceId, + select: [{ valueExpression: '' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + }); + + it.each([ + { output: 'duration', factor: 1e-9 }, + { output: 'data_rate' }, + { output: 'throughput' }, + ] as const)( + 'should round-trip numberFormat output "$output" through save and get', + async numberFormat => { + const sourceId = ctx.traceSource._id.toString(); + + const saveResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: `NumberFormat ${numberFormat.output}`, + tiles: [ + { + name: 'Number Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + numberFormat, + }, + }, + { + name: 'Line Tile', + config: { + displayType: 'line', + sourceId, + select: [ + { aggFn: 'count' }, + { + aggFn: 'avg', + valueExpression: 'Duration', + numberFormat, + }, + ], + }, + }, + ], + }, + ); + + expect(saveResult.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(saveResult)); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: saved.id, + }, + ); + expect(getResult.isError).toBeFalsy(); + const fetched = JSON.parse(getFirstText(getResult)); + + const numberTile = fetched.tiles.find( + (t: { name: string }) => t.name === 'Number Tile', + ); + expect(numberTile.config.numberFormat).toEqual(numberFormat); + + const lineTile = fetched.tiles.find( + (t: { name: string }) => t.name === 'Line Tile', + ); + expect(lineTile.config.select[1].numberFormat).toEqual(numberFormat); + }, + ); + + it('should reject numberFormat with an unknown output value', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Bad NumberFormat', + tiles: [ + { + name: 'Number Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + numberFormat: { output: 'not_a_real_output' }, + }, + }, + ], + }); + + expect(result.isError).toBe(true); + }); + + it('should reject heatmap tile on a non-Trace source', async () => { + // Create a Log source so the heatmap source-kind gate has + // something to reject. The schema accepts the tile shape (the + // sourceId is valid), so this exercises the runtime check that + // the REST POST path also runs. + const logSource = await Source.create({ + kind: SourceKind.Log, + team: ctx.team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: 'otel_logs', + }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Logs', + }); + + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Heatmap on Log Source', + tiles: [ + { + name: 'Heatmap', + config: { + displayType: 'heatmap', + sourceId: logSource._id.toString(), + select: [{ valueExpression: 'Duration' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toContain('Trace source'); + expect(text).toContain(logSource._id.toString()); + }); + + // Exercises the update-side source-kind gate via filterChangedHeatmapTiles + // (displayType changed to heatmap on an existing tile). + it('should reject update that changes a tile to heatmap on a non-Trace source', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: ctx.team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Logs', + }); + + const created = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Line on Log Source', + tiles: [ + { + name: 'Line', + config: { + displayType: 'line', + sourceId: logSource._id.toString(), + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + expect(created.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(created)); + + const update = await callTool(ctx.client!, 'clickstack_save_dashboard', { + id: saved.id, + name: 'Line on Log Source', + tiles: [ + { + ...saved.tiles[0], + config: { + ...saved.tiles[0].config, + displayType: 'heatmap', + select: [{ valueExpression: 'Duration' }], + }, + }, + ], + }); + expect(update.isError).toBe(true); + const text = getFirstText(update); + expect(text).toContain('Trace source'); + expect(text).toContain(logSource._id.toString()); + }); + + // Exercises the update-side source-kind gate via filterChangedHeatmapTiles + // (sourceId changed on an existing heatmap tile). + it('should reject update that changes a heatmap tile source to a non-Trace source', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: ctx.team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Logs', + }); + + const created = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Heatmap re-pointed at Log', + tiles: [ + { + name: 'Heatmap', + config: { + displayType: 'heatmap', + sourceId: ctx.traceSource._id.toString(), + select: [{ valueExpression: 'Duration' }], + }, + }, + ], + }); + expect(created.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(created)); + + const update = await callTool(ctx.client!, 'clickstack_save_dashboard', { + id: saved.id, + name: 'Heatmap re-pointed at Log', + tiles: [ + { + ...saved.tiles[0], + config: { + ...saved.tiles[0].config, + sourceId: logSource._id.toString(), + }, + }, + ], + }); + expect(update.isError).toBe(true); + const text = getFirstText(update); + expect(text).toContain('Trace source'); + expect(text).toContain(logSource._id.toString()); + }); + + // Asserts each tile's config survives the serializer/deserializer cycle + // independently when mixed with other displayTypes on the same dashboard. + it('should round-trip a heatmap alongside line and number tiles in one dashboard', async () => { + const sourceId = ctx.traceSource._id.toString(); + + const heatmapConfig = { + displayType: 'heatmap' as const, + sourceId, + select: [ + { valueExpression: 'Duration', heatmapScaleType: 'log' as const }, + ], + }; + const lineConfig = { + displayType: 'line' as const, + sourceId, + select: [{ aggFn: 'count' as const }], + groupBy: "SpanAttributes['service.name']", + }; + const numberConfig = { + displayType: 'number' as const, + sourceId, + select: [{ aggFn: 'count' as const }], + numberFormat: { output: 'number' as const, average: true }, + }; + + const save = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Mixed Tile Round-Trip', + tiles: [ + { name: 'Heatmap Tile', config: heatmapConfig }, + { name: 'Line Tile', config: lineConfig }, + { name: 'Number Tile', config: numberConfig }, + ], + }); + expect(save.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(save)); + expect(saved.tiles).toHaveLength(3); + + const byName: Record = + Object.fromEntries( + saved.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), + ); + expect(byName['Heatmap Tile'].config).toMatchObject(heatmapConfig); + expect(byName['Line Tile'].config).toMatchObject(lineConfig); + expect(byName['Number Tile'].config).toMatchObject(numberConfig); + + const fetched = JSON.parse( + getFirstText( + await callTool(ctx.client!, 'clickstack_get_dashboard', { + id: saved.id, + }), + ), + ); + const fetchedByName: Record = + Object.fromEntries( + fetched.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), + ); + expect(fetchedByName['Heatmap Tile'].config).toMatchObject(heatmapConfig); + expect(fetchedByName['Line Tile'].config).toMatchObject(lineConfig); + expect(fetchedByName['Number Tile'].config).toMatchObject(numberConfig); + }); + }); + + describe('containers and tabs', () => { + // Mirrors the v2 external-API "Containers and tabs" describe block so + // the MCP path enforces the same 5 cross-field rules: container id + // unique, tab id unique within a container, tile.containerId + // resolves, tile.tabId resolves to a tab on that container, and + // tile.tabId requires tile.containerId. + const buildTile = ( + sourceId: string, + overrides: Record, + ) => ({ + name: 'Tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line' as const, + sourceId, + select: [{ aggFn: 'count' as const, where: '' }], + }, + ...overrides, + }); + + it('should round-trip containers, tabs, and tile containerId/tabId on create and update', async () => { + const sourceId = ctx.traceSource._id.toString(); + const containers = [ + { + id: 'service-health', + title: 'Service Health', + collapsed: false, + collapsible: true, + bordered: true, + tabs: [ + { id: 'errors', title: 'Errors' }, + { id: 'latency', title: 'Latency' }, + ], + }, + { + id: 'overview', + title: 'Overview', + collapsed: true, + }, + ]; + + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Containers MCP Round-Trip', + tiles: [ + buildTile(sourceId, { + name: 'In Group, Tab A', + containerId: 'service-health', + tabId: 'errors', + }), + buildTile(sourceId, { + name: 'In Group, Tab B', + containerId: 'service-health', + tabId: 'latency', + }), + // Tile inside a tabbed container without a tabId renders in + // the container shell rather than under a tab. Guards that the + // schema does not accidentally require tabId for every tile in + // a tabbed container. + buildTile(sourceId, { + name: 'In Tabbed Group, No Tab', + containerId: 'service-health', + }), + buildTile(sourceId, { + name: 'In Plain Group', + containerId: 'overview', + }), + buildTile(sourceId, { name: 'Ungrouped' }), + ], + tags: ['containers-mcp'], + containers, + }, + ); + + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + expect(created.containers).toEqual(containers); + + // Typed as ExternalDashboardTileWithId so a typo on .containerId + // would fail typecheck instead of silently asserting undefined. + const createdTilesByName: Record = + Object.fromEntries( + created.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), + ); + expect(createdTilesByName['In Group, Tab A']).toMatchObject({ + containerId: 'service-health', + tabId: 'errors', + }); + expect(createdTilesByName['In Group, Tab B']).toMatchObject({ + containerId: 'service-health', + tabId: 'latency', + }); + expect(createdTilesByName['In Tabbed Group, No Tab']).toMatchObject({ + containerId: 'service-health', + }); + expect( + createdTilesByName['In Tabbed Group, No Tab'].tabId, + ).toBeUndefined(); + expect(createdTilesByName['In Plain Group']).toMatchObject({ + containerId: 'overview', + }); + expect(createdTilesByName['In Plain Group'].tabId).toBeUndefined(); + expect(createdTilesByName.Ungrouped.containerId).toBeUndefined(); + expect(createdTilesByName.Ungrouped.tabId).toBeUndefined(); + + // Verify a fresh GET via the get tool also returns the structure. + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: created.id, + }, + ); + const fetched = JSON.parse(getFirstText(getResult)); + expect(fetched.containers).toEqual(containers); + + // Update: rename a tab, drop the second container, re-home tiles. + const updatedContainers = [ + { + id: 'service-health', + title: 'Service Health', + collapsed: true, + tabs: [ + { id: 'errors', title: 'Error Rate' }, + { id: 'latency', title: 'Latency' }, + ], + }, + ]; + const reHomedUngrouped = { + ...createdTilesByName.Ungrouped, + containerId: 'service-health', + tabId: 'errors', + }; + const droppedContainerTile = { + ...createdTilesByName['In Plain Group'], + containerId: undefined, + tabId: undefined, + }; + + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'Containers MCP Round-Trip', + tiles: [ + createdTilesByName['In Group, Tab A'], + createdTilesByName['In Group, Tab B'], + createdTilesByName['In Tabbed Group, No Tab'], + droppedContainerTile, + reHomedUngrouped, + ], + tags: ['containers-mcp'], + containers: updatedContainers, + }, + ); + + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.containers).toEqual(updatedContainers); + const updatedTilesByName: Record = + Object.fromEntries( + updated.tiles.map((t: ExternalDashboardTileWithId) => [t.name, t]), + ); + expect(updatedTilesByName['In Plain Group'].containerId).toBeUndefined(); + expect(updatedTilesByName.Ungrouped).toMatchObject({ + containerId: 'service-health', + tabId: 'errors', + }); + expect(updatedTilesByName['In Tabbed Group, No Tab'].containerId).toBe( + 'service-health', + ); + expect( + updatedTilesByName['In Tabbed Group, No Tab'].tabId, + ).toBeUndefined(); + }); + + it('should reject duplicate container ids', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Duplicate Container Ids', + tiles: [buildTile(sourceId, {})], + containers: [ + { id: 'dupe', title: 'A', collapsed: false }, + { id: 'dupe', title: 'B', collapsed: false }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Container IDs must be unique'); + }); + + it('should reject duplicate tab ids within a container', async () => { + // Structural rules (duplicate container ids, duplicate tab ids) fire + // through the body schema, which is reported as a stringified issue + // array. Cross-tile-ref rules (covered below) fire through + // collectTileContainerRefIssues and produce plain prose. + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Duplicate Tab Ids', + tiles: [buildTile(sourceId, {})], + containers: [ + { + id: 'service-health', + title: 'Service Health', + collapsed: false, + tabs: [ + { id: 'errors', title: 'Errors' }, + { id: 'errors', title: 'Errors Two' }, + ], + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Duplicate tab id'); + expect(getFirstText(result)).toContain('errors'); + expect(getFirstText(result)).toContain('service-health'); + }); + + it('should reject a tile that supplies tabId without containerId', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Tab Without Container', + tiles: [buildTile(sourceId, { tabId: 'errors' })], + containers: [ + { + id: 'service-health', + title: 'Service Health', + collapsed: false, + tabs: [{ id: 'errors', title: 'Errors' }], + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'tabId requires containerId to be set', + ); + }); + + it('should reject a tile that references an unknown containerId', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Unknown Container', + tiles: [buildTile(sourceId, { containerId: 'does-not-exist' })], + containers: [{ id: 'real', title: 'Real', collapsed: false }], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'unknown containerId "does-not-exist"', + ); + }); + + it('should reject a tile that references an unknown tabId within a container', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Unknown Tab', + tiles: [ + buildTile(sourceId, { + containerId: 'service-health', + tabId: 'ghost', + }), + ], + containers: [ + { + id: 'service-health', + title: 'Service Health', + collapsed: false, + tabs: [{ id: 'errors', title: 'Errors' }], + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('unknown tabId "ghost"'); + }); + + it('should preserve existing containers on update when body omits the field', async () => { + // Exercises the `effectiveContainers = parsedContainers ?? existingDashboard.containers ?? []` + // fallback in the MCP update handler. Without it, a PUT that + // updates only `tiles` would reject because the tile's + // containerId reference would resolve against an empty array. + const sourceId = ctx.traceSource._id.toString(); + const containers = [ + { + id: 'service-health', + title: 'Service Health', + collapsed: false, + tabs: [{ id: 'errors', title: 'Errors' }], + }, + ]; + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'PUT-without-containers fallback', + tiles: [ + buildTile(sourceId, { + name: 'In Group', + containerId: 'service-health', + tabId: 'errors', + }), + ], + containers, + }, + ); + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'PUT-without-containers fallback', + tiles: created.tiles, + // containers intentionally omitted + }, + ); + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.containers).toEqual(containers); + expect(updated.tiles[0]).toMatchObject({ + containerId: 'service-health', + tabId: 'errors', + }); + }); + + it('should wipe persisted containers when update body sets containers: []', async () => { + // Mirror the v2 behavior: an explicit empty array clears the + // persisted containers, and the response normalizes [] back to + // absent on read. + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Wipe containers', + tiles: [buildTile(sourceId, { name: 'Tile' })], + containers: [{ id: 'overview', title: 'Overview', collapsed: false }], + }, + ); + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + expect(created.containers).toHaveLength(1); + + const wipedTile = { + ...created.tiles[0], + containerId: undefined, + tabId: undefined, + }; + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'Wipe containers', + tiles: [wipedTile], + containers: [], + }, + ); + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.containers).toBeUndefined(); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: created.id, + }, + ); + const fetched = JSON.parse(getFirstText(getResult)); + expect(fetched.containers).toBeUndefined(); + }); + + // Bounds mirror DASHBOARD_CONTAINER_ID_MAX (256) on the tile-level + // containerId / tabId. 256 chars must accept; 257 must reject. + // 257 trips the inputSchema's `.max(256)` and surfaces back as the + // MCP SDK's "Input validation error" envelope. + it('should accept a 256-char tile.containerId and reject 257', async () => { + const sourceId = ctx.traceSource._id.toString(); + const idAtMax = 'c'.repeat(256); + const okResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: '256-char containerId', + tiles: [buildTile(sourceId, { containerId: idAtMax })], + containers: [{ id: idAtMax, title: 'Max', collapsed: false }], + }, + ); + expect(okResult.isError).toBeFalsy(); + + const idTooLong = 'c'.repeat(257); + const tooLongResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: '257-char containerId', + tiles: [buildTile(sourceId, { containerId: idTooLong })], + containers: [{ id: idTooLong, title: 'Too long', collapsed: false }], + }, + ); + expect(tooLongResult.isError).toBe(true); + expect(getFirstText(tooLongResult)).toContain( + 'String must contain at most 256 character(s)', + ); + expect(getFirstText(tooLongResult)).toContain('containerId'); + }); + }); + + describe('dashboard filters', () => { + // The MCP input schema delegates to createDashboardBodySchema / + // updateDashboardBodySchema for the filter shape. These tests guard + // that the MCP path lights up the same filter round-trip the v2 REST + // path already covers: filters are saved verbatim, get back with an + // assigned id, survive an update, and reject obvious bad inputs. + + const traceTile = (sourceId: string) => ({ + name: 'Volume', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line' as const, + sourceId, + select: [{ aggFn: 'count' as const, where: '' }], + }, + }); + + it('should round-trip filters on create, get, and update', async () => { + const sourceId = ctx.traceSource._id.toString(); + + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Service Detail (MCP filter round-trip)', + tiles: [traceTile(sourceId)], + filters: [ + { + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + sourceId, + }, + { + type: 'QUERY_EXPRESSION', + name: 'Environment', + expression: 'deployment.environment', + sourceId, + where: "deployment.environment = 'production'", + whereLanguage: 'sql', + }, + ], + }, + ); + + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + expect(Array.isArray(created.filters)).toBe(true); + expect(created.filters).toHaveLength(2); + // The body schema assigns an id to each filter on create. Capture + // them so the update payload can include the same ids and the + // filter array round-trips identically (instead of being treated + // as wholesale replacement with new ids). + const [serviceFilter, envFilter] = created.filters; + expect(serviceFilter).toMatchObject({ + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + sourceId, + }); + expect(typeof serviceFilter.id).toBe('string'); + expect(serviceFilter.id.length).toBeGreaterThan(0); + expect(envFilter).toMatchObject({ + type: 'QUERY_EXPRESSION', + name: 'Environment', + expression: 'deployment.environment', + sourceId, + where: "deployment.environment = 'production'", + whereLanguage: 'sql', + }); + + // Fetch and assert the same shape. + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: created.id, + }, + ); + const fetched = JSON.parse(getFirstText(getResult)); + expect(fetched.filters).toEqual(created.filters); + + // Update: rename the first filter and drop the second. The first + // filter keeps its id, the second is dropped (not preserved). + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'Service Detail (MCP filter round-trip)', + tiles: [traceTile(sourceId)], + filters: [ + { + id: serviceFilter.id, + type: 'QUERY_EXPRESSION', + name: 'Service (renamed)', + expression: 'ServiceName', + sourceId, + }, + ], + }, + ); + + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.filters).toHaveLength(1); + expect(updated.filters[0]).toMatchObject({ + id: serviceFilter.id, + type: 'QUERY_EXPRESSION', + name: 'Service (renamed)', + expression: 'ServiceName', + sourceId, + }); + }); + + it('should accept a create payload with stray filter ids (copy-paste round-trip)', async () => { + // An LLM that copies a filter out of clickstack_get_dashboard and + // back into clickstack_save_dashboard for a NEW dashboard ships an + // `id` on the filter. The body schema for create is + // `.strict()` and rejects `id`, so saveDashboard normalizes by + // stripping ids before validation. Without that normalization + // this payload would 4xx with a confusing strict-validation + // error. + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Create with stray filter id', + tiles: [traceTile(sourceId)], + filters: [ + { + id: '999999999999999999999999', // simulates a copied id + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + sourceId, + }, + ], + }); + expect(result.isError).toBeFalsy(); + const dashboard = JSON.parse(getFirstText(result)); + expect(dashboard.filters).toHaveLength(1); + // The new dashboard gets a fresh id; the copied one is discarded. + expect(dashboard.filters[0].id).not.toBe('999999999999999999999999'); + expect(dashboard.filters[0]).toMatchObject({ + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + sourceId, + }); + }); + + it('should assign ids to new filters added during update (no-id round-trip)', async () => { + // An LLM updating a dashboard to ADD a brand-new filter ships + // a filter without an `id`. The body schema for update requires + // id, so saveDashboard fills one in. Without that normalization + // this payload would 4xx complaining that id is missing. + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Update adds new filter', + tiles: [traceTile(sourceId)], + filters: [ + { + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + sourceId, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const existingFilterId = created.filters[0].id; + + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'Update adds new filter', + tiles: [traceTile(sourceId)], + filters: [ + // Existing filter, preserved by id. + { + id: existingFilterId, + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + sourceId, + }, + // Brand-new filter, no id. saveDashboard assigns one. + { + type: 'QUERY_EXPRESSION', + name: 'Environment', + expression: 'deployment.environment', + sourceId, + }, + ], + }, + ); + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.filters).toHaveLength(2); + const envFilter = updated.filters.find( + (f: { name: string }) => f.name === 'Environment', + ); + expect(envFilter).toBeDefined(); + expect(typeof envFilter.id).toBe('string'); + expect(envFilter.id.length).toBeGreaterThan(0); + expect(envFilter.id).not.toBe(existingFilterId); + }); + + it('should round-trip a table tile that uses a having clause', async () => { + // mcpTableTileSchema exposes `having` so the service_detail + // example's "Top Error Messages" pattern (groupBy StatusMessage + // with having: "StatusMessage != ''") survives MCP authoring. + // Without `having` on the schema, Zod would strip it and the + // saved tile would include empty-message rows instead of the + // filtered set the prompt promises. + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Top Error Messages', + tiles: [ + { + name: 'Top Error Messages', + x: 0, + y: 0, + w: 12, + h: 6, + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count', alias: 'Count' }], + groupBy: 'StatusMessage', + having: "StatusMessage != ''", + orderBy: 'Count DESC', + groupByColumnsOnLeft: true, + }, + }, + ], + }); + expect(result.isError).toBeFalsy(); + const dashboard = JSON.parse(getFirstText(result)); + expect(dashboard.tiles[0].config.having).toBe("StatusMessage != ''"); + }); + }); + + describe('table onClick linking', () => { + // Mirrors the v2 external-API onClick tests so the MCP path + // enforces the same drill-down rules: row-click can target /search + // for a log/trace source or another dashboard, by concrete ID or + // by templated name, with optional whereTemplate/filters. + it('should round-trip a table tile with a search onClick by ID', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'OnClick search by id', + tiles: [ + { + name: 'Errors by Service', + x: 0, + y: 0, + w: 12, + h: 6, + config: { + displayType: 'table', + sourceId, + groupBy: "ResourceAttributes['service.name']", + select: [{ aggFn: 'count' }], + onClick: { + type: 'search', + target: { mode: 'id', id: sourceId }, + whereLanguage: 'sql', + filters: [ + { + kind: 'expressionTemplate', + expression: 'ServiceName', + template: "{{ResourceAttributes['service.name']}}", + }, + ], + }, + }, + }, + ], + }, + ); + + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + expect(created.tiles[0].config.onClick).toEqual({ + type: 'search', + target: { mode: 'id', id: sourceId }, + whereLanguage: 'sql', + filters: [ + { + kind: 'expressionTemplate', + expression: 'ServiceName', + template: "{{ResourceAttributes['service.name']}}", + }, + ], + }); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: created.id, + }, + ); + const fetched = JSON.parse(getFirstText(getResult)); + expect(fetched.tiles[0].config.onClick).toEqual( + created.tiles[0].config.onClick, + ); + }); + + it('should round-trip a table tile with a dashboard onClick by ID', async () => { + const sourceId = ctx.traceSource._id.toString(); + // Create the target dashboard the onClick will link to so the + // server-side `getMissingOnClickDashboards` check resolves. + const targetDashboard = await new Dashboard({ + name: 'Service Detail', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick dashboard by id', + tiles: [ + { + name: 'Top Services', + x: 0, + y: 0, + w: 12, + h: 6, + config: { + displayType: 'table', + sourceId, + groupBy: "ResourceAttributes['service.name']", + select: [{ aggFn: 'count' }], + onClick: { + type: 'dashboard', + target: { + mode: 'id', + id: targetDashboard._id.toString(), + }, + whereLanguage: 'sql', + filters: [ + { + kind: 'expressionTemplate', + expression: 'ServiceName', + template: "{{ResourceAttributes['service.name']}}", + }, + ], + }, + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles[0].config.onClick).toEqual({ + type: 'dashboard', + target: { mode: 'id', id: targetDashboard._id.toString() }, + whereLanguage: 'sql', + filters: [ + { + kind: 'expressionTemplate', + expression: 'ServiceName', + template: "{{ResourceAttributes['service.name']}}", + }, + ], + }); + }); + + it('should round-trip a templated dashboard onClick (mode=template)', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick dashboard by template', + tiles: [ + { + name: 'Service Picker', + x: 0, + y: 0, + w: 12, + h: 6, + config: { + displayType: 'table', + sourceId, + groupBy: 'SpanName', + select: [{ aggFn: 'count' }], + onClick: { + type: 'dashboard', + target: { + mode: 'template', + template: '{{TargetDashboard}}', + }, + whereLanguage: 'lucene', + whereTemplate: + "service.name:{{ResourceAttributes['service.name']}}", + }, + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles[0].config.onClick).toEqual({ + type: 'dashboard', + target: { mode: 'template', template: '{{TargetDashboard}}' }, + whereLanguage: 'lucene', + whereTemplate: "service.name:{{ResourceAttributes['service.name']}}", + }); + }); + + it('should round-trip onClick on a raw SQL table tile', async () => { + const connectionId = ctx.connection._id.toString(); + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'SQL onClick', + tiles: [ + { + name: 'Raw SQL Table', + x: 0, + y: 0, + w: 12, + h: 6, + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: + 'SELECT ServiceName, count() AS c FROM otel_traces GROUP BY ServiceName LIMIT 50', + onClick: { + type: 'search', + target: { mode: 'id', id: sourceId }, + whereLanguage: 'sql', + filters: [ + { + kind: 'expressionTemplate', + expression: 'ServiceName', + template: '{{ServiceName}}', + }, + ], + }, + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles[0].config.onClick).toEqual({ + type: 'search', + target: { mode: 'id', id: sourceId }, + whereLanguage: 'sql', + filters: [ + { + kind: 'expressionTemplate', + expression: 'ServiceName', + template: '{{ServiceName}}', + }, + ], + }); + }); + + it('should reject a table tile onClick referencing a non-existent dashboard', async () => { + const sourceId = ctx.traceSource._id.toString(); + const ghostDashboardId = new mongoose.Types.ObjectId().toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick missing dashboard', + tiles: [ + { + name: 'Top Services', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + onClick: { + type: 'dashboard', + target: { mode: 'id', id: ghostDashboardId }, + whereLanguage: 'sql', + }, + }, + }, + ], + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toContain('onClick dashboard'); + expect(text).toContain(ghostDashboardId); + }); + + it('should reject a table tile onClick search target with a non-log/trace source', async () => { + // /search only renders log/trace sources; targeting a metric + // source must be rejected at save time, matching the REST POST + // path. Create the metric source inline so this test is + // self-contained. + const metricSource = await Source.create({ + kind: SourceKind.Metric, + team: ctx.team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: '', + }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', + [MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum', + [MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram', + }, + timestampValueExpression: 'TimeUnix', + connection: ctx.connection._id, + name: 'Metrics', + }); + const sourceId = ctx.traceSource._id.toString(); + + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick metric source', + tiles: [ + { + name: 'Top Services', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + onClick: { + type: 'search', + target: { + mode: 'id', + id: metricSource._id.toString(), + }, + whereLanguage: 'sql', + }, + }, + }, + ], + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toContain('log or trace'); + expect(text).toContain(metricSource._id.toString()); + }); + + it('should reject a table tile onClick with an invalid target.id', async () => { + // target.id must be a valid ObjectId. Bad shape is caught by the + // input schema and surfaces as the SDK's "Input validation error" + // envelope rather than a custom 400 message. + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick bad object id', + tiles: [ + { + name: 'Top Services', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + onClick: { + type: 'dashboard', + target: { mode: 'id', id: 'not-a-valid-object-id' }, + whereLanguage: 'sql', + }, + }, + }, + ], + }); + + expect(result.isError).toBe(true); + }); + + it('should accept onClick without whereLanguage (it is optional at the schema layer)', async () => { + // SearchConditionLanguageSchema is `.optional()` in common-utils, + // so omitting whereLanguage is allowed. Pin that behavior so a + // future tightening of the schema doesn't quietly change it + // without updating the docs that callers rely on. + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick no whereLanguage', + tiles: [ + { + name: 'Top Services', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + onClick: { + type: 'search', + target: { mode: 'id', id: sourceId }, + // whereLanguage intentionally omitted + }, + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles[0].config.onClick).toMatchObject({ + type: 'search', + target: { mode: 'id', id: sourceId }, + }); + }); + + it('should reject an unknown onClick.type', async () => { + // The schema is a discriminated union on `type`; a bogus type + // must be rejected up front. + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'OnClick bad type', + tiles: [ + { + name: 'Top Services', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + onClick: { + type: 'navigate-to-mars', + target: { mode: 'id', id: sourceId }, + whereLanguage: 'sql', + }, + }, + }, + ], + }); + + expect(result.isError).toBe(true); + }); + + it('should validate onClick on update too', async () => { + // Round-trip a tile with no onClick, then update it to add an + // onClick pointing at a non-existent dashboard. The PUT path + // mirrors POST: missing dashboards are rejected with the same + // message. + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'OnClick update validation', + tiles: [ + { + name: 'Top Services', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + + const ghostDashboardId = new mongoose.Types.ObjectId().toString(); + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: created.id, + name: 'OnClick update validation', + tiles: [ + { + ...created.tiles[0], + config: { + ...created.tiles[0].config, + onClick: { + type: 'dashboard', + target: { mode: 'id', id: ghostDashboardId }, + whereLanguage: 'sql', + }, + }, + }, + ], + }, + ); + + expect(updateResult.isError).toBe(true); + const text = getFirstText(updateResult); + expect(text).toContain('onClick dashboard'); + expect(text).toContain(ghostDashboardId); + }); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/searchDashboards.test.ts b/packages/api/src/mcp/__tests__/dashboards/searchDashboards.test.ts new file mode 100644 index 0000000000..72a44d4d3f --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/searchDashboards.test.ts @@ -0,0 +1,261 @@ +import mongoose from 'mongoose'; + +import Dashboard from '@/models/dashboard'; + +import { callTool, getFirstText } from '../mcpTestUtils'; +import { setupDashboardTests } from './setup'; + +describe('MCP Dashboard Tools - clickstack_search_dashboards', () => { + const ctx = setupDashboardTests(); + + it('should find dashboards by name (case-insensitive)', async () => { + await new Dashboard({ + name: 'Service Overview', + tiles: [], + team: ctx.team._id, + tags: [], + }).save(); + await new Dashboard({ + name: 'Error Dashboard', + tiles: [], + team: ctx.team._id, + tags: [], + }).save(); + await new Dashboard({ + name: 'service metrics', + tiles: [], + team: ctx.team._id, + tags: [], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: 'service', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(2); + expect(output.map((d: { name: string }) => d.name).sort()).toEqual([ + 'Service Overview', + 'service metrics', + ]); + }); + + it('should find dashboards by tags', async () => { + await new Dashboard({ + name: 'Dashboard A', + tiles: [], + team: ctx.team._id, + tags: ['production', 'backend'], + }).save(); + await new Dashboard({ + name: 'Dashboard B', + tiles: [], + team: ctx.team._id, + tags: ['production', 'frontend'], + }).save(); + await new Dashboard({ + name: 'Dashboard C', + tiles: [], + team: ctx.team._id, + tags: ['staging'], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + tags: ['production'], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(2); + + // Search with multiple tags (AND) + const result2 = await callTool( + ctx.client!, + 'clickstack_search_dashboards', + { + tags: ['production', 'backend'], + }, + ); + + const output2 = JSON.parse(getFirstText(result2)); + expect(output2).toHaveLength(1); + expect(output2[0].name).toBe('Dashboard A'); + }); + + it('should combine name and tags filters', async () => { + await new Dashboard({ + name: 'API Service', + tiles: [], + team: ctx.team._id, + tags: ['production'], + }).save(); + await new Dashboard({ + name: 'API Errors', + tiles: [], + team: ctx.team._id, + tags: ['staging'], + }).save(); + await new Dashboard({ + name: 'Web Service', + tiles: [], + team: ctx.team._id, + tags: ['production'], + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: 'API', + tags: ['production'], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(1); + expect(output[0].name).toBe('API Service'); + }); + + it('should return empty array when no dashboards match', async () => { + await new Dashboard({ + name: 'Unrelated', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: 'nonexistent-dashboard-name', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(0); + }); + + it('should treat regex metacharacters as literal substrings', async () => { + await new Dashboard({ + name: 'API (v2) Service', + tiles: [], + team: ctx.team._id, + }).save(); + await new Dashboard({ + name: 'API v2 Service', + tiles: [], + team: ctx.team._id, + }).save(); + + // Parentheses are regex metacharacters — without escaping, "(v2)" + // would be treated as a capture group and match "v2" anywhere. + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: '(v2)', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + // Only the dashboard with literal "(v2)" should match + expect(output).toHaveLength(1); + expect(output[0].name).toBe('API (v2) Service'); + }); + + it('should not throw on regex-invalid characters like [', async () => { + await new Dashboard({ + name: 'Test Dashboard', + tiles: [], + team: ctx.team._id, + }).save(); + + // An unescaped "[" would throw a MongoDB BadValue error + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: '[invalid', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(0); + }); + + it('should match literal dots, not arbitrary characters', async () => { + await new Dashboard({ + name: 'api.v2.service', + tiles: [], + team: ctx.team._id, + }).save(); + await new Dashboard({ + name: 'apiXv2Xservice', + tiles: [], + team: ctx.team._id, + }).save(); + + // Unescaped "." in regex matches any character, so ".v2." would + // match "Xv2X". With escaping, only the literal dot matches. + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: '.v2.', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(1); + expect(output[0].name).toBe('api.v2.service'); + }); + + it('should reject empty query string', async () => { + await new Dashboard({ + name: 'Should Not Appear', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: '', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('at least one'); + }); + + it('should reject empty tags array', async () => { + await new Dashboard({ + name: 'Should Not Appear', + tiles: [], + team: ctx.team._id, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + tags: [], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('at least one'); + }); + + it('should reject empty query with empty tags', async () => { + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: '', + tags: [], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('at least one'); + }); + + it('should only return dashboards for the current team', async () => { + await new Dashboard({ + name: 'My Dashboard', + tiles: [], + team: ctx.team._id, + }).save(); + // Create a dashboard for a different team + const otherTeamId = new mongoose.Types.ObjectId(); + await new Dashboard({ + name: 'My Dashboard', + tiles: [], + team: otherTeamId, + }).save(); + + const result = await callTool(ctx.client!, 'clickstack_search_dashboards', { + query: 'My Dashboard', + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(1); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/setup.ts b/packages/api/src/mcp/__tests__/dashboards/setup.ts new file mode 100644 index 0000000000..46f1b3600a --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/setup.ts @@ -0,0 +1,81 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + DEFAULT_DATABASE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +import { McpContext } from '../../tools/types'; +import { createTestClient } from '../mcpTestUtils'; + +/** + * Shared setup/teardown for all dashboard MCP tool tests. + * + * Usage: + * const ctx = setupDashboardTests(); + * // Then access ctx.team, ctx.traceSource, ctx.connection, ctx.client + */ +export function setupDashboardTests() { + const server = getServer(); + const ctx = { + server, + team: null as any, + user: null as any, + traceSource: null as any, + connection: null as any, + client: null as Client | null, + }; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + ctx.team = result.team; + ctx.user = result.user; + + ctx.connection = await Connection.create({ + team: ctx.team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + ctx.traceSource = await Source.create({ + kind: SourceKind.Trace, + team: ctx.team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_TRACES_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Traces', + }); + + const mcpContext: McpContext = { + teamId: ctx.team._id.toString(), + userId: ctx.user._id.toString(), + }; + ctx.client = await createTestClient(mcpContext); + }); + + afterEach(async () => { + await ctx.client?.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + return ctx; +} diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index cb60705ffa..9b1cc2f730 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -821,7 +821,7 @@ Used in the "where" field of select items and search tiles. AND: service.name:api AND http.status_code:>=500 OR: level:error OR level:fatal NOT: NOT level:debug - Wildcard: service.name:front* + Wildcard: ServiceName:front* Phrase: Body:"connection refused" Exists: _exists_:http.route Range (numeric): Duration:>1000000000 @@ -831,16 +831,39 @@ Used in the "where" field of select items and search tiles. NOTE: In Lucene filters, use dot notation for attribute keys (service.name, http.method). This is different from valueExpression and groupBy which require bracket syntax (SpanAttributes['http.method']). -GOTCHA: Lucene queries that reference map-attribute keys via dotted paths (http.status_code, db.system, deployment.environment, anything under SpanAttributes / ResourceAttributes / LogAttributes) are NOT reliably translated to bracket access. Operators (>=, >, <=, <), wildcards (*), AND simple equality all hit translator gaps depending on the column. Symptoms: tile saves fine; clickstack_query_tile returns an error or returns zero rows when there should be matches; the dashboard renders "Error loading chart" on the tile. Do NOT trust dotted-path Lucene; use SQL with bracket access for ALL map-attribute filtering. +*** SUBSTRING MATCHING (field:value and field:value*) *** + +Lucene field:value does NOT mean exact equality. It translates to ilike(field, '%value%') -- a case-insensitive SUBSTRING match. This has two major consequences: + + 1. Plain value is a substring search: + SpanKind:Server matches "Server", "ServerStreaming", "InternalServer", etc. + SeverityText:error matches "error", "error_timeout", "critical_error", etc. + + 2. Trailing wildcard (field:val*) is prefix-within-substring, NOT a true prefix match: + ServiceName:api* does NOT mean "starts with api". It means "contains a substring starting with api". + In practice this often works for prefix matching, but it is NOT equivalent to SQL LIKE 'api%'. + + For exact matching or true prefix/suffix/contains patterns, use SQL instead: + Exact: whereLanguage: "sql", where: "SpanKind = 'Server'" + Prefix: whereLanguage: "sql", where: "ServiceName LIKE 'api%'" + Contains: whereLanguage: "sql", where: "Body LIKE '%timeout%'" + Regex: whereLanguage: "sql", where: "match(ServiceName, '^api-v[0-9]+')" + + When Lucene substring matching is FINE: free-text columns like Body, StatusMessage where substring is the intent. + When Lucene substring matching is WRONG: enum-like columns (SpanKind, SeverityText, StatusCode, ServiceName) where you want exact or true prefix match. + +*** MAP-ATTRIBUTE GOTCHA *** + +Lucene queries that reference map-attribute keys via dotted paths (http.status_code, db.system, deployment.environment, anything under SpanAttributes / ResourceAttributes / LogAttributes) are NOT reliably translated to bracket access. Operators (>=, >, <=, <), wildcards (*), AND simple equality all hit translator gaps depending on the column. Symptoms: tile saves fine; clickstack_query_tile returns an error or returns zero rows when there should be matches; the dashboard renders "Error loading chart" on the tile. Do NOT trust dotted-path Lucene; use SQL with bracket access for ALL map-attribute filtering. Unreliable in Lucene (any operation, any operator): http.status_code:>=500 (operator) - http.route:* (wildcard) + http.route:* (wildcard -- also hits the substring matching trap above) db.system:mongodb (simple equality; the translator does NOT map this to SpanAttributes['db.system']) Reliable in Lucene (top-level columns only): Duration:>1000000000 (top-level numeric range) - ServiceName:checkout (top-level string, see fuzzy-match note below for what this actually means) + ServiceName:checkout (top-level string -- but remember this is a substring match, not exact!) Workaround: switch to SQL with bracket access. Map values are stored as strings, so quote the comparison value. @@ -853,11 +876,6 @@ Workaround: switch to SQL with bracket access. Map values are stored as strings, whereLanguage: "sql" where: "SpanAttributes['http.route'] != ''" (filters out empty keys; map values are often empty) -FUZZY-MATCH NOTE: Lucene field:value on a top-level string column translates to ilike(field, '%value%'), not exact equality. That is fine for free-text columns (Body, StatusMessage) but surprising for enum-like columns (SpanKind, SeverityText, StatusCode). SpanKind:Server matches "Server" AND any other string containing "Server" as a substring. For exact-match semantics on top-level columns, use SQL: - - whereLanguage: "sql" - where: "SpanKind = 'Server'" - == SQL FILTER SYNTAX == Alternative to Lucene. Set whereLanguage: "sql" when using SQL syntax. @@ -1247,10 +1265,12 @@ Example: find top patterns for production services over the last 4 hours: Correct: where: "SpanAttributes['db.system'] = 'mongodb'" (whereLanguage: "sql") Lucene parses dotted-path queries happily at save time but the translator does not reliably map them to bracket access. Behavior varies by column and operator. Treat any Lucene filter on a map-attribute key as a save-time bug and use SQL with bracket access for ALL map-attribute filtering. Map values are stored as strings; quote the comparison value. -15. Lucene field:value on enum-like top-level columns (fuzzy substring match, not equality) - Wrong: where: "SpanKind:Server" (whereLanguage: "lucene"; translates to ilike(SpanKind, '%Server%') and matches "Server_Client", "Internal_Server_Error", etc.) - Correct: where: "SpanKind = 'Server'" (whereLanguage: "sql") - Lucene field:value is a substring match (ilike with surrounding wildcards), not exact equality. Fine for Body and StatusMessage; wrong for enum-like columns where exact match matters (SpanKind, SeverityText, StatusCode). When the column has discrete known values, use SQL with =. +15. Lucene field:value and field:value* (substring matching, not exact or true prefix) + Wrong: where: "SpanKind:Server" (whereLanguage: "lucene"; translates to ilike(SpanKind, '%Server%') and matches "ServerStreaming", "InternalServer", etc.) + Wrong: where: "ServiceName:api*" (whereLanguage: "lucene"; NOT a true prefix match -- it is prefix-within-substring) + Correct: where: "SpanKind = 'Server'" (whereLanguage: "sql"; exact match) + Correct: where: "ServiceName LIKE 'api%'" (whereLanguage: "sql"; true prefix match) + Lucene field:value is a substring match (ilike with surrounding wildcards), not exact equality. Lucene field:value* is prefix-within-substring, not SQL LIKE 'value%'. Fine for free-text columns (Body, StatusMessage); wrong for enum-like columns where exact match matters (SpanKind, SeverityText, StatusCode) and wrong when true prefix matching is needed. Use SQL for exact match (=), prefix (LIKE 'x%'), contains (LIKE '%x%'), or regex (match(col, 'pattern')). 16. Forgetting that map-attribute values are often empty strings Wrong: groupBy on SpanAttributes['db.collection.name'] without filtering empty values; the result has an empty-string bucket alongside the real ones, which renders as a blank row in tables and an unlabelled slice in pies. diff --git a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts new file mode 100644 index 0000000000..03f834809c --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts @@ -0,0 +1,83 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import mongoose from 'mongoose'; +import { z } from 'zod'; + +import Dashboard from '@/models/dashboard'; +import { convertToExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; + +export function registerGetDashboardTile( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'clickstack_get_dashboard_tile', + { + title: 'Get a Single Dashboard Tile', + description: + 'Retrieve a single tile from a dashboard by tileId. ' + + 'Useful for inspecting one tile without loading the full dashboard. ' + + 'Use clickstack_get_dashboard (without an ID) to list dashboards, ' + + 'then clickstack_get_dashboard (with an ID) to see all tile IDs.', + inputSchema: z.object({ + dashboardId: z.string().describe('Dashboard ID.'), + tileId: z + .string() + .describe( + 'Tile ID within the dashboard. ' + + 'Obtain from clickstack_get_dashboard.', + ), + }), + }, + withToolTracing( + 'clickstack_get_dashboard_tile', + context, + async ({ dashboardId, tileId }) => { + if (!mongoose.Types.ObjectId.isValid(dashboardId)) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], + }; + } + + const dashboard = await Dashboard.findOne({ + _id: dashboardId, + team: teamId, + }); + if (!dashboard) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + const externalDashboard = convertToExternalDashboard(dashboard); + const tile = externalDashboard.tiles.find(t => t.id === tileId); + if (!tile) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => `${t.id} (${t.name})`).join(', ')}`, + }, + ], + }; + } + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify(tile, null, 2), + }, + ], + }; + }, + ), + ); +} diff --git a/packages/api/src/mcp/tools/dashboards/index.ts b/packages/api/src/mcp/tools/dashboards/index.ts index a1e511e995..f479ae541b 100644 --- a/packages/api/src/mcp/tools/dashboards/index.ts +++ b/packages/api/src/mcp/tools/dashboards/index.ts @@ -3,8 +3,11 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { McpContext, ToolDefinition } from '../types'; import { registerDeleteDashboard } from './deleteDashboard'; import { registerGetDashboard } from './getDashboard'; +import { registerGetDashboardTile } from './getDashboardTile'; +import { registerPatchDashboard } from './patchDashboard'; import { registerQueryTile } from './queryTile'; import { registerSaveDashboard } from './saveDashboard'; +import { registerSearchDashboards } from './searchDashboards'; export * from './schemas'; @@ -13,8 +16,11 @@ const dashboardsTools: ToolDefinition = ( context: McpContext, ) => { registerGetDashboard(server, context); + registerGetDashboardTile(server, context); registerSaveDashboard(server, context); + registerPatchDashboard(server, context); registerDeleteDashboard(server, context); + registerSearchDashboards(server, context); registerQueryTile(server, context); }; diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts new file mode 100644 index 0000000000..0ee13f8cf9 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -0,0 +1,286 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { uniq } from 'lodash'; +import mongoose from 'mongoose'; + +import * as config from '@/config'; +import Dashboard from '@/models/dashboard'; +import { + cleanupDashboardAlerts, + convertToExternalDashboard, + convertToInternalTileConfig, + isConfigTile, + validateDashboardTiles, +} from '@/routers/external-api/v2/utils/dashboards'; +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { mcpPatchDashboardSchema } from './schemas'; + +export function registerPatchDashboard( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + const frontendUrl = config.FRONTEND_URL; + + server.registerTool( + 'clickstack_patch_dashboard', + { + title: 'Patch Dashboard', + description: + 'Make targeted updates to a dashboard without resubmitting the full object. ' + + 'You can update dashboard-level fields (name, tags) and/or replace a single ' + + 'tile by tileId \u2014 all in one call. Unmentioned tiles and fields are preserved. ' + + 'Use clickstack_get_dashboard_tile to inspect a tile before patching it. ' + + 'IMPORTANT: After patching a tile, run clickstack_query_tile to confirm the query still works.', + inputSchema: mcpPatchDashboardSchema, + }, + withToolTracing( + 'clickstack_patch_dashboard', + context, + async ({ dashboardId, name, tags, tileId, tile: inputTile }) => { + // Cross-field validation (kept in handler so the inputSchema + // stays a plain z.object and its properties are visible in the + // JSON Schema that the MCP SDK exposes to LLMs). + if ( + name === undefined && + tags === undefined && + (tileId === undefined || inputTile === undefined) + ) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: 'Provide at least one of: name, tags, or tileId+tile to patch.', + }, + ], + }; + } + if ((tileId === undefined) !== (inputTile === undefined)) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: 'tileId and tile must both be provided or both omitted.', + }, + ], + }; + } + + if (!mongoose.Types.ObjectId.isValid(dashboardId)) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], + }; + } + + const existingDashboard = await Dashboard.findOne({ + _id: dashboardId, + team: teamId, + }); + if (!existingDashboard) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + // Build the $set payload and the query filter. Metadata fields + // are simple top-level $set entries; the tile patch uses the + // positional $ operator matched by 'tiles.id' in the filter. + const setPayload: Record = {}; + const queryFilter: Record = { + _id: dashboardId, + team: teamId, + }; + + if (name !== undefined) { + setPayload.name = name; + } + if (tags !== undefined) { + setPayload.tags = uniq(tags); + } + + let patchedTile: ExternalDashboardTileWithId | undefined; + + // ── Tile-level patch ───────────────────────────────────────── + if (tileId !== undefined && inputTile !== undefined) { + // Work directly with the persisted internal tiles array so + // untouched tiles are never round-tripped through the external + // converter (which would strip orphaned container refs from + // unrelated tiles). + const internalTiles = + (existingDashboard.tiles as { id: string }[]) ?? []; + const existingIdx = internalTiles.findIndex(t => t.id === tileId); + if (existingIdx === -1) { + // Build a human-readable list of available tile IDs. + // Convert only for the error message (read-only, no write-back). + const externalDashboard = + convertToExternalDashboard(existingDashboard); + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => `${t.id} (${t.name})`).join(', ')}`, + }, + ], + }; + } + + // Read the existing tile's layout and container refs from the + // persisted internal format so fallback values are accurate + // regardless of the external converter's self-healing logic. + const existingInternalTile = internalTiles[existingIdx] as { + id: string; + x?: number; + y?: number; + w?: number; + h?: number; + containerId?: string; + tabId?: string; + config?: { name?: string }; + }; + + // Merge: the incoming tile definition replaces config/name, + // but layout and container refs fall back to the existing tile + // when omitted so the LLM doesn't have to re-specify them. + // Coerce legacy empty-string containerId/tabId to undefined so + // they don't trip the container-ref validator (mirrors the + // self-heal in convertTileToExternalChart). + const incoming = inputTile as Partial; + const existingContainerId = + existingInternalTile.containerId || undefined; + const existingTabId = existingInternalTile.tabId || undefined; + const mergedTile: ExternalDashboardTileWithId = { + id: tileId, + name: incoming.name ?? existingInternalTile.config?.name ?? '', + x: incoming.x ?? existingInternalTile.x ?? 0, + y: incoming.y ?? existingInternalTile.y ?? 0, + w: incoming.w ?? existingInternalTile.w ?? 12, + h: incoming.h ?? existingInternalTile.h ?? 4, + containerId: + 'containerId' in incoming + ? incoming.containerId + : existingContainerId, + tabId: 'tabId' in incoming ? incoming.tabId : existingTabId, + // The config comes from the incoming tile (validated by Zod). + config: incoming.config, + } as ExternalDashboardTileWithId; + + // Validate the patched tile using the shared validation helper. + const validationError = await validateDashboardTiles({ + teamId, + tiles: [mergedTile], + existingTiles: existingDashboard.tiles ?? [], + containers: existingDashboard.containers ?? [], + }); + if (validationError) { + return { + isError: true, + content: [{ type: 'text' as const, text: validationError }], + }; + } + + // Convert only the patched tile to internal format. + if (!isConfigTile(mergedTile)) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: 'Tile must have a config block.', + }, + ], + }; + } + const internalTile = convertToInternalTileConfig(mergedTile); + + // Use the positional $ operator matched by 'tiles.id' in the + // query filter. This targets the tile by its id field rather + // than a captured numeric index, so a concurrent save_dashboard + // that replaces the whole tiles array can't cause us to + // overwrite an unrelated tile at a stale index. + queryFilter['tiles.id'] = tileId; + setPayload['tiles.$'] = internalTile; + patchedTile = mergedTile; + } + + const updatedDashboard = await Dashboard.findOneAndUpdate( + queryFilter, + { $set: setPayload }, + { new: true }, + ); + + if (!updatedDashboard) { + // When a tile patch is in flight, a null result means the tile + // was removed or the dashboard was deleted between our read + // and this write. + if (tileId !== undefined) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: + `Tile ${tileId} was not found at write time (it may have been removed by a concurrent update). ` + + 'The entire update was rejected — name/tags changes (if any) were not applied. Resubmit.', + }, + ], + }; + } + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + // Reconcile alerts: if the tile's displayType changed to one + // that doesn't support alerts (e.g. raw SQL line/pie), clean up + // stale alert documents. Scope to just the patched tile — pass + // it as both the "new" tiles and "existing" ids so the helper + // checks whether the updated config still supports alerts. + if (tileId !== undefined) { + const existingTileIds = new Set([tileId]); + const patchedTileInDb = updatedDashboard.tiles.filter( + t => t.id === tileId, + ); + await cleanupDashboardAlerts({ + dashboardId, + teamId, + internalTiles: patchedTileInDb, + existingTileIds, + }); + } + + // Return a lightweight response: the patched tile (if any) plus + // updated dashboard metadata, without the full tile array. + const output: Record = { + id: updatedDashboard._id.toString(), + name: updatedDashboard.name, + tags: updatedDashboard.tags, + ...(frontendUrl + ? { url: `${frontendUrl}/dashboards/${updatedDashboard._id}` } + : {}), + }; + if (patchedTile) { + output.patchedTile = patchedTile; + output.hint = + 'Use clickstack_query_tile to test the patched tile query.'; + } + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify(output, null, 2), + }, + ], + }; + }, + ), + ); +} diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index 4e118c6212..84f19db437 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -8,20 +8,13 @@ import * as config from '@/config'; import Dashboard, { IDashboard } from '@/models/dashboard'; import { cleanupDashboardAlerts, - collectTileContainerRefIssues, convertExternalFiltersToInternal, convertExternalTilesToInternal, convertToExternalDashboard, createDashboardBodySchema, - fetchSourcesForValidation, - filterChangedHeatmapTiles, - getHeatmapTilesWithIncompatibleSources, - getInvalidOnClickSearchSources, - getMissingConnections, - getMissingOnClickDashboards, - getMissingSources, resolveSavedQueryLanguage, updateDashboardBodySchema, + validateDashboardTiles, } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardFilter, @@ -49,7 +42,8 @@ export function registerSaveDashboard( 'Call clickstack_list_sources first to obtain sourceId and connectionId values. ' + 'IMPORTANT: After saving a dashboard, always run clickstack_query_tile on each tile ' + 'to confirm the queries work and return expected data. Tiles can silently fail ' + - 'due to incorrect filter syntax, missing attributes, or wrong column names.', + 'due to incorrect filter syntax, missing attributes, or wrong column names. ' + + 'TIP: To update a single tile without resubmitting all tiles, use clickstack_patch_dashboard instead.', inputSchema: z.object({ id: z .string() @@ -180,105 +174,16 @@ async function createDashboard({ const { tiles, filters, containers: parsedContainers } = parsed.data; const tilesWithId = tiles as ExternalDashboardTileWithId[]; - // Mirror the v2 router POST handler: structural container checks ran - // through the body schema; per-tile containerId/tabId resolution runs - // against the request body's containers (no existing dashboard to fall - // back to on create). - const tileRefIssues = collectTileContainerRefIssues( - parsedContainers ?? [], - tilesWithId, - ); - if (tileRefIssues.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Validation error: ${tileRefIssues.join('; ')}`, - }, - ], - }; - } - - // Hoist the source fetch so missing-source and heatmap-source-kind - // checks share a single DB round-trip, mirroring the REST POST path. - const [ - sources, - missingConnections, - missingOnClickDashboards, - invalidOnClickSearchSources, - ] = await Promise.all([ - fetchSourcesForValidation(teamId), - getMissingConnections(teamId, tilesWithId), - getMissingOnClickDashboards(teamId, tilesWithId), - getInvalidOnClickSearchSources(teamId, tilesWithId), - ]); - - const missingSources = getMissingSources(sources, tilesWithId, filters); - if (missingSources.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Could not find source IDs: ${missingSources.join(', ')}`, - }, - ], - }; - } - if (missingConnections.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Could not find connection IDs: ${missingConnections.join(', ')}`, - }, - ], - }; - } - - // Source-kind gate for heatmap tiles. Mirrors the REST POST path so - // an MCP-issued create cannot persist a heatmap that the REST PUT - // would reject with 400 on the next round-trip. - const heatmapNonTraceSources = getHeatmapTilesWithIncompatibleSources( - sources, - tilesWithId, - ); - if (heatmapNonTraceSources.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapNonTraceSources.join(', ')}`, - }, - ], - }; - } - - // Validate that a table tile's row-click will not land on a - // missing dashboard or a non-log/trace source. - if (missingOnClickDashboards.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`, - }, - ], - }; - } - if (invalidOnClickSearchSources.length > 0) { + const validationError = await validateDashboardTiles({ + teamId, + tiles: tilesWithId, + filters, + containers: parsedContainers ?? [], + }); + if (validationError) { return { isError: true, - content: [ - { - type: 'text' as const, - text: `The following onClick search source IDs are not log or trace sources: ${invalidOnClickSearchSources.join(', ')}`, - }, - ], + content: [{ type: 'text' as const, text: validationError }], }; } @@ -373,53 +278,10 @@ async function updateDashboard({ const { tiles, filters, containers: parsedContainers } = parsed.data; const tilesWithId = tiles as ExternalDashboardTileWithId[]; - // Hoist sources, connections, and existing-dashboard fetches into a - // single Promise.all so the source list is shared across helpers and - // the heatmap source-kind check can scope itself to tiles whose - // sourceId/displayType actually changed in this request, matching - // the REST PUT path. `containers` is in the projection so the - // container/tab ref check can fall back to the persisted containers - // when the payload omits them. - const [ - sources, - missingConnections, - missingOnClickDashboards, - invalidOnClickSearchSources, - existingDashboard, - ] = await Promise.all([ - fetchSourcesForValidation(teamId), - getMissingConnections(teamId, tilesWithId), - getMissingOnClickDashboards(teamId, tilesWithId), - getInvalidOnClickSearchSources(teamId, tilesWithId), - Dashboard.findOne( - { _id: dashboardId, team: teamId }, - { tiles: 1, filters: 1, containers: 1 }, - ).lean(), - ]); - - const missingSources = getMissingSources(sources, tilesWithId, filters); - if (missingSources.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Could not find source IDs: ${missingSources.join(', ')}`, - }, - ], - }; - } - if (missingConnections.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Could not find connection IDs: ${missingConnections.join(', ')}`, - }, - ], - }; - } + const existingDashboard = await Dashboard.findOne( + { _id: dashboardId, team: teamId }, + { tiles: 1, filters: 1, containers: 1 }, + ).lean(); if (!existingDashboard) { return { @@ -428,70 +290,19 @@ async function updateDashboard({ }; } - // Mirror the v2 router PUT handler: resolve tile container/tab refs - // against an effective container set so a payload that omits the - // `containers` field falls back to the persisted dashboard rather - // than an empty fallback. Otherwise a tile pointing at a preserved - // container would be rejected with "Tile references unknown - // containerId". const effectiveContainers = parsedContainers ?? existingDashboard.containers ?? []; - const tileRefIssues = collectTileContainerRefIssues( - effectiveContainers, - tilesWithId, - ); - if (tileRefIssues.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Validation error: ${tileRefIssues.join('; ')}`, - }, - ], - }; - } - - // Source-kind gate, scoped to heatmap tiles whose source or - // displayType changed in this request. Mirrors the REST PUT path. - const heatmapNonTraceSources = getHeatmapTilesWithIncompatibleSources( - sources, - filterChangedHeatmapTiles(tilesWithId, existingDashboard.tiles ?? []), - ); - if (heatmapNonTraceSources.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapNonTraceSources.join(', ')}`, - }, - ], - }; - } - - // Validate that a table tile's row-click will not land on a - // missing dashboard or a non-log/trace source. - if (missingOnClickDashboards.length > 0) { - return { - isError: true, - content: [ - { - type: 'text' as const, - text: `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`, - }, - ], - }; - } - if (invalidOnClickSearchSources.length > 0) { + const validationError = await validateDashboardTiles({ + teamId, + tiles: tilesWithId, + filters, + existingTiles: existingDashboard.tiles ?? [], + containers: effectiveContainers, + }); + if (validationError) { return { isError: true, - content: [ - { - type: 'text' as const, - text: `The following onClick search source IDs are not log or trace sources: ${invalidOnClickSearchSources.join(', ')}`, - }, - ], + content: [{ type: 'text' as const, text: validationError }], }; } diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index fc692acbb7..e8c25b8997 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -311,7 +311,7 @@ const mcpOnClickSchema = z ); const mcpTileLayoutSchema = z.object({ - name: z.string().describe('Tile title shown on the dashboard'), + name: z.string().min(1).describe('Tile title shown on the dashboard'), x: z .number() .min(0) @@ -612,6 +612,46 @@ const mcpTileSchema = z.union([ mcpSqlTileSchema, ]); +// Layout schema without defaults for the patch tool: layout fields stay +// truly `undefined` when omitted so the merge logic can fall back to the +// existing tile's values instead of Zod filling in 0/12/4. +const mcpPatchTileLayoutSchema = z.object({ + name: z + .string() + .min(1) + .optional() + .describe('Tile title. Omit to keep the existing title.'), + x: z.number().min(0).max(23).optional(), + y: z.number().min(0).optional(), + w: z.number().min(1).max(24).optional(), + h: z.number().min(1).optional(), + containerId: z.string().min(1).max(DASHBOARD_CONTAINER_ID_MAX).optional(), + tabId: z.string().min(1).max(DASHBOARD_CONTAINER_ID_MAX).optional(), +}); + +// Build the patch tile union by extending the default-free layout with +// each tile type's config shape. We only need the `config` field from +// each tile schema; the layout wrapper is replaced. +const mcpPatchTileSchema = z.union([ + mcpPatchTileLayoutSchema.extend({ config: mcpLineTileSchema.shape.config }), + mcpPatchTileLayoutSchema.extend({ config: mcpBarTileSchema.shape.config }), + mcpPatchTileLayoutSchema.extend({ config: mcpTableTileSchema.shape.config }), + mcpPatchTileLayoutSchema.extend({ + config: mcpNumberTileSchema.shape.config, + }), + mcpPatchTileLayoutSchema.extend({ config: mcpPieTileSchema.shape.config }), + mcpPatchTileLayoutSchema.extend({ + config: mcpHeatmapTileSchema.shape.config, + }), + mcpPatchTileLayoutSchema.extend({ + config: mcpSearchTileSchema.shape.config, + }), + mcpPatchTileLayoutSchema.extend({ + config: mcpMarkdownTileSchema.shape.config, + }), + mcpPatchTileLayoutSchema.extend({ config: mcpSqlTileSchema.shape.config }), +]); + export const mcpTilesParam = z .array(mcpTileSchema) .describe( @@ -735,6 +775,49 @@ export const mcpFiltersParam = z ']', ); +export const mcpPatchDashboardSchema = z.object({ + dashboardId: z.string().describe('Dashboard ID.'), + name: z + .string() + .min(1) + .optional() + .describe('New dashboard name. Omit to keep the current name.'), + tags: z + .array(z.string()) + .optional() + .describe( + 'New tags array (replaces all existing tags). Omit to keep the current tags.', + ), + tileId: z + .string() + .optional() + .describe( + 'ID of the tile to replace. Must be paired with `tile`. ' + + 'Obtain tile IDs from clickstack_get_dashboard.', + ), + tile: mcpPatchTileSchema + .optional() + .describe( + 'The full replacement tile definition. Replaces the tile matched by tileId. ' + + 'Layout fields (x, y, w, h), name, and containerId/tabId default to the ' + + "existing tile's values when omitted, so you only need to specify what changed.", + ), +}); + +export const mcpSearchDashboardsSchema = z.object({ + query: z + .string() + .max(200) + .optional() + .describe( + 'Search term to match against dashboard names (case-insensitive substring match).', + ), + tags: z + .array(z.string().min(1)) + .optional() + .describe('Filter to dashboards that have ALL of these tags.'), +}); + export const mcpContainersParam = z .array(DashboardContainerSchema) .max(DASHBOARD_MAX_CONTAINERS) diff --git a/packages/api/src/mcp/tools/dashboards/searchDashboards.ts b/packages/api/src/mcp/tools/dashboards/searchDashboards.ts new file mode 100644 index 0000000000..fa77915fd3 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/searchDashboards.ts @@ -0,0 +1,115 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { escapeRegExp } from 'lodash'; + +import * as config from '@/config'; +import Dashboard from '@/models/dashboard'; +import logger from '@/utils/logger'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { mcpSearchDashboardsSchema } from './schemas'; + +const SEARCH_RESULTS_LIMIT = 100; + +export function registerSearchDashboards( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + const frontendUrl = config.FRONTEND_URL; + + server.registerTool( + 'clickstack_search_dashboards', + { + title: 'Search Dashboards', + description: + 'Search dashboards by name and/or tags. Returns matching dashboards with ' + + 'their IDs, names, and tags. More targeted than clickstack_get_dashboard (which ' + + 'lists all dashboards). At least one of query or tags must be provided.', + inputSchema: mcpSearchDashboardsSchema, + }, + withToolTracing( + 'clickstack_search_dashboards', + context, + async ({ query, tags }) => { + const hasQuery = typeof query === 'string' && query.length > 0; + const hasTags = Array.isArray(tags) && tags.length > 0; + + if (!hasQuery && !hasTags) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: 'Provide at least one of: query (non-empty string) or tags (non-empty array).', + }, + ], + }; + } + + const filter: Record = { team: teamId }; + + if (hasQuery) { + filter.name = { $regex: escapeRegExp(query), $options: 'i' }; + } + if (hasTags) { + filter.tags = { $all: tags }; + } + + try { + const dashboards = await Dashboard.find(filter) + .select({ name: 1, tags: 1 }) + .limit(SEARCH_RESULTS_LIMIT + 1) + .lean(); + + const truncated = dashboards.length > SEARCH_RESULTS_LIMIT; + const results = truncated + ? dashboards.slice(0, SEARCH_RESULTS_LIMIT) + : dashboards; + + const output = results.map(d => ({ + id: d._id.toString(), + name: d.name, + tags: d.tags, + ...(frontendUrl + ? { url: `${frontendUrl}/dashboards/${d._id}` } + : {}), + })); + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + truncated + ? { + results: output, + truncated: true, + hint: `Returned first ${SEARCH_RESULTS_LIMIT} results. Narrow your query or tags to see more.`, + } + : output, + null, + 2, + ), + }, + ], + }; + } catch (err) { + logger.error( + { err, teamId, query, tags }, + 'clickstack_search_dashboards: query failed', + ); + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Search failed: ${err instanceof Error ? err.message : String(err)}`, + }, + ], + }; + } + }, + ), + ); +} diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 5fe673da7f..19abd9c71c 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -13,7 +13,10 @@ const WHERE_DESCRIPTION = ' Lucene uses DOT notation: SpanAttributes.http.method:GET\n' + " SQL uses BRACKET notation: SpanAttributes['http.method'] = 'GET'\n\n" + "WRONG: SpanAttributes['key']:value (Lucene cannot parse bracket syntax)\n" + - 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")'; + 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")\n\n' + + 'SUBSTRING TRAP: Lucene field:value matches ANY row containing "value" as a substring, not exact equality.\n' + + ' SpanKind:Server matches "Server", "ServerStreaming", "InternalServer", etc.\n' + + " For exact match, use SQL: SpanKind = 'Server'"; const WHERE_LANGUAGE_DESCRIPTION = 'Query language for the "where" filter. Default: lucene.\n' + @@ -21,8 +24,10 @@ const WHERE_LANGUAGE_DESCRIPTION = ' Lucene (default): Column:value, Column.mapKey:value, Column:>100\n' + " SQL: Column = 'value', SpanAttributes['key'] = 'value'\n\n" + 'Lucene supports comparisons (>= > < <=), wildcards (field:val*), ranges ([1 TO 5]), ' + - 'and map attributes via dot notation. ' + - 'Use "sql" for IN(...) lists, complex expressions, or function calls.\n\n' + + 'and map attributes via dot notation. Use "sql" for IN(...) lists, complex expressions, or function calls.\n' + + 'IMPORTANT: Lucene field:value is a SUBSTRING match (ilike), not exact equality. ' + + 'field:val* is prefix-within-substring, not a true prefix match. ' + + "For exact matching or reliable wildcards, use SQL: WHERE field = 'value' or WHERE field LIKE 'val%'.\n\n" + 'Common mistake: writing Column:value (Lucene) but setting whereLanguage to "sql". ' + 'If your filter uses colon syntax, leave whereLanguage as "lucene" (the default).'; diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index cb3735ae82..5b83bcc41d 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -9,23 +9,13 @@ import { ExternalDashboardTileWithId, objectIdSchema } from '@/utils/zod'; import { cleanupDashboardAlerts, - collectTileContainerRefIssues, convertExternalFiltersToInternal, convertExternalTilesToInternal, convertToExternalDashboard, createDashboardBodySchema, - fetchSourcesForValidation, - filterChangedHeatmapTiles, - getHeatmapTilesWithIncompatibleSources, - getInvalidOnClickSearchSources, - getMissingConnections, - getMissingOnClickDashboards, - getMissingSources, - isConfigTile, - isRawSqlExternalTileConfig, resolveSavedQueryLanguage, - SourceForValidation, updateDashboardBodySchema, + validateDashboardTiles, } from './utils/dashboards'; /** @@ -45,29 +35,6 @@ const EXTERNAL_DASHBOARD_PROJECTION = { containers: 1, } as const; -function getSourceConnectionMismatches( - sources: SourceForValidation[], - tiles: ExternalDashboardTileWithId[], -): string[] { - const sourceById = new Map(sources.map(s => [s._id.toString(), s])); - - const sourcesWithInvalidConnections: string[] = []; - for (const tile of tiles) { - if ( - isConfigTile(tile) && - isRawSqlExternalTileConfig(tile.config) && - tile.config.sourceId - ) { - const source = sourceById.get(tile.config.sourceId); - if (source && source.connection.toString() !== tile.config.connectionId) { - sourcesWithInvalidConnections.push(tile.config.sourceId); - } - } - } - - return sourcesWithInvalidConnections; -} - /** * @openapi * components: @@ -1922,90 +1889,14 @@ router.post( containers, } = req.body; - // Tile-level container/tab ref resolution: container structure - // (duplicate ids, tab uniqueness) was checked by the body schema; - // tile-resolution runs here against the request body's containers - // (POST has no existing dashboard to fall back to). - const tileRefIssues = collectTileContainerRefIssues( - containers ?? [], - tiles, - ); - if (tileRefIssues.length > 0) { - return res.status(400).json({ - message: tileRefIssues.join('; '), - }); - } - - // Hoist the source fetch so the source-derived validators - // (missing sources, source/connection mismatch, heatmap source - // kind) reuse one query result instead of each re-running - // `getSources(team)`. onClick helpers stay separate because - // they own their own dashboard lookup as well as a sources - // lookup; firing them inside the same Promise.all keeps the - // request latency unchanged. - const [ - sources, - missingConnections, - missingOnClickDashboards, - invalidOnClickSearchSources, - ] = await Promise.all([ - fetchSourcesForValidation(teamId), - getMissingConnections(teamId, tiles), - getMissingOnClickDashboards(teamId, tiles), - getInvalidOnClickSearchSources(teamId, tiles), - ]); - - const missingSources = getMissingSources(sources, tiles, filters); - const sourceConnectionMismatches = getSourceConnectionMismatches( - sources, + const validationError = await validateDashboardTiles({ + teamId: teamId.toString(), tiles, - ); - const heatmapNonTraceSources = getHeatmapTilesWithIncompatibleSources( - sources, - tiles, - ); - - if (missingSources.length > 0) { - return res.status(400).json({ - message: `Could not find the following source IDs: ${missingSources.join( - ', ', - )}`, - }); - } - if (missingConnections.length > 0) { - return res.status(400).json({ - message: `Could not find the following connection IDs: ${missingConnections.join( - ', ', - )}`, - }); - } - if (sourceConnectionMismatches.length > 0) { - return res.status(400).json({ - message: `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join( - ', ', - )}`, - }); - } - if (heatmapNonTraceSources.length > 0) { - return res.status(400).json({ - message: `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapNonTraceSources.join( - ', ', - )}`, - }); - } - if (missingOnClickDashboards.length > 0) { - return res.status(400).json({ - message: `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join( - ', ', - )}`, - }); - } - if (invalidOnClickSearchSources.length > 0) { - return res.status(400).json({ - message: `The following onClick search source IDs are not log or trace sources: ${invalidOnClickSearchSources.join( - ', ', - )}`, - }); + filters, + containers: containers ?? [], + }); + if (validationError) { + return res.status(400).json({ message: validationError }); } const internalTiles = convertExternalTilesToInternal(tiles); @@ -2212,122 +2103,35 @@ router.put( containers, } = req.body ?? {}; - // Hoist the source fetch and the existing-dashboard read so the - // source-derived validators reuse one query result and the - // heatmap source-kind check can scope itself to tiles whose - // sourceId/displayType actually changed in this request (a - // source whose `kind` was changed after the dashboard was - // originally accepted shouldn't wedge unrelated edits). onClick - // helpers stay separate because they own their own dashboard - // lookup as well as a sources lookup; firing them inside the - // same Promise.all keeps the request latency unchanged. - const [ - sources, - missingConnections, - missingOnClickDashboards, - invalidOnClickSearchSources, - existingDashboard, - ] = await Promise.all([ - fetchSourcesForValidation(teamId), - getMissingConnections(teamId, tiles), - getMissingOnClickDashboards(teamId, tiles), - getInvalidOnClickSearchSources(teamId, tiles), - Dashboard.findOne( - { _id: dashboardId, team: teamId }, - { tiles: 1, filters: 1, containers: 1 }, - ).lean(), - ]); - - const missingSources = getMissingSources(sources, tiles, filters); - const sourceConnectionMismatches = getSourceConnectionMismatches( - sources, - tiles, - ); - const heatmapNonTraceSources = getHeatmapTilesWithIncompatibleSources( - sources, - filterChangedHeatmapTiles(tiles, existingDashboard?.tiles ?? []), - ); + const existingDashboard = await Dashboard.findOne( + { _id: dashboardId, team: teamId }, + { tiles: 1, filters: 1, containers: 1 }, + ).lean(); - if (missingSources.length > 0) { - return res.status(400).json({ - message: `Could not find the following source IDs: ${missingSources.join( - ', ', - )}`, - }); - } - if (missingConnections.length > 0) { - return res.status(400).json({ - message: `Could not find the following connection IDs: ${missingConnections.join( - ', ', - )}`, - }); - } - if (sourceConnectionMismatches.length > 0) { - return res.status(400).json({ - message: `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join( - ', ', - )}`, - }); - } - if (heatmapNonTraceSources.length > 0) { - return res.status(400).json({ - message: `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapNonTraceSources.join( - ', ', - )}`, - }); - } - if (missingOnClickDashboards.length > 0) { - return res.status(400).json({ - message: `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join( - ', ', - )}`, - }); + if (existingDashboard == null) { + return res.sendStatus(404); } - if (invalidOnClickSearchSources.length > 0) { - return res.status(400).json({ - message: `The following onClick search source IDs are not log or trace sources: ${invalidOnClickSearchSources.join( - ', ', - )}`, - }); + + const effectiveContainers = + containers ?? existingDashboard.containers ?? []; + const validationError = await validateDashboardTiles({ + teamId: teamId.toString(), + tiles, + filters, + existingTiles: existingDashboard.tiles ?? [], + containers: effectiveContainers, + }); + if (validationError) { + return res.status(400).json({ message: validationError }); } - // PUT performs a read-modify-write on the dashboard doc with no - // optimistic-concurrency check (no `If-Match` / `updatedAt` - // guard). Concurrent PUTs may silently overwrite each other, - // which can leave orphan tile->container refs on layout-shape - // edits. The endpoint contract is at-most-one-writer; the - // OpenAPI description above documents this. Adding ETag-style - // concurrency would be a breaking change to the request shape - // and is tracked separately. `existingDashboard` is fetched in - // the hoisted Promise.all above and reused here for tile-ref - // resolution against the effective container set. const existingTileIds = new Set( - (existingDashboard?.tiles ?? []).map((t: { id: string }) => t.id), + (existingDashboard.tiles ?? []).map((t: { id: string }) => t.id), ); const existingFilterIds = new Set( - (existingDashboard?.filters ?? []).map((f: { id: string }) => f.id), + (existingDashboard.filters ?? []).map((f: { id: string }) => f.id), ); - // Tile-level container/tab ref resolution: container structure - // was checked by the body schema; tile-resolution runs here - // against the effective container set so an omitted `containers` - // body field (which preserves the existing array on write) is - // resolved against the doc's existing containers rather than an - // empty fallback. Without this, a PUT that updates only `tiles` - // and leaves a tile pointing at a real preserved container - // would be rejected with "Tile references unknown containerId". - const effectiveContainers = - containers ?? existingDashboard?.containers ?? []; - const tileRefIssues = collectTileContainerRefIssues( - effectiveContainers, - tiles, - ); - if (tileRefIssues.length > 0) { - return res.status(400).json({ - message: tileRefIssues.join('; '), - }); - } - const internalTiles = convertExternalTilesToInternal( tiles, existingTileIds, diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 25c59b35f2..844a92e194 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -1,9 +1,13 @@ -import { displayTypeSupportsRawSqlAlerts } from '@hyperdx/common-utils/dist/core/utils'; +import { + displayTypeSupportsBuilderAlerts, + displayTypeSupportsRawSqlAlerts, +} from '@hyperdx/common-utils/dist/core/utils'; import { validateDashboardContainersStructure, validateDashboardTileContainerRefs, } from '@hyperdx/common-utils/dist/dashboardValidation'; import { + isBuilderSavedChartConfig, isHeatmapCompatibleSource, isPromqlSavedChartConfig, isRawSqlSavedChartConfig, @@ -70,7 +74,7 @@ export type ConfigTile = ExternalDashboardTileWithId & { config: Exclude; }; -export function isRawSqlExternalTileConfig( +function isRawSqlExternalTileConfig( config: ExternalDashboardTileConfig, ): config is ExternalDashboardRawSqlTileConfig { return 'configType' in config && config.configType === 'sql'; @@ -767,15 +771,13 @@ export function convertToInternalTileConfig( * router can pass a single fetched array into multiple helpers without * pulling in `controllers/sources` for the type alone. */ -export type SourceForValidation = Awaited< - ReturnType ->[number]; +type SourceForValidation = Awaited>[number]; /** Fetches sources for a team. Re-exports the controller call so callers * outside `controllers/sources` don't need a second import for the * validation flow. The return type is the awaited shape of `getSources` * (an array of Source documents) so callers can `await` it directly. */ -export async function fetchSourcesForValidation( +async function fetchSourcesForValidation( team: string | mongoose.Types.ObjectId, ): Promise { return getSources(team.toString()); @@ -792,7 +794,7 @@ function getTileOnClick(tile: ExternalDashboardTileWithId) { } /** Returns source IDs referenced in tiles/filters that do not exist for the team */ -export function getMissingSources( +function getMissingSources( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], @@ -840,7 +842,7 @@ export function getMissingSources( * `packages/common-utils/src/guards.ts` and `ChartEditorControls.tsx`), so * UI and API gates move together. */ -export function getHeatmapTilesWithIncompatibleSources( +function getHeatmapTilesWithIncompatibleSources( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], ): string[] { @@ -874,7 +876,7 @@ export function getHeatmapTilesWithIncompatibleSources( * tiles, tiles whose displayType just changed to heatmap, and tiles * whose `sourceId` changed all flow through the check. */ -export function filterChangedHeatmapTiles( +function filterChangedHeatmapTiles( requestTiles: ExternalDashboardTileWithId[], existingTiles: DashboardDocument['tiles'], ): ExternalDashboardTileWithId[] { @@ -918,7 +920,7 @@ export function filterChangedHeatmapTiles( * Sources that don't exist are ignored here, getMissingSources handles that * case separately with a clearer error message. */ -export async function getInvalidOnClickSearchSources( +async function getInvalidOnClickSearchSources( team: string | mongoose.Types.ObjectId, tiles: ExternalDashboardTileWithId[], ): Promise { @@ -943,7 +945,7 @@ export async function getInvalidOnClickSearchSources( * Returns dashboard IDs referenced by tile OnClick link-outs (mode=id, * type=dashboard) that do not exist for the team. */ -export async function getMissingOnClickDashboards( +async function getMissingOnClickDashboards( team: string | mongoose.Types.ObjectId, tiles: ExternalDashboardTileWithId[], ): Promise { @@ -969,7 +971,7 @@ export async function getMissingOnClickDashboards( } /** Returns connection IDs referenced in tiles that do not belong to the team */ -export async function getMissingConnections( +async function getMissingConnections( team: string | mongoose.Types.ObjectId, tiles: ExternalDashboardTileWithId[], ): Promise { @@ -1074,8 +1076,122 @@ export function convertExternalFiltersToInternal( } /** - * Delete alerts for tiles that were removed or converted to raw SQL - * (which doesn't support alerts). + * Returns source IDs on raw SQL tiles whose connection doesn't match + * the source's persisted connection. Catches copy-paste errors where + * the LLM mixes up sourceId and connectionId from different sources. + */ +function getSourceConnectionMismatches( + sources: SourceForValidation[], + tiles: ExternalDashboardTileWithId[], +): string[] { + const sourceById = new Map(sources.map(s => [s._id.toString(), s])); + + const mismatched: string[] = []; + for (const tile of tiles) { + if ( + isConfigTile(tile) && + isRawSqlExternalTileConfig(tile.config) && + tile.config.sourceId + ) { + const source = sourceById.get(tile.config.sourceId); + if (source && source.connection.toString() !== tile.config.connectionId) { + mismatched.push(tile.config.sourceId); + } + } + } + + return mismatched; +} + +// ── Shared tile validation ─────────────────────────────────────────────── + +export type TileValidationContext = { + teamId: string; + tiles: ExternalDashboardTileWithId[]; + /** Filters to check for missing source IDs (create/full-update paths). */ + filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[]; + /** Existing internal tiles for scoping heatmap change detection (update paths). */ + existingTiles?: DashboardDocument['tiles']; + /** Container set to validate tile containerId/tabId refs against. */ + containers: DashboardContainer[]; +}; + +/** + * Run the full suite of tile validation checks (sources, connections, + * heatmap source-kind, onClick targets, container/tab refs). Returns + * `null` when all checks pass, or an error message string on failure. + * + * Consolidates the ~95-line validation block that was previously + * duplicated across REST v2 POST/PUT, MCP save (create/update), and + * MCP patch handlers. + */ +export async function validateDashboardTiles( + ctx: TileValidationContext, +): Promise { + const { teamId, tiles, filters, existingTiles, containers } = ctx; + + // Container/tab ref resolution. + const tileRefIssues = collectTileContainerRefIssues(containers, tiles); + if (tileRefIssues.length > 0) { + return tileRefIssues.join('; '); + } + + // Fetch sources/connections/onClick targets in parallel. + const [ + sources, + missingConnections, + missingOnClickDashboards, + invalidOnClickSearchSources, + ] = await Promise.all([ + fetchSourcesForValidation(teamId), + getMissingConnections(teamId, tiles), + getMissingOnClickDashboards(teamId, tiles), + getInvalidOnClickSearchSources(teamId, tiles), + ]); + + const missingSources = getMissingSources(sources, tiles, filters); + if (missingSources.length > 0) { + return `Could not find the following source IDs: ${missingSources.join(', ')}`; + } + if (missingConnections.length > 0) { + return `Could not find the following connection IDs: ${missingConnections.join(', ')}`; + } + + const sourceConnectionMismatches = getSourceConnectionMismatches( + sources, + tiles, + ); + if (sourceConnectionMismatches.length > 0) { + return `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(', ')}`; + } + + // Heatmap source-kind gate. On create (no existingTiles), validate all + // tiles. On update, scope to tiles whose sourceId/displayType changed. + const heatmapTilesToCheck = existingTiles + ? filterChangedHeatmapTiles(tiles, existingTiles) + : tiles; + const heatmapNonTraceSources = getHeatmapTilesWithIncompatibleSources( + sources, + heatmapTilesToCheck, + ); + if (heatmapNonTraceSources.length > 0) { + return `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapNonTraceSources.join(', ')}`; + } + + if (missingOnClickDashboards.length > 0) { + return `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`; + } + if (invalidOnClickSearchSources.length > 0) { + return `The following onClick search source IDs are not log or trace sources: ${invalidOnClickSearchSources.join(', ')}`; + } + + return null; +} + +/** + * Delete alerts for tiles that were removed or whose config no longer + * supports alerts (raw SQL with incompatible displayType, or builder + * tiles with incompatible displayType like Pie/Table/Heatmap/etc.). */ export async function cleanupDashboardAlerts({ dashboardId, @@ -1090,13 +1206,19 @@ export async function cleanupDashboardAlerts({ }) { const newTileIdSet = new Set(internalTiles.map(t => t.id)); const tileIdsToDeleteAlerts = [ + // Tiles whose config no longer supports alerts (raw SQL or builder). ...internalTiles - .filter( - tile => - isRawSqlSavedChartConfig(tile.config) && - !displayTypeSupportsRawSqlAlerts(tile.config.displayType), - ) + .filter(tile => { + if (isRawSqlSavedChartConfig(tile.config)) { + return !displayTypeSupportsRawSqlAlerts(tile.config.displayType); + } + if (isBuilderSavedChartConfig(tile.config)) { + return !displayTypeSupportsBuilderAlerts(tile.config.displayType); + } + return false; + }) .map(tile => tile.id), + // Tiles that were completely removed. ...[...existingTileIds].filter(id => !newTileIdSet.has(id)), ]; if (tileIdsToDeleteAlerts.length > 0) { From 0e68d6f1e48240f99d31b45998a812b7d36c6e8f Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 2 Jun 2026 14:14:10 -0400 Subject: [PATCH 012/728] fix: apply Zod defaults in External API dashboard routes (#2403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The External API dashboard endpoints (`POST /api/v2/dashboards` and `PUT /api/v2/dashboards/:id`) used `validateRequestWithEnhancedErrors`, which validates request data but does **not** mutate `req.body` with the parsed result. This meant Zod defaults (like `where: ''`) were computed during validation but discarded, causing `undefined` values to be stored in MongoDB. This caused a concrete bug: creating a heatmap tile without an explicit `where` field would store `undefined` in MongoDB, and subsequent UI-based dashboard updates would fail because the UI-facing API rejects `undefined` where fields. **Fix:** Switch the dashboard routes from `validateRequestWithEnhancedErrors` to `processRequestWithEnhancedErrors` — a sibling middleware already used by other External API routes (e.g. `/api/v2/search`, `/api/v2/alerts`) that assigns the parsed data (including Zod defaults and transforms) back to `req.body`/`req.params`/`req.query`. The one-line change in the import alias ensures all Zod schema defaults are applied before the route handler processes the request. ### How to test on Vercel preview N/A — non-UI change ### References - Linear Issue: HDX-4432 Linear Issue: [HDX-4432](https://linear.app/clickhouse/issue/HDX-4432/bug-external-api-allows-heatmap-tile-with-undefined-where)
Open in Web Open in Cursor 
Co-authored-by: Cursor Agent <199161495+cursoragent@users.noreply.github.com> --- .../external-api/__tests__/dashboards.test.ts | 60 +++++++++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts index 4d8cbb7d44..3f9c075b23 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts @@ -2573,6 +2573,66 @@ describe('External API v2 Dashboards - new format', () => { ); }); + it('persists where as empty string when omitted from heatmap tile', async () => { + const heatmapNoWhere = { + name: 'Heatmap Without Where', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'heatmap', + sourceId: traceSource._id.toString(), + select: [{ valueExpression: 'Duration' }], + }, + }; + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard Heatmap No Where', + tiles: [heatmapNoWhere], + tags: [], + }) + .expect(200); + + const dashboardInDb = await Dashboard.findById( + response.body.data.id, + ).lean(); + + expect((dashboardInDb!.tiles[0].config as any).where).toBe(''); + }); + + it('persists select-item where as empty string when omitted', async () => { + const lineChartNoWhere = { + name: 'Line Chart Without Where', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + }, + }; + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard Line No Where', + tiles: [lineChartNoWhere], + tags: [], + }) + .expect(200); + + const dashboardInDb = await Dashboard.findById( + response.body.data.id, + ).lean(); + + expect( + (dashboardInDb!.tiles[0].config as any).select[0].aggCondition, + ).toBe(''); + }); + it('does not silently downgrade a corrupted heatmap to line on GET', async () => { // Seed a Dashboard directly via Mongo with a heatmap tile whose // select[0] lacks a non-empty valueExpression. The current API diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 5b83bcc41d..9dd32fbead 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -4,7 +4,7 @@ import { z } from 'zod'; import { deleteDashboard } from '@/controllers/dashboard'; import Dashboard, { IDashboard } from '@/models/dashboard'; -import { validateRequestWithEnhancedErrors as validateRequest } from '@/utils/enhancedErrors'; +import { processRequestWithEnhancedErrors as validateRequest } from '@/utils/enhancedErrors'; import { ExternalDashboardTileWithId, objectIdSchema } from '@/utils/zod'; import { From 9119de5fbee8e371020b9a53079be5d195b04dcf Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 2 Jun 2026 12:23:37 -0600 Subject: [PATCH 013/728] [HDX-4403] Add unique MongoDB index on accessKey field in User model (#2397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Add a unique MongoDB index on the `accessKey` field in the User schema (`packages/api/src/models/user.ts`) to eliminate full collection scans during API key authentication. The `User.findOne({ accessKey })` query is called by the `validateUserAccessKey` middleware on **every** External API v2 request and MCP request. Without an index, this results in a full collection scan on the users collection. Adding `{ accessKey: 1 }` with `unique: true` follows the same pattern as the existing `email` index. ### Deployment note The original backfill migration (`20231130053610`) calls `uuidv4()` once and `$set`s the same value on every user, so any install that ran it with more than one user will have duplicate `accessKey` values. The `unique: true` index will fail with `E11000` on startup in that case. If you hit an `E11000 duplicate key error` on the `accessKey_1` index after deploying, run the following script in `mongosh` to deduplicate and build the index: ```js // 1. Find all duplicate accessKey groups const dupes = db.users.aggregate([ { $match: { accessKey: { $exists: true, $ne: null } } }, { $group: { _id: "$accessKey", ids: { $push: "$_id" }, count: { $sum: 1 } } }, { $match: { count: { $gt: 1 } } } ]).toArray(); print(`Found ${dupes.length} duplicate accessKey group(s)`); // 2. For each group, keep the first user's key and regenerate the rest dupes.forEach(group => { const [keep, ...rest] = group.ids; print(` accessKey "${group._id.substring(0, 8)}..." shared by ${group.count} users — keeping ${keep}, regenerating ${rest.length}`); rest.forEach(id => { db.users.updateOne({ _id: id }, { $set: { accessKey: UUID().toString() } }); }); }); // 3. Backfill any users missing an accessKey entirely const missing = db.users.find({ $or: [{ accessKey: { $exists: false } }, { accessKey: null }] }).toArray(); if (missing.length > 0) { print(`Found ${missing.length} user(s) with missing accessKey, assigning new keys...`); missing.forEach(u => { db.users.updateOne({ _id: u._id }, { $set: { accessKey: UUID().toString() } }); }); } // 4. Build the unique index (Mongoose will also do this on next startup) db.users.createIndex({ accessKey: 1 }, { unique: true }); print("Done — unique index created."); ``` ### How to test on Vercel preview N/A — non-UI change ### References - Linear Issue: https://linear.app/clickhouse/issue/HDX-4403/add-mongodb-index-on-accesskey-field-in-user-model --- .changeset/add-accesskey-index.md | 5 +++++ packages/api/src/models/user.ts | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/add-accesskey-index.md diff --git a/.changeset/add-accesskey-index.md b/.changeset/add-accesskey-index.md new file mode 100644 index 0000000000..4a9b44d73c --- /dev/null +++ b/.changeset/add-accesskey-index.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +Add unique MongoDB index on accessKey field in User model to eliminate full collection scans during API key authentication. This could cause startup failures if any existing users share duplicate accessKey values. diff --git a/packages/api/src/models/user.ts b/packages/api/src/models/user.ts index b7067b20c3..8f60fadeb5 100644 --- a/packages/api/src/models/user.ts +++ b/packages/api/src/models/user.ts @@ -47,5 +47,6 @@ UserSchema.plugin(passportLocalMongoose, { }); UserSchema.index({ email: 1 }, { unique: true }); +UserSchema.index({ accessKey: 1 }, { unique: true }); export default mongoose.model('User', UserSchema); From 418567ffed4af5ada2891f3eb734f3b8ca8ace3e Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:50:28 +0200 Subject: [PATCH 014/728] feat: trace panel inline split detail (#2402) --- .changeset/tough-ads-pay.md | 5 + packages/app/src/LogSidePanelElements.tsx | 83 +++--- packages/app/src/SessionSidePanel.tsx | 6 +- .../app/src/components/DBRowDataPanel.tsx | 28 ++ .../src/components/DBRowJsonViewer.test.tsx | 21 +- .../app/src/components/DBRowJsonViewer.tsx | 37 +-- .../app/src/components/DBRowSidePanel.tsx | 34 +-- .../src/components/DBRowSidePanelHeader.tsx | 34 ++- packages/app/src/components/DBTracePanel.tsx | 261 +++++++++++++----- .../src/components/DBTraceWaterfallChart.tsx | 82 ++++-- packages/app/src/components/DrawerUtils.tsx | 16 +- .../TimelineChart/TimelineChart.tsx | 1 - .../TimelineChart/TimelineXAxis.tsx | 21 +- .../TimelineChart/__tests__/utils.test.ts | 34 ++- .../app/src/components/TimelineChart/utils.ts | 33 ++- .../__tests__/DBTracePanel.test.tsx | 12 + .../app/styles/ResizablePanel.module.scss | 22 ++ 17 files changed, 494 insertions(+), 236 deletions(-) create mode 100644 .changeset/tough-ads-pay.md diff --git a/.changeset/tough-ads-pay.md b/.changeset/tough-ads-pay.md new file mode 100644 index 0000000000..792a3b8c3d --- /dev/null +++ b/.changeset/tough-ads-pay.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": minor +--- + +feat: trace panel inline split detail diff --git a/packages/app/src/LogSidePanelElements.tsx b/packages/app/src/LogSidePanelElements.tsx index 00586e1b75..4a9e08a51d 100644 --- a/packages/app/src/LogSidePanelElements.tsx +++ b/packages/app/src/LogSidePanelElements.tsx @@ -3,7 +3,16 @@ import * as React from 'react'; import Link from 'next/link'; import cx from 'classnames'; import { JSONTree } from 'react-json-tree'; -import { Alert, Button, CloseButton, Kbd, Text, Tooltip } from '@mantine/core'; +import { + Alert, + Button, + Group, + Kbd, + Modal, + Stack, + Text, + Tooltip, +} from '@mantine/core'; import { IconChevronDown, IconChevronRight, @@ -485,37 +494,49 @@ export const NetworkBody = ({ /** * Keyboard shortcuts */ -export const LogSidePanelKbdShortcuts = () => { - const [isDismissed, setDismissed] = useLocalStorage( - 'kbd-shortcuts-dismissed', - false, - ); - - const handleDismiss = React.useCallback(() => { - setDismissed(true); - }, [setDismissed]); - - if (isDismissed) { - return null; - } - +const SHORTCUTS = [ + { keys: ['esc'], label: 'Close panel' }, + { keys: ['←', '→'], label: 'Navigate between events' }, + { keys: ['↑', '↓'], label: 'Navigate between events' }, + { keys: ['k', 'j'], label: 'Navigate between events' }, + { keys: ['⌘/Ctrl', 'scroll'], label: 'Zoom trace timeline' }, +] as const; + +export const KeyboardShortcutsModal = ({ + opened, + onClose, +}: { + opened: boolean; + onClose: () => void; +}) => { return ( -
-
-
-
- Use ← - → arrow keys or k - j to move through events -
-
-
- ESC to close -
-
- -
-
+ + + {SHORTCUTS.map(({ keys, label }) => ( + + + {keys.map((key, i) => ( + + {i > 0 && ( + + + + + )} + {key} + + ))} + + {label} + + ))} + + ); }; diff --git a/packages/app/src/SessionSidePanel.tsx b/packages/app/src/SessionSidePanel.tsx index ddc26e3cca..2cee84a423 100644 --- a/packages/app/src/SessionSidePanel.tsx +++ b/packages/app/src/SessionSidePanel.tsx @@ -13,7 +13,7 @@ import { IconLink, IconX } from '@tabler/icons-react'; import { DrawerFullWidthToggle, - getInitialDrawerWidthPercent, + INITIAL_DRAWER_WIDTH_PERCENT, } from '@/components/DrawerUtils'; import useResizable from '@/hooks/useResizable'; import { @@ -54,11 +54,11 @@ export default function SessionSidePanel({ const [subDrawerOpen, setSubDrawerOpen] = useState(false); const { size, setSize, startResize } = useResizable( - getInitialDrawerWidthPercent(), + INITIAL_DRAWER_WIDTH_PERCENT, ); const isFullWidth = size >= 99; const toggleFullWidth = useCallback(() => { - setSize(isFullWidth ? getInitialDrawerWidthPercent() : 100); + setSize(isFullWidth ? INITIAL_DRAWER_WIDTH_PERCENT : 100); }, [isFullWidth, setSize]); useHotkeys( diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index 2a38f2d105..ef2bd8ed1c 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -190,6 +190,34 @@ export function useRowData({ }; } +// Detects whether a normalized row carries Kubernetes resource attributes, used +// to conditionally surface the Infrastructure tab/panel. Requires the source to +// expose resource attributes; returns false (rather than throwing) on any gap. +export function rowHasK8sContext( + source: TSource | null | undefined, + normalizedRow: Record | null | undefined, +): boolean { + try { + if ( + source == null || + !('resourceAttributesExpression' in source) || + !source.resourceAttributesExpression || + !normalizedRow + ) { + return false; + } + + const resourceAttrs = normalizedRow[ROW_DATA_ALIASES.RESOURCE_ATTRIBUTES]; + return ( + resourceAttrs?.['k8s.pod.uid'] != null || + resourceAttrs?.['k8s.node.name'] != null + ); + } catch (e) { + console.error(e); + return false; + } +} + export function getJSONColumnNames(meta: ResponseJSON['meta'] | undefined) { return ( meta diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index 161a5abfe6..f0a1165d6b 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -61,11 +61,28 @@ describe('DBRowJsonViewer', () => { ); }; + // Line action buttons are now icon-only; locate them by their `title` + // tooltip. Maps the friendly action name to a unique title substring. + const ACTION_TITLE: Record = { + Search: 'search for this value only', + 'Add to Filters': 'add to filters', + Column: 'column to results table', + 'Copy Object': 'copy object', + 'Copy Value': 'copy value', + }; + + const findActionButton = (line: HTMLElement, buttonText: string) => { + const needle = (ACTION_TITLE[buttonText] ?? buttonText).toLowerCase(); + return within(line).getByTitle((content: string) => + (content ?? '').toLowerCase().includes(needle), + ); + }; + // Helper to click a button on a line const clickLineButton = (fieldText: string, buttonText: string) => { const line = screen.getByText(fieldText).closest('.line')! as HTMLElement; fireEvent.mouseEnter(line); - const button = within(line).getByText(buttonText); + const button = findActionButton(line, buttonText); fireEvent.click(button); }; @@ -84,7 +101,7 @@ describe('DBRowJsonViewer', () => { .getByText(childField) .closest('.line')! as HTMLElement; fireEvent.mouseEnter(childLine); - const button = within(childLine).getByText(buttonText); + const button = findActionButton(childLine, buttonText); fireEvent.click(button); }; diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index 65abd56115..fd9b4c61b0 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -384,12 +384,7 @@ export function DBRowJsonViewer({ ) { actions.push({ key: 'add-to-search', - label: ( - - - Add to Filters - - ), + label: , title: 'Add to Filters', onClick: () => { let filterFieldPath = fieldPath; @@ -428,12 +423,7 @@ export function DBRowJsonViewer({ if (generateSearchUrl && typeof value !== 'object') { actions.push({ key: 'search', - label: ( - - - Search - - ), + label: , title: 'Search for this value only', onClick: () => { let searchFieldPath = fieldPath; @@ -532,17 +522,7 @@ export function DBRowJsonViewer({ const isIncluded = displayedColumns?.includes(columnFieldPath); actions.push({ key: 'toggle-column', - label: isIncluded ? ( - - - Column - - ) : ( - - - Column - - ), + label: isIncluded ? : , title: isIncluded ? `Remove ${fieldPath} column from results table` : `Add ${fieldPath} column to results table`, @@ -589,18 +569,15 @@ export function DBRowJsonViewer({ if (typeof value === 'object') { actions.push({ key: 'copy-object', - label: 'Copy Object', + label: , + title: 'Copy object', onClick: handleCopyObject, }); } else { actions.push({ key: 'copy-value', - label: ( - - - Copy Value - - ), + label: , + title: 'Copy value', onClick: async () => { const copied = await copyTextToClipboard( typeof value === 'string' diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index 1bd6dcde7d..1b1929dc28 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -31,7 +31,6 @@ import DBRowSidePanelHeader, { import useResizable from '@/hooks/useResizable'; import { WithClause } from '@/hooks/useRowWhere'; import useWaterfallSearchState from '@/hooks/useWaterfallSearchState'; -import { LogSidePanelKbdShortcuts } from '@/LogSidePanelElements'; import { getEventBody } from '@/source'; import TabBar from '@/TabBar'; import { SearchConfig } from '@/types'; @@ -41,11 +40,11 @@ import { useZIndex, ZIndexContext } from '@/zIndex'; import ServiceMapSidePanel from './ServiceMap/ServiceMapSidePanel'; import ContextSubpanel from './ContextSidePanel'; import DBInfraPanel from './DBInfraPanel'; -import { RowDataPanel, useRowData } from './DBRowDataPanel'; +import { RowDataPanel, rowHasK8sContext, useRowData } from './DBRowDataPanel'; import { RowOverviewPanel } from './DBRowOverviewPanel'; import { DBSessionPanel, useSessionId } from './DBSessionPanel'; import DBTracePanel from './DBTracePanel'; -import { getInitialDrawerWidthPercent } from './DrawerUtils'; +import { INITIAL_DRAWER_WIDTH_PERCENT } from './DrawerUtils'; import styles from '@/../styles/LogSidePanel.module.scss'; @@ -289,26 +288,10 @@ const DBRowSidePanel = ({ enabled: rowId != null, }); - const hasK8sContext = useMemo(() => { - try { - if ( - !('resourceAttributesExpression' in source) || - !source?.resourceAttributesExpression || - !normalizedRow - ) { - return false; - } - - const resourceAttrs = normalizedRow['__hdx_resource_attributes']; - return ( - resourceAttrs?.['k8s.pod.uid'] != null || - resourceAttrs?.['k8s.node.name'] != null - ); - } catch (e) { - console.error(e); - return false; - } - }, [source, normalizedRow]); + const hasK8sContext = useMemo( + () => rowHasK8sContext(source, normalizedRow), + [source, normalizedRow], + ); const initialRowHighlightHint = useMemo(() => { if (normalizedRow) { @@ -554,7 +537,6 @@ const DBRowSidePanel = ({ )} - ); }; @@ -572,12 +554,12 @@ export default function DBRowSidePanelErrorBoundary({ const drawerZIndex = contextZIndex + 10; const { size, setSize, startResize } = useResizable( - getInitialDrawerWidthPercent(), + INITIAL_DRAWER_WIDTH_PERCENT, ); const isFullWidth = size >= 99; const toggleFullWidth = useCallback(() => { - setSize(isFullWidth ? getInitialDrawerWidthPercent() : 100); + setSize(isFullWidth ? INITIAL_DRAWER_WIDTH_PERCENT : 100); }, [isFullWidth, setSize]); // Keep track of sub-drawers so we can disable closing this root drawer diff --git a/packages/app/src/components/DBRowSidePanelHeader.tsx b/packages/app/src/components/DBRowSidePanelHeader.tsx index b017905909..b37a1f383a 100644 --- a/packages/app/src/components/DBRowSidePanelHeader.tsx +++ b/packages/app/src/components/DBRowSidePanelHeader.tsx @@ -6,10 +6,12 @@ import React, { useState, } from 'react'; import { + ActionIcon, Box, Breadcrumbs, Button, Flex, + Group, Paper, Text, Tooltip, @@ -18,8 +20,10 @@ import { import { IconArrowsDiagonal, IconArrowsDiagonalMinimize2, + IconKeyboard, } from '@tabler/icons-react'; +import { KeyboardShortcutsModal } from '@/LogSidePanelElements'; import { FormatTime } from '@/useFormatTime'; import { useUserPreferences } from '@/useUserPreferences'; import { formatDistanceToNowStrictShort } from '@/utils'; @@ -157,6 +161,7 @@ export default function DBRowSidePanelHeader({ onToggleFullWidth?: () => void; }) { const [bodyExpanded, setBodyExpanded] = React.useState(false); + const [shortcutsOpen, setShortcutsOpen] = useState(false); const { generateSearchUrl } = useContext(RowSidePanelContext); const isContentTruncated = mainContent.length > MAX_MAIN_CONTENT_LENGTH; @@ -239,13 +244,30 @@ export default function DBRowSidePanelHeader({ )} - {onToggleFullWidth && ( - - )} + + + setShortcutsOpen(true)} + aria-label="Keyboard shortcuts" + > + + + + {onToggleFullWidth && ( + + )} + + setShortcutsOpen(false)} + /> {!bodyConfigured ? null : mainContent ? ( (); +}; + +const eventRowWhereParser = parseAsJsonEncoded(); -enum Tab { +enum SpanDetailTab { Overview = 'overview', Parsed = 'parsed', + Infrastructure = 'infrastructure', +} + +// Renders the inline detail for the currently-selected span. Mounted only while +// a span is selected, so it can call useRowData with a real source rather than +// the parent passing a placeholder when nothing is selected. Owns the active +// tab, which resets to Overview when the panel is closed and reopened. +function SpanDetailPanel({ + source, + rowId, + aliasWith, + onClose, +}: { + source: TSource; + rowId: string; + aliasWith?: WithClause[]; + onClose: () => void; +}) { + const [displayedTab, setDisplayedTab] = useState( + SpanDetailTab.Overview, + ); + + const { data: rowData } = useRowData({ source, rowId, aliasWith }); + const normalizedRow = rowData?.data?.[0]; + + const hasK8sContext = useMemo( + () => rowHasK8sContext(source, normalizedRow), + [source, normalizedRow], + ); + + // If the selected span loses k8s context (e.g. switching spans) while the + // Infrastructure tab is active, fall back to Overview so we don't show a + // blank panel. Derived rather than synced via an effect. + const effectiveTab = + displayedTab === SpanDetailTab.Infrastructure && !hasK8sContext + ? SpanDetailTab.Overview + : displayedTab; + + return ( + <> +
+ setDisplayedTab(v)} + /> + + + + + +
+ {effectiveTab === SpanDetailTab.Overview && ( + + )} + {effectiveTab === SpanDetailTab.Parsed && ( + + )} + {effectiveTab === SpanDetailTab.Infrastructure && hasK8sContext && ( + + + + )} + + ); } export default function DBTracePanel({ @@ -146,9 +244,33 @@ export default function DBTracePanel({ const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = useState(false); - const [displayedTab, setDisplayedTab] = useState(Tab.Overview); + // Parent owns the horizontal split sizing; the waterfall lives on the left + // and the selected span's detail renders inline on the right. + const { size: rightPanelSize, startResize: startHorizontalResize } = + useResizable(35, 'right'); + + const handleCloseSpanDetails = useCallback(() => { + setEventRowWhere(null); + }, [setEventRowWhere]); + + const selectedSpanSource = useMemo(() => { + if (!eventRowWhere) return null; + if (eventRowWhere.type === SourceKind.Log && logSourceData) { + return logSourceData; + } + return traceSourceData; + }, [eventRowWhere, logSourceData, traceSourceData]); + return ( -
+
@@ -238,70 +360,67 @@ export default function DBTracePanel({ )} - {traceSourceData?.kind === SourceKind.Trace && traceId && ( - - )} - {traceSourceData != null && eventRowWhere != null && ( - <> - - Event Details - - setDisplayedTab(v)} - /> - {displayedTab === Tab.Overview && ( - +
+ {traceSourceData?.kind === SourceKind.Trace && traceId && ( + )} - {displayedTab === Tab.Parsed && ( - +
+ + {eventRowWhere != null && ( + + )} + + {traceSourceData != null && + eventRowWhere != null && + selectedSpanSource != null && ( +
+ +
)} - - )} - {traceSourceData != null && !eventRowWhere && traceId && ( - -
- Please select a span above to view details. -
- - )} +
); } diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index f5383204e7..be1c75a2fa 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -1,4 +1,11 @@ -import { ReactNode, useCallback, useEffect, useMemo, useState } from 'react'; +import { + ReactNode, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import _, { omit } from 'lodash'; import { useForm } from 'react-hook-form'; import SqlString from 'sqlstring'; @@ -19,13 +26,12 @@ import { Center, Chip, Code, - Divider, Group, Kbd, Text, Tooltip, } from '@mantine/core'; -import { useDisclosure } from '@mantine/hooks'; +import { useDisclosure, useElementSize } from '@mantine/hooks'; import { IconChevronDown, IconChevronRight, @@ -36,7 +42,6 @@ import { ContactSupportText } from '@/components/ContactSupportText'; import SearchInputV2 from '@/components/SearchInput/SearchInputV2'; import { TimelineChart } from '@/components/TimelineChart'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; -import useResizable from '@/hooks/useResizable'; import useRowWhere, { WithClause } from '@/hooks/useRowWhere'; import useWaterfallSearchState from '@/hooks/useWaterfallSearchState'; import { @@ -63,7 +68,6 @@ import { import { DBHighlightedAttributesList } from './DBHighlightedAttributesList'; import styles from '@/../styles/LogSidePanel.module.scss'; -import resizeStyles from '@/../styles/ResizablePanel.module.scss'; export type SpanRow = { Body: string; @@ -451,7 +455,6 @@ export function DBTraceWaterfallChartContainer({ }; emptyState?: ReactNode; }) { - const { size, startResize } = useResizable(30, 'bottom'); const formatTime = useFormatTime(); const { @@ -569,23 +572,46 @@ export function DBTraceWaterfallChartContainer({ logRowsMeta, ]); + // Auto-select the originating span once when the panel opens — but only if + // nothing is already selected. Two cases must NOT trigger a (re-)select: + // - the user explicitly cleared the selection (closing the span detail), and + // - a selection was restored from the URL on load (deep link / reload). + // We mark the hint applied as soon as a selection exists or we apply it + // ourselves, so it never fires twice for the same hint; it re-fires only when + // the hint genuinely changes (different originating span / trace). + const appliedHighlightHintRef = useRef(null); useEffect(() => { - if (initialRowHighlightHint && onClick && highlightedRowWhere == null) { - const initialRowHighlightIndex = rows.findIndex(row => { - return ( - row.Timestamp === initialRowHighlightHint.timestamp && - row.SpanId === initialRowHighlightHint.spanId && - row.Body === initialRowHighlightHint.body - ); - }); + if (!initialRowHighlightHint || !onClick) { + return; + } - if (initialRowHighlightIndex !== -1) { - onClick?.({ - id: rows[initialRowHighlightIndex].id, - type: rows[initialRowHighlightIndex].type ?? '', - aliasWith: rows[initialRowHighlightIndex].aliasWith, - }); - } + const hintKey = `${initialRowHighlightHint.timestamp}|${initialRowHighlightHint.spanId}|${initialRowHighlightHint.body}`; + if (appliedHighlightHintRef.current === hintKey) { + return; + } + + // A selection already exists (restored from the URL, or user-chosen). Honor + // it and record the hint so we never override it — now or later. + if (highlightedRowWhere != null) { + appliedHighlightHintRef.current = hintKey; + return; + } + + const initialRowHighlightIndex = rows.findIndex(row => { + return ( + row.Timestamp === initialRowHighlightHint.timestamp && + row.SpanId === initialRowHighlightHint.spanId && + row.Body === initialRowHighlightHint.body + ); + }); + + if (initialRowHighlightIndex !== -1) { + appliedHighlightHintRef.current = hintKey; + onClick({ + id: rows[initialRowHighlightIndex].id, + type: rows[initialRowHighlightIndex].type ?? '', + aliasWith: rows[initialRowHighlightIndex].aliasWith, + }); } }, [initialRowHighlightHint, rows, onClick, highlightedRowWhere]); @@ -975,7 +1001,8 @@ export function DBTraceWaterfallChartContainer({ return v.id === highlightedRowWhere; }); - const heightPx = (size / 100) * window.innerHeight; + const { ref: timelineWrapperRef, height: timelineWrapperHeight } = + useElementSize(); return ( <> @@ -1103,10 +1130,11 @@ export function DBTraceWaterfallChartContainer({ )}
{isFetching ? ( @@ -1139,7 +1167,7 @@ export function DBTraceWaterfallChartContainer({ ) ) : ( { @@ -1154,12 +1182,6 @@ export function DBTraceWaterfallChartContainer({ /> )}
- ); } diff --git a/packages/app/src/components/DrawerUtils.tsx b/packages/app/src/components/DrawerUtils.tsx index 984dc027b1..0d898e6268 100644 --- a/packages/app/src/components/DrawerUtils.tsx +++ b/packages/app/src/components/DrawerUtils.tsx @@ -13,21 +13,7 @@ import { IconLayoutSidebarRightExpand, } from '@tabler/icons-react'; -const LARGE_SCREEN_BREAKPOINT_PX = 1440; -const LARGE_SCREEN_DRAWER_WIDTH_PERCENT = 60; -const SMALL_SCREEN_DRAWER_WIDTH_PERCENT = 80; - -export function getInitialDrawerWidthPercent(): number { - if (typeof window === 'undefined') { - return SMALL_SCREEN_DRAWER_WIDTH_PERCENT; - } - - if (window.innerWidth > LARGE_SCREEN_BREAKPOINT_PX) { - return LARGE_SCREEN_DRAWER_WIDTH_PERCENT; - } - - return SMALL_SCREEN_DRAWER_WIDTH_PERCENT; -} +export const INITIAL_DRAWER_WIDTH_PERCENT = 80; export const DrawerFullWidthToggle = React.memo<{ isFullWidth?: boolean; diff --git a/packages/app/src/components/TimelineChart/TimelineChart.tsx b/packages/app/src/components/TimelineChart/TimelineChart.tsx index f5fe68c0ec..8e35c0796a 100644 --- a/packages/app/src/components/TimelineChart/TimelineChart.tsx +++ b/packages/app/src/components/TimelineChart/TimelineChart.tsx @@ -274,7 +274,6 @@ export const TimelineChart = memo(function (props: TimelineChartProps) { maxVal={maxVal} heightRef={timelineHeightRef} labelWidth={labelWidth} - scaleRef={scaleRef} />
void; }; +// Minimum horizontal space reserved per tick label. The widest label we render +// (e.g. "1.234s") is ~45px; the extra headroom keeps adjacent labels from +// touching. Tune up for a sparser axis, down for a denser one. +const MIN_TICK_PX = 56; + export function TimelineXAxis({ maxVal, labelWidth, heightRef, - scaleRef, ref, }: { maxVal: number; labelWidth: number; heightRef: RefObject; - scaleRef: RefObject; ref: Ref; }) { const ticksContainerRef = useRef(null); @@ -45,11 +48,17 @@ export function TimelineXAxis({ return; } - const scale = scaleRef.current ?? 1; const height = heightRef.current ?? 0; const max = maxValRef.current; - const scaledMaxVal = max / scale; - const interval = calculateInterval(scaledMaxVal); + + // Budget ticks by the pixels actually available. The ticks container spans + // the (zoom-scaled) events area, so dividing its width by a minimum label + // width gives how many labels fit without overlapping. Measuring here keeps + // the axis correct on both panel resize and zoom, since both re-invoke + // recompute after the layout/scale has changed. + const ticksWidthPx = container.getBoundingClientRect().width; + const maxTicks = Math.max(1, Math.floor(ticksWidthPx / MIN_TICK_PX)); + const interval = calculateInterval(max, maxTicks); const numTicks = Math.floor(max / interval); const percSpacing = (interval / max) * 100; @@ -78,7 +87,7 @@ export function TimelineXAxis({ tick.style.marginLeft = i === 0 ? '0' : `${percSpacing.toFixed(6)}%`; label.textContent = renderMs(i * interval); } - }, [maxValRef, heightRef, scaleRef]); + }, [maxValRef, heightRef]); useImperativeHandle(ref, () => ({ recompute }), [recompute]); diff --git a/packages/app/src/components/TimelineChart/__tests__/utils.test.ts b/packages/app/src/components/TimelineChart/__tests__/utils.test.ts index b51746d61d..ddba012174 100644 --- a/packages/app/src/components/TimelineChart/__tests__/utils.test.ts +++ b/packages/app/src/components/TimelineChart/__tests__/utils.test.ts @@ -52,8 +52,9 @@ describe('calculateInterval', () => { expect(calculateInterval(10)).toBe(1); }); - it('returns 2x magnitude for value 25', () => { - expect(calculateInterval(25)).toBe(2); + it('snaps up to 5x magnitude for value 25', () => { + // value/10 = 2.5 → smallest fitting bucket is 5 (snap up), giving 5 ticks + expect(calculateInterval(25)).toBe(5); }); it('returns 5x magnitude for value 50', () => { @@ -75,4 +76,33 @@ describe('calculateInterval', () => { it('returns magnitude 100 for value 1000', () => { expect(calculateInterval(1000)).toBe(100); }); + + it('never produces more than maxTicks steps', () => { + for (const [value, maxTicks] of [ + [1000, 10], + [1000, 5], + [1000, 2], + [1000, 1], + [37, 6], + [250, 3], + ] as const) { + const interval = calculateInterval(value, maxTicks); + expect(Math.floor(value / interval)).toBeLessThanOrEqual(maxTicks); + } + }); + + it('thins ticks as the budget shrinks', () => { + // A tighter tick budget must yield a coarser (>=) interval. + expect(calculateInterval(1000, 5)).toBeGreaterThanOrEqual( + calculateInterval(1000, 10), + ); + expect(calculateInterval(1000, 2)).toBeGreaterThanOrEqual( + calculateInterval(1000, 5), + ); + }); + + it('returns a safe fallback for non-positive ranges', () => { + expect(calculateInterval(0)).toBe(1); + expect(calculateInterval(-5)).toBe(1); + }); }); diff --git a/packages/app/src/components/TimelineChart/utils.ts b/packages/app/src/components/TimelineChart/utils.ts index dc1fe17570..998827e997 100644 --- a/packages/app/src/components/TimelineChart/utils.ts +++ b/packages/app/src/components/TimelineChart/utils.ts @@ -18,21 +18,28 @@ export function renderMs(ms: number) { return `${(ms / 1000).toFixed(3)}s`; } -export function calculateInterval(value: number) { - // Calculate the approximate interval by dividing the value by 10 - const interval = value / 10; +// Returns the smallest "human-friendly" interval (a 1/2/5 × 10ⁿ bucket) that +// divides `value` into at most `maxTicks` steps. Snapping up rather than down +// guarantees the resulting tick count never exceeds maxTicks, so callers can +// budget ticks by the available pixel width to avoid overlapping labels. +export function calculateInterval(value: number, maxTicks = 10) { + if (value <= 0 || maxTicks < 1) { + return value > 0 ? value : 1; + } - // Round the interval to the nearest power of 10 to make it a human-friendly number - const magnitude = Math.pow(10, Math.floor(Math.log10(interval))); + // The finest interval that would still fit within the tick budget; snap it up + // to the nearest human-friendly bucket. + const rough = value / maxTicks; + const magnitude = Math.pow(10, Math.floor(Math.log10(rough))); - // Adjust the interval to the nearest standard bucket size - let bucketSize = magnitude; - if (interval >= 2 * magnitude) { - bucketSize = 2 * magnitude; + if (rough <= magnitude) { + return magnitude; } - if (interval >= 5 * magnitude) { - bucketSize = 5 * magnitude; + if (rough <= 2 * magnitude) { + return 2 * magnitude; } - - return bucketSize; + if (rough <= 5 * magnitude) { + return 5 * magnitude; + } + return 10 * magnitude; } diff --git a/packages/app/src/components/__tests__/DBTracePanel.test.tsx b/packages/app/src/components/__tests__/DBTracePanel.test.tsx index ca95483eb4..9c67cbe944 100644 --- a/packages/app/src/components/__tests__/DBTracePanel.test.tsx +++ b/packages/app/src/components/__tests__/DBTracePanel.test.tsx @@ -36,6 +36,18 @@ jest.mock('../SourceSelect', () => ({ SourceSelectControlled: () =>
source select
, })); +// useRowData runs unconditionally (for the Infrastructure tab / k8s detection) +// and would otherwise need a QueryClient provider; stub it for this unit test. +jest.mock('../DBRowDataPanel', () => ({ + useRowData: () => ({ data: undefined }), + RowDataPanel: () =>
row data panel
, +})); + +jest.mock('../DBInfraPanel', () => ({ + __esModule: true, + default: () =>
infra panel
, +})); + jest.mock('../SourceSchemaPreview', () => ({ __esModule: true, default: () =>
, diff --git a/packages/app/styles/ResizablePanel.module.scss b/packages/app/styles/ResizablePanel.module.scss index fe44b09a1e..6f04532888 100644 --- a/packages/app/styles/ResizablePanel.module.scss +++ b/packages/app/styles/ResizablePanel.module.scss @@ -20,6 +20,28 @@ } } +.resizeHandleInline { + position: relative; + right: auto; + top: auto; + bottom: auto; + width: 6px; + flex-shrink: 0; + cursor: col-resize; + transition: all 150ms ease; + z-index: 10; + + &:hover, + &:active { + background-color: var(--color-bg-neutral); + } + + &:active { + background-color: var(--color-bg-neutral); + width: 8px; + } +} + .resizeYHandle { position: absolute; bottom: -3px; From 19bcaed480eac297f8528b7fddee94edd5b096ae Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 3 Jun 2026 08:33:53 -0400 Subject: [PATCH 015/728] test: Add E2E tests for materialized views (#2406) ## Summary This PR adds the following E2E tests covering materialized views 1. MV indicator shows Accelerated in the chart editor: count() over the MV-backed source shows the green "Accelerated" badge. 2. MV indicator + chart data on a saved dashboard tile: the saved tile renders a recharts chart (data populated from the MV) and shows the accelerated icon. Covers both the dashboard-tile indicator and "charts populated from MV data." 3. MV modal shows correct config: opens the modal from the indicator and verifies the active MV (e2e_otel_traces_1m), "1 minute" granularity, dimension columns (ServiceName/StatusCode), and the aggregated-columns table (count + avg/max/quantile of Duration). 4. Configuring an MV with auto-population: adds an MV in the source form, picks the MV table, and verifies inference fills granularity, dimension columns, timestamp column, and the 4 aggregated-column rows. ### Screenshots or video No behavior changes, just tests and test ids ### References - Linear Issue: Closes HDX-4436 - Related PRs: --- docker/clickhouse/local/init-db-e2e.sh | 35 ++- .../MVOptimizationIndicator.tsx | 16 +- .../MaterializedViews/MVOptimizationModal.tsx | 141 +++++++----- .../app/src/components/Sources/SourceForm.tsx | 27 ++- .../e2e/components/ChartEditorComponent.ts | 9 +- .../components/MaterializedViewComponent.ts | 90 ++++++++ .../e2e/components/SourceFormComponent.ts | 70 ++++++ .../dashboard-template-import.spec.ts | 15 +- .../e2e/features/materialized-views.spec.ts | 208 ++++++++++++++++++ .../e2e/features/traces-workflow.spec.ts | 7 +- .../app/tests/e2e/fixtures/e2e-fixtures.json | 73 ++++++ .../e2e/page-objects/ServicesDashboardPage.ts | 4 +- packages/app/tests/e2e/seed-clickhouse.ts | 7 + packages/app/tests/e2e/utils/constants.ts | 11 + 14 files changed, 633 insertions(+), 80 deletions(-) create mode 100644 packages/app/tests/e2e/components/MaterializedViewComponent.ts create mode 100644 packages/app/tests/e2e/components/SourceFormComponent.ts create mode 100644 packages/app/tests/e2e/features/materialized-views.spec.ts diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh index 0ac945bf9e..e6192e4875 100755 --- a/docker/clickhouse/local/init-db-e2e.sh +++ b/docker/clickhouse/local/init-db-e2e.sh @@ -209,5 +209,38 @@ ENGINE = MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) TTL toDate(TimeUnix) + toIntervalDay(30) -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1 +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- Materialized view rollup for traces, used by E2E tests covering MV +-- acceleration. The target table pre-aggregates Duration over 1-minute +-- buckets grouped by the dimension columns ServiceName and StatusCode. +-- The 'E2E Traces MV' fixture source references e2e_otel_traces_1m. +CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces_1m +( + \`Timestamp\` DateTime, + \`ServiceName\` LowCardinality(String), + \`StatusCode\` LowCardinality(String), + \`count\` SimpleAggregateFunction(sum, UInt64), + \`avg__Duration\` AggregateFunction(avg, UInt64), + \`max__Duration\` SimpleAggregateFunction(max, UInt64), + \`quantile__Duration\` AggregateFunction(quantile(0.95), UInt64) +) +ENGINE = AggregatingMergeTree +ORDER BY (Timestamp, ServiceName, StatusCode) +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.e2e_otel_traces_1m_mv TO ${DATABASE}.e2e_otel_traces_1m +AS SELECT + toStartOfMinute(Timestamp) AS Timestamp, + ServiceName, + StatusCode, + count() AS count, + avgState(Duration) AS avg__Duration, + maxSimpleState(Duration) AS max__Duration, + quantileState(0.95)(Duration) AS quantile__Duration +FROM ${DATABASE}.e2e_otel_traces +GROUP BY + Timestamp, + ServiceName, + StatusCode EOFSQL diff --git a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx index 95ce3ec505..47b6662388 100644 --- a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx +++ b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx @@ -23,13 +23,23 @@ function MVOptimizationIcon({ }) { return isInWarningState ? ( - + ) : ( - + @@ -48,6 +58,8 @@ function MVOptimizationBadge({ color={isInWarningState ? WARNING_COLOR : SUCCESS_COLOR} onClick={onClick} className="cursor-pointer" + data-testid="mv-optimization-indicator" + data-mv-accelerated={isInWarningState ? 'false' : 'true'} > {isInWarningState ? 'Not Accelerated' : 'Accelerated'} diff --git a/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx b/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx index e2b052ad06..b998ecad97 100644 --- a/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx +++ b/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx @@ -47,69 +47,90 @@ export default function MVOptimizationModal({ onClose={onClose} size="lg" > - - This source is configured with{' '} - {hasMultipleMVs ? 'materialized views' : 'a materialized view'} for - accelerating some aggregations. - +
+ + This source is configured with{' '} + {hasMultipleMVs ? 'materialized views' : 'a materialized view'} for + accelerating some aggregations. + - - {mvConfigs.map(config => { - const key = mvConfigToKey(config); - const explanation = explanationsByKey.get(key); - const hasErrors = !!explanation?.errors.length; - const isBeingUsedByOptimizedConfig = explanation?.success; - const rowEstimate = - explanation?.rowEstimate?.toLocaleString() ?? 'N/A'; + + {mvConfigs.map(config => { + const key = mvConfigToKey(config); + const explanation = explanationsByKey.get(key); + const hasErrors = !!explanation?.errors.length; + const isBeingUsedByOptimizedConfig = explanation?.success; + const rowEstimate = + explanation?.rowEstimate?.toLocaleString() ?? 'N/A'; - return ( - - - - {config.tableName} - {isBeingUsedByOptimizedConfig ? ( - - - Active - - - ) : hasErrors ? ( - - - Incompatible - - - ) : explanation ? ( - - - Skipped - - - ) : null} - - - - <> - - {hasErrors && ( - - - The query cannot be accelerated using this materialized - view for the following reason(s): - - {explanation.errors.map((error, idx) => ( - - {error} + return ( + + + + {config.tableName} + {isBeingUsedByOptimizedConfig ? ( + + + Active + + + ) : hasErrors ? ( + + + Incompatible + + + ) : explanation ? ( + + + Skipped + + + ) : null} + + + + <> + + {hasErrors && ( + + + The query cannot be accelerated using this + materialized view for the following reason(s): - ))} - - )} - - - - ); - })} - + {explanation.errors.map((error, idx) => ( + + {error} + + ))} + + )} + + + + ); + })} + +
); } diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index a73ed1ca40..6c4043afb1 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -550,6 +550,7 @@ function MaterializedViewsFormSection({ control, setValue }: TableModelProps) {
)} diff --git a/packages/app/src/components/DBHeatmapChart.tsx b/packages/app/src/components/DBHeatmapChart.tsx index 4deac54ea1..573c893089 100644 --- a/packages/app/src/components/DBHeatmapChart.tsx +++ b/packages/app/src/components/DBHeatmapChart.tsx @@ -3,26 +3,12 @@ import dynamic from 'next/dynamic'; import type { Plugin } from 'uplot'; import uPlot from 'uplot'; import UplotReact from 'uplot-react'; -import { - ClickHouseQueryError, - inferTimestampColumn, -} from '@hyperdx/common-utils/dist/clickhouse'; +import { inferTimestampColumn } from '@hyperdx/common-utils/dist/clickhouse'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { DisplayType } from '@hyperdx/common-utils/dist/types'; -import { - Box, - Button, - Code, - Divider, - Flex, - Group, - Modal, - Text, - useMantineColorScheme, -} from '@mantine/core'; -import { useDisclosure, useElementSize } from '@mantine/hooks'; -import { IconArrowsDiagonal } from '@tabler/icons-react'; +import { Divider, Flex, Text, useMantineColorScheme } from '@mantine/core'; +import { useElementSize } from '@mantine/hooks'; import { isAggregateFunction, timeBucketByGranularity } from '@/ChartUtils'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; @@ -31,7 +17,9 @@ import { FormatTime } from '@/useFormatTime'; import { formatDurationMsCompact, formatNumber } from '@/utils'; import ChartContainer from './charts/ChartContainer'; -import { SQLPreview } from './ChartSQLPreview'; +import ChartErrorState, { + ChartErrorStateVariant, +} from './charts/ChartErrorState'; type Mode2DataArray = [number[], number[], number[]]; @@ -619,6 +607,7 @@ function HeatmapContainer({ toolbarSuffix, scaleType = 'log', showLegend = false, + errorVariant, }: { config: HeatmapChartConfig; enabled?: boolean; @@ -637,6 +626,7 @@ function HeatmapContainer({ toolbarSuffix?: React.ReactNode[]; scaleType?: HeatmapScaleType; showLegend?: boolean; + errorVariant?: ChartErrorStateVariant; }) { const dateRange = config.dateRange; const granularity = convertDateRangeToGranularityString(dateRange, 245); @@ -662,8 +652,6 @@ function HeatmapContainer({ enabled: enabled, }); - const [errorModal, errorModalControls] = useDisclosure(); - // UInt64 are returned as strings; quantile returns floats const min = Number.parseFloat(minMaxData?.data?.[0]?.['min'] ?? '0'); const max = Number.parseFloat(minMaxData?.data?.[0]?.['max'] ?? '0'); @@ -799,49 +787,7 @@ function HeatmapContainer({ Loading... ) : _error ? ( - - - Error loading chart, please check your query or try again later. - - - errorModalControls.close()} - title="Error Details" - > - - - Error Message: - - - {_error.message} - - {_error instanceof ClickHouseQueryError && ( - <> - - Sent Query: - - - - )} - - - + ) : time.length < 2 || generatedTsBuckets?.length < 2 ? ( Not enough data points to render heatmap. Try expanding your search diff --git a/packages/app/src/components/DBHistogramChart.tsx b/packages/app/src/components/DBHistogramChart.tsx index ffd7a45769..c8fc48a19b 100644 --- a/packages/app/src/components/DBHistogramChart.tsx +++ b/packages/app/src/components/DBHistogramChart.tsx @@ -11,9 +11,7 @@ import { YAxis, } from 'recharts'; import { CategoricalChartState } from 'recharts/types/chart/types'; -import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; -import { Box, Code, Text } from '@mantine/core'; import { buildMVDateRangeIndicator } from '@/ChartUtils'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; @@ -21,8 +19,10 @@ import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanati import { useSource } from '@/source'; import ChartContainer from './charts/ChartContainer'; +import ChartErrorState, { + ChartErrorStateVariant, +} from './charts/ChartErrorState'; import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator'; -import { SQLPreview } from './ChartSQLPreview'; function HistogramChart({ graphResults, @@ -197,6 +197,7 @@ export default function DBHistogramChart({ toolbarPrefix, toolbarSuffix, showMVOptimizationIndicator = true, + errorVariant, }: { config: BuilderChartConfigWithDateRange; queryKeyPrefix?: string; @@ -205,6 +206,7 @@ export default function DBHistogramChart({ toolbarPrefix?: React.ReactNode[]; toolbarSuffix?: React.ReactNode[]; showMVOptimizationIndicator?: boolean; + errorVariant?: ChartErrorStateVariant; }) { const queriedConfig = omit(config, ['granularity']); const { data, isLoading, isError, error } = useQueriedChartConfig( @@ -272,32 +274,7 @@ export default function DBHistogramChart({ Loading Chart Data...
) : isError ? ( -
- - Error loading chart, please check your query or try again later. - - - - Error Message: - - - {error.message} - - {error instanceof ClickHouseQueryError && ( - <> - - Sent Query: - - - - )} - -
+ ) : data?.data.length === 0 ? (
No data found within time range. diff --git a/packages/app/src/components/DBListBarChart.tsx b/packages/app/src/components/DBListBarChart.tsx index a4d6eb6c8d..b2f7a9c077 100644 --- a/packages/app/src/components/DBListBarChart.tsx +++ b/packages/app/src/components/DBListBarChart.tsx @@ -1,10 +1,9 @@ import { useMemo } from 'react'; import Link from 'next/link'; import { omit } from 'lodash'; -import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import type { FloatingPosition } from '@mantine/core'; -import { Box, Code, Flex, HoverCard, Text } from '@mantine/core'; +import { Box, Flex, HoverCard, Text } from '@mantine/core'; import { buildMVDateRangeIndicator } from '@/ChartUtils'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; @@ -14,8 +13,10 @@ import type { NumberFormat } from '@/types'; import { formatNumber, semanticKeyedColor } from '@/utils'; import ChartContainer from './charts/ChartContainer'; +import ChartErrorState, { + ChartErrorStateVariant, +} from './charts/ChartErrorState'; import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator'; -import { SQLPreview } from './ChartSQLPreview'; function ListItem({ title, @@ -184,6 +185,7 @@ export default function DBListBarChart({ title, toolbarItems, showMVOptimizationIndicator = true, + errorVariant, }: { config: BuilderChartConfigWithDateRange; onSettled?: () => void; @@ -197,6 +199,7 @@ export default function DBListBarChart({ title?: React.ReactNode; toolbarItems?: React.ReactNode[]; showMVOptimizationIndicator?: boolean; + errorVariant?: ChartErrorStateVariant; }) { const queriedConfig = omit(config, ['granularity']); const { data, isLoading, isError, error } = useQueriedChartConfig( @@ -273,32 +276,7 @@ export default function DBListBarChart({ Loading Chart Data...
) : isError ? ( -
- - Error loading chart, please check your query or try again later. - - - - Error Message: - - - {error.message} - - {error instanceof ClickHouseQueryError && ( - <> - - Sent Query: - - - - )} - -
+ ) : data?.data.length === 0 ? (
No data found within time range. diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 9d2c8c31e6..04dd3d3974 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -38,7 +38,6 @@ import { } from '@hyperdx/common-utils/dist/types'; import { Box, - Code, Flex, Group, Modal, @@ -100,6 +99,9 @@ import { usePrevious, } from '@/utils'; +import ChartErrorState, { + ChartErrorStateVariant, +} from './charts/ChartErrorState'; import DBRowTableFieldWithPopover from './DBTable/DBRowTableFieldWithPopover'; import DBRowTableRowButtons from './DBTable/DBRowTableRowButtons'; import TableHeader from './DBTable/TableHeader'; @@ -342,6 +344,7 @@ export const RawLogTable = memo( dedupRows, isError, error, + errorVariant = 'inline', columnTypeMap, dateRange, loadingDate, @@ -378,6 +381,7 @@ export const RawLogTable = memo( isError?: boolean; error?: ClickHouseQueryError | Error; + errorVariant?: ChartErrorStateVariant; dateRange?: [Date, Date]; loadingDate?: Date; config?: BuilderChartConfigWithDateRange; @@ -1226,7 +1230,16 @@ export const RawLogTable = memo( })} -
+
{isLoading ? (
@@ -1265,40 +1278,8 @@ export const RawLogTable = memo( isLoading == false && dedupedRows.length > 0 ? (
End of Results
- ) : isError ? ( -
- - Error loading results, please check your query or - try again. - - - - - {error?.message} - - - {error instanceof ClickHouseQueryError && ( - <> - - Sent Query: - - - - - - )} - -
+ ) : isError && error ? ( + ) : hasNextPage == false && isLoading == false && dedupedRows.length === 0 ? ( @@ -1531,6 +1512,7 @@ function DBSqlRowTableComponent({ variant = 'default', enableSmallFirstWindow, tableId, + errorVariant, }: { config: BuilderChartConfigWithDateRange; sourceId?: string; @@ -1556,6 +1538,7 @@ function DBSqlRowTableComponent({ variant?: DBRowTableVariant; enableSmallFirstWindow?: boolean; tableId?: string; + errorVariant?: ChartErrorStateVariant; }) { const { data: me } = api.useMe(); const { toggleColumn, displayedColumns: contextDisplayedColumns } = @@ -1850,6 +1833,7 @@ function DBSqlRowTableComponent({ generateRowId={getRowWhere} isError={isError} error={error ?? undefined} + errorVariant={errorVariant} columnTypeMap={columnMap} dateRange={config.dateRange} loadingDate={loadingDate} diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index 083205df77..928a73eb49 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -13,6 +13,7 @@ import TabBar from '@/TabBar'; import { useLocalStorage } from '@/utils'; import { parseAsStringEncoded } from '@/utils/queryParsers'; +import { ChartErrorStateVariant } from './charts/ChartErrorState'; import { useNestedPanelState } from './ContextSidePanel'; import { RowDataPanel } from './DBRowDataPanel'; import { RowOverviewPanel } from './DBRowOverviewPanel'; @@ -44,6 +45,7 @@ interface Props { variant?: DBRowTableVariant; enableSmallFirstWindow?: boolean; tableId?: string; + errorVariant?: ChartErrorStateVariant; } export default function DBSqlRowTableWithSideBar({ @@ -65,6 +67,7 @@ export default function DBSqlRowTableWithSideBar({ variant, enableSmallFirstWindow, tableId, + errorVariant, }: Props) { const { data: sourceData } = useSource({ id: sourceId }); const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded); @@ -145,6 +148,7 @@ export default function DBSqlRowTableWithSideBar({ variant={variant} enableSmallFirstWindow={enableSmallFirstWindow} tableId={tableId} + errorVariant={errorVariant} /> ); diff --git a/packages/app/src/components/DBTableChart.tsx b/packages/app/src/components/DBTableChart.tsx index 8949feadf4..41bd3f0f5b 100644 --- a/packages/app/src/components/DBTableChart.tsx +++ b/packages/app/src/components/DBTableChart.tsx @@ -1,5 +1,4 @@ import { useCallback, useMemo, useState } from 'react'; -import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { isRatioChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { isBuilderChartConfig, @@ -7,7 +6,7 @@ import { isRawSqlChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; -import { Box, Code, Text } from '@mantine/core'; +import { Text } from '@mantine/core'; import { SortingState } from '@tanstack/react-table'; import { @@ -22,9 +21,11 @@ import { useChartNumberFormats, useSource } from '@/source'; import { useIntersectionObserver } from '@/utils'; import ChartContainer from './charts/ChartContainer'; +import ChartErrorState, { + ChartErrorStateVariant, +} from './charts/ChartErrorState'; import { getClientSideSortingFn } from './DBTable/sorting'; import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator'; -import { SQLPreview } from './ChartSQLPreview'; export default function DBTableChart({ config, @@ -39,6 +40,7 @@ export default function DBTableChart({ toolbarSuffix, showMVOptimizationIndicator = true, variant, + errorVariant, }: { config: ChartConfigWithOptTimestamp; getRowSearchLink?: (row: any) => string | null; @@ -52,6 +54,7 @@ export default function DBTableChart({ toolbarSuffix?: React.ReactNode[]; showMVOptimizationIndicator?: boolean; variant?: TableVariant; + errorVariant?: ChartErrorStateVariant; }) { const [sort, setSort] = useState([]); @@ -230,32 +233,7 @@ export default function DBTableChart({ Loading Chart Data...
) : isError && error ? ( -
- - Error loading chart, please check your query or try again later. - - - - Error Message: - - - {error.message} - - {error instanceof ClickHouseQueryError && ( - <> - - Sent Query: - - - - )} - -
+ ) : data?.data.length === 0 ? (
No data found within time range. diff --git a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx index 82be32f4a5..32e8823079 100644 --- a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx +++ b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx @@ -1,17 +1,16 @@ import { pick } from 'lodash'; -import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { type Filter, pickSampleWeightExpressionProps, type TTraceSource, } from '@hyperdx/common-utils/dist/types'; -import { Box, Code, Group, Text } from '@mantine/core'; +import { Group, Text } from '@mantine/core'; import { ChartBox } from '@/components/ChartBox'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { useServiceDashboardExpressions } from '@/serviceDashboard'; -import { SQLPreview } from './ChartSQLPreview'; +import ChartErrorState from './charts/ChartErrorState'; import DBSqlRowTableWithSideBar from './DBSqlRowTableWithSidebar'; export default function SlowestEventsTile({ @@ -77,32 +76,7 @@ export default function SlowestEventsTile({ Loading Chart Data...
) : isError ? ( -
- - Error loading chart, please check your query or try again later. - - - - Error Message: - - - {error.message} - - {error instanceof ClickHouseQueryError && ( - <> - - Sent Query: - - - - )} - -
+ ) : data?.data.length === 0 ? (
No data found within time range. From f95687b099e22c0f016f8ea31ae0c4f6b46d218f Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 3 Jun 2026 13:06:10 -0400 Subject: [PATCH 017/728] fix: prevent source setup dropdowns from being clipped by the modal (HDX-4445) (#2411) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What & why In the **Configure New Source** modal, clicking the **Database** field opened a dropdown that was clipped at the top edge of the modal — with several databases the list was cut off and the top entries were unreachable (HDX-4445). Root cause: the Database, Table, and Server Connection pickers set `comboboxProps={{ withinPortal: false }}`, so their dropdowns were positioned inside the modal's DOM and clipped by the modal's bounds. (`maxDropdownHeight` already gives the list an internal scrollbar — the problem was purely positional clipping by the modal container.) ## Change - Flip `withinPortal: false → true` on the three source-form pickers (`DatabaseSelect`, `DBTableSelect`, `ConnectionSelect`) so their dropdowns render in a portal and escape the modal's clip box. Matches the in-modal precedent in `DashboardFiltersModal`; `true` is Mantine v9's default. `maxDropdownHeight={280}` is kept for the internal scroll. - Add a component test (`sourceFormPickerDropdowns.test.tsx`): mount each picker with a **20-item** list and assert (a) all 20 options render (no truncation) and (b) they're portaled **out** of the picker's own container, so a modal can't clip them. (b) is the regression guard — it fails on `withinPortal: false` regardless of option count; (a) guards against truncation. Verified red/green. Note: clipping is a visual `overflow` effect that doesn't change the DOM or an element's box and isn't measurable in jsdom, so the portal DOM-location is the deterministic signal a component test can assert. (True on-screen visual proof would need a real-browser E2E test.) ## Testing - `make ci-lint` ✅ (lint + tsc, all packages) - `yarn ci:unit` ✅ — full app unit suite, incl. the new test - Verified red/green: on `withinPortal: false` all 20 options still render but are nested in the container (clippable) → test fails; on `true` they're portaled out → test passes. Closes HDX-4445. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .changeset/source-form-dropdown-clipping.md | 5 + .../app/src/components/ConnectionSelect.tsx | 2 +- packages/app/src/components/DBTableSelect.tsx | 2 +- .../app/src/components/DatabaseSelect.tsx | 2 +- .../sourceFormPickerDropdowns.test.tsx | 150 ++++++++++++++++++ 5 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 .changeset/source-form-dropdown-clipping.md create mode 100644 packages/app/src/components/__tests__/sourceFormPickerDropdowns.test.tsx diff --git a/.changeset/source-form-dropdown-clipping.md b/.changeset/source-form-dropdown-clipping.md new file mode 100644 index 0000000000..08f54d1c83 --- /dev/null +++ b/.changeset/source-form-dropdown-clipping.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +Fix the database, table, and connection dropdowns being clipped inside the source setup modal. The dropdowns now render in a portal, so the full list is visible and scrollable when configuring or editing a source. diff --git a/packages/app/src/components/ConnectionSelect.tsx b/packages/app/src/components/ConnectionSelect.tsx index 476f254394..2422377cfb 100644 --- a/packages/app/src/components/ConnectionSelect.tsx +++ b/packages/app/src/components/ConnectionSelect.tsx @@ -26,7 +26,7 @@ export function ConnectionSelectControlled({ allowDeselect={false} data={values} // disabled={isDatabasesLoading} - comboboxProps={{ withinPortal: false }} + comboboxProps={{ withinPortal: true }} searchable placeholder="Connection" leftSection={} diff --git a/packages/app/src/components/DBTableSelect.tsx b/packages/app/src/components/DBTableSelect.tsx index 0228db08f7..8d715b6280 100644 --- a/packages/app/src/components/DBTableSelect.tsx +++ b/packages/app/src/components/DBTableSelect.tsx @@ -61,7 +61,7 @@ function DBTableSelect({ data={data} disabled={isTablesLoading} value={table} - comboboxProps={{ withinPortal: false }} + comboboxProps={{ withinPortal: true }} onChange={v => setTable(v ?? undefined)} onBlur={onBlur} name={name} diff --git a/packages/app/src/components/DatabaseSelect.tsx b/packages/app/src/components/DatabaseSelect.tsx index db48b2fd76..b2270a313a 100644 --- a/packages/app/src/components/DatabaseSelect.tsx +++ b/packages/app/src/components/DatabaseSelect.tsx @@ -41,7 +41,7 @@ function DatabaseSelect({ maxDropdownHeight={280} data={data} disabled={isDatabasesLoading} - comboboxProps={{ withinPortal: false }} + comboboxProps={{ withinPortal: true }} value={database} onChange={v => setDatabase(v ?? undefined)} onBlur={onBlur} diff --git a/packages/app/src/components/__tests__/sourceFormPickerDropdowns.test.tsx b/packages/app/src/components/__tests__/sourceFormPickerDropdowns.test.tsx new file mode 100644 index 0000000000..70d2841534 --- /dev/null +++ b/packages/app/src/components/__tests__/sourceFormPickerDropdowns.test.tsx @@ -0,0 +1,150 @@ +import { useForm } from 'react-hook-form'; +import { screen, within } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import { useDatabasesDirect, useTablesDirect } from '@/clickhouse'; +import { useConnections } from '@/connection'; + +import { ConnectionSelectControlled } from '../ConnectionSelect'; +import { DatabaseSelectControlled } from '../DatabaseSelect'; +import { DBTableSelectControlled } from '../DBTableSelect'; + +jest.mock('@/clickhouse', () => ({ + useDatabasesDirect: jest.fn(), + useTablesDirect: jest.fn(), +})); +jest.mock('@/connection', () => ({ + useConnections: jest.fn(), +})); +// DBTableSelect renders these next to the table picker; not relevant to this test. +jest.mock('../SourceSchemaPreview', () => ({ + __esModule: true, + default: () => null, + isSourceSchemaPreviewEnabled: () => false, +})); +jest.mock('../SourceSelect', () => ({ + SourceManagementMenu: () => null, +})); + +// Mantine's Combobox calls scrollIntoView when its dropdown opens; jsdom lacks it. +window.HTMLElement.prototype.scrollIntoView = jest.fn(); + +// The hooks return large `UseQueryResult`/connection types; the components only +// read a couple of fields, so cast to a loose mock for these minimal fixtures. +const asMock = (fn: unknown) => fn as jest.Mock; + +// Use a large list so the dropdown overflows like the bug report (HDX-4445). +const OPTION_COUNT = 20; +const databaseNames = Array.from( + { length: OPTION_COUNT }, + (_, i) => `db_${String(i).padStart(2, '0')}`, +); +const tableNames = Array.from( + { length: OPTION_COUNT }, + (_, i) => `table_${String(i).padStart(2, '0')}`, +); +const connections = Array.from({ length: OPTION_COUNT }, (_, i) => ({ + id: `conn-${i}`, + name: `Connection ${String(i).padStart(2, '0')}`, +})); + +beforeEach(() => { + asMock(useDatabasesDirect).mockReturnValue({ + data: { data: databaseNames.map(name => ({ name })) }, + isLoading: false, + }); + asMock(useTablesDirect).mockReturnValue({ + data: { data: tableNames.map(name => ({ name })) }, + isLoading: false, + }); + asMock(useConnections).mockReturnValue({ data: connections }); +}); + +function DatabaseHarness() { + const { control } = useForm(); + return ( + + ); +} + +function TableHarness() { + const { control } = useForm(); + return ( + + ); +} + +function ConnectionHarness() { + const { control } = useForm(); + return ; +} + +/** + * HDX-4445: these pickers live inside the source-setup modal. With many entries + * the dropdown must (a) render every option and (b) render them in a portal, so + * the modal's overflow can't clip the list. + * + * `hidden: true` — jsdom has no layout, so the portaled dropdown computes as + * "hidden"; the default role query would skip it. + * + * Note on what jsdom can/can't prove: clipping is a *visual* effect of + * `overflow:hidden`, which does not change the DOM or an element's box, and + * jsdom has no layout — so pixel visibility ("is option 20 actually on screen") + * is not assertable here. The two checks below are what's both meaningful and + * deterministic: every option is rendered (no truncation/virtualization), and + * the options are portaled OUT of the picker's container so a modal can't clip + * them. The container check is what flips between fixed (`withinPortal: true`) + * and broken (`false`) — independent of option count. True on-screen proof + * would require a real-browser (E2E) test. + */ +async function expectAllOptionsRenderedInPortal( + container: HTMLElement, + trigger: HTMLElement, +) { + await userEvent.click(trigger); + + // (a) All options are rendered — none dropped by truncation/virtualization. + const allOptions = await screen.findAllByRole('option', { hidden: true }); + expect(allOptions).toHaveLength(OPTION_COUNT); + + // (b) None are nested inside the picker's own container: they are portaled + // out, so the modal's overflow cannot clip them. Fails on withinPortal:false. + expect( + within(container).queryAllByRole('option', { hidden: true }), + ).toHaveLength(0); +} + +describe('source form picker dropdowns render all options in a portal', () => { + it(`DatabaseSelect renders all ${OPTION_COUNT} databases in a portal`, async () => { + const { container } = renderWithMantine(); + await expectAllOptionsRenderedInPortal( + container, + screen.getByPlaceholderText('Database'), + ); + }); + + it(`DBTableSelect renders all ${OPTION_COUNT} tables in a portal`, async () => { + const { container } = renderWithMantine(); + await expectAllOptionsRenderedInPortal( + container, + screen.getByPlaceholderText('Table'), + ); + }); + + it(`ConnectionSelect renders all ${OPTION_COUNT} connections in a portal`, async () => { + const { container } = renderWithMantine(); + await expectAllOptionsRenderedInPortal( + container, + screen.getByPlaceholderText('Connection'), + ); + }); +}); From d1d91d74c0797d108bd69fc9260db9b406fe6200 Mon Sep 17 00:00:00 2001 From: Mike Shi Date: Thu, 4 Jun 2026 05:22:32 -0700 Subject: [PATCH 018/728] feat(service-map): server-side filtering, latency percentiles, throughput & focus (#2387) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What & why The Service Map showed topology plus only part of RED (request counts + error rate). This brings it closer to a full APM-style service map: filter large maps down to what you care about, see latency (the missing **D** in RED) and throughput, and pivot/inspect a service. ### Commit 1 — Server-side filtering - A Lucene/SQL `where` input and a service-name **MultiSelect** scope which spans/services the map is built from, with inbound/outbound neighbor expansion so a focused service still shows its immediate dependencies. ### Commit 2 — Latency percentiles, throughput, focus & node sizing - **p50 / p95 / p99 latency** per node and per edge, computed server-side via a single `GROUPING SETS` query that emits both rolled-up node-level rows and per-edge rows (percentiles can't be combined client-side). Errors now come from `countIf`, dropping the old per-status map. - **Throughput (req/s)**, labeled, shown alongside percentiles in tooltips. - A **Focus** action in the node tooltip drives the service filter to that service and its immediate neighbors (clicking again clears it). Clicking a node just selects it / opens the tooltip — it does not auto-focus. - **Node size scales with total throughput** (incoming + outgoing requests). ## Testing - 68 unit tests covering the aggregation and the pure helpers (`rawDurationToMs`, `getRequestsPerSecond`, `formatRate`, `getNodeSize`). - `tsc`, eslint, and stylelint are clean. - The `GROUPING SETS` query was executed against a live ClickHouse to confirm it parses/runs (`GROUPING()`, `countIf`, `quantile`). - Not yet visually validated against real trace data. ## Notes for reviewers - Edge latency is the **server-side processing time** of calls from that caller (server spans grouped by caller), not client-observed round-trip latency. - Percentiles are approximate (computed over sampled spans with approximate quantiles) and are marked with a `~` in the UI. - `DBServiceMapPage.tsx` is now ~390 lines; extracting the filter bar into its own component is a reasonable follow-up. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../service-map-percentiles-filtering.md | 12 + packages/app/src/DBServiceMapPage.tsx | 169 ++++- .../src/components/ServiceMap/ServiceMap.tsx | 87 ++- .../components/ServiceMap/ServiceMapEdge.tsx | 19 + .../components/ServiceMap/ServiceMapNode.tsx | 43 +- .../ServiceMap/ServiceMapTooltip.tsx | 93 ++- .../ServiceMap/__tests__/utils.test.ts | 157 +++++ .../app/src/components/ServiceMap/utils.ts | 106 +++ .../src/hooks/__tests__/useServiceMap.test.ts | 601 ++++++------------ packages/app/src/hooks/useServiceMap.tsx | 273 +++++--- 10 files changed, 997 insertions(+), 563 deletions(-) create mode 100644 .changeset/service-map-percentiles-filtering.md diff --git a/.changeset/service-map-percentiles-filtering.md b/.changeset/service-map-percentiles-filtering.md new file mode 100644 index 0000000000..2e80fab9b0 --- /dev/null +++ b/.changeset/service-map-percentiles-filtering.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': patch +--- + +feat(service-map): server-side filtering, latency percentiles, throughput & focus + +The Service Map gains server-side filtering (Lucene/SQL `where` plus a +service-name multi-select with inbound/outbound neighbor expansion), latency +percentiles (p50/p95/p99) and request throughput (req/s) in node and edge +tooltips, a "Focus" action to scope the map to a service and its immediate +dependencies, and node sizing by total throughput (incoming + outgoing). +Percentiles are computed server-side via a single GROUPING SETS query. diff --git a/packages/app/src/DBServiceMapPage.tsx b/packages/app/src/DBServiceMapPage.tsx index 31e4d5ea0b..e717c31921 100644 --- a/packages/app/src/DBServiceMapPage.tsx +++ b/packages/app/src/DBServiceMapPage.tsx @@ -1,16 +1,37 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import dynamic from 'next/dynamic'; import Head from 'next/head'; -import { parseAsInteger, useQueryState } from 'nuqs'; +import { + parseAsArrayOf, + parseAsInteger, + parseAsString, + parseAsStringEnum, + useQueryState, + useQueryStates, +} from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; +import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; -import { Button, Group, Modal, Slider, Text } from '@mantine/core'; +import { + Button, + Flex, + Group, + Modal, + MultiSelect, + Slider, + Text, +} from '@mantine/core'; import { IconConnection } from '@tabler/icons-react'; import EmptyState from '@/components/EmptyState'; import { PageLayout } from '@/components/PageLayout'; +import SearchWhereInput, { + getStoredLanguage, +} from '@/components/SearchInput/SearchWhereInput'; import { IS_LOCAL_MODE } from '@/config'; +import { useGetKeyValues } from '@/hooks/useMetadata'; import { withAppNav } from '@/layout'; +import { parseAsStringEncoded } from '@/utils/queryParsers'; import OnboardingModal from './components/OnboardingModal'; import ServiceMap from './components/ServiceMap/ServiceMap'; @@ -54,6 +75,12 @@ const defaultTimeRange = parseTimeQuery(DEFAULT_INTERVAL, false) as [ Date, ]; +const searchQueryStateMap = { + where: parseAsStringEncoded, + whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), + services: parseAsArrayOf(parseAsString), +}; + function DBServiceMapPage() { const brandName = useBrandDisplayName(); @@ -61,6 +88,9 @@ function DBServiceMapPage() { const [sourceId, setSourceId] = useQueryState('source'); const [isCreateSourceModalOpen, setIsCreateSourceModalOpen] = useState(false); + const [searchedConfig, setSearchedConfig] = + useQueryStates(searchQueryStateMap); + const [displayedTimeInputValue, setDisplayedTimeInputValue] = useState(DEFAULT_INTERVAL); @@ -81,9 +111,12 @@ function DBServiceMapPage() { ) ?? defaultSource) : defaultSource; - const { control } = useForm({ + const { control, handleSubmit, setValue } = useForm({ values: { source: source?.id, + where: searchedConfig.where ?? '', + whereLanguage: + searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', }, }); @@ -97,6 +130,73 @@ function DBServiceMapPage() { } }, [watchedSource, sourceId, setSourceId]); + const sourceTableConnection = useMemo(() => tcFromSource(source), [source]); + + const serviceNameKey = source?.serviceNameExpression ?? 'ServiceName'; + const serviceNamesChartConfig = useMemo( + () => + source + ? { + from: source.from, + connection: source.connection, + timestampValueExpression: source.timestampValueExpression, + where: '', + select: '', + dateRange: searchedTimeRange, + } + : undefined, + [source, searchedTimeRange], + ); + const { data: serviceNameValues, isLoading: isServiceNamesLoading } = + useGetKeyValues( + { + chartConfig: serviceNamesChartConfig, + keys: [serviceNameKey], + disableRowLimit: true, + limit: 10000, + }, + { enabled: !!source }, + ); + const serviceNameOptions = useMemo( + () => + (serviceNameValues?.[0]?.value ?? []) + .map(v => String(v)) + .sort((a, b) => a.localeCompare(b)), + [serviceNameValues], + ); + + const selectedServices = searchedConfig.services ?? []; + const setSelectedServices = useCallback( + (values: string[]) => { + setSearchedConfig(prev => ({ + ...prev, + services: values.length > 0 ? values : null, + })); + }, + [setSearchedConfig], + ); + + // Clicking a node focuses on that service (and, via the server-side filter's + // neighbor expansion, its immediate callers/callees). Clicking the currently + // focused service clears the focus, so clicks toggle. + const onFocusService = useCallback( + (serviceName: string) => { + setSearchedConfig(prev => { + const current = prev.services ?? []; + const isFocused = current.length === 1 && current[0] === serviceName; + return { ...prev, services: isFocused ? null : [serviceName] }; + }); + }, + [setSearchedConfig], + ); + + const onSubmit = useCallback(() => { + onSearch(displayedTimeInputValue); + handleSubmit(({ where, whereLanguage }) => { + setSearchedConfig(prev => ({ ...prev, where, whereLanguage })); + })(); + }, [handleSubmit, setSearchedConfig, displayedTimeInputValue, onSearch]); + const [samplingFactor, setSamplingFactor] = useQueryState( 'samplingFactor', parseAsInteger.withDefault(10), @@ -230,11 +330,62 @@ function DBServiceMapPage() { actions={headerActions} fillViewport content={ - + <> + + + + setValue('whereLanguage', lang, { shouldDirty: true }) + } + enableHotkey + size="xs" + data-testid="service-map-search-input" + dateRange={searchedTimeRange} + sourceId={source?.id} + lucenePlaceholder="Filter spans w/ Lucene (ex. http.method:GET)" + sqlPlaceholder="SQL WHERE to filter spans (ex. Duration > 1000000)" + minWidth="min(500px, 100%)" + /> + + 0 ? selectedServices : undefined + } + onFocusService={onFocusService} + /> + } /> diff --git a/packages/app/src/components/ServiceMap/ServiceMap.tsx b/packages/app/src/components/ServiceMap/ServiceMap.tsx index 29dc7030e0..92a09ac211 100644 --- a/packages/app/src/components/ServiceMap/ServiceMap.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMap.tsx @@ -77,6 +77,7 @@ interface ServiceMapPresentationProps { dateRange: [Date, Date]; source: TTraceSource; isSingleTrace?: boolean; + onFocusService?: (serviceName: string) => void; } function ServiceMapPresentation({ @@ -86,6 +87,7 @@ function ServiceMapPresentation({ dateRange, source, isSingleTrace, + onFocusService, }: ServiceMapPresentationProps) { const [nodes, setNodes] = useState([]); const [edges, setEdges] = useState([]); @@ -108,12 +110,16 @@ function ServiceMapPresentation({ [], ); - const maxErrorPercentage = useMemo(() => { + const { maxErrorPercentage, maxThroughput } = useMemo(() => { let maxError = 0; + let maxTput = 0; for (const service of services?.values() ?? []) { maxError = Math.max(service.incomingRequests.errorPercentage, maxError); + const throughput = + service.incomingRequests.totalRequests + service.outgoingRequests; + maxTput = Math.max(throughput, maxTput); } - return maxError; + return { maxErrorPercentage: maxError, maxThroughput: maxTput }; }, [services]); useEffect(() => { @@ -125,7 +131,9 @@ function ServiceMapPresentation({ dateRange, source, maxErrorPercentage, + maxThroughput, isSingleTrace, + onFocusService, }, position: { x: index * 150, y: 100 }, type: 'service', @@ -135,37 +143,48 @@ function ServiceMapPresentation({ services?.values() ?? [], ) .filter(service => service.incomingRequestsByClient.size > 0) - .flatMap( - ({ - serviceName, - incomingRequestsByClient: requestCountPerClientPerStatus, - }) => - Array.from(requestCountPerClientPerStatus.entries()).map( - ([clientServiceName, { totalRequests, errorPercentage }]) => { - return { - id: `${serviceName}-${clientServiceName}`, - source: clientServiceName, - target: serviceName, - animated: true, - type: 'request', - data: { - totalRequests, - errorPercentage, - source, - dateRange, - serviceName, - isSingleTrace, - }, - }; - }, - ), + .flatMap(({ serviceName, incomingRequestsByClient: requestsByClient }) => + Array.from(requestsByClient.entries()).map( + ([ + clientServiceName, + { totalRequests, errorPercentage, p50, p95, p99, hasLatency }, + ]) => { + return { + id: `${serviceName}-${clientServiceName}`, + source: clientServiceName, + target: serviceName, + animated: true, + type: 'request', + data: { + totalRequests, + errorPercentage, + p50, + p95, + p99, + hasLatency, + source, + dateRange, + serviceName, + isSingleTrace, + }, + }; + }, + ), ); const nodeWithLayout = getGraphLayout(nodes, edges); setNodes(nodeWithLayout); setEdges(edges); - }, [services, dateRange, source, maxErrorPercentage, isSingleTrace]); + }, [ + services, + dateRange, + source, + maxErrorPercentage, + maxThroughput, + isSingleTrace, + onFocusService, + ]); if (isLoading) { return ( @@ -246,6 +265,12 @@ interface ServiceMapProps { dateRange: [Date, Date]; samplingFactor?: number; isSingleTrace?: boolean; + where?: string; + whereLanguage?: 'sql' | 'lucene'; + serviceNames?: string[]; + // Called when a node is clicked, e.g. to drive the service filter to focus + // on that service and its immediate dependencies. + onFocusService?: (serviceName: string) => void; } export default function ServiceMap({ @@ -254,6 +279,10 @@ export default function ServiceMap({ dateRange, samplingFactor = 1, isSingleTrace, + where, + whereLanguage, + serviceNames, + onFocusService, }: ServiceMapProps) { const { isLoading, @@ -264,6 +293,9 @@ export default function ServiceMap({ source: traceTableSource, dateRange, samplingFactor, + where, + whereLanguage, + serviceNames, }); useEffect(() => { @@ -285,6 +317,7 @@ export default function ServiceMap({ dateRange={dateRange} source={traceTableSource} isSingleTrace={isSingleTrace} + onFocusService={onFocusService} /> ); diff --git a/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx b/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx index d688cb0f12..9e08473531 100644 --- a/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx @@ -8,10 +8,16 @@ import { } from '@xyflow/react'; import ServiceMapTooltip from './ServiceMapTooltip'; +import { deriveDisplayMetrics } from './utils'; export type ServiceMapEdgeData = { totalRequests: number; errorPercentage: number; + // Latency percentiles (raw duration units) for this client→server call. + p50: number; + p95: number; + p99: number; + hasLatency: boolean; dateRange: [Date, Date]; source: TTraceSource; serviceName: string; @@ -30,12 +36,23 @@ export default function ServiceMapEdge( const { totalRequests, errorPercentage, + p50, + p95, + p99, + hasLatency, dateRange, serviceName, source, isSingleTrace, } = props.data; + const { latencyMs, requestsPerSecond } = deriveDisplayMetrics( + { totalRequests, p50, p95, p99, hasLatency }, + source, + dateRange, + isSingleTrace, + ); + return ( <> @@ -48,6 +65,8 @@ export default function ServiceMapEdge( void; }; export default function ServiceMapNode( @@ -25,11 +30,18 @@ export default function ServiceMapNode( incomingRequests: { totalRequests: totalIncomingRequestCount, errorPercentage, + p50, + p95, + p99, + hasLatency, }, + outgoingRequests, source, dateRange, maxErrorPercentage, + maxThroughput, isSingleTrace, + onFocusService, } = data; const { backgroundColor, borderColor } = getNodeColors( @@ -38,16 +50,43 @@ export default function ServiceMapNode( props.selected, ); + // Fallback matches the schema default (3 = ms); in practice the field is + // always present on a parsed source. + const { latencyMs, requestsPerSecond } = deriveDisplayMetrics( + { + totalRequests: totalIncomingRequestCount, + p50, + p95, + p99, + hasLatency, + }, + source, + dateRange, + isSingleTrace, + ); + + const size = getNodeSize( + totalIncomingRequestCount + outgoingRequests, + maxThroughput, + ); + return ( <> onFocusService(serviceName) + : undefined + } />
@@ -64,6 +103,8 @@ export default function ServiceMapNode( style={{ backgroundColor, borderColor, + width: size, + height: size, }} />
diff --git a/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx b/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx index 918c14fb80..1aafa4d66b 100644 --- a/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx @@ -1,27 +1,48 @@ import { useCallback } from 'react'; import SqlString from 'sqlstring'; import { TTraceSource } from '@hyperdx/common-utils/dist/types'; -import { Button, Stack } from '@mantine/core'; -import { IconSearch } from '@tabler/icons-react'; +import { Button, Group, Stack, Text } from '@mantine/core'; +import { + IconActivity, + IconClock, + IconSearch, + IconTarget, +} from '@tabler/icons-react'; -import { formatApproximateNumber, navigateToTraceSearch } from './utils'; +import { formatDurationMs } from '@/utils'; + +import { + formatApproximateNumber, + formatRate, + navigateToTraceSearch, +} from './utils'; import styles from './ServiceMap.module.scss'; export default function ServiceMapTooltip({ totalRequests, errorPercentage, + latencyMs, + requestsPerSecond, source, dateRange, serviceName, isSingleTrace, + onFocus, }: { totalRequests: number; errorPercentage: number; + // p50/p95/p99 already converted to milliseconds; omitted when unavailable. + latencyMs?: { p50: number; p95: number; p99: number }; + // Throughput; omitted for single-trace maps where a rate is meaningless. + requestsPerSecond?: number; source: TTraceSource; dateRange: [Date, Date]; serviceName: string; isSingleTrace?: boolean; + // When provided, renders a "Focus" action that filters the map to this + // service and its immediate dependencies. + onFocus?: () => void; }) { const requestText = `${isSingleTrace ? totalRequests : formatApproximateNumber(totalRequests)} request${ totalRequests !== 1 ? 's' : '' @@ -56,29 +77,69 @@ export default function ServiceMapTooltip({ }); }, [dateRange, source, serviceName]); + const showMetrics = + totalRequests > 0 && (requestsPerSecond != null || latencyMs != null); + return ( - + {errorPercentage > 0 ? ( - <> - - + + ) : null} + {showMetrics ? ( + + {requestsPerSecond != null ? ( + + + {formatRate(requestsPerSecond)} + + ) : null} + {latencyMs != null ? ( + + + {/* Percentiles are estimated (approximate quantiles, over sampled + spans), so prefix each value with ~. */} + + p50 ~{formatDurationMs(latencyMs.p50)} · p95 ~ + {formatDurationMs(latencyMs.p95)} · p99 ~ + {formatDurationMs(latencyMs.p99)} + + + ) : null} + + ) : null} + {onFocus ? ( + ) : null} ); diff --git a/packages/app/src/components/ServiceMap/__tests__/utils.test.ts b/packages/app/src/components/ServiceMap/__tests__/utils.test.ts index ecabf40fb3..bb2a1c2a99 100644 --- a/packages/app/src/components/ServiceMap/__tests__/utils.test.ts +++ b/packages/app/src/components/ServiceMap/__tests__/utils.test.ts @@ -2,9 +2,14 @@ import router from 'next/router'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { + deriveDisplayMetrics, formatApproximateNumber, + formatRate, getNodeColors, + getNodeSize, + getRequestsPerSecond, navigateToTraceSearch, + rawDurationToMs, } from '../utils'; // Mock next/router @@ -346,3 +351,155 @@ describe('getNodeColors', () => { }); }); }); + +describe('rawDurationToMs', () => { + it('converts nanoseconds (precision 9) to milliseconds', () => { + // 1_000_000 ns = 1 ms + expect(rawDurationToMs(1_000_000, 9)).toBe(1); + expect(rawDurationToMs(2_500_000, 9)).toBe(2.5); + }); + + it('converts microseconds (precision 6) to milliseconds', () => { + // 1000 µs = 1 ms + expect(rawDurationToMs(1000, 6)).toBe(1); + expect(rawDurationToMs(500, 6)).toBe(0.5); + }); + + it('treats precision 3 as already-milliseconds', () => { + expect(rawDurationToMs(42, 3)).toBe(42); + }); + + it('scales up for precision below 3 (e.g. seconds)', () => { + // precision 0 = seconds: 7s -> 7000ms (divisor is 10^-3, i.e. multiply up) + expect(rawDurationToMs(7, 0)).toBe(7000); + }); + + it('handles zero', () => { + expect(rawDurationToMs(0, 9)).toBe(0); + }); +}); + +describe('getRequestsPerSecond', () => { + const oneHour: [Date, Date] = [ + new Date('2024-01-01T00:00:00.000Z'), + new Date('2024-01-01T01:00:00.000Z'), + ]; + + it('divides total requests by the window in seconds', () => { + // 3600 requests over 3600s = 1/s + expect(getRequestsPerSecond(3600, oneHour)).toBe(1); + expect(getRequestsPerSecond(7200, oneHour)).toBe(2); + }); + + it('returns 0 for a zero-length window', () => { + const t = new Date('2024-01-01T00:00:00.000Z'); + expect(getRequestsPerSecond(100, [t, t])).toBe(0); + }); + + it('returns 0 for an inverted window', () => { + expect(getRequestsPerSecond(100, [oneHour[1], oneHour[0]])).toBe(0); + }); +}); + +describe('formatRate', () => { + it('formats sub-1/s rates with two decimals and a req/s label', () => { + expect(formatRate(0)).toBe('0 req/s'); + expect(formatRate(0.2)).toBe('0.20 req/s'); + }); + + it('formats 1-999/s with one decimal', () => { + expect(formatRate(1)).toBe('1.0 req/s'); + expect(formatRate(3.45)).toBe('3.5 req/s'); + expect(formatRate(999)).toBe('999.0 req/s'); + }); + + it('formats >=1000/s in thousands', () => { + expect(formatRate(1000)).toBe('1.0k req/s'); + expect(formatRate(12345)).toBe('12.3k req/s'); + }); + + it('returns 0 req/s for non-finite or negative input', () => { + expect(formatRate(Infinity)).toBe('0 req/s'); + expect(formatRate(NaN)).toBe('0 req/s'); + expect(formatRate(-5)).toBe('0 req/s'); + }); +}); + +describe('getNodeSize', () => { + it('returns the minimum size when there is no traffic to compare', () => { + expect(getNodeSize(0, 0)).toBe(32); + expect(getNodeSize(0, 100)).toBe(32); + }); + + it('returns the maximum size for the busiest node', () => { + // ratio = sqrt(100/100) = 1 -> MAX + expect(getNodeSize(100, 100)).toBe(60); + }); + + it('scales by the square root of the request ratio', () => { + // sqrt(25/100) = 0.5 -> 32 + 0.5*(60-32) = 46 + expect(getNodeSize(25, 100)).toBe(46); + }); + + it('clamps requests above the max to the max size', () => { + expect(getNodeSize(200, 100)).toBe(60); + }); + + it('stays within [32, 60]', () => { + for (const r of [1, 10, 50, 99, 100]) { + const size = getNodeSize(r, 100); + expect(size).toBeGreaterThanOrEqual(32); + expect(size).toBeLessThanOrEqual(60); + } + }); +}); + +describe('deriveDisplayMetrics', () => { + const source = { durationPrecision: 9 } as unknown as TTraceSource; + const oneHour: [Date, Date] = [ + new Date('2024-01-01T00:00:00.000Z'), + new Date('2024-01-01T01:00:00.000Z'), + ]; + + it('converts percentiles to ms and computes throughput', () => { + const m = deriveDisplayMetrics( + { + totalRequests: 3600, + p50: 1_000_000, + p95: 5_000_000, + p99: 9_000_000, + hasLatency: true, + }, + source, + oneHour, + ); + expect(m.latencyMs).toEqual({ p50: 1, p95: 5, p99: 9 }); + expect(m.requestsPerSecond).toBe(1); + }); + + it('omits latency when hasLatency is false', () => { + const m = deriveDisplayMetrics( + { totalRequests: 10, p50: 0, p95: 0, p99: 0, hasLatency: false }, + source, + oneHour, + ); + expect(m.latencyMs).toBeUndefined(); + }); + + it('omits throughput for single-trace maps but keeps latency', () => { + const m = deriveDisplayMetrics( + { + totalRequests: 10, + p50: 1_000_000, + p95: 1_000_000, + p99: 1_000_000, + hasLatency: true, + }, + source, + oneHour, + true, + ); + expect(m.requestsPerSecond).toBeUndefined(); + expect(m.latencyMs).toBeDefined(); + }); +}); diff --git a/packages/app/src/components/ServiceMap/utils.ts b/packages/app/src/components/ServiceMap/utils.ts index 316c248c5f..bb131e8e5a 100644 --- a/packages/app/src/components/ServiceMap/utils.ts +++ b/packages/app/src/components/ServiceMap/utils.ts @@ -60,3 +60,109 @@ export function getNodeColors( borderColor, }; } + +/** + * Converts a raw duration column value into milliseconds using the source's + * `durationPrecision` (the base-10 exponent of the stored unit: 9 = ns, + * 6 = µs, 3 = ms). Mirrors the conversion used by the trace waterfall so the + * service map reports latency in the same units as the rest of the app. + */ +export function rawDurationToMs( + rawDuration: number, + durationPrecision: number, +): number { + // ms = raw * 10^(3 - precision). Multiplying keeps an exact integer factor + // for precision < 3 (e.g. 0 = seconds) instead of dividing by a fractional + // divisor; matches getDurationSecondsExpression's unit conversion. + return rawDuration * Math.pow(10, 3 - durationPrecision); +} + +/** + * Normalizes a total request count over a time window into a per-second rate + * (throughput). Returns 0 for non-positive windows. + */ +export function getRequestsPerSecond( + totalRequests: number, + dateRange: [Date, Date], +): number { + const windowSeconds = + (dateRange[1].getTime() - dateRange[0].getTime()) / 1000; + if (windowSeconds <= 0) { + return 0; + } + return totalRequests / windowSeconds; +} + +/** + * Formats a per-second request rate for display, with an explicit unit label, + * e.g. "1.2k req/s", "5.0 req/s", "0.20 req/s". + */ +export function formatRate(perSecond: number): string { + if (!Number.isFinite(perSecond) || perSecond <= 0) { + return '0 req/s'; + } + if (perSecond >= 1000) { + return `${(perSecond / 1000).toFixed(1)}k req/s`; + } + if (perSecond >= 1) { + return `${perSecond.toFixed(1)} req/s`; + } + return `${perSecond.toFixed(2)} req/s`; +} + +const MIN_NODE_SIZE = 32; +const MAX_NODE_SIZE = 60; + +/** + * Scales a node's diameter (px) by its total throughput (incoming + outgoing + * requests) relative to the busiest node, so heavier-traffic services read as + * larger. Uses a square-root scale so visual *area* tracks volume rather than + * diameter. Falls back to the minimum size when there's nothing to compare. + */ +export function getNodeSize(throughput: number, maxThroughput: number): number { + if (maxThroughput <= 0 || throughput <= 0) { + return MIN_NODE_SIZE; + } + const ratio = Math.sqrt(Math.min(throughput, maxThroughput) / maxThroughput); + return Math.round(MIN_NODE_SIZE + ratio * (MAX_NODE_SIZE - MIN_NODE_SIZE)); +} + +export type DisplayStats = { + totalRequests: number; + p50: number; + p95: number; + p99: number; + hasLatency: boolean; +}; + +/** + * Derives the latency/throughput a tooltip shows from a node's or edge's raw + * stats. Shared by ServiceMapNode and ServiceMapEdge so the two stay in sync: + * latency is converted to ms (only when available), throughput is omitted for + * single-trace maps where a per-second rate is meaningless. + */ +export function deriveDisplayMetrics( + stats: DisplayStats, + source: TTraceSource, + dateRange: [Date, Date], + isSingleTrace?: boolean, +): { + latencyMs?: { p50: number; p95: number; p99: number }; + requestsPerSecond?: number; +} { + // Fallback matches the schema default (3 = ms); in practice the field is + // always present on a parsed source. + const precision = source.durationPrecision ?? 3; + return { + latencyMs: stats.hasLatency + ? { + p50: rawDurationToMs(stats.p50, precision), + p95: rawDurationToMs(stats.p95, precision), + p99: rawDurationToMs(stats.p99, precision), + } + : undefined, + requestsPerSecond: isSingleTrace + ? undefined + : getRequestsPerSecond(stats.totalRequests, dateRange), + }; +} diff --git a/packages/app/src/hooks/__tests__/useServiceMap.test.ts b/packages/app/src/hooks/__tests__/useServiceMap.test.ts index c34534acb1..4418bf34e6 100644 --- a/packages/app/src/hooks/__tests__/useServiceMap.test.ts +++ b/packages/app/src/hooks/__tests__/useServiceMap.test.ts @@ -1,484 +1,261 @@ import { aggregateServiceMapData, SpanAggregationRow } from '../useServiceMap'; +// Convenience builders for the GROUPING SETS row shapes. +function nodeRow( + serverServiceName: string, + requestCount: number, + errorCount: number, + latency?: { p50: number; p95: number; p99: number }, +): SpanAggregationRow { + return { + serverServiceName, + isNodeLevel: true, + requestCount, + errorCount, + ...latency, + }; +} + +function edgeRow( + serverServiceName: string, + clientServiceName: string | undefined, + requestCount: number, + errorCount: number, + latency?: { p50: number; p95: number; p99: number }, +): SpanAggregationRow { + return { + serverServiceName, + clientServiceName, + isNodeLevel: false, + requestCount, + errorCount, + ...latency, + }; +} + describe('aggregateServiceMapData', () => { describe('basic aggregation', () => { - it('should return empty map for empty input', () => { - const result = aggregateServiceMapData([]); - expect(result.size).toBe(0); + it('returns an empty map for empty input', () => { + expect(aggregateServiceMapData([]).size).toBe(0); }); - it('should aggregate single service with single status', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); + it('builds a service from a node-level row', () => { + const result = aggregateServiceMapData([nodeRow('api-service', 100, 0)]); expect(result.size).toBe(1); - expect(result.has('api-service')).toBe(true); - const service = result.get('api-service')!; expect(service.serviceName).toBe('api-service'); expect(service.incomingRequests.totalRequests).toBe(100); - expect(service.incomingRequests.requestCountByStatus.get('Ok')).toBe(100); + expect(service.incomingRequests.errorCount).toBe(0); expect(service.incomingRequests.errorPercentage).toBe(0); expect(service.incomingRequestsByClient.size).toBe(0); }); - it('should aggregate multiple rows for same service and status', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 100, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 50, - }, - ]; - - const result = aggregateServiceMapData(data); + it('uses the node-level row as the rolled-up total (no client summing)', () => { + // Node-level totals come straight from the rolled-up row, independent of + // the per-edge rows. + const result = aggregateServiceMapData([ + nodeRow('api-service', 150, 15), + edgeRow('api-service', 'web', 100, 10), + edgeRow('api-service', 'worker', 50, 5), + ]); - expect(result.size).toBe(1); const service = result.get('api-service')!; expect(service.incomingRequests.totalRequests).toBe(150); - expect(service.incomingRequests.requestCountByStatus.get('Ok')).toBe(150); + expect(service.incomingRequests.errorCount).toBe(15); + expect(service.incomingRequestsByClient.size).toBe(2); }); + }); - it('should aggregate multiple status codes for same service', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 100, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - requestCount: 10, - }, - ]; - - const result = aggregateServiceMapData(data); - - expect(result.size).toBe(1); - const service = result.get('api-service')!; - expect(service.incomingRequests.totalRequests).toBe(110); - expect(service.incomingRequests.requestCountByStatus.get('Ok')).toBe(100); - expect(service.incomingRequests.requestCountByStatus.get('Error')).toBe( + describe('error percentage', () => { + it('computes errorPercentage from errorCount / totalRequests', () => { + const result = aggregateServiceMapData([nodeRow('api-service', 100, 10)]); + expect(result.get('api-service')!.incomingRequests.errorPercentage).toBe( 10, ); }); - it('should aggregate multiple services', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 100, - }, - { - serverServiceName: 'db-service', - serverStatusCode: 'Ok', - requestCount: 200, - }, - ]; - - const result = aggregateServiceMapData(data); - - expect(result.size).toBe(2); - expect(result.has('api-service')).toBe(true); - expect(result.has('db-service')).toBe(true); - expect(result.get('api-service')!.incomingRequests.totalRequests).toBe( + it('is 100 when all requests error', () => { + const result = aggregateServiceMapData([nodeRow('api-service', 30, 30)]); + expect(result.get('api-service')!.incomingRequests.errorPercentage).toBe( 100, ); - expect(result.get('db-service')!.incomingRequests.totalRequests).toBe( - 200, - ); }); - }); - describe('error percentage calculation', () => { - it('should calculate error percentage correctly', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 70, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Unset', - requestCount: 20, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - requestCount: 10, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; - - expect(service.incomingRequests.totalRequests).toBe(100); - expect(service.incomingRequests.errorPercentage).toBe(10); - }); - - it('should calculate 0% error when no errors', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; - - expect(service.incomingRequests.errorPercentage).toBe(0); - }); - - it('should calculate 100% error when all errors', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; - - expect(service.incomingRequests.errorPercentage).toBe(100); - }); - - it('should calculate precise error percentages', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 97, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - requestCount: 3, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; - - expect(service.incomingRequests.errorPercentage).toBe(3); - }); - - it('should handle 0 total requests without division by zero', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'client-service', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); - - // client-service is added but has no incoming requests - const clientService = result.get('client-service')!; - expect(clientService.incomingRequests.totalRequests).toBe(0); - expect(clientService.incomingRequests.errorPercentage).toBe(0); + it('avoids division by zero for zero-request services', () => { + // A client-only node, created from an edge row, has no incoming requests. + const result = aggregateServiceMapData([ + edgeRow('api-service', 'web', 100, 0), + ]); + const web = result.get('web')!; + expect(web.incomingRequests.totalRequests).toBe(0); + expect(web.incomingRequests.errorPercentage).toBe(0); }); }); - describe('client service aggregation', () => { - it('should aggregate requests by client service', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'web-service', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); + describe('edges (per-client stats)', () => { + it('records per-client stats and creates the client node', () => { + const result = aggregateServiceMapData([ + nodeRow('api-service', 100, 0), + edgeRow('api-service', 'web-service', 100, 0), + ]); expect(result.size).toBe(2); - expect(result.has('api-service')).toBe(true); expect(result.has('web-service')).toBe(true); const service = result.get('api-service')!; - expect(service.incomingRequests.totalRequests).toBe(100); - expect(service.incomingRequestsByClient.size).toBe(1); - expect(service.incomingRequestsByClient.has('web-service')).toBe(true); - - const clientStats = service.incomingRequestsByClient.get('web-service')!; - expect(clientStats.totalRequests).toBe(100); - expect(clientStats.requestCountByStatus.get('Ok')).toBe(100); + const fromWeb = service.incomingRequestsByClient.get('web-service')!; + expect(fromWeb.totalRequests).toBe(100); }); - it('should aggregate multiple clients calling same service', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'web-service', - requestCount: 100, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'mobile-service', - requestCount: 50, - }, - ]; - - const result = aggregateServiceMapData(data); + it('supports multiple clients calling the same service', () => { + const result = aggregateServiceMapData([ + nodeRow('api-service', 150, 0), + edgeRow('api-service', 'web', 100, 0), + edgeRow('api-service', 'mobile', 50, 5), + ]); - expect(result.size).toBe(3); // api-service, web-service, mobile-service const service = result.get('api-service')!; - expect(service.incomingRequests.totalRequests).toBe(150); expect(service.incomingRequestsByClient.size).toBe(2); - - const webStats = service.incomingRequestsByClient.get('web-service')!; - expect(webStats.totalRequests).toBe(100); - - const mobileStats = - service.incomingRequestsByClient.get('mobile-service')!; - expect(mobileStats.totalRequests).toBe(50); - }); - - it('should aggregate multiple requests from same client with different status codes', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'web-service', - requestCount: 90, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - clientServiceName: 'web-service', - requestCount: 10, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; - const clientStats = service.incomingRequestsByClient.get('web-service')!; - - expect(clientStats.totalRequests).toBe(100); - expect(clientStats.requestCountByStatus.get('Ok')).toBe(90); - expect(clientStats.requestCountByStatus.get('Error')).toBe(10); - expect(clientStats.errorPercentage).toBe(10); + expect( + service.incomingRequestsByClient.get('mobile')!.errorPercentage, + ).toBe(10); }); - it('should handle mix of requests with and without client service names', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'web-service', - requestCount: 100, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 50, // No client service (uninstrumented) - }, - ]; - - const result = aggregateServiceMapData(data); + it('ignores edge rows from uninstrumented (null) clients', () => { + // These are already counted in the node-level total; no edge is drawn. + const result = aggregateServiceMapData([ + nodeRow('api-service', 100, 0), + edgeRow('api-service', undefined, 40, 0), + ]); const service = result.get('api-service')!; - expect(service.incomingRequests.totalRequests).toBe(150); - expect(service.incomingRequestsByClient.size).toBe(1); - expect(service.incomingRequestsByClient.has('web-service')).toBe(true); + expect(service.incomingRequestsByClient.size).toBe(0); + expect(result.size).toBe(1); }); - it('should create client service entry even if client has no incoming requests', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'web-service', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); - - expect(result.has('web-service')).toBe(true); - const clientService = result.get('web-service')!; - expect(clientService.serviceName).toBe('web-service'); - expect(clientService.incomingRequests.totalRequests).toBe(0); - expect(clientService.incomingRequestsByClient.size).toBe(0); - expect(clientService.incomingRequests.errorPercentage).toBe(0); + it('builds a chain A -> B -> C', () => { + const result = aggregateServiceMapData([ + nodeRow('B', 100, 0), + edgeRow('B', 'A', 100, 0), + nodeRow('C', 80, 0), + edgeRow('C', 'B', 80, 0), + ]); + + expect(new Set(result.keys())).toEqual(new Set(['A', 'B', 'C'])); + expect(result.get('B')!.incomingRequestsByClient.has('A')).toBe(true); + expect(result.get('C')!.incomingRequestsByClient.has('B')).toBe(true); + // A is a leaf client with no incoming traffic. + expect(result.get('A')!.incomingRequests.totalRequests).toBe(0); }); }); - describe('complex scenarios', () => { - it('should handle service chain (A -> B -> C)', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'service-b', - serverStatusCode: 'Ok', - clientServiceName: 'service-a', - requestCount: 100, - }, - { - serverServiceName: 'service-c', - serverStatusCode: 'Ok', - clientServiceName: 'service-b', - requestCount: 100, - }, - ]; - - const result = aggregateServiceMapData(data); - - expect(result.size).toBe(3); - - // Service A (no incoming requests) - const serviceA = result.get('service-a')!; - expect(serviceA.incomingRequests.totalRequests).toBe(0); - - // Service B (receives from A, calls C) - const serviceB = result.get('service-b')!; - expect(serviceB.incomingRequests.totalRequests).toBe(100); - expect(serviceB.incomingRequestsByClient.has('service-a')).toBe(true); - - // Service C (receives from B) - const serviceC = result.get('service-c')!; - expect(serviceC.incomingRequests.totalRequests).toBe(100); - expect(serviceC.incomingRequestsByClient.has('service-b')).toBe(true); + describe('outgoing throughput', () => { + it('rolls edge volume up to the caller as outgoingRequests', () => { + const result = aggregateServiceMapData([ + nodeRow('api', 100, 0), + edgeRow('api', 'web', 100, 0), + ]); + + // web calls api 100x -> web has 100 outgoing, 0 incoming. + expect(result.get('web')!.outgoingRequests).toBe(100); + expect(result.get('web')!.incomingRequests.totalRequests).toBe(0); + // api receives but makes no calls of its own here. + expect(result.get('api')!.outgoingRequests).toBe(0); }); - it('should handle multiple clients with different error rates', () => { - const data: SpanAggregationRow[] = [ - // Client 1: 10% error rate - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'client-1', - requestCount: 90, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - clientServiceName: 'client-1', - requestCount: 10, - }, - // Client 2: 50% error rate - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - clientServiceName: 'client-2', - requestCount: 50, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - clientServiceName: 'client-2', - requestCount: 50, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; - - // Overall error rate: 60 errors out of 200 = 30% - expect(service.incomingRequests.totalRequests).toBe(200); - expect(service.incomingRequests.errorPercentage).toBe(30); + it('sums outgoing across multiple callees', () => { + const result = aggregateServiceMapData([ + nodeRow('api', 100, 0), + edgeRow('api', 'gateway', 100, 0), + nodeRow('db', 60, 0), + edgeRow('db', 'gateway', 60, 0), + ]); - // Client 1: 10% error rate - const client1Stats = service.incomingRequestsByClient.get('client-1')!; - expect(client1Stats.errorPercentage).toBe(10); - - // Client 2: 50% error rate - const client2Stats = service.incomingRequestsByClient.get('client-2')!; - expect(client2Stats.errorPercentage).toBe(50); + // gateway calls api (100) and db (60) -> 160 outgoing. + expect(result.get('gateway')!.outgoingRequests).toBe(160); }); - it('should handle large request counts', () => { - const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 1_000_000, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', - requestCount: 1_000, - }, - ]; + it('accumulates both incoming and outgoing for a mid-chain service', () => { + // A -> B -> C : B receives 100 and makes 80. + const result = aggregateServiceMapData([ + nodeRow('B', 100, 0), + edgeRow('B', 'A', 100, 0), + nodeRow('C', 80, 0), + edgeRow('C', 'B', 80, 0), + ]); + + expect(result.get('B')!.incomingRequests.totalRequests).toBe(100); + expect(result.get('B')!.outgoingRequests).toBe(80); + expect(result.get('A')!.outgoingRequests).toBe(100); + expect(result.get('C')!.outgoingRequests).toBe(0); + }); + }); - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; + describe('latency percentiles', () => { + it('passes through percentiles and flags hasLatency', () => { + const result = aggregateServiceMapData([ + nodeRow('api-service', 100, 0, { p50: 5, p95: 20, p99: 50 }), + ]); + + const stats = result.get('api-service')!.incomingRequests; + expect(stats.p50).toBe(5); + expect(stats.p95).toBe(20); + expect(stats.p99).toBe(50); + expect(stats.hasLatency).toBe(true); + }); - expect(service.incomingRequests.totalRequests).toBe(1_001_000); - expect(service.incomingRequests.errorPercentage).toBeCloseTo(0.0999, 4); + it('marks hasLatency false and zeroes percentiles when absent', () => { + const stats = aggregateServiceMapData([ + nodeRow('api-service', 100, 0), + ]).get('api-service')!.incomingRequests; + expect(stats.hasLatency).toBe(false); + expect(stats.p50).toBe(0); + expect(stats.p95).toBe(0); + expect(stats.p99).toBe(0); }); - it('should handle different status code strings', () => { - const data: SpanAggregationRow[] = [ + it('flags hasLatency from p50 even when p95/p99 are absent', () => { + // statsFromRow keys hasLatency off p50; missing p95/p99 default to 0. + const result = aggregateServiceMapData([ { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 50, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Error', + serverServiceName: 'api', + isNodeLevel: true, requestCount: 10, - }, - { - serverServiceName: 'api-service', - serverStatusCode: 'Unset', - requestCount: 40, - }, - ]; - - const result = aggregateServiceMapData(data); - const service = result.get('api-service')!; + errorCount: 0, + p50: 5, + }, + ]); + const stats = result.get('api')!.incomingRequests; + expect(stats.hasLatency).toBe(true); + expect(stats.p50).toBe(5); + expect(stats.p95).toBe(0); + expect(stats.p99).toBe(0); + }); - expect(service.incomingRequests.totalRequests).toBe(100); - expect(service.incomingRequests.requestCountByStatus.get('Ok')).toBe(50); - expect(service.incomingRequests.requestCountByStatus.get('Error')).toBe( - 10, - ); - expect(service.incomingRequests.requestCountByStatus.get('Unset')).toBe( - 40, - ); - expect(service.incomingRequests.errorPercentage).toBe(10); + it('records per-edge percentiles independently of the node', () => { + const result = aggregateServiceMapData([ + nodeRow('api-service', 100, 0, { p50: 5, p95: 20, p99: 50 }), + edgeRow('api-service', 'web', 100, 0, { p50: 8, p95: 30, p99: 60 }), + ]); + + const fromWeb = result + .get('api-service')! + .incomingRequestsByClient.get('web')!; + expect(fromWeb.p95).toBe(30); + expect(fromWeb.hasLatency).toBe(true); }); }); describe('data structure integrity', () => { - it('should not mutate input data', () => { + it('does not mutate the input rows', () => { const data: SpanAggregationRow[] = [ - { - serverServiceName: 'api-service', - serverStatusCode: 'Ok', - requestCount: 100, - }, + nodeRow('api-service', 100, 10, { p50: 1, p95: 2, p99: 3 }), + edgeRow('api-service', 'web', 100, 10), ]; - - const originalData = JSON.parse(JSON.stringify(data)); + const original = JSON.parse(JSON.stringify(data)); aggregateServiceMapData(data); - - expect(data).toEqual(originalData); + expect(data).toEqual(original); }); }); }); diff --git a/packages/app/src/hooks/useServiceMap.tsx b/packages/app/src/hooks/useServiceMap.tsx index 5fc6639d04..a220e340f4 100644 --- a/packages/app/src/hooks/useServiceMap.tsx +++ b/packages/app/src/hooks/useServiceMap.tsx @@ -11,9 +11,18 @@ import { useMetadataWithSettings } from './useMetadata'; export type SpanAggregationRow = { serverServiceName: string; - serverStatusCode: string; - requestCount: number; clientServiceName?: string; + // True for node-level (rolled-up across all callers) rows, false for + // edge-level (a specific client→server call) rows. Derived from GROUPING() + // over the client service in the GROUPING SETS query. + isNodeLevel: boolean; + requestCount: number; + errorCount: number; + // Latency percentiles in raw duration units. Undefined when the source has + // no duration expression. Convert with rawDurationToMs for display. + p50?: number; + p95?: number; + p99?: number; }; async function getServiceMapQuery({ @@ -22,12 +31,18 @@ async function getServiceMapQuery({ traceId, metadata, samplingFactor, + where, + whereLanguage, + serviceNames, }: { source: TTraceSource; dateRange: [Date, Date]; traceId?: string; metadata: Metadata; samplingFactor: number; + where?: string; + whereLanguage?: 'sql' | 'lucene'; + serviceNames?: string[]; }) { // Don't sample if we're looking for a specific trace const effectiveSamplingLevel = traceId ? 1 : samplingFactor; @@ -37,6 +52,11 @@ async function getServiceMapQuery({ connection: source.connection, dateRange, timestampValueExpression: source.timestampValueExpression, + ...(source.implicitColumnExpression != null + ? { implicitColumnExpression: source.implicitColumnExpression } + : {}), + where: where || '', + whereLanguage: whereLanguage ?? 'lucene', filters: [ // Sample a subset of traces, for performance in the following join { @@ -77,6 +97,16 @@ async function getServiceMapQuery({ valueExpression: source.statusCodeExpression ?? 'StatusCode', alias: 'statusCode', }, + // Carry the raw span duration through so we can aggregate latency. + // Only available when the source defines a duration expression. + ...(source.durationExpression + ? [ + { + valueExpression: source.durationExpression, + alias: 'duration', + }, + ] + : []), ], }; @@ -91,7 +121,6 @@ async function getServiceMapQuery({ condition: `${source.spanKindExpression} IN ('Server', 'Consumer', 'SPAN_KIND_SERVER', 'SPAN_KIND_CONSUMER')`, }, ], - where: '', }, metadata, source.querySettings, @@ -106,135 +135,151 @@ async function getServiceMapQuery({ condition: `${source.spanKindExpression} IN ('Client', 'Producer', 'SPAN_KIND_CLIENT', 'SPAN_KIND_PRODUCER')`, }, ], - where: '', }, metadata, source.querySettings, ), ]); - // Left join to support services which receive requests from clients that are not instrumented. - // Ordering helps ensure stable graph layout. + const serviceNameInList = serviceNames?.length + ? { UNSAFE_RAW_SQL: serviceNames.map(s => SqlString.escape(s)).join(', ') } + : null; + const serviceNameFilter = serviceNameInList + ? chSql`AND ( + ServerSpans.serviceName IN (${serviceNameInList}) + OR ClientSpans.serviceName IN (${serviceNameInList}) + )` + : chSql``; + + // Latency percentiles over the server spans. Empty fragment when the source + // has no duration expression. Percentiles can't be combined client-side, so + // they're computed server-side per grouping set (see GROUPING SETS below). + // One quantiles(...) call maintains a single reservoir sketch for all three + // percentiles, ~3x cheaper than three separate quantile() aggregations. + const latencySelect = source.durationExpression + ? chSql`, + quantiles(0.5, 0.95, 0.99)(ServerSpans.duration) as quantiles` + : chSql``; + + // Left join to support services which receive requests from clients that are + // not instrumented. GROUPING SETS emits, in one pass, both a per-edge row + // (server + client) and a rolled-up per-service node row (server only); + // GROUPING() flags which is which so node and edge percentiles are each + // computed over the right set of spans. Ordering helps stable graph layout. return chSql` - WITH + WITH ServerSpans AS (${serverCTE}), ClientSpans AS (${clientCTE}) SELECT ServerSpans.serviceName AS serverServiceName, - ServerSpans.statusCode AS serverStatusCode, ClientSpans.serviceName AS clientServiceName, - count(*) * ${{ Int64: effectiveSamplingLevel }} as requestCount + GROUPING(ClientSpans.serviceName) AS isNodeLevel, + count(*) * ${{ Int64: effectiveSamplingLevel }} as requestCount, + countIf(ServerSpans.statusCode = 'Error') * ${{ Int64: effectiveSamplingLevel }} as errorCount${latencySelect} FROM ServerSpans LEFT JOIN ClientSpans ON ServerSpans.traceId = ClientSpans.traceId AND ServerSpans.parentSpanId = ClientSpans.spanId WHERE (ClientSpans.serviceName IS NULL OR ServerSpans.serviceName != ClientSpans.serviceName) - GROUP BY serverServiceName, serverStatusCode, clientServiceName - ORDER BY serverServiceName, serverStatusCode, clientServiceName + ${serviceNameFilter} + GROUP BY GROUPING SETS ( + (ServerSpans.serviceName, ClientSpans.serviceName), + (ServerSpans.serviceName) + ) + ORDER BY serverServiceName, isNodeLevel, clientServiceName `; } -type IncomingRequestStats = { +export type IncomingRequestStats = { totalRequests: number; - requestCountByStatus: Map; + errorCount: number; errorPercentage: number; + // Latency percentiles in raw duration units. Convert with + // rawDurationToMs(value, source.durationPrecision) for display. + p50: number; + p95: number; + p99: number; + // Whether latency percentiles are available (source has a duration column). + hasLatency: boolean; }; export type ServiceAggregation = { serviceName: string; incomingRequests: IncomingRequestStats; incomingRequestsByClient: Map; + // Total requests this service makes to others (sum of edges where it is the + // client). Combined with incoming traffic to size the node by total + // throughput. + outgoingRequests: number; }; +function emptyStats(): IncomingRequestStats { + return { + totalRequests: 0, + errorCount: 0, + errorPercentage: 0, + p50: 0, + p95: 0, + p99: 0, + hasLatency: false, + }; +} + +function statsFromRow(row: SpanAggregationRow): IncomingRequestStats { + return { + totalRequests: row.requestCount, + errorCount: row.errorCount, + errorPercentage: + row.requestCount > 0 ? (row.errorCount / row.requestCount) * 100 : 0, + p50: row.p50 ?? 0, + p95: row.p95 ?? 0, + p99: row.p99 ?? 0, + hasLatency: row.p50 != null, + }; +} + export function aggregateServiceMapData(data: SpanAggregationRow[]) { - // Aggregate data by service const services = new Map(); - for (const row of data) { - const { - serverServiceName, - serverStatusCode, - clientServiceName, - requestCount, - } = row; - if (!services.has(serverServiceName)) { - services.set(serverServiceName, { - serviceName: serverServiceName, - incomingRequests: { - totalRequests: 0, - requestCountByStatus: new Map(), - errorPercentage: 0, - }, + const ensureService = (name: string): ServiceAggregation => { + let service = services.get(name); + if (!service) { + service = { + serviceName: name, + incomingRequests: emptyStats(), incomingRequestsByClient: new Map(), - }); + outgoingRequests: 0, + }; + services.set(name, service); } + return service; + }; - const service = services.get(serverServiceName)!; - - // Add to total incoming request count - service.incomingRequests.totalRequests += requestCount; - - // Add to request count per status - const currentStatusCount = - service.incomingRequests.requestCountByStatus.get(serverStatusCode) || 0; - service.incomingRequests.requestCountByStatus.set( - serverStatusCode, - currentStatusCount + requestCount, - ); - - // Add to request count per client per status - if (clientServiceName) { - if (!service.incomingRequestsByClient.has(clientServiceName)) { - service.incomingRequestsByClient.set(clientServiceName, { - totalRequests: 0, - requestCountByStatus: new Map(), - errorPercentage: 0, - }); - } - - const perClientStats = - service.incomingRequestsByClient.get(clientServiceName)!; - perClientStats.totalRequests += requestCount; + for (const row of data) { + const service = ensureService(row.serverServiceName); - const currentClientStatusCount = - perClientStats.requestCountByStatus.get(serverStatusCode) || 0; - perClientStats.requestCountByStatus.set( - serverStatusCode, - currentClientStatusCount + requestCount, + if (row.isNodeLevel) { + // Rolled-up totals across all callers for this service. + service.incomingRequests = statsFromRow(row); + } else if (row.clientServiceName) { + // Per-caller (edge) stats. Ensure the caller exists as a node too. + ensureService(row.clientServiceName); + service.incomingRequestsByClient.set( + row.clientServiceName, + statsFromRow(row), ); - - if (!services.has(clientServiceName)) { - services.set(clientServiceName, { - serviceName: clientServiceName, - incomingRequests: { - totalRequests: 0, - requestCountByStatus: new Map(), - errorPercentage: 0, - }, - incomingRequestsByClient: new Map(), - }); - } } + // Edge-level rows with no client service come from uninstrumented callers; + // they're already included in the node-level totals, so no edge is drawn. } - // Calculate error percentages for all services and their client stats + // Roll each edge's volume up to its caller's outgoing total. for (const service of services.values()) { - // Calculate error percentage for total incoming requests - const errorCount = - service.incomingRequests.requestCountByStatus.get('Error') || 0; - service.incomingRequests.errorPercentage = - service.incomingRequests.totalRequests > 0 - ? (errorCount / service.incomingRequests.totalRequests) * 100 - : 0; - - // Calculate error percentage for each client - for (const clientStats of service.incomingRequestsByClient.values()) { - const clientErrorCount = - clientStats.requestCountByStatus.get('Error') || 0; - clientStats.errorPercentage = - clientStats.totalRequests > 0 - ? (clientErrorCount / clientStats.totalRequests) * 100 - : 0; + for (const [clientName, stats] of service.incomingRequestsByClient) { + const client = services.get(clientName); + if (client) { + client.outgoingRequests += stats.totalRequests; + } } } @@ -246,17 +291,32 @@ export default function useServiceMap({ dateRange, traceId, samplingFactor, + where, + whereLanguage, + serviceNames, }: { source: TTraceSource; dateRange: [Date, Date]; traceId?: string; samplingFactor: number; + where?: string; + whereLanguage?: 'sql' | 'lucene'; + serviceNames?: string[]; }) { const client = useClickhouseClient(); const metadata = useMetadataWithSettings(); return useQuery({ - queryKey: ['serviceMapData', traceId, source, dateRange, samplingFactor], + queryKey: [ + 'serviceMapData', + traceId, + source, + dateRange, + samplingFactor, + where, + whereLanguage, + serviceNames, + ], queryFn: async ({ signal }) => { const query = await getServiceMapQuery({ source, @@ -264,6 +324,9 @@ export default function useServiceMap({ traceId, metadata, samplingFactor, + where, + whereLanguage, + serviceNames, }); const data = await client @@ -278,14 +341,28 @@ export default function useServiceMap({ join_algorithm: 'auto', }, }) - .then(res => res.json>()) + .then(res => res.json>()) .then(data => - data.data.map((row: Record) => ({ - serverServiceName: row.serverServiceName, - serverStatusCode: row.serverStatusCode, - clientServiceName: row.clientServiceName, - requestCount: Number.parseInt(row.requestCount), - })), + data.data.map((row: Record) => { + // quantiles(...) returns a [p50, p95, p99] array; absent when the + // source has no duration expression. + const quantiles = Array.isArray(row.quantiles) + ? (row.quantiles as unknown[]) + : undefined; + return { + serverServiceName: row.serverServiceName as string, + // serviceName is a non-nullable LowCardinality(String), so + // rolled-up node-level rows and unmatched LEFT JOINs come back as + // '' rather than null — normalize those to undefined (no edge). + clientServiceName: (row.clientServiceName as string) || undefined, + isNodeLevel: Number(row.isNodeLevel) === 1, + requestCount: Number.parseInt(row.requestCount as string, 10), + errorCount: Number.parseInt(row.errorCount as string, 10), + p50: quantiles ? Number(quantiles[0]) : undefined, + p95: quantiles ? Number(quantiles[1]) : undefined, + p99: quantiles ? Number(quantiles[2]) : undefined, + }; + }), ); return aggregateServiceMapData(data); From 747352f3202646992e28293f9f8068047a1f8e24 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:59:15 -0400 Subject: [PATCH 019/728] feat(common-utils): apply direct_read KV items optimization to SQL filters (#2405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary SQL `type: 'sql'` filters on Map columns now get the same has()/hasAny() rewrite that Lucene filters already use when a KV items column with a text(tokenizer=array) skip index exists. - Map['k'] = 'v' → has(Items, concat('k', '=', 'v')) - Map['k'] IN ('a','b') → hasAny(Items, array(concat(...), ...)) Extracts buildKvItemsLookup from CustomSchemaSQLSerializerV2 into a shared export. Removes debug console.log('AVK filter'). ### References This accomplishes the `direct_read` optimization for filters instead of the approach reverted by https://github.com/hyperdxio/hyperdx/pull/2401 --- .changeset/many-comics-rule.md | 5 + packages/common-utils/CHANGELOG.md | 18 + .../src/__tests__/renderChartConfig.test.ts | 148 ++++++++ .../rewriteSqlFilterWithKvItems.test.ts | 327 ++++++++++++++++++ .../src/core/renderChartConfig.ts | 158 ++++++++- packages/common-utils/src/queryParser.ts | 160 +++++---- 6 files changed, 741 insertions(+), 75 deletions(-) create mode 100644 .changeset/many-comics-rule.md create mode 100644 packages/common-utils/src/__tests__/rewriteSqlFilterWithKvItems.test.ts diff --git a/.changeset/many-comics-rule.md b/.changeset/many-comics-rule.md new file mode 100644 index 0000000000..26eae6d49e --- /dev/null +++ b/.changeset/many-comics-rule.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": patch +--- + +feat: add direct_read optimization for filters diff --git a/packages/common-utils/CHANGELOG.md b/packages/common-utils/CHANGELOG.md index f75e287c9b..450dcf1b0e 100644 --- a/packages/common-utils/CHANGELOG.md +++ b/packages/common-utils/CHANGELOG.md @@ -4,6 +4,24 @@ ### Minor Changes +- feat: apply direct_read KV items optimization to SQL filters + + SQL `type: 'sql'` filters on Map columns (e.g. + `LogAttributes['key'] IN ('a', 'b')`) now get the same `has()` / + `hasAny()` rewrite that Lucene filters already use. When a KV items + column with a `text(tokenizer=array)` skip index exists for the Map, + the condition is rewritten at the filter site before rendering: + + - `Map['k'] = 'v'` → `has(Items, concat('k', '=', 'v'))` + - `Map['k'] IN ('a', 'b')` → `hasAny(Items, array(concat('k', '=', 'a'), concat('k', '=', 'b')))` + + Empty-string values are left unrewritten to preserve ClickHouse's + missing-key semantics (`Map(String, String)['absent'] = ''`). + + Also extracts `buildKvItemsLookup` from `CustomSchemaSQLSerializerV2` + into a shared top-level export so both the Lucene serializer and the + SQL filter rewriter can use the same lookup logic. + - 3123db53: feat: experimental promql support - dcab1cb6: feat: default the direct_read map column optimization on supported ClickHouse versions diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 7fe70ca760..164d43292d 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -984,6 +984,154 @@ describe('renderChartConfig', () => { }); }); + describe('SQL filter KV items direct_read optimization', () => { + const stubKvItemsMetadata = () => { + mockMetadata.getColumns = jest.fn().mockResolvedValue([ + { + name: 'LogAttributes', + type: 'Map(String, String)', + default_type: '', + default_expression: '', + }, + { + name: 'LogAttributeItems', + type: 'Array(String)', + default_type: 'MATERIALIZED', + default_expression: + "arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String)))", + }, + ]); + mockMetadata.getSkipIndices = jest.fn().mockResolvedValue([ + { + name: 'idx_log_attr_items', + type: 'text', + typeFull: 'text(tokenizer=array)', + expression: 'LogAttributeItems', + granularity: 10000000, + }, + ]); + mockMetadata.getServerVersion = jest + .fn() + .mockResolvedValue([26, 5, 0, 0]); + mockMetadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockResolvedValue(new Map()); + }; + + const buildConfig = (condition: string): ChartConfigWithOptDateRange => ({ + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + whereLanguage: 'sql', + filters: [{ type: 'sql', condition }], + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-01-01'), new Date('2025-01-02')], + granularity: '1 minute', + }); + + it('rewrites `Map[key] = value` to has() when a KV items column exists', async () => { + stubKvItemsMetadata(); + const sql = parameterizedQueryToSql( + await renderChartConfig( + buildConfig("LogAttributes['service.name'] = 'api'"), + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + "has(`LogAttributeItems`, concat('service.name', '=', 'api'))", + ); + expect(sql).not.toContain("LogAttributes['service.name'] = 'api'"); + }); + + it('rewrites `Map[key] IN (one)` to has()', async () => { + stubKvItemsMetadata(); + const sql = parameterizedQueryToSql( + await renderChartConfig( + buildConfig("LogAttributes['service.name'] IN ('api')"), + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + "has(`LogAttributeItems`, concat('service.name', '=', 'api'))", + ); + }); + + it('rewrites `Map[key] IN (many)` to hasAny(... array(...))', async () => { + stubKvItemsMetadata(); + const sql = parameterizedQueryToSql( + await renderChartConfig( + buildConfig("LogAttributes['k'] IN ('a', 'b', 'c')"), + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + "hasAny(`LogAttributeItems`, array(concat('k', '=', 'a'), concat('k', '=', 'b'), concat('k', '=', 'c')))", + ); + }); + + it('leaves the condition unchanged when no KV items column exists', async () => { + mockMetadata.getColumns = jest.fn().mockResolvedValue([ + { + name: 'LogAttributes', + type: 'Map(String, String)', + default_type: '', + default_expression: '', + }, + ]); + mockMetadata.getSkipIndices = jest.fn().mockResolvedValue([]); + mockMetadata.getServerVersion = jest + .fn() + .mockResolvedValue([26, 5, 0, 0]); + mockMetadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockResolvedValue(new Map()); + + const sql = parameterizedQueryToSql( + await renderChartConfig( + buildConfig("LogAttributes['k'] = 'v'"), + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain("LogAttributes['k'] = 'v'"); + expect(sql).not.toContain('has('); + }); + + it('does not rewrite when value is empty (Map[k]= preserves missing-key semantics)', async () => { + stubKvItemsMetadata(); + const sql = parameterizedQueryToSql( + await renderChartConfig( + buildConfig("LogAttributes['k'] = ''"), + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain("LogAttributes['k'] = ''"); + expect(sql).not.toContain('has('); + }); + + it('rewrites only the matching Map subscript in a compound AND condition', async () => { + stubKvItemsMetadata(); + const sql = parameterizedQueryToSql( + await renderChartConfig( + buildConfig( + "LogAttributes['service.name'] = 'api' AND SeverityText = 'error'", + ), + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + "has(`LogAttributeItems`, concat('service.name', '=', 'api'))", + ); + expect(sql).toContain("SeverityText = 'error'"); + }); + }); + describe('k8s semantic convention migrations', () => { it('should generate SQL with metricNameSql for k8s.pod.cpu.utilization gauge metric', async () => { const config: ChartConfigWithOptDateRange = { diff --git a/packages/common-utils/src/__tests__/rewriteSqlFilterWithKvItems.test.ts b/packages/common-utils/src/__tests__/rewriteSqlFilterWithKvItems.test.ts new file mode 100644 index 0000000000..5ed0d03523 --- /dev/null +++ b/packages/common-utils/src/__tests__/rewriteSqlFilterWithKvItems.test.ts @@ -0,0 +1,327 @@ +import { rewriteSqlFilterWithKvItems } from '@/core/renderChartConfig'; +import { KvItemsLookup } from '@/queryParser'; + +const makeLookup = ( + entries: Array<[string, { kvItemsColumn: string; separator: string }]>, +): KvItemsLookup => new Map(entries); + +const defaultLookup: KvItemsLookup = makeLookup([ + ['LogAttributes', { kvItemsColumn: 'LogAttributeItems', separator: '=' }], +]); + +describe('rewriteSqlFilterWithKvItems', () => { + describe('early returns', () => { + it('returns the condition verbatim when the lookup is empty', () => { + expect( + rewriteSqlFilterWithKvItems('this is not valid SQL', new Map()), + ).toBe('this is not valid SQL'); + + expect( + rewriteSqlFilterWithKvItems("LogAttributes['k'] = 'v'", new Map()), + ).toBe("LogAttributes['k'] = 'v'"); + }); + + it('returns the condition verbatim when the SQL fails to parse', () => { + const condition = 'this is not valid SQL ???'; + expect(rewriteSqlFilterWithKvItems(condition, defaultLookup)).toBe( + condition, + ); + }); + + it('returns the condition verbatim for an empty condition string', () => { + expect(rewriteSqlFilterWithKvItems('', defaultLookup)).toBe(''); + }); + }); + + describe('= operator', () => { + it("rewrites Map['key'] = 'value' to has(kvItems, concat(key, sep, value))", () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['service.name'] = 'api'", + defaultLookup, + ); + expect(result).toBe( + "has(`LogAttributeItems`, concat('service.name', '=', 'api'))", + ); + }); + + it('does not rewrite when the value is an empty string', () => { + // Map(String, String) subscript defaults to '' for absent keys, so + // `Map['k'] = ''` would silently match records where 'k' is unset if + // rewritten to has(items, 'k='). Same rationale as the source comment. + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = ''", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).toContain("LogAttributes['k'] = ''"); + }); + + it('does not rewrite when the right side is a numeric literal', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 5", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).toContain("LogAttributes['k'] = 5"); + }); + + it('does not rewrite when the right side is a column reference', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = Severity", + defaultLookup, + ); + expect(result).not.toContain('has('); + }); + + it('does not rewrite when the subscript appears on the right side', () => { + const result = rewriteSqlFilterWithKvItems( + "'api' = LogAttributes['k']", + defaultLookup, + ); + expect(result).not.toContain('has('); + }); + + it('does not rewrite when the map column is not in the lookup', () => { + const result = rewriteSqlFilterWithKvItems( + "ResourceAttributes['k'] = 'v'", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).toContain("ResourceAttributes['k'] = 'v'"); + }); + + it('does not rewrite plain column comparisons (no subscript)', () => { + const result = rewriteSqlFilterWithKvItems( + "Severity = 'error'", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).toContain("Severity = 'error'"); + }); + }); + + describe('IN operator', () => { + it("rewrites Map['key'] IN ('a') (single item) to has(...) not hasAny(...)", () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] IN ('a')", + defaultLookup, + ); + expect(result).toBe("has(`LogAttributeItems`, concat('k', '=', 'a'))"); + }); + + it("rewrites Map['key'] IN ('a','b','c') to hasAny(... array(...))", () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] IN ('a', 'b', 'c')", + defaultLookup, + ); + expect(result).toBe( + "hasAny(`LogAttributeItems`, array(concat('k', '=', 'a'), concat('k', '=', 'b'), concat('k', '=', 'c')))", + ); + }); + + it('does not rewrite when any IN value is an empty string', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] IN ('a', '')", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).not.toContain('hasAny('); + }); + + it('does not rewrite when any IN value is non-string', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] IN ('a', 5)", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).not.toContain('hasAny('); + }); + + it('does not rewrite NOT IN', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] NOT IN ('a', 'b')", + defaultLookup, + ); + expect(result).not.toContain('has('); + expect(result).not.toContain('hasAny('); + }); + }); + + describe('other operators are not rewritten', () => { + it.each([ + ["LogAttributes['k'] != 'v'", '!='], + ["LogAttributes['k'] < 'v'", '<'], + ["LogAttributes['k'] > 'v'", '>'], + ["LogAttributes['k'] <= 'v'", '<='], + ["LogAttributes['k'] >= 'v'", '>='], + ["LogAttributes['k'] LIKE '%v%'", 'LIKE'], + ["LogAttributes['k'] BETWEEN 'a' AND 'z'", 'BETWEEN'], + ])('leaves %s untouched (operator: %s)', condition => { + const result = rewriteSqlFilterWithKvItems(condition, defaultLookup); + expect(result).not.toContain('has('); + expect(result).not.toContain('hasAny('); + }); + }); + + describe('compound conditions', () => { + it('rewrites only the matching subscript in an AND chain', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['service.name'] = 'api' AND Severity = 'error'", + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('service.name', '=', 'api'))", + ); + expect(result).toContain("Severity = 'error'"); + expect(result).not.toContain("LogAttributes['service.name']"); + }); + + it('rewrites only the matching subscript in an OR chain', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v' OR Severity = 'error'", + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k', '=', 'v'))", + ); + expect(result).toContain("Severity = 'error'"); + }); + + it('rewrites every matching subscript when several appear together', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['a'] = 'x' AND LogAttributes['b'] = 'y'", + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('a', '=', 'x'))", + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('b', '=', 'y'))", + ); + expect(result).not.toContain("LogAttributes['"); + }); + + it('rewrites subscripts inside nested AND/OR groups', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v' AND (LogAttributes['k2'] = 'v2' OR Severity = 'x')", + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k', '=', 'v'))", + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k2', '=', 'v2'))", + ); + expect(result).toContain("Severity = 'x'"); + }); + + it('mixes = and IN rewrites in the same condition', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v' AND LogAttributes['env'] IN ('prod', 'staging')", + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k', '=', 'v'))", + ); + expect(result).toContain( + "hasAny(`LogAttributeItems`, array(concat('env', '=', 'prod'), concat('env', '=', 'staging')))", + ); + }); + }); + + describe('lookup configuration', () => { + it('uses the configured separator in the rewritten concat', () => { + const colonLookup = makeLookup([ + [ + 'LogAttributes', + { kvItemsColumn: 'LogAttributeItems', separator: ':' }, + ], + ]); + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v'", + colonLookup, + ); + expect(result).toBe("has(`LogAttributeItems`, concat('k', ':', 'v'))"); + }); + + it('uses the configured kv items column name (backtick-quoted)', () => { + const lookup = makeLookup([ + [ + 'LogAttributes', + { kvItemsColumn: 'CustomItemsColumn', separator: '=' }, + ], + ]); + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v'", + lookup, + ); + expect(result).toBe("has(`CustomItemsColumn`, concat('k', '=', 'v'))"); + }); + + it('applies independent lookup entries to each map column', () => { + const lookup = makeLookup([ + [ + 'LogAttributes', + { kvItemsColumn: 'LogAttributeItems', separator: '=' }, + ], + [ + 'ResourceAttributes', + { kvItemsColumn: 'ResourceAttributeItems', separator: ':' }, + ], + ]); + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v' AND ResourceAttributes['k2'] = 'v2'", + lookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k', '=', 'v'))", + ); + expect(result).toContain( + "has(`ResourceAttributeItems`, concat('k2', ':', 'v2'))", + ); + }); + + it('rewrites only map columns present in the lookup', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k'] = 'v' AND OtherMap['k2'] = 'v2'", + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k', '=', 'v'))", + ); + expect(result).toContain("OtherMap['k2'] = 'v2'"); + }); + }); + + describe('edge cases', () => { + it('preserves whitespace and special characters in map keys', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['key with spaces'] = 'value'", + defaultLookup, + ); + expect(result).toBe( + "has(`LogAttributeItems`, concat('key with spaces', '=', 'value'))", + ); + }); + + it('does not rewrite chained subscripts (Map[a][b])', () => { + const result = rewriteSqlFilterWithKvItems( + "LogAttributes['k']['k2'] = 'v'", + defaultLookup, + ); + expect(result).not.toContain('has('); + }); + + it('is idempotent on an already-rewritten has() condition', () => { + const alreadyRewritten = + "has(`LogAttributeItems`, concat('k', '=', 'v'))"; + const result = rewriteSqlFilterWithKvItems( + alreadyRewritten, + defaultLookup, + ); + expect(result).toContain( + "has(`LogAttributeItems`, concat('k', '=', 'v'))", + ); + }); + }); +}); diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 47e53bd649..b399a139fb 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -22,7 +22,12 @@ import { isRawSqlChartConfig, } from '@/guards'; import { replaceMacros } from '@/macros'; -import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser'; +import { + buildKvItemsLookup, + CustomSchemaSQLSerializerV2, + KvItemsLookup, + SearchQueryBuilder, +} from '@/queryParser'; import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@/rawSqlParams'; import { AggregateFunction, @@ -338,6 +343,136 @@ const fastifySQL = ({ } }; +export const rewriteSqlFilterWithKvItems = ( + condition: string, + kvItemsLookup: KvItemsLookup, +): string => { + if (kvItemsLookup.size === 0) return condition; + try { + const parser = new SQLParser.Parser(); + const prefix = 'SELECT 1 FROM `t` WHERE '; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + const ast = parser.astify(`${prefix}${condition}`, { + database: 'Postgresql', + }) as SQLParser.Select; + + const tryOptimize = ( + node: SQLParser.ExpressionValue | SQLParser.ExprList, + ): void => { + if (!('operator' in node)) return; + const op = String(node.operator ?? '').toUpperCase(); + if (op !== '=' && op !== 'IN') return; + const left = node.left; + if ( + left?.type !== 'column_ref' || + ('column' in left && typeof left.column === 'string') + ) { + return; + } + const mapColumn = left['column']?.expr?.value; + const arrIdx = left['array_index']; + if ( + typeof mapColumn !== 'string' || + !Array.isArray(arrIdx) || + arrIdx.length !== 1 + ) { + return; + } + const idxNode = arrIdx[0]?.index; + if ( + idxNode?.type !== 'single_quote_string' || + typeof idxNode.value !== 'string' + ) { + return; + } + const mapKey: string = idxNode.value; + const info = kvItemsLookup.get(mapColumn); + if (!info) return; + + let values: string[]; + if (op === '=') { + const right = node.right; + if ( + right?.type !== 'single_quote_string' || + typeof right.value !== 'string' + ) { + return; + } + values = [right.value]; + } else { + const right = node.right; + if (right?.type !== 'expr_list' || !Array.isArray(right.value)) return; + const collected: string[] = []; + for (const item of right.value) { + if ( + item?.type !== 'single_quote_string' || + typeof item.value !== 'string' + ) { + return; + } + collected.push(item.value); + } + values = collected; + } + // Bail on empty values: `Map['k']='' ` also matches absent keys because + // Map(String, String)'s subscript default is '', which `has(items, 'k=')` + // alone does not preserve. Same rationale for empty entries in IN lists. + if (values.length === 0 || values.some(v => v === '')) return; + + const replacement = + values.length === 1 + ? SqlString.format('has(??, concat(?, ?, ?))', [ + info.kvItemsColumn, + mapKey, + info.separator, + values[0], + ]) + : `hasAny(${SqlString.format('??', [ + info.kvItemsColumn, + ])}, array(${values + .map(v => + SqlString.format('concat(?, ?, ?)', [ + mapKey, + info.separator, + v, + ]), + ) + .join(', ')}))`; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- astify returns union type, we expect Select + const replAst = parser.astify(`${prefix}${replacement}`, { + database: 'Postgresql', + }) as SQLParser.Select; + const newWhere = replAst.where; + if (newWhere == null) return; + for (const k of Object.keys(node)) delete node[k]; + Object.assign(node, newWhere); + }; + + const traverse = ( + node: SQLParser.ExpressionValue | SQLParser.ExprList | null, + ): void => { + if (node == null) return; + if (node.type === 'binary_expr') { + if ('left' in node) { + traverse(node.left); + } + if ('right' in node) { + traverse(node.right); + } + tryOptimize(node); + } else if (node.type === 'expr_list' && Array.isArray(node.value)) { + node.value.forEach(traverse); + } + }; + traverse(ast.where); + + return parser.sqlify(ast).slice(prefix.length); + } catch { + return condition; + } +}; + const aggFnExpr = ({ fn, expr, @@ -966,6 +1101,21 @@ async function renderWhere( ).filter(v => v !== null) as ChSql[]; } + const hasSqlFilter = + chartConfig.filters?.some(f => f.type === 'sql') ?? false; + const kvItemsLookup: KvItemsLookup = + hasSqlFilter && + chartConfig.from.databaseName && + chartConfig.from.tableName && + !hasSubqueryCte(chartConfig.with) + ? await buildKvItemsLookup({ + metadata, + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + connectionId: chartConfig.connection, + }) + : new Map(); + const filterConditions = await Promise.all( (chartConfig.filters ?? []).map(async filter => { if (filter.type === 'sql_ast') { @@ -975,9 +1125,13 @@ async function renderWhere( ')', ); } else if (filter.type === 'lucene' || filter.type === 'sql') { + const condition = + filter.type === 'sql' + ? rewriteSqlFilterWithKvItems(filter.condition, kvItemsLookup) + : filter.condition; return wrapChSqlIfNotEmpty( await renderWhereExpression({ - condition: filter.condition, + condition, from: chartConfig.from, language: filter.type, implicitColumnExpression: chartConfig.implicitColumnExpression, diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index f96ffb635b..595a334dda 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -1075,6 +1075,85 @@ const KV_ITEMS_STRATEGIES = [ parseKvItemsCastExpression, ] as const; +/** + * Builds a lookup from map column name to KV items column name. + * A KV items column is an ALIAS/MATERIALIZED column whose expression is + * arrayMap((k,v)->concat(k,'=',v), mapKeys(X), mapValues(X)) and which has + * a text(tokenizer=array) skip index. + * + * The version gate (`supportsDirectReadMap`) only applies to ALIAS items + * columns: ALIAS columns are computed at query time, so `has(items, ...)` + * against an ALIAS only realizes its speedup when the server can perform a + * direct_read against the underlying Map's tuple storage. MATERIALIZED items + * columns are physically stored on disk, so `has()` reads them directly and + * is fast on any ClickHouse version that supports the text index itself. + * + * Returns an empty Map on any failure; never throws. + */ +export async function buildKvItemsLookup({ + metadata, + databaseName, + tableName, + connectionId, +}: { + metadata: Metadata; + databaseName: string; + tableName: string; + connectionId: string; +}): Promise { + const lookup: KvItemsLookup = new Map(); + try { + const [serverVersion, columns, skipIndices] = await Promise.all([ + metadata.getServerVersion({ connectionId }), + metadata.getColumns({ databaseName, tableName, connectionId }), + metadata + .getSkipIndices({ databaseName, tableName, connectionId }) + .catch(() => [] as SkipIndexMetadata[]), + ]); + + const directReadSupported = supportsDirectReadMap(serverVersion); + + const kvItemsCandidates = columns.filter( + c => + (c.default_type === 'ALIAS' || c.default_type === 'MATERIALIZED') && + c.default_expression, + ); + + for (const candidate of kvItemsCandidates) { + if (candidate.default_type === 'ALIAS' && !directReadSupported) { + continue; + } + + let parsed: { mapColumn: string; separator: string } | undefined; + for (const strategy of KV_ITEMS_STRATEGIES) { + parsed = strategy(candidate.default_expression); + if (parsed) break; + } + if (!parsed) continue; + + const candidateName = normalizeChExpression(candidate.name); + const candidateExpr = normalizeChExpression(candidate.default_expression); + const hasArrayTextIndex = skipIndices.some(idx => { + if (idx.type !== 'text') return false; + const tokenizer = parseTokenizerFromTextIndex(idx); + if (tokenizer?.type !== 'array') return false; + const idxExpr = normalizeChExpression(idx.expression); + return idxExpr === candidateName || idxExpr === candidateExpr; + }); + + if (hasArrayTextIndex) { + lookup.set(parsed.mapColumn, { + kvItemsColumn: candidate.name, + separator: parsed.separator, + }); + } + } + } catch (error) { + console.warn('Error building KV items lookup:', error); + } + return lookup; +} + export class CustomSchemaSQLSerializerV2 extends SQLSerializer { private metadata: Metadata; private tableName: string; @@ -1130,81 +1209,16 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { return false; }); - // Pre-fetch KV items lookup (map column -> KV items column with text(tokenizer=array) index) - this.kvItemsLookupPromise = this.buildKvItemsLookup().catch(error => { - console.warn('Error building KV items lookup:', error); - return new Map(); - }); + this.kvItemsLookupPromise = this.buildKvItemsLookup(); } - /** - * Builds a lookup from map column name to KV items column name. - * A KV items column is an ALIAS/MATERIALIZED column whose expression is - * arrayMap((k,v)->concat(k,'=',v), mapKeys(X), mapValues(X)) and which has - * a text(tokenizer=array) skip index. - * - * The version gate (`supportsDirectReadMap`) only applies to ALIAS items - * columns: ALIAS columns are computed at query time, so `has(items, ...)` - * against an ALIAS only realizes its speedup when the server can perform a - * direct_read against the underlying Map's tuple storage. MATERIALIZED items - * columns are physically stored on disk, so `has()` reads them directly and - * is fast on any ClickHouse version that supports the text index itself. - */ - private async buildKvItemsLookup(): Promise { - const [serverVersion, columns, skipIndices] = await Promise.all([ - this.metadata.getServerVersion({ connectionId: this.connectionId }), - this.metadata.getColumns({ - databaseName: this.databaseName, - tableName: this.tableName, - connectionId: this.connectionId, - }), - this.skipIndicesPromise ?? Promise.resolve([]), - ]); - - const directReadSupported = supportsDirectReadMap(serverVersion); - - const lookup: KvItemsLookup = new Map(); - - const kvItemsCandidates = columns.filter( - c => - (c.default_type === 'ALIAS' || c.default_type === 'MATERIALIZED') && - c.default_expression, - ); - - for (const candidate of kvItemsCandidates) { - if (candidate.default_type === 'ALIAS' && !directReadSupported) { - continue; - } - - const parsed = (() => { - let parsed: { mapColumn: string; separator: string } | undefined; - for (const strategy of KV_ITEMS_STRATEGIES) { - parsed = strategy(candidate.default_expression); - if (parsed) break; - } - return parsed; - })(); - if (!parsed) continue; - - const candidateName = normalizeChExpression(candidate.name); - const candidateExpr = normalizeChExpression(candidate.default_expression); - const hasArrayTextIndex = skipIndices.some(idx => { - if (idx.type !== 'text') return false; - const tokenizer = parseTokenizerFromTextIndex(idx); - if (tokenizer?.type !== 'array') return false; - const idxExpr = normalizeChExpression(idx.expression); - return idxExpr === candidateName || idxExpr === candidateExpr; - }); - - if (hasArrayTextIndex) { - lookup.set(parsed.mapColumn, { - kvItemsColumn: candidate.name, - separator: parsed.separator, - }); - } - } - - return lookup; + private buildKvItemsLookup(): Promise { + return buildKvItemsLookup({ + metadata: this.metadata, + databaseName: this.databaseName, + tableName: this.tableName, + connectionId: this.connectionId, + }); } /** From f6bda8c5f5be2b766d7975f578a373924b4f416c Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 4 Jun 2026 09:10:28 -0600 Subject: [PATCH 020/728] refactor: simplify MCP ObjectId validation with shared helpers and schema-level checks (#2412) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Simplifies ObjectId validation across MCP tool handlers by introducing shared utilities and moving validation into Zod schemas where possible. ## Why There were 11 identical inline ObjectId validation blocks (5 lines each) spread across 8 MCP handler files, all constructing the same `{ isError: true, content: [...] }` error object. No shared helpers existed for MCP error responses. ## How **Hybrid approach: schema-level + shared helpers** ### New utility: `packages/api/src/mcp/utils/errors.ts` - `mcpError(text)` — builds the standard MCP error response in one call - `validateObjectId(id, label)` — returns an MCP error if invalid, `null` if valid ### Schema-level validation (6 checks eliminated entirely) Moved `objectIdSchema` into Zod `inputSchema` for always-required ID fields so invalid IDs are rejected before the handler runs: - `dashboards/schemas.ts` — `mcpPatchDashboardSchema.dashboardId` - `dashboards/getDashboardTile.ts` — `dashboardId` - `dashboards/deleteDashboard.ts` — `id` - `dashboards/queryTile.ts` — `dashboardId` - `dashboards/saveDashboard.ts` — `id` (optional, for updates) - `savedSearches/schemas.ts` — `sourceId` ### Handler-level validation (5 checks simplified to one-liners) For optional/conditional ID fields (dual list/detail endpoints, update-only paths): - `getDashboard.ts`, `getSavedSearch.ts`, `getAlert.ts` — optional id for detail fetch - `saveSavedSearch.ts` — optional id for update path - `saveAlert.ts` — optional id for update path ### Bonus: `mcpError()` adoption Also replaced several other inline error object literals with `mcpError()` in `saveAlert.ts` and `saveSavedSearch.ts`. The helper can be adopted incrementally across the remaining error sites. ## Stats - **13 files changed**, 67 insertions, 102 deletions (~35 net lines removed) - **11 → 0** inline ObjectId validation blocks (6 moved to schemas, 5 reduced to one-liners) --- .changeset/mcp-objectid-validation.md | 10 +++++++ .../dashboards/getDashboardTile.test.ts | 4 ++- .../src/mcp/__tests__/savedSearches.test.ts | 4 ++- packages/api/src/mcp/tools/alerts/getAlert.ts | 9 ++---- .../api/src/mcp/tools/alerts/saveAlert.ts | 24 +++++---------- .../mcp/tools/dashboards/deleteDashboard.ts | 10 ++----- .../src/mcp/tools/dashboards/getDashboard.ts | 9 ++---- .../mcp/tools/dashboards/getDashboardTile.ts | 11 ++----- .../mcp/tools/dashboards/patchDashboard.ts | 8 ----- .../api/src/mcp/tools/dashboards/queryTile.ts | 11 ++----- .../src/mcp/tools/dashboards/saveDashboard.ts | 12 ++------ .../api/src/mcp/tools/dashboards/schemas.ts | 2 +- .../mcp/tools/savedSearches/getSavedSearch.ts | 10 ++----- .../tools/savedSearches/saveSavedSearch.ts | 23 ++++---------- .../src/mcp/tools/savedSearches/schemas.ts | 10 +++---- packages/api/src/mcp/utils/errors.ts | 30 +++++++++++++++++++ .../utils/__tests__/enhancedErrors.test.ts | 2 +- packages/api/src/utils/zod.ts | 2 +- 18 files changed, 85 insertions(+), 106 deletions(-) create mode 100644 .changeset/mcp-objectid-validation.md create mode 100644 packages/api/src/mcp/utils/errors.ts diff --git a/.changeset/mcp-objectid-validation.md b/.changeset/mcp-objectid-validation.md new file mode 100644 index 0000000000..6087e50143 --- /dev/null +++ b/.changeset/mcp-objectid-validation.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': patch +--- + +refactor(mcp): simplify ObjectId validation with shared helpers and schema-level checks + +Add `mcpError()` and `validateObjectId()` utilities to reduce boilerplate +across MCP tool handlers. Move ObjectId validation into Zod input schemas +for always-required ID fields, eliminating inline checks entirely. Remaining +conditional checks use the new one-liner helper. diff --git a/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts b/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts index 8cd5a655ec..5df31e3edb 100644 --- a/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/getDashboardTile.test.ts @@ -98,6 +98,8 @@ describe('MCP Dashboard Tools - clickstack_get_dashboard_tile', () => { ); expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Invalid dashboard ID'); + const text = getFirstText(result); + expect(text).toContain('Invalid ObjectId'); + expect(text).toContain('dashboardId'); }); }); diff --git a/packages/api/src/mcp/__tests__/savedSearches.test.ts b/packages/api/src/mcp/__tests__/savedSearches.test.ts index 81756fb156..8495ae674b 100644 --- a/packages/api/src/mcp/__tests__/savedSearches.test.ts +++ b/packages/api/src/mcp/__tests__/savedSearches.test.ts @@ -299,7 +299,9 @@ describe('MCP Saved Search Tools', () => { }); expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Invalid sourceId'); + const text = getFirstText(result); + expect(text).toContain('Invalid ObjectId'); + expect(text).toContain('sourceId'); }); it('should include url in response when FRONTEND_URL is set', async () => { diff --git a/packages/api/src/mcp/tools/alerts/getAlert.ts b/packages/api/src/mcp/tools/alerts/getAlert.ts index 6b39655d93..f1cf8eb0c3 100644 --- a/packages/api/src/mcp/tools/alerts/getAlert.ts +++ b/packages/api/src/mcp/tools/alerts/getAlert.ts @@ -12,6 +12,7 @@ import type { IDashboard } from '@/models/dashboard'; import type { ISavedSearch } from '@/models/savedSearch'; import { translateAlertDocumentToExternalAlert } from '@/utils/externalApi'; +import { validateObjectId } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; @@ -108,12 +109,8 @@ export function registerGetAlert(server: McpServer, context: McpContext): void { } // ── Get single alert (full detail) ── - if (!mongoose.Types.ObjectId.isValid(id)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid alert ID' }], - }; - } + const idError = validateObjectId(id, 'alert ID'); + if (idError) return idError; const alert = await getAlertById(id, teamId); if (!alert) { diff --git a/packages/api/src/mcp/tools/alerts/saveAlert.ts b/packages/api/src/mcp/tools/alerts/saveAlert.ts index 88e9b7635c..3642133711 100644 --- a/packages/api/src/mcp/tools/alerts/saveAlert.ts +++ b/packages/api/src/mcp/tools/alerts/saveAlert.ts @@ -13,6 +13,7 @@ import { type AlertChannel, AlertSource } from '@/models/alert'; import { BaseError } from '@/utils/errors'; import { translateAlertDocumentToExternalAlert } from '@/utils/externalApi'; +import { mcpError, validateObjectId } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; import { @@ -53,19 +54,14 @@ export function registerSaveAlert( // ── Runtime cross-field validation ── const validationError = validateSaveAlertInput(input); if (validationError) { - return { - isError: true, - content: [{ type: 'text' as const, text: validationError }], - }; + return mcpError(validationError); } // ── Validate ID for updates (early return narrows input.id to string) ── const alertId = input.id; - if (alertId != null && !mongoose.Types.ObjectId.isValid(alertId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid alert ID' }], - }; + if (alertId != null) { + const idError = validateObjectId(alertId, 'alert ID'); + if (idError) return idError; } // Build the alert input matching the shape expected by controllers. @@ -102,10 +98,7 @@ export function registerSaveAlert( : e instanceof Error ? e.message : String(e); - return { - isError: true, - content: [{ type: 'text' as const, text: msg }], - }; + return mcpError(msg); } const mongoUserId = new mongoose.Types.ObjectId(userId); @@ -114,10 +107,7 @@ export function registerSaveAlert( if (alertId) { const updated = await updateAlert(alertId, mongoTeamId, alertInput); if (!updated) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Alert not found' }], - }; + return mcpError('Alert not found'); } return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts b/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts index 0b00f6a4b1..e6da6a7521 100644 --- a/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts @@ -4,6 +4,7 @@ import { z } from 'zod'; import { deleteDashboard } from '@/controllers/dashboard'; import Dashboard from '@/models/dashboard'; +import { objectIdSchema } from '@/utils/zod'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; @@ -22,20 +23,13 @@ export function registerDeleteDashboard( 'Permanently delete a dashboard by ID. Also removes any alerts attached to its tiles. ' + 'Use clickstack_get_dashboard (without an ID) to list available dashboard IDs.', inputSchema: z.object({ - id: z.string().describe('Dashboard ID to delete.'), + id: objectIdSchema.describe('Dashboard ID to delete.'), }), }, withToolTracing( 'clickstack_delete_dashboard', context, async ({ id: dashboardId }) => { - if (!mongoose.Types.ObjectId.isValid(dashboardId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], - }; - } - const existing = await Dashboard.findOne({ _id: dashboardId, team: teamId, diff --git a/packages/api/src/mcp/tools/dashboards/getDashboard.ts b/packages/api/src/mcp/tools/dashboards/getDashboard.ts index 412b9e17b3..43eaa941f0 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboard.ts @@ -7,6 +7,7 @@ import { getDashboards } from '@/controllers/dashboard'; import Dashboard from '@/models/dashboard'; import { convertToExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; +import { validateObjectId } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; @@ -51,12 +52,8 @@ export function registerGetDashboard( }; } - if (!mongoose.Types.ObjectId.isValid(id)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], - }; - } + const idError = validateObjectId(id, 'dashboard ID'); + if (idError) return idError; const dashboard = await Dashboard.findOne({ _id: id, team: teamId }); if (!dashboard) { diff --git a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts index 03f834809c..921a979b90 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts @@ -1,9 +1,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import mongoose from 'mongoose'; import { z } from 'zod'; import Dashboard from '@/models/dashboard'; import { convertToExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; +import { objectIdSchema } from '@/utils/zod'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; @@ -24,7 +24,7 @@ export function registerGetDashboardTile( 'Use clickstack_get_dashboard (without an ID) to list dashboards, ' + 'then clickstack_get_dashboard (with an ID) to see all tile IDs.', inputSchema: z.object({ - dashboardId: z.string().describe('Dashboard ID.'), + dashboardId: objectIdSchema.describe('Dashboard ID.'), tileId: z .string() .describe( @@ -37,13 +37,6 @@ export function registerGetDashboardTile( 'clickstack_get_dashboard_tile', context, async ({ dashboardId, tileId }) => { - if (!mongoose.Types.ObjectId.isValid(dashboardId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], - }; - } - const dashboard = await Dashboard.findOne({ _id: dashboardId, team: teamId, diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index 0ee13f8cf9..f21782c85d 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -1,6 +1,5 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { uniq } from 'lodash'; -import mongoose from 'mongoose'; import * as config from '@/config'; import Dashboard from '@/models/dashboard'; @@ -70,13 +69,6 @@ export function registerPatchDashboard( }; } - if (!mongoose.Types.ObjectId.isValid(dashboardId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], - }; - } - const existingDashboard = await Dashboard.findOne({ _id: dashboardId, team: teamId, diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index 77b12df45b..5517ab54cf 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -1,9 +1,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import mongoose from 'mongoose'; import { z } from 'zod'; import Dashboard from '@/models/dashboard'; import { convertToExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; +import { objectIdSchema } from '@/utils/zod'; import { withToolTracing } from '../../utils/tracing'; import { parseTimeRange, runConfigTile } from '../query/helpers'; @@ -25,7 +25,7 @@ export function registerQueryTile( 'without rebuilding the query from scratch. ' + 'Use clickstack_get_dashboard with an ID to find tile IDs.', inputSchema: z.object({ - dashboardId: z.string().describe('Dashboard ID.'), + dashboardId: objectIdSchema.describe('Dashboard ID.'), tileId: z .string() .describe( @@ -58,13 +58,6 @@ export function registerQueryTile( } const { startDate, endDate } = timeRange; - if (!mongoose.Types.ObjectId.isValid(dashboardId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], - }; - } - const dashboard = await Dashboard.findOne({ _id: dashboardId, team: teamId, diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index 84f19db437..4fcee6d604 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -21,7 +21,9 @@ import type { ExternalDashboardFilterWithId, ExternalDashboardTileWithId, } from '@/utils/zod'; +import { objectIdSchema } from '@/utils/zod'; +import { mcpError } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; import { mcpContainersParam, mcpFiltersParam, mcpTilesParam } from './schemas'; @@ -45,8 +47,7 @@ export function registerSaveDashboard( 'due to incorrect filter syntax, missing attributes, or wrong column names. ' + 'TIP: To update a single tile without resubmitting all tiles, use clickstack_patch_dashboard instead.', inputSchema: z.object({ - id: z - .string() + id: objectIdSchema .optional() .describe( 'Dashboard ID. Omit to create a new dashboard, provide to update an existing one.', @@ -249,13 +250,6 @@ async function updateDashboard({ | (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] | undefined; }) { - if (!mongoose.Types.ObjectId.isValid(dashboardId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], - }; - } - const parsed = updateDashboardBodySchema.safeParse({ name, tiles: inputTiles, diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index e8c25b8997..a11e02ec80 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -776,7 +776,7 @@ export const mcpFiltersParam = z ); export const mcpPatchDashboardSchema = z.object({ - dashboardId: z.string().describe('Dashboard ID.'), + dashboardId: objectIdSchema.describe('Dashboard ID.'), name: z .string() .min(1) diff --git a/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts b/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts index 663566c691..ddc98427b4 100644 --- a/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts +++ b/packages/api/src/mcp/tools/savedSearches/getSavedSearch.ts @@ -1,11 +1,11 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import mongoose from 'mongoose'; import { z } from 'zod'; import * as config from '@/config'; import { getSavedSearch } from '@/controllers/savedSearch'; import { SavedSearch } from '@/models/savedSearch'; +import { validateObjectId } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; @@ -55,12 +55,8 @@ export function registerGetSavedSearch( } // ── Get single saved search ── - if (!mongoose.Types.ObjectId.isValid(id)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid saved search ID' }], - }; - } + const idError = validateObjectId(id, 'saved search ID'); + if (idError) return idError; const savedSearch = await getSavedSearch(teamId, id); if (!savedSearch) { diff --git a/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts b/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts index b8f81020c2..2e682f137b 100644 --- a/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts +++ b/packages/api/src/mcp/tools/savedSearches/saveSavedSearch.ts @@ -1,5 +1,4 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import mongoose from 'mongoose'; import * as config from '@/config'; import { @@ -9,6 +8,7 @@ import { } from '@/controllers/savedSearch'; import { getSource } from '@/controllers/sources'; +import { mcpError, validateObjectId } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; import { mcpSaveSavedSearchSchema } from './schemas'; @@ -34,26 +34,15 @@ export function registerSaveSavedSearch( const isUpdate = !!input.id; // ── Validate ID for updates ── - if (isUpdate && !mongoose.Types.ObjectId.isValid(input.id!)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid saved search ID' }], - }; + if (isUpdate) { + const idError = validateObjectId(input.id!, 'saved search ID'); + if (idError) return idError; } - // ── Validate sourceId ── - if (!mongoose.Types.ObjectId.isValid(input.sourceId)) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Invalid sourceId' }], - }; - } + // ── Validate sourceId (format validated by Zod schema, check existence) ── const source = await getSource(teamId, input.sourceId); if (!source) { - return { - isError: true, - content: [{ type: 'text' as const, text: 'Source not found' }], - }; + return mcpError('Source not found'); } // Build the saved search data matching what the controller expects. diff --git a/packages/api/src/mcp/tools/savedSearches/schemas.ts b/packages/api/src/mcp/tools/savedSearches/schemas.ts index bd5bbccbb3..41b42d56ca 100644 --- a/packages/api/src/mcp/tools/savedSearches/schemas.ts +++ b/packages/api/src/mcp/tools/savedSearches/schemas.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; +import { objectIdSchema } from '@/utils/zod'; + // --------------------------------------------------------------------------- // MCP-compatible Zod schemas for saved search tools. // --------------------------------------------------------------------------- @@ -57,11 +59,9 @@ export const mcpSaveSavedSearchSchema = z.object({ .string() .optional() .describe('Sort expression. Example: "Timestamp DESC"'), - sourceId: z - .string() - .describe( - 'Source ID — call clickstack_list_sources to find available sources.', - ), + sourceId: objectIdSchema.describe( + 'Source ID — call clickstack_list_sources to find available sources.', + ), tags: z .array(z.string()) .optional() diff --git a/packages/api/src/mcp/utils/errors.ts b/packages/api/src/mcp/utils/errors.ts new file mode 100644 index 0000000000..7c22a217b0 --- /dev/null +++ b/packages/api/src/mcp/utils/errors.ts @@ -0,0 +1,30 @@ +import mongoose from 'mongoose'; + +type McpErrorResult = { + isError: true; + content: [{ type: 'text'; text: string }]; +}; + +/** + * Build a standard MCP error response. + */ +export function mcpError(text: string): McpErrorResult { + return { + isError: true as const, + content: [{ type: 'text' as const, text }], + }; +} + +/** + * Validate that a string is a valid MongoDB ObjectId. + * Returns an MCP error result if invalid, or `null` if valid. + */ +export function validateObjectId( + id: string, + label: string, +): McpErrorResult | null { + if (!mongoose.Types.ObjectId.isValid(id)) { + return mcpError(`Invalid ${label}`); + } + return null; +} diff --git a/packages/api/src/utils/__tests__/enhancedErrors.test.ts b/packages/api/src/utils/__tests__/enhancedErrors.test.ts index 6892600777..517445f661 100644 --- a/packages/api/src/utils/__tests__/enhancedErrors.test.ts +++ b/packages/api/src/utils/__tests__/enhancedErrors.test.ts @@ -166,7 +166,7 @@ describe('enhancedErrors', () => { expect(response.status).toBe(400); expect(response.body.message).toEqual( - "Body validation failed: interval: Invalid enum value. Expected '1m' | '5m' | '15m' | '30m' | '1h' | '6h' | '12h' | '1d', received '99m'; Params validation failed: id: Invalid input", + "Body validation failed: interval: Invalid enum value. Expected '1m' | '5m' | '15m' | '30m' | '1h' | '6h' | '12h' | '1d', received '99m'; Params validation failed: id: Invalid ObjectId", ); }); }); diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index b2d3e53ebb..ac933f706b 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -23,7 +23,7 @@ import { AlertSource } from '@/models/alert'; export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); -}); +}, 'Invalid ObjectId'); // ================================ // Charts & Dashboards (old format) From 8164492f7155409a7f6a90ded11761d0e3435e97 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 4 Jun 2026 11:37:08 -0600 Subject: [PATCH 021/728] fix(mcp): improve alias descriptions and examples for readable chart legends (#2418) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes HDX-4453 ## Problem When the MCP generates dashboard tiles or query charts, series legends often show raw ClickHouse expressions like `count()` or `quantile(0.95)(Duration)` instead of human-readable labels. This happens because: 1. The `alias` field description in the JSON schema was minimal (`"Display label for this series"`) with no urgency to include it 2. All 5 example tiles in `mcpTilesParam` omitted aliases entirely — LLMs copy these patterns ## Changes - **Add aliases to every example tile** in `mcpTilesParam` (except heatmap which doesn't use aliases): `"Errors"`, `"Requests"`, `"Avg Duration"`, `"P95 Latency"` - **Expand the `alias` field description** in both dashboard and query schemas to explain what aliases are used for (legends, table headers, CSV exports, onClick templates) and what happens without them (raw ClickHouse expressions in the UI) These are description-only changes to the Zod schemas — no runtime behavior changes. The JSON schema `description` is what every MCP client sees on every tool call, making this the highest-impact place to guide LLMs toward always including aliases. --- .changeset/mcp-alias-descriptions.md | 5 +++++ .../api/src/mcp/tools/dashboards/schemas.ts | 20 +++++++++++++------ packages/api/src/mcp/tools/query/schemas.ts | 6 +++++- 3 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 .changeset/mcp-alias-descriptions.md diff --git a/.changeset/mcp-alias-descriptions.md b/.changeset/mcp-alias-descriptions.md new file mode 100644 index 0000000000..61cc48e4b8 --- /dev/null +++ b/.changeset/mcp-alias-descriptions.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': patch +--- + +fix(mcp): improve alias field descriptions and examples for readable chart legends diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index a11e02ec80..f88033a5a7 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -98,7 +98,15 @@ const mcpTileSelectItemSchema = z .default('') .describe('Filter in Lucene syntax. Example: "level:error"'), whereLanguage: SearchConditionLanguageSchema.optional().default('lucene'), - alias: z.string().optional().describe('Display label for this series'), + alias: z + .string() + .optional() + .describe( + 'Display label for this series — used in chart legends, table column headers, CSV exports, and onClick templates. ' + + 'Always set a short, human-readable alias (e.g. "Requests", "P95 Latency", "Error Rate"). ' + + 'Without an alias the UI shows the raw ClickHouse expression (e.g. count(), quantile(0.95)(Duration)) which is hard to read. ' + + 'Heatmap select items are the only exception (no alias needed).', + ), level: externalQuantileLevelSchema .optional() .describe('Percentile level for aggFn="quantile"'), @@ -659,15 +667,15 @@ export const mcpTilesParam = z 'The config block varies by displayType – use clickstack_list_sources for sourceId and connectionId values.\n\n' + 'Example tiles:\n' + '1. Line chart: { "name": "Error Rate", "config": { "displayType": "line", "sourceId": "", ' + - '"groupBy": "ResourceAttributes[\'service.name\']", "select": [{ "aggFn": "count", "where": "StatusCode:STATUS_CODE_ERROR" }] } }\n' + + '"groupBy": "ResourceAttributes[\'service.name\']", "select": [{ "aggFn": "count", "where": "StatusCode:STATUS_CODE_ERROR", "alias": "Errors" }] } }\n' + '2. Table: { "name": "Top Endpoints", "config": { "displayType": "table", "sourceId": "", ' + - '"groupBy": "SpanAttributes[\'http.route\']", "select": [{ "aggFn": "count" }, ' + - '{ "aggFn": "avg", "valueExpression": "Duration", "numberFormat": { "output": "duration", "factor": 0.000000001 } }] } }\n' + + '"groupBy": "SpanAttributes[\'http.route\']", "select": [{ "aggFn": "count", "alias": "Requests" }, ' + + '{ "aggFn": "avg", "valueExpression": "Duration", "alias": "Avg Duration", "numberFormat": { "output": "duration", "factor": 0.000000001 } }] } }\n' + ' (per-series numberFormat lets one column render as a duration while a sibling count column stays a plain number)\n' + '3. Number: { "name": "Total Requests", "config": { "displayType": "number", "sourceId": "", ' + - '"select": [{ "aggFn": "count" }], "numberFormat": { "output": "number", "average": true } } }\n' + + '"select": [{ "aggFn": "count", "alias": "Requests" }], "numberFormat": { "output": "number", "average": true } } }\n' + '4. Number (duration): { "name": "P95 Latency", "config": { "displayType": "number", "sourceId": "", ' + - '"select": [{ "aggFn": "quantile", "level": 0.95, "valueExpression": "Duration" }], ' + + '"select": [{ "aggFn": "quantile", "level": 0.95, "valueExpression": "Duration", "alias": "P95 Latency" }], ' + '"numberFormat": { "output": "duration", "factor": 0.000000001 } } }\n' + '5. Heatmap: { "name": "Latency Heatmap", "config": { "displayType": "heatmap", "sourceId": "", ' + '"select": [{ "valueExpression": "Duration" }], ' + diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 19abd9c71c..6bcdd91ab4 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -104,7 +104,11 @@ export const mcpSelectItemSchema = z.object({ alias: z .string() .optional() - .describe('Display label for this series. Example: "Error rate"'), + .describe( + 'Display label for this series — used in chart legends, table column headers, CSV exports, and onClick templates. ' + + 'Always set a short, human-readable alias (e.g. "Requests", "P95 Latency", "Error Rate"). ' + + 'Without an alias the UI shows the raw ClickHouse expression (e.g. count(), quantile(0.95)(Duration)) which is hard to read.', + ), level: z .union([z.literal(0.5), z.literal(0.9), z.literal(0.95), z.literal(0.99)]) .optional() From 53e8bd17f5b23d0ca07feff7fdd14dcdf91a4800 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 4 Jun 2026 14:19:51 -0400 Subject: [PATCH 022/728] fix: Fix height of source select RHS menu (#2419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR ensures the height of the RHS kebab menu on the source select component matches the height of the source select input. ### Screenshots or video Before: Screenshot 2026-06-04 at 1 25 37 PM After: Screenshot 2026-06-04 at 1 31 20 PM Screenshot 2026-06-04 at 1 31 14 PM ### How to test on Vercel preview This can be tested on the preview, wherever the source select is available. ### References - Linear Issue: - Related PRs: --- .changeset/smooth-turkeys-dream.md | 5 +++++ packages/app/src/components/SourceSelect.tsx | 5 ++++- packages/app/styles/SourceSelectControlled.module.scss | 9 ++++----- 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 .changeset/smooth-turkeys-dream.md diff --git a/.changeset/smooth-turkeys-dream.md b/.changeset/smooth-turkeys-dream.md new file mode 100644 index 0000000000..ddfa44ee5a --- /dev/null +++ b/.changeset/smooth-turkeys-dream.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: Fix height of source select RHS menu diff --git a/packages/app/src/components/SourceSelect.tsx b/packages/app/src/components/SourceSelect.tsx index d00e23e656..8b4f1b9410 100644 --- a/packages/app/src/components/SourceSelect.tsx +++ b/packages/app/src/components/SourceSelect.tsx @@ -46,6 +46,7 @@ interface SourceManagementMenuProps { */ onManageSources?: () => void; onCreate?: () => void; + size?: string; } /** @@ -63,6 +64,7 @@ interface SourceManagementMenuProps { */ export const SourceManagementMenu = ({ hasSelection, + size = 'sm', onSchemaPreview, isSchemaPreviewEnabled = true, onEdit, @@ -133,7 +135,7 @@ export const SourceManagementMenu = ({ Date: Fri, 5 Jun 2026 04:14:35 +0900 Subject: [PATCH 023/728] fix: use toJSONString for metric attribute hashing with JSON schema (#2087) (#2118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary When the ClickHouse OTel exporter is configured with `json: true` (`BETA_CH_OTEL_JSON_SCHEMA_ENABLED`), attribute columns (`Attributes`, `ScopeAttributes`, `ResourceAttributes`) are stored as `JSON` type instead of `Map(String, String)`. The existing metric bucketing queries always used `cityHash64(mapConcat(...))`, which fails with: > Function mapConcat requires at least one argument of type Map This fix detects the column type at query time via `metadata.getColumns()` and substitutes `mapConcat` with `toJSONString`-based hashing when JSON schema is active. The detection falls back to the original Map behaviour if the column lookup fails (e.g. table not yet created). Affected query paths: Gauge, Sum, and Histogram (count + quantile) metrics. ## Screenshots or video N/A — backend query generation change only, no UI changes. ## How to test locally 1. Enable JSON schema mode by setting `BETA_CH_OTEL_JSON_SCHEMA_ENABLED=true` in your ClickHouse exporter config (`json: true`). 2. Send metric data via the OTel Collector to populate `otel_metrics_gauge`, `otel_metrics_sum`, and `otel_metrics_histogram` tables. 3. Open the Metrics Explorer and chart any Gauge, Sum, or Histogram metric. 4. Verify the chart renders without the `Function mapConcat requires at least one argument of type Map` error. 5. Without the fix, the same steps produce the error above. ## References - Closes: #2087 - Related upstream OTel Collector bug (attributes stored as `{}`): https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/47582 - Related PR (JSON schema support for K8s Warning Events): #1605 Co-authored-by: Dan Hable <418679+dhable@users.noreply.github.com> --- .../renderChartConfig.test.ts.snap | 223 ++++++++++++++++++ .../src/__tests__/renderChartConfig.test.ts | 207 ++++++++++++++++ packages/common-utils/src/core/histogram.ts | 25 +- .../src/core/renderChartConfig.ts | 52 +++- 4 files changed, 500 insertions(+), 7 deletions(-) diff --git a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap index e97e977c7b..8c1cbb3b97 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -22,6 +22,229 @@ exports[`renderChartConfig HAVING clause should render HAVING clause with multip ),count(),endpoint FROM default.metrics WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY endpoint HAVING avg(response_time) > 500 AND count(*) > 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use direct cityHash64 for gauge metric when Attributes column is JSON type 1`] = ` +"WITH Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM default.otel_metrics_gauge + WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'system.cpu.utilization')) + ),Bucketed AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM Source + GROUP BY AttributesHash, __hdx_time_bucket2 + ORDER BY AttributesHash, __hdx_time_bucket2 + ) SELECT avg( + toFloat64OrDefault(toString(LastValue)) + ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use direct cityHash64 for histogram (count) metric when Attributes column is JSON type 1`] = ` +"WITH source AS ( + SELECT + TimeUnix, + AggregationTemporality, + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`, + + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS attr_hash, + cityHash64(ExplicitBounds) AS bounds_hash, + toInt64(Count) AS current_count, + lagInFrame(toNullable(current_count), 1, NULL) OVER ( + PARTITION BY attr_hash, bounds_hash, AggregationTemporality + ORDER BY TimeUnix + ) AS prev_count, + CASE + WHEN AggregationTemporality = 1 THEN current_count + WHEN AggregationTemporality = 2 THEN greatest(0, current_count - coalesce(prev_count, 0)) + ELSE 0 + END AS delta + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.request.count')) + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + + sum(delta) AS "Value" + FROM source + GROUP BY \`__hdx_time_bucket\` + ) SELECT \`__hdx_time_bucket\`, "Value" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use direct cityHash64 for histogram (quantile) metric when Attributes column is JSON type 1`] = ` +"WITH source AS ( + SELECT + MetricName, + ExplicitBounds, + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`, + + sumForEach(deltas) as rates + FROM ( + SELECT + TimeUnix, + MetricName, + ResourceAttributes, + Attributes, + ExplicitBounds, + attr_hash, + any(attr_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_attr_hash, + any(bounds_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_bounds_hash, + any(counts) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_counts, + counts, + IF( + AggregationTemporality = 1 + OR prev_attr_hash != attr_hash + OR bounds_hash != prev_bounds_hash + OR arrayExists((x) -> x.2 < x.1, arrayZip(prev_counts, counts)), + counts, + counts - prev_counts + ) AS deltas + FROM ( + SELECT + TimeUnix, + MetricName, + AggregationTemporality, + ExplicitBounds, + ResourceAttributes, + Attributes, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS attr_hash, + cityHash64(ExplicitBounds) AS bounds_hash, + CAST(BucketCounts AS Array(Int64)) counts + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.duration')) + ORDER BY attr_hash, TimeUnix ASC + ) + ) + GROUP BY \`__hdx_time_bucket\`, MetricName, ExplicitBounds + ORDER BY \`__hdx_time_bucket\` + ),points AS ( + SELECT + \`__hdx_time_bucket\`, + MetricName, + + arrayZipUnaligned(arrayCumSum(rates), ExplicitBounds) as point, + length(point) as n + FROM source + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + MetricName, + + point[n].1 AS total, + 0.95 * total AS rank, + arrayFirstIndex(x -> if(x.1 > rank, 1, 0), point) AS upper_idx, + point[upper_idx].1 AS upper_count, + ifNull(point[upper_idx].2, inf) AS upper_bound, + CASE + WHEN upper_idx > 1 THEN point[upper_idx - 1].2 + WHEN point[upper_idx].2 > 0 THEN 0 + ELSE inf + END AS lower_bound, + if ( + lower_bound = 0, + 0, + point[upper_idx - 1].1 + ) AS lower_count, + CASE + WHEN upper_bound = inf THEN point[upper_idx - 1].2 + WHEN lower_bound = inf THEN point[1].2 + ELSE lower_bound + (upper_bound - lower_bound) * ((rank - lower_count) / (upper_count - lower_count)) + END AS "Value" + FROM points + WHERE length(point) > 1 AND total > 0 + ) SELECT \`__hdx_time_bucket\`, "Value" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use direct cityHash64 for sum metric when Attributes column is JSON type 1`] = ` +"WITH Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash, + IF( + AggregationTemporality = 1, + Value, -- DELTA: Value is already the per-interval increase + greatest(Value - lagInFrame(toNullable(Value), 1, NULL) OVER (PARTITION BY AttributesHash ORDER BY TimeUnix), 0) + ) AS Rate, + IF( + AggregationTemporality = 1, + SUM(Value) OVER (PARTITION BY AttributesHash ORDER BY TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), + Value + ) AS Sum + FROM default.otel_metrics_sum + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 5 minute) - INTERVAL 5 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 5 minute) + INTERVAL 5 minute) AND ((MetricName = 'db.client.connections.usage'))),Bucketed AS ( + SELECT + \`__hdx_time_bucket2\`, + AttributesHash, + Rate, + Sum, + ResourceAttributes, + ResourceSchemaUrl, + ScopeName, + ScopeVersion, + ScopeAttributes, + ScopeDroppedAttrCount, + ScopeSchemaUrl, + ServiceName, + MetricName, + MetricDescription, + MetricUnit, + Attributes, + StartTimeUnix, + Flags, + AggregationTemporality, + IsMonotonic + FROM ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + -- Per-bucket increase: sum of raw per-row deltas. NULL + -- Source.Rate (first row of each partition) is ignored by sum(). + sum(Source.Rate) AS Rate, + -- Last cumulative reading in the bucket (by time), used by + -- the no-aggFn last_value(Sum) outer projection. argMax is + -- deterministic w.r.t. TimeUnix ordering unlike last_value + -- which in a GROUP BY context is anyLast (order-dependent). + argMax(Source.Sum, TimeUnix) AS Sum, + any(ResourceAttributes) AS ResourceAttributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeAttributes) AS ScopeAttributes, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricName) AS MetricName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(Attributes) AS Attributes, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags, + any(AggregationTemporality) AS AggregationTemporality, + any(IsMonotonic) AS IsMonotonic + FROM Source + GROUP BY AttributesHash, \`__hdx_time_bucket2\` + ORDER BY AttributesHash, \`__hdx_time_bucket2\` + ) + ) SELECT avg( + toFloat64OrDefault(toString(Rate)) + ) AS "Value",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + exports[`renderChartConfig SETTINGS clause should apply the "chart config" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; exports[`renderChartConfig SETTINGS clause should apply the "query settings" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 164d43292d..9816c0f935 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -3141,4 +3141,211 @@ describe('renderChartConfig', () => { expect(sql).toContain('toStartOfInterval(toDateTime(EventTime),'); }); }); + + describe('JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED)', () => { + // When the ClickHouse exporter uses json: true, attribute columns are JSON + // type instead of Map(String, String). mapConcat() fails on JSON columns, so + // cityHash64 receives the JSON columns directly as variadic arguments. + + let jsonSchemaMockMetadata: jest.Mocked; + + beforeEach(() => { + const jsonSchemaColumns = [ + { name: 'TimeUnix', type: 'DateTime64(9)' }, + { name: 'MetricName', type: 'LowCardinality(String)' }, + { name: 'Attributes', type: 'JSON' }, + { name: 'ScopeAttributes', type: 'JSON' }, + { name: 'ResourceAttributes', type: 'JSON' }, + { name: 'Value', type: 'Float64' }, + { name: 'AggregationTemporality', type: 'Int32' }, + ]; + jsonSchemaMockMetadata = { + getColumns: jest.fn().mockResolvedValue(jsonSchemaColumns), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest + .fn() + .mockImplementation(async ({ column }: { column: string }) => + jsonSchemaColumns.find(col => col.name === column), + ), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + }); + + const baseMetricConfig = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { + databaseName: 'default', + tableName: '', + }, + where: '', + whereLanguage: 'sql' as const, + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')] as [ + Date, + Date, + ], + granularity: '1 minute' as const, + limit: { limit: 10 }, + }; + + it('should use direct cityHash64 for gauge metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.cpu.utilization', + metricType: MetricsDataType.Gauge, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain( + 'cityHash64(ScopeAttributes, ResourceAttributes, Attributes)', + ); + expect(actual).not.toContain('toJSONString'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should use direct cityHash64 for sum metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + granularity: '5 minute', + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'db.client.connections.usage', + metricType: MetricsDataType.Sum, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain( + 'cityHash64(ScopeAttributes, ResourceAttributes, Attributes)', + ); + expect(actual).not.toContain('toJSONString'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should use direct cityHash64 for histogram (quantile) metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + granularity: '2 minute', + select: [ + { + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain( + 'cityHash64(ScopeAttributes, ResourceAttributes, Attributes)', + ); + expect(actual).not.toContain('toJSONString'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should use direct cityHash64 for histogram (count) metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + granularity: '2 minute', + select: [ + { + aggFn: 'count', + valueExpression: 'Value', + metricName: 'http.server.request.count', + metricType: MetricsDataType.Histogram, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain( + 'cityHash64(ScopeAttributes, ResourceAttributes, Attributes)', + ); + expect(actual).not.toContain('toJSONString'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should still use mapConcat when Attributes column is Map type (non-JSON schema)', async () => { + // Verify existing Map-schema behaviour is unchanged + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.cpu.utilization', + metricType: MetricsDataType.Gauge, + }, + ], + }; + + // mockMetadata returns Map-typed columns (default setup from beforeEach) + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain('mapConcat'); + expect(actual).not.toContain('toJSONString'); + }); + }); }); diff --git a/packages/common-utils/src/core/histogram.ts b/packages/common-utils/src/core/histogram.ts index d84453e8a1..3b1a3abc78 100644 --- a/packages/common-utils/src/core/histogram.ts +++ b/packages/common-utils/src/core/histogram.ts @@ -4,8 +4,21 @@ import { BuilderChartConfig } from '@/types'; type WithClauses = BuilderChartConfig['with']; type TemplatedInput = ChSql | string; +// Returns the SQL expression for hashing metric attributes into a unique key. +// When using the JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED), the attribute +// columns are JSON type instead of Map(String, String), so mapConcat() would +// fail with "Function mapConcat requires at least one argument of type Map". +// cityHash64 accepts JSON columns directly as variadic arguments, which avoids +// the per-row toJSONString serialization overhead (~28% faster). +export function attrHashExpr(isJsonSchema: boolean): string { + return isJsonSchema + ? 'cityHash64(ScopeAttributes, ResourceAttributes, Attributes)' + : 'cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes))'; +} + export const translateHistogram = ({ select, + isJsonSchema = false, ...rest }: { select: Exclude[number]; @@ -14,6 +27,7 @@ export const translateHistogram = ({ from: TemplatedInput; where: TemplatedInput; valueAlias: TemplatedInput; + isJsonSchema?: boolean; }) => { if (select.aggFn === 'quantile') { if (!('level' in select) || select.level === null) @@ -21,10 +35,11 @@ export const translateHistogram = ({ return translateHistogramQuantile({ ...rest, level: select.level, + isJsonSchema, }); } if (select.aggFn === 'count') { - return translateHistogramCount(rest); + return translateHistogramCount({ ...rest, isJsonSchema }); } throw new Error(`${select.aggFn} is not supported for histograms currently`); }; @@ -35,12 +50,14 @@ const translateHistogramCount = ({ from, where, valueAlias, + isJsonSchema = false, }: { timeBucketSelect: TemplatedInput; groupBy?: TemplatedInput; from: TemplatedInput; where: TemplatedInput; valueAlias: TemplatedInput; + isJsonSchema?: boolean; }): WithClauses => [ { name: 'source', @@ -50,7 +67,7 @@ const translateHistogramCount = ({ AggregationTemporality, ${timeBucketSelect}, ${groupBy ? chSql`[${groupBy}] AS group,` : ''} - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS attr_hash, + ${attrHashExpr(isJsonSchema)} AS attr_hash, cityHash64(ExplicitBounds) AS bounds_hash, toInt64(Count) AS current_count, lagInFrame(toNullable(current_count), 1, NULL) OVER ( @@ -86,6 +103,7 @@ const translateHistogramQuantile = ({ where, valueAlias, level, + isJsonSchema = false, }: { timeBucketSelect: TemplatedInput; groupBy?: TemplatedInput; @@ -93,6 +111,7 @@ const translateHistogramQuantile = ({ where: TemplatedInput; valueAlias: TemplatedInput; level: number; + isJsonSchema?: boolean; }): WithClauses => [ { name: 'source', @@ -131,7 +150,7 @@ const translateHistogramQuantile = ({ ExplicitBounds, ResourceAttributes, Attributes, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS attr_hash, + ${attrHashExpr(isJsonSchema)} AS attr_hash, cityHash64(ExplicitBounds) AS bounds_hash, CAST(BucketCounts AS Array(Int64)) counts FROM ${from} diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index b399a139fb..1ae4198c1b 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -2,8 +2,15 @@ import isPlainObject from 'lodash/isPlainObject'; import * as SQLParser from 'node-sql-parser'; import SqlString from 'sqlstring'; -import { ChSql, chSql, concatChSql, wrapChSqlIfNotEmpty } from '@/clickhouse'; -import { translateHistogram } from '@/core/histogram'; +import { + ChSql, + chSql, + concatChSql, + convertCHDataTypeToJSType, + JSDataType, + wrapChSqlIfNotEmpty, +} from '@/clickhouse'; +import { attrHashExpr, translateHistogram } from '@/core/histogram'; import { Metadata } from '@/core/metadata'; import { convertDateRangeToGranularityString, @@ -1472,6 +1479,42 @@ async function translateMetricChartConfig( ); } + // Detect whether the metric tables use the JSON schema + // (BETA_CH_OTEL_JSON_SCHEMA_ENABLED). When enabled, attribute columns + // (Attributes, ScopeAttributes, ResourceAttributes) are JSON type instead of + // Map(String, String), which means mapConcat() cannot be used. We detect this + // by inspecting the actual column type in ClickHouse. + let isJsonSchema = false; + const detectionTableName = + (metricType != null ? metricTables[metricType] : undefined) ?? + metricTables[MetricsDataType.Gauge] ?? + metricTables[MetricsDataType.Sum] ?? + metricTables[MetricsDataType.Histogram]; + if (detectionTableName && from.databaseName && chartConfig.connection) { + try { + const columns = await metadata.getColumns({ + databaseName: from.databaseName, + tableName: detectionTableName, + connectionId: chartConfig.connection, + }); + // We only check `Attributes` as a representative column — the OTel + // exporter sets all three attribute columns (Attributes, ScopeAttributes, + // ResourceAttributes) to the same type, so checking one is sufficient. + isJsonSchema = columns.some( + c => + c.name === 'Attributes' && + convertCHDataTypeToJSType(c.type) === JSDataType.JSON, + ); + } catch (e) { + // If column detection fails (e.g. table doesn't exist yet), fall back to + // the Map schema behaviour which was the original default. + console.warn( + 'Failed to detect metric table column types, falling back to Map schema', + e, + ); + } + } + if ( metricType === MetricsDataType.Gauge && metricName && @@ -1520,7 +1563,7 @@ async function translateMetricChartConfig( sql: chSql` SELECT *, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash + ${attrHashExpr(isJsonSchema)} AS AttributesHash FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] } })} WHERE ${where} `, @@ -1634,7 +1677,7 @@ async function translateMetricChartConfig( sql: chSql` SELECT *, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash, + ${attrHashExpr(isJsonSchema)} AS AttributesHash, IF( AggregationTemporality = 1, Value, -- DELTA: Value is already the per-interval increase @@ -1889,6 +1932,7 @@ async function translateMetricChartConfig( }), where, valueAlias, + isJsonSchema, }), select: `\`__hdx_time_bucket\`${groupBy ? ', group' : ''}, "${valueAlias}"`, from: { From 998ea5d07823265463238dbd3e2a2daa3541a823 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 4 Jun 2026 16:28:50 -0400 Subject: [PATCH 024/728] feat(charts): add 'Fit Y-Axis to Data' display setting for line charts (#2417) # Summary Adds an optional `fitYAxisToData` display setting for line charts. When enabled, the y-axis lower bound follows the minimum of the displayed data (with padding) instead of being pinned at zero, making small fluctuations between series easier to see. The default remains lower bound = 0. The setting appears in Display Settings for Line and Stacked Bar charts, persists on the chart config, and is exposed in the External API v2 line chart schemas. Closes #1619 ## Demo https://github.com/user-attachments/assets/7c4ab4c8-962d-4526-9dc2-6c997b477f9f ## How to test This can be tested in the preview environment Closes #1619 Closes HDX-3242 --- .changeset/breezy-students-clap.md | 7 ++ packages/api/openapi.json | 10 +++ .../external-api/__tests__/dashboards.test.ts | 76 +++++++++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 14 ++++ .../external-api/v2/utils/dashboards.ts | 4 + packages/api/src/utils/zod.ts | 2 + packages/app/src/HDXMultiSeriesTimeChart.tsx | 49 +++++++++--- .../components/ChartDisplaySettingsDrawer.tsx | 9 +++ .../DBEditTimeChartForm/EditTimeChartForm.tsx | 6 ++ packages/app/src/components/DBTimeChart.tsx | 1 + .../ChartDisplaySettingsDrawer.test.tsx | 64 ++++++++++++++++ packages/common-utils/src/types.ts | 1 + 12 files changed, 232 insertions(+), 11 deletions(-) create mode 100644 .changeset/breezy-students-clap.md diff --git a/.changeset/breezy-students-clap.md b/.changeset/breezy-students-clap.md new file mode 100644 index 0000000000..2e08cb5b03 --- /dev/null +++ b/.changeset/breezy-students-clap.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add option to fit time chart y-axis lower bound diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 6018c8300c..9bcad76117 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -1239,6 +1239,11 @@ "description": "Fill missing time buckets with zero instead of leaving gaps.", "default": true }, + "fitYAxisToData": { + "type": "boolean", + "description": "Set the y-axis lower bound to the minimum of the displayed data instead of zero, making small fluctuations between series easier to see.\n", + "default": false + }, "numberFormat": { "$ref": "#/components/schemas/NumberFormat", "description": "Number formatting options for displayed values." @@ -1676,6 +1681,11 @@ "type": "boolean", "description": "Expand date range boundaries to the query granularity interval.", "default": true + }, + "fitYAxisToData": { + "type": "boolean", + "description": "Set the y-axis lower bound to the minimum of the displayed data instead of zero, making small fluctuations between series easier to see.\n", + "default": false } } } diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts index 3f9c075b23..7aa2799d6a 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts @@ -2193,6 +2193,7 @@ describe('External API v2 Dashboards - new format', () => { displayType: 'line', asRatio: true, fillNulls: true, + fitYAxisToData: true, sourceId: traceSource._id.toString(), numberFormat: { output: 'time', @@ -2703,6 +2704,7 @@ describe('External API v2 Dashboards - new format', () => { compareToPreviousPeriod: true, fillNulls: true, alignDateRangeToGranularity: true, + fitYAxisToData: true, numberFormat: { output: 'number', mantissa: 2 }, }, }; @@ -2796,6 +2798,78 @@ describe('External API v2 Dashboards - new format', () => { expect(omit(response.body.data.tiles[4], ['id'])).toEqual(pieRawSql); }); + it('persists fitYAxisToData on line tiles only and reads it back on GET', async () => { + const sourceId = traceSource._id.toString(); + + // A line tile that opts into fitYAxisToData; a bar tile that attempts to + // set it (it is line-only, so it must be dropped); and a line tile that + // omits the field entirely to confirm it stays absent (optional, no + // default). + const fitLine: ExternalDashboardTile = { + name: 'Fit Line', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + sourceId, + fitYAxisToData: true, + select: [{ aggFn: 'count', where: '', whereLanguage: 'sql' }], + }, + }; + + const bar: ExternalDashboardTile = { + name: 'Bar', + x: 6, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'stacked_bar', + sourceId, + // fitYAxisToData only applies to line charts; setting it on a bar + // tile should be ignored rather than persisted. + fitYAxisToData: false, + select: [{ aggFn: 'count', where: '', whereLanguage: 'sql' }], + } as ExternalDashboardTile['config'], + }; + + const unsetLine: ExternalDashboardTile = { + name: 'Unset Line', + x: 12, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count', where: '', whereLanguage: 'sql' }], + }, + }; + + const createResponse = await authRequest('post', BASE_URL) + .send({ + name: 'fitYAxisToData dashboard', + tiles: [fitLine, bar, unsetLine], + tags: [], + }) + .expect(200); + + const { id } = createResponse.body.data; + + const getResponse = await authRequest('get', `${BASE_URL}/${id}`).expect( + 200, + ); + const tiles = getResponse.body.data.tiles; + + expect(tiles[0].config.fitYAxisToData).toBe(true); + // Bar charts never carry fitYAxisToData — it is dropped on write. + expect(tiles[1].config).not.toHaveProperty('fitYAxisToData'); + // Omitted on input → absent on read-back (optional, no default). + expect(tiles[2].config).not.toHaveProperty('fitYAxisToData'); + }); + it('should return 400 when source IDs do not exist', async () => { const nonExistentSourceId = new ObjectId().toString(); const mockDashboard = createMockDashboard(nonExistentSourceId); @@ -3628,6 +3702,7 @@ describe('External API v2 Dashboards - new format', () => { displayType: 'line', asRatio: true, fillNulls: true, + fitYAxisToData: true, sourceId: traceSource._id.toString(), numberFormat: { output: 'time', @@ -3889,6 +3964,7 @@ describe('External API v2 Dashboards - new format', () => { compareToPreviousPeriod: true, fillNulls: true, alignDateRangeToGranularity: true, + fitYAxisToData: true, numberFormat: { output: 'number', mantissa: 2 }, }, }; diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 9dd32fbead..9075e15388 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -479,6 +479,13 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: boolean * description: Fill missing time buckets with zero instead of leaving gaps. * default: true + * fitYAxisToData: + * type: boolean + * description: > + * Set the y-axis lower bound to the minimum of the displayed data + * instead of zero, making small fluctuations between series easier + * to see. + * default: false * numberFormat: * $ref: '#/components/schemas/NumberFormat' * description: Number formatting options for displayed values. @@ -835,6 +842,13 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: boolean * description: Expand date range boundaries to the query granularity interval. * default: true + * fitYAxisToData: + * type: boolean + * description: > + * Set the y-axis lower bound to the minimum of the displayed + * data instead of zero, making small fluctuations between + * series easier to see. + * default: false * * BarRawSqlChartConfig: * description: Raw SQL configuration for a stacked-bar time-series chart. diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 844a92e194..214899f85c 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -162,6 +162,7 @@ const convertToExternalTileChartConfig = ( sourceId: config.source, alignDateRangeToGranularity: config.alignDateRangeToGranularity, fillNulls: config.fillNulls !== false, + fitYAxisToData: config.fitYAxisToData, numberFormat: config.numberFormat, compareToPreviousPeriod: config.compareToPreviousPeriod, }; @@ -243,6 +244,7 @@ const convertToExternalTileChartConfig = ( config.select.length == 2, alignDateRangeToGranularity: config.alignDateRangeToGranularity, fillNulls: config.fillNulls !== false, + fitYAxisToData: config.fitYAxisToData, groupBy: stringValueOrDefault(config.groupBy, undefined), select: Array.isArray(config.select) ? config.select.map(convertToExternalSelectItem) @@ -541,6 +543,7 @@ export function convertToInternalTileConfig( 'numberFormat', 'alignDateRangeToGranularity', 'compareToPreviousPeriod', + 'fitYAxisToData', ]), displayType: externalConfig.displayType === 'stacked_bar' @@ -593,6 +596,7 @@ export function convertToInternalTileConfig( 'numberFormat', 'alignDateRangeToGranularity', 'compareToPreviousPeriod', + 'fitYAxisToData', ]), displayType: externalConfig.displayType === 'stacked_bar' diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index ac933f706b..d098e4d44e 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -255,6 +255,7 @@ const externalDashboardLineChartConfigSchema = externalDashboardTimeChartConfigSchema.extend({ displayType: z.literal('line'), compareToPreviousPeriod: z.boolean().optional(), + fitYAxisToData: z.boolean().optional(), }); const externalDashboardLineRawSqlChartConfigSchema = @@ -263,6 +264,7 @@ const externalDashboardLineRawSqlChartConfigSchema = compareToPreviousPeriod: z.boolean().optional(), fillNulls: z.boolean().optional(), alignDateRangeToGranularity: z.boolean().optional(), + fitYAxisToData: z.boolean().optional(), }); const externalDashboardBarChartConfigSchema = diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index 1ef8c840a5..f9bcb08566 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -354,6 +354,7 @@ export const MemoChart = memo(function MemoChart({ onToggleSeries, granularity, dateRangeEndInclusive = true, + fitYAxisToData = false, }: { graphResults: any[]; setIsClickActive: (v: any) => void; @@ -375,6 +376,11 @@ export const MemoChart = memo(function MemoChart({ onToggleSeries?: (seriesName: string, isShiftKey?: boolean) => void; granularity: string; dateRangeEndInclusive?: boolean; + /** + * When true, the y-axis lower bound is the minimum of the displayed data + * (with padding) instead of zero. + **/ + fitYAxisToData?: boolean; }) { const _id = useId(); const id = _id.replace(/:/g, ''); @@ -441,20 +447,29 @@ export const MemoChart = memo(function MemoChart({ const yAxisDomain: AxisDomain = useMemo(() => { const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; - if (!hasSelection) { - // No selection, let Recharts auto-calculate based on all data + // Fitting the y-axis lower bound to the data only applies to line charts. + // Bar charts are always anchored at zero so the bar lengths stay + // proportional to their values. + const shouldFitYAxis = + fitYAxisToData && displayType !== DisplayType.StackedBar; + + // The data min/max is only needed to either zoom into a selection or to + // fit the lower bound to the data. When neither applies, let Recharts + // auto-calculate the upper bound while pinning the lower bound to zero. + if (!hasSelection && !shouldFitYAxis) { return [0, 'auto']; } - // When series are selected, calculate domain based only on visible series + // Calculate domain based on visible series (all series when there's no + // explicit selection). let minValue = Infinity; let maxValue = -Infinity; graphResults.forEach(dataPoint => { lineData.forEach(ld => { const seriesName = ld.displayName || ld.dataKey; - // Only consider selected series - if (selectedSeriesNames.has(seriesName)) { + // Only consider visible series + if (!hasSelection || selectedSeriesNames.has(seriesName)) { const value = dataPoint[ld.dataKey]; if (typeof value === 'number' && !isNaN(value)) { minValue = Math.min(minValue, value); @@ -466,15 +481,27 @@ export const MemoChart = memo(function MemoChart({ // If we found valid values, return them with some padding if (minValue !== Infinity && maxValue !== -Infinity) { - const padding = (maxValue - minValue) * 0.1; // 10% padding - return [ - Math.max(0, minValue - padding), // Don't go below 0 - maxValue + padding, - ]; + const padding = (maxValue - minValue) * 0.05; // 5% padding + // When fitting to data, allow the lower bound to follow the data + // minimum; otherwise keep it pinned at zero. The 5% padding must not + // drag the axis below zero unless the data itself is negative, so + // clamp at zero whenever the minimum is non-negative. + const lowerBound = + shouldFitYAxis && minValue < 0 + ? minValue - padding + : Math.max(0, minValue - padding); + const upperBound = maxValue + padding; + return [lowerBound, upperBound]; } return ['auto', 'auto']; - }, [graphResults, lineData, selectedSeriesNames]); + }, [ + graphResults, + lineData, + selectedSeriesNames, + fitYAxisToData, + displayType, + ]); const sizeRef = useRef<[number, number]>([0, 0]); const [containerWidth, setContainerWidth] = useState(0); diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index 5f5f8005bc..46621da3e5 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -30,6 +30,7 @@ export type ChartConfigDisplaySettings = Pick< | 'alignDateRangeToGranularity' | 'fillNulls' | 'compareToPreviousPeriod' + | 'fitYAxisToData' | 'color' > & { groupByColumnsOnLeft?: boolean; @@ -63,6 +64,7 @@ function applyDefaultSettings( : settings.alignDateRangeToGranularity, fillNulls: settings.fillNulls ?? 0, compareToPreviousPeriod: settings.compareToPreviousPeriod ?? false, + fitYAxisToData: settings.fitYAxisToData ?? false, groupByColumnsOnLeft: settings.groupByColumnsOnLeft ?? false, color: settings.color, }; @@ -165,6 +167,13 @@ export default function ChartDisplaySettingsDrawer({ ) } /> + )} diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index ca42c9ba95..46c670096e 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -221,6 +221,7 @@ export default function EditTimeChartForm({ alignDateRangeToGranularity, fillNulls, compareToPreviousPeriod, + fitYAxisToData, numberFormat, groupByColumnsOnLeft, color, @@ -230,6 +231,7 @@ export default function EditTimeChartForm({ 'alignDateRangeToGranularity', 'fillNulls', 'compareToPreviousPeriod', + 'fitYAxisToData', 'numberFormat', 'groupByColumnsOnLeft', 'color', @@ -250,6 +252,7 @@ export default function EditTimeChartForm({ alignDateRangeToGranularity, fillNulls, compareToPreviousPeriod, + fitYAxisToData, numberFormat, groupByColumnsOnLeft, color, @@ -258,6 +261,7 @@ export default function EditTimeChartForm({ alignDateRangeToGranularity, fillNulls, compareToPreviousPeriod, + fitYAxisToData, numberFormat, groupByColumnsOnLeft, color, @@ -539,6 +543,7 @@ export default function EditTimeChartForm({ alignDateRangeToGranularity, fillNulls, compareToPreviousPeriod, + fitYAxisToData, groupByColumnsOnLeft, color, }: ChartConfigDisplaySettings) => { @@ -546,6 +551,7 @@ export default function EditTimeChartForm({ setValue('alignDateRangeToGranularity', alignDateRangeToGranularity); setValue('fillNulls', fillNulls); setValue('compareToPreviousPeriod', compareToPreviousPeriod); + setValue('fitYAxisToData', fitYAxisToData); setValue('groupByColumnsOnLeft', groupByColumnsOnLeft); setValue('color', color); onSubmit(); diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index d0d4dfa9e1..c0f2b26616 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -740,6 +740,7 @@ function DBTimeChartComponent({ onToggleSeries={handleToggleSeries} granularity={granularity} dateRangeEndInclusive={queriedConfig.dateRangeEndInclusive} + fitYAxisToData={queriedConfig.fitYAxisToData} /> )} diff --git a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx index db430e1ded..9b83e81dc2 100644 --- a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx +++ b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx @@ -109,4 +109,68 @@ describe('ChartDisplaySettingsDrawer', () => { }); }); }); + + describe('fit y-axis to data setting', () => { + it('shows the "Fit Y-Axis to Data" checkbox for line charts', () => { + renderWithMantine( + , + ); + + expect( + screen.getByRole('checkbox', { name: /fit y-axis to data/i }), + ).toBeInTheDocument(); + }); + + it('does not show the "Fit Y-Axis to Data" checkbox for table charts', () => { + renderWithMantine( + , + ); + + expect( + screen.queryByRole('checkbox', { name: /fit y-axis to data/i }), + ).not.toBeInTheDocument(); + }); + + it('defaults to unchecked (lower bound = 0)', () => { + renderWithMantine( + , + ); + + expect( + screen.getByRole('checkbox', { name: /fit y-axis to data/i }), + ).not.toBeChecked(); + }); + + it('calls onChange with fitYAxisToData = true when enabled and applied', async () => { + const onChange = jest.fn(); + const user = userEvent.setup(); + + renderWithMantine( + , + ); + + await user.click( + screen.getByRole('checkbox', { name: /fit y-axis to data/i }), + ); + await user.click(screen.getByRole('button', { name: /apply/i })); + + expect(onChange).toHaveBeenCalledTimes(1); + expect(onChange.mock.calls[0][0]).toMatchObject({ + fitYAxisToData: true, + }); + }); + }); }); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 64b5cde975..5666ce2cfb 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -861,6 +861,7 @@ const SharedChartSettingsSchema = z.object({ compareToPreviousPeriod: z.boolean().optional(), fillNulls: z.union([z.number(), z.literal(false)]).optional(), alignDateRangeToGranularity: z.boolean().optional(), + fitYAxisToData: z.boolean().optional(), onClick: OnClickSchema.optional(), // Palette-token color override. Applied by the renderer for number // tiles only (gated in `ChartDisplaySettingsDrawer`); other display From 89949b1bd6148e14e345f67daa609a30fe13fcfd Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 5 Jun 2026 07:54:56 +1000 Subject: [PATCH 025/728] feat: Added filters to dashboard exports (#2410) --- .changeset/good-swans-hammer.md | 5 + packages/app/src/DBDashboardImportPage.tsx | 180 ++++++++++-- packages/app/src/DBDashboardPage.tsx | 1 + packages/app/src/DashboardFilters.tsx | 34 ++- .../useDashboardFilterValues.test.tsx | 49 +++- .../src/hooks/useDashboardFilterValues.tsx | 53 ++-- .../dashboard-template-import.spec.ts | 178 ++++++++++++ .../e2e/page-objects/DashboardImportPage.ts | 20 ++ .../tests/e2e/page-objects/DashboardPage.ts | 39 +++ .../src/__tests__/filters.test.ts | 260 +++++++++++++++++- .../common-utils/src/__tests__/utils.test.ts | 222 +++++++++++++++ packages/common-utils/src/core/utils.ts | 18 ++ packages/common-utils/src/filters.ts | 164 ++++++++++- 13 files changed, 1169 insertions(+), 54 deletions(-) create mode 100644 .changeset/good-swans-hammer.md diff --git a/.changeset/good-swans-hammer.md b/.changeset/good-swans-hammer.md new file mode 100644 index 0000000000..a5ba3e0de5 --- /dev/null +++ b/.changeset/good-swans-hammer.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": patch +--- + +Adding filters to dashboard exports. Implemented validation on dashboard imports to catch potential issues with generated JSON or manually tweaked JSON. diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 9ce75353a5..0b8cd9dc07 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -15,6 +15,14 @@ import { Controller, useForm, useWatch } from 'react-hook-form'; import { z } from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { convertToDashboardDocument } from '@hyperdx/common-utils/dist/core/utils'; +import { + type DashboardFilterQueryIssue, + type SavedFilterValueIssue, + type SavedQueryIssue, + validateDashboardFilterQueries, + validateSavedFilterValues, + validateSavedQuery, +} from '@hyperdx/common-utils/dist/filters'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { type DashboardTemplate, @@ -65,6 +73,48 @@ import { getDashboardTemplate } from './dashboardTemplates'; import { withAppNav } from './layout'; import { useSources } from './source'; +/** + * A colored headline message with an optional collapsible "details" section, + * toggled by a chevron button. Shared by the import error and the various + * non-blocking import warnings so they stay visually consistent. The collapse + * state is local, so the details start collapsed each time it's rendered. + */ +function CollapsibleMessage({ + color, + message, + details, + 'data-testid': dataTestId, +}: { + color: string; + message: ReactNode; + details?: ReactNode; + 'data-testid'?: string; +}) { + const [detailsOpen, { toggle: toggleDetails }] = useDisclosure(false); + return ( +
+ {message} + {details != null && ( + <> + + {details} + + )} +
+ ); +} + function FileSelection({ onComplete, }: { @@ -79,7 +129,14 @@ function FileSelection({ message: string; details?: ReactNode; } | null>(null); - const [errorDetails, { toggle: toggleErrorDetails }] = useDisclosure(false); + const [filterWarnings, setFilterWarnings] = useState( + [], + ); + const [filterQueryWarnings, setFilterQueryWarnings] = useState< + DashboardFilterQueryIssue[] + >([]); + const [savedQueryWarning, setSavedQueryWarning] = + useState(null); const { control, handleSubmit } = useForm({ resolver: zodResolver(FormSchema), @@ -87,6 +144,9 @@ function FileSelection({ const onSubmit = async ({ file }: FormValues) => { setError(null); + setFilterWarnings([]); + setFilterQueryWarnings([]); + setSavedQueryWarning(null); if (!file) return; let data: unknown; @@ -120,6 +180,32 @@ function FileSelection({ return; } + // The schema only guarantees saved filter values are well-shaped, not that + // their condition strings parse. Surface a non-blocking warning so users + // who hand-edited or generated the file know a value won't actually filter, + // while still letting them proceed with the import. + setFilterWarnings( + validateSavedFilterValues(result.data.savedFilterValues ?? []), + ); + + // Import never runs the filters' values queries, so a filter whose `where` + // clause is malformed would otherwise only surface as a failed query after + // opening the dashboard. Statically validate the `where` clauses here so the + // problem is visible up front (non-blocking). + setFilterQueryWarnings( + validateDashboardFilterQueries(result.data.filters ?? []), + ); + + // The dashboard's default saved query is also free-form text. A malformed + // one is lower impact (it surfaces in the dashboard search bar where it can + // be edited), but warn about it for consistency (non-blocking). + setSavedQueryWarning( + validateSavedQuery( + result.data.savedQuery, + result.data.savedQueryLanguage, + ), + ); + onComplete(result.data); }; @@ -185,32 +271,72 @@ function FileSelection({ /> {error && ( -
- {error.message} - {error.details != null && ( - <> - - {error.details} - - )} -
+ + )} + + {filterWarnings.length > 0 && ( + + {filterWarnings.map(warning => ( + + Invalid {warning.language} condition: {warning.condition} + + ))} + + } + /> + )} + + {filterQueryWarnings.length > 0 && ( + + {filterQueryWarnings.map(warning => ( + + {warning.filterName}: invalid {warning.language} query:{' '} + {warning.where} + + ))} + + } + /> + )} + + {savedQueryWarning && ( + + Invalid {savedQueryWarning.language} query:{' '} + {savedQueryWarning.query} + + } + /> )} diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index ac648ab5f1..ec89bb5384 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -2353,6 +2353,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { {hasTiles && ( } + data-testid="export-dashboard-menu-item" onClick={() => { if (!sources || !dashboard) { notifications.show({ diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index 78a29dfdb6..f92ac131c8 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -1,7 +1,7 @@ import { FilterState } from '@hyperdx/common-utils/dist/filters'; import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; import { Group, Stack, Text, Tooltip } from '@mantine/core'; -import { IconHelp, IconRefresh } from '@tabler/icons-react'; +import { IconAlertTriangle, IconHelp, IconRefresh } from '@tabler/icons-react'; import { VirtualMultiSelect } from './components/VirtualMultiSelect/VirtualMultiSelect'; import { useDashboardFilterValues } from './hooks/useDashboardFilterValues'; @@ -12,6 +12,7 @@ interface DashboardFilterSelectProps { value: string[]; values?: string[]; isLoading?: boolean; + isError?: boolean; } const getAppliesToTooltip = (filter: DashboardFilter) => { @@ -26,6 +27,7 @@ const DashboardFilterSelect = ({ value, values, isLoading, + isError, }: DashboardFilterSelectProps) => { const sortedValues = values?.toSorted() || []; const tooltipText = getAppliesToTooltip(filter); @@ -43,12 +45,27 @@ const DashboardFilterSelect = ({ data-testid={`dashboard-filter-help-${filter.name}`} /> + {isError && ( + + + + )}
{ - const { data: filterValuesById, isFetching } = useDashboardFilterValues({ + const { + data: filterValuesById, + erroredFilterIds, + isFetching, + } = useDashboardFilterValues({ filters, dateRange, }); @@ -84,11 +105,18 @@ const DashboardFilters = ({ const selectedValues = included ? Array.from(included).map(v => v.toString()) : []; + // Fall back to the hook-level fetching state only until this filter's + // query has produced an entry; once it has (even with empty values), + // honor its own loading flag so a finished query never stays disabled. + const isLoadingValues = queriedFilterValues + ? queriedFilterValues.isLoading + : isFetching; return ( onSetFilterValue(filter.expression, values)} values={queriedFilterValues?.values} value={selectedValues} diff --git a/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx index 98837b38d8..6384fb7540 100644 --- a/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx @@ -881,8 +881,15 @@ describe('useDashboardFilterValues', () => { isLoading: false, }); - // filter2 (service.name) should not be in the map because the query failed - expect(result.current.data?.has('filter2')).toBe(false); + // filter2 (service.name) is still present with empty values (so the UI can + // keep the control interactive) and is flagged as errored so callers can + // surface a warning. + expect(result.current.data?.get('filter2')).toEqual({ + values: [], + isLoading: false, + }); + expect(result.current.erroredFilterIds.has('filter1')).toBe(false); + expect(result.current.erroredFilterIds.has('filter2')).toBe(true); // Overall error state should be true expect(result.current.isError).toBe(true); @@ -891,6 +898,44 @@ describe('useDashboardFilterValues', () => { expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(2); }); + it('keeps a filter that returned no values present and interactive', async () => { + // Arrange + jest.spyOn(sourceModule, 'useSources').mockReturnValue({ + data: mockSources, + isLoading: false, + } as any); + + jest + .mocked(optimizeGetKeyValuesCalls) + .mockImplementationOnce(async ({ chartConfig, keys }) => [ + { chartConfig, keys }, + ]); + + // Query succeeds but returns no rows for the requested key. + mockMetadata.getKeyValues.mockResolvedValueOnce([]); + + // Act + const { result } = renderHook( + () => + useDashboardFilterValues({ + filters: [mockFilters[0]], + dateRange: mockDateRange, + }), + { wrapper }, + ); + + // Assert + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + // Entry exists with empty values and is not loading → control stays usable. + expect(result.current.data?.get('filter1')).toEqual({ + values: [], + isLoading: false, + }); + expect(result.current.erroredFilterIds.has('filter1')).toBe(false); + expect(result.current.isError).toBe(false); + }); + it('should keep previous data while fetching new data (placeholderData behavior)', async () => { // Arrange jest.spyOn(sourceModule, 'useSources').mockReturnValue({ diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index 8e36a05668..710eecb930 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -217,29 +217,42 @@ export function useDashboardFilterValues({ // Map results by filter ID instead of expression so that two filters with // the same expression but different sources/WHERE clauses get distinct values. - const flattenedData = useMemo( - () => - new Map( - results.flatMap(({ isLoading, data = [] }, resultIndex) => { - const call = calls[resultIndex]; - return data.flatMap(({ key: expression, value }) => { - const keyIndex = call.keys.indexOf(expression); - const filterIds = call.filterIds?.[keyIndex] ?? []; - const entry = { - values: value.map(v => v.toString()), - isLoading, - }; - return filterIds.map( - filterId => [filterId, entry] as [string, typeof entry], - ); - }); - }), - ), - [results, calls], - ); + // + // We iterate over the *requested* keys (not just the returned rows) so that a + // filter whose query returned no rows — or failed outright — still gets an + // entry (with empty `values`). Without this, such filters would be missing + // from the map entirely, and any consumer that derives "is still loading" + // from the entry's absence would leave the control stuck disabled forever. + // Failed filter IDs are tracked separately so callers can surface a warning + // while keeping the control interactive. + const { data: flattenedData, erroredFilterIds } = useMemo(() => { + const map = new Map(); + const errored = new Set(); + results.forEach((result, resultIndex) => { + const call = calls[resultIndex]; + if (!call) return; + const { isLoading, isError, data = [] } = result; + const valuesByExpression = new Map( + data.map(({ key, value }) => [key, value.map(v => v.toString())]), + ); + call.keys.forEach((expression, keyIndex) => { + const filterIds = call.filterIds?.[keyIndex] ?? []; + const values = valuesByExpression.get(expression) ?? []; + for (const filterId of filterIds) { + map.set(filterId, { values, isLoading }); + if (isError) { + errored.add(filterId); + } + } + }); + }); + return { data: map, erroredFilterIds: errored }; + }, [results, calls]); return { data: flattenedData, + /** Filter IDs whose key-values query failed. */ + erroredFilterIds, isLoading: isLoadingOptimizedCalls || results.every(r => r.isLoading), isFetching: isFetchingOptimizedCalls || results.some(r => r.isFetching), isError: results.some(r => r.isError), diff --git a/packages/app/tests/e2e/features/dashboard-template-import.spec.ts b/packages/app/tests/e2e/features/dashboard-template-import.spec.ts index 87b27577c5..1595e50482 100644 --- a/packages/app/tests/e2e/features/dashboard-template-import.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-template-import.spec.ts @@ -608,6 +608,184 @@ test.describe('Dashboard Template Import', { tag: ['@dashboard'] }, () => { }, ); + test( + 'should round-trip a dashboard through export and re-import', + { tag: '@full-stack' }, + async ({ page }) => { + const ts = Date.now(); + const dashboardName = `E2E Export Import ${ts}`; + const tileName = `Logs Count ${ts}`; + const filterName = `Service ${ts}`; + + await test.step('Create a dashboard with a tile and a filter', async () => { + await dashboardPage.goto(); + await dashboardPage.createNewDashboard(); + await dashboardPage.editDashboardName(dashboardName); + await dashboardPage.addTileWithSource( + tileName, + DEFAULT_LOGS_SOURCE_NAME, + ); + + await dashboardPage.openEditFiltersModal(); + await dashboardPage.addFilterToDashboard( + filterName, + DEFAULT_LOGS_SOURCE_NAME, + 'ServiceName', + ); + await expect( + dashboardPage.getFilterItemByName(filterName), + ).toBeVisible(); + await dashboardPage.closeFiltersModal(); + // Export reads the in-memory dashboard state (not the persisted doc), + // so the tile + filter we just added are already present — no need to + // wait for the fire-and-forget PATCH to land. + }); + + let exported: Record = {}; + await test.step('Export the dashboard and verify the downloaded template', async () => { + exported = await dashboardPage.exportDashboard(); + expect(exported.name).toBe(dashboardName); + expect(exported.tiles).toHaveLength(1); + expect(exported.tiles[0].config.source).toBe(DEFAULT_LOGS_SOURCE_NAME); + expect(exported.filters).toHaveLength(1); + expect(exported.filters[0]).toMatchObject({ + name: filterName, + expression: 'ServiceName', + source: DEFAULT_LOGS_SOURCE_NAME, + }); + }); + + await test.step('Re-import the exported template and finish (auto-resolved mappings)', async () => { + await dashboardImportPage.gotoImport(); + await dashboardImportPage.uploadDashboardFile(JSON.stringify(exported)); + await expect(dashboardImportPage.mappingStepHeading).toBeVisible(); + await expect(dashboardImportPage.dashboardNameInput).toHaveValue( + dashboardName, + ); + await dashboardImportPage.finishImportButton.click(); + await expect( + dashboardImportPage.getImportSuccessNotification(), + ).toBeVisible(); + await page.waitForURL(/\/dashboards\/[a-f0-9]{24}/); + }); + + await test.step('Verify the re-imported dashboard matches the original', async () => { + const logSources = await getSources(page, 'log'); + const logsSourceId = logSources.find( + (s: { name: string }) => s.name === DEFAULT_LOGS_SOURCE_NAME, + ).id; + + const importedId = dashboardPage.getCurrentDashboardId(); + const imported = await fetchDashboardById(page, importedId); + expect(imported.name).toBe(dashboardName); + expect(imported.tiles).toHaveLength(1); + expect(imported.tiles[0].config.source).toBe(logsSourceId); + expect(imported.filters).toHaveLength(1); + expect(imported.filters[0]).toMatchObject({ + name: filterName, + expression: 'ServiceName', + source: logsSourceId, + }); + }); + }, + ); + + test( + 'should warn about invalid saved filter values and saved query without blocking import', + { tag: '@full-stack' }, + async ({ page }) => { + const ts = Date.now(); + const dashboardName = `E2E Import Invalid Saved Values ${ts}`; + const tileName = `Logs Count ${ts}`; + const invalidFilterCondition = 'ServiceName = = ='; + const invalidSavedQuery = 'ServiceName:((("broken'; + + // Tile uses the workspace logs source so its mapping auto-resolves and + // the import can be finished without manual mapping. The saved filter + // value and saved query carry deliberately malformed conditions. + const dashboardFile = { + version: '0.1.0', + name: dashboardName, + tiles: [ + { + id: 'tile-1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: tileName, + source: DEFAULT_LOGS_SOURCE_NAME, + displayType: 'number', + granularity: 'auto', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + whereLanguage: 'sql', + }, + }, + ], + savedFilterValues: [{ type: 'sql', condition: invalidFilterCondition }], + savedQuery: invalidSavedQuery, + savedQueryLanguage: 'lucene', + }; + + await test.step('Upload the dashboard file and reach the mapping step', async () => { + await dashboardImportPage.gotoImport(); + await dashboardImportPage.uploadDashboardFile( + JSON.stringify(dashboardFile), + ); + // Warnings are non-blocking: the mapping step is still reached. + await expect(dashboardImportPage.mappingStepHeading).toBeVisible(); + await expect(dashboardImportPage.dashboardNameInput).toHaveValue( + dashboardName, + ); + }); + + await test.step('Verify the invalid saved filter value warning is shown and lists the bad condition', async () => { + const warning = dashboardImportPage.getSavedFilterValuesWarning(); + await expect(warning).toBeVisible(); + await expect( + warning.getByText( + '1 saved filter value has an invalid condition and will not be applied.', + ), + ).toBeVisible(); + await dashboardImportPage.showDetails(warning); + await expect( + warning.getByText(`Invalid sql condition: ${invalidFilterCondition}`), + ).toBeVisible(); + }); + + await test.step('Verify the invalid saved query warning is shown and lists the bad query', async () => { + const warning = dashboardImportPage.getSavedQueryWarning(); + await expect(warning).toBeVisible(); + await expect( + warning.getByText( + "The dashboard's saved query has an invalid condition and will not be applied.", + ), + ).toBeVisible(); + await dashboardImportPage.showDetails(warning); + await expect( + warning.getByText(`Invalid lucene query: ${invalidSavedQuery}`), + ).toBeVisible(); + }); + + await test.step('Finish the import despite the warnings and verify success', async () => { + await dashboardImportPage.finishImportButton.click(); + await expect( + dashboardImportPage.getImportSuccessNotification(), + ).toBeVisible(); + await page.waitForURL(/\/dashboards\/[a-f0-9]{24}/); + }); + + await test.step('Verify the imported dashboard retained its name and tile', async () => { + const dashboardId = dashboardPage.getCurrentDashboardId(); + const dashboard = await fetchDashboardById(page, dashboardId); + expect(dashboard.name).toBe(dashboardName); + expect(dashboard.tiles).toHaveLength(1); + }); + }, + ); + test( 'should drop an unmapped onClick from the imported tile', { tag: '@full-stack' }, diff --git a/packages/app/tests/e2e/page-objects/DashboardImportPage.ts b/packages/app/tests/e2e/page-objects/DashboardImportPage.ts index 8a084807b9..8536d5e330 100644 --- a/packages/app/tests/e2e/page-objects/DashboardImportPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardImportPage.ts @@ -140,4 +140,24 @@ export class DashboardImportPage { getImportSuccessNotification() { return this.page.getByText('Import Successful!'); } + + /** The non-blocking warning shown when imported saved filter values have invalid conditions. */ + getSavedFilterValuesWarning() { + return this.page.getByTestId('import-warning-saved-filter-values'); + } + + /** The non-blocking warning shown when imported dashboard filters have invalid value queries. */ + getFilterQueriesWarning() { + return this.page.getByTestId('import-warning-filter-queries'); + } + + /** The non-blocking warning shown when the imported dashboard's saved query is invalid. */ + getSavedQueryWarning() { + return this.page.getByTestId('import-warning-saved-query'); + } + + /** Expand the "Show Details" section of one of the import warning/error blocks. */ + async showDetails(warning: Locator) { + await warning.getByRole('button', { name: 'Show Details' }).click(); + } } diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index 93c14f1e6a..1d2e9cb6c4 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -86,6 +86,7 @@ export class DashboardPage { private readonly dashboardMenuButton: Locator; private readonly saveDefaultQueryAndFiltersMenuItem: Locator; private readonly removeDefaultQueryAndFiltersMenuItem: Locator; + private readonly exportDashboardMenuItem: Locator; constructor(page: Page) { this.page = page; @@ -137,6 +138,9 @@ export class DashboardPage { this.removeDefaultQueryAndFiltersMenuItem = page.getByTestId( 'remove-default-query-filters-menu-item', ); + this.exportDashboardMenuItem = page.getByTestId( + 'export-dashboard-menu-item', + ); } /** @@ -536,6 +540,22 @@ export class DashboardPage { await this.saveTile(); } + /** + * Add a tile bound to an explicit source. Unlike addTileWithConfig (which + * relies on the editor's default source), this selects a known source so the + * exported tile carries a deterministic source name that auto-maps on import. + */ + async addTileWithSource(chartName: string, sourceName: string) { + await this.addTile(); + await expect(this.chartEditor.nameInput).toBeVisible(); + await this.chartEditor.waitForDataToLoad(); + await this.chartEditor.setChartName(chartName); + await this.chartEditor.selectSource(sourceName); + await this.chartEditor.runQuery(false); + await this.chartEditor.save(); + await expect(this.getTiles()).toHaveCount(1, { timeout: 10000 }); + } + /** * Get all dashboard tiles */ @@ -616,6 +636,25 @@ export class DashboardPage { await this.removeDefaultQueryAndFiltersMenuItem.click(); } + /** + * Export the current dashboard via the dashboard menu and return the parsed + * JSON that was downloaded. The export triggers a client-side anchor download + * (see `downloadObjectAsJson` in DBDashboardPage), so we capture the + * Playwright download event and read its stream. + */ + async exportDashboard(): Promise> { + await this.dashboardMenuButton.click(); + const downloadPromise = this.page.waitForEvent('download'); + await this.exportDashboardMenuItem.click(); + const download = await downloadPromise; + const stream = await download.createReadStream(); + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf-8')); + } + /** * Toggle live mode */ diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index 20253b4c8b..408750b7f5 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -1,4 +1,10 @@ -import { filtersToQuery } from '@/filters'; +import { + filtersToQuery, + validateDashboardFilterQueries, + validateSavedFilterValues, + validateSavedQuery, +} from '@/filters'; +import type { DashboardFilter, Filter } from '@/types'; describe('filters', () => { describe('filtersToQuery', () => { @@ -163,4 +169,256 @@ describe('filters', () => { ]); }); }); + + describe('validateSavedFilterValues', () => { + it('returns no issues for an empty array', () => { + expect(validateSavedFilterValues([])).toEqual([]); + }); + + it('accepts a valid single-value lucene condition', () => { + const filters: Filter[] = [ + { type: 'lucene', condition: 'ServiceName:"hdx-oss-dev-api"' }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('accepts a valid multi-value (OR) lucene condition', () => { + const filters: Filter[] = [ + { + type: 'lucene', + condition: + '(ServiceName:"hdx-oss-dev-api" OR ServiceName:"hdx-oss-dev-app")', + }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('accepts lucene conditions over map / bracket-notation keys', () => { + const filters: Filter[] = [ + { + type: 'lucene', + condition: 'ResourceAttributes.k8s\\.pod\\.name:"checkout-0"', + }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('accepts a valid sql condition', () => { + const filters: Filter[] = [ + { type: 'sql', condition: "ServiceName = 'hdx-oss-dev-api'" }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('accepts a valid sql condition over a map access column', () => { + const filters: Filter[] = [ + { + type: 'sql', + condition: "ResourceAttributes['service.name'] = 'checkout'", + }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('flags a malformed lucene condition', () => { + const filters: Filter[] = [ + { type: 'lucene', condition: 'ServiceName:((("broken' }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([ + { + index: 0, + language: 'lucene', + condition: 'ServiceName:((("broken', + }, + ]); + }); + + it('flags a malformed sql condition', () => { + const filters: Filter[] = [ + { type: 'sql', condition: 'ServiceName = = ' }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([ + { index: 0, language: 'sql', condition: 'ServiceName = = ' }, + ]); + }); + + it('treats empty / whitespace-only conditions as valid (no-ops)', () => { + const filters: Filter[] = [ + { type: 'lucene', condition: '' }, + { type: 'sql', condition: ' ' }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('treats structurally-valid sql_ast filters as valid', () => { + const filters: Filter[] = [ + { type: 'sql_ast', operator: '=', left: 'ServiceName', right: 'api' }, + ]; + expect(validateSavedFilterValues(filters)).toEqual([]); + }); + + it('reports the correct index for each invalid value in a mixed list', () => { + const filters: Filter[] = [ + { type: 'lucene', condition: 'ServiceName:"good"' }, + { type: 'lucene', condition: 'Bad:((("' }, + { type: 'sql', condition: "Level = 'error'" }, + { type: 'sql', condition: 'broken = = =' }, + ]; + const issues = validateSavedFilterValues(filters); + expect(issues).toEqual([ + { index: 1, language: 'lucene', condition: 'Bad:((("' }, + { index: 3, language: 'sql', condition: 'broken = = =' }, + ]); + }); + }); + + describe('validateSavedQuery', () => { + it('returns null for an empty / nullish query', () => { + expect(validateSavedQuery('', 'lucene')).toBeNull(); + expect(validateSavedQuery(' ', 'sql')).toBeNull(); + expect(validateSavedQuery(null, 'lucene')).toBeNull(); + expect(validateSavedQuery(undefined, 'sql')).toBeNull(); + }); + + it('accepts a valid lucene query', () => { + expect(validateSavedQuery('ServiceName:"api"', 'lucene')).toBeNull(); + }); + + it('accepts a valid sql query', () => { + expect(validateSavedQuery("ServiceName = 'api'", 'sql')).toBeNull(); + }); + + it('defaults a missing language to lucene', () => { + expect(validateSavedQuery('ServiceName:"api"', null)).toBeNull(); + expect(validateSavedQuery('ServiceName:"api"', undefined)).toBeNull(); + expect(validateSavedQuery('Bad:((("', undefined)).toEqual({ + language: 'lucene', + query: 'Bad:((("', + }); + }); + + it('treats promql as valid (not statically validated)', () => { + expect(validateSavedQuery('rate(foo[5m]', 'promql')).toBeNull(); + }); + + it('flags a malformed lucene query', () => { + expect(validateSavedQuery('ServiceName:((("broken', 'lucene')).toEqual({ + language: 'lucene', + query: 'ServiceName:((("broken', + }); + }); + + it('flags a malformed sql query', () => { + expect(validateSavedQuery('ServiceName = = ', 'sql')).toEqual({ + language: 'sql', + query: 'ServiceName = = ', + }); + }); + }); + + describe('validateDashboardFilterQueries', () => { + const filter = (overrides: Partial): DashboardFilter => ({ + id: 'f1', + type: 'QUERY_EXPRESSION', + name: 'ServiceName', + expression: 'ServiceName', + source: 'logs', + ...overrides, + }); + + it('returns no issues for an empty array', () => { + expect(validateDashboardFilterQueries([])).toEqual([]); + }); + + it('treats a filter with no where clause as valid', () => { + expect( + validateDashboardFilterQueries([filter({ whereLanguage: 'lucene' })]), + ).toEqual([]); + }); + + it('treats a whitespace-only where clause as valid', () => { + expect( + validateDashboardFilterQueries([ + filter({ where: ' ', whereLanguage: 'lucene' }), + ]), + ).toEqual([]); + }); + + it('accepts a valid lucene where clause', () => { + expect( + validateDashboardFilterQueries([ + filter({ where: 'ServiceName:*', whereLanguage: 'lucene' }), + ]), + ).toEqual([]); + }); + + it('accepts a valid sql where clause', () => { + expect( + validateDashboardFilterQueries([ + filter({ where: "ServiceName != ''", whereLanguage: 'sql' }), + ]), + ).toEqual([]); + }); + + it('flags a malformed lucene where clause', () => { + expect( + validateDashboardFilterQueries([ + filter({ + id: 'svc', + name: 'Service', + where: 'ServiceName:((("', + whereLanguage: 'lucene', + }), + ]), + ).toEqual([ + { + filterId: 'svc', + filterName: 'Service', + language: 'lucene', + where: 'ServiceName:((("', + }, + ]); + }); + + it('flags a malformed sql where clause', () => { + expect( + validateDashboardFilterQueries([ + filter({ + id: 'svc', + name: 'Service', + where: 'ServiceName = =', + whereLanguage: 'sql', + }), + ]), + ).toEqual([ + { + filterId: 'svc', + filterName: 'Service', + language: 'sql', + where: 'ServiceName = =', + }, + ]); + }); + + it('only reports the invalid filters in a mixed list', () => { + const issues = validateDashboardFilterQueries([ + filter({ id: 'a', where: 'ServiceName:*', whereLanguage: 'lucene' }), + filter({ + id: 'b', + name: 'Bad', + where: 'Bad:((("', + whereLanguage: 'lucene', + }), + filter({ id: 'c', where: "Level = 'error'", whereLanguage: 'sql' }), + ]); + expect(issues).toEqual([ + { + filterId: 'b', + filterName: 'Bad', + language: 'lucene', + where: 'Bad:((("', + }, + ]); + }); + }); }); diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 077a9ae2f9..d70aa40882 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -13,6 +13,7 @@ import { import { aliasMapToWithClauses, + convertToDashboardDocument, convertToDashboardTemplate, extractSettingsClauseFromEnd, findJsonExpressions, @@ -605,6 +606,227 @@ describe('utils', () => { }); }); + it('should include saved default query and filter values in the template', () => { + const dashboard: z.infer = { + id: 'dashboard1', + name: 'Dashboard With Defaults', + tags: [], + tiles: [ + { + id: 'tile1', + config: { + name: 'Log Tile', + source: 'source1', + select: '', + where: '', + }, + x: 0, + y: 0, + w: 6, + h: 6, + }, + ], + savedQuery: 'level:error', + savedQueryLanguage: 'lucene', + savedFilterValues: [ + { + type: 'lucene', + condition: 'ServiceName:"accounting"', + }, + ], + }; + + const sources: TSource[] = [ + { + id: 'source1', + name: 'Logs', + connection: 'connection1', + kind: SourceKind.Log, + from: { + databaseName: 'db1', + tableName: 'logs_table', + }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', + }, + ]; + + const template = convertToDashboardTemplate(dashboard, sources); + + expect(template.savedQuery).toBe('level:error'); + expect(template.savedQueryLanguage).toBe('lucene'); + expect(template.savedFilterValues).toEqual([ + { type: 'lucene', condition: 'ServiceName:"accounting"' }, + ]); + + const document = convertToDashboardDocument(template); + expect(document.savedQuery).toBe('level:error'); + expect(document.savedQueryLanguage).toBe('lucene'); + expect(document.savedFilterValues).toEqual([ + { type: 'lucene', condition: 'ServiceName:"accounting"' }, + ]); + }); + + describe('savedFilterValues export/import round trip', () => { + const baseSources: TSource[] = [ + { + id: 'source1', + name: 'Logs', + connection: 'connection1', + kind: SourceKind.Log, + from: { databaseName: 'db1', tableName: 'logs_table' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', + }, + ]; + + const makeDashboard = ( + savedFilterValues?: z.infer< + typeof DashboardSchema + >['savedFilterValues'], + ): z.infer => ({ + id: 'dashboard1', + name: 'Service Filter Dashboard', + tags: [], + tiles: [ + { + id: 'tile1', + config: { + name: 'Log Tile', + source: 'source1', + select: '', + where: '', + }, + x: 0, + y: 0, + w: 6, + h: 6, + }, + ], + filters: [ + { + id: 'filter1', + type: 'QUERY_EXPRESSION', + name: 'ServiceName Filter', + expression: 'ServiceName', + source: 'source1', + }, + ], + ...(savedFilterValues !== undefined ? { savedFilterValues } : {}), + }); + + it('omits savedFilterValues when no services are selected (undefined)', () => { + const dashboard = makeDashboard(undefined); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(template.savedFilterValues).toBeUndefined(); + expect('savedFilterValues' in template).toBe(false); + + const document = convertToDashboardDocument(template); + expect(document.savedFilterValues).toBeUndefined(); + expect('savedFilterValues' in document).toBe(false); + }); + + it('omits savedFilterValues when the selection is an empty array', () => { + const dashboard = makeDashboard([]); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(template.savedFilterValues).toBeUndefined(); + expect('savedFilterValues' in template).toBe(false); + + const document = convertToDashboardDocument(template); + expect(document.savedFilterValues).toBeUndefined(); + expect('savedFilterValues' in document).toBe(false); + }); + + it('carries a single selected service through the round trip', () => { + const savedFilterValues = [ + { + type: 'lucene' as const, + condition: 'ServiceName:"hdx-oss-dev-api"', + }, + ]; + const dashboard = makeDashboard(savedFilterValues); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(template.savedFilterValues).toEqual(savedFilterValues); + + const document = convertToDashboardDocument(template); + expect(document.savedFilterValues).toEqual(savedFilterValues); + }); + + it('carries multiple selected services through the round trip', () => { + const savedFilterValues = [ + { + type: 'lucene' as const, + condition: + '(ServiceName:"hdx-oss-dev-api" OR ServiceName:"hdx-oss-dev-app")', + }, + ]; + const dashboard = makeDashboard(savedFilterValues); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(template.savedFilterValues).toEqual(savedFilterValues); + + const document = convertToDashboardDocument(template); + expect(document.savedFilterValues).toEqual(savedFilterValues); + }); + + it('carries selected values across multiple filter expressions', () => { + const savedFilterValues = [ + { + type: 'lucene' as const, + condition: + '(ServiceName:"hdx-oss-dev-api" OR ServiceName:"hdx-oss-dev-app")', + }, + { type: 'lucene' as const, condition: 'SeverityText:"error"' }, + ]; + const dashboard = makeDashboard(savedFilterValues); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(template.savedFilterValues).toEqual(savedFilterValues); + + const document = convertToDashboardDocument(template); + expect(document.savedFilterValues).toEqual(savedFilterValues); + }); + + it('deep-clones savedFilterValues so the template does not alias the input', () => { + const savedFilterValues = [ + { + type: 'lucene' as const, + condition: 'ServiceName:"hdx-oss-dev-api"', + }, + ]; + const dashboard = makeDashboard(savedFilterValues); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(template.savedFilterValues).not.toBe( + dashboard.savedFilterValues, + ); + expect(template.savedFilterValues?.[0]).not.toBe(savedFilterValues[0]); + + // Mutating the source after export must not leak into the template. + savedFilterValues[0].condition = 'ServiceName:"mutated"'; + const exported = template.savedFilterValues?.[0]; + expect(exported).toEqual({ + type: 'lucene', + condition: 'ServiceName:"hdx-oss-dev-api"', + }); + }); + + it('produces a template that validates against DashboardTemplateSchema', () => { + const dashboard = makeDashboard([ + { + type: 'lucene' as const, + condition: 'ServiceName:"hdx-oss-dev-api"', + }, + ]); + + const template = convertToDashboardTemplate(dashboard, baseSources); + expect(() => DashboardTemplateSchema.parse(template)).not.toThrow(); + }); + }); + it('should remap filter.appliesToSourceIds from IDs to names', () => { const sources: TSource[] = [ { diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 5f95e6cb0b..2095771aca 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -672,6 +672,15 @@ export function convertToDashboardTemplate( output.containers = structuredClone(input.containers); } + if (input.savedQuery != null) { + output.savedQuery = input.savedQuery; + output.savedQueryLanguage = input.savedQueryLanguage ?? null; + } + + if (input.savedFilterValues && input.savedFilterValues.length > 0) { + output.savedFilterValues = structuredClone(input.savedFilterValues); + } + return output; } @@ -711,6 +720,15 @@ export function convertToDashboardDocument( output.containers = structuredClone(input.containers); } + if (input.savedQuery != null) { + output.savedQuery = input.savedQuery; + output.savedQueryLanguage = input.savedQueryLanguage ?? null; + } + + if (input.savedFilterValues && input.savedFilterValues.length > 0) { + output.savedFilterValues = structuredClone(input.savedFilterValues); + } + return output; } diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 9a7ac150b1..4bb8fe0d1e 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1,4 +1,8 @@ -import { Filter } from '@/types'; +import * as SQLParser from 'node-sql-parser'; + +import { replaceJsonExpressions } from '@/core/utils'; +import { parse } from '@/queryParser'; +import { DashboardFilter, Filter } from '@/types'; export type FilterState = { [key: string]: { @@ -52,3 +56,161 @@ export const filtersToQuery = ( return conditions; }); }; + +export type SavedFilterValueIssue = { + /** Index of the offending value within the input array */ + index: number; + /** Query language the condition claims to be written in */ + language: 'lucene' | 'sql'; + /** The raw condition string that failed to parse */ + condition: string; +}; + +const isParseableLucene = (condition: string): boolean => { + try { + parse(condition); + return true; + } catch { + return false; + } +}; + +// node-sql-parser can't handle ClickHouse map / array access (e.g. +// `LogAttributes['k']` or `arr[1]`), so swap those out for harmless literals +// before parsing — we only care whether the predicate is structurally valid. +const MAP_OR_ARRAY_ACCESS_REGEX = /\b[a-zA-Z0-9_]+\[([0-9]+|'[^']*')\]/g; + +const isParseableSql = (condition: string): boolean => { + try { + const { sqlWithReplacements } = replaceJsonExpressions(condition); + const sanitized = sqlWithReplacements.replace( + MAP_OR_ARRAY_ACCESS_REGEX, + "''", + ); + new SQLParser.Parser().astify(`SELECT 1 FROM t WHERE ${sanitized}`, { + database: 'Postgresql', + }); + return true; + } catch { + return false; + } +}; + +/** + * Whether a condition string parses as the given query language. Empty / + * whitespace-only conditions are considered valid (they're no-ops, not errors). + * `promql` is not statically validated here and is treated as valid. + */ +export function isValidFilterCondition( + condition: string, + language: 'lucene' | 'sql' | 'promql', +): boolean { + if (!condition.trim()) return true; + if (language === 'lucene') return isParseableLucene(condition); + if (language === 'sql') return isParseableSql(condition); + return true; +} + +/** + * Validate the condition strings carried in a dashboard's saved filter values. + * + * Schema validation only guarantees each value has a `{ type, condition }` + * shape — the condition text itself is a free-form string and may be broken in + * a hand-edited or machine-generated export. This returns one issue per value + * whose condition fails to parse as the language it claims to be, so callers + * can warn the user (e.g. on import) without hard-blocking the operation. + * + * Empty / whitespace-only conditions are treated as valid (they're no-ops at + * query time, not errors), as are structurally-validated `sql_ast` filters. + */ +export function validateSavedFilterValues( + filters: Filter[], +): SavedFilterValueIssue[] { + const issues: SavedFilterValueIssue[] = []; + filters.forEach((filter, index) => { + if (filter.type !== 'lucene' && filter.type !== 'sql') return; + const condition = filter.condition; + if (!condition.trim()) return; + if (!isValidFilterCondition(condition, filter.type)) { + issues.push({ index, language: filter.type, condition }); + } + }); + return issues; +} + +export type SavedQueryIssue = { + /** Query language the saved query claims to be written in */ + language: 'lucene' | 'sql'; + /** The raw saved query string that failed to parse */ + query: string; +}; + +/** + * Validate a dashboard's default saved query (the `where` clause applied to the + * whole dashboard). Like the other import-time validators this only checks that + * the query *parses* as its declared language. Returns a single issue or `null`. + * + * Empty / whitespace-only queries are treated as valid (no-ops), and a query in + * a non-statically-validated language (`promql`) is treated as valid. A missing + * language defaults to `lucene`, mirroring how the dashboard page resolves it. + * + * A malformed saved query is comparatively low impact at import time — it's + * surfaced in the dashboard's search bar where the user can see and edit it — + * but validating it keeps the import warnings consistent and avoids silently + * carrying over a broken default query. + */ +export function validateSavedQuery( + savedQuery: string | null | undefined, + language: 'lucene' | 'sql' | 'promql' | null | undefined, +): SavedQueryIssue | null { + if (!savedQuery?.trim()) return null; + const lang = language ?? 'lucene'; + if (lang !== 'lucene' && lang !== 'sql') return null; + if (isValidFilterCondition(savedQuery, lang)) return null; + return { language: lang, query: savedQuery }; +} + +export type DashboardFilterQueryIssue = { + /** ID of the offending dashboard filter */ + filterId: string; + /** Display name of the offending dashboard filter */ + filterName: string; + /** Query language of the filter's `where` clause */ + language: 'lucene' | 'sql'; + /** The raw `where` clause that failed to parse */ + where: string; +}; + +/** + * Validate the `where` clause of each dashboard filter *definition* (the query + * that scopes which values populate the filter's dropdown). + * + * Useful at import time, where no values query is actually run: a filter whose + * `where` clause is malformed would otherwise only surface as a failed query + * after opening the dashboard. Returns one issue per filter whose `where` + * fails to parse as its declared language. + * + * Note: this only checks that the `where` clause *parses*. It cannot catch a + * `where`/`expression` that references a non-existent column — that only fails + * when the query runs against ClickHouse. + */ +export function validateDashboardFilterQueries( + filters: DashboardFilter[], +): DashboardFilterQueryIssue[] { + const issues: DashboardFilterQueryIssue[] = []; + for (const filter of filters) { + const where = filter.where ?? ''; + if (!where.trim()) continue; + const language = filter.whereLanguage ?? 'sql'; + if (language !== 'lucene' && language !== 'sql') continue; + if (!isValidFilterCondition(where, language)) { + issues.push({ + filterId: filter.id, + filterName: filter.name, + language, + where, + }); + } + } + return issues; +} From e03971b09a3b9c8432f0df44f11d404a3a5a1404 Mon Sep 17 00:00:00 2001 From: Elizabet Oliveira Date: Fri, 5 Jun 2026 14:57:26 +0100 Subject: [PATCH 026/728] refactor(theme): rename chart palette tokens to hue names + unify across themes (#2362) Co-authored-by: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 Co-authored-by: Cursor --- .changeset/rename-chart-palette-tokens.md | 85 +++ agent_docs/data_viz_colors.md | 541 +++++++++++------- packages/api/src/controllers/dashboard.ts | 35 +- .../routers/api/__tests__/dashboard.test.ts | 75 +++ packages/api/src/routers/api/dashboards.ts | 30 + .../__tests__/provisionDashboards.test.ts | 31 + .../src/tasks/provisionDashboards/index.ts | 22 +- packages/app/src/DBDashboardImportPage.tsx | 9 + packages/app/src/HDXMultiSeriesTimeChart.tsx | 36 +- packages/app/src/Spotlights.tsx | 4 +- packages/app/src/__tests__/ChartUtils.test.ts | 16 +- .../__tests__/HDXMultiSeriesTimeChart.test.ts | 55 ++ .../app/src/__tests__/Spotlights.test.tsx | 6 +- .../src/__tests__/dashboard.remote.test.ts | 285 +++++++++ packages/app/src/__tests__/dashboard.test.ts | 81 +++ packages/app/src/__tests__/utils.test.ts | 67 +-- packages/app/src/api.ts | 65 +-- .../ChartEditor/__tests__/utils.test.ts | 4 +- .../components/ColorSwatchInput.stories.tsx | 2 +- .../src/components/ColorSwatchInput.test.tsx | 52 +- .../app/src/components/ColorSwatchInput.tsx | 30 +- packages/app/src/components/DBNumberChart.tsx | 20 +- packages/app/src/components/DBRowTable.tsx | 21 +- .../ChartDisplaySettingsDrawer.test.tsx | 8 +- .../__tests__/DBNumberChart.test.tsx | 20 + packages/app/src/dashboard.ts | 100 +++- .../app/src/theme/ChartColors.stories.tsx | 41 +- .../app/src/theme/semanticColorsGrouped.ts | 22 +- .../themes/_chart-categorical-tokens.scss | 62 ++ .../src/theme/themes/clickstack/_tokens.scss | 67 +-- .../app/src/theme/themes/hyperdx/_tokens.scss | 61 +- packages/app/src/utils.ts | 352 +++++++----- .../common-utils/src/__tests__/guards.test.ts | 161 +++++- packages/common-utils/src/types.ts | 211 ++++++- 34 files changed, 2014 insertions(+), 663 deletions(-) create mode 100644 .changeset/rename-chart-palette-tokens.md create mode 100644 packages/app/src/__tests__/HDXMultiSeriesTimeChart.test.ts create mode 100644 packages/app/src/__tests__/dashboard.remote.test.ts create mode 100644 packages/app/src/theme/themes/_chart-categorical-tokens.scss diff --git a/.changeset/rename-chart-palette-tokens.md b/.changeset/rename-chart-palette-tokens.md new file mode 100644 index 0000000000..cea4e38ac2 --- /dev/null +++ b/.changeset/rename-chart-palette-tokens.md @@ -0,0 +1,85 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +'@hyperdx/api': patch +--- + +refactor(theme): rename chart palette tokens from chart-1..10 to hue-named +(chart-blue, chart-orange, ...) and unify the categorical palette across HyperDX +and ClickStack + +Stored configs from the initial color picker (#2265) keep working. +`ChartPaletteTokenSchema` stays strict (a plain `z.enum`, so its `z.input` +matches `z.output` — wrapping it in `z.preprocess` would poison +`validateRequest`'s `req.body` inference all the way up to +`Dashboard.tiles[i].config.color`). Migration of legacy `chart-1` .. `chart-10` +happens at five complementary points so no entry or wire-format path can slip +through, all composing over a single shared walker +(`walkRawDashboardTileColors` in `common-utils`) so the per-tile traversal +stays in lockstep: + +- **Fetch-time / write-time (React)**: `normalizeDashboardTileColors` in + `packages/app/src/dashboard.ts` heals dashboards on read + (`useDashboards` / `fetchLocalDashboards` / `fetchDashboards`) and on write + (`useUpdateDashboard` / `useCreateDashboard`). Unresolvable color strings + (stale hexes, hand-edited values, forward-rolled future tokens) are + preserved so the user's chosen value survives a render pass — the strict + server-side schema surfaces a clear error on next save instead of the + normalizer quietly dropping the field. +- **JSON import**: `DBDashboardImportPage` runs + `normalizeRawDashboardTileColors` on the parsed JSON *before* the strict + `DashboardTemplateSchema.safeParse`, so templates exported from a + pre-rename deploy import cleanly. +- **Server-side GET response healing**: `getDashboards` / `getDashboard` in + `packages/api/src/controllers/dashboard.ts` rewrite legacy tile colors on + the way out. Pre-rename Mongo docs are served on the wire as + hue-named tokens so non-React HTTP clients (CI scripts, stale bundle + tabs during a rolling deploy, the external API) can round-trip + GET → PATCH without ever resurrecting `chart-N` through the strict + schema. +- **Server-side write shim**: the dashboards POST / PATCH routes mount + a request-body preprocessor that rewrites legacy tile colors before + `validateRequest` runs `ChartPaletteTokenSchema`. Catches non-React + HTTP callers (stale-bundle tabs during a rolling deploy, CI scripts, + MCP, the upcoming external-API parity work) for a one-release + deprecation window without weakening the schema's input/output equality. + The dashboard provisioner task applies the same shim before parsing + on-disk template files. +- **Render-time (belt-and-suspenders)**: `DBNumberChart` and + `ColorSwatchInput` also call `resolveChartPaletteToken` for tiles + constructed in memory between fetch and save (`ChartEditor` form + state, unit-test fixtures, hand-rolled `Tile` literals). + +The migration preserves the HyperDX slot ordering from #2265 (slot 1 = brand +green, slot 2 = blue, etc.). + +**ClickStack legacy color caveat:** Pre-rename ClickStack used a different slot +ordering than HyperDX (`--color-chart-1` was brand blue `#437eef`, not brand +green). The migration map uses HyperDX slot ordering, so any ClickStack +dashboard saved via #2265 with `color: 'chart-1'` will flip from blue to +Observable green after migration. We chose this trade-off deliberately over +branching the legacy map by active theme: `LEGACY_CHART_PALETTE_TOKEN_MAP` lives +in `common-utils` (shared with the API), and migration is one-shot persisted on +next save — theme-branching would couple common-utils to browser DOM state and +still produce wrong results for users whose active theme changed since the +original pick. Affected users can manually re-pick the desired hue via the (now +hue-labeled) color picker. + +The categorical palette is based on Observable 10, with `chart-blue` swapped to +`#437eef` to match the brand link color +(`--click-global-color-text-link-default`); all other hues are straight from +Observable 10. The palette resolves identically on both themes — picking +`chart-blue` always renders the brand blue. Brand identity for charts moves +entirely into the semantic layer: `--color-chart-success` and `--color-chart-info` +resolve to categorical `chart-green` (`#3ca951`) and `chart-blue` (`#437eef`) on +both HyperDX and ClickStack, so success fills, info-level logs, and the +matching multi-series slots all read consistently across brands. + +Internally, JS (`CATEGORICAL_HEX_BY_TOKEN` in `packages/app/src/utils.ts`) is +the source of truth for categorical hues — `getColorFromCSSVariable` and +`getColorFromCSSToken` skip `getComputedStyle` for categorical tokens since the +palette is unified across themes. The matching `--color-chart-{hue}` CSS vars in +`_tokens.scss` remain as a stylesheet-author affordance (inline `var()` use, +devtools inspection) and a hook for any future per-brand override. Semantic +tokens still resolve through `getComputedStyle` because they genuinely vary per +theme. diff --git a/agent_docs/data_viz_colors.md b/agent_docs/data_viz_colors.md index 72438f54c1..4f12ed08ea 100644 --- a/agent_docs/data_viz_colors.md +++ b/agent_docs/data_viz_colors.md @@ -1,114 +1,224 @@ # Data Visualization Colors -> Single source of truth for chart and visualization colors in HyperDX. -> Read this before adding, changing, or hard-coding a color in any chart, -> sparkline, heatmap, legend, status pill, or other data display. +> Single source of truth for chart and visualization colors in HyperDX. Read +> this before adding, changing, or hard-coding a color in any chart, sparkline, +> heatmap, legend, status pill, or other data display. ## TL;DR -There are **three** color systems for data viz, with three different -consumption patterns: +There are **three** color systems for data viz, with three different consumption +patterns: -| System | Use for | Source of truth | How to consume | -| ------------------------------ | ---------------------------------------- | ---------------------------------------- | --------------------------------------------- | -| **Categorical 1–10** | Multi-series line/bar/area/pie charts | CSS vars `--color-chart-1`..`-10` | `getColorProps(index, label)` in `utils.ts` | -| **Semantic (success/warn/err)**| Status indicators, log levels, deltas | CSS vars `--color-chart-{success,...}` | `getChartColorSuccess/Warning/Error()` | -| **Heatmap continuous** | `DBHeatmapChart` density gradients | `darkPalette`/`lightPalette` arrays | Imported directly from `DBHeatmapChart.tsx` | +| System | Use for | Source of truth | How to consume | +| ------------------------------------ | ------------------------------------- | ------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| **Categorical (10 hues)** | Multi-series line/bar/area/pie charts | `CATEGORICAL_HEX_BY_TOKEN` in `utils.ts` (CSS vars mirror it) | `getColorProps(index, label)` (positional) / `getColorFromCSSToken('chart-{hue}')` (by name) | +| **Semantic (success/warn/err/info)** | Status indicators, log levels, deltas | CSS vars `--color-chart-{success,warning,error,info}` | `getChartColor{Success,Warning,Error,Info}()` | +| **Heatmap continuous** | `DBHeatmapChart` density gradients | `darkPalette`/`lightPalette` arrays | Imported directly from `DBHeatmapChart.tsx` | **Hard rules**: - **Never** pass a hex color to a chart series. Always go through one of the helpers above so theme switching works. -- **Never** map log levels to raw Mantine colors (`red.5`, `yellow.6`). - Use `logLevelColor()` / `getColorProps()` — they pick the theme-correct - semantic chart color. +- **Never** map log levels to raw Mantine colors (`red.5`, `yellow.6`). Use + `logLevelColor()` / `getColorProps()` — they pick the theme-correct semantic + chart color. - The categorical palette and the heatmap palette are **different things**. - Don't reuse `--color-chart-N` for heatmap density; don't reuse the heatmap + Don't reuse `--color-chart-{hue}` for heatmap density; don't reuse the heatmap arrays for series colors. ## Where the colors live -### Categorical series palette (`--color-chart-1` through `--color-chart-10`) - -| Theme | File | Index 1 (primary) | -| ----------- | ----------------------------------------------------------------------- | ----------------- | -| HyperDX | `packages/app/src/theme/themes/hyperdx/_tokens.scss` (lines ~95–117) | `#00c28a` brand green | -| ClickStack | `packages/app/src/theme/themes/clickstack/_tokens.scss` (lines ~175–201)| `#437eef` Observable blue | - -The same ten slots in both themes use the same hue families (blue, orange, -red, cyan, green, pink, purple, light blue, brown, gray) — only **slot 1** -differs: - -- **HyperDX** leads with brand green, then Observable colors. -- **ClickStack** leads with Observable blue (Click UI accent yellow doesn't - pass contrast on a typical chart background, so we don't use it as a series - color — see "Per-theme considerations" below). - -The vars are defined identically inside the dark and light selectors. That -duplication is intentional and called out in `_tokens.scss`: CSS specificity -requires it because the parent selectors `[data-mantine-color-scheme='dark']` -and `[data-mantine-color-scheme='light']` would otherwise drop the vars on -scheme switch. - -### Semantic chart colors +### Categorical palette (`--color-chart-{hue}`) + +The categorical palette is **unified across themes** — based on Observable 10 +([d3 schemeObservable10](https://observablehq.com/@d3/color-schemes)), with +`chart-blue` swapped to `#437eef` so it matches the brand link color +(`--click-global-color-text-link-default` for ClickStack, same hue for HyperDX +series). All other hues are straight from Observable 10. Identical on HyperDX +and ClickStack: + +| Token | Hex | +| ------------------ | --------- | +| `chart-blue` | `#437eef` | +| `chart-orange` | `#efb118` | +| `chart-red` | `#ff725c` | +| `chart-cyan` | `#6cc5b0` | +| `chart-green` | `#3ca951` | +| `chart-pink` | `#ff8ab7` | +| `chart-purple` | `#a463f2` | +| `chart-light-blue` | `#97bbf5` | +| `chart-brown` | `#9c6b4e` | +| `chart-gray` | `#9498a0` | + +**Source of truth for categorical hues lives in JS.** The matching +`--color-chart-{hue}` CSS vars in both +`packages/app/src/theme/themes/hyperdx/_tokens.scss` and +`packages/app/src/theme/themes/clickstack/_tokens.scss` are a stylesheet-author +affordance only — `getColorFromCSSVariable` and `getColorFromCSSToken` skip +`getComputedStyle` for categorical tokens and return values straight from +`CATEGORICAL_HEX_BY_TOKEN` in `utils.ts`. The palette is the same on every theme +today, so a DOM round-trip per series buys nothing. + +The CSS vars exist for: + +- SCSS modules and inline `style={{ background: 'var(--color-chart-blue)' }}` + consumers (no JS import needed). +- Devtools inspection while debugging chart styling. +- Forward-compat: if a future brand wants to override hues, switch + `getColorFromCSSVariable`/`getColorFromCSSToken` back to reading the var and + add per-brand entries to `CATEGORICAL_HEX_BY_TOKEN`. + +The categorical hues and semantic chart colors live in a single shared partial, +`packages/app/src/theme/themes/_chart-categorical-tokens.scss` (`chart-categorical-tokens` +and `chart-semantic-tokens` mixins). Both brand themes `@use` it and `@include` +both mixins inside their per-theme `chart-tokens` mixin. Each theme's +`chart-tokens` mixin is then `@include`'d inside both `[data-mantine-color-scheme]` +selectors. Sass inlines the bodies at each call site, so the emitted CSS has the +same per-scheme specificity as a hand-duplicated block would — but the source +lives in **one place** for both layers. **If you change a categorical hex in the +shared partial, change it in `CATEGORICAL_HEX_BY_TOKEN` in `utils.ts` too. If +you change a semantic hex, update `SEMANTIC_CHART_PALETTE` in `utils.ts` too — +the SCSS and JS sources are intentionally mirrored.** + +Brand identity for charts is carried by non-chart UI chrome (Mantine accent, +sidebar gradient, Click UI globals), not by per-brand chart semantic colors — +`success` and `info` reuse categorical `chart-green` and `chart-blue` on both +brands. + +### Semantic chart colors (`--color-chart-{success|warning|error|info}`) ```text ---color-chart-success # success / OK / ingested / info-level fills +--color-chart-success # success / OK / ingested fills --color-chart-warning # warnings, throttling, slowdowns --color-chart-error # failures, errors, alerts firing +--color-chart-info # info-level logs, neutral "primary" series --color-chart-success-highlight # hover/selected variants (lighter shades) --color-chart-warning-highlight --color-chart-error-highlight ``` -Defined in both `_tokens.scss` files. **HyperDX success uses brand green -(`#00c28a`)**; **ClickStack success uses Observable green (`#3ca951`)** so it -doesn't collide with the yellow brand accent. Warning and error are the same -across themes (orange `#efb118`, red `#ff725c`). +Defined in both `_tokens.scss` files. **Unified on both brands**: `success` +reuses categorical `chart-green` (`#3ca951`); `info` reuses categorical +`chart-blue` (`#437eef`). Warning and error use the same Observable hexes across +themes. -### JavaScript fallback (`packages/app/src/utils.ts`) +Unlike the categorical hues, **the semantic CSS vars are read at runtime** via +`getComputedStyle` (see `getSemanticChartColor` in `utils.ts`). That keeps +inline `var(--color-chart-warning)` consumers like `LogLevel.tsx` reacting to +theme switches without a React re-render, and it lets JS callers like +`getChartColorSuccess()` return the correct hex for the active theme. -The CSS vars are the source of truth at runtime, but two palette objects in -`utils.ts` are the SSR fallback **and** the storybook reference: +### Single source of truth in TS (`packages/app/src/utils.ts`) ```text -CHART_PALETTE # HyperDX (green-first), lines ~356-374 -CLICKSTACK_CHART_PALETTE # ClickStack (blue-first), lines ~376-393 -COLORS # Exported, ordered, HyperDX-default array, lines ~398-409 +CATEGORICAL_HEX_BY_TOKEN # { 'chart-blue': '#437eef', ... } — file-private, + # authoritative for categorical hues +SEMANTIC_CHART_PALETTE # { hyperdx: {...}, clickstack: {...} } — file-private, + # SSR/fallback for the semantic CSS vars +COLORS # ordered hex array, derived from CATEGORICAL_PALETTE_TOKENS ``` -`COLORS[0]` corresponds to `--color-chart-1`, `COLORS[1]` to `--color-chart-2`, -and so on. **Keep them in sync.** If you change a hex in one place, change it -in all three (HyperDX SCSS, ClickStack SCSS, and `CHART_PALETTE` / -`CLICKSTACK_CHART_PALETTE` / `COLORS`). +`COLORS[i]` equals `CATEGORICAL_HEX_BY_TOKEN[CATEGORICAL_PALETTE_TOKENS[i]]`. +`COLORS` is what `getColorFromCSSVariable(i)` returns — on both server and +client, since categorical hues no longer round-trip through the CSS var. + +The hue-named `CHART_PALETTE_TOKENS`, `CATEGORICAL_PALETTE_TOKENS`, and +`SEMANTIC_PALETTE_TOKENS` constants live in `packages/common-utils/src/types.ts` +so the Zod schema can reference them (shared with the API). ### Reader functions (`packages/app/src/utils.ts`) These are the only functions React code should call: -| Function | Returns | -| -------------------------------- | ---------------------------------------------------- | -| `getColorProps(index, level)` | Categorical color, with log-level override applied | -| `semanticKeyedColor(key, index)` | Same, but driven by `key` (e.g. series name) | -| `getChartColorSuccess()` | `var(--color-chart-success)` resolved to a hex string| -| `getChartColorWarning()` | `var(--color-chart-warning)` resolved | -| `getChartColorError()` | `var(--color-chart-error)` resolved | -| `getChartColor*Highlight()` | Hover/selected variants | -| `logLevelColor(key)` | Maps `'error' \| 'warn' \| 'info'` → semantic color | -| `getLogLevelColorOrder()` | Stable ordering for log-level series | +| Function | Returns | +| -------------------------------- | ----------------------------------------------------------- | +| `getColorProps(index, level)` | Categorical color by index, with log-level override applied | +| `semanticKeyedColor(key, index)` | Same, but driven by `key` (e.g. series name) | +| `getColorFromCSSToken(token)` | Resolves any `ChartPaletteToken` (categorical or semantic) | +| `getChartColorSuccess()` | `var(--color-chart-success)` resolved to a hex string | +| `getChartColorWarning()` | `var(--color-chart-warning)` resolved | +| `getChartColorError()` | `var(--color-chart-error)` resolved | +| `getChartColorInfo()` | `var(--color-chart-info)` resolved (brand-primary for info) | +| `getChartColor*Highlight()` | Hover/selected variants | +| `logLevelColor(key)` | Maps `'error' \| 'warn' \| 'info'` → semantic color | +| `getLogLevelColorOrder()` | Stable ordering for log-level series | Internals worth knowing: -- `getColorFromCSSVariable(index)` reads `--color-chart-{index+1}` from - `documentElement` via `getComputedStyle`. On SSR or if the var is missing, - it falls back to `COLORS[index % COLORS.length]`. -- `getSemanticChartColor(varName, hyperdxFallback, clickstackFallback)` does - the same for the semantic vars and uses `detectActiveTheme()` (checks for - the `theme-clickstack` class on ``) to pick the correct fallback when - CSS isn't available. -- During SSR, semantic readers return the **HyperDX** default — the - hydration mismatch window is tiny because charts render after data fetch - on the client. +- `getColorFromCSSVariable(index)` returns `COLORS[index % 10]` directly — no + DOM read. Categorical hues are unified across themes, so the JS palette is + authoritative. +- `getColorFromCSSToken(token)` is split: + - Categorical tokens (`chart-blue`, etc.) come straight from + `CATEGORICAL_HEX_BY_TOKEN` — same shortcut, no `getComputedStyle`. + - Semantic tokens (`chart-success`, `-warning`, `-error`) read + `--color-{token}` from `documentElement` via `getComputedStyle` and fall + back to the active theme's `SEMANTIC_CHART_PALETTE` entry when running + server-side or when the DOM read fails. +- `getSemanticChartColor(varName, key)` is the shared helper that backs the + `getChartColor{Success,Warning,Error,Info}()` readers. It uses + `detectActiveTheme()` (checks for the `theme-clickstack` class on ``) to + pick the correct per-brand fallback. +- During SSR, semantic readers return the **HyperDX** default — the hydration + mismatch window is tiny because charts render after data fetch on the client. + +### Legacy `chart-1` … `chart-10` tokens + +The number-tile color picker (#2265) initially shipped with numeric tokens +(`chart-1` … `chart-10`). Renamed here to hue-named tokens so stored configs and +the upcoming external API surface are self-documenting. + +Existing stored configs keep working. The mapping: + +```text +chart-1 -> chart-green (was HyperDX brand green at slot 1) +chart-2 -> chart-blue (was HyperDX slot 2) +chart-3 -> chart-orange +... +chart-10 -> chart-gray +``` + +It preserves the HyperDX slot ordering from #2265, so HyperDX users see no +visual change. + +> **⚠️ ClickStack legacy color caveat.** Pre-rename ClickStack used a different +> slot ordering than HyperDX (`--color-chart-1` was brand blue `#437eef`, not +> brand green; `--color-chart-2` was orange, not blue; etc.). Because the +> migration map preserves HyperDX slot ordering, any ClickStack dashboard saved +> via #2265 will visually shift: stored `chart-1` flips from brand blue to +> Observable green, `chart-2` flips from orange to blue, and so on. We chose +> this trade-off deliberately over branching the legacy map by active theme: +> `LEGACY_CHART_PALETTE_TOKEN_MAP` lives in `common-utils` (shared with the +> API), and migration is one-shot persisted on next save — theme-branching would +> couple common-utils to browser DOM state and still produce wrong results for +> users whose active theme changed since the original pick. Affected users can +> manually re-pick the desired hue via the (now hue-labeled) color picker. Use +> `chart-info` semantic if you need the brand-primary appearance. + +**`ChartPaletteTokenSchema` itself stays strict** (a plain `z.enum`). Wrapping +it in `z.preprocess` would force the schema's `z.input` type to `unknown`, which +poisons `validateRequest`'s `req.body` inference in the API package all the way +up to `Dashboard.tiles[i].config.color`. Strict input/output equality is more +important than a one-line runtime migration buried in the schema. + +Migration happens at two complementary layers instead: + +1. **Fetch-time _and_ write-time** — `normalizeDashboardTileColors` in + `packages/app/src/dashboard.ts` walks every tile and rewrites any legacy + `config.color` to its hue-named equivalent via `resolveChartPaletteToken`. It + runs on every read (`useDashboards` / `fetchLocalDashboards`) and on every + write (`useUpdateDashboard` / `useCreateDashboard`). The write-time pass is + what lets JSON imports (`DBDashboardImportPage`), presets, and + MCP-constructed payloads pass the strict server-side validator, and it + converges the DB-side data on next save instead of leaving legacy tokens in + storage forever. +2. **Render-time** — `DBNumberChart` and `ColorSwatchInput` also call + `resolveChartPaletteToken` as belt-and-suspenders for tiles constructed in + memory between fetch and save (e.g. `ChartEditor` form state, unit-test + fixtures, hand-rolled `Tile` literals). + +`LEGACY_CHART_PALETTE_TOKEN_MAP` and `resolveChartPaletteToken` live in +`packages/common-utils/src/types.ts` next to the enum. ### Heatmap palette (component-local) @@ -119,9 +229,9 @@ Internals worth knowing: - Selected at the call site by `useMantineColorScheme()` and `colorScheme === 'light' ? lightPalette : darkPalette`. -These are **scheme-aware, not brand-aware**: HyperDX dark and ClickStack -dark share the same heatmap gradient, same for light. Red is intentionally -omitted from the high end so it can be reserved for error overlays. +These are **scheme-aware, not brand-aware**: HyperDX dark and ClickStack dark +share the same heatmap gradient, same for light. Red is intentionally omitted +from the high end so it can be reserved for error overlays. `DBHeatmapChart.tsx` re-exports `darkPalette` and `lightPalette`. `DBSearchHeatmapChart.tsx` imports them via that re-export — **do not** @@ -133,55 +243,31 @@ A few component-local accents that are **not** part of the categorical or semantic palettes: - `ALL_SPANS_COLOR = 'var(--mantine-color-blue-6)'` in - `packages/app/src/components/deltaChartUtils.ts` — the "all spans" - reference bar in `DBDeltaChart`. Keep using this var; don't replace it - with `--color-chart-1` (it's a comparison reference, not a series). -- Trace waterfall span tints in `DBTraceWaterfallChart.tsx` — derived from - span attributes, not from this palette. + `packages/app/src/components/deltaChartUtils.ts` — the "all spans" reference + bar in `DBDeltaChart`. Keep using this var; don't replace it with a + categorical token (it's a comparison reference, not a series). +- Trace waterfall span tints in `DBTraceWaterfallChart.tsx` — derived from span + attributes, not from this palette. ## Storybook reference -The visual reference for the categorical and semantic palettes is the -storybook story at: - -```1:22:packages/app/src/theme/ChartColors.stories.tsx -import React from 'react'; - -import { - COLORS, - getChartColorError, - getChartColorSuccess, - getChartColorWarning, -} from '@/utils'; - -// Labels for chart colors - brand green first, then Observable palette -const COLOR_LABELS = [ - 'Green (Brand)', - 'Blue', - 'Orange', - 'Red', - 'Cyan', - 'Pink', - 'Purple', - 'Light Blue', - 'Brown', - 'Gray', -]; -``` +The visual reference for the categorical and semantic palettes is the storybook +story at `packages/app/src/theme/ChartColors.stories.tsx`. It renders +`AllChartColors`, `BarChartPreview`, `LineChartPreview`, +`SemanticColorsPreview`, and `AccessibilityCheck`. Run storybook in the `app` +package to inspect both schemes side by side. -It renders `AllChartColors`, `BarChartPreview`, `LineChartPreview`, -`SemanticColorsPreview`, and `AccessibilityCheck`. Run storybook in the -`app` package to inspect both schemes side by side. +The number-tile color picker (`ColorSwatchInput.stories.tsx`) renders the same +tokens through the user-facing picker UI. ## How to consume (recipes) ### Multi-series time-series chart `ChartUtils.tsx → setLineColors` already wires the categorical palette per -series via `getColorProps(index, level)` in -`packages/app/src/ChartUtils.tsx`. **You should not have to think about -chart colors when adding a new chart that goes through `seriesToTimeSeries` -/ `setLineColors`** — they handle it. +series via `getColorProps(index, level)` in `packages/app/src/ChartUtils.tsx`. +**You should not have to think about chart colors when adding a new chart that +goes through `seriesToTimeSeries` / `setLineColors`** — they handle it. If you're rendering a custom chart outside that pipeline: @@ -195,9 +281,26 @@ const series = data.map((s, i) => ({ })); ``` -`level` (the second arg) is used to override with semantic colors when the -label looks like a log level (`'error'`, `'warn'`, `'info'`, etc.). Pass -`s.label` if it might encode a log level, otherwise pass an empty string. +`level` (the second arg) is used to override with semantic colors when the label +looks like a log level (`'error'`, `'warn'`, `'info'`, etc.). Pass `s.label` if +it might encode a log level, otherwise pass an empty string. + +### Identity color ("this thing is always blue") + +For UI surfaces that should always render a specific hue regardless of +multi-series ordering: + +```tsx +import { getColorFromCSSToken } from '@/utils'; + +Always blue; +``` + +Or directly in CSS: + +```tsx + +``` ### Status pill / delta indicator @@ -209,8 +312,8 @@ import { getChartColorError, getChartColorSuccess } from '@/utils'; ``` These functions return resolved hex strings, so they can be used in inline -styles or passed to libraries that don't understand CSS vars (e.g. -`uPlot`'s canvas fills). +styles or passed to libraries that don't understand CSS vars (e.g. `uPlot`'s +canvas fills). If you're styling a DOM element with regular CSS, prefer the var directly: @@ -218,9 +321,15 @@ If you're styling a DOM element with regular CSS, prefer the var directly: ``` -The var route reacts instantly to theme switches without re-running React. -Use the function form only when you need a string at compute time -(canvas/WebGL, library config objects, etc.). +The var route reacts instantly to theme switches without re-running React. Use +the function form only when you need a string at compute time (canvas/WebGL, +library config objects, etc.). + +### User-customizable chart color (number tile etc.) + +`ColorSwatchInput` stores the choice as a `ChartPaletteToken`. Resolve to a hex +at paint time via `getColorFromCSSToken(token)`. Never store hex strings in +chart configs — tokens reflow correctly across themes and color modes. ### Heatmap @@ -232,15 +341,15 @@ const { colorScheme } = useMantineColorScheme(); const palette = colorScheme === 'light' ? lightPalette : darkPalette; ``` -That's the only correct way to consume the heatmap gradient. Never -reconstruct it. +That's the only correct way to consume the heatmap gradient. Never reconstruct +it. ### Pie / donut where slice order matters -The categorical palette is **ordered for distinguishability** — adjacent -slots are designed to contrast. If you're drawing a pie chart where the -largest slice should always be the most prominent color, sort your data -first and let the index map to the palette naturally: +The categorical palette is **ordered for distinguishability** — adjacent slots +are designed to contrast. If you're drawing a pie chart where the largest slice +should always be the most prominent color, sort your data first and let the +index map to the palette naturally: ```tsx const sorted = data.toSorted((a, b) => b.value - a.value); @@ -250,125 +359,143 @@ const slices = sorted.map((d, i) => ({ })); ``` -`ChartUtils.tsx → buildPieChartData` already does this — the comment "Sort -in descending order so the largest slice is always first and gets the -first color in the palette" is at line ~444. +`ChartUtils.tsx → buildPieChartData` already does this. ## Per-theme considerations -### HyperDX (green-first) +The categorical palette is identical on both themes — Observable 10. Semantic +`success` and `info` also reuse categorical hues on both brands. + +### HyperDX -- Slot 1 is brand green (`#00c28a`) so single-series charts feel - on-brand without any extra config. -- Semantic success **also** uses brand green, so success indicators and - primary series share a hue. This is intentional but worth knowing: - if a chart juxtaposes a "success" pill with a green series, that's - expected, not a bug. +- `--color-chart-success` uses categorical `chart-green` (`#3ca951`). +- `--color-chart-info` uses categorical `chart-blue` (`#437eef`), so info-level + logs and `getChartColorInfo()` render the same blue as multi-series slot 0. +- Multi-series charts start at brand blue (slot 0, `#437eef`) and proceed + through the canonical palette — brand identity is preserved via the Mantine + green accent and sidebar gradient, not via chart semantic colors. -### ClickStack (blue-first) +### ClickStack -- The brand accent is **yellow** (`--palette-brand-300: #faff69`). It is - **not** in the chart palette and should not be added — yellow on a - light background fails contrast, and yellow as a series color reads as - "warning" in most contexts. -- Slot 1 falls back to Observable blue (`#437eef`). -- Semantic success is **Observable green** (`#3ca951`), distinct from - the yellow brand accent. Don't try to "brand" success with yellow. +- The brand accent is **yellow** (`--palette-brand-300: #faff69`). It is **not** + in the chart palette and should not be added — yellow on a light background + fails contrast, and yellow as a series color reads as "warning" in most + contexts. +- `--color-chart-success` uses categorical `chart-green` (`#3ca951`). +- `--color-chart-info` uses categorical `chart-blue` (`#437eef`, same as + `--click-global-color-text-link-default`), so info-level logs and + `getChartColorInfo()` render the brand blue. ### Both themes -- The dark/light scheme split is handled by CSS (the same vars get - redefined inside each scheme selector). React code does not need to - branch on scheme except for the heatmap palette. -- The chart vars in `_tokens.scss` are intentionally duplicated across - the dark and light blocks. If you change one, change the other. +- The dark/light scheme split is handled by CSS (the same vars get redefined + inside each scheme selector). React code does not need to branch on scheme + except for the heatmap palette. +- The chart vars in `_tokens.scss` are intentionally duplicated across the dark + and light blocks. If you change one, change the other. ## Adding new entries -### A new categorical slot (slot 11+) +### A new categorical hue (11th token) -Don't, unless you have a real need. Ten distinguishable hues is the upper -bound for readable categorical legends — beyond that, viewers can't tell -slices apart, and color-blind viewers definitely can't. Solutions in -descending order of preference: +Don't, unless you have a real need. Ten distinguishable hues is the upper bound +for readable categorical legends — beyond that, viewers can't tell slices apart, +and color-blind viewers definitely can't. Solutions in descending order of +preference: 1. **Group small categories** into "Other" before charting. 2. **Reuse slots** with patterns/strokes/labels for disambiguation. -3. If you really must extend: add the same `--color-chart-N` var to - **all four** SCSS blocks (HyperDX dark, HyperDX light, ClickStack - dark, ClickStack light), append the hex to `CHART_PALETTE` / - `CLICKSTACK_CHART_PALETTE`, append to `COLORS`, append a label to - `COLOR_LABELS` in `ChartColors.stories.tsx`. +3. If you really must extend: add `--color-chart-{newhue}` to the shared + `_chart-categorical-tokens.scss` partial — one edit covers both brands and + both schemes via the existing `@include` chain. Then append + `'chart-{newhue}'` to `CHART_PALETTE_TOKENS` in `common-utils/src/types.ts`, + add the hex to `CATEGORICAL_HEX_BY_TOKEN` in `utils.ts`, and add a label + entry in `ColorSwatchInput.tsx` → `TOKEN_LABELS` and + `ChartColors.stories.tsx` → `COLOR_LABELS`. ### A new semantic color (e.g. `--color-chart-pending`) -1. Pick the hex per theme (HyperDX often uses brand variants; - ClickStack uses Observable variants). -2. Add `--color-chart-pending` and (if needed) - `--color-chart-pending-highlight` to **all four** SCSS blocks. -3. Add a hex to `CHART_PALETTE` and `CLICKSTACK_CHART_PALETTE`. -4. Add a `getChartColorPending()` reader in `utils.ts` that calls - `getSemanticChartColor('--color-chart-pending', hyperdxHex, clickstackHex)`. -5. Update `ChartColors.stories.tsx` `SEMANTIC_CHART_COLORS` so the new - color shows up in the design-tokens story. -6. If it should override based on a label / status string, extend +1. Pick the hex (unified across brands unless you have a deliberate per-brand split). +2. Add `--color-chart-pending` and (if needed) `--color-chart-pending-highlight` + to the `@mixin chart-semantic-tokens` block in + `_chart-categorical-tokens.scss`. The mixin is `@include`'d in each theme's + `chart-tokens` mixin, which is `@include`'d in each scheme selector — one edit + covers dark and light for both brands. For a per-brand override only, declare + the var in that brand's `chart-tokens` mixin after the shared `@include`. +3. Add `pending` (and optionally `pendingHighlight`) to + `SEMANTIC_CHART_PALETTE.hyperdx` and `.clickstack` in `utils.ts`. +4. Append `'chart-pending'` to `CHART_PALETTE_TOKENS` (and + `SEMANTIC_PALETTE_TOKENS` slice will pick it up automatically) in + `common-utils/src/types.ts`. +5. Add a `getChartColorPending()` reader in `utils.ts` that calls + `getSemanticChartColor('--color-chart-pending', 'pending')`. +6. Update `ChartColors.stories.tsx` `SEMANTIC_CHART_COLORS` so the new color + shows up in the design-tokens story. +7. Add a `'chart-pending': 'Pending'` entry to `TOKEN_LABELS` in + `ColorSwatchInput.tsx`. +8. If it should override based on a label / status string, extend `getLevelColor` / `logLevelColor` accordingly. ### A new heatmap palette The current arrays were tuned to (a) be visible on the chart's dark/light -background, (b) avoid red at the high end so error overlays stay -readable, and (c) follow a perceptually monotonic luminance ramp. -Don't add a new palette without those three properties verified — both -"physically dim" and "perceptually noisy" gradients hurt readability. +background, (b) avoid red at the high end so error overlays stay readable, and +(c) follow a perceptually monotonic luminance ramp. Don't add a new palette +without those three properties verified — both "physically dim" and +"perceptually noisy" gradients hurt readability. If you do need to add one (e.g. for a different chart type), keep it -component-local like the existing ones, and re-use the same scheme-pick -pattern (`useMantineColorScheme` + ternary). +component-local like the existing ones, and re-use the same scheme-pick pattern +(`useMantineColorScheme` + ternary). ## Anti-patterns ```tsx - // hex string in a chart - // CSS color keyword - // raw Mantine palette for a status - -import { CHART_PALETTE } from '@/utils'; // CHART_PALETTE is not exported + // hex string in a chart + // CSS color keyword + // raw Mantine palette for a status + // palette object is module-private ``` Why each is wrong: -- **Hex / keyword**: bypasses theme switching. ClickStack users will see - HyperDX colors, dark/light won't react. -- **Raw Mantine**: bypasses semantic mapping. `green.5` is not the same - as `--color-chart-success` once we tweak the brand palette. -- **Importing palette objects**: they're module-private. The exported - surface is the reader functions and `COLORS` (the SSR fallback array, - not for direct use in components). +- **Hex / keyword**: bypasses theme switching. Dark/light won't react, and any + future per-theme tweak won't propagate. +- **Raw Mantine**: bypasses semantic mapping. `green.5` is not the same as + `--color-chart-success` once we tweak the brand palette. +- **Importing palette objects**: `CATEGORICAL_HEX_BY_TOKEN` and + `SEMANTIC_CHART_PALETTE` are module-private. The exported surface is the + reader functions, the token enums, and `COLORS` (the SSR fallback array — + prefer `getColorProps` / `getColorFromCSSToken` over indexing it directly). ## Pre-merge checklist for chart-touching PRs -- [ ] Toggled HyperDX ↔ ClickStack theme — series colors change as - expected (slot 1 swaps green ↔ blue). -- [ ] Toggled dark ↔ light — categorical and semantic colors stay legible - on both backgrounds; heatmap gradient flips palettes. -- [ ] Status indicators use `getChartColor*()` / `var(--color-chart-*)`, - not raw Mantine colors. +- [ ] Toggled HyperDX ↔ ClickStack theme — semantic success and info stay + unified (chart-green / chart-blue); categorical palette stays identical. +- [ ] Toggled dark ↔ light — categorical and semantic colors stay legible on + both backgrounds; heatmap gradient flips palettes. +- [ ] Status indicators use `getChartColor*()` / `var(--color-chart-*)`, not raw + Mantine colors. - [ ] No new hex strings in chart components — all colors flow through `utils.ts` helpers or CSS vars. -- [ ] If you added or changed a hex, changed it in **all four** places - (HyperDX SCSS, ClickStack SCSS, `CHART_PALETTE` / - `CLICKSTACK_CHART_PALETTE`, `COLORS`). +- [ ] If you added or changed a categorical hex, changed it in **both** places + (`_chart-categorical-tokens.scss` shared partial, + `CATEGORICAL_HEX_BY_TOKEN` in `utils.ts`). +- [ ] If you added or changed a semantic hex, changed it in **both** places + (`chart-semantic-tokens` mixin in `_chart-categorical-tokens.scss`, + `SEMANTIC_CHART_PALETTE.{theme}` in `utils.ts`). - [ ] Storybook `Design Tokens / Chart Colors` still renders correctly. ## File reference summary -| What | Where | -| ------------------------------------------ | --------------------------------------------------------------------------- | -| HyperDX chart vars (dark + light) | `packages/app/src/theme/themes/hyperdx/_tokens.scss` | -| ClickStack chart vars (dark + light) | `packages/app/src/theme/themes/clickstack/_tokens.scss` | -| JS palettes + reader functions | `packages/app/src/utils.ts` | -| Multi-series wiring (`setLineColors` etc.) | `packages/app/src/ChartUtils.tsx` | -| Heatmap palettes | `packages/app/src/components/DBHeatmapChart.tsx` (`darkPalette`, `lightPalette`) | -| Storybook visual reference | `packages/app/src/theme/ChartColors.stories.tsx` | -| Delta "all spans" reference color | `packages/app/src/components/deltaChartUtils.ts` (`ALL_SPANS_COLOR`) | +| What | Where | +| --------------------------------------------------- | -------------------------------------------------------------------------------- | +| Shared chart vars (10 categorical hues + semantics) | `packages/app/src/theme/themes/_chart-categorical-tokens.scss` | +| Per-theme `chart-tokens` mixin (`@include` chain) | `packages/app/src/theme/themes/hyperdx/_tokens.scss`, `clickstack/_tokens.scss` | +| JS palette objects + reader functions | `packages/app/src/utils.ts` | +| Palette token enum + legacy migration | `packages/common-utils/src/types.ts` | +| Multi-series wiring (`setLineColors` etc.) | `packages/app/src/ChartUtils.tsx` | +| Number-tile color picker | `packages/app/src/components/ColorSwatchInput.tsx` | +| Heatmap palettes | `packages/app/src/components/DBHeatmapChart.tsx` (`darkPalette`, `lightPalette`) | +| Storybook visual reference | `packages/app/src/theme/ChartColors.stories.tsx` | +| Delta "all spans" reference color | `packages/app/src/components/deltaChartUtils.ts` (`ALL_SPANS_COLOR`) | diff --git a/packages/api/src/controllers/dashboard.ts b/packages/api/src/controllers/dashboard.ts index 92a93e4c17..01ab5c6e77 100644 --- a/packages/api/src/controllers/dashboard.ts +++ b/packages/api/src/controllers/dashboard.ts @@ -1,7 +1,9 @@ import { DashboardWithoutIdSchema, + resolveChartPaletteToken, SavedChartConfig, Tile, + walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import { map, partition, uniq } from 'lodash'; import { z } from 'zod'; @@ -25,6 +27,32 @@ function pickAlertsByTile(tiles: Tile[]) { }, {}); } +/** + * Rewrite any legacy `chart-1`..`chart-10` tile colors from #2265 in + * an already-serialized dashboard JSON to their hue-named equivalents + * before it leaves the server. Keeps the wire format on a single + * canonical vocabulary so non-React HTTP clients (CI scripts, stale + * bundle tabs during a rolling deploy, the upcoming external API + * surface) never have to know about the legacy values, and so a + * GET → unmodified PATCH round-trip on a Mongo-seeded legacy doc can + * never resurrect the legacy tokens through the strict server-side + * `ChartPaletteTokenSchema`. The React-side + * `normalizeDashboardTileColors` becomes redundant for the wire path + * after this lands but stays in place as defense in depth for + * `IS_LOCAL_MODE` and in-memory tile literals. + * + * Unresolvable strings (stale hexes, hand-edited values, forward-rolled + * future tokens) pass through untouched so the user's data is not + * silently dropped; the strict schema surfaces a clear error on next + * save. + */ +function healLegacyDashboardTileColors(dashboard: T): T { + return walkRawDashboardTileColors(dashboard, current => { + const resolved = resolveChartPaletteToken(current); + return resolved ?? current; + }) as T; +} + type TileForAlertSync = Pick & { config?: Pick | { alert?: IAlert | AlertDocument }; }; @@ -107,7 +135,8 @@ export async function getDashboards(teamId: ObjectId) { alert: alerts[`${d._id.toString()}:${t.id}`]?.[0], }, })), - })); + })) + .map(healLegacyDashboardTileColors); return dashboards; } @@ -120,13 +149,13 @@ export async function getDashboard(dashboardId: string, teamId: ObjectId) { getDashboardAlertsByTile(teamId, dashboardId), ]); - return { + return healLegacyDashboardTileColors({ ..._dashboard?.toJSON(), tiles: _dashboard?.tiles.map(t => ({ ...t, config: { ...t.config, alert: alerts[t.id]?.[0] }, })), - }; + }); } export async function createDashboard( diff --git a/packages/api/src/routers/api/__tests__/dashboard.test.ts b/packages/api/src/routers/api/__tests__/dashboard.test.ts index 415c95aa9d..6cafc53da3 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.test.ts @@ -80,6 +80,81 @@ describe('dashboard router', () => { ); }); + // Server-side migration shim for legacy `chart-1`..`chart-10` tokens + // shipped by #2265. Stale-bundle React clients during a rolling deploy + // and non-React HTTP callers (CI scripts, MCP, future external API + // writes) can still send the numeric tokens; the route normalizes + // them to hue-named equivalents *before* the strict + // `ChartPaletteTokenSchema` runs, otherwise the request would 400. + it('migrates legacy chart-N tile colors on POST', async () => { + const tileWithLegacyColor = makeTile(); + tileWithLegacyColor.config = { + ...tileWithLegacyColor.config, + color: 'chart-1' as any, + }; + + const created = await agent + .post('/dashboards') + .send({ + name: 'Legacy Color Dashboard', + tiles: [tileWithLegacyColor], + tags: [], + }) + .expect(200); + + expect(created.body.tiles[0].config.color).toBe('chart-green'); + }); + + it('migrates legacy chart-N tile colors on PATCH', async () => { + const created = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + + const patchedTile = { + ...created.body.tiles[0], + config: { ...created.body.tiles[0].config, color: 'chart-10' }, + }; + + const updated = await agent + .patch(`/dashboards/${created.body.id}`) + .send({ tiles: [patchedTile, ...created.body.tiles.slice(1)] }) + .expect(200); + + expect(updated.body.tiles[0].config.color).toBe('chart-gray'); + }); + + // Wire-format guarantee: GET emits canonical hue-named tokens even for + // dashboards that pre-date the rename and still hold `chart-1`..`chart-10` + // in Mongo. Without this, a non-React client (or a stale React bundle + // that bypasses `normalizeDashboardTileColors`) could GET a legacy + // color and round-trip it back through PATCH where the strict schema + // would 400. Bypasses the API to seed the legacy value directly into + // Mongo because the POST/PATCH middleware would otherwise rewrite it + // before it ever reached the DB. + // + // The dashboards router currently only exposes a list GET (no + // `/:id` single GET handler — single-dashboard reads on the React + // side go through `useDashboards` and filter client-side). The + // controller's `getDashboard` healer is still exercised in the same + // process: `updateDashboard` calls `getDashboard` internally before + // PATCH, so a follow-up no-op PATCH would surface a regression. + it('returns hue-named tokens on GET (list) for a Mongo-seeded legacy chart-N tile', async () => { + const tileWithLegacy = makeTile(); + (tileWithLegacy.config as any).color = 'chart-1'; + const seeded = await Dashboard.create({ + name: 'Pre-rename Dashboard', + team: team._id, + tiles: [tileWithLegacy], + tags: [], + }); + + const list = await agent.get('/dashboards').expect(200); + const fromList = list.body.find(d => d._id === seeded._id.toString()); + expect(fromList).toBeDefined(); + expect(fromList.tiles[0].config.color).toBe('chart-green'); + }); + it('sets createdBy and updatedBy on create and populates them in GET', async () => { const created = await agent .post('/dashboards') diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 8ac2ea2f30..71112b756a 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -3,6 +3,8 @@ import { DashboardWithoutIdSchema, PresetDashboard, PresetDashboardFilterSchema, + resolveChartPaletteToken, + walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import express from 'express'; import _ from 'lodash'; @@ -29,6 +31,32 @@ import { objectIdSchema } from '@/utils/zod'; // create routes that will get and update dashboards const router = express.Router(); +/** + * Heal legacy `chart-1`..`chart-10` tile colors from #2265 on the request + * body *before* `validateRequest` runs `ChartPaletteTokenSchema`. Keeps the + * schema strict (so `z.input` == `z.output` and `req.body` infers cleanly) + * while still accepting payloads from any non-React HTTP client whose + * stored values haven't yet been healed by the app-side normalizer + * (`normalizeDashboardTileColors` in `packages/app/src/dashboard.ts`). + * + * This is a one-release deprecation shim — once stored data has converged + * on the hue-named tokens, it can be removed in favor of straight-strict + * validation. The actual walk delegates to `walkRawDashboardTileColors` + * in common-utils so this middleware, the app-side normalizer, the JSON + * import path, and the provisioner all share the same per-tile traversal. + */ +const migrateLegacyDashboardTileColors: express.RequestHandler = ( + req, + _res, + next, +) => { + req.body = walkRawDashboardTileColors(req.body, current => { + const resolved = resolveChartPaletteToken(current); + return resolved ?? current; + }); + next(); +}; + router.get('/', async (req, res, next) => { try { const { teamId } = getNonNullUserWithTeam(req); @@ -43,6 +71,7 @@ router.get('/', async (req, res, next) => { router.post( '/', + migrateLegacyDashboardTileColors, validateRequest({ body: DashboardWithoutIdSchema, }), @@ -63,6 +92,7 @@ router.post( router.patch( '/:id', + migrateLegacyDashboardTileColors, validateRequest({ params: z.object({ id: objectIdSchema, diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.test.ts index 69ad2d3fd0..5991156d6f 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.test.ts @@ -81,6 +81,37 @@ describe('provisionDashboards', () => { expect(result).toHaveLength(1); expect(result[0].name).toBe('Test'); }); + + // Pins the inlined `migrateLegacyDashboardTileColorsRaw` walker + // (which delegates to `walkRawDashboardTileColors` in common-utils). + // Without the migration the strict `DashboardWithoutIdSchema` + // rejects the legacy enum and `readDashboardFiles` skips the file + // outright, so a provisioned dashboard authored against #2265 + // wouldn't ship at all. + it('migrates legacy chart-N tile colors before schema validation', () => { + const tile = makeTile(); + (tile.config as any).color = 'chart-1'; + fs.writeFileSync( + path.join(tmpDir, 'legacy.json'), + JSON.stringify({ name: 'Legacy', tiles: [tile], tags: [] }), + ); + const result = readDashboardFiles(tmpDir); + expect(result).toHaveLength(1); + expect((result[0].tiles[0].config as any).color).toBe('chart-green'); + }); + + // Early-return for files whose `tiles` is non-array (or missing). + // The walker leaves the payload untouched, so the file behaves + // exactly like any other schema-invalid input: skipped, not + // crashed. + it('does not crash on a file whose tiles is not an array', () => { + fs.writeFileSync( + path.join(tmpDir, 'tiles-string.json'), + JSON.stringify({ name: 'Bad Tiles', tiles: 'not-an-array', tags: [] }), + ); + expect(() => readDashboardFiles(tmpDir)).not.toThrow(); + expect(readDashboardFiles(tmpDir)).toEqual([]); + }); }); describe('syncDashboards', () => { diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index eb8c096792..79ac66ca7e 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -1,6 +1,8 @@ import { DashboardWithoutId, DashboardWithoutIdSchema, + resolveChartPaletteToken, + walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; import path from 'path'; @@ -12,6 +14,20 @@ import type { HdxTask } from '@/tasks/types'; import { ProvisionDashboardsTaskArgs } from '@/tasks/types'; import logger from '@/utils/logger'; +// Heal legacy `chart-1`..`chart-10` tile colors from #2265 before the +// strict `DashboardWithoutIdSchema` parse rejects them. Same policy as +// the React `normalizeDashboardTileColors` and the API router's +// `migrateLegacyDashboardTileColors`: hue tokens pass through, legacy +// numeric tokens are rewritten to hue-named equivalents, and unknown +// strings are left intact so the schema's native enum error surfaces +// in the warn log instead of silently dropping the field. +function migrateLegacyDashboardTileColorsRaw(raw: unknown): unknown { + return walkRawDashboardTileColors(raw, current => { + const resolved = resolveChartPaletteToken(current); + return resolved ?? current; + }); +} + export function readDashboardFiles(dir: string): DashboardWithoutId[] { let files: string[]; try { @@ -24,10 +40,12 @@ export function readDashboardFiles(dir: string): DashboardWithoutId[] { const dashboards: DashboardWithoutId[] = []; for (const file of files) { try { - const raw = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')); + const raw = migrateLegacyDashboardTileColorsRaw( + JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')), + ) as Record | null | undefined; const parsed = DashboardWithoutIdSchema.safeParse({ tags: [], - ...raw, + ...(raw as object), }); if (!parsed.success) { logger.warn( diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 0b8cd9dc07..25ad1fcca4 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -65,6 +65,7 @@ import { useBrandDisplayName } from './theme/ThemeProvider'; import api from './api'; import { useConnections } from './connection'; import { + normalizeRawDashboardTileColors, useCreateDashboard, useDashboards, useUpdateDashboard, @@ -162,6 +163,14 @@ function FileSelection({ return; } + // Heal legacy `chart-1`..`chart-10` tile colors from #2265 *before* + // the strict `DashboardTemplateSchema.safeParse` runs. The schema's + // `ChartPaletteTokenSchema` is a plain `z.enum` (no `z.preprocess`), + // so a template exported from a pre-rename deploy would otherwise + // fail import with an opaque enum error and never reach the + // write-time normalizer in `useCreateDashboard`. + data = normalizeRawDashboardTileColors(data); + const result = DashboardTemplateSchema.safeParse(data); if (!result.success) { onComplete(null); diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index f9bcb08566..1a08dc1785 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -333,6 +333,32 @@ const StackedBarWithOverlap = (props: BarProps) => { ); }; +/** + * Compute the unique set of hexes referenced by `` defs + * inside MemoChart. Exported so a unit test can pin the dedup-and-union + * behavior without standing up a full recharts render (which jsdom + * struggles with at the container-sized SVG layer). + * + * Includes every categorical hex up front so any positional `` + * fill resolves, then unions in semantic hexes returned by the + * `getChartColor{Info,Success,Warning,Error}` helpers — those land in + * `lineData[].color` and would otherwise be missing a matching def. + * `undefined` colors are filtered so `c.replace('#', '')` can't throw + * on a future caller that leaves a series color unset. + */ +export function collectMemoChartGradientHexes( + lineData: { color?: string }[], +): string[] { + return Array.from( + new Set([ + ...COLORS, + ...lineData + .map(ld => ld.color) + .filter((c): c is string => typeof c === 'string'), + ]), + ); +} + export const MemoChart = memo(function MemoChart({ graphResults, setIsClickActive, @@ -702,7 +728,15 @@ export const MemoChart = memo(function MemoChart({ }} > - {COLORS.map(c => { + {/* Gradient defs cover every hex that any fill may reference. + `COLORS` (the unified categorical palette) is included up-front + as a baseline; semantic colors returned by the + `getChartColor{Info,Success,Warning,Error}` helpers can also + appear in `lineData[].color` (e.g. info-level log series + resolve to `--color-chart-info`, chart blue `#437eef`, on both + brands, which matches categorical slot 0). Union them here so the + referenced `url(#time-chart-lin-grad-…)` always exists. */} + {collectMemoChartGradientHexes(lineData).map(c => { return ( { const logomark = useLogomark({ size: 16 }); const { data: logViewsData } = useSavedSearches(); - const { data: dashboardsData } = api.useDashboards(); + const { data: dashboardsData } = useDashboards(); const actions = React.useMemo(() => { const logViews = logViewsData ?? []; diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index e2c1081e8d..2d3d5834ed 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -11,7 +11,15 @@ import { formatResponseForPieChart, formatResponseForTimeChart, } from '@/ChartUtils'; -import { COLORS, getChartColorError } from '@/utils'; +import { COLORS } from '@/utils'; + +// Anchor info/error to concrete hexes rather than `getChartColorInfo()` / +// `getChartColorError()` so a regression that breaks the helpers can't +// move expected and actual in lockstep. Keep in sync with +// `_chart-categorical-tokens.scss` (`chart-semantic-tokens` mixin) and +// `SEMANTIC_CHART_PALETTE` in `packages/app/src/utils.ts`. +const SEMANTIC_INFO_HEX = '#437eef'; +const SEMANTIC_ERROR_HEX = '#ff725c'; describe('ChartUtils', () => { describe('formatResponseForTimeChart', () => { @@ -306,7 +314,7 @@ describe('ChartUtils', () => { expect(actual.lineData).toEqual([ { - color: COLORS[0], + color: SEMANTIC_INFO_HEX, dataKey: 'info', currentPeriodKey: 'info', previousPeriodKey: 'info (previous)', @@ -315,7 +323,7 @@ describe('ChartUtils', () => { isDashed: false, }, { - color: COLORS[0], + color: SEMANTIC_INFO_HEX, dataKey: 'debug', currentPeriodKey: 'debug', previousPeriodKey: 'debug (previous)', @@ -324,7 +332,7 @@ describe('ChartUtils', () => { isDashed: false, }, { - color: getChartColorError(), + color: SEMANTIC_ERROR_HEX, dataKey: 'error', currentPeriodKey: 'error', previousPeriodKey: 'error (previous)', diff --git a/packages/app/src/__tests__/HDXMultiSeriesTimeChart.test.ts b/packages/app/src/__tests__/HDXMultiSeriesTimeChart.test.ts new file mode 100644 index 0000000000..aff4692293 --- /dev/null +++ b/packages/app/src/__tests__/HDXMultiSeriesTimeChart.test.ts @@ -0,0 +1,55 @@ +// Targeted unit test for the `` defs union that backs +// `` lookups inside `MemoChart`. The +// helper is exported so we can pin the union/dedup behavior without +// rendering recharts in jsdom (which struggles with sized SVG +// containers). Covers the regression flagged in the deep review on +// #2362 where a semantic-hex `lineData[].color` (e.g. the output of +// `getChartColorInfo()` on HyperDX) would not have a matching gradient +// def after the `COLORS` palette was unified to Observable 10. +import { collectMemoChartGradientHexes } from '../HDXMultiSeriesTimeChart'; +import { COLORS } from '../utils'; + +describe('collectMemoChartGradientHexes', () => { + it('includes every categorical hex from COLORS up front', () => { + const hexes = collectMemoChartGradientHexes([]); + for (const c of COLORS) { + expect(hexes).toContain(c); + } + expect(hexes).toHaveLength(COLORS.length); + }); + + it('unions in semantic hexes that lineData[].color introduces', () => { + // `#00c28a` is HyperDX brand green — historically returned by + // `getChartColorInfo()` for info-level series. After unifying + // `COLORS` to Observable 10 it is no longer in the categorical + // palette, so the gradient def must come from the lineData union. + const semanticHex = '#00c28a'; + const hexes = collectMemoChartGradientHexes([{ color: semanticHex }]); + expect(hexes).toContain(semanticHex); + expect(hexes).toHaveLength(COLORS.length + 1); + }); + + it('dedupes a lineData hex that is already in COLORS', () => { + const dup = COLORS[0]; + const hexes = collectMemoChartGradientHexes([ + { color: dup }, + { color: dup }, + ]); + expect(hexes.filter(h => h === dup)).toHaveLength(1); + expect(hexes).toHaveLength(COLORS.length); + }); + + it('filters out undefined and non-string colors', () => { + // The downstream `c.replace('#', '')` would throw if `undefined` + // sneaked through; this guards the defensive filter from being + // accidentally removed. + const hexes = collectMemoChartGradientHexes([ + { color: undefined }, + { color: '#abcdef' }, + // @ts-expect-error — intentionally exercising the runtime guard. + { color: 42 }, + ]); + expect(hexes).toContain('#abcdef'); + expect(hexes.every(h => typeof h === 'string')).toBe(true); + }); +}); diff --git a/packages/app/src/__tests__/Spotlights.test.tsx b/packages/app/src/__tests__/Spotlights.test.tsx index f796298e5d..7e7142cf67 100644 --- a/packages/app/src/__tests__/Spotlights.test.tsx +++ b/packages/app/src/__tests__/Spotlights.test.tsx @@ -1,6 +1,6 @@ import { renderHook } from '@testing-library/react'; -import api from '../api'; +import { useDashboards } from '../dashboard'; import { useSavedSearches } from '../savedSearch'; import { useSpotlightActions } from '../Spotlights'; @@ -22,10 +22,10 @@ jest.mock('next/router', () => ({ }, })); jest.mock('../savedSearch'); -jest.mock('../api'); +jest.mock('../dashboard'); const mockUseSavedSearches = useSavedSearches as jest.Mock; -const mockUseDashboards = api.useDashboards as jest.Mock; +const mockUseDashboards = useDashboards as jest.Mock; describe('useSpotlightActions', () => { const mockSavedSearches = { diff --git a/packages/app/src/__tests__/dashboard.remote.test.ts b/packages/app/src/__tests__/dashboard.remote.test.ts new file mode 100644 index 0000000000..93c6730cf9 --- /dev/null +++ b/packages/app/src/__tests__/dashboard.remote.test.ts @@ -0,0 +1,285 @@ +// Mirrors the local-mode tests in `dashboard.test.ts` but exercises the +// non-local branch of `fetchDashboards`: `hdxServer('dashboards').json<>()` +// followed by the same `normalizeDashboardTileColors` pass. The two files +// are split because `IS_LOCAL_MODE` is bound at module load and the two +// branches need different top-level mocks; `jest.doMock` inside +// `jest.isolateModules` did not override the hoisted `jest.mock` factory +// reliably enough to share a file. +jest.mock('../config', () => ({ IS_LOCAL_MODE: false })); +jest.mock('../api', () => ({ hdxServer: jest.fn() })); +jest.mock('@mantine/notifications', () => ({ + notifications: { show: jest.fn() }, +})); +jest.mock('nuqs', () => ({ + parseAsJson: jest.fn(), + useQueryState: jest.fn(), +})); + +// Capture each mutation's `mutationFn` so the tests can invoke it directly +// without standing up a full React Query provider. Each `useMutation()` +// call appends its config to `mutationFnCalls`; tests pull the most +// recently registered fn and invoke it as if `mutate({...})` had run. +const mutationFnCalls: Array<(input: any) => any> = []; +jest.mock('@tanstack/react-query', () => ({ + useQuery: jest.fn(), + useMutation: jest.fn((cfg: { mutationFn: (input: any) => any }) => { + mutationFnCalls.push(cfg.mutationFn); + return { mutate: jest.fn(), mutateAsync: jest.fn() }; + }), + useQueryClient: jest.fn(() => ({ invalidateQueries: jest.fn() })), +})); +jest.mock('@/utils', () => ({ hashCode: jest.fn(() => 0) })); + +import { LEGACY_CHART_PALETTE_TOKEN_MAP } from '@hyperdx/common-utils/dist/types'; + +import { hdxServer } from '../api'; +import { + fetchDashboards, + normalizeRawDashboardTileColors, + useCreateDashboard, + useUpdateDashboard, +} from '../dashboard'; + +const hdxServerMock = hdxServer as jest.Mock; + +const LEGACY_TO_HUE_CASES = Object.entries(LEGACY_CHART_PALETTE_TOKEN_MAP); + +const remoteDashboardWithTileColor = (color: unknown) => [ + { + id: 'a', + name: 'A', + tiles: [{ id: 't1', x: 0, y: 0, w: 4, h: 4, config: { color } }], + tags: [], + }, +]; + +const setRemotePayload = (payload: unknown) => { + hdxServerMock.mockReturnValue({ + json: jest.fn().mockResolvedValue(payload), + }); +}; + +beforeEach(() => { + hdxServerMock.mockReset(); + mutationFnCalls.length = 0; +}); + +describe('fetchDashboards (remote path)', () => { + // Stored configs from #2265 (the initial number-tile color picker) + // contain `color: 'chart-1'..'chart-10'`. The fetch-time normalizer + // heals those values for any tile that comes back from the API, so + // downstream consumers see the canonical hue tokens that + // `ChartPaletteTokenSchema` accepts. Symmetric coverage with the + // local-path suite in `dashboard.test.ts`. + it.each(LEGACY_TO_HUE_CASES)( + 'migrates legacy %s → %s from a remote payload', + async (legacy, hue) => { + setRemotePayload(remoteDashboardWithTileColor(legacy)); + + const result = await fetchDashboards(); + + expect(hdxServerMock).toHaveBeenCalledWith('dashboards'); + expect(result[0].tiles[0].config).toMatchObject({ color: hue }); + }, + ); + + it('passes through hue-named tokens unchanged', async () => { + setRemotePayload(remoteDashboardWithTileColor('chart-orange')); + + const result = await fetchDashboards(); + + expect(result[0].tiles[0].config).toMatchObject({ color: 'chart-orange' }); + }); + + it('leaves unresolvable color strings intact (no silent data loss)', async () => { + setRemotePayload(remoteDashboardWithTileColor('chart-future-magenta')); + + const result = await fetchDashboards(); + + expect(result[0].tiles[0].config).toMatchObject({ + color: 'chart-future-magenta', + }); + }); + + it('does not touch tiles whose config has no color field', async () => { + setRemotePayload([ + { + id: 'a', + name: 'A', + tiles: [ + { id: 't1', x: 0, y: 0, w: 4, h: 4, config: { displayType: 1 } }, + ], + tags: [], + }, + ]); + + const result = await fetchDashboards(); + + expect(result[0].tiles[0].config).toEqual({ displayType: 1 }); + }); +}); + +// Symmetric write-time coverage: dashboards constructed outside the +// fetch path (JSON import, presets, MCP payloads) hit +// `useCreateDashboard` / `useUpdateDashboard` directly. The strict +// server-side `ChartPaletteTokenSchema` would 400 a legacy `chart-N` +// here, so the mutations also call `normalizeDashboardTileColors` +// before serializing the body. These tests pin that contract. +describe('useCreateDashboard / useUpdateDashboard write-time normalization', () => { + const captureMutation = ( + hookFactory: () => unknown, + ): ((input: any) => any) => { + hookFactory(); + const fn = mutationFnCalls[mutationFnCalls.length - 1]; + expect(fn).toBeDefined(); + return fn; + }; + + beforeEach(() => { + // hdxServer is invoked as `hdxServer(url, opts).json()` for POST + // but as `hdxServer(url, opts)` for PATCH; mock to handle both. + hdxServerMock.mockReturnValue({ + json: jest.fn().mockResolvedValue({ id: 'a' }), + }); + }); + + it('rewrites legacy chart-N to hue tokens before POST in useCreateDashboard', async () => { + const create = captureMutation(useCreateDashboard); + + await create({ + name: 'D', + tiles: [ + { id: 't1', x: 0, y: 0, w: 4, h: 4, config: { color: 'chart-1' } }, + ], + tags: [], + }); + + expect(hdxServerMock).toHaveBeenCalledWith( + 'dashboards', + expect.objectContaining({ + method: 'POST', + json: expect.objectContaining({ + tiles: [ + expect.objectContaining({ config: { color: 'chart-green' } }), + ], + }), + }), + ); + }); + + it('preserves unresolvable color through POST so the server can surface a clear schema error', async () => { + const create = captureMutation(useCreateDashboard); + + await create({ + name: 'D', + tiles: [ + { + id: 't1', + x: 0, + y: 0, + w: 4, + h: 4, + config: { color: 'chart-future-magenta' }, + }, + ], + tags: [], + }); + + const call = hdxServerMock.mock.calls[0]; + expect(call[1].json.tiles[0].config).toMatchObject({ + color: 'chart-future-magenta', + }); + }); + + it('rewrites legacy chart-N to hue tokens before PATCH in useUpdateDashboard', async () => { + const update = captureMutation(useUpdateDashboard); + + await update({ + id: 'a', + tiles: [ + { id: 't1', x: 0, y: 0, w: 4, h: 4, config: { color: 'chart-10' } }, + ], + }); + + expect(hdxServerMock).toHaveBeenCalledWith( + 'dashboards/a', + expect.objectContaining({ + method: 'PATCH', + json: expect.objectContaining({ + tiles: [expect.objectContaining({ config: { color: 'chart-gray' } })], + }), + }), + ); + }); + + it('passes hue tokens through unchanged in useUpdateDashboard', async () => { + const update = captureMutation(useUpdateDashboard); + + await update({ + id: 'a', + tiles: [ + { + id: 't1', + x: 0, + y: 0, + w: 4, + h: 4, + config: { color: 'chart-orange' }, + }, + ], + }); + + const call = hdxServerMock.mock.calls[0]; + expect(call[1].json.tiles[0].config).toEqual({ color: 'chart-orange' }); + }); +}); + +// Pre-validation walker used by `DBDashboardImportPage`. Operates on +// `unknown` so JSON-imported templates can be healed *before* the strict +// `DashboardTemplateSchema.safeParse` rejects legacy `chart-N` with an +// opaque enum error. Distinct from `normalizeDashboardTileColors` in +// that unresolvable strings are left in place so the schema can report +// the bad value via its native error path. +describe('normalizeRawDashboardTileColors', () => { + it('rewrites legacy chart-N inside a tile config', () => { + const input = { + name: 'D', + tiles: [{ config: { color: 'chart-2' } }], + }; + expect(normalizeRawDashboardTileColors(input)).toEqual({ + name: 'D', + tiles: [{ config: { color: 'chart-blue' } }], + }); + }); + + it('leaves unresolvable color strings in place for the schema to flag', () => { + const input = { tiles: [{ config: { color: 'chart-future-magenta' } }] }; + const result = normalizeRawDashboardTileColors(input) as { + tiles: Array<{ config: { color: string } }>; + }; + expect(result.tiles[0].config.color).toBe('chart-future-magenta'); + }); + + it('returns the input untouched when tiles is missing or non-array', () => { + expect(normalizeRawDashboardTileColors({ name: 'D' })).toEqual({ + name: 'D', + }); + expect(normalizeRawDashboardTileColors({ tiles: 'oops' })).toEqual({ + tiles: 'oops', + }); + expect(normalizeRawDashboardTileColors(null)).toBeNull(); + expect(normalizeRawDashboardTileColors('not-an-object')).toBe( + 'not-an-object', + ); + }); + + it('preserves referential equality when nothing changes', () => { + const input = { tiles: [{ config: { color: 'chart-orange' } }] }; + expect(normalizeRawDashboardTileColors(input)).toBe(input); + }); + + it('skips tiles whose config has no color', () => { + const input = { tiles: [{ config: { displayType: 'line' } }] }; + expect(normalizeRawDashboardTileColors(input)).toBe(input); + }); +}); diff --git a/packages/app/src/__tests__/dashboard.test.ts b/packages/app/src/__tests__/dashboard.test.ts index 5e5ba38773..2cad91cf30 100644 --- a/packages/app/src/__tests__/dashboard.test.ts +++ b/packages/app/src/__tests__/dashboard.test.ts @@ -14,10 +14,14 @@ jest.mock('@tanstack/react-query', () => ({ })); jest.mock('@/utils', () => ({ hashCode: jest.fn(() => 0) })); +import { LEGACY_CHART_PALETTE_TOKEN_MAP } from '@hyperdx/common-utils/dist/types'; + import { fetchLocalDashboards, getLocalDashboardTags } from '../dashboard'; const STORAGE_KEY = 'hdx-local-dashboards'; +const LEGACY_TO_HUE_CASES = Object.entries(LEGACY_CHART_PALETTE_TOKEN_MAP); + beforeEach(() => { localStorage.clear(); }); @@ -35,6 +39,83 @@ describe('fetchLocalDashboards', () => { localStorage.setItem(STORAGE_KEY, JSON.stringify(dashboards)); expect(fetchLocalDashboards()).toHaveLength(2); }); + + describe('legacy tile color migration (fetch-time normalizer)', () => { + // Stored configs from #2265 (the initial number-tile color picker) + // contain `color: 'chart-1'..'chart-10'`. The rename refactor swapped + // those numeric tokens for hue-named ones and kept `ChartPaletteToken + // Schema` strict, so legacy values must be healed at load time. + const storeDashboardWithTileColor = (color: unknown) => { + localStorage.setItem( + STORAGE_KEY, + JSON.stringify([ + { + id: 'a', + name: 'A', + tiles: [ + { + id: 't1', + x: 0, + y: 0, + w: 4, + h: 4, + config: { color }, + }, + ], + tags: [], + }, + ]), + ); + }; + + // Drives every slot off the canonical `LEGACY_CHART_PALETTE_TOKEN_MAP` + // so a future tweak to the mapping is caught here instead of by a + // stored dashboard silently recoloring on the user's next reload. + it.each(LEGACY_TO_HUE_CASES)('migrates legacy %s → %s', (legacy, hue) => { + storeDashboardWithTileColor(legacy); + expect(fetchLocalDashboards()[0].tiles[0].config).toMatchObject({ + color: hue, + }); + }); + + it('passes through hue-named tokens unchanged', () => { + storeDashboardWithTileColor('chart-orange'); + expect(fetchLocalDashboards()[0].tiles[0].config).toMatchObject({ + color: 'chart-orange', + }); + }); + + it('leaves unresolvable color strings intact (no silent data loss)', () => { + // A stale hex / hand-edited / future-rollback value is preserved + // so the user's choice survives a render pass — the strict + // server-side `ChartPaletteTokenSchema` surfaces a clear error on + // next save instead of the normalizer quietly dropping the field + // and irreversibly overwriting the user's pick. + storeDashboardWithTileColor('chart-future-magenta'); + expect(fetchLocalDashboards()[0].tiles[0].config).toMatchObject({ + color: 'chart-future-magenta', + }); + }); + + it('does not touch tiles whose config has no color field', () => { + localStorage.setItem( + STORAGE_KEY, + JSON.stringify([ + { + id: 'a', + name: 'A', + tiles: [ + { id: 't1', x: 0, y: 0, w: 4, h: 4, config: { displayType: 1 } }, + ], + tags: [], + }, + ]), + ); + expect(fetchLocalDashboards()[0].tiles[0].config).toEqual({ + displayType: 1, + }); + }); + }); }); describe('getLocalDashboardTags', () => { diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index bedea9c851..5d8eeb68aa 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -1,4 +1,8 @@ -import { NumericUnit, TSource } from '@hyperdx/common-utils/dist/types'; +import { + ChartPaletteTokenSchema, + NumericUnit, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { SortingState } from '@tanstack/react-table'; import { act, renderHook } from '@testing-library/react'; @@ -1210,13 +1214,19 @@ describe('getColorFromCSSToken', () => { jest.restoreAllMocks(); }); - it('returns the CSS variable value when getComputedStyle provides one', () => { - jest.spyOn(global, 'getComputedStyle').mockReturnValue({ - getPropertyValue: (name: string) => - name === '--color-chart-1' ? '#custom-green' : '', - } as unknown as CSSStyleDeclaration); + it('returns the categorical hex directly from CATEGORICAL_HEX_BY_TOKEN without reading CSS', () => { + // Categorical tokens are unified across themes, so the resolver + // intentionally skips getComputedStyle to avoid a per-series + // layout read. A CSS-var override has no effect on the returned + // value (and shouldn't be relied upon by JS callers). + const getComputedStyleSpy = jest + .spyOn(global, 'getComputedStyle') + .mockReturnValue({ + getPropertyValue: () => '#should-be-ignored', + } as unknown as CSSStyleDeclaration); - expect(getColorFromCSSToken('chart-1')).toBe('#custom-green'); + expect(getColorFromCSSToken('chart-blue')).toBe(COLORS[0]); + expect(getComputedStyleSpy).not.toHaveBeenCalled(); }); it('returns the CSS variable value for semantic tokens when provided', () => { @@ -1228,40 +1238,33 @@ describe('getColorFromCSSToken', () => { expect(getColorFromCSSToken('chart-success')).toBe('#theme-green'); }); - it('falls back to COLORS[0] for chart-1 when the CSS variable is empty', () => { - jest.spyOn(global, 'getComputedStyle').mockReturnValue({ - getPropertyValue: () => '', - } as unknown as CSSStyleDeclaration); - - expect(getColorFromCSSToken('chart-1')).toBe(COLORS[0]); - }); - - it('falls back to the SSR palette when getComputedStyle throws', () => { + it('falls back to SEMANTIC_CHART_PALETTE when getComputedStyle throws for semantic tokens', () => { jest.spyOn(global, 'getComputedStyle').mockImplementation(() => { throw new Error('getComputedStyle unavailable'); }); - // Semantic tokens fall back to their designated palette colors. - // These hex values are the CHART_PALETTE constants used in paletteTokenSSRFallback. - expect(getColorFromCSSToken('chart-success')).toBe('#00c28a'); + // Defaults to HyperDX in jsdom because the document has no + // theme-clickstack class. + expect(getColorFromCSSToken('chart-success')).toBe('#3ca951'); expect(getColorFromCSSToken('chart-warning')).toBe('#efb118'); expect(getColorFromCSSToken('chart-error')).toBe('#ff725c'); - // Categorical token falls back to the COLORS array by index. - expect(getColorFromCSSToken('chart-1')).toBe(COLORS[0]); - expect(getColorFromCSSToken('chart-10')).toBe(COLORS[9]); }); - it('falls back to the SSR palette for all categorical indices (chart-1 through chart-10)', () => { - // Verify every categorical token resolves to its expected COLORS entry. - // The throw-branch exercises the same paletteTokenSSRFallback code as the - // SSR branch (window === undefined), which jsdom prevents from being - // simulated via Object.defineProperty. - jest.spyOn(global, 'getComputedStyle').mockImplementation(() => { - throw new Error('not available'); + it('returns the canonical hex for every categorical token in CATEGORICAL_PALETTE_TOKENS', () => { + utils.CATEGORICAL_PALETTE_TOKENS.forEach((token, i) => { + expect(getColorFromCSSToken(token)).toBe(COLORS[i]); }); + }); - for (let i = 1; i <= 10; i++) { - expect(getColorFromCSSToken(`chart-${i}` as any)).toBe(COLORS[i - 1]); - } + it('schema rejects legacy chart-1..10; render-time consumers rely on resolveChartPaletteToken instead', () => { + // The schema is deliberately strict (no `z.preprocess`) so that + // its `z.input` type matches its `z.output` type — otherwise + // `validateRequest` in the API would infer `req.body.tiles[i] + // .config.color` as `unknown`. Legacy migration for stored + // configs from #2265 happens at fetch time via + // `normalizeDashboardTileColors` and at render time via + // `resolveChartPaletteToken`. + expect(() => ChartPaletteTokenSchema.parse('chart-1')).toThrow(); + expect(() => ChartPaletteTokenSchema.parse('chart-10')).toThrow(); }); }); diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index f67574b18d..78f3aa8413 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -21,15 +21,10 @@ import type { WebhookTestApiResponse, WebhookUpdateApiResponse, } from '@hyperdx/common-utils/dist/types'; -import type { UseQueryOptions } from '@tanstack/react-query'; import { useMutation, useQuery } from '@tanstack/react-query'; import { IS_LOCAL_MODE } from './config'; -import { - Dashboard, - fetchLocalDashboards, - getLocalDashboardTags, -} from './dashboard'; +import { getLocalDashboardTags } from './dashboard'; type ServicesResponse = { data: Record< string, @@ -127,64 +122,6 @@ const api = { }, }); }, - useDashboards(options?: UseQueryOptions) { - return useQuery({ - queryKey: [`dashboards`], - queryFn: IS_LOCAL_MODE - ? async () => fetchLocalDashboards() - : () => hdxServer(`dashboards`, { method: 'GET' }).json(), - ...options, - }); - }, - useCreateDashboard() { - return useMutation({ - mutationFn: async ({ - name, - charts, - query, - tags, - }: { - name: string; - charts: Dashboard['tiles']; - query: string; - tags: string[]; - }) => - hdxServer(`dashboards`, { - method: 'POST', - json: { name, charts, query, tags }, - }).json(), - }); - }, - useUpdateDashboard() { - return useMutation({ - mutationFn: async ({ - id, - name, - charts, - query, - tags, - }: { - id: string; - name: string; - charts: Dashboard['tiles']; - query: string; - tags: string[]; - }) => - hdxServer(`dashboards/${id}`, { - method: 'PUT', - json: { name, charts, query, tags }, - }).json(), - }); - }, - useDeleteDashboard() { - return useMutation({ - mutationFn: async ({ id }: { id: string }) => { - await hdxServer(`dashboards/${id}`, { - method: 'DELETE', - }); - }, - }); - }, usePresetDashboardFilters( presetDashboard: PresetDashboard, sourceId: string, diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index 1e45cfbb34..9167387d83 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -1352,12 +1352,12 @@ describe('color round-trip (sql/promql Number tile)', () => { displayType: DisplayType.Number, sqlTemplate: 'SELECT count() FROM logs', connection: 'conn-1', - color: 'chart-2', + color: 'chart-orange', series: [], }; const result = convertFormStateToChartConfig(form, dateRange, undefined); expect(result).toBeDefined(); - expect((result as any).color).toBe('chart-2'); + expect((result as any).color).toBe('chart-orange'); }); it('preserves color through convertFormStateToSavedChartConfig for promql Number tile', () => { diff --git a/packages/app/src/components/ColorSwatchInput.stories.tsx b/packages/app/src/components/ColorSwatchInput.stories.tsx index 98c0abb5f5..08a2e1f1a9 100644 --- a/packages/app/src/components/ColorSwatchInput.stories.tsx +++ b/packages/app/src/components/ColorSwatchInput.stories.tsx @@ -26,7 +26,7 @@ export const Default = () => { export const Selected = () => { const [value, setValue] = React.useState( - 'chart-1', + 'chart-blue', ); return ; }; diff --git a/packages/app/src/components/ColorSwatchInput.test.tsx b/packages/app/src/components/ColorSwatchInput.test.tsx index e56cf43748..224cf16c01 100644 --- a/packages/app/src/components/ColorSwatchInput.test.tsx +++ b/packages/app/src/components/ColorSwatchInput.test.tsx @@ -25,7 +25,7 @@ describe('ColorSwatchInput', () => { await user.click(screen.getByTestId('color-swatch-input-trigger')); expect( - await screen.findByTestId('color-swatch-option-chart-1'), + await screen.findByTestId('color-swatch-option-chart-blue'), ).toBeInTheDocument(); expect( screen.getByTestId('color-swatch-option-chart-success'), @@ -72,12 +72,14 @@ describe('ColorSwatchInput', () => { it('marks only the selected swatch as pressed', async () => { const user = userEvent.setup(); - renderWithMantine(); + renderWithMantine(); await user.click(screen.getByTestId('color-swatch-input-trigger')); - const selected = await screen.findByTestId('color-swatch-option-chart-2'); - const other = await screen.findByTestId('color-swatch-option-chart-3'); + const selected = await screen.findByTestId( + 'color-swatch-option-chart-orange', + ); + const other = await screen.findByTestId('color-swatch-option-chart-red'); expect(selected).toHaveAttribute('aria-pressed', 'true'); expect(other).toHaveAttribute('aria-pressed', 'false'); }); @@ -85,7 +87,9 @@ describe('ColorSwatchInput', () => { it('clears the selection via the Clear button', async () => { const onChange = jest.fn(); const user = userEvent.setup(); - renderWithMantine(); + renderWithMantine( + , + ); await user.click(screen.getByTestId('color-swatch-input-trigger')); await user.click(await screen.findByTestId('color-swatch-input-clear')); @@ -101,7 +105,7 @@ describe('ColorSwatchInput', () => { await user.click(screen.getByTestId('color-swatch-input-trigger')); expect( - await screen.findByTestId('color-swatch-option-chart-1'), + await screen.findByTestId('color-swatch-option-chart-blue'), ).toBeInTheDocument(); expect( screen.queryByTestId('color-swatch-input-clear'), @@ -135,6 +139,30 @@ describe('ColorSwatchInput', () => { } }); + it('migrates a legacy chart-1..10 value to the matching hue swatch', async () => { + // Defense in depth: `normalizeDashboardTileColors` heals stored + // legacy tokens at fetch time, so the picker should usually only + // see hue-named values. But preset / in-memory tiles or any code + // path that bypasses the fetch normalizer can still pass a + // legacy `chart-1` here, so the picker also resolves it before + // matching against the swatches. + const user = userEvent.setup(); + renderWithMantine( + , + ); + + const trigger = screen.getByTestId('color-swatch-input-trigger'); + expect(trigger).toHaveTextContent(/green/i); + + await user.click(trigger); + expect( + await screen.findByTestId('color-swatch-option-chart-green'), + ).toHaveAttribute('aria-pressed', 'true'); + }); + it('respects disabled and does not open the popover', async () => { const user = userEvent.setup(); renderWithMantine(); @@ -144,7 +172,7 @@ describe('ColorSwatchInput', () => { await user.click(trigger); expect( - screen.queryByTestId('color-swatch-option-chart-1'), + screen.queryByTestId('color-swatch-option-chart-blue'), ).not.toBeInTheDocument(); }); @@ -153,11 +181,13 @@ describe('ColorSwatchInput', () => { renderWithMantine(); await user.click(screen.getByTestId('color-swatch-input-trigger')); - await user.click(await screen.findByTestId('color-swatch-option-chart-3')); + await user.click( + await screen.findByTestId('color-swatch-option-chart-red'), + ); await waitFor(() => { expect( - screen.queryByTestId('color-swatch-option-chart-3'), + screen.queryByTestId('color-swatch-option-chart-red'), ).not.toBeInTheDocument(); }); }); @@ -180,11 +210,11 @@ describe('ColorSwatchInput', () => { fireEvent.click(trigger); const firstSwatch = await screen.findByTestId( - 'color-swatch-option-chart-1', + 'color-swatch-option-chart-blue', ); firstSwatch.focus(); fireEvent.click(firstSwatch); - expect(onChange).toHaveBeenCalledWith('chart-1'); + expect(onChange).toHaveBeenCalledWith('chart-blue'); }); }); diff --git a/packages/app/src/components/ColorSwatchInput.tsx b/packages/app/src/components/ColorSwatchInput.tsx index 2ce27a9ecf..5b82741f6b 100644 --- a/packages/app/src/components/ColorSwatchInput.tsx +++ b/packages/app/src/components/ColorSwatchInput.tsx @@ -18,7 +18,7 @@ import { CATEGORICAL_PALETTE_TOKENS, ChartPaletteToken, getColorFromCSSToken, - isChartPaletteToken, + resolveChartPaletteToken, SEMANTIC_PALETTE_TOKENS, } from '@/utils'; @@ -27,16 +27,16 @@ import classes from './ColorSwatchInput.module.scss'; const Z_INDEX = 9999; const TOKEN_LABELS: Record = { - 'chart-1': 'Color 1', - 'chart-2': 'Color 2', - 'chart-3': 'Color 3', - 'chart-4': 'Color 4', - 'chart-5': 'Color 5', - 'chart-6': 'Color 6', - 'chart-7': 'Color 7', - 'chart-8': 'Color 8', - 'chart-9': 'Color 9', - 'chart-10': 'Color 10', + 'chart-blue': 'Blue', + 'chart-orange': 'Orange', + 'chart-red': 'Red', + 'chart-cyan': 'Cyan', + 'chart-green': 'Green', + 'chart-pink': 'Pink', + 'chart-purple': 'Purple', + 'chart-light-blue': 'Light Blue', + 'chart-brown': 'Brown', + 'chart-gray': 'Gray', 'chart-success': 'Success', 'chart-warning': 'Warning', 'chart-error': 'Error', @@ -73,8 +73,12 @@ export const ColorSwatchInput = ({ }: ColorSwatchInputProps) => { const [opened, setOpened] = React.useState(false); - // Guard against legacy values that are not in the current palette. - const safeValue = value && isChartPaletteToken(value) ? value : undefined; + // Accept both current hue-named tokens and legacy `chart-1`..`chart-10` + // values from #2265. The fetch normalizer in `dashboard.ts` usually + // heals stored data before it reaches us, but in-memory tiles or + // preset configs can still arrive with legacy values. Truly unknown + // values fall through to "no selection". + const safeValue = resolveChartPaletteToken(value); const handleChange = (next?: ChartPaletteToken) => { onChange?.(next); diff --git a/packages/app/src/components/DBNumberChart.tsx b/packages/app/src/components/DBNumberChart.tsx index fe38c26a9a..5c33d784f9 100644 --- a/packages/app/src/components/DBNumberChart.tsx +++ b/packages/app/src/components/DBNumberChart.tsx @@ -10,7 +10,7 @@ import { } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithDateRange, - isChartPaletteToken, + resolveChartPaletteToken, } from '@hyperdx/common-utils/dist/types'; import { Flex, Text } from '@mantine/core'; @@ -259,12 +259,18 @@ export default function DBNumberChart({ }); // Tile-level color override resolved at render time so token choices - // reflow correctly across light / dark / IDE themes. Unknown tokens - // (legacy strings, schema gaps) fall back to the default text color. - const tileColor = - config.color && isChartPaletteToken(config.color) - ? getColorFromCSSToken(config.color) - : undefined; + // reflow correctly across light / dark / IDE themes. + // `resolveChartPaletteToken` accepts both current hue-named tokens and + // legacy `chart-1`..`chart-10` values from stored configs. The fetch + // path (`normalizeDashboardTileColors`) already heals stored data, so + // in practice this resolver only ever sees the migrated hue tokens — + // but we keep the call as defense in depth against any tile that gets + // constructed in memory without going through the fetch normalizer. + // Unknown strings fall back to the default text color. + const resolvedColorToken = resolveChartPaletteToken(config.color); + const tileColor = resolvedColorToken + ? getColorFromCSSToken(resolvedColorToken) + : undefined; const toolbarItemsMemo = useMemo(() => { const allToolbarItems = []; diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 04dd3d3974..54106c3210 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -92,7 +92,7 @@ import { import { FormatTime } from '@/useFormatTime'; import { useUserPreferences } from '@/useUserPreferences'; import { - COLORS, + getChartColorInfo, getLogLevelClass, logLevelColor, useLocalStorage, @@ -271,16 +271,19 @@ const PatternTrendChart = ({ isAnimationActive={false} dataKey="count" stackId="a" - fill={color || COLORS[0]} + // `getChartColorInfo()` resolves a CSS var via + // `getComputedStyle(document.documentElement)` and is + // invoked once per row render. Kept inline (instead of + // hoisted into a memo) because memoizing would either + // require a stable theme-class subscription this + // component doesn't already have, or risk a stale + // value on theme toggle. The per-row cost is acceptable: + // pattern rows render in a virtualized list and the + // `getComputedStyle` read is sub-microsecond. Revisit + // if this surfaces in a profile. + fill={color || getChartColorInfo()} maxBarSize={24} /> - {/* */} } /> diff --git a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx index 9b83e81dc2..ec6b681bcd 100644 --- a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx +++ b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx @@ -69,12 +69,14 @@ describe('ChartDisplaySettingsDrawer', () => { await user.click(screen.getByTestId('color-swatch-input-trigger')); // Pick a categorical token. - const swatch = await screen.findByTestId('color-swatch-option-chart-1'); + const swatch = await screen.findByTestId( + 'color-swatch-option-chart-blue', + ); await user.click(swatch); // Verify the trigger shows the selected token. const trigger = screen.getByTestId('color-swatch-input-trigger'); - expect(trigger).toHaveTextContent(/color 1/i); + expect(trigger).toHaveTextContent(/blue/i); // Apply the settings. await user.click(screen.getByRole('button', { name: /apply/i })); @@ -82,7 +84,7 @@ describe('ChartDisplaySettingsDrawer', () => { expect(onChange).toHaveBeenCalledTimes(1); // react-hook-form passes (values, event) to the onSubmit handler; // check the values argument directly. - expect(onChange.mock.calls[0][0]).toMatchObject({ color: 'chart-1' }); + expect(onChange.mock.calls[0][0]).toMatchObject({ color: 'chart-blue' }); }); it('calls onChange with a semantic token when Apply is clicked', async () => { diff --git a/packages/app/src/components/__tests__/DBNumberChart.test.tsx b/packages/app/src/components/__tests__/DBNumberChart.test.tsx index 7a727e7bcd..cc175e3292 100644 --- a/packages/app/src/components/__tests__/DBNumberChart.test.tsx +++ b/packages/app/src/components/__tests__/DBNumberChart.test.tsx @@ -348,6 +348,26 @@ describe('DBNumberChart', () => { expect(mockGetColorFromCSSToken).not.toHaveBeenCalled(); expect(screen.getByText('1234')).toBeInTheDocument(); }); + + it('migrates legacy chart-1..10 tokens to their hue-named equivalent at render time', () => { + // Defense in depth: in practice `normalizeDashboardTileColors` in + // `packages/app/src/dashboard.ts` heals legacy tokens at fetch + // time, so renderers should always see hue-named values. But any + // tile constructed in memory (e.g. from a preset or a unit test) + // can still carry a legacy `chart-1`, so the renderer also + // resolves through `resolveChartPaletteToken` before the + // CSS-token lookup. + const config = { + ...baseTestConfig, + color: 'chart-1' as any, + }; + + renderWithMantine(); + + expect(mockGetColorFromCSSToken).toHaveBeenCalledWith('chart-green'); + const textEl = screen.getByText('1234'); + expect(textEl).toHaveStyle({ color: 'rgb(0, 255, 0)' }); + }); }); describe('auto-sized font', () => { diff --git a/packages/app/src/dashboard.ts b/packages/app/src/dashboard.ts index 0a4851c4ce..8e52d3813a 100644 --- a/packages/app/src/dashboard.ts +++ b/packages/app/src/dashboard.ts @@ -4,8 +4,10 @@ import { DashboardContainer, DashboardFilter, Filter, + resolveChartPaletteToken, SavedChartConfig, SearchConditionLanguage, + walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import { notifications } from '@mantine/notifications'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; @@ -46,11 +48,76 @@ export type Dashboard = { const localDashboards = createEntityStore('hdx-local-dashboards'); -async function fetchDashboards(): Promise { +/** + * Resolution policy shared by both the typed normalizer below and the + * `unknown`-walking JSON-import variant: hue tokens pass through + * unchanged, legacy `chart-N` from #2265 are rewritten to their + * hue-named equivalents, and anything else (stale hexes, hand-edited + * values, future tokens from a forward-rolled deploy) is left as-is so + * the strict server-side `ChartPaletteTokenSchema` surfaces a clear + * error on save rather than silently dropping the user's chosen color. + * Render-time consumers (`DBNumberChart`, `ColorSwatchInput`) still + * call `resolveChartPaletteToken` directly so the live chart falls + * back gracefully even while the unresolved value is in flight. + */ +const migrateOrPreserveColor = (current: string): string => { + const resolved = resolveChartPaletteToken(current); + return resolved ?? current; +}; + +/** + * Heal legacy `chart-1`..`chart-10` colors stored on tiles by #2265 + * into their hue-named equivalents. Applied at fetch time so every + * downstream consumer (renderers, the color picker, save mutations) + * sees the canonical hue tokens that `ChartPaletteTokenSchema` + * accepts, AND at write time so dashboards constructed outside the + * fetch path (JSON imports via `DBDashboardImportPage`, presets, MCP + * payloads) don't trip the strict server-side enum validator and + * return a Zod 400. Symmetric application also lets the DB-side data + * converge on next save instead of holding legacy tokens forever. + * + * Delegates the per-tile walk to `walkRawDashboardTileColors` in + * common-utils so the unknown-import path and the API-side migration + * shim stay in lockstep with this typed variant. The double cast is + * unavoidable: the shared walker is generic over `unknown` so the + * provisioner / API can call it, and TypeScript can't carry the + * `tiles[]` element type through. The return shape is structurally + * identical to the input. + */ +function normalizeDashboardTileColors( + dashboard: T, +): T { + if (!dashboard.tiles || dashboard.tiles.length === 0) return dashboard; + return walkRawDashboardTileColors(dashboard, migrateOrPreserveColor) as T; +} + +/** + * Walks a parsed-but-not-yet-validated JSON payload and rewrites every + * `tiles[i].config.color` that points at a legacy `chart-1`..`chart-10` + * value to its hue-named equivalent. Same policy as + * `normalizeDashboardTileColors` (legacy → hue, unknown left intact for + * the schema to flag) but exposed as `unknown -> unknown` so the + * JSON-import flow in `DBDashboardImportPage` can heal legacy values + * *before* the strict `DashboardTemplateSchema` parse (which would + * otherwise reject the legacy enum and trip an error toast). + */ +export function normalizeRawDashboardTileColors(input: unknown): unknown { + return walkRawDashboardTileColors(input, migrateOrPreserveColor); +} + +/** + * Shared queryFn behind `useDashboards`. Exported so tests can call it + * directly (notably `dashboard.remote.test.ts`, which exercises the + * non-local branch in isolation). React components should keep going + * through `useDashboards` so React Query caching and invalidation + * stays uniform. + */ +export async function fetchDashboards(): Promise { if (IS_LOCAL_MODE) { - return localDashboards.getAll(); + return localDashboards.getAll().map(normalizeDashboardTileColors); } - return hdxServer('dashboards').json(); + const dashboards = await hdxServer('dashboards').json(); + return dashboards.map(normalizeDashboardTileColors); } export function useUpdateDashboard() { @@ -60,14 +127,15 @@ export function useUpdateDashboard() { mutationFn: async ( dashboard: Partial & { id: Dashboard['id'] }, ) => { + const normalized = normalizeDashboardTileColors(dashboard); if (IS_LOCAL_MODE) { - const { id, ...updates } = dashboard; + const { id, ...updates } = normalized; localDashboards.update(id, updates); return; } - await hdxServer(`dashboards/${dashboard.id}`, { + await hdxServer(`dashboards/${normalized.id}`, { method: 'PATCH', - json: dashboard, + json: normalized, }); }, onSuccess: () => { @@ -81,12 +149,13 @@ export function useCreateDashboard() { return useMutation({ mutationFn: async (dashboard: Omit) => { + const normalized = normalizeDashboardTileColors(dashboard); if (IS_LOCAL_MODE) { - return localDashboards.create(dashboard); + return localDashboards.create(normalized); } return hdxServer('dashboards', { method: 'POST', - json: dashboard, + json: normalized, }).json(); }, onSuccess: () => { @@ -143,7 +212,12 @@ export function useDashboard({ const dashboard: Dashboard | undefined = useMemo(() => { if (isLocalDashboard) { - return localDashboard ?? defaultDashboard; + // URL-state local dashboards bypass `fetchDashboards`, so heal + // any legacy `chart-N` here too (symmetric with the write-time + // pass in `setDashboard` below). + return localDashboard + ? normalizeDashboardTileColors(localDashboard) + : defaultDashboard; } return remoteDashboard; }, [isLocalDashboard, localDashboard, defaultDashboard, remoteDashboard]); @@ -155,7 +229,11 @@ export function useDashboard({ onError?: VoidFunction, ) => { if (isLocalDashboard) { - setLocalDashboard(newDashboard); + // Normalize on write too so the URL-state local dashboard never + // holds a legacy `chart-N` after a no-fetch path (e.g. a tile + // inserted via a preset literal) and matches the canonical hue + // tokens used by the renderers. + setLocalDashboard(normalizeDashboardTileColors(newDashboard)); onSuccess?.(); } else { setIsSettingDashboard(true); @@ -196,7 +274,7 @@ export function useDashboard({ } export function fetchLocalDashboards(): Dashboard[] { - return localDashboards.getAll(); + return localDashboards.getAll().map(normalizeDashboardTileColors); } export function getLocalDashboardTags(): string[] { diff --git a/packages/app/src/theme/ChartColors.stories.tsx b/packages/app/src/theme/ChartColors.stories.tsx index 56e17a2568..bdd50500bd 100644 --- a/packages/app/src/theme/ChartColors.stories.tsx +++ b/packages/app/src/theme/ChartColors.stories.tsx @@ -1,19 +1,21 @@ import React from 'react'; import { + CATEGORICAL_PALETTE_TOKENS, COLORS, getChartColorError, + getChartColorInfo, getChartColorSuccess, getChartColorWarning, } from '@/utils'; -// Labels for chart colors - brand green first, then Observable palette +// Pretty labels for the picker UI, in the same order as CATEGORICAL_PALETTE_TOKENS. const COLOR_LABELS = [ - 'Green (Brand)', 'Blue', 'Orange', 'Red', 'Cyan', + 'Green', 'Pink', 'Purple', 'Light Blue', @@ -21,11 +23,12 @@ const COLOR_LABELS = [ 'Gray', ]; -// Derive chart colors from the single source of truth in utils.ts -const CHART_COLORS = COLORS.map((hex, i) => ({ - name: `color-chart-${i + 1}`, - hex, - label: COLOR_LABELS[i] || `Color ${i + 1}`, +// Derive chart colors from the single source of truth in utils.ts; the token +// (e.g. `chart-blue`) is the CSS-var slug too. +const CHART_COLORS = CATEGORICAL_PALETTE_TOKENS.map((token, i) => ({ + name: `color-${token}`, + hex: COLORS[i], + label: COLOR_LABELS[i] ?? token, })); const SEMANTIC_CHART_COLORS = [ @@ -44,6 +47,11 @@ const SEMANTIC_CHART_COLORS = [ hex: getChartColorError(), label: 'Error (Red)', }, + { + name: 'color-chart-info', + hex: getChartColorInfo(), + label: 'Info (Blue)', + }, ]; const story = { @@ -124,6 +132,18 @@ export const AllChartColors = () => (

Semantic Chart Colors

+

+ Semantic chart colors are unified across HyperDX and ClickStack — + `success` aliases categorical `chart-green` and `info` aliases + `chart-blue`. Brand identity for charts is carried by non-chart UI chrome + (Mantine accent, sidebar gradient), not by these tokens. +

{SEMANTIC_CHART_COLORS.map(({ name, hex, label }) => ( @@ -321,7 +341,7 @@ export const SemanticColorsPreview = () => ( fontSize: 24, }} > - {i === 0 ? '98%' : i === 1 ? '15' : '3'} + {i === 0 ? '98%' : i === 1 ? '15' : i === 2 ? '3' : '42'}
{label}
@@ -352,7 +372,7 @@ export const AccessibilityCheck = () => ( border: '1px solid var(--color-border)', }} > - {CHART_COLORS.map(({ hex, label }, i) => ( + {CHART_COLORS.map(({ hex, label }) => (
( gap: 2, }} > - {i + 1} - {label} + {label}
))}
diff --git a/packages/app/src/theme/semanticColorsGrouped.ts b/packages/app/src/theme/semanticColorsGrouped.ts index 19da9b1d4d..5c21f10527 100644 --- a/packages/app/src/theme/semanticColorsGrouped.ts +++ b/packages/app/src/theme/semanticColorsGrouped.ts @@ -57,19 +57,21 @@ export const semanticColorsGrouped = { 'color-json-punctuation', ], charts: [ - 'color-chart-1', - 'color-chart-2', - 'color-chart-3', - 'color-chart-4', - 'color-chart-5', - 'color-chart-6', - 'color-chart-7', - 'color-chart-8', - 'color-chart-9', - 'color-chart-10', + 'color-chart-blue', + 'color-chart-orange', + 'color-chart-red', + 'color-chart-cyan', + 'color-chart-green', + 'color-chart-pink', + 'color-chart-purple', + 'color-chart-light-blue', + 'color-chart-brown', + 'color-chart-gray', 'color-chart-success', 'color-chart-warning', 'color-chart-error', + 'color-chart-info', + 'color-chart-success-highlight', 'color-chart-error-highlight', 'color-chart-warning-highlight', ], diff --git a/packages/app/src/theme/themes/_chart-categorical-tokens.scss b/packages/app/src/theme/themes/_chart-categorical-tokens.scss new file mode 100644 index 0000000000..4c9454babf --- /dev/null +++ b/packages/app/src/theme/themes/_chart-categorical-tokens.scss @@ -0,0 +1,62 @@ +/* Shared Chart Tokens - used by both theme files */ + +/* + * Categorical hues and semantic chart colors are unified across HyperDX and + * ClickStack. This file is the SCSS source of truth for both layers; each + * brand theme `@use`s it and `@include`s the mixins inside its per-theme + * `chart-tokens` mixin (categorical first, then semantic — semantics reference + * `var(--color-chart-green)` / `var(--color-chart-blue)`). + * + * Per-brand overrides: if a future brand needs a different semantic hex, + * declare it in that theme's `chart-tokens` mixin *after* the shared + * `@include chart-semantic-tokens` call. + * + * Categorical hues — React reads these from `CATEGORICAL_HEX_BY_TOKEN` in + * `packages/app/src/utils.ts` directly (`getColorFromCSSVariable` / + * `getColorFromCSSToken` skip `getComputedStyle` for categorical tokens). + * The CSS vars exist for SCSS modules, inline `var()` consumers, and devtools. + * If you change a categorical hex here, update `CATEGORICAL_HEX_BY_TOKEN` + * in `utils.ts` to match. + * + * Semantic tokens — read at runtime via `getComputedStyle` in `utils.ts`. + * If you change a semantic hex here, update `SEMANTIC_CHART_PALETTE` in + * `utils.ts` to match. + * + * `chart-blue` is `#437eef` (the brand link color, same hue as + * `--click-global-color-text-link-default`) rather than Observable blue + * (`#4269d0`); all other hues are straight from Observable 10 + * (https://observablehq.com/@d3/color-schemes). + */ +@mixin chart-categorical-tokens { + --color-chart-blue: #437eef; + --color-chart-orange: #efb118; + --color-chart-red: #ff725c; + --color-chart-cyan: #6cc5b0; + --color-chart-green: #3ca951; + --color-chart-pink: #ff8ab7; + --color-chart-purple: #a463f2; + --color-chart-light-blue: #97bbf5; + --color-chart-brown: #9c6b4e; + --color-chart-gray: #9498a0; +} + +/* + * Semantic chart colors — unified on both brands. All four semantic vars + * alias their matching categorical hue var so a future tweak to e.g. + * `--color-chart-orange` propagates to `--color-chart-warning` without + * a second edit. Must be `@include`d after `chart-categorical-tokens`. + * + * Highlight variants stay as concrete hexes — they're a tint of the + * base hue, not an alias, so they can't ride the var. + */ +@mixin chart-semantic-tokens { + --color-chart-success: var(--color-chart-green); + --color-chart-warning: var(--color-chart-orange); + --color-chart-error: var(--color-chart-red); + --color-chart-info: var(--color-chart-blue); + + /* Highlighted variants (hover/selection states) */ + --color-chart-success-highlight: #80d9b3; + --color-chart-warning-highlight: #f5c94d; + --color-chart-error-highlight: #ffa090; +} diff --git a/packages/app/src/theme/themes/clickstack/_tokens.scss b/packages/app/src/theme/themes/clickstack/_tokens.scss index 0651f8ef9d..72b36cca71 100644 --- a/packages/app/src/theme/themes/clickstack/_tokens.scss +++ b/packages/app/src/theme/themes/clickstack/_tokens.scss @@ -4,6 +4,23 @@ /* Uses Click UI design tokens */ +/* + * `chart-tokens` is split out so the chart palette is defined once, + * not duplicated across the dark/light blocks. Both scheme selectors + * @include it so specificity stays identical to declaring the vars + * inline — Sass inlines the body of the mixin at each call site, so + * the resulting CSS is byte-identical to the previous hand-duplicated + * shape. Categorical and semantic chart tokens come from the shared + * `chart-categorical-tokens` partial (two mixins: hues, then semantics). + */ + +@use '../chart-categorical-tokens' as shared-chart; + +@mixin chart-tokens { + @include shared-chart.chart-categorical-tokens; + @include shared-chart.chart-semantic-tokens; +} + /* Dark Mode */ .theme-clickstack[data-mantine-color-scheme='dark'] { /* Brand Palette - Yellow/Gold */ @@ -172,33 +189,7 @@ --color-json-array: #ffd966; --color-json-punctuation: #666980; - /* - * Chart Colors - Observable 10 categorical palette - * NOTE: These colors are intentionally duplicated in both dark and light mode sections. - * CSS specificity requires them to be defined within each [data-mantine-color-scheme] selector - * to ensure they're applied correctly. A shared .theme-clickstack section would have lower - * specificity and could be overridden by other styles. - */ - --color-chart-1: #437eef; /* Blue - Primary */ - --color-chart-2: #efb118; /* Orange */ - --color-chart-3: #ff725c; /* Red */ - --color-chart-4: #6cc5b0; /* Cyan */ - --color-chart-5: #3ca951; /* Green */ - --color-chart-6: #ff8ab7; /* Pink */ - --color-chart-7: #a463f2; /* Purple */ - --color-chart-8: #97bbf5; /* Light Blue */ - --color-chart-9: #9c6b4e; /* Brown */ - --color-chart-10: #9498a0; /* Gray */ - - /* Chart Semantic Colors */ - --color-chart-success: #3ca951; /* Green */ - --color-chart-warning: #efb118; /* Orange */ - --color-chart-error: #ff725c; /* Red */ - - /* Chart Semantic Colors - Highlighted (for hover/selection states) */ - --color-chart-success-highlight: #80d9b3; - --color-chart-warning-highlight: #f5c94d; - --color-chart-error-highlight: #ffa090; + @include chart-tokens; /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ --color-viz-range-handle: var(--palette-brand-300); @@ -390,27 +381,7 @@ --color-json-array: #997300; --color-json-punctuation: #868e96; - /* Chart Colors - See dark mode section for explanation of why duplication is required */ - --color-chart-1: #437eef; /* Blue - Primary */ - --color-chart-2: #efb118; /* Orange */ - --color-chart-3: #ff725c; /* Red */ - --color-chart-4: #6cc5b0; /* Cyan */ - --color-chart-5: #3ca951; /* Green */ - --color-chart-6: #ff8ab7; /* Pink */ - --color-chart-7: #a463f2; /* Purple */ - --color-chart-8: #97bbf5; /* Light Blue */ - --color-chart-9: #9c6b4e; /* Brown */ - --color-chart-10: #9498a0; /* Gray */ - - /* Chart Semantic Colors */ - --color-chart-success: #3ca951; /* Green */ - --color-chart-warning: #efb118; /* Orange */ - --color-chart-error: #ff725c; /* Red */ - - /* Chart Semantic Colors - Highlighted (for hover/selection states) */ - --color-chart-success-highlight: #80d9b3; - --color-chart-warning-highlight: #f5c94d; - --color-chart-error-highlight: #ffa090; + @include chart-tokens; /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ --color-viz-range-handle: var(--mantine-color-indigo-8); diff --git a/packages/app/src/theme/themes/hyperdx/_tokens.scss b/packages/app/src/theme/themes/hyperdx/_tokens.scss index 88f8e6ff8b..3ec5280fe0 100644 --- a/packages/app/src/theme/themes/hyperdx/_tokens.scss +++ b/packages/app/src/theme/themes/hyperdx/_tokens.scss @@ -6,8 +6,23 @@ * These mixins define all design tokens for HyperDX theme. * They are used both here (for .theme-hyperdx scoped selectors) and * in _base-tokens.scss (for unscoped fallback selectors during SSR). + * + * `chart-tokens` is split out so the chart palette is defined once, + * not duplicated across the dark/light blocks. Both dark and light + * @include it so specificity stays identical to declaring the vars + * inline — Sass inlines the body of the mixin at each call site, so + * the resulting CSS is byte-identical to the previous hand-duplicated + * shape. Categorical and semantic chart tokens come from the shared + * `chart-categorical-tokens` partial (two mixins: hues, then semantics). */ +@use '../chart-categorical-tokens' as shared-chart; + +@mixin chart-tokens { + @include shared-chart.chart-categorical-tokens; + @include shared-chart.chart-semantic-tokens; +} + @mixin dark-mode-tokens { /* Backgrounds */ --color-bg-body: var(--mantine-color-dark-9); @@ -92,29 +107,7 @@ --color-json-array: var(--mantine-color-green-3); --color-json-punctuation: var(--mantine-color-dark-4); - /* Chart Colors - Brand green first, then Observable palette - Note: 1-indexed CSS vars map to 0-indexed COLORS array in utils.ts - e.g., --color-chart-1 corresponds to COLORS[0] */ - --color-chart-1: #00c28a; /* Green - Brand (primary) */ - --color-chart-2: #4269d0; /* Blue */ - --color-chart-3: #efb118; /* Orange */ - --color-chart-4: #ff725c; /* Red */ - --color-chart-5: #6cc5b0; /* Cyan */ - --color-chart-6: #ff8ab7; /* Pink */ - --color-chart-7: #a463f2; /* Purple */ - --color-chart-8: #97bbf5; /* Light Blue */ - --color-chart-9: #9c6b4e; /* Brown */ - --color-chart-10: #9498a0; /* Gray */ - - /* Chart Semantic Colors */ - --color-chart-success: #00c28a; /* Green - Brand */ - --color-chart-warning: #efb118; /* Orange */ - --color-chart-error: #ff725c; /* Red */ - - /* Chart Semantic Colors - Highlighted (for hover/selection states) */ - --color-chart-success-highlight: #80d9b3; - --color-chart-warning-highlight: #f5c94d; - --color-chart-error-highlight: #ffa090; + @include chart-tokens; /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ --color-viz-range-handle: var(--mantine-color-dark-2); @@ -217,27 +210,7 @@ --color-json-array: var(--mantine-color-green-7); --color-json-punctuation: var(--mantine-color-dark-5); - /* Chart Colors - Brand green first, then Observable palette (same for light mode) */ - --color-chart-1: #00c28a; /* Green - Brand (primary) */ - --color-chart-2: #4269d0; /* Blue */ - --color-chart-3: #efb118; /* Orange */ - --color-chart-4: #ff725c; /* Red */ - --color-chart-5: #6cc5b0; /* Cyan */ - --color-chart-6: #ff8ab7; /* Pink */ - --color-chart-7: #a463f2; /* Purple */ - --color-chart-8: #97bbf5; /* Light Blue */ - --color-chart-9: #9c6b4e; /* Brown */ - --color-chart-10: #9498a0; /* Gray */ - - /* Chart Semantic Colors */ - --color-chart-success: #00c28a; /* Green - Brand */ - --color-chart-warning: #efb118; /* Orange */ - --color-chart-error: #ff725c; /* Red */ - - /* Chart Semantic Colors - Highlighted (for hover/selection states) */ - --color-chart-success-highlight: #80d9b3; - --color-chart-warning-highlight: #f5c94d; - --color-chart-error-highlight: #ffa090; + @include chart-tokens; /* Visualization Range Selector (minimap viewports, brush ranges, etc.) */ --color-viz-range-handle: var(--mantine-color-gray-5); diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index e80d6a25ee..8bbac30acf 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -6,6 +6,7 @@ import type { SetStateAction } from 'react'; import TimestampNano from 'timestamp-nano'; import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata'; import { + CATEGORICAL_PALETTE_TOKENS, ChartPaletteToken, NumericUnit, SourceKind, @@ -354,60 +355,128 @@ export const getLogLevelClass = (lvl: string | undefined) => { : undefined; }; -// Chart color palette - single source of truth -// Colors from Observable categorical palette, with custom brand green -// https://observablehq.com/@d3/color-schemes -const CHART_PALETTE = { - green: '#00c28a', // Brand green (Mantine green.5) - used as primary chart color - blue: '#4269d0', - orange: '#efb118', - red: '#ff725c', - cyan: '#6cc5b0', - pink: '#ff8ab7', - purple: '#a463f2', - lightBlue: '#97bbf5', - brown: '#9c6b4e', - gray: '#9498a0', - // Highlighted variants (lighter shades for hover/selection states) - greenHighlight: '#80d9b3', - redHighlight: '#ffa090', - orangeHighlight: '#f5c94d', -} as const; - -// ClickStack theme chart color palette - Observable 10 categorical palette -// https://observablehq.com/@d3/color-schemes -const CLICKSTACK_CHART_PALETTE = { - blue: '#437EEF', // Primary color for ClickStack - orange: '#efb118', - red: '#ff725c', - cyan: '#6cc5b0', - green: '#3ca951', - pink: '#ff8ab7', - purple: '#a463f2', - lightBlue: '#97bbf5', - brown: '#9c6b4e', - gray: '#9498a0', - // Highlighted variants (lighter shades for hover/selection states) - greenHighlight: '#80d9b3', - redHighlight: '#ffa090', - orangeHighlight: '#f5c94d', -} as const; - -// Ordered array for chart series - green first for brand consistency (HyperDX default) -// Maps to CSS variables: COLORS[0] -> --color-chart-1, COLORS[1] -> --color-chart-2, etc. -// NOTE: This is a fallback for SSR. In browser, getColorFromCSSVariable() reads from CSS variables -export const COLORS = [ - CHART_PALETTE.green, // 1 - Brand green (primary) - HyperDX default - CHART_PALETTE.blue, // 2 - CHART_PALETTE.orange, // 3 - CHART_PALETTE.red, // 4 - CHART_PALETTE.cyan, // 5 - CHART_PALETTE.pink, // 6 - CHART_PALETTE.purple, // 7 - CHART_PALETTE.lightBlue, // 8 - CHART_PALETTE.brown, // 9 - CHART_PALETTE.gray, // 10 -]; +/** + * Chart Categorical Palette - ten distinguishable hues based on + * Observable 10 (https://observablehq.com/@d3/color-schemes), with + * `chart-blue` swapped to `#437eef` to match the brand link color + * (`--click-global-color-text-link-default`). All other hues are + * straight from Observable 10. Unified across themes. + * + * **JS is the source of truth for categorical hues.** Both HyperDX and + * ClickStack resolve `--color-chart-{hue}` to the same hex today, so + * the JS readers (`getColorFromCSSVariable`, `getColorFromCSSToken`) + * return values directly from this object without round-tripping + * through `getComputedStyle`. The matching `--color-chart-{hue}` CSS + * vars in `_tokens.scss` exist as a stylesheet-author affordance only + * (inline `var()` use, devtools inspection); they are NOT read back by + * the React rendering path. Per-brand identity is carried by the + * semantic chart tokens below and by non-chart UI chrome (Mantine + * accent, sidebar gradient, etc.). + * + * Keep in sync with `CATEGORICAL_PALETTE_TOKENS` in + * `@hyperdx/common-utils/dist/types` and with the `--color-chart-{hue}` + * vars in `packages/app/src/theme/themes/_chart-categorical-tokens.scss` + * (the single shared SCSS source for categorical hues; both brand + * themes `@use` it). + */ +type CategoricalChartPaletteToken = (typeof CATEGORICAL_PALETTE_TOKENS)[number]; + +const CATEGORICAL_HEX_BY_TOKEN = { + 'chart-blue': '#437eef', + 'chart-orange': '#efb118', + 'chart-red': '#ff725c', + 'chart-cyan': '#6cc5b0', + 'chart-green': '#3ca951', + 'chart-pink': '#ff8ab7', + 'chart-purple': '#a463f2', + 'chart-light-blue': '#97bbf5', + 'chart-brown': '#9c6b4e', + 'chart-gray': '#9498a0', +} as const satisfies Record; + +// Reverse-direction completeness check: if `CATEGORICAL_HEX_BY_TOKEN` +// ever grows an extra key that's not in `CATEGORICAL_PALETTE_TOKENS` +// (e.g. a deprecated hex stuck around after dropping the token from the +// shared enum), this type collapses to `never` and the assignment below +// becomes a compile error. The `satisfies` above already enforces the +// forward direction (every categorical token has a hex), so together +// they pin the two structures to a 1:1 mapping at build time. +type _CategoricalHexCompleteness = + Exclude< + keyof typeof CATEGORICAL_HEX_BY_TOKEN, + CategoricalChartPaletteToken + > extends never + ? true + : never; +const _categoricalHexCompletenessCheck: _CategoricalHexCompleteness = true; +void _categoricalHexCompletenessCheck; + +type SemanticChartColorKey = + | 'success' + | 'warning' + | 'error' + | 'info' + | 'successHighlight' + | 'warningHighlight' + | 'errorHighlight'; + +/** + * Per-brand semantic chart palette. SSR / `getComputedStyle` fallback + * for `getChartColor{Success,Warning,Error,Info,*Highlight}` helpers. + * Live values come from `--color-chart-{success|warning|error|info}[-highlight]` + * in `_chart-categorical-tokens.scss` (`chart-semantic-tokens` mixin). + * + * Kept per-brand (instead of collapsed to one object) so the + * `Record<'hyperdx' | 'clickstack', SemanticChartHexes>` constraint + * forces both brands to declare every semantic key — dropping or + * renaming one in either entry becomes a compile error rather than a + * silent runtime divergence between SSR and client. The two entries + * are byte-identical today; collapse to a flat object if and when a + * brand actually needs to diverge. + */ +type SemanticChartHexes = Readonly>; + +const SEMANTIC_CHART_PALETTE: Readonly< + Record<'hyperdx' | 'clickstack', SemanticChartHexes> +> = { + hyperdx: { + success: '#3ca951', + warning: '#efb118', + error: '#ff725c', + info: '#437eef', + successHighlight: '#80d9b3', + warningHighlight: '#f5c94d', + errorHighlight: '#ffa090', + }, + clickstack: { + success: '#3ca951', + warning: '#efb118', + error: '#ff725c', + info: '#437eef', + successHighlight: '#80d9b3', + warningHighlight: '#f5c94d', + errorHighlight: '#ffa090', + }, +}; + +/** + * Ordered hex array for positional series assignment. + * `COLORS[i]` === `CATEGORICAL_HEX_BY_TOKEN[CATEGORICAL_PALETTE_TOKENS[i]]`. + * Returned directly by `getColorFromCSSVariable(i)` on both server and + * client — the categorical palette is unified across themes, so there's + * no benefit to reading the matching CSS var via `getComputedStyle`. + * + * Typed as `readonly string[]` (not `string[]`) because the array is a + * derived snapshot of `CATEGORICAL_HEX_BY_TOKEN` — mutating it in place + * would desync the two structures the completeness check above pins + * together. `CATEGORICAL_PALETTE_TOKENS` is a `readonly` tuple of + * `CategoricalChartPaletteToken` already, so the `.map` callback's + * `token` parameter is the narrow union and `CATEGORICAL_HEX_BY_TOKEN` + * index lookup is exhaustive without further assertion. + */ +export const COLORS: readonly string[] = CATEGORICAL_PALETTE_TOKENS.map( + token => CATEGORICAL_HEX_BY_TOKEN[token], +); /** * Palette token types and runtime guards live in common-utils so the @@ -421,7 +490,7 @@ export const COLORS = [ export { CATEGORICAL_PALETTE_TOKENS, CHART_PALETTE_TOKENS, - isChartPaletteToken, + resolveChartPaletteToken, SEMANTIC_PALETTE_TOKENS, } from '@hyperdx/common-utils/dist/types'; export type { ChartPaletteToken }; @@ -448,59 +517,53 @@ function detectActiveTheme(): 'clickstack' | 'hyperdx' { } /** - * Reads chart color from CSS variable based on index. - * CSS variables handle theme switching automatically via theme classes on documentElement. - * Falls back to COLORS array if CSS variable is not available (SSR or getComputedStyle fails). + * Returns the Nth categorical chart hex by series index. Index wraps + * modulo `CATEGORICAL_PALETTE_TOKENS.length`. * - * Note on SSR/Hydration: During SSR, this returns fallback colors (HyperDX green palette). - * On client hydration, it reads from CSS variables which may differ for ClickStack theme. - * This is expected behavior - charts typically render after data fetching (client-side), - * so hydration mismatches are rare. If needed, wrap chart components with suppressHydrationWarning. + * Reads from the JS palette directly. The matching `--color-chart-{hue}` + * CSS var resolves to the same hex on every theme today, so the + * previous `getComputedStyle` round-trip added a layout read per series + * with no functional benefit. If a future brand wants to override hues, + * reintroduce the DOM read here (and add per-brand entries to + * `CATEGORICAL_HEX_BY_TOKEN`). */ function getColorFromCSSVariable(index: number): string { - const colorArrayLength = COLORS.length; - - if (typeof window === 'undefined') { - // SSR: fallback to default colors (HyperDX palette) - return COLORS[index % colorArrayLength]; - } - - try { - const cssVarName = `--color-chart-${(index % colorArrayLength) + 1}`; - // Read from documentElement - CSS variables cascade from theme classes - const computedStyle = getComputedStyle(document.documentElement); - const color = computedStyle.getPropertyValue(cssVarName).trim(); - - // Only use CSS variable if it's actually set (non-empty) - if (color && color !== '') { - return color; - } - } catch { - // Fallback if getComputedStyle fails - } - - // Fallback to default colors - return COLORS[index % colorArrayLength]; + const i = index % CATEGORICAL_PALETTE_TOKENS.length; + return COLORS[i]; } /** - * Reads a chart color from a palette token by resolving the matching - * CSS variable (`--color-chart-` for categorical - * tokens, `--color-chart-{success|warning|error}` for semantic tokens). + * Resolves a chart palette token to a hex string. + * + * Categorical hue tokens (`chart-blue`, `chart-orange`, ...) come + * straight from `CATEGORICAL_HEX_BY_TOKEN` — the palette is unified + * across themes, so the matching `--color-chart-{hue}` CSS var would + * always resolve to the same value, and skipping `getComputedStyle` + * avoids an unnecessary layout read per series. * - * Falls back to the appropriate slot in `COLORS` (categorical) or the - * HyperDX semantic palette (success / warning / error) when running - * server-side or when `getComputedStyle` fails. + * Semantic tokens (`chart-success`, `-warning`, `-error`) DO vary per + * brand, so they read the matching CSS var (`--color-chart-{name}`) + * via `getComputedStyle` and fall back to the active theme's entry in + * `SEMANTIC_CHART_PALETTE` for SSR / `getComputedStyle` failures. * * @example - * getColorFromCSSToken('chart-1') // brand green + * getColorFromCSSToken('chart-blue') // Observable blue (both themes) * getColorFromCSSToken('chart-warning') // theme-aware warning */ export function getColorFromCSSToken(token: ChartPaletteToken): string { - const cssVarName = `--color-${token}`; + if (isCategoricalChartPaletteToken(token)) { + return CATEGORICAL_HEX_BY_TOKEN[token]; + } + + // After the categorical short-circuit, `token` is narrowed to a + // semantic token (`chart-success`/`chart-warning`/`chart-error`) + // — the parameter type that `semanticTokenFallback` enforces via + // its exhaustiveness check. + const semanticToken = token; + const cssVarName = `--color-${semanticToken}`; if (typeof window === 'undefined') { - return paletteTokenSSRFallback(token); + return semanticTokenFallback(semanticToken); } try { @@ -513,21 +576,38 @@ export function getColorFromCSSToken(token: ChartPaletteToken): string { // Fallback if getComputedStyle fails } - return paletteTokenSSRFallback(token); + return semanticTokenFallback(semanticToken); } -function paletteTokenSSRFallback(token: ChartPaletteToken): string { +function isCategoricalChartPaletteToken( + token: ChartPaletteToken, +): token is keyof typeof CATEGORICAL_HEX_BY_TOKEN { + return Object.prototype.hasOwnProperty.call(CATEGORICAL_HEX_BY_TOKEN, token); +} + +function semanticTokenFallback( + token: Exclude, +): string { switch (token) { case 'chart-success': - return CHART_PALETTE.green; case 'chart-warning': - return CHART_PALETTE.orange; - case 'chart-error': - return CHART_PALETTE.red; + case 'chart-error': { + const theme = detectActiveTheme(); + const key = token.slice('chart-'.length) as + | 'success' + | 'warning' + | 'error'; + return SEMANTIC_CHART_PALETTE[theme][key]; + } default: { - // Categorical token: chart-N where N is 1..10 - const index = Number(token.slice('chart-'.length)) - 1; - return COLORS[index] ?? COLORS[0]; + // Exhaustiveness assertion: if a new semantic token lands on + // `ChartPaletteToken` without a matching case above, this line + // becomes a compile error. The fallback was previously + // `COLORS[0]` with a "brand-primary" comment, but that path is + // unreachable through the parameter type and silently masked + // future drift. + const _exhaustive: never = token; + throw new Error(`Unhandled semantic chart token: ${_exhaustive}`); } } } @@ -546,21 +626,19 @@ export function hashCode(str: string) { } /** - * Gets theme-aware chart color from CSS variable or falls back to palette. - * Reads from --color-chart-{type} CSS variable, falls back to theme-appropriate palette. + * Theme-aware semantic chart color resolver. Reads `cssVarName` from + * `documentElement` and falls back to the active theme's value from + * `SEMANTIC_CHART_PALETTE` (HyperDX during SSR). * - * Note on SSR/Hydration: During SSR, returns HyperDX colors as default. - * On client, reads from CSS variables for accurate theme colors. - * Charts typically render client-side after data fetching, minimizing hydration issues. + * Charts typically render client-side after data fetching, so hydration + * mismatches between the SSR fallback and the live var are rare. */ function getSemanticChartColor( cssVarName: string, - hyperdxColor: string, - clickstackColor: string, + key: SemanticChartColorKey, ): string { if (typeof window === 'undefined') { - // SSR: use HyperDX as default (can't detect theme without DOM) - return hyperdxColor; + return SEMANTIC_CHART_PALETTE.hyperdx[key]; } try { @@ -573,59 +651,48 @@ function getSemanticChartColor( // Fallback if getComputedStyle fails } - // Fallback to theme-appropriate palette - const activeTheme = detectActiveTheme(); - return activeTheme === 'clickstack' ? clickstackColor : hyperdxColor; + return SEMANTIC_CHART_PALETTE[detectActiveTheme()][key]; } -// Semantic colors for log levels (theme-aware) -// These are functions that read from CSS variables with theme-appropriate fallbacks +// Semantic chart colors (theme-aware). Read from CSS variables with +// per-theme fallbacks in `SEMANTIC_CHART_PALETTE`. export function getChartColorSuccess(): string { - return getSemanticChartColor( - '--color-chart-success', - CHART_PALETTE.green, - CLICKSTACK_CHART_PALETTE.green, - ); + return getSemanticChartColor('--color-chart-success', 'success'); } export function getChartColorWarning(): string { - return getSemanticChartColor( - '--color-chart-warning', - CHART_PALETTE.orange, - CLICKSTACK_CHART_PALETTE.orange, - ); + return getSemanticChartColor('--color-chart-warning', 'warning'); } export function getChartColorError(): string { - return getSemanticChartColor( - '--color-chart-error', - CHART_PALETTE.red, - CLICKSTACK_CHART_PALETTE.red, - ); + return getSemanticChartColor('--color-chart-error', 'error'); +} + +/** Chart blue used for info-level logs and similar "neutral / default" + * series. Same hue as categorical `chart-blue` on both brands. */ +export function getChartColorInfo(): string { + return getSemanticChartColor('--color-chart-info', 'info'); } // Highlighted variants (theme-aware) export function getChartColorSuccessHighlight(): string { return getSemanticChartColor( '--color-chart-success-highlight', - CHART_PALETTE.greenHighlight, - CLICKSTACK_CHART_PALETTE.greenHighlight, + 'successHighlight', ); } export function getChartColorErrorHighlight(): string { return getSemanticChartColor( '--color-chart-error-highlight', - CHART_PALETTE.redHighlight, - CLICKSTACK_CHART_PALETTE.redHighlight, + 'errorHighlight', ); } export function getChartColorWarningHighlight(): string { return getSemanticChartColor( '--color-chart-warning-highlight', - CHART_PALETTE.orangeHighlight, - CLICKSTACK_CHART_PALETTE.orangeHighlight, + 'warningHighlight', ); } @@ -640,8 +707,7 @@ export const semanticKeyedColor = ( ? getChartColorError() : logLevel === 'warn' ? getChartColorWarning() - : // Info-level logs use primary chart color (blue for ClickStack, green for HyperDX) - getColorFromCSSVariable(0); + : getChartColorInfo(); } // Use CSS variable for theme-aware colors, fallback to hardcoded array @@ -654,8 +720,7 @@ export const logLevelColor = (key: string | number | undefined) => { ? getChartColorError() : logLevel === 'warn' ? getChartColorWarning() - : // Info-level logs use primary chart color (blue for ClickStack, green for HyperDX) - getColorFromCSSVariable(0); + : getChartColorInfo(); }; // order of colors for sorting. primary color (blue/green) on bottom, then yellow, then red @@ -672,8 +737,7 @@ const getLevelColor = (logLevel?: string) => { ? getChartColorError() : logLevel === 'warn' ? getChartColorWarning() - : // Info-level logs use primary chart color (blue for ClickStack, green for HyperDX) - getColorFromCSSVariable(0); + : getChartColorInfo(); }; export const getColorProps = (index: number, level: string): string => { diff --git a/packages/common-utils/src/__tests__/guards.test.ts b/packages/common-utils/src/__tests__/guards.test.ts index 0b9ee56241..0e04ca17bd 100644 --- a/packages/common-utils/src/__tests__/guards.test.ts +++ b/packages/common-utils/src/__tests__/guards.test.ts @@ -3,7 +3,14 @@ import { isBuilderSavedChartConfig, isRawSqlSavedChartConfig, } from '@/guards'; -import { DisplayType, isChartPaletteToken } from '@/types'; +import { + CATEGORICAL_PALETTE_TOKENS, + ChartPaletteTokenSchema, + DisplayType, + isChartPaletteToken, + resolveChartPaletteToken, + walkRawDashboardTileColors, +} from '@/types'; describe('isRawSqlSavedChartConfig', () => { it('returns true when configType is "sql"', () => { @@ -170,9 +177,9 @@ describe('displayTypeRequiresSource', () => { }); describe('isChartPaletteToken', () => { - it('returns true for all categorical tokens', () => { - for (let i = 1; i <= 10; i++) { - expect(isChartPaletteToken(`chart-${i}`)).toBe(true); + it('returns true for every hue-named categorical token', () => { + for (const token of CATEGORICAL_PALETTE_TOKENS) { + expect(isChartPaletteToken(token)).toBe(true); } }); @@ -182,6 +189,15 @@ describe('isChartPaletteToken', () => { expect(isChartPaletteToken('chart-error')).toBe(true); }); + it('returns false for legacy numeric tokens (handled by resolveChartPaletteToken, not the guard)', () => { + // The guard checks the current ChartPaletteToken enum strictly. + // Migration of legacy `chart-1`..`chart-10` is owned by + // `resolveChartPaletteToken` (render-time) and + // `normalizeDashboardTileColors` (fetch-time, in the app package). + expect(isChartPaletteToken('chart-1')).toBe(false); + expect(isChartPaletteToken('chart-10')).toBe(false); + }); + it('returns false for a raw hex string', () => { expect(isChartPaletteToken('#00c28a')).toBe(false); expect(isChartPaletteToken('#ff725c')).toBe(false); @@ -202,14 +218,14 @@ describe('isChartPaletteToken', () => { }); it('is case-sensitive (no uppercase matches)', () => { - expect(isChartPaletteToken('Chart-1')).toBe(false); + expect(isChartPaletteToken('Chart-Blue')).toBe(false); expect(isChartPaletteToken('CHART-SUCCESS')).toBe(false); expect(isChartPaletteToken('chart-Success')).toBe(false); }); - it('returns false for an out-of-range categorical index', () => { - expect(isChartPaletteToken('chart-0')).toBe(false); - expect(isChartPaletteToken('chart-11')).toBe(false); + it('returns false for an out-of-range categorical hue', () => { + expect(isChartPaletteToken('chart-magenta')).toBe(false); + expect(isChartPaletteToken('chart-teal')).toBe(false); }); it('returns false for strings that look similar but are not tokens', () => { @@ -218,3 +234,132 @@ describe('isChartPaletteToken', () => { expect(isChartPaletteToken('chart-neutral')).toBe(false); }); }); + +describe('ChartPaletteTokenSchema', () => { + it('accepts current hue-named and semantic tokens', () => { + expect(ChartPaletteTokenSchema.parse('chart-blue')).toBe('chart-blue'); + expect(ChartPaletteTokenSchema.parse('chart-light-blue')).toBe( + 'chart-light-blue', + ); + expect(ChartPaletteTokenSchema.parse('chart-success')).toBe( + 'chart-success', + ); + }); + + it('rejects legacy chart-1..10 — migration is owned by the app-side normalizer, not the schema', () => { + // Keeping the schema strict (no `z.preprocess`) keeps its `z.input` + // type equal to its `z.output` type. Wrapping the enum in + // `z.preprocess` would force the input to `unknown`, which + // poisons `validateRequest`'s `req.body` inference all the way + // up to `Dashboard.tiles[i].config.color`. Stored legacy values + // are healed at fetch time by `normalizeDashboardTileColors` in + // `packages/app/src/dashboard.ts`. + expect(() => ChartPaletteTokenSchema.parse('chart-1')).toThrow(); + expect(() => ChartPaletteTokenSchema.parse('chart-10')).toThrow(); + }); + + it('rejects unknown strings', () => { + expect(() => ChartPaletteTokenSchema.parse('chart-magenta')).toThrow(); + expect(() => ChartPaletteTokenSchema.parse('chart-11')).toThrow(); + expect(() => ChartPaletteTokenSchema.parse('#ff0000')).toThrow(); + }); +}); + +describe('resolveChartPaletteToken', () => { + it('returns hue-named tokens unchanged', () => { + expect(resolveChartPaletteToken('chart-blue')).toBe('chart-blue'); + expect(resolveChartPaletteToken('chart-green')).toBe('chart-green'); + expect(resolveChartPaletteToken('chart-light-blue')).toBe( + 'chart-light-blue', + ); + }); + + it('returns semantic tokens unchanged', () => { + expect(resolveChartPaletteToken('chart-success')).toBe('chart-success'); + expect(resolveChartPaletteToken('chart-warning')).toBe('chart-warning'); + expect(resolveChartPaletteToken('chart-error')).toBe('chart-error'); + }); + + it('migrates legacy chart-1..10 to their HyperDX-slot-order hue equivalents', () => { + expect(resolveChartPaletteToken('chart-1')).toBe('chart-green'); + expect(resolveChartPaletteToken('chart-2')).toBe('chart-blue'); + expect(resolveChartPaletteToken('chart-3')).toBe('chart-orange'); + expect(resolveChartPaletteToken('chart-4')).toBe('chart-red'); + expect(resolveChartPaletteToken('chart-5')).toBe('chart-cyan'); + expect(resolveChartPaletteToken('chart-6')).toBe('chart-pink'); + expect(resolveChartPaletteToken('chart-7')).toBe('chart-purple'); + expect(resolveChartPaletteToken('chart-8')).toBe('chart-light-blue'); + expect(resolveChartPaletteToken('chart-9')).toBe('chart-brown'); + expect(resolveChartPaletteToken('chart-10')).toBe('chart-gray'); + }); + + it('returns undefined for unknown strings, hex values, and non-strings', () => { + expect(resolveChartPaletteToken('chart-magenta')).toBeUndefined(); + expect(resolveChartPaletteToken('chart-11')).toBeUndefined(); + expect(resolveChartPaletteToken('#ff0000')).toBeUndefined(); + expect(resolveChartPaletteToken('')).toBeUndefined(); + expect(resolveChartPaletteToken(undefined)).toBeUndefined(); + expect(resolveChartPaletteToken(null)).toBeUndefined(); + expect(resolveChartPaletteToken(1)).toBeUndefined(); + }); +}); + +// `walkRawDashboardTileColors` is the single shared per-tile traversal +// behind four sites: the React app's fetch- and write-time normalizer, +// the JSON-import pre-validation pass, the API dashboards-route +// middleware, and the dashboard provisioner. The tests below pin its +// contract directly; the per-callsite suites exercise the policy +// composed over it (legacy → hue, unknown preserved, etc). +describe('walkRawDashboardTileColors', () => { + const identity = (current: string) => current; + + it('rewrites string colors via onColor', () => { + const input = { tiles: [{ config: { color: 'chart-1' } }] }; + const result = walkRawDashboardTileColors(input, () => 'chart-green') as { + tiles: Array<{ config: { color: string } }>; + }; + expect(result.tiles[0].config.color).toBe('chart-green'); + }); + + it('strips the color field when onColor returns undefined', () => { + const input = { + tiles: [{ id: 't1', config: { color: 'bad', other: 'kept' } }], + }; + const result = walkRawDashboardTileColors(input, () => undefined) as { + tiles: Array<{ id: string; config: { color?: string; other?: string } }>; + }; + expect(result.tiles[0].config).not.toHaveProperty('color'); + expect(result.tiles[0].config.other).toBe('kept'); + expect(result.tiles[0].id).toBe('t1'); + }); + + it('preserves referential identity when the callback returns the same string', () => { + const input = { tiles: [{ config: { color: 'chart-orange' } }] }; + expect(walkRawDashboardTileColors(input, identity)).toBe(input); + }); + + it('skips tiles whose config has a non-string color', () => { + const input = { tiles: [{ config: { color: 42 } }] }; + expect(walkRawDashboardTileColors(input, () => 'chart-blue')).toBe(input); + }); + + it('returns the input unchanged when tiles is missing, non-array, null, or a primitive', () => { + expect(walkRawDashboardTileColors({ name: 'D' }, identity)).toEqual({ + name: 'D', + }); + expect(walkRawDashboardTileColors({ tiles: 'nope' }, identity)).toEqual({ + tiles: 'nope', + }); + expect(walkRawDashboardTileColors(null, identity)).toBeNull(); + expect(walkRawDashboardTileColors('hello', identity)).toBe('hello'); + expect(walkRawDashboardTileColors(undefined, identity)).toBeUndefined(); + }); + + it('handles tiles whose config is missing or non-object', () => { + const input = { + tiles: [{ id: 'a' }, { id: 'b', config: null }, { id: 'c', config: 7 }], + }; + // No color fields to touch → identity output. + expect(walkRawDashboardTileColors(input, () => 'x')).toBe(input); + }); +}); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 5666ce2cfb..50315a3dba 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -795,8 +795,13 @@ export function isOnClickDashboardById( * * Tokens map to CSS variables in * `packages/app/src/theme/themes//_tokens.scss`: - * chart-1 .. chart-10 -> --color-chart-1 .. --color-chart-10 - * chart-success/warning/error -> --color-chart-{success|warning|error} + * chart-{hue} -> --color-chart-{hue} (10 hues, unified across themes) + * chart-success/warning/error -> --color-chart-{success|warning|error} (semantic; unified across brands) + * + * `chart-info` is a render-time CSS variable (defined in the shared + * `chart-semantic-tokens` SCSS mixin) but is intentionally *not* in the + * picker enum — it's consumed only by code paths that always want + * brand-primary (e.g. info-level log series, `getChartColorInfo()`). * * Storing tokens (not hex) lets user choices reflow correctly across * themes and color modes; see notes/repo-conventions/hyperdx/tile-styling.md. @@ -806,34 +811,87 @@ export function isOnClickDashboardById( * stays in `packages/app/src/utils.ts` because it depends on * `getComputedStyle(document.documentElement)`. */ -export const CHART_PALETTE_TOKENS = [ - 'chart-1', - 'chart-2', - 'chart-3', - 'chart-4', - 'chart-5', - 'chart-6', - 'chart-7', - 'chart-8', - 'chart-9', - 'chart-10', +/** Categorical tokens (10 hues). Tuple literal so the element type + * stays narrow (`'chart-blue' | 'chart-orange' | ...`) rather than + * widening to `ChartPaletteToken`; downstream consumers like + * `CATEGORICAL_HEX_BY_TOKEN` in `packages/app/src/utils.ts` rely on + * the narrow element type to enforce 1:1 coverage at compile time. */ +export const CATEGORICAL_PALETTE_TOKENS = [ + 'chart-blue', + 'chart-orange', + 'chart-red', + 'chart-cyan', + 'chart-green', + 'chart-pink', + 'chart-purple', + 'chart-light-blue', + 'chart-brown', + 'chart-gray', +] as const; + +/** Semantic tokens (success / warning / error). Tuple literal for the + * same narrow-element-type reason as the categorical list above. */ +export const SEMANTIC_PALETTE_TOKENS = [ 'chart-success', 'chart-warning', 'chart-error', ] as const; +export const CHART_PALETTE_TOKENS = [ + ...CATEGORICAL_PALETTE_TOKENS, + ...SEMANTIC_PALETTE_TOKENS, +] as const; + export type ChartPaletteToken = (typeof CHART_PALETTE_TOKENS)[number]; -/** Categorical tokens (chart-1 .. chart-10). */ -export const CATEGORICAL_PALETTE_TOKENS = CHART_PALETTE_TOKENS.slice( - 0, - 10, -) as readonly ChartPaletteToken[]; +/** Numeric tokens (`chart-1` .. `chart-10`) shipped in #2265. */ +type LegacyChartPaletteTokenKey = + | 'chart-1' + | 'chart-2' + | 'chart-3' + | 'chart-4' + | 'chart-5' + | 'chart-6' + | 'chart-7' + | 'chart-8' + | 'chart-9' + | 'chart-10'; -/** Semantic tokens (success / warning / error). */ -export const SEMANTIC_PALETTE_TOKENS = CHART_PALETTE_TOKENS.slice( - 10, -) as readonly ChartPaletteToken[]; +/** + * Legacy numeric tokens (`chart-1` .. `chart-10`) shipped in the initial + * release of the number-tile color picker (#2265). Renamed to hue-named + * tokens here to make stored configs and the external API schema + * self-documenting; mapped at parse time so saved tiles keep working. + * + * Mapping preserves the HyperDX slot ordering from #2265 (slot 1 was + * brand green, slot 2 was blue, and so on through the Observable 10 + * palette). + * + * ⚠️ ClickStack caveat: pre-rename ClickStack resolved `--color-chart-1` + * to brand blue, not brand green, so a ClickStack tile saved with the + * old "Color 1" will visually shift after migration. The trade-off + * (and why we don't theme-branch this map) is documented in + * `agent_docs/data_viz_colors.md` and the changeset for #2362. + * + * Keyed by the narrow `LegacyChartPaletteTokenKey` union (rather than + * `string`) so a typo in a legacy slot at edit time becomes a compile + * error. + */ +export const LEGACY_CHART_PALETTE_TOKEN_MAP = { + 'chart-1': 'chart-green', + 'chart-2': 'chart-blue', + 'chart-3': 'chart-orange', + 'chart-4': 'chart-red', + 'chart-5': 'chart-cyan', + 'chart-6': 'chart-pink', + 'chart-7': 'chart-purple', + 'chart-8': 'chart-light-blue', + 'chart-9': 'chart-brown', + 'chart-10': 'chart-gray', +} as const satisfies Record; + +export type LegacyChartPaletteToken = + keyof typeof LEGACY_CHART_PALETTE_TOKEN_MAP; /** Type guard for runtime validation of an unknown token string. */ export function isChartPaletteToken( @@ -845,7 +903,114 @@ export function isChartPaletteToken( ); } -/** Zod schema that accepts only the curated palette tokens above. */ +function isLegacyChartPaletteToken( + value: unknown, +): value is LegacyChartPaletteToken { + return ( + typeof value === 'string' && + Object.prototype.hasOwnProperty.call(LEGACY_CHART_PALETTE_TOKEN_MAP, value) + ); +} + +/** + * Resolve any string to a canonical `ChartPaletteToken`, accepting both + * current hue-named tokens and legacy numeric tokens (`chart-1` .. + * `chart-10`) from #2265. Returns `undefined` for anything else. + * + * Use this at every render-time consumption point (dashboard tile + * renderers like `DBNumberChart`, the color picker's `safeValue` guard + * in `ColorSwatchInput`, etc.). The app's normalizer + * (`normalizeDashboardTileColors` in `packages/app/src/dashboard.ts`) + * heals dashboards both on fetch (`useDashboards` / + * `fetchLocalDashboards`) and on write (`useUpdateDashboard` / + * `useCreateDashboard`), so the DB-side data converges on next save + * and JSON imports / preset constructions don't trip the strict + * `ChartPaletteTokenSchema`. Render-time consumers still call this + * helper as defense in depth for tiles built in memory between fetch + * and save (`ChartEditor` form state, unit-test fixtures). + */ +export function resolveChartPaletteToken( + value: unknown, +): ChartPaletteToken | undefined { + if (typeof value !== 'string') return undefined; + if (isLegacyChartPaletteToken(value)) { + return LEGACY_CHART_PALETTE_TOKEN_MAP[value]; + } + return isChartPaletteToken(value) ? value : undefined; +} + +/** + * Walk a parsed-but-not-yet-typed dashboard payload and yield each + * `tiles[i].config.color` that holds a string, asking `onColor` what + * the new value should be. The walker is the single shared + * implementation behind: + * - the React app's fetch- / write-time normalizer + * (`normalizeDashboardTileColors` in `packages/app/src/dashboard.ts`) + * - the JSON-import pre-validation pass + * (`normalizeRawDashboardTileColors`, same file) + * - the API dashboards route middleware + * (`migrateLegacyDashboardTileColors` in + * `packages/api/src/routers/api/dashboards.ts`) + * - the dashboard-provisioner task + * (`packages/api/src/tasks/provisionDashboards/index.ts`) + * + * `onColor` receives the current string and returns one of: + * - `undefined` → strip the `color` field from that tile's config. + * - a string identical to `current` → leave the tile untouched + * (preserves referential identity so React reconciliation stays + * cheap). + * - a different string → rewrite `config.color` to the new value. + * + * Returns the (possibly new) `input` reference. When nothing changed, + * the same `input` is returned so `===` callers can short-circuit. + * Inputs that aren't an object, or whose `tiles` isn't an array, are + * returned unchanged. + */ +export function walkRawDashboardTileColors( + input: unknown, + onColor: (current: string) => string | undefined, +): unknown { + if (!input || typeof input !== 'object') return input; + const root = input as { tiles?: unknown }; + const tiles = root.tiles; + if (!Array.isArray(tiles)) return input; + let changed = false; + const nextTiles = (tiles as unknown[]).map(tile => { + if (!tile || typeof tile !== 'object') return tile; + const t = tile as { config?: unknown }; + const config = t.config; + if (!config || typeof config !== 'object') return tile; + const c = config as { color?: unknown }; + const current = c.color; + if (typeof current !== 'string') return tile; + const next = onColor(current); + if (next === current) return tile; + changed = true; + if (next === undefined) { + const { color: _drop, ...rest } = c; + return { ...t, config: rest }; + } + return { ...t, config: { ...c, color: next } }; + }); + return changed ? { ...root, tiles: nextTiles } : input; +} + +/** + * Strict Zod schema for the curated palette tokens. Intentionally + * does NOT accept legacy numeric tokens (`chart-1` .. `chart-10`) + * from #2265 — wrapping the enum in `z.preprocess` would force the + * schema's input type to `unknown`, which breaks downstream `z.infer` + * consumers (e.g. `validateRequest` in the API handlers infers + * `req.body` as `unknown` for any field reached through this schema). + * + * Legacy data is healed at load time instead: see + * `normalizeDashboardTileColors` in `packages/app/src/dashboard.ts`, + * which walks `tiles[i].config.color` and replaces any legacy token + * with its hue-named equivalent via `resolveChartPaletteToken`. + * Render-time consumers also call `resolveChartPaletteToken` as + * belt-and-suspenders against any data path that bypasses the + * fetch-time normalizer. + */ export const ChartPaletteTokenSchema = z.enum(CHART_PALETTE_TOKENS); // When making changes here, consider if they need to be made to the external API From 56c58663b1b7729afe89b163fe8141ae85575407 Mon Sep 17 00:00:00 2001 From: Mike Shi Date: Fri, 5 Jun 2026 07:30:45 -0700 Subject: [PATCH 027/728] fix(search-filters): prevent nested filter dropdowns from disappearing on reopen (#2421) ## Summary Nested filter dropdowns in ClickStack Search (e.g. `ResourceAttributes`, `LogAttributes`) showed no filters after being closed and reopened. ## Root Cause The bug was introduced by PR #2096 (Mantine v7 to v9 upgrade). In Mantine v9, `Accordion.Panel` uses a `Collapse` component that transitions through `display: none` state when closed. The `NestedFilterGroup` component previously used `{isExpanded && ...}` to conditionally mount its panel content, including the scroll container used by `@tanstack/react-virtual`. When reopening: 1. `isExpanded` becomes true - content mounts inside the Panel 2. But the Collapse wrapper still has `display: none` (until an async `requestAnimationFrame` fires) 3. The virtualizer observes the scroll element via `useLayoutEffect` and calls `getBoundingClientRect()` - returns `{width: 0, height: 0}` because the element is inside `display: none` 4. With `outerSize = 0`, the virtualizer's `calculateRange()` returns `null` - `getVirtualItems()` returns `[]` 5. No child filter items are rendered In Mantine v7, this pattern worked because the Panel either fully unmounted its content or didn't transition through `display: none`, so the virtualizer always had valid dimensions when content was present. ## Fix Remove the `{isExpanded && ...}` conditional wrapper from `NestedFilterGroup`. The scroll container now stays mounted inside the `Accordion.Panel` at all times, maintaining the virtualizer's connection to it. This is safe because: - The virtualizer already uses `count: isExpanded ? childFilters.length : 0` to prevent rendering items when collapsed - The `Accordion.Panel`'s built-in `Collapse` animation handles visibility - When reopening, the `ResizeObserver` detects the scroll element's dimension change (from `display: none` to `display: block`) and the virtualizer renders items correctly ## Testing - All 1208 unit tests pass - Added a regression test that verifies open - close - reopen behavior keeps the panel accessible - Lint and TypeScript checks pass Linear Issue: [HDX-4471](https://linear.app/clickhouse/issue/HDX-4471/dropdown-filters-disappear-when-reopened-in-clickstack-search)
Open in Web Open in Cursor 
Co-authored-by: Cursor Agent <199161495+cursoragent@users.noreply.github.com> Co-authored-by: Brandon Pereira <7552738+brandon-pereira@users.noreply.github.com> --- .changeset/angry-boats-lie.md | 5 + .../DBSearchPageFilters/NestedFilterGroup.tsx | 209 +++++++++--------- .../__tests__/DBSearchPageFilters.test.tsx | 23 ++ 3 files changed, 129 insertions(+), 108 deletions(-) create mode 100644 .changeset/angry-boats-lie.md diff --git a/.changeset/angry-boats-lie.md b/.changeset/angry-boats-lie.md new file mode 100644 index 0000000000..a5e9d5fc74 --- /dev/null +++ b/.changeset/angry-boats-lie.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix(search-filters): prevent nested filter dropdowns from disappearing on reopen diff --git a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx index 8a3106b5cc..945b9d60e5 100644 --- a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx +++ b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx @@ -149,122 +149,115 @@ export const NestedFilterGroup = ({ pb={0} pl="xs" > - {isExpanded && ( -
- {childFilters.length === 0 ? ( - - - No properties found - - - ) : ( +
+ {childFilters.length === 0 ? ( + + + No properties found + + + ) : ( +
-
- {rowVirtualizer.getVirtualItems().map(virtualRow => { - const child = childFilters[virtualRow.index]; - const childSelectedValues = selectedValues[ - child.key - ] || { - included: new Set(), - excluded: new Set(), - }; - const childHasSelections = - childSelectedValues.included.size + - childSelectedValues.excluded.size > - 0; + {rowVirtualizer.getVirtualItems().map(virtualRow => { + const child = childFilters[virtualRow.index]; + const childSelectedValues = selectedValues[child.key] || { + included: new Set(), + excluded: new Set(), + }; + const childHasSelections = + childSelectedValues.included.size + + childSelectedValues.excluded.size > + 0; - return ( -
+ ({ + value: value, + label: value.toString(), + }))} + optionsLoading={false} + selectedValues={childSelectedValues} + onChange={value => onChange(child.key, value)} + onClearClick={() => onClearClick(child.key)} + onOnlyClick={value => onOnlyClick(child.key, value)} + onExcludeClick={value => + onExcludeClick(child.key, value) + } + valuePins={{ + onPinClick: value => onPinClick(child.key, value), + isPinned: value => isPinned(child.key, value), + onSharedPinClick: onSharedPinClick + ? value => onSharedPinClick(child.key, value) + : undefined, + isSharedPinned: isSharedPinned + ? value => isSharedPinned(child.key, value) + : undefined, }} - > - ({ - value: value, - label: value.toString(), - }))} - optionsLoading={false} - selectedValues={childSelectedValues} - onChange={value => onChange(child.key, value)} - onClearClick={() => onClearClick(child.key)} - onOnlyClick={value => - onOnlyClick(child.key, value) - } - onExcludeClick={value => - onExcludeClick(child.key, value) - } - valuePins={{ - onPinClick: value => - onPinClick(child.key, value), - isPinned: value => isPinned(child.key, value), - onSharedPinClick: onSharedPinClick - ? value => onSharedPinClick(child.key, value) - : undefined, - isSharedPinned: isSharedPinned - ? value => isSharedPinned(child.key, value) - : undefined, - }} - fieldPins={{ - onFieldPinClick: () => - onFieldPinClick?.(child.key), - isFieldPinned: isFieldPinned?.(child.key), - onToggleSharedFieldPin: () => - onToggleSharedFieldPin?.(child.key), - isSharedFieldPinned: isSharedFieldPinned?.( - child.key, - ), - }} - onColumnToggle={ - onColumnToggle - ? () => onColumnToggle(child.key) - : undefined - } - isColumnDisplayed={displayedColumns?.includes( + fieldPins={{ + onFieldPinClick: () => + onFieldPinClick?.(child.key), + isFieldPinned: isFieldPinned?.(child.key), + onToggleSharedFieldPin: () => + onToggleSharedFieldPin?.(child.key), + isSharedFieldPinned: isSharedFieldPinned?.( child.key, - )} - onLoadMore={() => onLoadMore(child.key)} - loadMoreLoading={ - loadMoreLoading[child.key] || false - } - hasLoadedMore={hasLoadedMore[child.key] || false} - showFilterCounts={showFilterCounts} - isDefaultExpanded={childHasSelections} - chartConfig={chartConfig} - isLive={isLive} - /> -
- ); - })} -
+ ), + }} + onColumnToggle={ + onColumnToggle + ? () => onColumnToggle(child.key) + : undefined + } + isColumnDisplayed={displayedColumns?.includes( + child.key, + )} + onLoadMore={() => onLoadMore(child.key)} + loadMoreLoading={ + loadMoreLoading[child.key] || false + } + hasLoadedMore={hasLoadedMore[child.key] || false} + showFilterCounts={showFilterCounts} + isDefaultExpanded={childHasSelections} + chartConfig={chartConfig} + isLive={isLive} + /> +
+ ); + })}
- )} -
- )} +
+ )} +
diff --git a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx index f1c54849b9..a275dbcec0 100644 --- a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx +++ b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx @@ -691,4 +691,27 @@ describe('NestedFilterGroup', () => { expect(screen.queryAllByTestId(/nested-filter-group-attr/)).toHaveLength(0); }); + + it('should allow reopening after close (panel stays accessible)', async () => { + renderWithMantine( + , + ); + + // Initially closed + const panel = screen.getByTestId('nested-filter-group-panel'); + expect(panel).toHaveAttribute('aria-hidden', 'true'); + + // Open the group + const control = screen.getByTestId('nested-filter-group-control'); + await userEvent.click(control); + expect(panel).toHaveAttribute('aria-hidden', 'false'); + + // Close the group + await userEvent.click(control); + expect(panel).toHaveAttribute('aria-hidden', 'true'); + + // Reopen the group — should not be stuck closed + await userEvent.click(control); + expect(panel).toHaveAttribute('aria-hidden', 'false'); + }); }); From ee907386437585031362f0baf0ee24acf668085e Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 5 Jun 2026 11:40:31 -0400 Subject: [PATCH 028/728] fix: Add sourceId to MCP Raw SQL Tile schema (#2424) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR updates the MCP server's dashboard schemas/prompts to allow agents to set source ID for raw sql chart configs. This is required for certain macros to work. ### Screenshots or video Claude now generates dashboards with Raw SQL Tiles with the Source ID set: Screenshot 2026-06-05 at 10 55 29 AM ### How to test on Vercel preview This can be tested locally by asking your agent to generate a dashboard with some Raw SQL Tiles. ### References - Linear Issue: Closes HDX-4463 - Related PRs: --- .changeset/chilled-drinks-pay.md | 6 + .../dashboards/patchDashboard.test.ts | 95 ++++++++++ .../dashboards/saveDashboard.test.ts | 109 +++++++++++ .../__tests__/dashboards/validation.test.ts | 169 ++++++++++++++++++ .../api/src/mcp/prompts/dashboards/content.ts | 36 ++-- .../mcp/tools/dashboards/patchDashboard.ts | 11 ++ .../src/mcp/tools/dashboards/saveDashboard.ts | 11 ++ .../api/src/mcp/tools/dashboards/schemas.ts | 14 +- .../src/mcp/tools/dashboards/validation.ts | 61 +++++++ .../external-api/v2/utils/dashboards.ts | 2 +- packages/common-utils/src/macros.ts | 16 ++ 11 files changed, 518 insertions(+), 12 deletions(-) create mode 100644 .changeset/chilled-drinks-pay.md create mode 100644 packages/api/src/mcp/__tests__/dashboards/validation.test.ts create mode 100644 packages/api/src/mcp/tools/dashboards/validation.ts diff --git a/.changeset/chilled-drinks-pay.md b/.changeset/chilled-drinks-pay.md new file mode 100644 index 0000000000..d5a8b7d1b9 --- /dev/null +++ b/.changeset/chilled-drinks-pay.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +--- + +fix: Add sourceId to MCP Raw SQL Tile schema diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts index 666f82edaf..98d4927c3c 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts @@ -294,6 +294,101 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { expect(getFirstText(result)).toContain('source'); }); + it('should reject patching a tile to raw SQL with $__filters but no sourceId', async () => { + const sourceId = ctx.traceSource._id.toString(); + const connectionId = ctx.connection._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'SQL Macro Patch Test', + tiles: [ + { + name: 'Valid Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const tileId = created.tiles[0].id; + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: created.id, + tileId, + tile: { + name: 'Filtered SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: + 'SELECT ServiceName, count() AS c FROM otel_traces WHERE $__timeFilter(Timestamp) AND $__filters GROUP BY ServiceName LIMIT 10', + }, + }, + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toContain('sourceId'); + expect(text).toContain('$__filters'); + expect(text).toContain('Filtered SQL'); + + // The original tile must be untouched in the database. + const dashboard = await Dashboard.findById(created.id); + const persistedTile = dashboard?.tiles?.find( + (t: { id: string }) => t.id === tileId, + ); + expect(persistedTile?.config?.name).toBe('Valid Tile'); + }); + + it('should allow patching a tile to raw SQL with $__filters when a sourceId is set', async () => { + const sourceId = ctx.traceSource._id.toString(); + const connectionId = ctx.connection._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'SQL Macro Patch Test (valid)', + tiles: [ + { + name: 'Valid Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: created.id, + tileId: created.tiles[0].id, + tile: { + name: 'Filtered SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sourceId, + sqlTemplate: + 'SELECT ServiceName, count() AS c FROM otel_traces WHERE $__timeFilter(Timestamp) AND $__filters GROUP BY ServiceName LIMIT 10', + }, + }, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.patchedTile.config.sourceId).toBe(sourceId); + }); + it('should not modify other tiles in the database (positional update)', async () => { const sourceId = ctx.traceSource._id.toString(); const createResult = await callTool( diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts index d0cfe53521..5b58b03e9f 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts @@ -262,6 +262,115 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { expect(output.tiles).toHaveLength(1); }); + it('should persist sourceId on a raw SQL tile that uses $__filters', async () => { + const connectionId = ctx.connection._id.toString(); + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'SQL Dashboard with filters', + tiles: [ + { + name: 'Filtered SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sourceId, + sqlTemplate: + 'SELECT ServiceName, count() AS c FROM otel_traces WHERE $__timeFilter(Timestamp) AND $__filters GROUP BY ServiceName LIMIT 10', + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + // sourceId must round-trip so the $__filters macro can resolve. + expect(output.tiles[0].config.sourceId).toBe(sourceId); + + const dashboard = await Dashboard.findById(output.id); + expect(dashboard?.tiles?.[0]?.config?.source?.toString()).toBe(sourceId); + }); + + it('should reject a raw SQL tile that uses $__filters without a sourceId', async () => { + const connectionId = ctx.connection._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'SQL Dashboard missing source', + tiles: [ + { + name: 'Filtered SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: + 'SELECT ServiceName, count() AS c FROM otel_traces WHERE $__timeFilter(Timestamp) AND $__filters GROUP BY ServiceName LIMIT 10', + }, + }, + ], + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toContain('sourceId'); + expect(text).toContain('$__filters'); + expect(text).toContain('Filtered SQL'); + + // Nothing should have been persisted. + expect(await Dashboard.countDocuments({})).toBe(0); + }); + + it('should reject a raw SQL tile that uses $__sourceTable without a sourceId', async () => { + const connectionId = ctx.connection._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'SQL Dashboard missing source', + tiles: [ + { + name: 'Source Table SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: + 'SELECT ServiceName, count() AS c FROM $__sourceTable WHERE $__timeFilter(Timestamp) GROUP BY ServiceName LIMIT 10', + }, + }, + ], + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toContain('sourceId'); + expect(text).toContain('$__sourceTable'); + expect(text).toContain('Source Table SQL'); + + // Nothing should have been persisted. + expect(await Dashboard.countDocuments({})).toBe(0); + }); + + it('should allow a raw SQL tile without a sourceId when it does not use $__filters', async () => { + const connectionId = ctx.connection._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Multi-table SQL Dashboard', + tiles: [ + { + name: 'Joined SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: + 'SELECT t.ServiceName, count() AS c FROM otel_traces t JOIN otel_logs l ON t.TraceId = l.TraceId GROUP BY t.ServiceName LIMIT 10', + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + }); + it('should create a dashboard with a heatmap tile on a Trace source', async () => { const sourceId = ctx.traceSource._id.toString(); const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts new file mode 100644 index 0000000000..88d0a1910f --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -0,0 +1,169 @@ +import mongoose from 'mongoose'; + +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +import { + getRawSqlMissingSourceError, + getRawSqlTilesMissingRequiredSource, +} from '../../tools/dashboards/validation'; + +const connectionId = new mongoose.Types.ObjectId().toString(); +const sourceId = new mongoose.Types.ObjectId().toString(); + +function makeSqlTile(overrides: { + name?: string; + sqlTemplate: string; + sourceId?: string; +}): ExternalDashboardTileWithId { + return { + id: 'sql-tile', + x: 0, + y: 0, + w: 12, + h: 4, + name: overrides.name ?? 'SQL Tile', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: overrides.sqlTemplate, + ...(overrides.sourceId ? { sourceId: overrides.sourceId } : {}), + }, + } as ExternalDashboardTileWithId; +} + +describe('getRawSqlTilesMissingRequiredSource', () => { + it('flags a raw SQL tile that uses $__filters without a sourceId', () => { + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ + name: 'Errors by Service', + sqlTemplate: + 'SELECT count() FROM otel_traces WHERE $__timeFilter(Timestamp) AND $__filters', + }), + ]), + ).toEqual([{ tile: 'Errors by Service', macros: ['$__filters'] }]); + }); + + it('flags a raw SQL tile that uses $__sourceTable without a sourceId', () => { + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ + name: 'Span Count', + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE $__timeFilter(Timestamp)', + }), + ]), + ).toEqual([{ tile: 'Span Count', macros: ['$__sourceTable'] }]); + }); + + it('reports every source-dependent macro used by a tile', () => { + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ + name: 'Both Macros', + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE $__timeFilter(Timestamp) AND $__filters', + }), + ]), + ).toEqual([ + { tile: 'Both Macros', macros: ['$__filters', '$__sourceTable'] }, + ]); + }); + + it('does not flag a raw SQL tile that sets a sourceId', () => { + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ + sourceId, + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE $__timeFilter(Timestamp) AND $__filters', + }), + ]), + ).toEqual([]); + }); + + it('does not flag a raw SQL tile that uses no source-dependent macro', () => { + // A multi-table query that legitimately omits sourceId and references + // neither $__filters nor $__sourceTable must not be rejected. + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ + sqlTemplate: + 'SELECT count() FROM otel_traces t JOIN otel_logs l ON t.TraceId = l.TraceId WHERE $__timeFilter(t.Timestamp) LIMIT 10', + }), + ]), + ).toEqual([]); + }); + + it('falls back to a positional label when the tile has no name', () => { + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ + name: '', + sqlTemplate: 'SELECT count() FROM otel_traces WHERE $__filters', + }), + ]), + ).toEqual([{ tile: 'tile #1', macros: ['$__filters'] }]); + }); + + it('ignores builder (non-SQL) tiles entirely', () => { + const builderTile = { + id: 'builder', + x: 0, + y: 0, + w: 12, + h: 4, + name: 'Line', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count', valueExpression: '', where: '' }], + }, + } as unknown as ExternalDashboardTileWithId; + expect(getRawSqlTilesMissingRequiredSource([builderTile])).toEqual([]); + }); + + it('collects every offending tile across a mixed set', () => { + expect( + getRawSqlTilesMissingRequiredSource([ + makeSqlTile({ name: 'A', sqlTemplate: 'SELECT 1 WHERE $__filters' }), + makeSqlTile({ + name: 'B', + sourceId, + sqlTemplate: 'SELECT 1 WHERE $__filters', + }), + makeSqlTile({ + name: 'C', + sqlTemplate: 'SELECT 1 FROM $__sourceTable', + }), + ]), + ).toEqual([ + { tile: 'A', macros: ['$__filters'] }, + { tile: 'C', macros: ['$__sourceTable'] }, + ]); + }); +}); + +describe('getRawSqlMissingSourceError', () => { + it('returns null when all tiles are valid', () => { + expect( + getRawSqlMissingSourceError([ + makeSqlTile({ sqlTemplate: 'SELECT 1 LIMIT 1' }), + ]), + ).toBeNull(); + }); + + it('builds a message naming the offending tiles and macros', () => { + const message = getRawSqlMissingSourceError([ + makeSqlTile({ + name: 'Errors by Service', + sqlTemplate: 'SELECT count() FROM otel_traces WHERE $__filters', + }), + ]); + expect(message).toContain('sourceId'); + expect(message).toContain('$__filters'); + expect(message).toContain('$__sourceTable'); + expect(message).toContain('Errors by Service (uses $__filters)'); + }); +}); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 9b1cc2f730..a79fde71ab 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -67,6 +67,9 @@ Use BUILDER tiles (with sourceId) for most cases: Use RAW SQL tiles (with connectionId) only for queries the builder cannot express: Requires configType: "sql" plus a displayType (line, stacked_bar, table, number, pie). Use when you need JOINs, sub-queries, CTEs, or expressions the builder does not generate. + ALWAYS set sourceId on a raw SQL tile (in addition to connectionId) UNLESS the query reads + from multiple tables (e.g. JOINs across sources). sourceId enables the $__filters and + $__sourceTable macros. When you can set sourceId, include "AND $__filters" in the WHERE clause. == COLUMN NAMING == @@ -677,7 +680,7 @@ copy the right ID and discover which filters are available. examples['infrastructure_sql'] = ` == INFRASTRUCTURE (Raw SQL) == -When to use: the user wants metric-like infrastructure visibility (request rate, ingestion volume, table sizes) but the builder tile types do not express the exact aggregation. Raw SQL tiles take connectionId (not sourceId) and a sqlTemplate with HyperDX macros. +When to use: the user wants metric-like infrastructure visibility (request rate, ingestion volume, table sizes) but the builder tile types do not express the exact aggregation. Raw SQL tiles take connectionId, a sqlTemplate with HyperDX macros, and (when the query reads a single table) a sourceId. Set sourceId and add "AND $__filters" so dashboard filters apply. { name: "Infrastructure (SQL)", @@ -689,7 +692,8 @@ When to use: the user wants metric-like infrastructure visibility (request rate, configType: "sql", displayType: "line", connectionId: "${connectionId}", - sqlTemplate: "SELECT $__timeInterval(Timestamp) AS ts, count() AS logs_per_interval FROM otel_logs WHERE $__timeFilter(Timestamp) GROUP BY ts ORDER BY ts" + sourceId: "${logSourceId}", + sqlTemplate: "SELECT $__timeInterval(Timestamp) AS ts, count() AS logs_per_interval FROM otel_logs WHERE $__timeFilter(Timestamp) AND $__filters GROUP BY ts ORDER BY ts" } }, { @@ -699,7 +703,8 @@ When to use: the user wants metric-like infrastructure visibility (request rate, configType: "sql", displayType: "table", connectionId: "${connectionId}", - sqlTemplate: "SELECT ServiceName, count() AS span_count, avg(Duration) AS avg_duration FROM otel_traces WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName ORDER BY span_count DESC LIMIT 20" + sourceId: "${traceSourceId}", + sqlTemplate: "SELECT ServiceName, count() AS span_count, avg(Duration) AS avg_duration FROM otel_traces WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) AND $__filters GROUP BY ServiceName ORDER BY span_count DESC LIMIT 20" } }, { @@ -709,7 +714,8 @@ When to use: the user wants metric-like infrastructure visibility (request rate, configType: "sql", displayType: "line", connectionId: "${connectionId}", - sqlTemplate: "SELECT $__timeInterval(Timestamp) AS ts, ServiceName, countIf(StatusCode = 'STATUS_CODE_ERROR') / count() AS error_rate FROM otel_traces WHERE $__timeFilter(Timestamp) GROUP BY ServiceName, ts ORDER BY ts" + sourceId: "${traceSourceId}", + sqlTemplate: "SELECT $__timeInterval(Timestamp) AS ts, ServiceName, countIf(StatusCode = 'STATUS_CODE_ERROR') / count() AS error_rate FROM otel_traces WHERE $__timeFilter(Timestamp) AND $__filters GROUP BY ServiceName, ts ORDER BY ts" } } ] @@ -725,7 +731,10 @@ SQL TEMPLATE REFERENCE: $__fromTime / $__toTime start/end as DateTime values $__fromTime_ms / $__toTime_ms start/end as DateTime64 values $__interval_s raw interval in seconds - $__filters dashboard filter conditions (resolves to 1=1 when none) + $__filters dashboard filter conditions (resolves to 1=1 when none). + REQUIRES sourceId on the tile. Without it, filters never apply. + $__sourceTable the source's \`database\`.\`table\`. + REQUIRES sourceId on the tile. Without it, the query fails to run. Query parameters: {startDateMilliseconds:Int64} start of date range in milliseconds @@ -901,7 +910,11 @@ For configType: "sql" tiles, write ClickHouse SQL with template macros: $__fromTime / $__toTime start/end as DateTime values $__fromTime_ms / $__toTime_ms start/end as DateTime64 values $__interval_s raw interval in seconds (for arithmetic) - $__filters dashboard filter conditions (1=1 when none) + $__filters dashboard filter conditions (1=1 when none). + REQUIRES sourceId on the tile. Set sourceId (unless the + query reads multiple tables) so dashboard filters apply. + $__sourceTable the source's \`database\`.\`table\`. + REQUIRES sourceId on the tile. Without it, the query fails to run. QUERY PARAMETERS (ClickHouse parameterized syntax): {startDateMilliseconds:Int64} @@ -1097,6 +1110,7 @@ Workaround: when you need a row-click drill-down from a map-attribute groupBy, a Raw SQL, alias the column: configType: "sql", displayType: "table", + sourceId: "", sqlTemplate: "SELECT SpanAttributes['http.route'] AS Route, count() AS Requests FROM otel_traces WHERE $__timeFilter(TimestampTime) AND $__filters GROUP BY Route ORDER BY Requests DESC LIMIT 100" onClick: { type: "search", target: { mode: "id", id: "" }, whereLanguage: "sql", @@ -1232,9 +1246,11 @@ Example: find top patterns for production services over the last 4 hours: 8. Forgetting to validate tiles after saving Always call clickstack_query_tile on EVERY tile after clickstack_save_dashboard, not just one. Save validates input shape but not query semantics. Several known gaps (Lucene comparison/wildcard on map attributes, builder tiles on metric sources, malformed having) pass save and fail at render time. A dashboard with one bad tile renders the whole page in a degraded state; the user sees "Error loading chart" with no way to know which tile broke unless you validated. If query_tile returns an error, fix the where / SQL and re-save before declaring the dashboard ready. -9. Using sourceId with SQL tiles or connectionId with builder tiles - Builder tiles (line, table, etc.) use sourceId. - SQL tiles (configType: "sql") use connectionId. +9. Using connectionId with builder tiles, or omitting connectionId or sourceId on a single-table SQL tile + Builder tiles (line, table, etc.) use sourceId (no connectionId). + SQL tiles (configType: "sql") REQUIRE connectionId, and should ALSO set sourceId + whenever the query reads a single table. SourceId is what makes $__filters work. + Only omit sourceId on a SQL tile when the query spans multiple tables (e.g. JOINs). 10. Assuming StatusCode or SeverityText values Values like STATUS_CODE_ERROR, Ok, error, fatal vary by deployment. @@ -1313,6 +1329,6 @@ External docs for the syntax and macros referenced in this guide: ClickHouse SQL functions (the "where" field with whereLanguage: "sql", and any sqlTemplate body): https://clickhouse.com/docs/sql-reference - Dashboard time / filter macros used in raw-SQL tile sqlTemplate ($__timeFilter, $__timeFilter_ms, $__timeInterval, $__timeInterval_ms, $__filters): + Dashboard time / filter macros used in raw-SQL tile sqlTemplate ($__timeFilter, $__timeFilter_ms, $__timeInterval, $__timeInterval_ms, $__filters, $__sourceTable): https://clickhouse.com/docs/use-cases/observability/clickstack/dashboards/sql-visualizations`; } diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index f21782c85d..9cc0a8483a 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -12,9 +12,11 @@ import { } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { mcpError } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; import { mcpPatchDashboardSchema } from './schemas'; +import { getRawSqlMissingSourceError } from './validation'; export function registerPatchDashboard( server: McpServer, @@ -163,6 +165,15 @@ export function registerPatchDashboard( config: incoming.config, } as ExternalDashboardTileWithId; + // Error on raw SQL tiles that have no source defined but which use + // macros which require a source to be set + const sqlFilterSourceError = getRawSqlMissingSourceError([ + mergedTile, + ]); + if (sqlFilterSourceError) { + return mcpError(sqlFilterSourceError); + } + // Validate the patched tile using the shared validation helper. const validationError = await validateDashboardTiles({ teamId, diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index 4fcee6d604..725db4a8d6 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -27,6 +27,7 @@ import { mcpError } from '../../utils/errors'; import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; import { mcpContainersParam, mcpFiltersParam, mcpTilesParam } from './schemas'; +import { getRawSqlMissingSourceError } from './validation'; export function registerSaveDashboard( server: McpServer, @@ -175,6 +176,11 @@ async function createDashboard({ const { tiles, filters, containers: parsedContainers } = parsed.data; const tilesWithId = tiles as ExternalDashboardTileWithId[]; + const sqlFilterSourceError = getRawSqlMissingSourceError(tilesWithId); + if (sqlFilterSourceError) { + return mcpError(sqlFilterSourceError); + } + const validationError = await validateDashboardTiles({ teamId, tiles: tilesWithId, @@ -272,6 +278,11 @@ async function updateDashboard({ const { tiles, filters, containers: parsedContainers } = parsed.data; const tilesWithId = tiles as ExternalDashboardTileWithId[]; + const sqlFilterSourceError = getRawSqlMissingSourceError(tilesWithId); + if (sqlFilterSourceError) { + return mcpError(sqlFilterSourceError); + } + const existingDashboard = await Dashboard.findOne( { _id: dashboardId, team: teamId }, { tiles: 1, filters: 1, containers: 1 }, diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index f88033a5a7..d3a306f426 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -589,6 +589,16 @@ const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ .describe( 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections', ), + sourceId: z + .string() + .optional() + .describe( + 'Source ID for the table this query reads from (call clickstack_list_sources). ' + + 'ALWAYS set this for raw SQL tiles UNLESS the query reads from multiple tables ' + + '(e.g. JOINs or sub-queries spanning several sources), in which case omit it. ' + + 'sourceId is REQUIRED by two macros: $__filters and $__sourceTable. ' + + 'The sourceId must belong to the same connection as connectionId.', + ), sqlTemplate: z .string() .describe( @@ -597,7 +607,9 @@ const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ '{intervalSeconds:Int64}, {intervalMilliseconds:Int64}.\n' + 'Or use macros: $__timeFilter(col), $__timeFilter_ms(col), $__dateFilter(col), ' + '$__fromTime, $__toTime, $__fromTime_ms, $__toTime_ms, ' + - '$__timeInterval(col), $__timeInterval_ms(col), $__interval_s, $__filters.\n' + + '$__timeInterval(col), $__timeInterval_ms(col), $__interval_s, $__filters, $__sourceTable.\n' + + 'IMPORTANT: $__filters and $__sourceTable both require sourceId to be set on this tile. ' + + 'Prefer including "AND $__filters" in the WHERE clause so dashboard filters apply.\n' + 'Example: "SELECT $__timeInterval(TimestampTime) AS ts, ServiceName, count() ' + 'FROM otel_logs WHERE $__timeFilter(TimestampTime) AND $__filters ' + 'GROUP BY ServiceName, ts ORDER BY ts"', diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts new file mode 100644 index 0000000000..2cdbbab170 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -0,0 +1,61 @@ +import { SOURCE_DEPENDENT_MACROS } from '@hyperdx/common-utils/dist/macros'; + +import { + isConfigTile, + isRawSqlExternalTileConfig, +} from '@/routers/external-api/v2/utils/dashboards'; +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +/** + * Returns one entry per raw SQL tile that uses a source-dependent macro but + * does not set a `sourceId`, recording which macro(s) triggered it so callers + * can build a precise error message. + */ +export function getRawSqlTilesMissingRequiredSource( + tiles: ExternalDashboardTileWithId[], +): { tile: string; macros: string[] }[] { + const offending: { tile: string; macros: string[] }[] = []; + tiles.forEach((tile, index) => { + if ( + !isConfigTile(tile) || + !isRawSqlExternalTileConfig(tile.config) || + tile.config.sourceId + ) { + return; + } + const { sqlTemplate } = tile.config; + const macros = SOURCE_DEPENDENT_MACROS.filter(macro => + sqlTemplate.includes(macro), + ); + if (macros.length > 0) { + offending.push({ + tile: tile.name?.trim() || `tile #${index + 1}`, + macros: [...macros], + }); + } + }); + return offending; +} + +/** + * MCP-only guard: a raw SQL tile that uses a source-dependent macro + * ($__filters or $__sourceTable) must set a `sourceId`. + * + * Returns a human-readable error message, or `null` when all tiles are valid. + */ +export function getRawSqlMissingSourceError( + tiles: ExternalDashboardTileWithId[], +): string | null { + const offending = getRawSqlTilesMissingRequiredSource(tiles); + if (offending.length === 0) return null; + const list = offending + .map(({ tile, macros }) => `${tile} (uses ${macros.join(', ')})`) + .join('; '); + return ( + 'Raw SQL tiles that use the $__filters or $__sourceTable macro must set a sourceId. ' + + "Without a source, $__filters cannot resolve dashboard filters against the source's " + + 'columns and $__sourceTable fails at query time. Add a sourceId to the following tiles ' + + '(call clickstack_list_sources to find it), or remove the macro if the query reads ' + + `from multiple tables: ${list}` + ); +} diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 214899f85c..a8506a3e6b 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -74,7 +74,7 @@ export type ConfigTile = ExternalDashboardTileWithId & { config: Exclude; }; -function isRawSqlExternalTileConfig( +export function isRawSqlExternalTileConfig( config: ExternalDashboardTileConfig, ): config is ExternalDashboardRawSqlTileConfig { return 'configType' in config && config.configType === 'sql'; diff --git a/packages/common-utils/src/macros.ts b/packages/common-utils/src/macros.ts index 297ebd1c7e..142cb84724 100644 --- a/packages/common-utils/src/macros.ts +++ b/packages/common-utils/src/macros.ts @@ -161,6 +161,22 @@ export const MACRO_SUGGESTIONS = [ })), ]; +/** + * Macro tokens (in `$__name` form) whose expansion depends on the chart's + * source being configured. With no source: + * - `$__filters` cannot resolve dashboard filters against the source's + * columns and falls back to `(1=1)`, so the filters are silently dropped. + * - `$__sourceTable` throws at render time ("requires a source to be + * selected"), so the query fails outright. + * + * Every other macro only needs the dashboard time range / interval and takes + * its column as an argument, so it does not require a source. + */ +export const SOURCE_DEPENDENT_MACROS = [ + '$__filters', + '$__sourceTable', +] as const; + type MacroMatch = { full: string; args: string[]; From 20fabc65879c5c89590e56589ffc6bd09f36af5b Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 5 Jun 2026 11:18:20 -0700 Subject: [PATCH 029/728] feat(team-settings): connect your AI assistant [HDX-4440] (#2407) ## Summary Adds a "Connect your AI Agents" section to the Team Settings page in a new "API & Agents" tab, alongside the existing API Keys card. Users get a one-click install path for the ClickStack MCP server in Claude Code, Cursor, VS Code, Codex CLI, OpenCode, or any MCP-compatible host without hand-rolling JSON. The MCP server resolves the active team from the bearer token, so the install snippet uses a single fixed `clickstack` server name; no per-tenant suffix needed on the client. Header values are shell-escaped against `\"`, `$`, backtick, and `\\` so a future access-key format with metacharacters can't turn a copy-paste install into a shell-injection vector. ## What's here - `packages/app/src/components/ClickStackOnboarding/` (new directory). Houses the install panel and its pieces: - `installSnippets.ts`: pure per-host snippet builders. Unit-tested round-trip through `buildAllSnippets`. - `McpInstallPanel.tsx`: host picker (Mantine `SegmentedControl`) + render branch per host. Exhaustive switch via `satisfies never` so adding a host variant fails the compile. - `CopySnippet.tsx`: code block with a `` affordance. - `DeeplinkInstall.tsx`: primary "Add to " button + `Manual setup` toggle that reveals a fallback JSON snippet. - `packages/app/src/components/TeamSettings/McpServerSection.tsx`: wraps the panel, derives the deployment shape from `useMe()`, mounts in the new tab. - `packages/app/src/TeamPage.tsx`: introduces a new "API & Agents" tab (URL value `api-agents`) that houses the API Keys card and the AI Agents install panel. The Integrations tab keeps webhooks only. - `.changeset/connect-your-ai-assistant.md`: user-facing change note. ## Hosts covered - Claude Code (CLI one-liner) - Cursor (`cursor://` deep link, JSON fallback behind `Manual setup`) - VS Code (`vscode:mcp/install` deep link, JSON fallback; requires Copilot Chat 1.99+) - Codex CLI (one-liner mirroring Claude Code; documented at https://developers.openai.com/codex/mcp) - OpenCode (JSON config under an `mcp` key, `type: "remote"`; documented at https://opencode.ai/docs/mcp-servers/) - Other (canonical `mcpServers` JSON block for Claude Desktop, Continue, Cline, and the long tail) ## Test plan - [x] `yarn jest --testPathPatterns="installSnippets|McpServerSection"` (25 tests pass). - [x] ESLint clean on every touched file. - [x] `tsc --noEmit` clean on every touched file. - [x] `prose-lint` clean on every touched file. - [x] Predicted tier: Tier 3. - [ ] Manual: bring up a self-managed deployment, navigate to Team Settings -> API & Agents, copy each host's snippet, install in the host, confirm the host connects against `/api/mcp`. Holding this checkbox until I drive the flow once. ## Notes - Branch prefix `alex/` (human prefix) keeps the classifier on the standard human thresholds. - Initial revision wrapped the e2e suite in `test.fixme(true, ...)` because the e2e webServer runs in `IS_LOCAL_MODE`, which short-circuits `useMe`/`useTeam` before the route mocks can land. That suite was deleted in the review-feedback fix-pack; component coverage in `__tests__/McpServerSection.test.tsx` exercises every render branch. A real e2e spec can come back once the e2e build-flag work lands. Linear: HDX-4440 Co-Authored-By: Claude Opus --- .changeset/connect-your-ai-assistant.md | 11 + packages/app/src/TeamPage.tsx | 19 +- .../ClickStackOnboarding/CopySnippet.tsx | 49 ++++ .../ClickStackOnboarding/DeeplinkInstall.tsx | 55 +++++ .../ClickStackOnboarding/McpInstallPanel.tsx | 209 ++++++++++++++++ .../__tests__/installSnippets.test.ts | 197 +++++++++++++++ .../ClickStackOnboarding/installSnippets.ts | 231 ++++++++++++++++++ .../TeamSettings/McpServerSection.tsx | 55 +++++ .../__tests__/McpServerSection.test.tsx | 190 ++++++++++++++ packages/app/tests/e2e/features/team.spec.ts | 26 +- .../app/tests/e2e/page-objects/TeamPage.ts | 19 ++ 11 files changed, 1050 insertions(+), 11 deletions(-) create mode 100644 .changeset/connect-your-ai-assistant.md create mode 100644 packages/app/src/components/ClickStackOnboarding/CopySnippet.tsx create mode 100644 packages/app/src/components/ClickStackOnboarding/DeeplinkInstall.tsx create mode 100644 packages/app/src/components/ClickStackOnboarding/McpInstallPanel.tsx create mode 100644 packages/app/src/components/ClickStackOnboarding/__tests__/installSnippets.test.ts create mode 100644 packages/app/src/components/ClickStackOnboarding/installSnippets.ts create mode 100644 packages/app/src/components/TeamSettings/McpServerSection.tsx create mode 100644 packages/app/src/components/TeamSettings/__tests__/McpServerSection.test.tsx diff --git a/.changeset/connect-your-ai-assistant.md b/.changeset/connect-your-ai-assistant.md new file mode 100644 index 0000000000..e39775e851 --- /dev/null +++ b/.changeset/connect-your-ai-assistant.md @@ -0,0 +1,11 @@ +--- +"@hyperdx/app": patch +--- + +feat: add a "Connect your AI assistant" section to Team Settings + +A new section on the Team Settings page (Integrations tab, above the API Keys +card) lets a user install the HyperDX MCP server in Claude Code, Cursor, +VS Code + Copilot, Codex CLI, or any MCP-compatible host without hand-rolling +JSON. Per-host snippets carry the user's personal access key so the install +works against the existing `/api/mcp` route without extra setup. diff --git a/packages/app/src/TeamPage.tsx b/packages/app/src/TeamPage.tsx index 4a9c2bffd4..f5dfb6e7f2 100644 --- a/packages/app/src/TeamPage.tsx +++ b/packages/app/src/TeamPage.tsx @@ -20,6 +20,7 @@ import { PageHeader } from './components/PageHeader'; import ApiKeysSection from './components/TeamSettings/ApiKeysSection'; import ConnectionsSection from './components/TeamSettings/ConnectionsSection'; import IntegrationsSection from './components/TeamSettings/IntegrationsSection'; +import McpServerSection from './components/TeamSettings/McpServerSection'; import SecurityPoliciesSection from './components/TeamSettings/SecurityPoliciesSection'; import SourcesSection from './components/TeamSettings/SourcesSection'; import TeamMembersSection from './components/TeamSettings/TeamMembersSection'; @@ -131,6 +132,20 @@ export default function TeamPage() { }, ] : []), + { + value: 'api-agents', + label: 'API & Agents', + sections: [ + { + id: 'team-api-agents-api-keys', + content: , + }, + { + id: 'team-api-agents-mcp-server', + content: , + }, + ], + }, { value: 'integrations', label: 'Integrations', @@ -139,10 +154,6 @@ export default function TeamPage() { id: 'team-integrations-webhooks', content: , }, - { - id: 'team-integrations-api-keys', - content: , - }, ], }, { diff --git a/packages/app/src/components/ClickStackOnboarding/CopySnippet.tsx b/packages/app/src/components/ClickStackOnboarding/CopySnippet.tsx new file mode 100644 index 0000000000..9f4b884ba7 --- /dev/null +++ b/packages/app/src/components/ClickStackOnboarding/CopySnippet.tsx @@ -0,0 +1,49 @@ +import { Button, Code, CopyButton, Group, Stack, Text } from '@mantine/core'; +import { IconCheck, IconCopy } from '@tabler/icons-react'; + +interface CopySnippetProps { + label: string; + snippet: string; +} + +/** + * Pre-formatted snippet with a copy-to-clipboard button. The copy + * affordance handles its own `Copied` affirmation via Mantine's + * ``. + */ +export function CopySnippet({ label, snippet }: CopySnippetProps) { + return ( + + + {label} + + + + {snippet} + + + {({ copied, copy }) => ( + + )} + + + + ); +} diff --git a/packages/app/src/components/ClickStackOnboarding/DeeplinkInstall.tsx b/packages/app/src/components/ClickStackOnboarding/DeeplinkInstall.tsx new file mode 100644 index 0000000000..ea742f7801 --- /dev/null +++ b/packages/app/src/components/ClickStackOnboarding/DeeplinkInstall.tsx @@ -0,0 +1,55 @@ +import { ReactNode, useState } from 'react'; +import { Anchor, Button, Collapse, Group, Stack, Tooltip } from '@mantine/core'; + +import { CopySnippet } from './CopySnippet'; + +interface DeeplinkInstallProps { + buttonLabel: string; + deeplink: string; + fallbackLabel: string; + fallbackSnippet: string; + note?: ReactNode; +} + +/** + * One-click "Add to " deep-link install for hosts that + * support it (Cursor, VS Code + Copilot). The manual fallback + * snippet stays tucked behind a `Manual setup` toggle so the + * primary affordance is always the deep link, with the JSON + * paste-it-yourself path available for users who can't or won't + * use the deep link. + */ +export function DeeplinkInstall({ + buttonLabel, + deeplink, + fallbackLabel, + fallbackSnippet, + note, +}: DeeplinkInstallProps) { + const [manualOpen, setManualOpen] = useState(false); + return ( + + + + + + setManualOpen(v => !v)} + > + {manualOpen ? 'Hide manual setup' : 'Manual setup'} + + + {note} + + + + + ); +} diff --git a/packages/app/src/components/ClickStackOnboarding/McpInstallPanel.tsx b/packages/app/src/components/ClickStackOnboarding/McpInstallPanel.tsx new file mode 100644 index 0000000000..25c6bda9f9 --- /dev/null +++ b/packages/app/src/components/ClickStackOnboarding/McpInstallPanel.tsx @@ -0,0 +1,209 @@ +import { useMemo, useState } from 'react'; +import { Group, SegmentedControl, Stack, Text } from '@mantine/core'; +import { + IconBraces, + IconBrandOpenai, + IconBrandVisualStudio, + IconCode, + IconRobot, + IconTerminal2, +} from '@tabler/icons-react'; + +import { CopySnippet } from './CopySnippet'; +import { DeeplinkInstall } from './DeeplinkInstall'; +import { + buildAllSnippets, + type BuiltSnippets, + type DeploymentShape, +} from './installSnippets'; + +/** + * Agent hosts the install panel covers. Five named installs (Claude + * Code, Cursor, VS Code, Codex CLI, OpenCode) plus "Other" as the + * JSON-fallback escape hatch for any other MCP-compatible host + * (Claude Desktop, Continue, Cline, ...). + * + * ChatGPT is intentionally absent: native MCP isn't there yet, and + * bridges are a user-side decision better tracked in the docs than + * in this UI surface. + */ +type AgentHost = + | 'claude-code' + | 'cursor' + | 'vscode-copilot' + | 'codex-cli' + | 'opencode' + | 'other'; + +interface HostChoice { + id: AgentHost; + label: string; +} + +const CHOICES: HostChoice[] = [ + { id: 'claude-code', label: 'Claude Code' }, + { id: 'cursor', label: 'Cursor' }, + { id: 'vscode-copilot', label: 'VS Code' }, + { id: 'codex-cli', label: 'Codex CLI' }, + { id: 'opencode', label: 'OpenCode' }, + { id: 'other', label: 'Other' }, +]; + +const HOST_IDS = new Set(CHOICES.map(c => c.id)); + +function isAgentHost(value: string): value is AgentHost { + return HOST_IDS.has(value); +} + +interface McpInstallPanelProps { + /** + * Deployment shape derived from `useMe()` in the caller. The + * caller is responsible for not mounting this panel until the + * deployment is ready (matching the convention in + * `ApiKeysSection`), so the type here is non-nullable. + */ + deployment: DeploymentShape; +} + +/** + * Renders the host picker plus the install primitive (CLI command, + * deep link, or JSON block) for the chosen host. Presentational; + * the deployment shape comes in via props so the same component + * can render from any surface that resolves a deployment + access + * key. + * + * The access key is inlined in the rendered snippet to match the + * existing API Keys card pattern, which shows the key in plain + * text. A follow-up will introduce a shared mask + reveal-to-copy + * affordance across every credential surface in Team Settings. + */ +export default function McpInstallPanel({ deployment }: McpInstallPanelProps) { + const [host, setHost] = useState('claude-code'); + + const snippets = useMemo(() => buildAllSnippets(deployment), [deployment]); + + return ( + + { + // Narrow the SegmentedControl's `string` callback against + // the CHOICES set so a future out-of-band value cannot + // silently install an invalid host. CHOICES is the source + // of truth for the option list. + if (isAgentHost(value)) { + setHost(value); + } + }} + data={CHOICES.map(c => ({ + value: c.id, + label: ( + + + {c.label} + + ), + }))} + aria-label="MCP host" + /> + + + + ); +} + +function HostIcon({ id }: { id: AgentHost }) { + switch (id) { + case 'claude-code': + return ; + case 'cursor': + return ; + case 'vscode-copilot': + return ; + case 'codex-cli': + return ; + case 'opencode': + return ; + case 'other': + return ; + } + // Exhaustiveness check via `satisfies never`: adding a new + // AgentHost variant without extending the switch fails the + // compile here. Defensive `return null` (instead of throwing) + // keeps a runtime-only unknown variant from crashing the panel. + id satisfies never; + return null; +} + +interface HostInstallProps { + host: AgentHost; + snippets: BuiltSnippets; +} + +function HostInstall({ host, snippets }: HostInstallProps) { + switch (host) { + case 'claude-code': + return ( + + ); + + case 'cursor': + return ( + + ); + + case 'vscode-copilot': + return ( + + Requires VS Code 1.99+ with the Copilot Chat MCP feature enabled. + + } + /> + ); + + case 'codex-cli': + return ( + + ); + + case 'opencode': + return ( + + ); + + case 'other': + return ( + + ); + } + // Exhaustiveness check via `satisfies never`: adding a new + // AgentHost variant without extending the switch fails the + // compile here. Defensive `return null` (instead of throwing) + // keeps a runtime-only unknown variant from crashing the panel. + host satisfies never; + return null; +} diff --git a/packages/app/src/components/ClickStackOnboarding/__tests__/installSnippets.test.ts b/packages/app/src/components/ClickStackOnboarding/__tests__/installSnippets.test.ts new file mode 100644 index 0000000000..4636579c9f --- /dev/null +++ b/packages/app/src/components/ClickStackOnboarding/__tests__/installSnippets.test.ts @@ -0,0 +1,197 @@ +import { + buildAllSnippets, + type DeploymentShape, + SERVER_NAME, +} from '../installSnippets'; + +const DEPLOYMENT: DeploymentShape = { + apiUrl: 'https://hyperdx.example.com/api', + accessKey: 'k_abcdef123456', +}; + +describe('buildAllSnippets > Claude Code', () => { + it('emits the documented Claude Code MCP install one-liner', () => { + const { claudeCode } = buildAllSnippets(DEPLOYMENT); + + expect(claudeCode).toBe( + `claude mcp add ${SERVER_NAME} --transport http https://hyperdx.example.com/api/mcp --header "Authorization: Bearer k_abcdef123456"`, + ); + }); +}); + +describe('buildAllSnippets > Codex CLI', () => { + it('emits the OpenAI Codex CLI mcp add command', () => { + const { codexCli } = buildAllSnippets(DEPLOYMENT); + + expect(codexCli).toBe( + `codex mcp add ${SERVER_NAME} --transport http https://hyperdx.example.com/api/mcp --header "Authorization: Bearer k_abcdef123456"`, + ); + }); +}); + +describe('buildAllSnippets > Cursor', () => { + it('emits a cursor:// URL with a URL-safe base64-encoded config that round-trips', () => { + const { cursor } = buildAllSnippets(DEPLOYMENT); + + expect( + cursor.startsWith( + `cursor://anysphere.cursor-deeplink/mcp/install?name=${SERVER_NAME}&config=`, + ), + ).toBe(true); + + const encoded = cursor.split('config=')[1]; + // base64url accepts both the URL-safe alphabet (`-`, `_`, no + // padding) and standard base64; using it explicitly is the + // documented decoder for the URL-safe variant we emit. + const decoded = JSON.parse( + Buffer.from(encoded, 'base64url').toString('utf8'), + ); + expect(decoded).toMatchObject({ + type: 'http', + url: 'https://hyperdx.example.com/api/mcp', + headers: { Authorization: 'Bearer k_abcdef123456' }, + }); + }); + + it('produces a Cursor config value that only uses the URL-safe alphabet', () => { + // The standard base64 characters `+` / `/` / `=` all carry + // special meaning inside a query-string value (`+` decodes as + // space under form-urlencoded), so the deep link must use the + // URL-safe variant. This guards every input that flows through + // `buildAllSnippets`, not just the canonical fixture. + const inputs: DeploymentShape[] = [ + DEPLOYMENT, + // Inputs chosen to maximise the chance that the standard + // base64 alphabet would emit `+` or `/`. Every byte > 0x3E or + // > 0x3F flips a `+` or `/` somewhere in the encoded output. + { + apiUrl: 'https://hyperdx.example.com/api', + accessKey: '????>>>>????>>>>', + }, + { + apiUrl: 'https://hyperdx.example.com/api', + accessKey: 'ÿÿÿÿÿÿ', + }, + ]; + for (const deployment of inputs) { + const { cursor } = buildAllSnippets(deployment); + const encoded = cursor.split('config=')[1]; + expect(encoded).toMatch(/^[-A-Za-z0-9_]+$/); + } + }); +}); + +describe('buildAllSnippets > VS Code', () => { + it('emits a vscode:mcp/install URL with a URL-encoded JSON config that round-trips', () => { + const { vscode } = buildAllSnippets(DEPLOYMENT); + + expect(vscode.startsWith('vscode:mcp/install?')).toBe(true); + + const encoded = vscode.replace(/^vscode:mcp\/install\?/, ''); + const decoded = JSON.parse(decodeURIComponent(encoded)); + expect(decoded).toMatchObject({ + name: SERVER_NAME, + type: 'http', + url: 'https://hyperdx.example.com/api/mcp', + headers: { Authorization: 'Bearer k_abcdef123456' }, + }); + }); +}); + +describe('buildAllSnippets > OpenCode', () => { + it('emits OpenCode JSON config under an `mcp` block with type: "remote"', () => { + const { openCode } = buildAllSnippets(DEPLOYMENT); + const parsed = JSON.parse(openCode); + + // OpenCode reads MCP servers from the `mcp` key (not + // `mcpServers` like the canonical block) and uses + // `type: "remote"` for HTTP transport. Verified empirically + // against a running ClickStack instance 2026-06-04. + expect(parsed).toMatchObject({ + mcp: { + [SERVER_NAME]: { + type: 'remote', + url: 'https://hyperdx.example.com/api/mcp', + headers: { Authorization: 'Bearer k_abcdef123456' }, + }, + }, + }); + }); + + it('does NOT emit type: "http" or an `mcpServers` key (those would be the wrong shape for OpenCode)', () => { + const { openCode } = buildAllSnippets(DEPLOYMENT); + + expect(openCode).not.toContain('"type": "http"'); + expect(openCode).not.toContain('"mcpServers"'); + }); +}); + +describe('buildAllSnippets > JSON block', () => { + it('emits canonical mcpServers JSON keyed on the fixed server name', () => { + const { jsonBlock } = buildAllSnippets(DEPLOYMENT); + const parsed = JSON.parse(jsonBlock); + + expect(parsed).toMatchObject({ + mcpServers: { + [SERVER_NAME]: { + url: 'https://hyperdx.example.com/api/mcp', + type: 'http', + headers: { Authorization: 'Bearer k_abcdef123456' }, + }, + }, + }); + }); + + it('renders pretty-printed JSON with two-space indent', () => { + const { jsonBlock } = buildAllSnippets(DEPLOYMENT); + + expect(jsonBlock).toContain('\n "mcpServers": {'); + }); +}); + +describe('buildAllSnippets > placeholder fallbacks', () => { + it('falls back to in the snippet when the key is empty', () => { + const { claudeCode } = buildAllSnippets({ ...DEPLOYMENT, accessKey: '' }); + + expect(claudeCode).toContain('Bearer '); + }); + + it('escapes shell metacharacters in header values', () => { + // Today's access keys are UUIDv4 with no metacharacters; the + // escape is defensive against future formats that allow `"`, + // `$`, `\`, or backtick. Any of those would otherwise turn a + // copy-paste install into a shell-injection vector. + const { claudeCode } = buildAllSnippets({ + ...DEPLOYMENT, + accessKey: 'k"$`\\suffix', + }); + + expect(claudeCode).toContain('Bearer k\\"\\$\\`\\\\suffix"'); + }); +}); + +describe('buildAllSnippets > host coverage', () => { + it('returns a populated string for every supported host', () => { + const all = buildAllSnippets(DEPLOYMENT); + + expect(all).toMatchObject({ + claudeCode: expect.stringContaining(`claude mcp add ${SERVER_NAME}`), + cursor: expect.stringContaining('cursor://'), + vscode: expect.stringContaining('vscode:mcp/install'), + codexCli: expect.stringContaining(`codex mcp add ${SERVER_NAME}`), + openCode: expect.stringContaining(`"${SERVER_NAME}"`), + jsonBlock: expect.stringContaining(`"${SERVER_NAME}"`), + }); + }); + + it('keys the JSON block on the same fixed server name in every output', () => { + const all = buildAllSnippets(DEPLOYMENT); + + expect(all.claudeCode).toContain(SERVER_NAME); + expect(all.codexCli).toContain(SERVER_NAME); + expect(all.cursor).toContain(`name=${SERVER_NAME}`); + expect(all.vscode).toContain(SERVER_NAME); + expect(all.openCode).toContain(`"${SERVER_NAME}"`); + expect(all.jsonBlock).toContain(`"${SERVER_NAME}"`); + }); +}); diff --git a/packages/app/src/components/ClickStackOnboarding/installSnippets.ts b/packages/app/src/components/ClickStackOnboarding/installSnippets.ts new file mode 100644 index 0000000000..61689516c6 --- /dev/null +++ b/packages/app/src/components/ClickStackOnboarding/installSnippets.ts @@ -0,0 +1,231 @@ +// Per-host snippet builders for the "Connect your AI assistant" +// section. Each builder takes a `DeploymentShape` (URL + access +// key) and returns the install primitive for one host: a CLI +// one-liner, a deep link, or a JSON config block. +// +// Pure functions only. Component-level state (host picker) lives +// in `McpInstallPanel.tsx`. Keeping the builders here makes unit +// tests cheap and keeps the snippet round-trip stable regardless +// of which surface renders them. + +/** + * MCP server name registered in the host's config. A single fixed + * value works because the access key carries the team context to + * the MCP server: every install snippet reaches the same + * `/api/mcp` endpoint and the server resolves the active team + * from the bearer token. A user with multiple ClickStack tenants + * authenticates as a different identity per host config and the + * server routes accordingly, so we don't need to disambiguate by + * name on the client. + */ +export const SERVER_NAME = 'clickstack'; + +export interface DeploymentShape { + /** Origin used to build the MCP URL, e.g. `https://example.com/api`. */ + apiUrl: string; + /** Per-user access key from `useMe().accessKey`. */ + accessKey: string; +} + +export interface BuiltSnippets { + /** Claude Code CLI one-liner. */ + claudeCode: string; + /** Cursor `cursor://` deep link. */ + cursor: string; + /** VS Code + Copilot `vscode:mcp/install` deep link. */ + vscode: string; + /** OpenAI Codex CLI one-liner. */ + codexCli: string; + /** OpenCode JSON config under an `mcp` block (uses `type: "remote"`). */ + openCode: string; + /** Canonical `mcpServers` JSON block for any other host. */ + jsonBlock: string; +} + +/** + * Encodes a config object the way the Cursor MCP deep link expects: + * `cursor://anysphere.cursor-deeplink/mcp/install?name=...&config=`. + * Runs in both the browser (`btoa`) and Node (`Buffer`) so the + * snippet builders are usable from Jest tests without a JSDOM + * polyfill. + * + * UTF-8-encodes the input before `btoa` so a future access-key or + * origin format that includes a code point > 0xFF cannot raise + * `InvalidCharacterError` and blank the whole panel (since + * `buildAllSnippets` builds every host eagerly). Today's inputs + * (UUIDv4 key + browser-normalised origin) are all Latin1 so the + * defensive path is unreachable, but the encode is cheap. + */ +function base64(value: string): string { + if (typeof window !== 'undefined' && typeof window.btoa === 'function') { + const utf8 = new TextEncoder().encode(value); + let binary = ''; + for (let i = 0; i < utf8.length; i++) { + binary += String.fromCharCode(utf8[i]); + } + return window.btoa(binary); + } + return Buffer.from(value, 'utf8').toString('base64'); +} + +/** + * URL-safe base64. Standard base64 emits `+`, `/`, `=`, all of + * which carry special meaning in a query-string value (`+` decodes + * as space on the deep-link host side, breaking the JSON payload). + * Cursor's deep-link decoder accepts the URL-safe alphabet; emitting + * it directly avoids a separate `encodeURIComponent` round on the + * already-encoded value. + */ +function base64UrlSafe(value: string): string { + return base64(value) + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, ''); +} + +/** + * Returns the headers map for the MCP HTTP transport. The + * deployment shape exposed by `useMe()` carries only the bearer + * access key, so `Authorization` is the only header emitted here. + */ +function buildHeaders(deployment: DeploymentShape): Record { + return { + Authorization: `Bearer ${deployment.accessKey || ''}`, + }; +} + +/** + * Returns the MCP URL the host connects to. `apiUrl` is the API + * origin derived from the active page; the MCP transport lives at + * `/mcp` underneath it. + */ +function buildUrl(deployment: DeploymentShape): string { + const base = deployment.apiUrl.endsWith('/') + ? deployment.apiUrl.slice(0, -1) + : deployment.apiUrl; + return `${base}/mcp`; +} + +/** + * Quote and escape a header value for safe inclusion inside a + * double-quoted shell argument. Defensive: today's `accessKey` is + * a UUIDv4 with no shell metacharacters, but a future format that + * permits `"`, `$`, `\`, or backtick would otherwise turn a + * copy-paste install into a shell-injection vector. + */ +function shellQuoteHeader(name: string, value: string): string { + const escaped = value.replace(/(["\\$`])/g, '\\$1'); + return `--header "${name}: ${escaped}"`; +} + +function headerArgs(headers: Record): string { + return Object.entries(headers) + .map(([key, value]) => shellQuoteHeader(key, value)) + .join(' '); +} + +/** + * Shared one-liner builder for CLI hosts whose `mcp add` primitive + * matches Claude Code's documented shape: ` mcp add + * --transport http --header "..."`. Codex CLI mirrors this + * verbatim (see https://developers.openai.com/codex/mcp), so the + * two hosts share the same generator and differ only in the binary + * name. + */ +function buildCliOneLiner(binary: string, deployment: DeploymentShape): string { + const url = buildUrl(deployment); + const headers = buildHeaders(deployment); + return `${binary} mcp add ${SERVER_NAME} --transport http ${url} ${headerArgs(headers)}`; +} + +/** + * Cursor `cursor://` deep link. Documented Cursor MCP install + * scheme: name in the query string, config as base64-encoded JSON. + * + * Uses URL-safe base64 (`-`, `_`, no padding) for the `config` value + * so the standard alphabet's `+` / `/` / `=` cannot be re-interpreted + * by the deep-link host's URL parser. Notably, `+` decodes as space + * under `application/x-www-form-urlencoded`, which corrupts the + * embedded JSON. + */ +function buildCursorDeeplink(deployment: DeploymentShape): string { + const config = { + type: 'http', + url: buildUrl(deployment), + headers: buildHeaders(deployment), + }; + const encoded = base64UrlSafe(JSON.stringify(config)); + return `cursor://anysphere.cursor-deeplink/mcp/install?name=${SERVER_NAME}&config=${encoded}`; +} + +/** + * VS Code + Copilot `vscode:mcp/install` deep link. Requires VS + * Code 1.99+ with the Copilot Chat MCP feature enabled. + */ +function buildVSCodeDeeplink(deployment: DeploymentShape): string { + const config = { + name: SERVER_NAME, + type: 'http', + url: buildUrl(deployment), + headers: buildHeaders(deployment), + }; + return `vscode:mcp/install?${encodeURIComponent(JSON.stringify(config))}`; +} + +/** + * OpenCode JSON config block. OpenCode's MCP config lives under an + * `mcp` key (not `mcpServers`) and uses `type: "remote"` for HTTP + * transport (documented at https://opencode.ai/docs/mcp-servers/). + * Verified empirically against a running ClickStack instance on + * 2026-06-04: OpenCode's `type: "remote"` connects successfully to + * our Streamable HTTP server. + * + * Users paste this into `opencode.json` in their project root, or + * into `~/.config/opencode/config.json` for a global install. + */ +function buildOpenCodeJsonBlock(deployment: DeploymentShape): string { + const block = { + mcp: { + [SERVER_NAME]: { + type: 'remote', + url: buildUrl(deployment), + headers: buildHeaders(deployment), + }, + }, + }; + return JSON.stringify(block, null, 2); +} + +/** + * Canonical `mcpServers` JSON block. Covers every MCP-compatible + * host that doesn't have a CLI primitive or deep link yet (Claude + * Desktop, Continue, Cline, and the long tail). + */ +function buildMcpJsonBlock(deployment: DeploymentShape): string { + const block = { + mcpServers: { + [SERVER_NAME]: { + url: buildUrl(deployment), + type: 'http', + headers: buildHeaders(deployment), + }, + }, + }; + return JSON.stringify(block, null, 2); +} + +/** + * Build every host's snippet in one call. The component pulls the + * field matching the current host out of the result; tests assert + * round-trip shape through this entry point. + */ +export function buildAllSnippets(deployment: DeploymentShape): BuiltSnippets { + return { + claudeCode: buildCliOneLiner('claude', deployment), + cursor: buildCursorDeeplink(deployment), + vscode: buildVSCodeDeeplink(deployment), + codexCli: buildCliOneLiner('codex', deployment), + openCode: buildOpenCodeJsonBlock(deployment), + jsonBlock: buildMcpJsonBlock(deployment), + }; +} diff --git a/packages/app/src/components/TeamSettings/McpServerSection.tsx b/packages/app/src/components/TeamSettings/McpServerSection.tsx new file mode 100644 index 0000000000..a100d19ac0 --- /dev/null +++ b/packages/app/src/components/TeamSettings/McpServerSection.tsx @@ -0,0 +1,55 @@ +import { useMemo } from 'react'; +import { Box, Card, Divider, Text } from '@mantine/core'; + +import api from '@/api'; + +import { type DeploymentShape } from '../ClickStackOnboarding/installSnippets'; +import McpInstallPanel from '../ClickStackOnboarding/McpInstallPanel'; + +/** + * Renders the "Connect your AI Agents" section on the Team + * Settings page (Integrations tab). Self-managed OSS deployments + * mount the MCP server at `/api/mcp` with the per-user + * access key as bearer; the MCP server resolves the active team + * from the token, so the install snippet doesn't need to encode + * tenant context on the client. + * + * Structure mirrors `IntegrationsSection.tsx` and + * `ApiKeysSection.tsx`: outer `` with a `` + * header and ``, then a single `` wrapping the + * install panel. No subtitle below the header so the visual + * rhythm matches the rest of the Integrations tab. + * + * Auth gating matches `ApiKeysSection`: render nothing until `me` + * resolves with a non-empty access key. `MeApiResponseSchema` + * declares `accessKey` as `z.string()`, and the User model + * generates one on account creation, so the empty-key branch is + * defensive against a state the schema disallows; we still skip + * mounting the panel in that case rather than rendering a snippet + * with an empty bearer. + */ +export default function McpServerSection() { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + + const deployment = useMemo(() => { + if (!me?.accessKey) return null; + return { + apiUrl: `${window.location.origin}/api`, + accessKey: me.accessKey, + }; + }, [me]); + + if (isLoadingMe || !deployment) { + return null; + } + + return ( + + Connect your AI Agents + + + + + + ); +} diff --git a/packages/app/src/components/TeamSettings/__tests__/McpServerSection.test.tsx b/packages/app/src/components/TeamSettings/__tests__/McpServerSection.test.tsx new file mode 100644 index 0000000000..ab3d636bde --- /dev/null +++ b/packages/app/src/components/TeamSettings/__tests__/McpServerSection.test.tsx @@ -0,0 +1,190 @@ +import { MantineProvider } from '@mantine/core'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import api from '@/api'; + +import McpServerSection from '../McpServerSection'; + +jest.mock('@/api', () => ({ + __esModule: true, + default: { + useMe: jest.fn(), + }, + hdxServer: jest.fn(), +})); + +const mockUseMe = jest.mocked(api.useMe); + +function setMe(accessKey: string | null, isLoading = false) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + mockUseMe.mockReturnValue({ + data: + accessKey === null + ? null + : { + id: 'u1', + email: 'a@b.com', + accessKey, + name: 'User', + createdAt: '', + }, + isLoading, + } as ReturnType); +} + +function renderSection() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + + , + ); +} + +describe('McpServerSection', () => { + beforeEach(() => { + jest.clearAllMocks(); + setMe('k_test'); + }); + + it('renders the section header and the install panel', () => { + renderSection(); + + expect(screen.getByTestId('mcp-server-section')).toBeInTheDocument(); + expect(screen.getByText(/^Connect your AI Agents$/)).toBeInTheDocument(); + }); + + it('renders nothing while the me payload is still loading', () => { + setMe(null, true); + + renderSection(); + + expect(screen.queryByTestId('mcp-server-section')).not.toBeInTheDocument(); + }); + + it('renders all six host options when the deployment shape is valid', () => { + renderSection(); + + expect(screen.getByText('Claude Code')).toBeInTheDocument(); + expect(screen.getByText('Cursor')).toBeInTheDocument(); + expect(screen.getByText('VS Code')).toBeInTheDocument(); + expect(screen.getByText('Codex CLI')).toBeInTheDocument(); + expect(screen.getByText('OpenCode')).toBeInTheDocument(); + expect(screen.getByText('Other')).toBeInTheDocument(); + }); + + // Auth-gate parity with `ApiKeysSection`: when `me` resolves to + // `null` (user not signed in, or `IS_LOCAL_MODE` short-circuit), + // the section silently doesn't render. No "sign in" alert. + it('renders nothing when me is null', () => { + setMe(null); + + renderSection(); + + expect(screen.queryByTestId('mcp-server-section')).not.toBeInTheDocument(); + }); + + it('renders the fixed clickstack server name in the install snippet', () => { + renderSection(); + + expect(screen.getByText(/claude mcp add clickstack /)).toBeInTheDocument(); + }); + + // `MeApiResponseSchema.accessKey` is `z.string()` non-nullable + // and the User model generates one at account creation, so an + // empty access key isn't reachable from the API in practice. + // The defensive guard in `McpServerSection` still skips render + // rather than emitting a snippet with an empty bearer. + it('renders nothing when accessKey is empty', () => { + setMe(''); + + renderSection(); + + expect(screen.queryByTestId('mcp-server-section')).not.toBeInTheDocument(); + }); + + it('switches to the Codex CLI snippet when the host picker selects Codex CLI', async () => { + const user = userEvent.setup(); + renderSection(); + + await user.click(screen.getByText('Codex CLI')); + + expect(screen.getByText(/codex mcp add clickstack /)).toBeInTheDocument(); + expect( + screen.queryByText(/^claude mcp add clickstack /), + ).not.toBeInTheDocument(); + }); + + it('renders the Cursor deeplink button with a cursor:// href when Cursor is selected', async () => { + const user = userEvent.setup(); + renderSection(); + + await user.click(screen.getByText('Cursor')); + + const button = screen.getByRole('link', { name: /Add to Cursor/i }); + const href = button.getAttribute('href') ?? ''; + expect(href).toMatch( + /^cursor:\/\/anysphere\.cursor-deeplink\/mcp\/install\?name=clickstack&config=[-A-Za-z0-9_]+$/, + ); + }); + + it('renders the VS Code deeplink button with a vscode:mcp/install href when VS Code is selected', async () => { + const user = userEvent.setup(); + renderSection(); + + await user.click(screen.getByText('VS Code')); + + const button = screen.getByRole('link', { name: /Add to VS Code/i }); + expect(button.getAttribute('href') ?? '').toMatch(/^vscode:mcp\/install\?/); + }); + + it('renders the canonical JSON block when Other is selected', async () => { + const user = userEvent.setup(); + renderSection(); + + await user.click(screen.getByText('Other')); + + expect(screen.getByText(/"mcpServers":/)).toBeInTheDocument(); + }); + + it('renders the OpenCode JSON block (`mcp` key with `type: "remote"`) when OpenCode is selected', async () => { + const user = userEvent.setup(); + renderSection(); + + await user.click(screen.getByText('OpenCode')); + + // OpenCode's shape diverges from `Other`: outer key is `mcp` + // (not `mcpServers`) and the server entry uses `type: "remote"` + // (not `type: "http"`). + expect(screen.getByText(/"mcp":/)).toBeInTheDocument(); + expect(screen.getByText(/"type": "remote"/)).toBeInTheDocument(); + }); + + it('reveals the manual JSON fallback when the Manual setup toggle is clicked on a deeplink host', async () => { + const user = userEvent.setup(); + renderSection(); + + await user.click(screen.getByText('Cursor')); + // The fallback JSON lives inside Mantine's ``, which + // keeps the child mounted and animates max-height + visibility + // for the open transition. JSDOM does not run CSS transitions, + // so the canonical "is the user seeing it" canary is the toggle + // label: it flips synchronously with React state. + expect(screen.getByText(/^Manual setup$/i)).toBeInTheDocument(); + expect(screen.queryByText(/^Hide manual setup$/i)).not.toBeInTheDocument(); + expect(screen.getByText(/"mcpServers":/)).not.toBeVisible(); + + await user.click(screen.getByText(/^Manual setup$/i)); + expect(screen.getByText(/^Hide manual setup$/i)).toBeInTheDocument(); + expect(screen.queryByText(/^Manual setup$/i)).not.toBeInTheDocument(); + + await user.click(screen.getByText(/^Hide manual setup$/i)); + expect(screen.getByText(/^Manual setup$/i)).toBeInTheDocument(); + }); +}); diff --git a/packages/app/tests/e2e/features/team.spec.ts b/packages/app/tests/e2e/features/team.spec.ts index 2d4ed69262..e7dc837e52 100644 --- a/packages/app/tests/e2e/features/team.spec.ts +++ b/packages/app/tests/e2e/features/team.spec.ts @@ -20,6 +20,7 @@ test.describe('Team Settings Page', { tag: ['@team', '@full-stack'] }, () => { await test.step('Verify team settings tabs are visible', async () => { await expect(teamPage.dataTab).toBeVisible(); await expect(teamPage.teamTab).toBeVisible(); + await expect(teamPage.apiAndAgentsTab).toBeVisible(); await expect(teamPage.integrationsTab).toBeVisible(); await expect(teamPage.advancedTab).toBeVisible(); }); @@ -63,19 +64,30 @@ test.describe('Team Settings Page', { tag: ['@team', '@full-stack'] }, () => { } }); + await test.step('Verify API & Agents tab sections are visible', async () => { + await teamPage.openApiAndAgentsTab(); + await expect(teamPage.apiKeys).toBeVisible(); + await expect(teamPage.mcpServer).toBeVisible(); + }); + + await test.step('Verify API & Agents tab headings exist', async () => { + await expect( + teamPage.apiKeys.getByText('API Keys', { exact: true }), + ).toBeVisible(); + await expect( + teamPage.mcpServer.getByText('Connect your AI Agents', { exact: true }), + ).toBeVisible(); + }); + await test.step('Verify integrations tab sections are visible', async () => { await teamPage.openIntegrationsTab(); await expect(teamPage.integrations).toBeVisible(); - await expect(teamPage.apiKeys).toBeVisible(); }); - await test.step('Verify integrations tab headings exist', async () => { + await test.step('Verify integrations tab heading exists', async () => { await expect( teamPage.integrations.getByText('Integrations', { exact: true }), ).toBeVisible(); - await expect( - teamPage.apiKeys.getByText('API Keys', { exact: true }), - ).toBeVisible(); }); await test.step('Verify advanced tab content is visible', async () => { @@ -142,7 +154,7 @@ test.describe('Team Settings Page', { tag: ['@team', '@full-stack'] }, () => { test('should display API keys', async () => { await test.step('Scroll to API keys section', async () => { - await teamPage.openIntegrationsTab(); + await teamPage.openApiAndAgentsTab(); await teamPage.apiKeys.scrollIntoViewIfNeeded(); }); @@ -162,7 +174,7 @@ test.describe('Team Settings Page', { tag: ['@team', '@full-stack'] }, () => { test('should open and cancel rotate API key modal', async () => { await test.step('Open rotate API key modal', async () => { - await teamPage.openIntegrationsTab(); + await teamPage.openApiAndAgentsTab(); await teamPage.clickRotateApiKey(); }); diff --git a/packages/app/tests/e2e/page-objects/TeamPage.ts b/packages/app/tests/e2e/page-objects/TeamPage.ts index df418ba6e6..07acbf0ef3 100644 --- a/packages/app/tests/e2e/page-objects/TeamPage.ts +++ b/packages/app/tests/e2e/page-objects/TeamPage.ts @@ -13,6 +13,7 @@ export class TeamPage { private readonly dataTabButton: Locator; private readonly teamTabButton: Locator; private readonly accessTabButton: Locator; + private readonly apiAndAgentsTabButton: Locator; private readonly integrationsTabButton: Locator; private readonly advancedTabButton: Locator; @@ -22,6 +23,7 @@ export class TeamPage { private readonly integrationsSection: Locator; private readonly teamNameSection: Locator; private readonly apiKeysSection: Locator; + private readonly mcpServerSection: Locator; private readonly teamMembersSection: Locator; private readonly securityPoliciesHeading: Locator; private readonly querySettingsHeading: Locator; @@ -65,6 +67,10 @@ export class TeamPage { name: 'Access', exact: true, }); + this.apiAndAgentsTabButton = page.getByRole('tab', { + name: 'API & Agents', + exact: true, + }); this.integrationsTabButton = page.getByRole('tab', { name: 'Integrations', exact: true, @@ -79,6 +85,7 @@ export class TeamPage { this.integrationsSection = page.getByTestId('integrations-section'); this.teamNameSection = page.getByTestId('team-name-section'); this.apiKeysSection = page.getByTestId('api-keys-section'); + this.mcpServerSection = page.getByTestId('mcp-server-section'); this.teamMembersSection = page.getByTestId('team-members-section'); this.securityPoliciesHeading = page.getByText('Security Policies', { exact: true, @@ -128,6 +135,10 @@ export class TeamPage { await this.openTab(this.teamTabButton, this.teamMembersSection); } + async openApiAndAgentsTab() { + await this.openTab(this.apiAndAgentsTabButton, this.apiKeysSection); + } + async openIntegrationsTab() { await this.openTab(this.integrationsTabButton, this.integrationsSection); } @@ -329,6 +340,10 @@ export class TeamPage { return this.accessTabButton; } + get apiAndAgentsTab() { + return this.apiAndAgentsTabButton; + } + get integrationsTab() { return this.integrationsTabButton; } @@ -373,6 +388,10 @@ export class TeamPage { return this.apiKeysSection; } + get mcpServer() { + return this.mcpServerSection; + } + get rotateButton() { return this.rotateApiKeyButton; } From 3f21e2e208a8ca4bd9372405581e2f2838ae3aea Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:13:18 -0700 Subject: [PATCH 030/728] [HDX-4406] Number tile conditional color rules (ordered thresholds) (#2386) ## Summary This PR adds conditional color rules to number tiles. Define an ordered list of conditions; the last matching rule's color wins, so list rules in ascending priority order with the highest-priority rule last. If no rule matches, the tile falls back to its static color (set previously in #2265), then to the default text color. The schema is intentionally generic at the shared level so a future table-tile slice can attach per-column rules without a schema change. > **Stacking note:** the branch was cut from `alex/HDX-4405-fix-stranded-tooltip-in-virtual-table` (PR #2380), so the file list includes 4 unrelated files from that PR: `HDXMultiSeriesTableChart.tsx`, `HDXMultiSeriesTableChart.test.tsx`, `dashboard-table-linking.spec.ts`, `DashboardPage.ts`. Those files belong to #2380 and will drop out of this diff once #2380 merges. The HDX-4406 review surface is the other 10 files. ### What changed **common-utils (schema)** - Added `ColorConditionSchema` (discriminated union of `gt`, `gte`, `lt`, `lte`, `between`, `eq`, `neq`, `contains`, `startsWith`, `endsWith`, `regex` operators) - Added `colorRules` (optional, max 10) to `SharedChartSettingsSchema` alongside the existing `color` field **app (resolver)** - `evaluateColorCondition`: evaluates a single rule against a runtime value with proper type guards (numeric ops reject strings, string ops reject numbers, bad regex returns false) - `resolveConditionalColor`: iterates rules in order, last match wins, falls back to `fallback` (the static color) when nothing matches - String data values (ClickHouse returns `UInt64` counts as JSON strings) are coerced to numbers before rule evaluation **app (UI)** - New `ColorRulesEditor` component: sortable rule list via `@dnd-kit/sortable`, per-operator value inputs (single number, two-number range for `between`, text-or-number for `eq`/`neq`), `ColorSwatchInput` per rule, add/delete buttons; "Add rule" disables at 10 - `ChartDisplaySettingsDrawer`: added "Conditional colors" section gated on `DisplayType.Number`, placed below the existing static color picker - `EditTimeChartForm`: `colorRules` wired through `useWatch`, `displaySettings` memo, and `handleUpdateDisplaySettings` - `DBNumberChart`: resolves color via `resolveConditionalColor(rawValue, config.colorRules, config.color)` at render time **Tests** - Schema: positive + negative cases for all operators and array length constraints - Resolver: `evaluateColorCondition` per-operator, `resolveConditionalColor` including last-match-wins, string coercion, and the canonical success/warning/error scenario - UI: add/delete/operator shape/color swatch/render cases for `ColorRulesEditor` - Integration: `DBNumberChart` with `color: 'chart-success'` + two rules confirms value 50 -> success, 200 -> warning, 1000 -> error; also covers string UInt64 input No changes to `packages/api` schemas or external API (separate follow-up ticket, mirrors HDX-4378). ### Screenshots or video | Before | After | | :----- | :---- | | Number tile has only a static color picker | Number tile shows a "Conditional colors" section with add/reorder/delete rule controls below the static color picker | ### How to test on Vercel preview **Preview routes:** /dashboards **Steps:** 1. Open or create a dashboard and add a number tile (or edit an existing one). 2. Click the "Display Settings" button to open the drawer. 3. Confirm the "Conditional colors" section appears below the "Color" picker. 4. Click "Add rule" and verify a rule row appears with operator >, a value input, a color swatch, and a delete button. 5. Change the operator to >= and set value to 100; pick the Warning color swatch. 6. Click "Add rule" again; set operator >=, value 500; pick the Error color swatch. 7. Click "Apply". Verify the tile persists the rules (re-open Display Settings and confirm the two rules are still there). 8. With a tile value below 100, confirm it renders in the static (or default) color. 9. With a value between 100 and 499, confirm warning color. 10. With a value >= 500, confirm error color. 11. Click "Add rule" 8 more times to reach 10 rules total. Verify the button becomes disabled. 12. Drag a rule handle to reorder and click Apply; confirm the new order is reflected on re-open. ### References - Linear Issue: https://linear.app/clickhouse/issue/HDX-4406 - Related PRs: #2265 (static number-tile color picker, precedent); #2380 (tooltip fix; this branch is stacked on it) --- packages/app/src/HDXMultiSeriesTableChart.tsx | 132 +++--- .../HDXMultiSeriesTableChart.test.tsx | 55 +++ packages/app/src/__tests__/utils.test.ts | 319 +++++++++++++++ .../components/ChartDisplaySettingsDrawer.tsx | 52 ++- .../app/src/components/ColorRulesEditor.tsx | 387 ++++++++++++++++++ .../DBEditTimeChartForm/EditTimeChartForm.tsx | 6 + packages/app/src/components/DBNumberChart.tsx | 58 ++- .../__tests__/ColorRulesEditor.test.tsx | 178 ++++++++ .../__tests__/DBNumberChart.test.tsx | 138 +++++++ packages/app/src/utils.ts | 80 ++++ .../features/dashboard-table-linking.spec.ts | 45 ++ .../tests/e2e/page-objects/DashboardPage.ts | 39 ++ .../common-utils/src/__tests__/types.test.ts | 284 +++++++++++++ packages/common-utils/src/types.ts | 72 ++++ 14 files changed, 1771 insertions(+), 74 deletions(-) create mode 100644 packages/app/src/components/ColorRulesEditor.tsx create mode 100644 packages/app/src/components/__tests__/ColorRulesEditor.test.tsx create mode 100644 packages/common-utils/src/__tests__/types.test.ts diff --git a/packages/app/src/HDXMultiSeriesTableChart.tsx b/packages/app/src/HDXMultiSeriesTableChart.tsx index 78641fe7e9..604e0c3140 100644 --- a/packages/app/src/HDXMultiSeriesTableChart.tsx +++ b/packages/app/src/HDXMultiSeriesTableChart.tsx @@ -307,6 +307,33 @@ export const Table = ({ ); const [wrapLinesEnabled, setWrapLinesEnabled] = useState(false); + // Store the virtual index of the hovered row (not its description string) + // so the label re-derives on every render. If the virtualiser replaces the + // row at that index (scroll re-virtualisation, auto-refetch) the label + // reflects the new row immediately rather than showing stale text. Storing + // the index also makes the safety-net `onMouseLeave` on correct: + // it sets null rather than the prior row's description. See HDX-4405. + const [hoveredVirtualIndex, setHoveredVirtualIndex] = useState( + null, + ); + + // Derive the label from whichever row currently occupies hoveredVirtualIndex. + // Returns null when no row is hovered, the index is out of range, or the + // row's action has no URL (error-toast rows show no hint). + const hoveredRowDescription = useMemo(() => { + if (hoveredVirtualIndex == null) return null; + const virtualRow = items.find(v => v.index === hoveredVirtualIndex); + if (!virtualRow) return null; + const row = rows[virtualRow.index] as TableRow | undefined; + if (!row) return null; + const rowAction = getRowAction ? getRowAction(row.original) : null; + return rowAction?.url != null && rowAction.description + ? rowAction.description + : null; + }, [hoveredVirtualIndex, items, rows, getRowAction]); + + const clearHovered = useCallback(() => setHoveredVirtualIndex(null), []); + const { csvData } = useCsvExport( truncatedData, columns.map(col => ({ @@ -371,65 +398,62 @@ export const Table = ({ ))} - - {paddingTop > 0 && ( - - - - )} - {items.map(virtualRow => { - const row = rows[virtualRow.index] as TableRow; - // Compute the action once per row so the row-level HoverCard - // sees the same description and per-cell renders share the - // memoized result from useOnClickLinkBuilder. - const rowAction = getRowAction ? getRowAction(row.original) : null; - const tr = ( - - {row.getVisibleCells().map(cell => { - return ( - - {flexRender( - cell.column.columnDef.cell, - cell.getContext(), - )} - - ); - })} + {/* Single Tooltip.Floating wrapping the whole so the hint + follows the cursor without being tied to the lifecycle of any + individual virtual row. Per-row Tooltip.Floating instances get + stranded in the Portal when a row unmounts before onMouseLeave + fires (rapid mouse movement in a virtualised list). With this + approach the tooltip state lives on , which never unmounts, + and the label is re-derived from hoveredVirtualIndex each render so + scroll re-virtualisation never shows stale text. See HDX-4405. */} + {hoveredRowDescription} + } + withinPortal + disabled={!hoveredRowDescription} + > + {/* onMouseLeave on is a safety net: if a virtual row + unmounts before its own onMouseLeave fires (rapid cursor + movement or re-virtualisation), leaving the table body still + clears the hovered index. */} + + {paddingTop > 0 && ( + + - ); - // Row-level Tooltip.Floating so the hint follows the cursor - // and anchors near the cell the user is over, not at the row's - // center-top. Tooltip.Floating tracks the cursor via floating-ui - // and stays within the row's bounding box; one tooltip per row - // means no flicker as the cursor moves between cells. - // - // The hint is suppressed when rowAction.url === null because - // the click only fires an error toast on those rows, so showing - // "Open in search" would mislead the user. - if (rowAction && rowAction.url) { + )} + {items.map(virtualRow => { + const row = rows[virtualRow.index] as TableRow; return ( - setHoveredVirtualIndex(virtualRow.index)} + onMouseLeave={clearHovered} > - {tr} - + {row.getVisibleCells().map(cell => { + return ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + ); + })} + ); - } - return tr; - })} - {paddingBottom > 0 && ( - - - - )} - + })} + {paddingBottom > 0 && ( + + + + )} + + {isTruncated && (
diff --git a/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx b/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx index c63e72941b..8947a731dc 100644 --- a/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx +++ b/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx @@ -102,6 +102,61 @@ describe('HDXMultiSeriesTableChart ', () => { { timeout: 1000 }, ); }); + + it('hides the hint when the hovered virtual index maps to a no-URL row (HDX-4405)', async () => { + // Regression for the virtualiser race: the hovered can unmount + // before its onMouseLeave fires (rapid movement / data refresh). The + // old per-row Tooltip.Floating was stranded in the Portal; the fix + // stores a virtual index and re-derives the label via useMemo each + // render. When the row at that index no longer has a URL, the tooltip + // hides without a leave event from the (now-gone) element. + // + // We verify the key invariant structurally: hovering index 0 (URL row) + // shows the hint; hovering index 1 (no-URL row) hides it — no + // mouseLeave fires between the two enterevents, simulating the + // cursor jumping over the table faster than leave events dispatch. + const multiRowData = [ + { ServiceName: 'web', Count: 10 }, + { ServiceName: 'api', Count: 5 }, + ]; + const getRowAction = jest.fn((row: { ServiceName: string }) => + row.ServiceName === 'web' + ? { url: '/search?source=src_1&where=', description: 'Search Logs' } + : { url: null, description: '', onClickError: jest.fn() }, + ); + + renderWithMantine( +
{}} + />, + ); + + const webRow = screen.getByText('web').closest('tr')!; + const apiRow = screen.getByText('api').closest('tr')!; + + // Hover the URL row — tooltip must appear + fireEvent.mouseEnter(webRow); + await waitFor(() => { + const hint = screen.getByTestId('row-action-hint'); + const tooltipBox = hint.closest('[style*="display"]'); + expect(tooltipBox?.style.display).toBe('block'); + }); + + // Hover the no-URL row WITHOUT firing mouseLeave on the first row. + // This simulates the cursor jumping faster than leave events dispatch. + fireEvent.mouseEnter(apiRow); + + // The label must derive to null (apiRow has url:null) so tooltip hides. + await waitFor(() => { + const hint = screen.getByTestId('row-action-hint'); + const tooltipBox = hint.closest('[style*="display"]'); + expect(tooltipBox?.style.display).toBe('none'); + }); + }); }); describe('getRowAction failure path', () => { diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index 5d8eeb68aa..63b77d78ea 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -1,5 +1,6 @@ import { ChartPaletteTokenSchema, + ColorCondition, NumericUnit, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -10,6 +11,7 @@ import { MetricsDataType, NumberFormat } from '../types'; import * as utils from '../utils'; import { COLORS, + evaluateColorCondition, formatAttributeClause, formatDurationMs, formatDurationMsCompact, @@ -20,6 +22,7 @@ import { mapKeyBy, orderByStringToSortingState, parseTimestampToMs, + resolveConditionalColor, sortingStateToOrderByString, stripTrailingSlash, useQueryHistory, @@ -1268,3 +1271,319 @@ describe('getColorFromCSSToken', () => { expect(() => ChartPaletteTokenSchema.parse('chart-10')).toThrow(); }); }); + +// ─── evaluateColorCondition ─────────────────────────────────────────────────── + +describe('evaluateColorCondition', () => { + describe('numeric ordered operators', () => { + it('gt: returns true when value > rule.value', () => { + const rule: ColorCondition = { + operator: 'gt', + value: 10, + color: 'chart-blue', + }; + expect(evaluateColorCondition(11, rule)).toBe(true); + expect(evaluateColorCondition(10, rule)).toBe(false); + expect(evaluateColorCondition(9, rule)).toBe(false); + }); + + it('gte: returns true when value >= rule.value', () => { + const rule: ColorCondition = { + operator: 'gte', + value: 10, + color: 'chart-blue', + }; + expect(evaluateColorCondition(10, rule)).toBe(true); + expect(evaluateColorCondition(11, rule)).toBe(true); + expect(evaluateColorCondition(9, rule)).toBe(false); + }); + + it('lt: returns true when value < rule.value', () => { + const rule: ColorCondition = { + operator: 'lt', + value: 10, + color: 'chart-blue', + }; + expect(evaluateColorCondition(9, rule)).toBe(true); + expect(evaluateColorCondition(10, rule)).toBe(false); + }); + + it('lte: returns true when value <= rule.value', () => { + const rule: ColorCondition = { + operator: 'lte', + value: 10, + color: 'chart-blue', + }; + expect(evaluateColorCondition(10, rule)).toBe(true); + expect(evaluateColorCondition(9, rule)).toBe(true); + expect(evaluateColorCondition(11, rule)).toBe(false); + }); + + it('numeric operators return false for string values', () => { + const rule: ColorCondition = { + operator: 'gt', + value: 10, + color: 'chart-blue', + }; + expect(evaluateColorCondition('15', rule)).toBe(false); + }); + }); + + describe('between operator', () => { + it('returns true when value is within [lo, hi]', () => { + const rule: ColorCondition = { + operator: 'between', + value: [10, 100], + color: 'chart-blue', + }; + expect(evaluateColorCondition(50, rule)).toBe(true); + expect(evaluateColorCondition(10, rule)).toBe(true); + expect(evaluateColorCondition(100, rule)).toBe(true); + expect(evaluateColorCondition(9, rule)).toBe(false); + expect(evaluateColorCondition(101, rule)).toBe(false); + }); + + it('handles inverted range (first > second) by normalising to [lo, hi]', () => { + const rule: ColorCondition = { + operator: 'between', + value: [100, 10], + color: 'chart-blue', + }; + expect(evaluateColorCondition(50, rule)).toBe(true); + expect(evaluateColorCondition(5, rule)).toBe(false); + }); + + it('returns false for string values', () => { + const rule: ColorCondition = { + operator: 'between', + value: [10, 100], + color: 'chart-blue', + }; + expect(evaluateColorCondition('50', rule)).toBe(false); + }); + }); + + describe('eq / neq operators', () => { + it('eq: returns true on strict equality (number)', () => { + const rule: ColorCondition = { + operator: 'eq', + value: 5, + color: 'chart-blue', + }; + expect(evaluateColorCondition(5, rule)).toBe(true); + expect(evaluateColorCondition(6, rule)).toBe(false); + }); + + it('eq: returns true on strict equality (string)', () => { + const rule: ColorCondition = { + operator: 'eq', + value: 'CRIT', + color: 'chart-blue', + }; + expect(evaluateColorCondition('CRIT', rule)).toBe(true); + expect(evaluateColorCondition('crit', rule)).toBe(false); + }); + + it('eq: cross-type mismatch returns false ("5" vs 5)', () => { + const rule: ColorCondition = { + operator: 'eq', + value: '5', + color: 'chart-blue', + }; + expect(evaluateColorCondition(5, rule)).toBe(false); + }); + + it('neq: returns true when value differs', () => { + const rule: ColorCondition = { + operator: 'neq', + value: 0, + color: 'chart-blue', + }; + expect(evaluateColorCondition(1, rule)).toBe(true); + expect(evaluateColorCondition(0, rule)).toBe(false); + }); + + it('neq: cross-type mismatch returns false (number vs string)', () => { + const rule: ColorCondition = { + operator: 'neq', + value: 'none', + color: 'chart-blue', + }; + // Without the typeof guard, `42 !== 'none'` is true, which would make + // the rule match every numeric value. Guarding keeps the docstring + // contract: cross-type mismatches return false. + expect(evaluateColorCondition(42, rule)).toBe(false); + }); + + it('neq: cross-type mismatch returns false (string vs number)', () => { + const rule: ColorCondition = { + operator: 'neq', + value: 42, + color: 'chart-blue', + }; + expect(evaluateColorCondition('none', rule)).toBe(false); + }); + }); + + describe('string operators', () => { + it('contains: returns true when string includes value', () => { + const rule: ColorCondition = { + operator: 'contains', + value: 'error', + color: 'chart-error', + }; + expect(evaluateColorCondition('fatal error occurred', rule)).toBe(true); + expect(evaluateColorCondition('warning', rule)).toBe(false); + }); + + it('contains: returns false for number values', () => { + const rule: ColorCondition = { + operator: 'contains', + value: 'error', + color: 'chart-error', + }; + expect(evaluateColorCondition(42, rule)).toBe(false); + }); + + it('startsWith: matches prefix', () => { + const rule: ColorCondition = { + operator: 'startsWith', + value: 'ERR', + color: 'chart-error', + }; + expect(evaluateColorCondition('ERR_500', rule)).toBe(true); + expect(evaluateColorCondition('WARN_ERR', rule)).toBe(false); + }); + + it('endsWith: matches suffix', () => { + const rule: ColorCondition = { + operator: 'endsWith', + value: 'CRIT', + color: 'chart-error', + }; + expect(evaluateColorCondition('ALERT_CRIT', rule)).toBe(true); + expect(evaluateColorCondition('CRIT_OK', rule)).toBe(false); + }); + + it('regex: matches valid pattern', () => { + const rule: ColorCondition = { + operator: 'regex', + value: '^err.*', + color: 'chart-error', + }; + expect(evaluateColorCondition('error123', rule)).toBe(true); + expect(evaluateColorCondition('warning', rule)).toBe(false); + }); + + it('regex: bad pattern returns false without throwing', () => { + const rule = { + operator: 'regex' as const, + value: '[invalid', + color: 'chart-error' as const, + }; + expect(() => evaluateColorCondition('test', rule)).not.toThrow(); + expect(evaluateColorCondition('test', rule)).toBe(false); + }); + }); +}); + +// ─── resolveConditionalColor ────────────────────────────────────────────────── + +describe('resolveConditionalColor', () => { + it('returns fallback when rules is undefined', () => { + expect(resolveConditionalColor(50, undefined, 'chart-success')).toBe( + 'chart-success', + ); + }); + + it('returns fallback when rules is empty', () => { + expect(resolveConditionalColor(50, [], 'chart-success')).toBe( + 'chart-success', + ); + }); + + it('returns fallback when value is null', () => { + const rules: ColorCondition[] = [ + { operator: 'gte', value: 0, color: 'chart-warning' }, + ]; + expect(resolveConditionalColor(null, rules, 'chart-success')).toBe( + 'chart-success', + ); + }); + + it('returns fallback when value is undefined', () => { + const rules: ColorCondition[] = [ + { operator: 'gte', value: 0, color: 'chart-warning' }, + ]; + expect(resolveConditionalColor(undefined, rules, 'chart-success')).toBe( + 'chart-success', + ); + }); + + it('returns the matching rule color when one rule matches', () => { + const rules: ColorCondition[] = [ + { operator: 'gte', value: 100, color: 'chart-warning' }, + ]; + expect(resolveConditionalColor(200, rules, 'chart-success')).toBe( + 'chart-warning', + ); + }); + + it('returns the LAST matching rule color (last-match-wins)', () => { + // value 1000: both rules match; last (chart-error) wins + const rules: ColorCondition[] = [ + { operator: 'gte', value: 100, color: 'chart-warning' }, + { operator: 'gte', value: 500, color: 'chart-error' }, + ]; + expect(resolveConditionalColor(1000, rules, 'chart-success')).toBe( + 'chart-error', + ); + }); + + it('returns fallback when no rule matches', () => { + const rules: ColorCondition[] = [ + { operator: 'gte', value: 100, color: 'chart-warning' }, + { operator: 'gte', value: 500, color: 'chart-error' }, + ]; + // value 50: no rule matches, return fallback + expect(resolveConditionalColor(50, rules, 'chart-success')).toBe( + 'chart-success', + ); + }); + + it('covers the DBNumberChart success/warning/error scenario', () => { + const rules: ColorCondition[] = [ + { operator: 'gte', value: 100, color: 'chart-warning' }, + { operator: 'gte', value: 500, color: 'chart-error' }, + ]; + // 50 → no match → static color + expect(resolveConditionalColor(50, rules, 'chart-success')).toBe( + 'chart-success', + ); + // 200 → rule 1 matches, rule 2 doesn't → chart-warning + expect(resolveConditionalColor(200, rules, 'chart-success')).toBe( + 'chart-warning', + ); + // 1000 → both match → last match = chart-error + expect(resolveConditionalColor(1000, rules, 'chart-success')).toBe( + 'chart-error', + ); + }); + + it('string rules do not match numeric values', () => { + const rules: ColorCondition[] = [ + { operator: 'contains', value: 'err', color: 'chart-error' }, + ]; + // numeric value, string rule: no match + expect(resolveConditionalColor(42, rules, 'chart-success')).toBe( + 'chart-success', + ); + }); + + it('returns undefined fallback when fallback is undefined and no rule matches', () => { + const rules: ColorCondition[] = [ + { operator: 'gte', value: 100, color: 'chart-warning' }, + ]; + expect(resolveConditionalColor(50, rules, undefined)).toBeUndefined(); + }); +}); diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index 46621da3e5..04660d7be1 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -20,6 +20,12 @@ import { import { shouldFillNullsWithZero } from '@/ChartUtils'; import { FormatTime } from '@/useFormatTime'; +import { + attachLocalIds, + ColorRulesEditor, + ColorRuleWithId, + stripLocalIds, +} from './ColorRulesEditor'; import { ColorSwatchInput } from './ColorSwatchInput'; import { CheckBoxControlled } from './InputControlled'; import { DEFAULT_NUMBER_FORMAT, NumberFormatForm } from './NumberFormat'; @@ -32,10 +38,19 @@ export type ChartConfigDisplaySettings = Pick< | 'compareToPreviousPeriod' | 'fitYAxisToData' | 'color' + | 'colorRules' > & { groupByColumnsOnLeft?: boolean; }; +/** + * Internal form shape: `colorRules` is stored with `localId`s for dnd-kit + * stability; they are stripped before the settings are passed to `onChange`. + */ +type DrawerFormValues = Omit & { + colorRules?: ColorRuleWithId[]; +}; + interface ChartDisplaySettingsDrawerProps { opened: boolean; settings: ChartConfigDisplaySettings; @@ -54,7 +69,7 @@ interface ChartDisplaySettingsDrawerProps { function applyDefaultSettings( settings: ChartConfigDisplaySettings, fallbackNumberFormat?: NumberFormat, -): ChartConfigDisplaySettings { +): DrawerFormValues { return { numberFormat: settings.numberFormat ?? fallbackNumberFormat ?? DEFAULT_NUMBER_FORMAT, @@ -67,6 +82,9 @@ function applyDefaultSettings( fitYAxisToData: settings.fitYAxisToData ?? false, groupByColumnsOnLeft: settings.groupByColumnsOnLeft ?? false, color: settings.color, + colorRules: settings.colorRules + ? attachLocalIds(settings.colorRules) + : undefined, }; } @@ -86,10 +104,9 @@ export default function ChartDisplaySettingsDrawer({ [settings, defaultNumberFormat], ); - const { control, handleSubmit, reset, setValue } = - useForm({ - defaultValues: appliedDefaults, - }); + const { control, handleSubmit, reset, setValue } = useForm({ + defaultValues: appliedDefaults, + }); useEffect(() => { reset(appliedDefaults); @@ -104,12 +121,24 @@ export default function ChartDisplaySettingsDrawer({ }, [onClose, reset, appliedDefaults]); const applyChanges = useCallback(() => { - handleSubmit(onChange)(); + handleSubmit(formValues => { + // Strip client-side localIds before passing rules to the config. + const { colorRules, ...rest } = formValues; + onChange({ + ...rest, + colorRules: colorRules ? stripLocalIds(colorRules) : undefined, + }); + })(); onClose(); }, [onChange, handleSubmit, onClose]); const resetToDefaults = useCallback(() => { - reset(applyDefaultSettings({}, defaultNumberFormat)); + reset( + applyDefaultSettings( + {} as ChartConfigDisplaySettings, + defaultNumberFormat, + ), + ); }, [reset, defaultNumberFormat]); const isTimeChart = @@ -208,6 +237,15 @@ export default function ChartDisplaySettingsDrawer({ )} /> + + ( + + )} + /> + )} diff --git a/packages/app/src/components/ColorRulesEditor.tsx b/packages/app/src/components/ColorRulesEditor.tsx new file mode 100644 index 0000000000..a6990e97a6 --- /dev/null +++ b/packages/app/src/components/ColorRulesEditor.tsx @@ -0,0 +1,387 @@ +import React, { useCallback } from 'react'; +import { + DndContext, + DragEndEvent, + MouseSensor, + TouchSensor, + useSensor, + useSensors, +} from '@dnd-kit/core'; +import { + arrayMove, + SortableContext, + useSortable, + verticalListSortingStrategy, +} from '@dnd-kit/sortable'; +import { CSS } from '@dnd-kit/utilities'; +import type { + ChartPaletteToken, + ColorCondition, +} from '@hyperdx/common-utils/dist/types'; +import { + ActionIcon, + Box, + Button, + Group, + NumberInput, + Select, + Stack, + Text, + TextInput, +} from '@mantine/core'; +import { IconGripVertical, IconTrash } from '@tabler/icons-react'; + +import { ColorSwatchInput } from './ColorSwatchInput'; + +// ─── Types ─────────────────────────────────────────────────────────────────── + +/** A ColorCondition with a client-side `localId` used as a stable dnd-kit key. */ +export type ColorRuleWithId = ColorCondition & { localId: string }; + +type ColorRulesEditorProps = { + value: ColorRuleWithId[]; + onChange: (rules: ColorRuleWithId[]) => void; +}; + +// ─── Operator options (number-tile subset) ──────────────────────────────────── + +const OPERATOR_OPTIONS = [ + { value: 'gt', label: '>' }, + { value: 'gte', label: '>=' }, + { value: 'lt', label: '<' }, + { value: 'lte', label: '<=' }, + { value: 'between', label: 'between' }, + { value: 'eq', label: '=' }, + { value: 'neq', label: '≠' }, +] as const; + +type NumericTileOperator = (typeof OPERATOR_OPTIONS)[number]['value']; + +/** Default rule added when the user clicks "Add rule". */ +function makeDefaultRule(): ColorRuleWithId { + return { + localId: crypto.randomUUID(), + operator: 'gt', + value: 0, + color: 'chart-blue', + }; +} + +// ─── Single sortable rule row ───────────────────────────────────────────────── + +function SortableRuleRow({ + rule, + index, + onUpdate, + onDelete, +}: { + rule: ColorRuleWithId; + index: number; + onUpdate: (index: number, next: ColorRuleWithId) => void; + onDelete: (index: number) => void; +}) { + const { + attributes, + listeners, + setNodeRef, + transform, + transition, + isDragging, + } = useSortable({ id: rule.localId }); + + const style: React.CSSProperties = { + transform: CSS.Transform.toString(transform), + transition, + opacity: isDragging ? 0.5 : 1, + }; + + const handleOperatorChange = useCallback( + (op: string | null) => { + if (!op) return; + const operator = op as NumericTileOperator; + // When switching to/from 'between', reset the value to a valid shape. + if (operator === 'between') { + onUpdate(index, { + ...rule, + operator: 'between', + value: [0, 100], + } as ColorRuleWithId); + } else if (operator === 'eq' || operator === 'neq') { + // eq/neq accept number or string; default to number 0 + const currentVal = + rule.operator !== 'between' && typeof rule.value === 'number' + ? rule.value + : 0; + onUpdate(index, { + ...rule, + operator, + value: currentVal, + } as ColorRuleWithId); + } else { + // Numeric ordered: gt/gte/lt/lte need a number + const currentVal = + rule.operator !== 'between' && typeof rule.value === 'number' + ? rule.value + : 0; + onUpdate(index, { + ...rule, + operator, + value: currentVal, + } as ColorRuleWithId); + } + }, + [index, onUpdate, rule], + ); + + const handleColorChange = useCallback( + (color?: ChartPaletteToken) => { + onUpdate(index, { ...rule, color: color ?? 'chart-blue' }); + }, + [index, onUpdate, rule], + ); + + const handleDelete = useCallback(() => onDelete(index), [index, onDelete]); + + // Value inputs differ by operator + let valueInputs: React.ReactNode; + if (rule.operator === 'between') { + const [lo, hi] = rule.value; + valueInputs = ( + + + onUpdate(index, { + ...rule, + operator: 'between', + value: [typeof v === 'number' ? v : 0, hi], + } as ColorRuleWithId) + } + aria-label={`Rule ${index + 1} lower bound`} + w={72} + /> + + to + + + onUpdate(index, { + ...rule, + operator: 'between', + value: [lo, typeof v === 'number' ? v : 0], + } as ColorRuleWithId) + } + aria-label={`Rule ${index + 1} upper bound`} + w={72} + /> + + ); + } else if (rule.operator === 'eq' || rule.operator === 'neq') { + // Accept text for eq/neq; if parseable as number convert it, else keep string + const displayVal = + typeof rule.value === 'number' + ? String(rule.value) + : (rule.value as string); + valueInputs = ( + { + const raw = e.currentTarget.value; + const num = Number(raw); + const coerced = + raw !== '' && !Number.isNaN(num) && Number.isFinite(num) + ? num + : raw; + onUpdate(index, { + ...rule, + operator: rule.operator, + value: coerced, + } as ColorRuleWithId); + }} + aria-label={`Rule ${index + 1} value`} + w={120} + /> + ); + } else { + // Numeric ordered: gt/gte/lt/lte + valueInputs = ( + + onUpdate(index, { + ...rule, + value: typeof v === 'number' ? v : 0, + } as ColorRuleWithId) + } + aria-label={`Rule ${index + 1} value`} + w={120} + /> + ); + } + + return ( + + + {/* Drag handle */} + + + + + {/* Operator */} + so its state is + // never tied to a virtual row's lifecycle. + const ts = Date.now(); + + await test.step('Create a table tile with a Search row-click action', async () => { + await addTableTile(`E2E Tooltip ${ts}`); + await dashboardPage.chartEditor.openRowClickDrawer(); + await dashboardPage.chartEditor.setRowClickMode('Search'); + await dashboardPage.chartEditor.fillRowClickTemplate( + DEFAULT_LOGS_SOURCE_NAME, + ); + await dashboardPage.chartEditor.applyRowClickDrawer(); + await dashboardPage.saveTile(); + }); + + await test.step('Set time range to Last 6 hours so rows render', async () => { + await dashboardPage.timePicker.selectRelativeTime('Last 6 hours'); + }); + + await dashboardPage.waitForTableTileRows(0); + + await test.step('Hover over first row — tooltip hint must appear', async () => { + await dashboardPage.hoverFirstTableRowAndGetTooltip(0); + // hoverFirstTableRowAndGetTooltip already asserts visibility before + // returning; reaching here means the tooltip appeared successfully. + }); + + await test.step('Move mouse away from the table — tooltip must disappear', async () => { + // Move to a neutral area well outside the table. The + // onMouseLeave safety net clears hoveredVirtualIndex, which makes + // hoveredRowDescription derive to null, disabling the shared + // Tooltip.Floating (display:none in the Portal). + await page.mouse.move(10, 10); + await expect(page.getByTestId('row-action-hint')).toBeHidden({ + timeout: 3000, + }); + }); + }); }, ); diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index 1d2e9cb6c4..a07b9b2d0e 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -1009,6 +1009,45 @@ export class DashboardPage { .click(); } + /** + * Return the first data row () of the table in the + * given tile. Used for hover-based interactions (e.g. tooltip tests). + */ + getFirstTableRow(tileIndex = 0): Locator { + return this.getTile(tileIndex) + .locator('table tbody tr[data-index]') + .first(); + } + + /** + * Hover over the first data row of a table tile and wait for the + * floating tooltip to appear. Returns the tooltip locator so callers + * can make further assertions. + * + * Tooltip.Floating renders its content inside a Portal at the document + * body level. Mantine uses CSS modules (hashed classes) so we locate the + * floating popup by text content that matches the known hint patterns used + * by describeOnClick (e.g. "Open in search", "Search ", + * "Open dashboard"). The tooltip is shown via `display: block` on the + * Portal div, so Playwright's `toBeVisible` checks that correctly. + */ + async hoverFirstTableRowAndGetTooltip(tileIndex = 0): Promise { + const row = this.getFirstTableRow(tileIndex); + await row.hover(); + // Trigger a mousemove inside the row so Tooltip.Floating's internal + // mousemove handler has a coordinate to position against. + const box = await row.boundingBox(); + if (box) { + await this.page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + } + // The Tooltip.Floating label is a . + // The portal div uses display:none when disabled, so Playwright's + // toBeVisible() correctly reflects the open/closed state. + const tooltip = this.page.getByTestId('row-action-hint'); + await tooltip.waitFor({ state: 'visible', timeout: 5000 }); + return tooltip; + } + /** * Locator for the Mantine toast raised by useOnClickLinkBuilder when the * configured onClick action fails (unknown source, missing row column, etc). diff --git a/packages/common-utils/src/__tests__/types.test.ts b/packages/common-utils/src/__tests__/types.test.ts new file mode 100644 index 0000000000..e93357798a --- /dev/null +++ b/packages/common-utils/src/__tests__/types.test.ts @@ -0,0 +1,284 @@ +import { z } from 'zod'; + +import { ColorConditionSchema } from '../types'; + +describe('ColorConditionSchema', () => { + // ─── Positive cases ───────────────────────────────────────────────────────── + + describe('numeric ordered operators', () => { + it.each(['gt', 'gte', 'lt', 'lte'] as const)( + 'parses operator %s with a valid numeric value', + operator => { + const result = ColorConditionSchema.safeParse({ + operator, + value: 42, + color: 'chart-success', + }); + expect(result.success).toBe(true); + }, + ); + + it('parses with an optional label', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'gte', + value: 100, + color: 'chart-warning', + label: 'High', + }); + expect(result.success).toBe(true); + }); + }); + + describe('between operator', () => { + it('parses a valid between rule', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'between', + value: [10, 100], + color: 'chart-blue', + }); + expect(result.success).toBe(true); + }); + + it('allows inverted between (first > second)', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'between', + value: [100, 10], + color: 'chart-blue', + }); + expect(result.success).toBe(true); + }); + }); + + describe('eq / neq operators', () => { + it('parses eq with a number value', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'eq', + value: 5, + color: 'chart-error', + }); + expect(result.success).toBe(true); + }); + + it('parses eq with a string value', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'eq', + value: 'CRIT', + color: 'chart-error', + }); + expect(result.success).toBe(true); + }); + + it('parses neq with a number value', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'neq', + value: 0, + color: 'chart-orange', + }); + expect(result.success).toBe(true); + }); + }); + + describe('string operators', () => { + it.each(['contains', 'startsWith', 'endsWith'] as const)( + 'parses operator %s with a non-empty string value', + operator => { + const result = ColorConditionSchema.safeParse({ + operator, + value: 'error', + color: 'chart-error', + }); + expect(result.success).toBe(true); + }, + ); + + it('parses regex with a valid pattern', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'regex', + value: '^error.*', + color: 'chart-error', + }); + expect(result.success).toBe(true); + }); + }); + + it('parses with all palette tokens', () => { + const tokens = [ + 'chart-blue', + 'chart-orange', + 'chart-red', + 'chart-cyan', + 'chart-green', + 'chart-pink', + 'chart-purple', + 'chart-light-blue', + 'chart-brown', + 'chart-gray', + 'chart-success', + 'chart-warning', + 'chart-error', + ] as const; + for (const token of tokens) { + const result = ColorConditionSchema.safeParse({ + operator: 'gt', + value: 0, + color: token, + }); + expect(result.success).toBe(true); + } + }); + + // ─── Negative cases ────────────────────────────────────────────────────────── + + it('rejects an unknown operator', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'notAnOp', + value: 1, + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects NaN on numeric operators', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'gt', + value: Number.NaN, + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects Infinity on numeric operators', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'lt', + value: Infinity, + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects a string value on a numeric operator (gt)', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'gt', + value: 'oops', + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects a number value on a string operator (contains)', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'contains', + value: 42, + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects an invalid palette token', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'gt', + value: 1, + color: 'not-a-token', + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty string on contains', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'contains', + value: '', + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty string on startsWith', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'startsWith', + value: '', + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty string on endsWith', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'endsWith', + value: '', + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty string on regex', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'regex', + value: '', + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects an unparseable regex pattern', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'regex', + value: '[invalid', + color: 'chart-blue', + }); + expect(result.success).toBe(false); + }); + + it('rejects a label longer than 40 characters', () => { + const result = ColorConditionSchema.safeParse({ + operator: 'gt', + value: 1, + color: 'chart-blue', + label: 'a'.repeat(41), + }); + expect(result.success).toBe(false); + }); +}); + +describe('colorRules array in SharedChartSettingsSchema', () => { + // Test array constraints directly with a z.array(ColorConditionSchema).max(10) schema, + // mirroring how SharedChartSettingsSchema declares colorRules. + const rulesSchema = z.array(ColorConditionSchema).max(10).optional(); + + it('accepts 0 rules', () => { + expect(rulesSchema.safeParse([]).success).toBe(true); + }); + + it('accepts 1 rule', () => { + expect( + rulesSchema.safeParse([{ operator: 'gt', value: 0, color: 'chart-blue' }]) + .success, + ).toBe(true); + }); + + it('accepts 5 rules', () => { + const rules = Array.from({ length: 5 }, (_, i) => ({ + operator: 'gt' as const, + value: i * 10, + color: 'chart-blue' as const, + })); + expect(rulesSchema.safeParse(rules).success).toBe(true); + }); + + it('accepts 10 rules', () => { + const rules = Array.from({ length: 10 }, (_, i) => ({ + operator: 'gte' as const, + value: i * 10, + color: 'chart-blue' as const, + })); + expect(rulesSchema.safeParse(rules).success).toBe(true); + }); + + it('rejects 11 rules', () => { + const rules = Array.from({ length: 11 }, (_, i) => ({ + operator: 'gte' as const, + value: i * 10, + color: 'chart-blue' as const, + })); + expect(rulesSchema.safeParse(rules).success).toBe(false); + }); +}); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 50315a3dba..95c0f157df 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1013,6 +1013,73 @@ export function walkRawDashboardTileColors( */ export const ChartPaletteTokenSchema = z.enum(CHART_PALETTE_TOKENS); +/** + * A single conditional color rule. Rules are evaluated in order against + * the tile's displayed value; the LAST matching rule's color wins + * (last-match-wins: higher-priority rules go last). If no rule matches, + * the tile's static `color` applies; if that is unset, the default text + * color applies. + * + * String operators (`contains`, `startsWith`, `endsWith`, `regex`) are + * included at the schema level so a future table-tile slice can reuse + * the same type without a schema change. The number-tile UI only exposes + * numeric / equality operators. + * + * Lives in common-utils so both the app and a future external-API parity + * PR can import it. + */ +export const ColorConditionSchema = z.discriminatedUnion('operator', [ + // Numeric ordered operators + z.object({ + operator: z.enum(['gt', 'gte', 'lt', 'lte']), + value: z.number().finite(), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), + }), + z.object({ + operator: z.literal('between'), + value: z.tuple([z.number().finite(), z.number().finite()]), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), + }), + // Equality (number OR string) + z.object({ + operator: z.enum(['eq', 'neq']), + value: z.union([z.number().finite(), z.string().max(200)]), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), + }), + // String operators (allowed at schema level for future table-tile reuse) + z.object({ + operator: z.enum(['contains', 'startsWith', 'endsWith']), + value: z.string().min(1).max(200), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), + }), + z.object({ + operator: z.literal('regex'), + value: z + .string() + .min(1) + .max(500) + .refine( + v => { + try { + new RegExp(v); + return true; + } catch { + return false; + } + }, + { message: 'Invalid regex pattern' }, + ), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), + }), +]); + +export type ColorCondition = z.infer; + // When making changes here, consider if they need to be made to the external API // schema as well (packages/api/src/utils/zod.ts). /** @@ -1036,6 +1103,11 @@ const SharedChartSettingsSchema = z.object({ // also a Number-tile-only field stored at shared level and gated in // the UI. color: ChartPaletteTokenSchema.optional(), + // Ordered conditional color rules for number tiles. Last matching rule + // wins (higher-priority rules go last). Kept at shared level so a future + // table-tile slice can attach per-column rules without a schema change. + // The UI gates the section on `displayType === DisplayType.Number`. + colorRules: z.array(ColorConditionSchema).max(10).optional(), }); export const _ChartConfigSchema = SharedChartSettingsSchema.extend({ From 6795826a9ae54d1a1c43ebcb836fb2b14c821e65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:59:43 +0000 Subject: [PATCH 031/728] chore(deps): bump hono from 4.12.16 to 4.12.23 (#2420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [hono](https://github.com/honojs/hono) from 4.12.16 to 4.12.23. e notes

Sourced from hono's releases.

v4.12.23

What's Changed

Full Changelog: https://github.com/honojs/hono/compare/v4.12.22...v4.12.23

v4.12.22

What's Changed

New Contributors

Full Changelog: https://github.com/honojs/hono/compare/v4.12.21...v4.12.22

v4.12.21

Security fixes

This release includes fixes for the following security issues:

app.mount() strips mount prefix using undecoded path, causing incorrect routing for percent-encoded paths

Affects: app.mount(). Fixes prefix stripping using the raw URL pathname instead of the decoded path, where percent-encoded characters in the mount prefix or path could cause the prefix to be removed at the wrong position, resulting in the sub-application receiving an incorrect path. GHSA-2gcr-mfcq-wcc3

IP Restriction bypasses static deny rules for non-canonical IPv6

Affects: hono/ip-restriction. Fixes IP address comparison using string equality, where non-canonical IPv6 representations of a denied address — such as compressed forms or hex-notation IPv4-mapped addresses — could bypass static deny rules. GHSA-xrhx-7g5j-rcj5

Cookie helper does not sanitize sameSite and priority, allowing Set-Cookie injection

Affects: hono/cookie. Fixes missing validation of sameSite and priority options against injection characters (;, \r, \n), where user-controlled input passed to either option could inject additional attributes into the Set-Cookie response header. GHSA-3hrh-pfw6-9m5x

JWT middleware accepts any Authorization scheme, not only Bearer

Affects: hono/jwt, hono/jwk. Fixes missing scheme validation in the Authorization header, where any two-part header value was accepted regardless of the scheme name, allowing non-Bearer schemes to pass JWT authentication. GHSA-f577-qrjj-4474

Users who use app.mount(), hono/ip-restriction, hono/cookie, or hono/jwt/hono/jwk are encouraged to upgrade to this version.

... (truncated)

Commits
  • 83bfb3b 4.12.23
  • bcd290a fix(utils/ipaddr): do not compress a single 0 group to :: (#4971)
  • c968177 feat(compress): add contentTypeFilter option and `COMPRESSIBLE_CONTENT_TYPE_R...
  • 0265a54 docs(contribution): add AI Usage Policy (#4970)
  • c84c5d2 feat(context): export the Context class publicly (#4543)
  • 82dad62 fix(serve-static): normalize all backslashes in file paths, not just the firs...
  • 2f01b77 4.12.22
  • 6bc0dff feat: add msgpack as a compressible content type (#4957)
  • 7e0555d fix(deno): echo negotiated WebSocket subprotocol in upgrade response (#4955)
  • f0ed246 fix(compress): respect Accept-Encoding when encoding option is set (#4951)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=hono&package-manager=npm_and_yarn&previous-version=4.12.16&new-version=4.12.23)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/hyperdxio/hyperdx/network/alerts).
--- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7b481164c2..a5a567ff49 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17513,9 +17513,9 @@ __metadata: linkType: hard "hono@npm:^4.11.4": - version: 4.12.16 - resolution: "hono@npm:4.12.16" - checksum: 10c0/3afee13722bf574780a641bd6d8812663c650fb7ac86df390f2d90293e1a6e2413aa9c45e4bc5b626a29c1b534fdb8353dd2151aab09bc4a95cd277aad4bd5c7 + version: 4.12.23 + resolution: "hono@npm:4.12.23" + checksum: 10c0/58945bb3aeb16d710b4a6c2809ba02b86b7269f6a3a67e126fc81cee5e9ae8ad2d9aa553db03c4f1a104ec03e423072e33932cb23eb3bbc48774acc72fc9c346 languageName: node linkType: hard From 9af8cbab30a30d9b761638c10ea38e23d5ecd657 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 8 Jun 2026 11:11:20 -0400 Subject: [PATCH 032/728] feat: add Browser RUM dashboard template (#2413) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds a **Browser RUM** template to the dashboards gallery (`Dashboards → Templates`) for browser sessions instrumented with the HyperDX Browser SDK — or any OpenTelemetry browser instrumentation that emits a `rum.sessionId` resource attribute. It fills a gap: HyperDX ships a browser SDK but had no out-of-the-box RUM dashboard. The template is purely declarative JSON validated by the existing `dashboardTemplates` schema test; the only code change is registering it in `dashboardTemplates/index.ts`, plus a changeset. The dashboard is organized into three sections: **Performance Overview** (page-view/session/error KPIs, Core Web Vitals LCP/INP/CLS p75, median/p90/p99 page-load percentiles, long tasks), **Page Views Breakdown** (traffic by URL, browser, country, and device size derived from `screen.xy`), and a tabbed **Errors** section (overview, JS exceptions by message and by page, failing API calls). It also defines five dashboard-level filters: Service, Environment, Service Version, Page URL, and Country. ### Screenshots or video image image image [Tab-1780519086079.webm](https://github.com/user-attachments/assets/1d9eafe2-b5be-40e0-a679-3a45c4e585be) ### How to test locally or on Vercel 1. `Dashboards → Templates → Browser RUM → Import`, then map each tile/filter to your Traces source (auto-maps if a source named "Traces" exists). 2. Point a browser app instrumented with `@hyperdx/browser` at your collector (or seed `webvitals` / `documentLoad` / fetch+xhr / exception spans carrying `rum.sessionId`). Verify the KPIs, Core Web Vitals, breakdown tables, and Errors tabs populate, and that the five filters apply. 3. **Top Browsers / Top Countries** tiles and the **Browser / Country** filters only populate when the collector's `useragent` and `geoip` processors are enabled (noted in the tile titles + dashboard description). ### References - Linear Issue: HDX-4451 - Related PRs: hyperdxio/hyperdx-js#240 (adds FCP/TTFB to the browser SDK, which extends this dashboard's Core Web Vitals coverage) --- .changeset/browser-rum-dashboard-template.md | 12 + .../src/dashboardTemplates/browser-rum.json | 970 ++++++++++++++++++ packages/app/src/dashboardTemplates/index.ts | 2 + 3 files changed, 984 insertions(+) create mode 100644 .changeset/browser-rum-dashboard-template.md create mode 100644 packages/app/src/dashboardTemplates/browser-rum.json diff --git a/.changeset/browser-rum-dashboard-template.md b/.changeset/browser-rum-dashboard-template.md new file mode 100644 index 0000000000..fd16b607c6 --- /dev/null +++ b/.changeset/browser-rum-dashboard-template.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': minor +--- + +feat: add Browser RUM dashboard template + +- New "Browser RUM" template in the dashboards gallery for browser sessions instrumented with the HyperDX Browser SDK (or any OTel browser instrumentation emitting a `rum.sessionId` resource attribute) +- Performance Overview section: page-view/session/error KPIs, Core Web Vitals (LCP/INP/CLS) p75, median/p75/p90 page-load percentiles, and long-task health +- Page Views Breakdown section: traffic grouped by URL, browser (parsed from the `http.user_agent` the document-load instrumentation emits), country, and device size (derived from `screen.xy`) +- Errors section with tabs for an overview, JS exceptions (by message and by page), and failing API calls +- Five dashboard-level filters: Service, Environment, Service Version, Page URL, and Country +- Top Countries tile and the Country filter populate when the OTel collector's `geoip` processor is enabled (geo can't be derived in the browser) diff --git a/packages/app/src/dashboardTemplates/browser-rum.json b/packages/app/src/dashboardTemplates/browser-rum.json new file mode 100644 index 0000000000..9799e45a3b --- /dev/null +++ b/packages/app/src/dashboardTemplates/browser-rum.json @@ -0,0 +1,970 @@ +{ + "version": "0.1.0", + "name": "Browser RUM", + "description": "Core Web Vitals, page-load percentiles, traffic breakdowns, and error tracking for browser sessions instrumented with the HyperDX Browser SDK (or any OpenTelemetry RUM instrumentation emitting rum.sessionId).", + "tags": ["Browser / RUM"], + "filters": [ + { + "id": "rum-filter-service", + "type": "QUERY_EXPRESSION", + "name": "Service", + "expression": "ServiceName", + "source": "Traces", + "where": "ResourceAttributes.rum.sessionId:*", + "whereLanguage": "lucene" + }, + { + "id": "rum-filter-environment", + "type": "QUERY_EXPRESSION", + "name": "Environment", + "expression": "ResourceAttributes['deployment.environment']", + "source": "Traces", + "where": "ResourceAttributes.rum.sessionId:* AND ResourceAttributes.deployment.environment:*", + "whereLanguage": "lucene" + }, + { + "id": "rum-filter-version", + "type": "QUERY_EXPRESSION", + "name": "Service Version", + "expression": "ResourceAttributes['service.version']", + "source": "Traces", + "where": "ResourceAttributes.rum.sessionId:* AND ResourceAttributes.service.version:*", + "whereLanguage": "lucene" + }, + { + "id": "rum-filter-url", + "type": "QUERY_EXPRESSION", + "name": "Page URL", + "expression": "SpanAttributes['location.href']", + "source": "Traces", + "where": "ResourceAttributes.rum.sessionId:* AND SpanAttributes.location.href:*", + "whereLanguage": "lucene" + }, + { + "id": "rum-filter-country", + "type": "QUERY_EXPRESSION", + "name": "Country", + "expression": "ResourceAttributes['geo.country.name']", + "source": "Traces", + "where": "ResourceAttributes.rum.sessionId:* AND ResourceAttributes.geo.country.name:*", + "whereLanguage": "lucene" + } + ], + "containers": [ + { + "id": "rum-perf", + "title": "Performance Overview", + "collapsed": false + }, + { + "id": "rum-breakdown", + "title": "Page Views Breakdown", + "collapsed": false + }, + { + "id": "rum-errors", + "title": "Errors", + "collapsed": false, + "tabs": [ + { + "id": "rum-errors-overview", + "title": "Overview" + }, + { + "id": "rum-errors-js", + "title": "JS Exceptions" + }, + { + "id": "rum-errors-api", + "title": "API Failures" + } + ] + } + ], + "tiles": [ + { + "id": "rum-006", + "containerId": "rum-perf", + "x": 0, + "y": 0, + "w": 6, + "h": 4, + "config": { + "name": "Page Views", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "aggCondition": "SpanName:\"documentLoad\" OR SpanName:\"routeChange\" OR SpanAttributes.component:\"page-view\" OR SpanAttributes.component:\"navigation\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-017", + "containerId": "rum-perf", + "x": 6, + "y": 0, + "w": 6, + "h": 4, + "config": { + "name": "Median Page Load", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "quantile", + "level": 0.5, + "valueExpression": "Duration", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene" + } + }, + { + "id": "rum-018", + "containerId": "rum-perf", + "x": 12, + "y": 0, + "w": 6, + "h": 4, + "config": { + "name": "p90 Page Load", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "quantile", + "level": 0.9, + "valueExpression": "Duration", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene" + } + }, + { + "id": "rum-001", + "containerId": "rum-perf", + "x": 18, + "y": 0, + "w": 6, + "h": 4, + "config": { + "name": "LCP p75 (ms)", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "none", + "valueExpression": "quantile(0.75)(toFloat64OrZero(SpanAttributes['lcp']))", + "aggCondition": "SpanName:\"webvitals\" AND SpanAttributes.lcp:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-002", + "containerId": "rum-perf", + "x": 0, + "y": 4, + "w": 6, + "h": 4, + "config": { + "name": "INP p75 (ms)", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "none", + "valueExpression": "quantile(0.75)(toFloat64OrZero(SpanAttributes['inp']))", + "aggCondition": "SpanName:\"webvitals\" AND SpanAttributes.inp:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-003", + "containerId": "rum-perf", + "x": 6, + "y": 4, + "w": 6, + "h": 4, + "config": { + "name": "CLS p75", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "none", + "valueExpression": "quantile(0.75)(toFloat64OrZero(SpanAttributes['cls']))", + "aggCondition": "SpanName:\"webvitals\" AND SpanAttributes.cls:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 3, + "thousandSeparated": true + } + } + }, + { + "id": "rum-005", + "containerId": "rum-perf", + "x": 12, + "y": 4, + "w": 6, + "h": 4, + "config": { + "name": "Active Sessions", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "aggCondition": "ResourceAttributes.rum.sessionId:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-019", + "containerId": "rum-perf", + "x": 18, + "y": 4, + "w": 6, + "h": 4, + "config": { + "name": "Sessions w/ Errors", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND StatusCode:error", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-020", + "containerId": "rum-perf", + "x": 0, + "y": 8, + "w": 24, + "h": 7, + "config": { + "name": "Page Load — median, p90, p99", + "source": "Traces", + "displayType": "line", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "quantile", + "level": 0.5, + "valueExpression": "Duration", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene", + "alias": "median" + }, + { + "aggFn": "quantile", + "level": 0.9, + "valueExpression": "Duration", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene", + "alias": "p90" + }, + { + "aggFn": "quantile", + "level": 0.99, + "valueExpression": "Duration", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene", + "alias": "p99" + } + ], + "where": "", + "whereLanguage": "lucene" + } + }, + { + "id": "rum-021", + "containerId": "rum-perf", + "x": 0, + "y": 15, + "w": 12, + "h": 6, + "config": { + "name": "Page Views over Time", + "source": "Traces", + "displayType": "line", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "alias": "Page Views", + "aggCondition": "SpanName:\"documentLoad\" OR SpanName:\"routeChange\" OR SpanAttributes.component:\"page-view\" OR SpanAttributes.component:\"navigation\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-024", + "containerId": "rum-perf", + "x": 12, + "y": 15, + "w": 12, + "h": 6, + "config": { + "name": "Long Tasks over Time", + "source": "Traces", + "displayType": "stacked_bar", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "alias": "Long Tasks", + "aggCondition": "SpanName:\"longtask\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-014", + "containerId": "rum-breakdown", + "x": 0, + "y": 0, + "w": 8, + "h": 8, + "config": { + "name": "Top URLs", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "alias": "Views", + "aggCondition": "SpanName:\"documentLoad\" OR SpanName:\"routeChange\" OR SpanAttributes.component:\"page-view\" OR SpanAttributes.component:\"navigation\"", + "aggConditionLanguage": "lucene" + }, + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "SpanName:\"documentLoad\" OR SpanName:\"routeChange\" OR SpanAttributes.component:\"page-view\" OR SpanAttributes.component:\"navigation\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "coalesce(nullif(SpanAttributes['http.url'], ''), nullif(SpanAttributes['page.url'], ''), nullif(SpanAttributes['location.href'], '')) AS \"URL\"", + "orderBy": "Views DESC", + "limit": { + "limit": 20 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "(SpanAttributes.http.url:\"{{URL}}\" OR SpanAttributes.page.url:\"{{URL}}\" OR SpanAttributes.location.href:\"{{URL}}\")", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-026", + "containerId": "rum-breakdown", + "x": 8, + "y": 0, + "w": 8, + "h": 8, + "config": { + "name": "Top Browsers", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND SpanAttributes.http.user_agent:*", + "aggConditionLanguage": "lucene" + }, + { + "aggFn": "count", + "valueExpression": "", + "alias": "Page Loads", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND SpanAttributes.http.user_agent:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "multiIf(positionCaseInsensitive(SpanAttributes['http.user_agent'], 'Edg/') > 0, 'Edge', positionCaseInsensitive(SpanAttributes['http.user_agent'], 'OPR/') > 0, 'Opera', positionCaseInsensitive(SpanAttributes['http.user_agent'], 'Firefox/') > 0, 'Firefox', positionCaseInsensitive(SpanAttributes['http.user_agent'], 'Chrome/') > 0, 'Chrome', positionCaseInsensitive(SpanAttributes['http.user_agent'], 'Safari/') > 0, 'Safari', 'Other') AS \"Browser\"", + "orderBy": "Sessions DESC", + "limit": { + "limit": 10 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "positionCaseInsensitive(SpanAttributes['http.user_agent'], '{{Browser}}') > 0", + "whereLanguage": "sql" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-027", + "containerId": "rum-breakdown", + "x": 16, + "y": 0, + "w": 8, + "h": 8, + "config": { + "name": "Top Countries (requires geoip processor)", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "ResourceAttributes.rum.sessionId:*", + "aggConditionLanguage": "lucene" + }, + { + "aggFn": "count", + "valueExpression": "", + "alias": "Views", + "aggCondition": "ResourceAttributes.rum.sessionId:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "coalesce(nullif(ResourceAttributes['geo.country.name'], ''), nullif(SpanAttributes['geo.country.name'], ''), nullif(ResourceAttributes['geo.country.iso_code'], ''), nullif(SpanAttributes['geo.country.iso_code'], '')) AS \"Country\"", + "orderBy": "Sessions DESC", + "limit": { + "limit": 10 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "ResourceAttributes.geo.country.name:\"{{Country}}\"", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-028", + "containerId": "rum-breakdown", + "x": 0, + "y": 8, + "w": 12, + "h": 8, + "config": { + "name": "Top Device Sizes", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "SpanName:\"documentLoad\" AND SpanAttributes.screen.xy:*", + "aggConditionLanguage": "lucene" + }, + { + "aggFn": "count", + "valueExpression": "", + "alias": "Views", + "aggCondition": "SpanName:\"documentLoad\" AND SpanAttributes.screen.xy:*", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "SpanAttributes['screen.xy'] AS \"Device\"", + "orderBy": "Sessions DESC", + "limit": { + "limit": 10 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "SpanName:\"documentLoad\" AND SpanAttributes.screen.xy:\"{{Device}}\"", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-011", + "containerId": "rum-breakdown", + "x": 12, + "y": 8, + "w": 12, + "h": 8, + "config": { + "name": "Slowest Pages (p90 page load)", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "quantile", + "level": 0.9, + "valueExpression": "Duration", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene", + "alias": "Page Load p90" + }, + { + "aggFn": "count", + "valueExpression": "", + "alias": "Views", + "aggCondition": "SpanName:\"documentLoad\"", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "coalesce(nullif(SpanAttributes['http.url'], ''), nullif(SpanAttributes['page.url'], ''), nullif(SpanAttributes['location.href'], '')) AS \"URL\"", + "orderBy": "`Page Load p90` DESC", + "limit": { + "limit": 20 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "SpanName:\"documentLoad\" AND (SpanAttributes.http.url:\"{{URL}}\" OR SpanAttributes.page.url:\"{{URL}}\" OR SpanAttributes.location.href:\"{{URL}}\")", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-016", + "containerId": "rum-breakdown", + "x": 0, + "y": 16, + "w": 24, + "h": 8, + "config": { + "name": "Top Errored Sessions", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "aggCondition": "StatusCode:error", + "aggConditionLanguage": "lucene", + "alias": "Errors" + }, + { + "aggFn": "count_distinct", + "valueExpression": "TraceId", + "alias": "Traces" + }, + { + "aggFn": "any", + "valueExpression": "nullif(SpanAttributes['userEmail'], '')", + "alias": "User" + }, + { + "aggFn": "any", + "valueExpression": "ServiceName", + "alias": "Service" + } + ], + "where": "ResourceAttributes.rum.sessionId:*", + "whereLanguage": "lucene", + "groupBy": "ResourceAttributes['rum.sessionId'] AS \"Session\"", + "having": "Errors > 0", + "havingLanguage": "sql", + "orderBy": "Errors DESC", + "limit": { + "limit": 20 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "ResourceAttributes.rum.sessionId:\"{{Session}}\"", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-007", + "containerId": "rum-errors", + "tabId": "rum-errors-overview", + "x": 0, + "y": 0, + "w": 12, + "h": 4, + "config": { + "name": "JS Errors", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND StatusCode:error AND NOT (SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\")", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-008", + "containerId": "rum-errors", + "tabId": "rum-errors-overview", + "x": 12, + "y": 0, + "w": 12, + "h": 4, + "config": { + "name": "AJAX Errors", + "source": "Traces", + "displayType": "number", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "aggCondition": "ResourceAttributes['rum.sessionId'] != '' AND (SpanAttributes['component'] = 'fetch' OR SpanAttributes['component'] = 'xml-http-request') AND (toUInt16OrZero(SpanAttributes['http.status_code']) >= 400 OR StatusCode = 'STATUS_CODE_ERROR')", + "aggConditionLanguage": "sql" + } + ], + "where": "", + "whereLanguage": "sql", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-010", + "containerId": "rum-errors", + "tabId": "rum-errors-overview", + "x": 0, + "y": 4, + "w": 24, + "h": 7, + "config": { + "name": "Errors over Time", + "source": "Traces", + "displayType": "line", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "aggCondition": "StatusCode:error AND NOT (SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\")", + "aggConditionLanguage": "lucene", + "alias": "JS Errors" + }, + { + "aggFn": "count", + "valueExpression": "", + "aggCondition": "(SpanAttributes['component'] = 'fetch' OR SpanAttributes['component'] = 'xml-http-request') AND (toUInt16OrZero(SpanAttributes['http.status_code']) >= 400 OR StatusCode = 'STATUS_CODE_ERROR')", + "aggConditionLanguage": "sql", + "alias": "AJAX Errors" + } + ], + "where": "ResourceAttributes.rum.sessionId:*", + "whereLanguage": "lucene", + "numberFormat": { + "output": "number", + "mantissa": 0, + "thousandSeparated": true + } + } + }, + { + "id": "rum-012", + "containerId": "rum-errors", + "tabId": "rum-errors-js", + "x": 0, + "y": 0, + "w": 12, + "h": 8, + "config": { + "name": "Top JS Errors (by message)", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "alias": "Occurrences", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND StatusCode:error AND NOT (SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\")", + "aggConditionLanguage": "lucene" + }, + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND StatusCode:error AND NOT (SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\")", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "coalesce(nullif(SpanAttributes['exception.message'], ''), nullif(SpanAttributes['message'], ''), SpanName) AS \"Error\"", + "orderBy": "Occurrences DESC", + "limit": { + "limit": 20 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "StatusCode:error AND (SpanAttributes.exception.message:\"{{Error}}\" OR SpanAttributes.message:\"{{Error}}\" OR SpanName:\"{{Error}}\")", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-015", + "containerId": "rum-errors", + "tabId": "rum-errors-js", + "x": 12, + "y": 0, + "w": 12, + "h": 8, + "config": { + "name": "Errors per Page", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "alias": "Errors", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND StatusCode:error AND NOT (SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\")", + "aggConditionLanguage": "lucene" + }, + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "ResourceAttributes.rum.sessionId:* AND StatusCode:error AND NOT (SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\")", + "aggConditionLanguage": "lucene" + } + ], + "where": "", + "whereLanguage": "lucene", + "groupBy": "coalesce(nullif(SpanAttributes['http.url'], ''), nullif(SpanAttributes['page.url'], ''), nullif(SpanAttributes['location.href'], '')) AS \"URL\"", + "orderBy": "Errors DESC", + "limit": { + "limit": 20 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "StatusCode:error AND (SpanAttributes.http.url:\"{{URL}}\" OR SpanAttributes.page.url:\"{{URL}}\" OR SpanAttributes.location.href:\"{{URL}}\")", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + }, + { + "id": "rum-013", + "containerId": "rum-errors", + "tabId": "rum-errors-api", + "x": 0, + "y": 0, + "w": 24, + "h": 8, + "config": { + "name": "Top Failing API Calls", + "source": "Traces", + "displayType": "table", + "granularity": "auto", + "alignDateRangeToGranularity": true, + "select": [ + { + "aggFn": "count", + "valueExpression": "", + "alias": "Failures", + "aggCondition": "ResourceAttributes['rum.sessionId'] != '' AND (SpanAttributes['component'] = 'fetch' OR SpanAttributes['component'] = 'xml-http-request') AND (toUInt16OrZero(SpanAttributes['http.status_code']) >= 400 OR StatusCode = 'STATUS_CODE_ERROR')", + "aggConditionLanguage": "sql" + }, + { + "aggFn": "count_distinct", + "valueExpression": "ResourceAttributes['rum.sessionId']", + "alias": "Sessions", + "aggCondition": "ResourceAttributes['rum.sessionId'] != '' AND (SpanAttributes['component'] = 'fetch' OR SpanAttributes['component'] = 'xml-http-request') AND (toUInt16OrZero(SpanAttributes['http.status_code']) >= 400 OR StatusCode = 'STATUS_CODE_ERROR')", + "aggConditionLanguage": "sql" + } + ], + "where": "", + "whereLanguage": "sql", + "groupBy": "coalesce(nullif(SpanAttributes['http.url'], ''), '') AS \"Endpoint\"", + "orderBy": "Failures DESC", + "limit": { + "limit": 20 + }, + "onClick": { + "type": "search", + "target": { + "mode": "id", + "id": "Traces" + }, + "whereTemplate": "(SpanAttributes.component:\"fetch\" OR SpanAttributes.component:\"xml-http-request\") AND SpanAttributes.http.url:\"{{Endpoint}}\"", + "whereLanguage": "lucene" + }, + "groupByColumnsOnLeft": true + } + } + ] +} diff --git a/packages/app/src/dashboardTemplates/index.ts b/packages/app/src/dashboardTemplates/index.ts index 609affa295..58e84c5684 100644 --- a/packages/app/src/dashboardTemplates/index.ts +++ b/packages/app/src/dashboardTemplates/index.ts @@ -3,6 +3,7 @@ import { DashboardTemplateSchema, } from '@hyperdx/common-utils/dist/types'; +import browserRum from './browser-rum.json'; import dotnetRuntime from './dotnet-runtime.json'; import goRuntime from './go-runtime.json'; import jvmRuntimeMetrics from './jvm-runtime-metrics.json'; @@ -23,6 +24,7 @@ function parseTemplate( } const templates: Record = { + 'browser-rum': browserRum, 'dotnet-runtime': dotnetRuntime, 'go-runtime': goRuntime, 'jvm-runtime-metrics': jvmRuntimeMetrics, From 2cecc9f40e1aa714fcb863054ce60fad53d29a7f Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:20:29 -0700 Subject: [PATCH 033/728] [HDX-4405] fix(app): prevent stranded tooltip in virtualised table rows (#2380) ## Summary Pivot of the original HDX-4405 fix per review feedback. The first round of this PR fixed the stranded-tooltip bug by hoisting `Tooltip.Floating` to the ``. That dismissed the bug but the resulting surface still felt off: the tooltip followed the cursor too closely. I've replaced it with a hover-revealed trailing chevron icon anchored to the row's last cell, wrapped in an anchored Mantine `Tooltip`. Mirrors the `.rowButtons` pattern from Search and Patterns. This eliminates the stranded-tooltip bug class by construction: the anchored Tooltip's open state is tied to the icon's lifecycle, so a virtual row unmounting mid-hover can no longer leave a popup behind. ### What changed (vs the previous commit on this branch) - Dropped the ``-level `Tooltip.Floating`, the `hoveredVirtualIndex` / `hoveredRowDescription` state, the `` `onMouseLeave` safety net, and per-row `onMouseEnter` / `onMouseLeave` handlers. - Added a per-row trailing `IconChevronRight` in the last `` does not reliably form a positioning context across browsers). - Tooltip configured `position="left"`, `withArrow`, `openDelay={300}`, `closeDelay={100}`, `fz="xs"` to match the established `DBRowTableIconButton` convention. - Suppressed on failure rows (`rowAction.url === null`) so the icon never promises a destination the click cannot deliver. ### Tests - Unit tests in `HDXMultiSeriesTableChart.test.tsx` rewritten: trailing-chevron rendering on success rows, dashboard-variant tooltip text, no chevron on failure rows, no chevron on the legacy `getRowSearchLink` path, no chevron when no action is configured. Dropped the inline-style `closest('[style*="display"]')` walk flagged in the previous deep review. - E2E spec `Trailing chevron hint appears on row hover...` now asserts opacity-gated visibility, tooltip text on row hover, and click-on-chevron navigation. Replaces the prior stranded-tooltip dismiss test. - `DashboardPage.ts` hint helper hovers the row to reveal the icon, then hovers the icon to open the anchored Tooltip. Docstring rewritten. ### How to test on Vercel preview **Preview route:** `/dashboards` 1. Open a dashboard with a Table tile configured with an `onClick` action (Search or Dashboard). 2. Hover a row. The chevron icon fades in at the right edge of the row. 3. Hover the chevron. After 300 ms the tooltip appears to the left with the resolved action description (for example `Search HyperDX Logs` or `Open dashboard "API Latency"`). 4. Click the chevron. Navigation behaves identically to clicking a cell. 5. Cmd-click / middle-click the chevron. New tab opens. 6. Right-click the chevron. Native context menu with "Open Link in New Tab" / "Copy Link Address". 7. Move the cursor rapidly across multiple rows. No stranded tooltip from earlier rows. 8. Configure an `onClick` whose template references an unknown column. Hover the resulting row. The chevron is not rendered. ### References - Linear: https://linear.app/clickhouse/issue/HDX-4405/ui-bug-tooltip-on-table-with-custom-actions-stays-rendered-when-moving - Related PRs: #2321 (introduced the row-click hint), this PR's previous commit (`24d8552c`) which fixed the stranding bug via tbody-`Tooltip.Floating`. --- .changeset/hdx-4405-trailing-action-hint.md | 5 + .../src/HDXMultiSeriesTableChart.module.scss | 68 +++++++ packages/app/src/HDXMultiSeriesTableChart.tsx | 187 ++++++++++-------- .../HDXMultiSeriesTableChart.test.tsx | 184 +++++++++++------ .../features/dashboard-table-linking.spec.ts | 55 ++++-- .../tests/e2e/page-objects/DashboardPage.ts | 50 +++-- 6 files changed, 370 insertions(+), 179 deletions(-) create mode 100644 .changeset/hdx-4405-trailing-action-hint.md create mode 100644 packages/app/src/HDXMultiSeriesTableChart.module.scss diff --git a/.changeset/hdx-4405-trailing-action-hint.md b/.changeset/hdx-4405-trailing-action-hint.md new file mode 100644 index 0000000000..2dba609e2b --- /dev/null +++ b/.changeset/hdx-4405-trailing-action-hint.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +Dashboard table tiles configured with a row-click action now show a trailing arrow-up-right icon at the right edge of each row, revealed on hover, with a small tooltip that names the destination. Actionable rows get a stronger background highlight on hover to reinforce interactivity before the user sees the arrow fade in. The icon click navigates to the same URL as a row click, with all the standard native browser behaviors (cmd-click new tab, middle-click new tab, right-click context menu). diff --git a/packages/app/src/HDXMultiSeriesTableChart.module.scss b/packages/app/src/HDXMultiSeriesTableChart.module.scss new file mode 100644 index 0000000000..416ffe0642 --- /dev/null +++ b/packages/app/src/HDXMultiSeriesTableChart.module.scss @@ -0,0 +1,68 @@ +// Trailing arrow hint for dashboard table tile rows with a +// configured onClick row action. The icon sits at the right edge of +// the last cell and is revealed on row hover. Mirrors the +// .rowButtons pattern from LogTable.module.scss (lines 156-172), +// scoped down to a single icon. See HDX-4405. + +.tableRow { + // Row hover is the visibility trigger for the trailing-icon hint. + // We rely on a descendant selector rather than wrapping a + // tooltip because anchored visibility is tied to the row's own + // lifecycle and never strands a popup after a virtual row + // unmounts. + &:hover .rowActionHint { + opacity: 1; + pointer-events: auto; + } +} + +.actionableRow { + // Stronger hover background on rows that resolve to a click + // destination. Non-actionable rows keep the default + // `bg-muted-hover` utility (`--color-bg-muted`), so the visual + // delta between the two reinforces interactivity before the + // user even sees the trailing arrow fade in. + &:hover { + background-color: var(--color-bg-highlighted); + } +} + +.lastCell { + // Containing block for the absolute-positioned .rowActionHint. + // does not reliably form a positioning context across + // browsers, so we attach `position: relative` to the last correct: - // it sets null rather than the prior row's description. See HDX-4405. - const [hoveredVirtualIndex, setHoveredVirtualIndex] = useState( - null, - ); - - // Derive the label from whichever row currently occupies hoveredVirtualIndex. - // Returns null when no row is hovered, the index is out of range, or the - // row's action has no URL (error-toast rows show no hint). - const hoveredRowDescription = useMemo(() => { - if (hoveredVirtualIndex == null) return null; - const virtualRow = items.find(v => v.index === hoveredVirtualIndex); - if (!virtualRow) return null; - const row = rows[virtualRow.index] as TableRow | undefined; - if (!row) return null; - const rowAction = getRowAction ? getRowAction(row.original) : null; - return rowAction?.url != null && rowAction.description - ? rowAction.description - : null; - }, [hoveredVirtualIndex, items, rows, getRowAction]); - - const clearHovered = useCallback(() => setHoveredVirtualIndex(null), []); - const { csvData } = useCsvExport( truncatedData, columns.map(col => ({ @@ -398,62 +376,105 @@ export const Table = ({ ))} - {/* Single Tooltip.Floating wrapping the whole so the hint - follows the cursor without being tied to the lifecycle of any - individual virtual row. Per-row Tooltip.Floating instances get - stranded in the Portal when a row unmounts before onMouseLeave - fires (rapid mouse movement in a virtualised list). With this - approach the tooltip state lives on , which never unmounts, - and the label is re-derived from hoveredVirtualIndex each render so - scroll re-virtualisation never shows stale text. See HDX-4405. */} - {hoveredRowDescription} - } - withinPortal - disabled={!hoveredRowDescription} - > - {/* onMouseLeave on is a safety net: if a virtual row - unmounts before its own onMouseLeave fires (rapid cursor - movement or re-virtualisation), leaving the table body still - clears the hovered index. */} - - {paddingTop > 0 && ( - - - )} - {items.map(virtualRow => { - const row = rows[virtualRow.index] as TableRow; - return ( - setHoveredVirtualIndex(virtualRow.index)} - onMouseLeave={clearHovered} - > - {row.getVisibleCells().map(cell => { - return ( - - ); - })} - - ); - })} - {paddingBottom > 0 && ( - - + {paddingTop > 0 && ( + + + )} + {items.map(virtualRow => { + const row = rows[virtualRow.index] as TableRow; + // Compute the action once per row so the trailing-icon hint + // shares the memoized result from useOnClickLinkBuilder with + // the per-cell renders. The hook keys its cache off the row + // reference via WeakMap (see useOnClickLinkBuilder), so this + // extra call is an O(1) lookup when the per-cell renders + // populate the entry first, and one extra compute per row + // only when the WeakMap entry is cold (e.g. fresh data). + const rowAction = getRowAction ? getRowAction(row.original) : null; + // Narrow `rowAction.url` to a non-null `string` once per row so + // the trailing-icon guard below (and the `` + // sink inside it) doesn't need an `as string` cast and stays + // type-safe under future changes to the `RowAction` shape. + const actionUrl = rowAction?.url ?? null; + const isActionable = actionUrl !== null; + const visibleCells = row.getVisibleCells(); + const lastCellIndex = visibleCells.length - 1; + return ( + + {visibleCells.map((cell, cellIndex) => { + const isLastCell = cellIndex === lastCellIndex; + return ( + + ); + })} - )} - - + ); + })} + {paddingBottom > 0 && ( + + + )} +
` of clickable rows, wrapped in a Next.js ``. The Link reuses the row's existing URL so cmd-click / middle-click / right-click "Open in New Tab" / status-bar URL preview all work on the icon. `tabIndex={-1}` and `aria-hidden="true"` keep sequential focus order and screen-reader traversal anchored to the per-cell Links so the icon doesn't double-count. - New `HDXMultiSeriesTableChart.module.scss` owns `.tableRow`, `.lastCell`, `.rowActionHint`. The icon is `opacity: 0` by default and fades in on `.tableRow:hover` via a 0.15s transition. `.lastCell` adds `position: relative` to the last `` only (the `
+ // only. The icon then anchors to the cell's right edge, which + // visually maps to the row's right edge because the last data + // column claims UNDEFINED_WIDTH (see reactTableColumns above). + position: relative; +} + +.rowActionHint { + position: absolute; + top: 50%; + right: 6px; + transform: translateY(-50%); + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + border-radius: 4px; + color: var(--color-text-muted); + background: transparent; + opacity: 0; + pointer-events: none; + transition: opacity 0.15s ease-in-out; + text-decoration: none; + + &:hover, + &:focus-visible { + color: var(--color-text-primary); + background-color: var(--color-bg-muted); + } + + &:focus-visible { + outline: 2px solid var(--color-outline-focus); + outline-offset: 1px; + } +} diff --git a/packages/app/src/HDXMultiSeriesTableChart.tsx b/packages/app/src/HDXMultiSeriesTableChart.tsx index 604e0c3140..5cd3441470 100644 --- a/packages/app/src/HDXMultiSeriesTableChart.tsx +++ b/packages/app/src/HDXMultiSeriesTableChart.tsx @@ -2,7 +2,11 @@ import { useCallback, useMemo, useRef, useState } from 'react'; import Link from 'next/link'; import cx from 'classnames'; import { Tooltip, UnstyledButton } from '@mantine/core'; -import { IconDownload, IconTextWrap } from '@tabler/icons-react'; +import { + IconArrowUpRight, + IconDownload, + IconTextWrap, +} from '@tabler/icons-react'; import { flexRender, getCoreRowModel, @@ -28,6 +32,7 @@ import type { NumberFormat } from './types'; import { formatNumber } from './utils'; import focusStyles from '../styles/focus.module.scss'; +import styles from './HDXMultiSeriesTableChart.module.scss'; export type TableVariant = 'default' | 'muted'; @@ -307,33 +312,6 @@ export const Table = ({ ); const [wrapLinesEnabled, setWrapLinesEnabled] = useState(false); - // Store the virtual index of the hovered row (not its description string) - // so the label re-derives on every render. If the virtualiser replaces the - // row at that index (scroll re-virtualisation, auto-refetch) the label - // reflects the new row immediately rather than showing stale text. Storing - // the index also makes the safety-net `onMouseLeave` on
-
- {flexRender( - cell.column.columnDef.cell, - cell.getContext(), - )} -
+
+
+ {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + {/* Trailing arrow hint: anchored Mantine Tooltip + wrapping a Next.js Link in the last cell of + rows that resolve to a URL. The icon is hidden + (opacity: 0) by default and revealed on row + hover via the .tableRow:hover .rowActionHint + rule. The Link inherits the same native + cmd-click / middle-click / right-click + semantics as the per-cell Link in the row body. + Suppressed when the row's templates failed + (rowAction.url === null) so the icon never + promises a destination the click won't open. + The arrow-up-right shape signals "navigate + elsewhere" without colliding with the + chevron-right used by sidebar group collapse + / expand affordances. See HDX-4405. */} + {isLastCell && actionUrl !== null && rowAction && ( + + + + + + )} +
+
{isTruncated && (
diff --git a/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx b/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx index 8947a731dc..13fc33b8a1 100644 --- a/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx +++ b/packages/app/src/__tests__/HDXMultiSeriesTableChart.test.tsx @@ -1,5 +1,6 @@ import React from 'react'; -import { fireEvent, screen, waitFor } from '@testing-library/react'; +import { screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; import { Table } from '@/HDXMultiSeriesTableChart'; @@ -76,7 +77,59 @@ describe('HDXMultiSeriesTableChart ', () => { }); }); - it('reveals the hint at the cursor when hovering a success row', async () => { + it('marks actionable rows with the actionableRow class so they hover to --color-bg-highlighted', () => { + // Rows with a resolved click destination get a stronger hover + // background than non-actionable rows. The differentiation is + // applied via the `actionableRow` CSS module class on the ; + // non-actionable rows fall through to the global + // `bg-muted-hover` utility (a lighter muted hover). + const getRowAction = jest.fn().mockReturnValue({ + url: '/search?source=src_1&where=', + description: 'Search HyperDX Logs', + }); + + const { container } = renderWithMantine( +
{}} + />, + ); + + const rows = container.querySelectorAll('tbody tr[data-index]'); + expect(rows.length).toBeGreaterThan(0); + rows.forEach(row => { + expect(row.className).toContain('actionableRow'); + expect(row.className).not.toContain('bg-muted-hover'); + }); + }); + + it('renders a trailing arrow hint in the last cell of a success row', () => { + const getRowAction = jest.fn().mockReturnValue({ + url: '/search?source=src_1&where=', + description: 'Search HyperDX Logs', + }); + + renderWithMantine( +
{}} + />, + ); + + const hint = screen.getByTestId('row-action-hint'); + expect(hint.tagName).toBe('A'); + expect(hint.getAttribute('href')).toContain('/search?source=src_1'); + expect(hint.getAttribute('aria-hidden')).toBe('true'); + }); + + it('shows the description in an anchored tooltip when the arrow is hovered', async () => { + const user = userEvent.setup(); const getRowAction = jest.fn().mockReturnValue({ url: '/search?source=src_1&where=', description: 'Search HyperDX Logs', @@ -92,42 +145,31 @@ describe('HDXMultiSeriesTableChart
', () => { />, ); - const row = screen.getByText('web').closest('tr')!; - fireEvent.mouseEnter(row); + const hint = screen.getByTestId('row-action-hint'); + await user.hover(hint); + // Mantine renders the Tooltip label as an element with role="tooltip" + // (with the label text as its accessible name) once opened. await waitFor( () => { - expect(screen.getByText('Search HyperDX Logs')).toBeInTheDocument(); + expect( + screen.getByRole('tooltip', { name: 'Search HyperDX Logs' }), + ).toBeInTheDocument(); }, - { timeout: 1000 }, + { timeout: 2000 }, ); }); - it('hides the hint when the hovered virtual index maps to a no-URL row (HDX-4405)', async () => { - // Regression for the virtualiser race: the hovered can unmount - // before its onMouseLeave fires (rapid movement / data refresh). The - // old per-row Tooltip.Floating was stranded in the Portal; the fix - // stores a virtual index and re-derives the label via useMemo each - // render. When the row at that index no longer has a URL, the tooltip - // hides without a leave event from the (now-gone) element. - // - // We verify the key invariant structurally: hovering index 0 (URL row) - // shows the hint; hovering index 1 (no-URL row) hides it — no - // mouseLeave fires between the two enterevents, simulating the - // cursor jumping over the table faster than leave events dispatch. - const multiRowData = [ - { ServiceName: 'web', Count: 10 }, - { ServiceName: 'api', Count: 5 }, - ]; - const getRowAction = jest.fn((row: { ServiceName: string }) => - row.ServiceName === 'web' - ? { url: '/search?source=src_1&where=', description: 'Search Logs' } - : { url: null, description: '', onClickError: jest.fn() }, - ); + it('renders the dashboard variant description in the tooltip', async () => { + const user = userEvent.setup(); + const getRowAction = jest.fn().mockReturnValue({ + url: '/dashboards/dash_1?where=', + description: 'Open dashboard "API Latency"', + }); renderWithMantine(
', () => { />, ); - const webRow = screen.getByText('web').closest('tr')!; - const apiRow = screen.getByText('api').closest('tr')!; + const hint = screen.getByTestId('row-action-hint'); + await user.hover(hint); - // Hover the URL row — tooltip must appear - fireEvent.mouseEnter(webRow); - await waitFor(() => { - const hint = screen.getByTestId('row-action-hint'); - const tooltipBox = hint.closest('[style*="display"]'); - expect(tooltipBox?.style.display).toBe('block'); - }); - - // Hover the no-URL row WITHOUT firing mouseLeave on the first row. - // This simulates the cursor jumping faster than leave events dispatch. - fireEvent.mouseEnter(apiRow); - - // The label must derive to null (apiRow has url:null) so tooltip hides. - await waitFor(() => { - const hint = screen.getByTestId('row-action-hint'); - const tooltipBox = hint.closest('[style*="display"]'); - expect(tooltipBox?.style.display).toBe('none'); - }); + await waitFor( + () => { + expect( + screen.getByRole('tooltip', { + name: 'Open dashboard "API Latency"', + }), + ).toBeInTheDocument(); + }, + { timeout: 2000 }, + ); }); }); describe('getRowAction failure path', () => { - it('renders the cell as a
', () => { expect(btn.hasAttribute('href')).toBe(false); }); - fireEvent.click(buttons[0]); + await user.click(buttons[0]); expect(onClickError).toHaveBeenCalledTimes(1); }); - it('does not reveal a hint when the row action has no url', async () => { + it('leaves non-actionable rows on bg-muted-hover (no actionableRow class)', () => { + // Mirror of the success-path actionableRow test: rows whose + // action returns `url: null` keep the default muted hover + // utility and never gain the stronger `actionableRow` class. const getRowAction = jest.fn().mockReturnValue({ url: null, description: 'Search HyperDX Logs', onClickError: jest.fn(), }); - renderWithMantine( + const { container } = renderWithMantine(
', () => { />, ); - const row = screen.getByText('web').closest('tr')!; - fireEvent.mouseEnter(row); + const rows = container.querySelectorAll('tbody tr[data-index]'); + expect(rows.length).toBeGreaterThan(0); + rows.forEach(row => { + expect(row.className).not.toContain('actionableRow'); + expect(row.className).toContain('bg-muted-hover'); + }); + }); - // Give the tooltip a chance to mount; assert it never does. - await new Promise(resolve => setTimeout(resolve, 250)); - expect(screen.queryByText('Search HyperDX Logs')).not.toBeInTheDocument(); + it('does not render a trailing arrow hint when rowAction.url is null', () => { + const getRowAction = jest.fn().mockReturnValue({ + url: null, + description: 'Search HyperDX Logs', + onClickError: jest.fn(), + }); + + renderWithMantine( +
{}} + />, + ); + + // The icon must not be in the DOM at all on failure rows: showing + // an arrow would promise a destination the click can't deliver, + // because the click only fires an error toast. + expect(screen.queryByTestId('row-action-hint')).toBeNull(); }); }); describe('legacy getRowSearchLink fallback', () => { - it('renders the cell as a Link without a HoverCard when only getRowSearchLink is provided', () => { + it('renders the cell as a Link without a trailing arrow hint when only getRowSearchLink is provided', () => { const getRowSearchLink = jest .fn() .mockReturnValue('/search?source=legacy&where='); @@ -240,11 +301,15 @@ describe('HDXMultiSeriesTableChart
', () => { expect(link.getAttribute('data-shape')).toBe('link'); expect(link.getAttribute('href')).toContain('/search?source=legacy'); }); + + // No trailing arrow on the legacy path; it's an additive surface + // tied to the new getRowAction prop only. + expect(screen.queryByTestId('row-action-hint')).toBeNull(); }); }); describe('no action configured', () => { - it('renders plain cells with no Link, button, or HoverCard', () => { + it('renders plain cells with no Link, button, or trailing arrow', () => { renderWithMantine(
', () => { expect( screen.queryAllByTestId('dashboard-table-row-action'), ).toHaveLength(0); + expect(screen.queryByTestId('row-action-hint')).toBeNull(); }); }); }); diff --git a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts index 2f96395a5c..689544080b 100644 --- a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts @@ -752,14 +752,15 @@ test.describe( }); }); - test('Tooltip hint appears on hover and disappears on mouse-leave — no stranded tooltip (HDX-4405)', async ({ + test('Trailing arrow hint appears on row hover and navigates to the action URL (HDX-4405)', async ({ page, }) => { - // Regression test for HDX-4405: Tooltip.Floating instances mounted - // per virtual row were getting stranded in their Portal when the row - // unmounted before onMouseLeave fired (rapid mouse movement). The fix - // moves a single shared Tooltip.Floating to so its state is - // never tied to a virtual row's lifecycle. + // Pivot of the original HDX-4405 regression test: the row-click hint + // is now an anchored Mantine Tooltip wrapping a trailing arrow-up-right + // icon in the last cell, rather than a Tooltip.Floating that tracks the + // cursor. The arrow is hidden until the row is hovered and the + // tooltip is tied to the icon's lifecycle, so no stranded popup is + // possible by construction. const ts = Date.now(); await test.step('Create a table tile with a Search row-click action', async () => { @@ -779,21 +780,39 @@ test.describe( await dashboardPage.waitForTableTileRows(0); - await test.step('Hover over first row — tooltip hint must appear', async () => { - await dashboardPage.hoverFirstTableRowAndGetTooltip(0); - // hoverFirstTableRowAndGetTooltip already asserts visibility before - // returning; reaching here means the tooltip appeared successfully. + await test.step('Trailing arrow is hidden until the row is hovered', async () => { + const hint = dashboardPage.getRowActionHint(0); + // The icon is in the DOM but its visibility is gated by the + // .tableRow:hover .rowActionHint CSS opacity transition. + await expect(hint).toHaveCSS('opacity', '0'); }); - await test.step('Move mouse away from the table — tooltip must disappear', async () => { - // Move to a neutral area well outside the table. The - // onMouseLeave safety net clears hoveredVirtualIndex, which makes - // hoveredRowDescription derive to null, disabling the shared - // Tooltip.Floating (display:none in the Portal). + await test.step('Hovering the row reveals the arrow and the tooltip describes the action', async () => { + const tooltip = await dashboardPage.hoverFirstTableRowAndGetTooltip(0); + await expect(tooltip).toContainText(/Search|Open/); + }); + + await test.step('Moving the cursor away closes the tooltip', async () => { + // Move to a neutral area; Mantine Tooltip closes on mouseleave + // (closeDelay=100ms in the component). Narrow the role match to + // the row-action tooltip text so the assertion does not flake on + // unrelated header / resize-handle tooltips that may also live in + // the portal at the moment of the check. await page.mouse.move(10, 10); - await expect(page.getByTestId('row-action-hint')).toBeHidden({ - timeout: 3000, - }); + await expect( + page.getByRole('tooltip', { name: /Search|Open/ }), + ).toBeHidden({ timeout: 3000 }); + }); + + await test.step('Clicking the trailing arrow navigates to the same destination as the row body', async () => { + // Re-hover the row, then click the arrow. The Link inside the + // tooltip-wrapped arrow points to the same href as the per-cell + // links, so click navigates exactly as a row click would. + const row = dashboardPage.getFirstTableRow(0); + await row.hover(); + const hint = dashboardPage.getRowActionHint(0); + await hint.click(); + await expect(page).toHaveURL(/\/search\?/, { timeout: 10000 }); }); }); }, diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index a07b9b2d0e..4bbf8b532b 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -1020,30 +1020,42 @@ export class DashboardPage { } /** - * Hover over the first data row of a table tile and wait for the - * floating tooltip to appear. Returns the tooltip locator so callers - * can make further assertions. + * Locator for the trailing arrow hint element rendered in the last + * cell of clickable rows. The icon (arrow-up-right) carries + * `data-testid="row-action-hint"` and is the trigger element for the + * anchored Mantine Tooltip describing the row's onClick destination. + */ + getRowActionHint(tileIndex = 0): Locator { + return this.getTile(tileIndex) + .locator('table tbody tr[data-index]') + .first() + .getByTestId('row-action-hint'); + } + + /** + * Hover the first data row of a table tile, then hover its trailing + * arrow hint so the anchored Mantine Tooltip opens. Returns the + * tooltip locator so callers can assert on the description text. * - * Tooltip.Floating renders its content inside a Portal at the document - * body level. Mantine uses CSS modules (hashed classes) so we locate the - * floating popup by text content that matches the known hint patterns used - * by describeOnClick (e.g. "Open in search", "Search ", - * "Open dashboard"). The tooltip is shown via `display: block` on the - * Portal div, so Playwright's `toBeVisible` checks that correctly. + * The arrow icon (`data-testid="row-action-hint"`) is hidden + * (`opacity: 0`) until the row is hovered. Hovering the row reveals + * the icon via the `.tableRow:hover .rowActionHint` CSS rule. The + * Mantine Tooltip wrapping the icon then opens when the cursor moves + * to the icon itself, rendering its label in a portal at the body. + * + * The returned locator narrows the role match by name so the assertion + * does not collide with header-cell or resize-handle tooltips that + * may also live in the portal at the moment of the check + * (Search-suggestion onClick wording, dashboard-open wording, etc.). */ async hoverFirstTableRowAndGetTooltip(tileIndex = 0): Promise { const row = this.getFirstTableRow(tileIndex); await row.hover(); - // Trigger a mousemove inside the row so Tooltip.Floating's internal - // mousemove handler has a coordinate to position against. - const box = await row.boundingBox(); - if (box) { - await this.page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); - } - // The Tooltip.Floating label is a . - // The portal div uses display:none when disabled, so Playwright's - // toBeVisible() correctly reflects the open/closed state. - const tooltip = this.page.getByTestId('row-action-hint'); + const hint = this.getRowActionHint(tileIndex); + // Hover the icon directly so the anchored Tooltip's mouseEnter + // listener fires; row-hover alone only fades the icon in. + await hint.hover(); + const tooltip = this.page.getByRole('tooltip', { name: /Search|Open/ }); await tooltip.waitFor({ state: 'visible', timeout: 5000 }); return tooltip; } From cdd7ca07be8506be58eed2563e3be3d98731b2ed Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 8 Jun 2026 13:43:26 -0600 Subject: [PATCH 034/728] fix(mcp): reduce describe_source timeouts by using rollup tables for map key discovery (#2426) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `clickstack_describe_source` can time out (10s wall-clock limit) when discovering map attribute keys on large tables. Stage 2 (map key discovery) was always falling back to a full main-table scan—up to 3M rows per Map column—even when rollup tables were available, because `metadataMVs` and `dateRange` weren't computed yet at that point in the pipeline. This hoists the rollup config and date range above Stage 2 so `getMapKeys` can use the fast rollup path. On the traces table in large environments this reduces Stage 2 from ~5-10s to <1s, keeping the full describe call well within the 10s budget. Additionally, `getMapValues` now accepts an `AbortSignal` so that any per-key fallback queries are cancelled when the timeout fires, rather than running to completion against ClickHouse after the caller has already given up. --- .changeset/fix-describe-source-timeout.md | 5 ++++ .../src/mcp/tools/sources/describeSource.ts | 27 +++++++++++-------- packages/common-utils/src/core/metadata.ts | 9 +++++++ 3 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 .changeset/fix-describe-source-timeout.md diff --git a/.changeset/fix-describe-source-timeout.md b/.changeset/fix-describe-source-timeout.md new file mode 100644 index 0000000000..716b355bc0 --- /dev/null +++ b/.changeset/fix-describe-source-timeout.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +fix(mcp): reduce describe_source timeouts by using rollup tables for map key discovery diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index a8249ae329..5674c7e2cf 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -143,6 +143,19 @@ async function describeSourceSchema( // Track which sampling stages were skipped due to timeout const skippedStages: string[] = []; + // Shared by stages 2–4 so map-key discovery can use rollup tables + // instead of falling back to expensive main-table scans. + const metadataMVs = + 'metadataMaterializedViews' in source + ? source.metadataMaterializedViews + : undefined; + + const now = new Date(); + const dateRange: [Date, Date] = [ + new Date(now.getTime() - VALUE_SAMPLE_LOOKBACK_MS), + now, + ]; + // ── 1. Column schema ────────────────────────────────────────────────── const columns = await metadata.getColumns({ databaseName, @@ -170,6 +183,9 @@ async function describeSourceSchema( column: col.name, maxKeys: 50, connectionId, + metadataMVs, + dateRange, + signal, }); mapKeysResults[col.name] = keys; } catch (e) { @@ -200,17 +216,6 @@ async function describeSourceSchema( const lowCardinalityValues: Record = {}; - const metadataMVs = - 'metadataMaterializedViews' in source - ? source.metadataMaterializedViews - : undefined; - - const now = new Date(); - const dateRange: [Date, Date] = [ - new Date(now.getTime() - VALUE_SAMPLE_LOOKBACK_MS), - now, - ]; - if (lcColumns.length > 0 && !signal.aborted) { try { const results = await metadata.getAllKeyValues({ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index a80feb1fad..d6e2b10e34 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -351,6 +351,7 @@ export class Metadata { metadataMVs, dateRange, timestampValueExpression, + signal, }: { databaseName: string; tableName: string; @@ -361,6 +362,7 @@ export class Metadata { metadataMVs?: MetadataMaterializedViews; dateRange?: [Date, Date]; timestampValueExpression?: string; + signal?: AbortSignal; }) { // Align date range to rollup granularity for consistent cache keys const alignedDateRange = @@ -419,6 +421,7 @@ export class Metadata { max_execution_time: 15, max_rows_to_read: '0', }, + abort_signal: signal, }) .then(res => res.json<{ Key: string }>()) .then(d => d.data.map(row => row.Key).filter(k => k)); @@ -524,6 +527,7 @@ export class Metadata { // Set the value to 0 (unlimited) so that the LIMIT is used instead max_rows_to_read: '0', }, + abort_signal: signal, }) .then(res => res.json<{ keysArr?: string[]; key?: string }>()) .then(d => { @@ -640,6 +644,7 @@ export class Metadata { connectionId, dateRange, timestampValueExpression, + signal, }: { databaseName: string; tableName: string; @@ -649,6 +654,7 @@ export class Metadata { dateRange?: [Date, Date]; timestampValueExpression?: string; connectionId: string; + signal?: AbortSignal; }) { const dateRangeCacheSuffix = dateRange && timestampValueExpression @@ -719,6 +725,7 @@ export class Metadata { read_overflow_mode: 'break', ...this.getClickHouseSettings(), }, + abort_signal: signal, }) .then(res => res.json<{ value: string }>()) .then(d => d.data.map(row => row.value)); @@ -1464,6 +1471,7 @@ export class Metadata { connectionId, dateRange, timestampValueExpression, + signal, }); return { key: p.keyExpression, value: fallback }; }), @@ -1482,6 +1490,7 @@ export class Metadata { connectionId, dateRange, timestampValueExpression, + signal, }); return { key: p.keyExpression, value }; }), From f113ea364853e80cf59e42774c6de07d09332537 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 9 Jun 2026 11:04:15 -0600 Subject: [PATCH 035/728] fix(mcp): add ClickHouse safety settings (max_execution_time, max_result_rows, readonly) (#2427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary MCP query tools (`clickstack_sql`, `clickstack_timeseries`, `clickstack_table`, `clickstack_search`, `clickstack_query_tile`) now enforce ClickHouse safety limits that were previously missing. A runaway agent-supplied query could hold a connection for up to 1 hour (the default `requestTimeout`) with no row cap and no protection against DDL/DML injection. **Closes HDX-4476** ## What changed A single constant governs all MCP query execution in `runConfigTile()`: | Setting | Value | Purpose | |---|---|---| | `max_execution_time` | 30s | Kills long-running queries — matches eventDeltas, eventPatterns, Prometheus | | `max_result_rows` | 100,000 | Prevents serialization OOM on the Node process — matches Prometheus | | `readonly` | 1 | Blocks CREATE, INSERT, ALTER, DROP — matches trace tools | | `requestTimeout` | 32s | HTTP timeout = 30s + 2s buffer for clean ClickHouse error — matches external API pattern | Applied to both the builder config path and the raw SQL path. The settings constant is shared so they can't drift. ## Tests Integration tests against real ClickHouse verify actual enforcement: - **readonly** — `CREATE TABLE`, `INSERT`, and `DROP TABLE` via `clickstack_sql` are rejected with a readonly error; `SELECT` still works - **max_execution_time** — `sleep(3)` with `max_execution_time: 1` throws `TIMEOUT_EXCEEDED` --- .changeset/nasty-crews-shop.md | 5 + packages/api/src/mcp/__tests__/query.test.ts | 13 +++ .../api/src/mcp/__tests__/queryTool.test.ts | 107 ++++++++++++++++++ packages/api/src/mcp/tools/query/helpers.ts | 26 +++++ 4 files changed, 151 insertions(+) create mode 100644 .changeset/nasty-crews-shop.md diff --git a/.changeset/nasty-crews-shop.md b/.changeset/nasty-crews-shop.md new file mode 100644 index 0000000000..0b53ead24f --- /dev/null +++ b/.changeset/nasty-crews-shop.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +fix(mcp): add ClickHouse safety settings (max_execution_time, max_result_rows, readonly) for MCP query execution diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index 214adc058b..1a9c7ac55c 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -225,6 +225,19 @@ describe('errorHint', () => { expect(hint).toContain('LIMIT'); }); + it('should match RESULT_IS_TOO_LARGE errors', () => { + const hint = errorHint('Code: 396. DB::Exception: RESULT_IS_TOO_LARGE'); + expect(hint).not.toBeNull(); + expect(hint).toContain('100,000 rows'); + expect(hint).toContain('LIMIT'); + }); + + it('should match TOO_MANY_ROWS_OR_BYTES errors', () => { + const hint = errorHint('Code: 396. DB::Exception: TOO_MANY_ROWS_OR_BYTES'); + expect(hint).not.toBeNull(); + expect(hint).toContain('100,000 rows'); + }); + it('should return null for unrecognized errors', () => { const hint = errorHint('Connection refused'); expect(hint).toBeNull(); diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index b5264580b5..4cdfd6563a 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -1,3 +1,4 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; @@ -583,4 +584,110 @@ describe('MCP Query Tools', () => { expect(getFirstText(result)).toContain('Invalid'); }); }); + + // ─── Safety settings (readonly + max_execution_time) ───────────────────────── + + describe('ClickHouse safety settings', () => { + describe('readonly enforcement via clickstack_sql', () => { + it('should reject CREATE TABLE (DDL)', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: 'CREATE TABLE __mcp_test_ddl (id UInt64) ENGINE = Memory', + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toMatch(/readonly/i); + }); + + it('should reject INSERT (DML)', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: `INSERT INTO ${DEFAULT_DATABASE}.${DEFAULT_LOGS_TABLE} (Body) VALUES ('injected')`, + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toMatch(/readonly/i); + }); + + it('should reject DROP TABLE (DDL)', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: `DROP TABLE IF EXISTS ${DEFAULT_DATABASE}.${DEFAULT_LOGS_TABLE}`, + }); + + expect(result.isError).toBe(true); + const text = getFirstText(result); + expect(text).toMatch(/readonly/i); + }); + + it('should allow SELECT (read-only query)', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: 'SELECT 1 AS value', + }); + + expect(result.isError).toBeFalsy(); + }); + }); + + describe('max_execution_time enforcement', () => { + it('should kill a query that exceeds max_execution_time', async () => { + // Test directly with ClickhouseClient to use a short timeout (1s) + // instead of waiting for the full 30s MCP default. + const clickhouseClient = new ClickhouseClient({ + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + requestTimeout: 5_000, + }); + + await expect( + clickhouseClient.query({ + query: 'SELECT sleep(3)', + format: 'JSON', + clickhouse_settings: { + max_execution_time: 1, + }, + }), + ).rejects.toThrow(/TIMEOUT_EXCEEDED|timeout/i); + }); + }); + + describe('settings propagation via clickstack_sql', () => { + it('should propagate max_execution_time=30 to ClickHouse', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: "SELECT getSetting('max_execution_time') AS value", + }); + + expect(result.isError).toBeFalsy(); + const parsed = JSON.parse(getFirstText(result)); + expect(parsed.result?.data?.[0]?.value).toBe(30); + }); + + it('should propagate max_result_rows=100000 to ClickHouse', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: "SELECT getSetting('max_result_rows') AS value", + }); + + expect(result.isError).toBeFalsy(); + const parsed = JSON.parse(getFirstText(result)); + expect(parsed.result?.data?.[0]?.value).toBe(100000); + }); + + it('should propagate readonly=1 to ClickHouse', async () => { + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: "SELECT getSetting('readonly') AS value", + }); + + expect(result.isError).toBeFalsy(); + const parsed = JSON.parse(getFirstText(result)); + expect(parsed.result?.data?.[0]?.value).toBe(1); + }); + }); + }); }); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 7fbd3d200a..c87fc82066 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -27,6 +27,22 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +// ─── Safety limits ─────────────────────────────────────────────────────────── + +/** ClickHouse settings applied to all MCP query-tool executions. */ +const MCP_CLICKHOUSE_SETTINGS = { + max_execution_time: 30, + max_result_rows: '100000', + readonly: 1, +} as const; + +/** + * HTTP request timeout for MCP query-tool ClickHouse clients. + * Set slightly above max_execution_time so ClickHouse can return a clean + * timeout error before the HTTP connection is aborted. + */ +const MCP_REQUEST_TIMEOUT = 32_000; // 30s query limit + 2s buffer + // ─── Where merging ─────────────────────────────────────────────────────────── export interface MergeWhereResult { @@ -243,6 +259,7 @@ export async function runConfigTile( host: connection.host, username: connection.username, password: connection.password, + requestTimeout: MCP_REQUEST_TIMEOUT, }); const isSearch = builderConfig.displayType === DisplayType.Search; @@ -295,6 +312,7 @@ export async function runConfigTile( config: chartConfig, metadata, querySettings: source.querySettings, + opts: { clickhouse_settings: MCP_CLICKHOUSE_SETTINGS }, }); return formatQueryResult(result); } catch (e) { @@ -349,6 +367,7 @@ export async function runConfigTile( host: connection.host, username: connection.username, password: connection.password, + requestTimeout: MCP_REQUEST_TIMEOUT, }); const chartConfig = { @@ -363,6 +382,7 @@ export async function runConfigTile( config: chartConfig, metadata, querySettings: undefined, + opts: { clickhouse_settings: MCP_CLICKHOUSE_SETTINGS }, }); return formatQueryResult(result); } catch (e) { @@ -419,5 +439,11 @@ export function errorHint(msg: string): string | null { 'The result row count is too large to serialize back to the agent.' ); } + if (/TOO_MANY_ROWS_OR_BYTES|RESULT_IS_TOO_LARGE/i.test(msg)) { + return ( + 'The query returned more than 100,000 rows. ' + + 'Add a LIMIT, narrow the time range, or add filters to reduce the result set.' + ); + } return null; } From 750b8afe85c870ee9b11033bde726275e50454d3 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 9 Jun 2026 13:50:20 -0600 Subject: [PATCH 036/728] feat(mcp): add denoise option to hyperdx_search tool (#2371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds a `denoise` boolean parameter to the MCP `hyperdx_search` tool that automatically filters out high-frequency repetitive event patterns from search results, mirroring the web app's "Denoise Results" checkbox. ## Why When investigating issues via MCP, LLM agents get back raw search results that are often dominated by repetitive log noise. This forces agents to either sift through noisy results or make a separate `hyperdx_event_patterns` call and then manually filter. The `denoise` option makes this a single-call workflow. ## How it works When `denoise=true`: 1. Runs the normal search query to get result rows 2. In parallel, samples 10k random events from the same source/time range and counts total events 3. Mines patterns using the shared Drain algorithm (`common-utils`) 4. Identifies "noisy" patterns — those accounting for >10% of sampled events (same threshold as the web app) 5. Rebuilds a `TemplateMiner`, matches each search result row against learned patterns 6. Filters out rows matching noisy patterns 7. Returns filtered rows plus a `denoised` metadata block listing removed patterns, original row count, and filtered row count Graceful degradation: if source/connection/body column can't be resolved, or if pattern sampling fails, the original results are returned unmodified. ## Changes | File | Change | |------|--------| | `packages/api/src/mcp/tools/query/denoise.ts` | **New** — Core `denoiseSearchResults()` function | | `packages/api/src/mcp/tools/query/search.ts` | Added `denoise` schema param + post-processing logic | | `packages/api/src/mcp/tools/query/helpers.ts` | Extracted shared `resolveBodyExpression()` + `SAFE_BODY_EXPR_CHARS` | | `packages/api/src/mcp/tools/query/runEventPatterns.ts` | Imports from helpers instead of local definitions | ## Example response (with denoise=true) ```json { "result": { "data": [...filtered rows...] }, "denoised": { "removedPatterns": [ { "pattern": "GET /health <*> <*>", "estimatedCount": 45000, "sampleCount": 4500 } ], "originalRowCount": 50, "filteredRowCount": 12 } } ``` Screenshot 2026-05-29 at 8 42 07 AM ## Performance Adds ~1-2s latency when `denoise=true` due to the pattern sampling queries. No impact when `denoise=false` (the default). Closes HDX-4346 --- .changeset/mcp-search-denoise.md | 20 ++ .../api/src/mcp/__tests__/queryTool.test.ts | 104 +++++++ packages/api/src/mcp/tools/query/denoise.ts | 274 ++++++++++++++++++ packages/api/src/mcp/tools/query/helpers.ts | 36 ++- .../src/mcp/tools/query/runEventPatterns.ts | 37 +-- packages/api/src/mcp/tools/query/search.ts | 145 ++++++++- packages/app/src/components/DBRowTable.tsx | 10 +- packages/common-utils/src/drain/index.ts | 7 +- .../common-utils/src/drain/mine-patterns.ts | 9 + 9 files changed, 604 insertions(+), 38 deletions(-) create mode 100644 .changeset/mcp-search-denoise.md create mode 100644 packages/api/src/mcp/tools/query/denoise.ts diff --git a/.changeset/mcp-search-denoise.md b/.changeset/mcp-search-denoise.md new file mode 100644 index 0000000000..53b5495980 --- /dev/null +++ b/.changeset/mcp-search-denoise.md @@ -0,0 +1,20 @@ +--- +'@hyperdx/api': patch +'@hyperdx/app': patch +'@hyperdx/common-utils': patch +--- + +feat(mcp): add denoise option to clickstack_search tool + +Add a `denoise` boolean parameter to the MCP `clickstack_search` tool that +automatically filters out high-frequency repetitive event patterns from +search results, mirroring the web app's "Denoise Results" feature. + +When enabled, the tool samples 10k random events, mines patterns using +the Drain algorithm, identifies noisy patterns (>10% of sample), and +filters them out of result rows. Returns filtered rows plus metadata +listing removed patterns with estimated counts. + +Extracts shared denoise constants (`DENOISE_SAMPLE_SIZE`, +`DENOISE_NOISE_THRESHOLD`) into `@hyperdx/common-utils` so the web app +and MCP server use the same values. diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index 4cdfd6563a..a63571f1d7 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -357,6 +357,110 @@ describe('MCP Query Tools', () => { expect(result.isError).toBe(true); expect(getFirstText(result)).toMatch(/sourceId/i); }); + + it('should expose denoise property in schema', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === 'clickstack_search'); + expect(tool).toBeDefined(); + const props = Object.keys(tool!.inputSchema.properties ?? {}); + expect(props).toContain('denoise'); + }); + + it('should emit denoised block when denoise=true on empty results', async () => { + const result = await callTool(client, 'clickstack_search', { + sourceId: logSource._id.toString(), + denoise: true, + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + // With no data, the denoised block should not appear because the + // search result itself has no rows to process (early return path). + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveProperty('result'); + }); + + describe('denoise with seeded data', () => { + const now = new Date(); + const fiveMinAgo = new Date(now.getTime() - 5 * 60 * 1000); + + beforeEach(async () => { + const logs: Parameters[0] = []; + + // Noisy pattern: "Health check OK from " — 80 rows (>10% threshold) + for (let i = 0; i < 80; i++) { + logs.push({ + Body: `Health check OK from 10.0.${Math.floor(i / 256)}.${i % 256}`, + ServiceName: 'loadbalancer', + SeverityText: 'INFO', + Timestamp: new Date(fiveMinAgo.getTime() + i * 100), + }); + } + + // Unique/rare events — 5 rows (well below 10% threshold) + for (let i = 0; i < 5; i++) { + logs.push({ + Body: `Rare event type ${String.fromCharCode(65 + i)} occurred in subsystem`, + ServiceName: 'worker', + SeverityText: 'WARN', + Timestamp: new Date(fiveMinAgo.getTime() + (80 + i) * 1000), + }); + } + + await bulkInsertLogs(logs); + }); + + it('should filter noisy patterns and emit denoised metadata', async () => { + const result = await callTool(client, 'clickstack_search', { + sourceId: logSource._id.toString(), + denoise: true, + maxResults: 200, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + + // Must have a denoised block + expect(output).toHaveProperty('denoised'); + expect(output.denoised).toHaveProperty('removedPatterns'); + expect(output.denoised).toHaveProperty('returnedRowCountBeforeDenoise'); + expect(output.denoised).toHaveProperty('filteredRowCount'); + + // Should not have a skipped reason + expect(output.denoised.skipped).toBeUndefined(); + + // The noisy health check pattern should be in removedPatterns + expect(output.denoised.removedPatterns.length).toBeGreaterThanOrEqual( + 1, + ); + const healthPattern = output.denoised.removedPatterns.find( + (p: { pattern: string }) => p.pattern.includes('Health check'), + ); + expect(healthPattern).toBeDefined(); + + // Filtered count should be less than original + expect(output.denoised.filteredRowCount).toBeLessThan( + output.denoised.returnedRowCountBeforeDenoise, + ); + }); + + it('should return results without denoised block when denoise=false', async () => { + const result = await callTool(client, 'clickstack_search', { + sourceId: logSource._id.toString(), + denoise: false, + maxResults: 200, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).not.toHaveProperty('denoised'); + }); + }); }); // ─── clickstack_event_patterns ───────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts new file mode 100644 index 0000000000..00d5f1bff2 --- /dev/null +++ b/packages/api/src/mcp/tools/query/denoise.ts @@ -0,0 +1,274 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; +import { + DENOISE_NOISE_THRESHOLD, + DENOISE_SAMPLE_SIZE, + flattenBody, + minePatterns, + TemplateMiner, + TemplateMinerConfig, +} from '@hyperdx/common-utils/dist/drain'; +import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; + +import { resolveBodyExpression } from './helpers'; + +// ─── Types ─────────────────────────────────────────────────────────────────── + +export interface DenoiseResult { + /** Filtered rows with noisy patterns removed. */ + rows: Record[]; + /** Patterns that were identified as noisy and removed. */ + removedPatterns: Array<{ + pattern: string; + estimatedCount: number; + sampleCount: number; + }>; + /** + * When non-null, denoising was skipped and rows are returned unmodified. + * The value describes why (e.g. "body_column_not_in_results"). + */ + skipped?: string; +} + +// ─── Core denoising function ───────────────────────────────────────────────── + +/** + * Denoise search results by mining patterns from a random sample, identifying + * "noisy" patterns (those accounting for >10% of the sample), and filtering + * them out of the result rows. + * + * This mirrors the web app's "Denoise Results" feature + * (packages/app/src/components/DBRowTable.tsx) but runs server-side using + * the shared TypeScript Drain implementation. + */ +export async function denoiseSearchResults( + teamId: string, + sourceId: string, + startDate: Date, + endDate: Date, + rows: Record[], + options?: { + where?: string; + whereLanguage?: 'lucene' | 'sql'; + }, +): Promise { + if (rows.length === 0) { + return { rows, removedPatterns: [], skipped: 'no_rows' }; + } + + // ── Resolve source & connection ── + const source = await getSource(teamId, sourceId); + if (!source) { + return { rows, removedPatterns: [], skipped: 'source_not_found' }; + } + + const bodyColumn = resolveBodyExpression(source); + if (!bodyColumn) { + return { rows, removedPatterns: [], skipped: 'no_body_column' }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { rows, removedPatterns: [], skipped: 'connection_not_found' }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + + const tsExpr = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + const implicitColumn = + 'implicitColumnExpression' in source + ? source.implicitColumnExpression + : undefined; + const useTextIndexForImplicitColumn = + 'useTextIndexForImplicitColumn' in source + ? source.useTextIndexForImplicitColumn + : undefined; + + // ── Query: Random sample of events for pattern learning ── + const sampleConfig = { + displayType: DisplayType.Search, + source: source._id.toString(), + select: `${bodyColumn} as __hdx_pattern_body, ${tsExpr} as __hdx_pattern_ts`, + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + useTextIndexForImplicitColumn, + orderBy: [{ ordering: 'DESC' as const, valueExpression: 'rand()' }], + limit: { limit: DENOISE_SAMPLE_SIZE, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + // ── Query: Total count for sample multiplier ── + const countConfig = { + displayType: DisplayType.Table, + source: source._id.toString(), + select: 'count() as total', + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + useTextIndexForImplicitColumn, + limit: { limit: 1, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + let sampleResult: Awaited< + ReturnType + >; + let countResult: Awaited< + ReturnType + >; + try { + [sampleResult, countResult] = await Promise.all([ + clickhouseClient.queryChartConfig({ + config: sampleConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + clickhouseClient.queryChartConfig({ + config: countConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + ]); + } catch { + // If sampling fails, return rows unmodified rather than failing the search + return { rows, removedPatterns: [], skipped: 'sampling_failed' }; + } + + const sampleRows = sampleResult.data; + const totalCount = Number(countResult.data?.[0]?.total ?? 0); + + if (!sampleRows || sampleRows.length === 0) { + return { rows, removedPatterns: [], skipped: 'no_sample_data' }; + } + + // ── Mine patterns from the sample ── + // Note: maxSamples: 1 — minePatterns always keeps at least one sample per + // cluster internally; we just minimize memory overhead. + const { patterns } = minePatterns(sampleRows, { + totalCount, + startDate, + endDate, + maxSamples: 1, + getBody: row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }, + getTimestamp: row => { + const tsRaw = row.__hdx_pattern_ts; + return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; + }, + }); + + if (patterns.length === 0) { + return { rows, removedPatterns: [] }; + } + + // ── Identify noisy patterns (>10% of sampled events) ── + // Key by template string rather than cluster ID so we are not coupled to + // the auto-incrementing IDs generated inside minePatterns(). The matching + // miner below produces its own IDs; comparing template strings is stable. + const sampledRowCount = sampleRows.length; + const noisyTemplates = new Set(); + const removedPatterns: DenoiseResult['removedPatterns'] = []; + + for (const p of patterns) { + if (p.sampleCount / sampledRowCount > DENOISE_NOISE_THRESHOLD) { + noisyTemplates.add(p.pattern); + removedPatterns.push({ + pattern: p.pattern, + estimatedCount: p.estimatedCount, + sampleCount: p.sampleCount, + }); + } + } + + if (noisyTemplates.size === 0) { + return { rows, removedPatterns: [] }; + } + + // ── Build a miner trained on the same sample for row matching ── + const drainConfig = new TemplateMinerConfig(); + const miner = new TemplateMiner(drainConfig); + for (const row of sampleRows) { + const raw = row.__hdx_pattern_body; + const bodyText = flattenBody(raw != null ? String(raw) : ''); + miner.addLogMessage(bodyText); + } + + // ── Match each result row and filter out noisy ones ── + const bodyColumnKey = findBodyColumnKey(rows[0], bodyColumn); + if (!bodyColumnKey) { + return { + rows, + removedPatterns: [], + skipped: 'body_column_not_in_results', + }; + } + + const filteredRows = rows.filter(row => { + const bodyValue = row[bodyColumnKey]; + if (bodyValue == null) return true; // Keep rows with no body + const bodyText = flattenBody(String(bodyValue)); + const match = miner.match(bodyText, 'fallback'); + if (!match) return true; // No pattern match — keep the row + return !noisyTemplates.has(match.getTemplate()); + }); + + return { + rows: filteredRows, + removedPatterns, + }; +} + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +/** + * Find the key in a result row that corresponds to the body column expression. + * ClickHouse may return the column under its expression name or a simplified alias. + */ +function findBodyColumnKey( + row: Record, + bodyColumn: string, +): string | null { + // Direct match (e.g. "Body", "SpanName") + if (bodyColumn in row) return bodyColumn; + + // Case-insensitive match + const lowerBody = bodyColumn.toLowerCase(); + for (const key of Object.keys(row)) { + if (key.toLowerCase() === lowerBody) return key; + } + + return null; +} diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index c87fc82066..c6782a4227 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -1,6 +1,9 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; +import { + getFirstTimestampValueExpression, + splitAndTrimWithBracket, +} from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderSavedChartConfig, isRawSqlSavedChartConfig, @@ -27,6 +30,37 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +// ─── Source body expression helpers ────────────────────────────────────────── + +export interface SourceBodyFields { + kind: string; + spanNameExpression?: string; + bodyExpression?: string; + implicitColumnExpression?: string; +} + +/** + * Resolve the body column expression for pattern mining from a source. + * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). + */ +export function resolveBodyExpression( + source: SourceBodyFields, +): string | undefined { + let expression: string | undefined; + if (source.kind === SourceKind.Trace) { + expression = source.spanNameExpression; + } else if (source.kind === SourceKind.Log) { + expression = source.bodyExpression ?? source.implicitColumnExpression; + } + if (!expression) return undefined; + const multiExpr = splitAndTrimWithBracket(expression); + return multiExpr.length === 1 ? expression : multiExpr[0]; +} + +/** Reject bodyExpression values containing SQL-unsafe characters. */ +// eslint-disable-next-line no-useless-escape +export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; + // ─── Safety limits ─────────────────────────────────────────────────────────── /** ClickHouse settings applied to all MCP query-tool executions. */ diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index 47f757cea6..7fb9a7617d 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -6,42 +6,17 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { minePatterns } from '@hyperdx/common-utils/dist/drain'; import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; -import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; import { trimToolResponse } from '@/utils/trimToolResponse'; -import { clickHouseErrorResult } from './helpers'; - -// ─── Source helpers ────────────────────────────────────────────────────────── - -interface SourceBodyFields { - kind: string; - spanNameExpression?: string; - bodyExpression?: string; - implicitColumnExpression?: string; -} - -/** - * Resolve the body column expression for pattern mining from a source. - * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). - */ -function resolveBodyExpression(source: SourceBodyFields): string | undefined { - let expression: string | undefined; - if (source.kind === SourceKind.Trace) { - expression = source.spanNameExpression; - } else if (source.kind === SourceKind.Log) { - expression = source.bodyExpression ?? source.implicitColumnExpression; - } - if (!expression) return undefined; - const multiExpr = splitAndTrimWithBracket(expression); - return multiExpr.length === 1 ? expression : multiExpr[0]; -} - -/** Reject bodyExpression values containing SQL-unsafe characters. */ -// eslint-disable-next-line no-useless-escape -const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; +import { + clickHouseErrorResult, + resolveBodyExpression, + SAFE_BODY_EXPR_CHARS, +} from './helpers'; // ─── Event pattern mining ──────────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 624639ffd7..6625b883dd 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -1,8 +1,12 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; +import logger from '@/utils/logger'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; +import { denoiseSearchResults } from './denoise'; import { buildTile, parseTimeRange, runConfigTile } from './helpers'; import { endTimeSchema, @@ -36,6 +40,16 @@ const searchSchema = z.object({ 'Maximum number of rows to return (1-200). Default: 50. ' + 'Use smaller values to reduce response size.', ), + denoise: z + .boolean() + .optional() + .default(false) + .describe( + 'When true, automatically removes events matching high-frequency patterns ' + + '(those accounting for >10% of sampled events) from the results. ' + + 'Useful for cutting through log noise to find unusual or interesting events. ' + + 'Adds ~1-2s of latency for the pattern sampling queries.', + ), startTime: startTimeSchema, endTime: endTimeSchema, }); @@ -56,6 +70,8 @@ export function registerSearch(server: McpServer, context: McpContext) { 'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' + 'For aggregated metrics, use clickstack_table instead. ' + 'For pattern discovery, use clickstack_event_patterns instead.\n\n' + + 'Set denoise=true to automatically filter out high-frequency repetitive patterns, ' + + 'surfacing only unusual or interesting events.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + "Map attributes use bracket syntax: SpanAttributes['http.method'].", inputSchema: searchSchema, @@ -78,9 +94,132 @@ export function registerSearch(server: McpServer, context: McpContext) { whereLanguage: input.whereLanguage, }); - return runConfigTile(teamId.toString(), tile, startDate, endDate, { - maxResults: input.maxResults, - }); + const result = await runConfigTile( + teamId.toString(), + tile, + startDate, + endDate, + { + maxResults: input.maxResults, + }, + ); + + // ── Denoising post-processing ── + if (!input.denoise || ('isError' in result && result.isError)) { + return result; + } + + // Extract the raw result data from the formatted response. + // runConfigTile returns { content: [{ type: "text", text: JSON }] }. + const resultText = result.content?.[0]?.text; + if (!resultText) return result; + + let parsed: { result?: { data?: Record[] } }; + try { + parsed = JSON.parse(resultText); + } catch { + return result; + } + + const resultData = parsed.result; + const rows = (resultData as Record | undefined)?.data as + | Record[] + | undefined; + if (!rows || !Array.isArray(rows) || rows.length === 0) { + return result; + } + + let denoised; + try { + denoised = await denoiseSearchResults( + teamId.toString(), + input.sourceId, + startDate, + endDate, + rows, + { + where: input.where, + whereLanguage: input.whereLanguage, + }, + ); + } catch (err) { + // Denoise is a post-processing enhancement — a failure here must + // never discard the already-successful search result. + logger.warn( + { err, sourceId: input.sourceId }, + 'denoiseSearchResults failed; returning raw results', + ); + + const { data: trimmedResult, isTrimmed } = trimToolResponse(resultData); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedResult, + denoised: { + removedPatterns: [], + returnedRowCountBeforeDenoise: rows.length, + filteredRowCount: rows.length, + skipped: 'denoise_failed', + }, + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; + } + + // Replace rows in the result with denoised rows and add metadata. + // Always emit a `denoised` block when denoise=true so callers can + // distinguish "no noisy patterns" from "denoise was not requested". + const denoisedResult = { + ...resultData, + data: denoised.rows, + }; + const { data: trimmedResult, isTrimmed } = + trimToolResponse(denoisedResult); + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedResult, + denoised: { + removedPatterns: denoised.removedPatterns, + // rows.length is the count returned by runConfigTile + // (already subject to maxResults and trim limits). + returnedRowCountBeforeDenoise: rows.length, + filteredRowCount: denoised.rows.length, + ...(denoised.skipped ? { skipped: denoised.skipped } : {}), + }, + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + ...(denoised.rows.length === 0 && !denoised.skipped + ? { + hint: 'All events matched noisy patterns and were removed. Try narrowing filters or disabling denoise to see all events.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; }), ); } diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 54106c3210..23f5877a6a 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -30,6 +30,10 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { + DENOISE_NOISE_THRESHOLD, + DENOISE_SAMPLE_SIZE, +} from '@hyperdx/common-utils/dist/drain'; import { BuilderChartConfigWithDateRange, SelectList, @@ -1716,7 +1720,7 @@ function DBSqlRowTableComponent({ const patternColumn = columns[columns.length - 1]; const groupedPatterns = useGroupedPatterns({ config, - samples: 10_000, + samples: DENOISE_SAMPLE_SIZE, bodyValueExpression: patternColumn ?? '', severityTextExpression: (source?.kind === SourceKind.Log @@ -1729,7 +1733,9 @@ function DBSqlRowTableComponent({ queryKey: ['noisy-patterns', config], queryFn: async () => { return Object.values(groupedPatterns.data).filter( - p => p.count / (groupedPatterns.sampledRowCount ?? 1) > 0.1, + p => + p.count / (groupedPatterns.sampledRowCount ?? 1) > + DENOISE_NOISE_THRESHOLD, ); }, enabled: diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index 937aaa82da..f735377f06 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -10,7 +10,12 @@ export type { PatternGroup, TrendBucket, } from './mine-patterns'; -export { flattenBody, minePatterns } from './mine-patterns'; +export { + DENOISE_NOISE_THRESHOLD, + DENOISE_SAMPLE_SIZE, + flattenBody, + minePatterns, +} from './mine-patterns'; export { Node } from './node'; export type { AddLogMessageResult, ExtractedParameter } from './template-miner'; export { TemplateMiner } from './template-miner'; diff --git a/packages/common-utils/src/drain/mine-patterns.ts b/packages/common-utils/src/drain/mine-patterns.ts index 82c4abd9ce..05583db7fd 100644 --- a/packages/common-utils/src/drain/mine-patterns.ts +++ b/packages/common-utils/src/drain/mine-patterns.ts @@ -6,6 +6,15 @@ import { import { TemplateMinerConfig } from './config'; import { TemplateMiner } from './template-miner'; +// ─── Denoise constants ─────────────────────────────────────────────────────── +// Shared between the web app and MCP server denoise paths. + +/** Number of random rows to sample for pattern learning. */ +export const DENOISE_SAMPLE_SIZE = 10_000; + +/** Patterns matching more than this fraction of sampled events are "noisy". */ +export const DENOISE_NOISE_THRESHOLD = 0.1; + // ─── Body normalization ────────────────────────────────────────────────────── /** Collapse newlines and runs of whitespace into single spaces. */ From 5bd1c681e94c0c97b92de433b59729fb78becb06 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 9 Jun 2026 14:26:41 -0600 Subject: [PATCH 037/728] feat: AI eval framework for benchmarking MCP servers (#2414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds `packages/hdx-eval` — an eval framework for benchmarking AI agents against observability MCP servers. The framework generates deterministic synthetic telemetry with planted anomalies, spawns Claude Code as an SRE agent, records full trajectories, and grades answers using programmatic checks + LLM-as-judge. ## Key Features - **MCP-agnostic** — compare any combination of MCPs (HyperDX vs ClickHouse, feature-branch vs main, two HyperDX instances, or any N-way comparison) - **5 scenarios** covering error root-cause, latency spikes, noisy signal triage, segmented regression, and service health checks - **Deterministic seeding** — mulberry32 PRNG produces byte-identical data for fair comparison - **Blinded LLM judging** — brand terms and tool names are redacted so the judge cannot tell which MCP produced the answer - **Baseline + challengers** reporting model with delta columns - **Web viewer** for browsing comparison dashboards, per-scenario breakdowns, and individual run trajectories - **Dual-slot setup** for A/B comparison of two HyperDX branches running simultaneously ## What is included - `packages/hdx-eval/` — full eval package (CLI, harness, generators, grading, reports, viewer) - `packages/hdx-eval/README.md` — comprehensive docs covering setup, config, scenarios, CLI reference, and scoring - `.opencode/commands/eval-summary.md` — eval analysis skill for reviewing results - `AGENTS.md` — minor addition documenting common utility locations to check before writing new functions ## Usage ```bash yarn workspace @hyperdx/hdx-eval dev setup-hyperdx yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --volume-factor 0.1 yarn workspace @hyperdx/hdx-eval dev run error-root-cause yarn workspace @hyperdx/hdx-eval viewer ``` See `packages/hdx-eval/README.md` for full documentation. --- .changeset/add-hdx-eval-framework.md | 15 + .opencode/commands/eval-summary.md | 157 +++ AGENTS.md | 6 +- knip.json | 4 + packages/hdx-eval/.gitignore | 2 + packages/hdx-eval/README.md | 371 ++++++ packages/hdx-eval/eslint.config.mjs | 60 + packages/hdx-eval/eval.config.example.json | 76 ++ packages/hdx-eval/jest.config.js | 15 + packages/hdx-eval/package.json | 42 + packages/hdx-eval/scripts/env.sh | 20 + .../hdx-eval/scripts/viewer/public/app.js | 1104 ++++++++++++++++ .../hdx-eval/scripts/viewer/public/index.html | 70 + .../hdx-eval/scripts/viewer/public/style.css | 679 ++++++++++ packages/hdx-eval/scripts/viewer/server.js | 183 +++ .../hdx-eval/src/__tests__/aggregate.test.ts | 264 ++++ packages/hdx-eval/src/__tests__/blind.test.ts | 102 ++ .../src/__tests__/error-root-cause.test.ts | 228 ++++ packages/hdx-eval/src/__tests__/grade.test.ts | 169 +++ .../src/__tests__/judgePrompt.test.ts | 62 + .../src/__tests__/latency-spike.test.ts | 200 +++ .../hdx-eval/src/__tests__/markdown.test.ts | 116 ++ .../hdx-eval/src/__tests__/mcpConfig.test.ts | 189 +++ .../src/__tests__/noisy-signals.test.ts | 273 ++++ .../src/__tests__/programmatic.test.ts | 140 ++ .../hdx-eval/src/__tests__/runRun.test.ts | 138 ++ .../hdx-eval/src/__tests__/seeded.test.ts | 66 + .../__tests__/segmented-regression.test.ts | 157 +++ .../__tests__/service-health-check.test.ts | 208 +++ .../src/__tests__/streamParser.test.ts | 127 ++ .../src/__tests__/systemPrompt.test.ts | 40 + packages/hdx-eval/src/cli.ts | 894 +++++++++++++ packages/hdx-eval/src/clickhouse/client.ts | 35 + packages/hdx-eval/src/clickhouse/insert.ts | 106 ++ packages/hdx-eval/src/clickhouse/schema.ts | 368 ++++++ packages/hdx-eval/src/generators/logs.ts | 30 + packages/hdx-eval/src/generators/templates.ts | 1162 +++++++++++++++++ packages/hdx-eval/src/generators/traces.ts | 50 + packages/hdx-eval/src/generators/types.ts | 39 + packages/hdx-eval/src/grading/blind.ts | 63 + packages/hdx-eval/src/grading/grade.ts | 268 ++++ packages/hdx-eval/src/grading/judge.ts | 259 ++++ packages/hdx-eval/src/grading/judgePrompt.ts | 77 ++ packages/hdx-eval/src/grading/programmatic.ts | 42 + packages/hdx-eval/src/grading/rubric.ts | 96 ++ packages/hdx-eval/src/grading/types.ts | 100 ++ packages/hdx-eval/src/harness/claudeSpawn.ts | 185 +++ packages/hdx-eval/src/harness/mcpConfig.ts | 45 + packages/hdx-eval/src/harness/runRun.ts | 235 ++++ packages/hdx-eval/src/harness/settingsFile.ts | 79 ++ packages/hdx-eval/src/harness/streamParser.ts | 156 +++ packages/hdx-eval/src/harness/systemPrompt.ts | 129 ++ packages/hdx-eval/src/harness/types.ts | 94 ++ packages/hdx-eval/src/hyperdx/api.ts | 203 +++ packages/hdx-eval/src/hyperdx/config.ts | 185 +++ packages/hdx-eval/src/hyperdx/setup.ts | 374 ++++++ packages/hdx-eval/src/index.ts | 10 + packages/hdx-eval/src/reports/aggregate.ts | 235 ++++ packages/hdx-eval/src/reports/markdown.ts | 264 ++++ packages/hdx-eval/src/reports/store.ts | 60 + packages/hdx-eval/src/rng/seeded.ts | 57 + packages/hdx-eval/src/runs/instrument.ts | 327 +++++ packages/hdx-eval/src/runs/path.ts | 26 + packages/hdx-eval/src/runs/store.ts | 64 + .../scenarios/error-root-cause/generate.ts | 736 +++++++++++ .../error-root-cause/ground-truth.json | 173 +++ packages/hdx-eval/src/scenarios/index.ts | 26 + .../src/scenarios/latency-spike/generate.ts | 554 ++++++++ .../scenarios/latency-spike/ground-truth.json | 136 ++ .../src/scenarios/noisy-signals/generate.ts | 415 ++++++ .../scenarios/noisy-signals/ground-truth.json | 247 ++++ .../hdx-eval/src/scenarios/seedScenario.ts | 58 + .../segmented-regression/generate.ts | 299 +++++ .../segmented-regression/ground-truth.json | 121 ++ .../service-health-check/generate.ts | 431 ++++++ .../service-health-check/ground-truth.json | 164 +++ packages/hdx-eval/src/scenarios/types.ts | 47 + packages/hdx-eval/tsconfig.build.json | 4 + packages/hdx-eval/tsconfig.json | 23 + yarn.lock | 62 + 80 files changed, 15095 insertions(+), 1 deletion(-) create mode 100644 .changeset/add-hdx-eval-framework.md create mode 100644 .opencode/commands/eval-summary.md create mode 100644 packages/hdx-eval/.gitignore create mode 100644 packages/hdx-eval/README.md create mode 100644 packages/hdx-eval/eslint.config.mjs create mode 100644 packages/hdx-eval/eval.config.example.json create mode 100644 packages/hdx-eval/jest.config.js create mode 100644 packages/hdx-eval/package.json create mode 100755 packages/hdx-eval/scripts/env.sh create mode 100644 packages/hdx-eval/scripts/viewer/public/app.js create mode 100644 packages/hdx-eval/scripts/viewer/public/index.html create mode 100644 packages/hdx-eval/scripts/viewer/public/style.css create mode 100644 packages/hdx-eval/scripts/viewer/server.js create mode 100644 packages/hdx-eval/src/__tests__/aggregate.test.ts create mode 100644 packages/hdx-eval/src/__tests__/blind.test.ts create mode 100644 packages/hdx-eval/src/__tests__/error-root-cause.test.ts create mode 100644 packages/hdx-eval/src/__tests__/grade.test.ts create mode 100644 packages/hdx-eval/src/__tests__/judgePrompt.test.ts create mode 100644 packages/hdx-eval/src/__tests__/latency-spike.test.ts create mode 100644 packages/hdx-eval/src/__tests__/markdown.test.ts create mode 100644 packages/hdx-eval/src/__tests__/mcpConfig.test.ts create mode 100644 packages/hdx-eval/src/__tests__/noisy-signals.test.ts create mode 100644 packages/hdx-eval/src/__tests__/programmatic.test.ts create mode 100644 packages/hdx-eval/src/__tests__/runRun.test.ts create mode 100644 packages/hdx-eval/src/__tests__/seeded.test.ts create mode 100644 packages/hdx-eval/src/__tests__/segmented-regression.test.ts create mode 100644 packages/hdx-eval/src/__tests__/service-health-check.test.ts create mode 100644 packages/hdx-eval/src/__tests__/streamParser.test.ts create mode 100644 packages/hdx-eval/src/__tests__/systemPrompt.test.ts create mode 100644 packages/hdx-eval/src/cli.ts create mode 100644 packages/hdx-eval/src/clickhouse/client.ts create mode 100644 packages/hdx-eval/src/clickhouse/insert.ts create mode 100644 packages/hdx-eval/src/clickhouse/schema.ts create mode 100644 packages/hdx-eval/src/generators/logs.ts create mode 100644 packages/hdx-eval/src/generators/templates.ts create mode 100644 packages/hdx-eval/src/generators/traces.ts create mode 100644 packages/hdx-eval/src/generators/types.ts create mode 100644 packages/hdx-eval/src/grading/blind.ts create mode 100644 packages/hdx-eval/src/grading/grade.ts create mode 100644 packages/hdx-eval/src/grading/judge.ts create mode 100644 packages/hdx-eval/src/grading/judgePrompt.ts create mode 100644 packages/hdx-eval/src/grading/programmatic.ts create mode 100644 packages/hdx-eval/src/grading/rubric.ts create mode 100644 packages/hdx-eval/src/grading/types.ts create mode 100644 packages/hdx-eval/src/harness/claudeSpawn.ts create mode 100644 packages/hdx-eval/src/harness/mcpConfig.ts create mode 100644 packages/hdx-eval/src/harness/runRun.ts create mode 100644 packages/hdx-eval/src/harness/settingsFile.ts create mode 100644 packages/hdx-eval/src/harness/streamParser.ts create mode 100644 packages/hdx-eval/src/harness/systemPrompt.ts create mode 100644 packages/hdx-eval/src/harness/types.ts create mode 100644 packages/hdx-eval/src/hyperdx/api.ts create mode 100644 packages/hdx-eval/src/hyperdx/config.ts create mode 100644 packages/hdx-eval/src/hyperdx/setup.ts create mode 100644 packages/hdx-eval/src/index.ts create mode 100644 packages/hdx-eval/src/reports/aggregate.ts create mode 100644 packages/hdx-eval/src/reports/markdown.ts create mode 100644 packages/hdx-eval/src/reports/store.ts create mode 100644 packages/hdx-eval/src/rng/seeded.ts create mode 100644 packages/hdx-eval/src/runs/instrument.ts create mode 100644 packages/hdx-eval/src/runs/path.ts create mode 100644 packages/hdx-eval/src/runs/store.ts create mode 100644 packages/hdx-eval/src/scenarios/error-root-cause/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/index.ts create mode 100644 packages/hdx-eval/src/scenarios/latency-spike/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/noisy-signals/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/seedScenario.ts create mode 100644 packages/hdx-eval/src/scenarios/segmented-regression/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/service-health-check/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/types.ts create mode 100644 packages/hdx-eval/tsconfig.build.json create mode 100644 packages/hdx-eval/tsconfig.json diff --git a/.changeset/add-hdx-eval-framework.md b/.changeset/add-hdx-eval-framework.md new file mode 100644 index 0000000000..9d7f75966b --- /dev/null +++ b/.changeset/add-hdx-eval-framework.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/hdx-eval': minor +--- + +feat: add AI eval framework for benchmarking MCP servers + +New `@hyperdx/hdx-eval` package for benchmarking AI agents against +observability MCP servers. Generates deterministic synthetic telemetry +with planted anomalies, spawns Claude Code as an SRE agent, records full +trajectories, and grades answers using programmatic checks and an +LLM-as-judge. + +Includes 5 scenarios (error-root-cause, latency-spike, noisy-signals, +segmented-regression, service-health-check), MCP-agnostic N-way +comparison, blinded judging, and a web viewer for browsing results. diff --git a/.opencode/commands/eval-summary.md b/.opencode/commands/eval-summary.md new file mode 100644 index 0000000000..5f2dfede07 --- /dev/null +++ b/.opencode/commands/eval-summary.md @@ -0,0 +1,157 @@ +--- +description: Analyze HyperDX eval benchmark results and produce a summary report +--- + +You are an eval benchmark analyst for HyperDX. Your job is to review eval run +results and produce a structured summary comparing HyperDX MCP performance +against the ClickHouse baseline MCP. + +## Input + +The user will provide either: + +- Raw eval output (pasted terminal logs from running the eval harness) +- A batch directory path under `packages/hdx-eval/runs/` +- A `$ARGUMENTS` referencing a specific batch timestamp (e.g. + `2026-05-21T16-34-35-417Z`) + +If the user provides a batch timestamp or directory path, locate the run files +under `packages/hdx-eval/runs//`. If they paste terminal output, +extract the batch path from the "Results written under" line. + +If no batch is specified, list `packages/hdx-eval/runs/` and use the **most +recent** batch directory (sorted by timestamp). + +## Analysis Steps + +Perform these steps in order. Use the Task tool with parallel subagents where +possible to speed up analysis. + +### 1. Read Summary Files + +Read `_summary.md` and `_summary.json` from the batch directory. These contain +aggregated metrics across all runs. + +### 2. Analyze Run Transcripts + +Use **parallel Task subagents** — one per MCP (hyperdx, clickhouse). Each +subagent reads all run JSON files for its MCP +(`//0.json`, `1.json`, `2.json`, etc.) and extracts: + +- **Tool call sequence** — ordered list of tool names called with index numbers +- **Tool errors** — any failed tool calls, including error messages and index +- **Backtracking patterns** — sequences where the model retried, searched for + unavailable tools, or re-ran queries with different parameters +- **Final answer** — whether the run produced one (final_answer vs max_turns) +- **Wasted tool calls** — calls that didn't contribute to the final answer (e.g. + ToolSearch for tools that don't exist in the sandbox, queries on empty tables, + dead-end hypothesis investigations) +- **Turn budget usage** — how many calls were used vs budget, and whether the + model showed awareness of remaining turns +- **Interesting observations** — anything unusual, surprising, or noteworthy + about the run (clever investigation strategies, unexpected tool behavior, + novel failure modes, etc.) + +### 3. Read Grade Files and Ground Truth + +Use a **single Task subagent** to read all `*.grade.json` files, the scenario's +`ground-truth.json` from +`packages/hdx-eval/src/scenarios//ground-truth.json`, and the system +prompt at `packages/hdx-eval/src/harness/systemPrompt.ts`. Extract: + +- Per-run combined, programmatic, and judge scores +- Per-check pass/fail with weights +- Judge reasoning/rationale per criterion +- Ground truth expected answers, checks, and distractors +- System prompt text and any variant-specific guidance + +## Default Output + +By default, produce the **Quick Stats** output below. Then prompt the user +asking if they want to see more. + +### Quick Stats + +Four bullets covering: + +- **Accuracy** — combined score comparison with relative percentage +- **Consistency** — completion rate (final_answer vs max_turns) and score + variance across runs +- **Speed** — wall clock, tool calls, and token usage with relative percentages +- **Overall** — one-line headline with the key deltas + +Then a **Notable Findings** section: 3-5 short bullets highlighting the most +interesting or surprising things you found in the transcripts. These should be +specific, concrete observations — not generic patterns. Examples: + +- "HyperDX Run 1 scored 79.8% despite 2 SQL errors — it recovered in 2 calls" +- "ClickHouse Run 2 was the single best run (89.8%) but the other 2 CH runs + both hit max_turns" +- "The feature_flag distractor tripped HyperDX Run 1 because event_deltas + surfaced it as a top-changing attribute" + +Finally, end with: + +> Want to dig deeper? I can show: +> - **Detailed stats** — full per-run breakdown, per-check wins, and failure +> analysis +> - **Recommendations** — specific tool/prompt changes to improve HyperDX scores +> - **Both** + +### Detailed Stats (on request) + +When the user asks for detailed stats, produce: + +#### Section 1: Top-Line Stats + +A table of metrics where HyperDX beat ClickHouse. For each, compute the +percentage improvement (relative, not absolute). Focus on: + +- Combined score delta +- Completion rate (final_answer vs max_turns) +- Programmatic accuracy delta +- Judge criteria where HyperDX scored notably higher +- Tool call efficiency (fewer calls = better) + +Format as a markdown table with columns: Metric, HyperDX, ClickHouse, Delta, +Relative. + +#### Section 2: Per-Check Wins + +A table of the programmatic checks where HyperDX outperformed ClickHouse by the +largest margin. Include both positive checks (identified the right pattern) and +negative checks (avoided false positives). Only include checks with a gap of +20pp or more. + +For each check with a large gap, add a brief explanation of **why** the gap +exists. + +#### Section 3: Failure Analysis + +For any run (either MCP) that scored below 50% combined or hit max_turns, +explain what went wrong. Reference specific tool call sequences and errors. +Contrast with how the other MCP handled the same challenge. + +### Recommendations (on request) + +When the user asks for recommendations, analyze the run transcripts and produce +concrete improvement opportunities for the HyperDX MCP tools or system prompt. +For each opportunity: + +1. **Problem** — What happened in the runs (cite specific tool call sequences) +2. **Impact** — How many tool calls / seconds / score points this costs +3. **Fix** — A specific, actionable change to the tool, system prompt, or eval + harness + +Do NOT use a pre-built checklist of patterns to look for. Derive all findings +directly from the transcript data for this specific batch. Each recommendation +should be grounded in something that actually happened in the runs. + +## Guidelines + +- Be precise with numbers. Always compute relative deltas (not just absolute). +- Cite specific run IDs and tool call numbers when describing patterns. +- Focus on actionable insights, not just observations. +- If the eval only has one scenario, that's fine — analyze it thoroughly. If + there are multiple scenarios, analyze each separately and then synthesize. +- Keep the tone technical and direct. No filler. diff --git a/AGENTS.md b/AGENTS.md index 70dbe826a2..8cb3ebe993 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ schema-agnostic design, and correlation across all telemetry types in one place. ## Architecture (WHAT) -This is a **monorepo** with five packages: +This is a **monorepo** with six packages: - `packages/app` - Next.js frontend (TypeScript, Mantine UI, TanStack Query) - `packages/api` - Express backend (Node.js 22+, MongoDB for metadata, @@ -26,6 +26,10 @@ This is a **monorepo** with five packages: - `packages/otel-collector` - Custom-built OpenTelemetry Collector (Go, OCB). See its [`README.md`](packages/otel-collector/README.md) for architecture, included components, and upgrade procedures. +- `packages/hdx-eval` - AI eval framework for benchmarking MCP servers against + observability scenarios. Generates deterministic synthetic telemetry, spawns + agents, and grades with programmatic checks + LLM-as-judge. See its + [`README.md`](packages/hdx-eval/README.md) for setup and usage. **Data flow**: Apps → OpenTelemetry Collector → ClickHouse (telemetry data) / MongoDB (configuration/metadata) diff --git a/knip.json b/knip.json index 7868b33e02..ed30a02bb6 100644 --- a/knip.json +++ b/knip.json @@ -26,6 +26,10 @@ "packages/common-utils": { "entry": ["src/**/*.ts", "!src/__tests__/**", "!src/**/*.test.*"], "project": ["src/**/*.ts"] + }, + "packages/hdx-eval": { + "project": ["src/**/*.ts"], + "ignoreDependencies": ["tsx"] } }, "ignore": ["scripts/dev-portal/**", ".github/scripts/**"], diff --git a/packages/hdx-eval/.gitignore b/packages/hdx-eval/.gitignore new file mode 100644 index 0000000000..423f3d8fde --- /dev/null +++ b/packages/hdx-eval/.gitignore @@ -0,0 +1,2 @@ +/runs/ +eval.config.json diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md new file mode 100644 index 0000000000..8edd9b5d59 --- /dev/null +++ b/packages/hdx-eval/README.md @@ -0,0 +1,371 @@ +# @hyperdx/hdx-eval + +Eval framework for benchmarking AI agents against observability MCP servers. +Generates deterministic synthetic telemetry (traces + logs) with planted +anomalies, spawns Claude Code as an SRE agent with access to any configured +MCP, records full trajectories, and grades answers using programmatic checks and +an LLM-as-judge. The framework is **MCP-agnostic** — compare any combination of +MCPs: HyperDX vs ClickHouse, feature-branch vs main, two different HyperDX +instances, or any N-way comparison. MCPs are defined in a config registry and +selected at run time. + +## Prerequisites + +- **`yarn dev` running** on at least one worktree (ClickHouse, MongoDB, API) +- **`claude` CLI installed** (the harness spawns it in streaming JSON mode) +- **`ANTHROPIC_API_KEY`** or **`AI_API_KEY`** in `.env.local` at the monorepo + root (for `run` and `grade` commands) +- **[`uv`](https://docs.astral.sh/uv/)** installed (only needed if you + configure a `stdio`-type ClickHouse MCP — it launches via + `uv run --with mcp-clickhouse`) + +## Quick Start + +All eval commands derive ClickHouse and API ports from the `HDX_DEV_SLOT` +environment variable. If your dev stack is running in a different worktree, +the auto-detected slot won't match. Set `HDX_DEV_SLOT` to the slot your dev +stack is using — you can find this in the banner printed by `yarn dev`. + +```bash +# Set the slot for your session (must match the running dev stack) +export HDX_DEV_SLOT=80 + +# 1. One-time setup: register eval account + create Sources + write config +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx + +# 2. Seed a scenario (use --volume-factor 0.1 for faster iteration) +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --volume-factor 0.1 + +# 3. Run the eval (all enabled MCPs, 3 runs each) +# First run saves an anchorTime to eval.config.json automatically. +yarn workspace @hyperdx/hdx-eval dev run error-root-cause + +# 4. Re-run later — reuses saved anchor, skips reseed +yarn workspace @hyperdx/hdx-eval dev run error-root-cause + +# 5. Grade + report (auto-runs after step 3 by default) +yarn workspace @hyperdx/hdx-eval dev grade +yarn workspace @hyperdx/hdx-eval dev report + +# 6. Browse results in the web viewer +yarn workspace @hyperdx/hdx-eval viewer +``` + +## Recommended: Dual-Slot Eval Setup + +For reliable A/B comparison of two HyperDX branches (e.g. feature vs main), +we recommend reserving two fixed dev slots. This gives you two fully isolated +HyperDX stacks (separate ClickHouse, MongoDB, API) that can run +simultaneously. + +### 1. Start two dev stacks with fixed slots + +```bash +# Slot 98: "baseline" — run from your main branch checkout +HDX_DEV_SLOT=98 yarn dev + +# Slot 99: "candidate" — run from your feature branch checkout +HDX_DEV_SLOT=99 yarn dev +``` + +This gives you: + +| | Slot 98 (baseline) | Slot 99 (candidate) | +|---|---|---| +| API | `http://localhost:30198` | `http://localhost:30199` | +| App | `http://localhost:30298` | `http://localhost:30299` | +| ClickHouse | `http://localhost:30598` | `http://localhost:30599` | +| MCP | `http://localhost:30198/mcp` | `http://localhost:30199/mcp` | + +### 2. Run `setup-hyperdx` against each instance + +```bash +# Setup baseline (slot 98) +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --api-url http://localhost:30198 + +# Setup candidate (slot 99) +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --api-url http://localhost:30199 +``` + +After setup, `eval.config.json` will have the `hyperdx` MCP pointing at the +slot it was set up against. You can then manually add the second MCP entry +(see [MCP Configuration](#mcp-configuration) below). + +### 3. Seed both ClickHouse instances + +Both instances need the eval data. Use `HDX_DEV_SLOT` to target each: + +```bash +HDX_DEV_SLOT=98 yarn workspace @hyperdx/hdx-eval dev seed error-root-cause +HDX_DEV_SLOT=99 yarn workspace @hyperdx/hdx-eval dev seed error-root-cause +``` + +### 4. Run the comparison + +```bash +yarn workspace @hyperdx/hdx-eval dev run error-root-cause \ + --mcp hyperdx-main,hyperdx-feature \ + --baseline hyperdx-main \ + --runs 5 +``` + +## MCP Configuration + +MCPs are defined in `eval.config.json` under the `mcps` key. Each entry +fully specifies how to connect, which tools to allow/deny, and how to +anonymize the MCP identity for fair LLM judging. + +### Config structure + +```json +{ + "mcps": { + "": { + "type": "http" | "stdio", + "url": "...", // http only + "headers": { ... }, // http only + "command": "...", // stdio only + "args": ["..."], // stdio only + "env": { ... }, // stdio only + "toolPattern": "mcp____*", + "label": "Display Name", + "brandTerms": ["Brand", "brand"], + "deniedTools": ["mcp____tool_to_hide"], + "enabled": true + } + }, + "scenarios": { ... }, + "hyperdxApi": { ... }, + "clickhouse": { ... } +} +``` + +### Field reference + +| Field | Required | Description | +|---|---|---| +| `type` | yes | `"http"` for HTTP/SSE transport, `"stdio"` for subprocess | +| `url` | http | MCP server URL | +| `headers` | no | HTTP headers (e.g. `Authorization: Bearer ...`) | +| `command` | stdio | Executable to spawn | +| `args` | no | Command arguments | +| `env` | no | Environment variables for the subprocess | +| `toolPattern` | yes | Glob for allowed tools (e.g. `mcp__myname__*`) | +| `label` | yes | Human-readable name for reports | +| `brandTerms` | no | Terms to redact when blinding answers for the LLM judge | +| `deniedTools` | no | Tool names to deny (reduces tool count for faster schema loading) | +| `enabled` | no | `false` to exclude from `--mcp all` (default: `true`). Can still be explicitly named with `--mcp name`. | + +### Top-level config sections + +| Section | Required | Purpose | +|---|---|---| +| `mcps` | yes | MCP registry (see above) | +| `scenarios` | no | Per-scenario HyperDX Source IDs (populated by `setup-hyperdx`) | +| `hyperdxApi` | no | HyperDX API URL + access key (used only by `setup-hyperdx`) | +| `clickhouse` | no | ClickHouse connection for `seed`, `drop`, `runs-instrument` commands | +| `anchorTime` | no | Fixed "now" anchor (ISO 8601). Auto-generated and saved on first `run`. See [Anchor Time](#anchor-time). | + +### Example: HTTP MCP (HyperDX) + +```json +{ + "type": "http", + "url": "http://localhost:30198/mcp", + "headers": { "Authorization": "Bearer " }, + "toolPattern": "mcp__hyperdx__*", + "label": "HyperDX", + "brandTerms": ["HyperDX", "hyperdx"], + "deniedTools": [ + "mcp__hyperdx__hyperdx_delete_dashboard", + "mcp__hyperdx__hyperdx_get_dashboard" + ] +} +``` + +### Example: stdio MCP (raw ClickHouse) + +```json +{ + "type": "stdio", + "command": "uv", + "args": ["run", "--with", "mcp-clickhouse==0.3.0", "--python", "3.10", "mcp-clickhouse"], + "env": { + "CLICKHOUSE_HOST": "localhost", + "CLICKHOUSE_PORT": "30598", + "CLICKHOUSE_USER": "default", + "CLICKHOUSE_PASSWORD": "" + }, + "toolPattern": "mcp__clickhouse__*", + "label": "ClickHouse MCP", + "brandTerms": ["ClickHouse MCP", "clickhouse"] +} +``` + +## Scenarios + +Each scenario writes into its own ClickHouse tables +(`eval__otel_traces` / `eval__otel_logs`) whose schema +mirrors HyperDX's `otel_traces` / `otel_logs` exactly. + +| Scenario | Agent Prompt | What's Planted | +|---|---|---| +| `error-root-cause` | "Checkout requests are failing..." | `payment-service` DB timeout cascading into `checkout-api` 5xx. 6M+ spans, 12M logs, 5 distractors. | +| `latency-spike` | "p99 latency on api-server has jumped..." | `/api/orders/search` p99 spike for enterprise tenants. 12M+ spans, 5 regions, 5 distractors. | +| `noisy-signals` | "We want to cut log ingest cost..." | ~16M logs across composite cells. Each noisy cell has a load-bearing pattern mixed in. | +| `segmented-regression` | "API error rate has gone up..." | Enterprise x cache-miss errors at ~12%. Single-axis aggregates dilute the signal. | +| `service-health-check` | "Generate a service health report..." | Peace-time report — no incident, but 4 novel signals to call out. | + +Use `--volume-factor` to scale background row counts for faster iteration. +Planted anomaly counts stay fixed. + +```bash +# Full volume (default) — realistic needle-in-haystack difficulty +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause + +# 10% volume — much faster, still structurally valid +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --volume-factor 0.1 +``` + +## CLI Reference + +All commands are run via `yarn workspace @hyperdx/hdx-eval dev `. + +### Setup + +```bash +dev setup-hyperdx # one-time: register + create Sources +dev setup-hyperdx --api-url http://... # explicit API URL +dev setup-hyperdx --check # validate existing config +dev setup-hyperdx --reset-sources # recreate all Sources +``` + +### Data Management + +```bash +dev list # list scenarios + tables +dev seed # seed with defaults (seed=42, now=Date.now()) +dev seed --seed 7 # custom PRNG seed +dev seed --now 2026-05-08T15:00:00Z # fixed anchor time +dev seed --volume-factor 0.1 # 10% background volume +dev seed --ch-url http://localhost:30599 # target specific CH +dev drop # drop scenario tables +``` + +### Running Evals + +```bash +dev run # all enabled MCPs, 3 runs each +dev run --mcp hyperdx # single MCP +dev run --mcp hyperdx-main,hyperdx-feature # compare two +dev run --mcp all # all enabled MCPs (default) +dev run --baseline hyperdx-main # set baseline for delta reports +dev run --runs 5 # 5 runs per cell +dev run --model claude-opus-4-6 +dev run --max-turns 20 # default: 15 +dev run --concurrency 4 # parallel cells +dev run --prompt-variant hypothesis # hypothesis-first variant +dev run --reseed # re-seed data before running +dev run --anchor-time 2026-05-08T15:00:00Z # override + save anchor +dev run --live # wall-clock now, no anchor, forces reseed +dev run --no-grade # skip auto-grading +dev run --no-judge # programmatic checks only (faster) +``` + +### Inspecting Results + +```bash +dev runs-list # list recorded batches +dev runs-show # summary of a run +dev runs-show --queries # every tool call + SQL +dev runs-show --final-answer # print the agent's answer +dev runs-instrument # enrich with ClickHouse query_log timings +``` + +### Grading & Reporting + +```bash +dev grade # programmatic + LLM-as-judge +dev grade --judge-model claude-sonnet-4-6 +dev grade --no-judge # programmatic checks only +dev grade --rerun-judge # re-run judge even if grades exist +dev report # render _summary.md + _summary.json +dev report --baseline hyperdx-main # override baseline for deltas +``` + +### Viewer + +```bash +yarn workspace @hyperdx/hdx-eval viewer # web UI at http://localhost:5176 +``` + +The viewer shows a **comparison dashboard** when you select a batch: + +- **Top-line verdict table** — combined scores per MCP with delta columns +- **Per-scenario breakdown** — metrics, judge criterion heatmap, programmatic + check pass rates +- **Side-by-side runs** — click any run to drill into the full trajectory + +## Scoring + +Combined score = `0.4 * programmatic + 0.6 * judge - toolErrorPenalty` + +- **Programmatic (40%)** — regex-based keyword checks against the final + answer, defined per-scenario in `ground-truth.json`. Includes both positive + checks (must find the right thing) and negative checks (must not blame a + distractor). +- **LLM-as-judge (60%)** — multi-criteria rubric scored 0-5 by Claude Opus + (configurable). Criteria are scenario-specific. Answers are **blinded** — + MCP tool names and brand terms are redacted so the judge can't tell which + MCP produced the answer. +- **Tool error penalty** — up to 20% deducted for high tool-call error rates. + +### Baseline + Challengers + +Reports use a **baseline + challengers** model. One MCP is designated as the +baseline (default: first MCP alphabetically, or set explicitly with +`--baseline`). All other MCPs show deltas relative to it: + +``` +| Metric | hyperdx-main | hyperdx-feature | Δ (hyperdx-feature) | +|----------------|-------------|-----------------|---------------------| +| Combined score | 85% | 91% | +6% | +``` + +## Determinism + +All randomness flows through a `mulberry32(seed)` PRNG. Same `(seed, now)` +produces byte-identical row content. This makes side-by-side comparison +meaningful — both MCPs query the exact same data. + +### Anchor Time + +The anchor time is the "now" reference for both seeded data timestamps and +the agent's system prompt. It is persisted in `eval.config.json` under +`anchorTime` so that subsequent runs automatically reuse the same value. + +- **First run**: if `anchorTime` is not set in the config, the current + wall-clock time is saved and used. If the scenario tables are empty or + missing, data is auto-seeded — no separate `seed` step required. +- **Subsequent runs**: the saved `anchorTime` is read from config. Data + is verified to exist before running; if it's missing (e.g. after a + `drop`), it's auto-seeded again. +- **`--anchor-time `**: override the saved value with a specific + timestamp. The new value is saved to config for future runs. Pair with + `--reseed` if the data needs to be regenerated to match. +- **`--live`**: ignore the saved anchor and use wall-clock time. The agent + does NOT receive a `FIXED CURRENT TIME` system prompt block. Implies + `--reseed` since the data must match the current time. +- **`--reseed`**: force re-seed even when data already exists (e.g. after + changing the seed value or anchor time). + +## Tests + +```bash +yarn workspace @hyperdx/hdx-eval ci:unit # all unit tests +yarn workspace @hyperdx/hdx-eval dev:unit # watch mode +``` + +Unit tests verify PRNG determinism, planted anomaly invariants, grading +pipeline, stream parsing, prompt construction, blinding, and report +aggregation. No ClickHouse needed. diff --git a/packages/hdx-eval/eslint.config.mjs b/packages/hdx-eval/eslint.config.mjs new file mode 100644 index 0000000000..c04226006b --- /dev/null +++ b/packages/hdx-eval/eslint.config.mjs @@ -0,0 +1,60 @@ +import js from '@eslint/js'; +import tseslint from 'typescript-eslint'; +import prettierConfig from 'eslint-config-prettier'; +import prettierPlugin from 'eslint-plugin-prettier'; +import simpleImportSort from 'eslint-plugin-simple-import-sort'; + +export default [ + js.configs.recommended, + ...tseslint.configs.recommended, + prettierConfig, + { + ignores: [ + 'node_modules/**', + 'dist/**', + 'coverage/**', + '**/*.config.mjs', + '**/*.config.js', + 'jest.config.js', + 'scripts/viewer/**', + ], + }, + { + files: ['src/**/*.ts'], + plugins: { + '@typescript-eslint': tseslint.plugin, + 'simple-import-sort': simpleImportSort, + prettier: prettierPlugin, + }, + rules: { + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unused-vars': [ + 'warn', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + }, + ], + 'prettier/prettier': 'error', + 'simple-import-sort/imports': 'error', + 'simple-import-sort/exports': 'error', + }, + languageOptions: { + parser: tseslint.parser, + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + project: './tsconfig.json', + tsconfigRootDir: import.meta.dirname, + }, + globals: { + console: 'readonly', + process: 'readonly', + setTimeout: 'readonly', + clearTimeout: 'readonly', + Buffer: 'readonly', + URL: 'readonly', + }, + }, + }, +]; diff --git a/packages/hdx-eval/eval.config.example.json b/packages/hdx-eval/eval.config.example.json new file mode 100644 index 0000000000..b71d7a3448 --- /dev/null +++ b/packages/hdx-eval/eval.config.example.json @@ -0,0 +1,76 @@ +{ + "_comment": "This file is generated by `hdx-eval setup-hyperdx`. You can also edit it by hand to add/modify MCPs.", + "mcps": { + "hyperdx": { + "type": "http", + "url": "http://localhost:30196/mcp", + "headers": { + "Authorization": "Bearer uuid-from-/me-endpoint" + }, + "toolPattern": "mcp__hyperdx__*", + "label": "HyperDX", + "brandTerms": ["HyperDX", "hyperdx"], + "deniedTools": [ + "mcp__hyperdx__hyperdx_delete_dashboard", + "mcp__hyperdx__hyperdx_get_dashboard", + "mcp__hyperdx__hyperdx_save_dashboard", + "mcp__hyperdx__hyperdx_query_tile", + "mcp__hyperdx__hyperdx_get_saved_search", + "mcp__hyperdx__hyperdx_save_saved_search", + "mcp__hyperdx__hyperdx_get_alert", + "mcp__hyperdx__hyperdx_get_webhook", + "mcp__hyperdx__hyperdx_save_alert" + ] + }, + "clickhouse": { + "type": "stdio", + "command": "uv", + "args": [ + "run", + "--with", + "mcp-clickhouse==0.3.0", + "--python", + "3.10", + "mcp-clickhouse" + ], + "env": { + "CLICKHOUSE_HOST": "localhost", + "CLICKHOUSE_PORT": "30596", + "CLICKHOUSE_USER": "default", + "CLICKHOUSE_PASSWORD": "", + "CLICKHOUSE_SECURE": "false", + "CLICKHOUSE_VERIFY": "false", + "CLICKHOUSE_SEND_RECEIVE_TIMEOUT": "10" + }, + "toolPattern": "mcp__clickhouse__*", + "label": "ClickHouse MCP", + "brandTerms": ["ClickHouse MCP", "clickhouse"] + } + }, + "scenarios": { + "error-root-cause": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + }, + "latency-spike": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + }, + "noisy-signals": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + } + }, + "hyperdxApi": { + "apiUrl": "http://localhost:30196", + "accessKey": "uuid-from-/me-endpoint", + "connectionId": "mongo-objectid" + }, + "clickhouse": { + "host": "localhost", + "port": "30596", + "user": "default", + "password": "" + }, + "anchorTime": "2026-06-01T00:00:00.000Z" +} diff --git a/packages/hdx-eval/jest.config.js b/packages/hdx-eval/jest.config.js new file mode 100644 index 0000000000..4bdc2f8362 --- /dev/null +++ b/packages/hdx-eval/jest.config.js @@ -0,0 +1,15 @@ +const { createJsWithTsPreset } = require('ts-jest'); + +const tsJestTransformCfg = createJsWithTsPreset(); + +/** @type {import("jest").Config} **/ +module.exports = { + ...tsJestTransformCfg, + testEnvironment: 'node', + rootDir: './src', + testMatch: ['**/__tests__/*.test.ts'], + testTimeout: 15000, + moduleNameMapper: { + '@/(.*)$': '/$1', + }, +}; diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json new file mode 100644 index 0000000000..570971fa95 --- /dev/null +++ b/packages/hdx-eval/package.json @@ -0,0 +1,42 @@ +{ + "name": "@hyperdx/hdx-eval", + "version": "0.1.0", + "license": "MIT", + "private": true, + "engines": { + "node": ">=22.16.0" + }, + "bin": "./dist/cli.js", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist/*" + ], + "dependencies": { + "@anthropic-ai/sdk": "^0.92.0", + "@clickhouse/client": "^1.12.1", + "commander": "^12.1.0", + "ms": "^2.1.3" + }, + "devDependencies": { + "@dotenvx/dotenvx": "^1.51.1", + "@types/jest": "^29.5.14", + "@types/ms": "^0.7.31", + "@types/node": "^22.15.18", + "jest": "^30.2.0", + "rimraf": "^4.4.1", + "ts-jest": "^29.4.5", + "tsx": "^4.19.0", + "typescript": "^5.9.3" + }, + "scripts": { + "build": "rimraf ./dist && tsc -p tsconfig.build.json", + "dev": "NODE_OPTIONS=--max-old-space-size=8192 ./scripts/env.sh tsx src/cli.ts", + "lint": "npx eslint --quiet . --ext .ts", + "lint:fix": "npx eslint . --ext .ts --fix", + "ci:lint": "yarn lint && yarn tsc --noEmit", + "ci:unit": "jest --ci --forceExit", + "dev:unit": "jest --watch", + "viewer": "node scripts/viewer/server.js" + } +} diff --git a/packages/hdx-eval/scripts/env.sh b/packages/hdx-eval/scripts/env.sh new file mode 100755 index 0000000000..a30d58c915 --- /dev/null +++ b/packages/hdx-eval/scripts/env.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Set up the full environment for hdx-eval, then exec the remaining args. +# +# 1. Source dev-env.sh from the monorepo root for slot-based port vars +# 2. Wrap the command with dotenvx to load .env / .env.local +# +# Usage: +# scripts/env.sh tsx src/cli.ts seed error-root-cause +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +# Slot-based port env vars (HYPERDX_API_PORT, HDX_DEV_CH_HTTP_PORT, etc.) +_HDX_EVAL_CWD="$PWD" +cd "$REPO_ROOT" && source ./scripts/dev-env.sh > /dev/null 2>&1 +cd "$_HDX_EVAL_CWD" +unset _HDX_EVAL_CWD + +# Load .env.local from the monorepo root (existing vars take precedence) +exec "$REPO_ROOT/node_modules/.bin/dotenvx" run \ + -f "$REPO_ROOT/.env.local" \ + --ignore=MISSING_ENV_FILE --quiet -- "$@" diff --git a/packages/hdx-eval/scripts/viewer/public/app.js b/packages/hdx-eval/scripts/viewer/public/app.js new file mode 100644 index 0000000000..9efc87e8d9 --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/public/app.js @@ -0,0 +1,1104 @@ +(() => { + const $ = (sel) => document.querySelector(sel); + const el = (tag, attrs = {}, ...children) => { + const n = document.createElement(tag); + for (const [k, v] of Object.entries(attrs)) { + if (k === 'class') n.className = v; + else if (k.startsWith('on')) n.addEventListener(k.slice(2), v); + else if (v === true) n.setAttribute(k, ''); + else if (v != null && v !== false) n.setAttribute(k, v); + } + for (const c of children.flat()) { + if (c == null || c === false) continue; + n.appendChild(typeof c === 'string' ? document.createTextNode(c) : c); + } + return n; + }; + + const state = { + batches: [], + batch: null, + cells: [], + summary: null, + selectedRun: null, // { scenario, mcp, idx } + runData: null, // { trajectory, grade } + activeTab: 'trajectory', + view: 'summary', // 'summary' | 'run' + }; + + // ----- formatting helpers ----- + + const fmtMs = (ms) => { + if (ms == null) return '—'; + if (ms < 1000) return `${ms}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + return `${(ms / 60_000).toFixed(1)}m`; + }; + + const fmtScore = (s) => (s == null ? '—' : s.toFixed(2)); + + const scoreClass = (s, max = 1) => { + if (s == null) return ''; + const n = s / max; + if (n <= 0.01) return 'bad'; + if (n < 0.5) return 'warn'; + return 'good'; + }; + + const criterionClass = (s) => { + if (s == null) return ''; + if (s === 0) return 'score-0'; + if (s <= 1) return 'score-low'; + if (s <= 2) return 'score-mid'; + return 'score-high'; + }; + + const stringifyMaybe = (v) => { + if (v == null) return ''; + if (typeof v === 'string') return v; + try { + return JSON.stringify(v, null, 2); + } catch { + return String(v); + } + }; + + // Tool results come back in a few shapes: + // - already an object/array (from toolCalls[].input) + // - a JSON-encoded string + // - newline-delimited JSON (ToolSearch output) + // - an array of {type:"text", text:"..."} content blocks (raw tool_result) + // Render them as pretty-printed JSON where possible, with text blocks unwrapped. + const tryParseJson = (s) => { + if (typeof s !== 'string') return undefined; + const t = s.trim(); + if (!t || (t[0] !== '{' && t[0] !== '[' && t[0] !== '"')) return undefined; + try { + return JSON.parse(t); + } catch { + return undefined; + } + }; + + const prettyJson = (v) => { + try { + return JSON.stringify(v, null, 2); + } catch { + return String(v); + } + }; + + const formatToolPayload = (v) => { + if (v == null) return ''; + + // Object / array: pretty-print directly, but unwrap content-block arrays. + if (typeof v === 'object') { + if ( + Array.isArray(v) && + v.length > 0 && + v.every( + (b) => b && typeof b === 'object' && b.type === 'text' && 'text' in b, + ) + ) { + return v + .map((b) => { + const inner = tryParseJson(b.text); + return inner !== undefined ? prettyJson(inner) : b.text; + }) + .join('\n\n'); + } + return prettyJson(v); + } + + if (typeof v !== 'string') return String(v); + + const trimmed = v.trim(); + if (!trimmed) return v; + + // Whole-string JSON. + const parsed = tryParseJson(trimmed); + if (parsed !== undefined) return prettyJson(parsed); + + // Newline-delimited JSON (e.g. ToolSearch). + const lines = trimmed.split('\n'); + if (lines.length > 1) { + const objs = []; + let allParsed = true; + for (const line of lines) { + const p = tryParseJson(line); + if (p === undefined) { + allParsed = false; + break; + } + objs.push(p); + } + if (allParsed) return objs.map(prettyJson).join('\n'); + } + + // wrapper with an embedded JSON preview. + const previewMatch = v.match( + /^([\s\S]*?Preview \(first [^)]+\):\n)([\s\S]*?)(\n\.\.\.\n<\/persisted-output>)$/, + ); + if (previewMatch) { + const [, header, body, footer] = previewMatch; + const previewParsed = tryParseJson(body); + if (previewParsed !== undefined) { + return header + prettyJson(previewParsed) + footer; + } + } + + return v; + }; + + // ----- data loading ----- + + async function fetchJson(url) { + const r = await fetch(url); + if (!r.ok) throw new Error(`${r.status} ${r.statusText}`); + return r.json(); + } + + async function loadBatches() { + const data = await fetchJson('/api/batches'); + state.batches = data.batches; + renderBatchPicker(); + if (state.batches.length) await loadBatch(state.batches[0].name); + } + + async function loadBatch(batchName) { + const data = await fetchJson(`/api/batches/${encodeURIComponent(batchName)}`); + state.batch = data.batch; + state.cells = data.cells || []; + state.summary = data.summary; + state.selectedRun = null; + state.runData = null; + renderBatchMeta(); + renderNav(); + showView('summary'); + } + + async function loadRun(scenario, mcp, idx) { + state.selectedRun = { scenario, mcp, idx }; + const u = `/api/batches/${encodeURIComponent(state.batch)}/runs/${encodeURIComponent(scenario)}/${encodeURIComponent(mcp)}/${idx}`; + state.runData = await fetchJson(u); + renderNav(); + showView('run'); + renderRun(); + } + + // ----- rendering: header ----- + + function renderBatchPicker() { + const sel = $('#batch-select'); + sel.innerHTML = ''; + for (const b of state.batches) { + sel.appendChild(el('option', { value: b.name }, b.name)); + } + sel.onchange = () => loadBatch(sel.value); + } + + function renderBatchMeta() { + const sc = state.summary?.scenarios?.length ?? state.cells.length; + const cells = state.cells.length; + const meta = $('#batch-meta'); + meta.innerHTML = ''; + meta.appendChild(document.createTextNode(`${sc} scenario(s) \u00B7 ${cells} cell(s) `)); + const link = el('a', { + href: '#', + class: 'back-link', + onclick: (e) => { + e.preventDefault(); + state.selectedRun = null; + renderNav(); + showView('summary'); + }, + }, '\u2190 Summary'); + meta.appendChild(link); + } + + // ----- rendering: sidebar nav ----- + + function renderNav() { + const root = $('#nav'); + root.innerHTML = ''; + + const byScenario = new Map(); + for (const c of state.cells) { + if (!byScenario.has(c.scenario)) byScenario.set(c.scenario, []); + byScenario.get(c.scenario).push(c); + } + + for (const [scenario, cells] of byScenario) { + const sNode = el( + 'div', + { class: 'nav-scenario' }, + el('div', { class: 'head' }, scenario), + ); + for (const c of cells) { + const mNode = el( + 'div', + { class: 'nav-mcp' }, + el('div', { class: 'head' }, c.mcp), + ); + for (const r of c.runs) { + const active = + state.selectedRun && + state.selectedRun.scenario === scenario && + state.selectedRun.mcp === c.mcp && + state.selectedRun.idx === r.idx; + const chip = el( + 'span', + { class: `score-chip ${scoreClass(r.combinedScore)}` }, + fmtScore(r.combinedScore), + ); + const errBadge = + r.toolErrors > 0 + ? el('span', { class: 'tag error' }, `${r.toolErrors}!`) + : null; + const termBadge = + r.termination && r.termination !== 'final_answer' + ? el('span', { class: 'tag' }, r.termination) + : null; + const node = el( + 'div', + { + class: `nav-run ${active ? 'active' : ''}`, + onclick: () => loadRun(scenario, c.mcp, r.idx), + }, + el( + 'span', + {}, + `#${r.idx}`, + ' ', + el('span', { class: 'muted small' }, `${r.toolCalls ?? '?'} calls`), + ), + el('span', {}, termBadge, ' ', errBadge, ' ', chip), + ); + mNode.appendChild(node); + } + sNode.appendChild(mNode); + } + root.appendChild(sNode); + } + } + + // ----- rendering: run ----- + + function renderRun() { + $('#summary').hidden = true; + $('#empty').hidden = true; + $('#run').hidden = false; + + const { trajectory: t, grade } = state.runData; + + // header strip + const hdr = $('#run-header'); + hdr.innerHTML = ''; + const fields = [ + ['scenario', t.scenario], + ['mcp', t.mcp], + ['run', `#${t.runIndex} · seed ${t.seed ?? '—'}`], + ['model', t.model || '—'], + ['termination', t.termination || '—'], + ['duration', fmtMs(t.durationMs)], + ['tool calls', String(t.toolCalls?.length ?? 0)], + [ + 'tool errors', + grade?.toolErrors + ? `${grade.toolErrors.errors}/${grade.toolErrors.total}` + : '—', + ], + [ + 'combined', + grade?.combinedScore != null ? grade.combinedScore.toFixed(3) : '—', + ], + ]; + for (const [k, v] of fields) { + hdr.appendChild( + el( + 'div', + {}, + el('div', { class: 'label' }, k), + el('div', { class: 'value' }, String(v)), + ), + ); + } + + renderTrajectory(); + renderAnswer(); + renderGrading(); + renderRaw(); + + activateTab(state.activeTab); + } + + function activateTab(name) { + state.activeTab = name; + for (const t of document.querySelectorAll('.tab')) { + t.classList.toggle('active', t.dataset.tab === name); + } + for (const p of document.querySelectorAll('.tab-pane')) { + p.classList.toggle('active', p.id === `tab-${name}`); + } + } + + // Build a unified trajectory from trajectory.messages, mixing thinking text, + // assistant text, and tool calls (with their results inline). Falls back to + // toolCalls[] when messages aren't structured the expected way. + function buildEvents(t) { + const events = []; + const toolResults = new Map(); // tool_use_id -> content + + if (Array.isArray(t.messages)) { + // First pass: collect tool_result blocks (sent in user messages) + for (const msg of t.messages) { + if (msg.type === 'user' && msg.message?.content) { + const content = msg.message.content; + if (Array.isArray(content)) { + for (const b of content) { + if (b.type === 'tool_result') { + toolResults.set(b.tool_use_id, { + content: b.content, + isError: !!b.is_error, + }); + } + } + } + } + } + // Second pass: emit thinking/text/tool_use in assistant order + for (const msg of t.messages) { + if (msg.type !== 'assistant') continue; + const content = msg.message?.content; + if (!Array.isArray(content)) continue; + for (const b of content) { + if (b.type === 'thinking') { + events.push({ kind: 'thinking', text: b.thinking }); + } else if (b.type === 'text') { + events.push({ kind: 'text', text: b.text }); + } else if (b.type === 'tool_use') { + const result = toolResults.get(b.id); + events.push({ + kind: 'tool', + name: b.name, + input: b.input, + output: result?.content, + isError: result?.isError || false, + id: b.id, + }); + } + } + } + } + + // Fallback: messages didn't give us tool calls — use toolCalls array. + const toolEventCount = events.filter((e) => e.kind === 'tool').length; + if (toolEventCount === 0 && Array.isArray(t.toolCalls)) { + for (const c of t.toolCalls) { + events.push({ + kind: 'tool', + name: c.name, + input: c.input, + output: c.output, + isError: c.isError || false, + }); + } + } + return events; + } + + function renderTrajectory() { + const t = state.runData.trajectory; + const events = buildEvents(t); + const list = $('#trajectory'); + + const hideToolSearch = $('#hide-toolsearch').checked; + const errorsOnly = $('#errors-only').checked; + const filter = $('#traj-filter').value.trim().toLowerCase(); + + list.innerHTML = ''; + let toolIdx = 0; + for (const e of events) { + let visible = true; + if (e.kind === 'tool') { + toolIdx++; + if (hideToolSearch && e.name === 'ToolSearch') visible = false; + if (errorsOnly && !e.isError) visible = false; + if (filter) { + const hay = ( + e.name + + '\n' + + formatToolPayload(e.input) + + '\n' + + formatToolPayload(e.output) + ).toLowerCase(); + if (!hay.includes(filter)) visible = false; + } + } else { + if (errorsOnly) visible = false; + if (filter && !e.text?.toLowerCase().includes(filter)) visible = false; + } + if (!visible) continue; + + if (e.kind === 'thinking') { + list.appendChild( + el('li', {}, el('div', { class: 'thinking' }, e.text)), + ); + } else if (e.kind === 'text') { + list.appendChild( + el('li', {}, el('div', { class: 'assistant-text' }, e.text)), + ); + } else if (e.kind === 'tool') { + list.appendChild(renderCall(toolIdx, e)); + } + } + } + + function renderCall(idx, c) { + const inputStr = formatToolPayload(c.input); + const outputStr = formatToolPayload(c.output); + const summary = + inputStr.length > 200 ? inputStr.slice(0, 200) + '…' : inputStr; + + const body = el( + 'div', + { class: 'body' }, + el('h4', {}, 'Input'), + el('pre', { class: 'code' }, inputStr || '(none)'), + el('h4', {}, c.isError ? 'Output (ERROR)' : 'Output'), + el( + 'pre', + { class: 'code' + (c.isError ? ' error' : '') }, + outputStr || '(none)', + ), + ); + + const li = el( + 'li', + { class: 'call' + (c.isError ? ' error' : '') }, + el( + 'div', + { + class: 'head', + onclick: (ev) => { + ev.currentTarget.parentElement.classList.toggle('open'); + }, + }, + el('span', { class: 'idx' }, `#${idx}`), + el('span', { class: 'name' + (c.isError ? ' error' : '') }, c.name), + el('span', { class: 'summary' }, summary.replace(/\s+/g, ' ')), + c.isError ? el('span', { class: 'tag error' }, 'ERROR') : null, + ), + body, + ); + return li; + } + + function renderAnswer() { + const t = state.runData.trajectory; + const pane = $('#tab-answer'); + pane.innerHTML = ''; + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'Agent Prompt'), + el('pre', {}, t.agentPrompt || '(none)'), + ), + ); + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'Final Answer'), + el('pre', {}, t.finalAnswer || '(no final answer)'), + ), + ); + if (t.systemPromptAppend) { + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'System Prompt Append'), + el('pre', {}, t.systemPromptAppend), + ), + ); + } + } + + function renderGrading() { + const g = state.runData.grade; + const pane = $('#tab-grading'); + pane.innerHTML = ''; + + if (!g) { + pane.appendChild(el('div', { class: 'muted' }, 'No grade file.')); + return; + } + + // Combined + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el( + 'h3', + {}, + 'Combined Score', + el('span', { class: 'score' }, fmtScore(g.combinedScore)), + ), + el( + 'div', + { class: 'muted' }, + `Programmatic ${fmtScore(g.programmatic?.score)} · Judge ${fmtScore(g.judge?.weightedScore)} · Tool error penalty ${fmtScore(g.toolErrors?.penalty)}`, + ), + ), + ); + + // Programmatic checks + if (g.programmatic?.hits) { + const checks = el('div', { class: 'grade-section' }); + checks.appendChild( + el( + 'h3', + {}, + 'Programmatic checks', + el( + 'span', + { class: 'score' }, + fmtScore(g.programmatic.score), + ), + ), + ); + for (const h of g.programmatic.hits) { + const good = !!h.satisfied; + checks.appendChild( + el( + 'div', + { class: 'check' + (h.negative ? ' negative' : '') }, + el( + 'div', + { class: 'marker ' + (good ? 'good' : 'bad') }, + good ? '✓' : '✗', + ), + el('div', { class: 'id' }, h.id), + el( + 'div', + { class: 'weight' }, + `w${h.weight}${h.matched != null ? ` · matched=${h.matched}` : ''}`, + ), + ), + ); + } + checks.appendChild( + el( + 'div', + { class: 'muted small', style: 'margin-top:0.6rem; font-size:11px' }, + 'Negative checks score points when NOT matched — they punish false attribution to distractors.', + ), + ); + pane.appendChild(checks); + } + + // Judge scores + if (g.judge?.scores) { + const judge = el('div', { class: 'grade-section' }); + judge.appendChild( + el( + 'h3', + {}, + `LLM Judge (${g.judge.model || 'model?'})`, + el('span', { class: 'score' }, fmtScore(g.judge.weightedScore)), + ), + ); + for (const [name, val] of Object.entries(g.judge.scores)) { + judge.appendChild( + el( + 'div', + { class: 'criterion' }, + el( + 'div', + { class: 'criterion-head' }, + el('div', { class: 'criterion-name' }, name), + el( + 'div', + { class: 'criterion-score ' + criterionClass(val.score) }, + `${val.score} / 4`, + ), + ), + el('div', { class: 'rationale' }, val.rationale || ''), + ), + ); + } + pane.appendChild(judge); + } + + // Tool errors detail + if (g.toolErrors) { + const te = el('div', { class: 'grade-section' }); + te.appendChild( + el( + 'h3', + {}, + 'Tool errors', + el( + 'span', + { class: 'score' }, + `${g.toolErrors.errors}/${g.toolErrors.total} (penalty ${fmtScore(g.toolErrors.penalty)})`, + ), + ), + ); + if (g.toolErrors.samples?.length) { + te.appendChild( + el( + 'pre', + { class: 'code' }, + stringifyMaybe(g.toolErrors.samples), + ), + ); + } else { + te.appendChild(el('div', { class: 'muted' }, 'No error samples.')); + } + pane.appendChild(te); + } + } + + function renderRaw() { + const pane = $('#tab-raw'); + pane.innerHTML = ''; + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'trajectory.json'), + el('pre', {}, stringifyMaybe(state.runData.trajectory)), + ), + ); + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'grade.json'), + el('pre', {}, stringifyMaybe(state.runData.grade)), + ), + ); + } + + // ----- view switching ----- + + function showView(view) { + state.view = view; + const summaryEl = $('#summary'); + const runEl = $('#run'); + const emptyEl = $('#empty'); + if (view === 'summary') { + summaryEl.hidden = false; + runEl.hidden = true; + emptyEl.hidden = true; + renderSummary(); + } else { + summaryEl.hidden = true; + runEl.hidden = false; + emptyEl.hidden = true; + } + } + + // ----- rendering: summary comparison dashboard ----- + + const pct = (x) => (x == null ? '—' : `${(x * 100).toFixed(0)}%`); + const signedPct = (x) => { + if (x == null) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${(x * 100).toFixed(0)}%`; + }; + const signedNum = (x, d = 1) => { + if (x == null) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${x.toFixed(d)}`; + }; + const deltaClass = (x, invert = false) => { + if (x == null) return ''; + const v = invert ? -x : x; + if (Math.abs(v) < 0.005) return 'delta-neutral'; + return v > 0 ? 'delta-pos' : 'delta-neg'; + }; + + function renderSummary() { + const root = $('#summary-content'); + root.innerHTML = ''; + + const s = state.summary; + if (!s || !s.scenarios || s.scenarios.length === 0) { + root.className = 'muted center'; + root.textContent = 'No summary data. Run grading and report first.'; + return; + } + root.className = 'summary-dashboard'; + + const mcpOrder = s.mcpOrder || Object.keys(s.scenarios[0]?.cells || {}).sort(); + const baseline = s.baseline || mcpOrder[0]; + const challengers = mcpOrder.filter((m) => m !== baseline); + + // Header info + root.appendChild( + el( + 'div', + { class: 'summary-header' }, + el('span', { class: 'label' }, 'Baseline:'), + el('span', { class: 'value accent' }, baseline), + el('span', { class: 'label' }, 'MCPs:'), + el('span', { class: 'value' }, mcpOrder.join(', ')), + s.generatedAt + ? el( + 'span', + { class: 'muted small' }, + new Date(s.generatedAt).toLocaleString(), + ) + : null, + ), + ); + + // Top-line verdict table + root.appendChild(renderVerdictTable(s, mcpOrder, baseline, challengers)); + + // Per-scenario breakdowns + for (const scenario of s.scenarios) { + root.appendChild( + renderScenarioBreakdown(scenario, mcpOrder, baseline, challengers), + ); + } + } + + function renderVerdictTable(s, mcpOrder, baseline, challengers) { + const headerCells = [el('th', {}, 'Scenario')]; + for (const m of mcpOrder) { + headerCells.push( + el('th', { class: m === baseline ? 'baseline-col' : '' }, m), + ); + } + for (const m of challengers) { + headerCells.push(el('th', { class: 'delta-col' }, `\u0394 ${m}`)); + } + + const rows = s.scenarios.map((sc) => { + const cells = [el('td', { class: 'scenario-name' }, sc.scenario)]; + for (const m of mcpOrder) { + const c = sc.cells?.[m]; + const score = c?.combinedScore?.mean; + cells.push( + el( + 'td', + { class: `score-cell ${scoreClass(score)}` }, + pct(score), + ), + ); + } + for (const m of challengers) { + const d = sc.deltas?.[m]?.combinedScore; + cells.push( + el('td', { class: `delta-cell ${deltaClass(d)}` }, signedPct(d)), + ); + } + return el('tr', {}, ...cells); + }); + + return el( + 'div', + { class: 'summary-section' }, + el('h2', {}, 'Top-line Verdict'), + el( + 'table', + { class: 'summary-table' }, + el('thead', {}, el('tr', {}, ...headerCells)), + el('tbody', {}, ...rows), + ), + ); + } + + function renderScenarioBreakdown(scenario, mcpOrder, baseline, challengers) { + const section = el('div', { class: 'summary-section scenario-breakdown' }); + section.appendChild(el('h2', {}, scenario.scenario)); + + // Metrics table + const metrics = [ + { + label: 'Combined score', + get: (c) => c?.combinedScore?.mean, + fmt: pct, + dKey: 'combinedScore', + dFmt: signedPct, + }, + { + label: 'Programmatic', + get: (c) => c?.programmatic?.mean, + fmt: pct, + dKey: 'programmaticScore', + dFmt: signedPct, + }, + { + label: 'Judge (weighted)', + get: (c) => c?.judge?.weightedMean, + fmt: pct, + dKey: 'judgeWeightedMean', + dFmt: signedPct, + }, + { + label: 'Tool calls', + get: (c) => c?.toolCalls?.mean, + fmt: (x) => (x == null ? '—' : x.toFixed(1)), + dKey: 'toolCalls', + dFmt: (x) => signedNum(x, 1), + invert: true, + }, + { + label: 'Tool errors', + get: (c) => c?.toolErrors?.mean, + fmt: (x) => (x == null ? '—' : x.toFixed(1)), + }, + { + label: 'Error penalty', + get: (c) => c?.toolErrors?.penaltyMean, + fmt: (x) => (x == null ? '—' : `${(x * 100).toFixed(0)}pp`), + }, + { + label: 'Output tokens', + get: (c) => c?.tokens?.output, + fmt: (x) => (x == null ? '—' : Math.round(x).toLocaleString()), + dKey: 'outputTokens', + dFmt: (x) => signedNum(x, 0), + invert: true, + }, + { + label: 'Duration (s)', + get: (c) => c?.durationMs?.mean, + fmt: (x) => (x == null ? '—' : (x / 1000).toFixed(1)), + dKey: 'durationMs', + dFmt: (x) => (x == null ? '—' : signedNum(x / 1000, 1)), + invert: true, + }, + { + label: 'Termination', + get: (c) => c?.termination, + fmt: (x) => { + if (!x) return '—'; + return Object.entries(x) + .sort((a, b) => b[1] - a[1]) + .map(([k, v]) => `${k}: ${v}`) + .join(', '); + }, + }, + { + label: 'N', + get: (c) => c?.n, + fmt: (x) => (x == null ? '—' : String(x)), + }, + ]; + + const headerCells = [el('th', {}, 'Metric')]; + for (const m of mcpOrder) { + headerCells.push( + el('th', { class: m === baseline ? 'baseline-col' : '' }, m), + ); + } + for (const m of challengers) { + headerCells.push(el('th', { class: 'delta-col' }, `\u0394 ${m}`)); + } + + const metricRows = metrics.map((metric) => { + const cells = [el('td', { class: 'metric-label' }, metric.label)]; + for (const m of mcpOrder) { + const c = scenario.cells?.[m]; + const v = metric.get(c); + cells.push(el('td', { class: 'metric-value' }, metric.fmt(v))); + } + for (const m of challengers) { + if (metric.dKey) { + const d = scenario.deltas?.[m]?.[metric.dKey]; + cells.push( + el( + 'td', + { class: `delta-cell ${deltaClass(d, metric.invert)}` }, + metric.dFmt(d), + ), + ); + } else { + cells.push(el('td', { class: 'delta-cell' }, '—')); + } + } + return el('tr', {}, ...cells); + }); + + section.appendChild( + el( + 'table', + { class: 'summary-table metrics-table' }, + el('thead', {}, el('tr', {}, ...headerCells)), + el('tbody', {}, ...metricRows), + ), + ); + + // Judge per-criterion heatmap + const allCriteria = new Set(); + for (const cell of Object.values(scenario.cells || {})) { + if (cell?.judge?.perCriterion) { + for (const id of Object.keys(cell.judge.perCriterion)) allCriteria.add(id); + } + } + if (allCriteria.size > 0) { + const jHeader = [el('th', {}, 'Criterion')]; + for (const m of mcpOrder) jHeader.push(el('th', {}, m)); + const jRows = [...allCriteria].sort().map((id) => { + const cells = [el('td', { class: 'metric-label' }, id)]; + for (const m of mcpOrder) { + const v = scenario.cells?.[m]?.judge?.perCriterion?.[id]; + const cls = criterionHeatClass(v); + cells.push( + el('td', { class: `heatmap-cell ${cls}` }, v != null ? `${v.toFixed(1)}/5` : '—'), + ); + } + return el('tr', {}, ...cells); + }); + section.appendChild( + el( + 'div', + { class: 'sub-table' }, + el('h3', {}, 'Judge per-criterion (0\u20135)'), + el( + 'table', + { class: 'summary-table heatmap' }, + el('thead', {}, el('tr', {}, ...jHeader)), + el('tbody', {}, ...jRows), + ), + ), + ); + } + + // Programmatic per-check + const allChecks = new Set(); + for (const cell of Object.values(scenario.cells || {})) { + if (cell?.programmatic?.perCheck) { + for (const id of Object.keys(cell.programmatic.perCheck)) allChecks.add(id); + } + } + if (allChecks.size > 0) { + const pHeader = [el('th', {}, 'Check')]; + for (const m of mcpOrder) pHeader.push(el('th', {}, m)); + const pRows = [...allChecks].sort().map((id) => { + const isNeg = id.startsWith('false_'); + const label = isNeg ? `${id} (neg)` : id; + const cells = [el('td', { class: 'metric-label' }, label)]; + for (const m of mcpOrder) { + const v = scenario.cells?.[m]?.programmatic?.perCheck?.[id]; + const cls = v != null ? (v >= 0.99 ? 'check-pass' : v >= 0.5 ? 'check-partial' : 'check-fail') : ''; + cells.push( + el('td', { class: `heatmap-cell ${cls}` }, v != null ? pct(v) : '—'), + ); + } + return el('tr', {}, ...cells); + }); + section.appendChild( + el( + 'div', + { class: 'sub-table' }, + el('h3', {}, 'Programmatic per-check (pass rate)'), + el( + 'table', + { class: 'summary-table heatmap' }, + el('thead', {}, el('tr', {}, ...pHeader)), + el('tbody', {}, ...pRows), + ), + ), + ); + } + + // Side-by-side run comparison + const scenarioCells = state.cells.filter((c) => c.scenario === scenario.scenario); + if (scenarioCells.length > 0) { + const maxRuns = Math.max(...scenarioCells.map((c) => c.runs.length)); + const runHeader = [el('th', {}, 'Run')]; + for (const c of scenarioCells) { + runHeader.push(el('th', {}, c.mcp)); + } + const runRows = []; + for (let i = 0; i < maxRuns; i++) { + const cells = [el('td', { class: 'metric-label' }, `#${i}`)]; + for (const c of scenarioCells) { + const r = c.runs[i]; + if (!r) { + cells.push(el('td', {}, '—')); + continue; + } + const chip = el( + 'span', + { class: `score-chip ${scoreClass(r.combinedScore)}` }, + fmtScore(r.combinedScore), + ); + const errBit = + r.toolErrors > 0 + ? el('span', { class: 'tag error small' }, `${r.toolErrors}err`) + : null; + const termBit = + r.termination !== 'final_answer' + ? el('span', { class: 'tag small' }, r.termination) + : null; + cells.push( + el( + 'td', + { + class: 'run-cell clickable', + onclick: () => loadRun(scenario.scenario, c.mcp, r.idx), + }, + chip, + ' ', + el('span', { class: 'muted small' }, `${r.toolCalls ?? '?'}c ${fmtMs(r.durationMs)}`), + termBit ? el('span', {}, ' ', termBit) : null, + errBit ? el('span', {}, ' ', errBit) : null, + ), + ); + } + runRows.push(el('tr', {}, ...cells)); + } + section.appendChild( + el( + 'div', + { class: 'sub-table' }, + el('h3', {}, 'Runs (click to inspect)'), + el( + 'table', + { class: 'summary-table runs-table' }, + el('thead', {}, el('tr', {}, ...runHeader)), + el('tbody', {}, ...runRows), + ), + ), + ); + } + + return section; + } + + function criterionHeatClass(v) { + if (v == null) return ''; + if (v >= 4) return 'heat-high'; + if (v >= 3) return 'heat-mid'; + if (v >= 2) return 'heat-low'; + return 'heat-bad'; + } + + // ----- wiring ----- + + for (const btn of document.querySelectorAll('.tab')) { + btn.addEventListener('click', () => activateTab(btn.dataset.tab)); + } + for (const id of ['hide-toolsearch', 'errors-only']) { + $('#' + id).addEventListener('change', () => { + if (state.runData) renderTrajectory(); + }); + } + $('#traj-filter').addEventListener('input', () => { + if (state.runData) renderTrajectory(); + }); + + loadBatches().catch((e) => { + $('#summary-content').textContent = `Error loading batches: ${e.message}`; + }); +})(); diff --git a/packages/hdx-eval/scripts/viewer/public/index.html b/packages/hdx-eval/scripts/viewer/public/index.html new file mode 100644 index 0000000000..ea1b626c65 --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/public/index.html @@ -0,0 +1,70 @@ + + + + + + HDX Eval Viewer + + + +
+
hdx-eval viewer
+
+ + + +
+
+ +
+ + +
+
+
+ Loading... +
+
+ + +
+
+ + + + diff --git a/packages/hdx-eval/scripts/viewer/public/style.css b/packages/hdx-eval/scripts/viewer/public/style.css new file mode 100644 index 0000000000..3a01855d18 --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/public/style.css @@ -0,0 +1,679 @@ +:root { + --bg: #0f1419; + --panel: #161c23; + --panel-2: #1d242d; + --border: #2a323d; + --text: #d9e1ea; + --muted: #7a8896; + --accent: #7cc4ff; + --good: #6cd97e; + --bad: #ff6b6b; + --warn: #ffc857; + --code-bg: #0b0f14; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + background: var(--bg); + color: var(--text); + font: 13px/1.45 -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, + sans-serif; + height: 100%; +} + +body { + display: flex; + flex-direction: column; +} + +header { + display: flex; + align-items: center; + gap: 1.2rem; + padding: 0.6rem 1rem; + background: var(--panel); + border-bottom: 1px solid var(--border); +} + +.title { + font-weight: 600; + letter-spacing: 0.02em; +} + +.batch-picker { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.back-link { + color: var(--accent); + text-decoration: none; + font-size: 12px; + margin-left: 0.5rem; +} +.back-link:hover { + text-decoration: underline; +} + +select, +input[type="search"] { + background: var(--panel-2); + color: var(--text); + border: 1px solid var(--border); + border-radius: 4px; + padding: 0.25rem 0.5rem; + font: inherit; +} + +select { + min-width: 280px; +} + +main { + display: flex; + flex: 1; + min-height: 0; +} + +#sidebar { + width: 280px; + border-right: 1px solid var(--border); + background: var(--panel); + padding: 0.6rem; + overflow-y: auto; +} + +#sidebar .small { + margin-bottom: 0.4rem; + text-transform: uppercase; + letter-spacing: 0.06em; + font-size: 11px; +} + +#content { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.muted { + color: var(--muted); +} + +.center { + margin: auto; + padding: 2rem; +} + +#run { + display: flex; + flex-direction: column; + flex: 1; + min-height: 0; +} + +#run-header { + padding: 0.6rem 1rem; + border-bottom: 1px solid var(--border); + background: var(--panel); + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); + gap: 0.5rem 1rem; +} + +#run-header .label { + color: var(--muted); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +#run-header .value { + font-weight: 500; +} + +.tabs { + display: flex; + gap: 0; + border-bottom: 1px solid var(--border); + background: var(--panel); +} + +.tab { + background: transparent; + color: var(--muted); + border: 0; + border-bottom: 2px solid transparent; + padding: 0.5rem 1rem; + cursor: pointer; + font: inherit; +} + +.tab.active { + color: var(--text); + border-bottom-color: var(--accent); +} + +.tab-pane { + display: none; + flex: 1; + overflow: auto; + padding: 0.8rem 1rem; +} + +.tab-pane.active { + display: block; +} + +/* sidebar nav */ +.nav-scenario { + margin-top: 0.4rem; +} +.nav-scenario > .head { + font-weight: 600; + padding: 0.25rem 0; + color: var(--text); +} +.nav-mcp { + margin-left: 0.6rem; +} +.nav-mcp > .head { + color: var(--accent); + padding: 0.2rem 0; +} +.nav-run { + margin-left: 1.4rem; + display: flex; + justify-content: space-between; + align-items: center; + gap: 0.4rem; + padding: 0.2rem 0.4rem; + border-radius: 3px; + cursor: pointer; + user-select: none; +} +.nav-run:hover { + background: var(--panel-2); +} +.nav-run.active { + background: var(--panel-2); + outline: 1px solid var(--accent); +} +.score-chip { + font-variant-numeric: tabular-nums; + padding: 0 0.4rem; + border-radius: 10px; + background: var(--panel-2); + border: 1px solid var(--border); + font-size: 11px; +} +.score-chip.good { + color: var(--good); + border-color: var(--good); +} +.score-chip.bad { + color: var(--bad); + border-color: var(--bad); +} +.score-chip.warn { + color: var(--warn); + border-color: var(--warn); +} + +/* trajectory */ +#traj-controls { + display: flex; + gap: 1rem; + align-items: center; + margin-bottom: 0.6rem; + font-size: 12px; +} +#traj-controls input[type="search"] { + flex: 1; + max-width: 320px; +} + +#trajectory { + list-style: none; + padding: 0; + margin: 0; +} + +.call { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 4px; + margin-bottom: 0.4rem; +} + +.call.error { + border-color: var(--bad); +} + +.call > .head { + display: flex; + align-items: center; + gap: 0.6rem; + padding: 0.4rem 0.6rem; + cursor: pointer; + user-select: none; +} + +.call > .head .idx { + color: var(--muted); + font-variant-numeric: tabular-nums; + min-width: 2.5em; +} + +.call > .head .name { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; +} + +.call > .head .name.error { + color: var(--bad); +} + +.call > .head .summary { + color: var(--muted); + font-size: 12px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + flex: 1; +} + +.call > .body { + display: none; + padding: 0 0.6rem 0.6rem 0.6rem; + border-top: 1px solid var(--border); +} + +.call.open > .body { + display: block; +} + +.call h4 { + margin: 0.6rem 0 0.2rem; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--muted); + font-weight: 600; +} + +pre.code { + background: var(--code-bg); + border: 1px solid var(--border); + border-radius: 3px; + padding: 0.5rem 0.7rem; + margin: 0; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; + max-height: 400px; + overflow: auto; +} + +pre.code.error { + border-color: var(--bad); +} + +.thinking { + background: var(--panel-2); + border-left: 3px solid var(--accent); + padding: 0.5rem 0.8rem; + margin: 0.4rem 0; + font-style: italic; + color: var(--muted); + white-space: pre-wrap; +} + +.assistant-text { + background: var(--panel-2); + border-left: 3px solid var(--good); + padding: 0.5rem 0.8rem; + margin: 0.4rem 0; + white-space: pre-wrap; +} + +/* answer/grading */ +#tab-answer pre { + background: var(--code-bg); + padding: 1rem; + border-radius: 4px; + white-space: pre-wrap; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12.5px; +} + +.grade-section { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 4px; + padding: 0.8rem 1rem; + margin-bottom: 0.8rem; +} +.grade-section h3 { + margin: 0 0 0.5rem 0; + font-size: 13px; + display: flex; + justify-content: space-between; + align-items: baseline; +} +.grade-section h3 .score { + font-variant-numeric: tabular-nums; + font-weight: 600; +} + +.check { + display: grid; + grid-template-columns: 1.4rem 1fr auto; + gap: 0.4rem; + padding: 0.25rem 0; + border-top: 1px solid var(--border); + align-items: center; +} +.check:first-of-type { + border-top: 0; +} +.check .marker.good { + color: var(--good); +} +.check .marker.bad { + color: var(--bad); +} +.check .id { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; +} +.check .weight { + color: var(--muted); + font-size: 11px; +} +.check.negative .id::after { + content: " (negative)"; + color: var(--warn); + font-size: 10px; +} + +.criterion { + border-top: 1px solid var(--border); + padding: 0.5rem 0; +} +.criterion:first-of-type { + border-top: 0; +} +.criterion-head { + display: flex; + justify-content: space-between; + align-items: baseline; +} +.criterion-name { + font-weight: 600; +} +.criterion-score { + font-variant-numeric: tabular-nums; +} +.criterion .rationale { + color: var(--muted); + margin-top: 0.2rem; + font-size: 12.5px; +} + +/* color the criterion score */ +.score-0 { + color: var(--bad); +} +.score-low { + color: var(--warn); +} +.score-mid { + color: var(--accent); +} +.score-high { + color: var(--good); +} + +#tab-raw pre { + background: var(--code-bg); + padding: 1rem; + border-radius: 4px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 11.5px; + white-space: pre-wrap; + word-break: break-word; +} + +.tag { + display: inline-block; + background: var(--panel-2); + border: 1px solid var(--border); + border-radius: 3px; + padding: 0 0.4rem; + font-size: 11px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; +} +.tag.error { + color: var(--bad); + border-color: var(--bad); +} +.tag.small { + font-size: 10px; + padding: 0 0.3rem; +} +.small { + font-size: 11px; +} + +/* ===== Summary Dashboard ===== */ + +#summary { + display: block; + flex: 1; + overflow: auto; + padding: 0; +} + +#summary[hidden] { + display: none; +} + +.summary-dashboard { + padding: 1rem 1.5rem 2rem; + max-width: 1200px; +} + +.summary-header { + display: flex; + align-items: center; + gap: 0.8rem; + margin-bottom: 1.2rem; + flex-wrap: wrap; +} +.summary-header .label { + color: var(--muted); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; +} +.summary-header .value { + font-weight: 500; +} +.summary-header .accent { + color: var(--accent); +} + +.summary-section { + margin-bottom: 1.5rem; +} +.summary-section h2 { + font-size: 15px; + font-weight: 600; + margin: 0 0 0.6rem 0; + color: var(--text); +} + +.sub-table { + margin-top: 0.8rem; +} +.sub-table h3 { + font-size: 12px; + font-weight: 600; + color: var(--muted); + text-transform: uppercase; + letter-spacing: 0.04em; + margin: 0 0 0.4rem 0; +} + +/* Summary tables */ +.summary-table { + width: 100%; + border-collapse: collapse; + font-variant-numeric: tabular-nums; + font-size: 12.5px; +} +.summary-table th, +.summary-table td { + padding: 0.4rem 0.7rem; + text-align: right; + border-bottom: 1px solid var(--border); +} +.summary-table th { + background: var(--panel); + color: var(--muted); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.04em; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} +.summary-table th:first-child, +.summary-table td:first-child { + text-align: left; +} +.summary-table tbody tr:hover { + background: var(--panel-2); +} +.summary-table .baseline-col { + color: var(--accent); +} +.summary-table .delta-col { + color: var(--muted); + font-size: 10px; +} +.scenario-name { + font-weight: 600; +} +.metric-label { + color: var(--text); + font-weight: 500; +} +.metric-value { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; +} + +/* Score cells */ +.score-cell { + font-weight: 600; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; +} +.score-cell.good { + color: var(--good); +} +.score-cell.warn { + color: var(--warn); +} +.score-cell.bad { + color: var(--bad); +} + +/* Delta cells */ +.delta-cell { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; + font-weight: 600; +} +.delta-pos { + color: var(--good); +} +.delta-neg { + color: var(--bad); +} +.delta-neutral { + color: var(--muted); +} + +/* Heatmap cells */ +.heatmap-cell { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; + text-align: center !important; +} +.heat-high { + color: var(--good); + background: rgba(108, 217, 126, 0.08); +} +.heat-mid { + color: var(--accent); + background: rgba(124, 196, 255, 0.06); +} +.heat-low { + color: var(--warn); + background: rgba(255, 200, 87, 0.06); +} +.heat-bad { + color: var(--bad); + background: rgba(255, 107, 107, 0.08); +} + +/* Programmatic check cells */ +.check-pass { + color: var(--good); + background: rgba(108, 217, 126, 0.08); +} +.check-partial { + color: var(--warn); + background: rgba(255, 200, 87, 0.06); +} +.check-fail { + color: var(--bad); + background: rgba(255, 107, 107, 0.08); +} + +/* Runs comparison table */ +.runs-table td { + vertical-align: middle; +} +.run-cell { + white-space: nowrap; +} +.run-cell.clickable { + cursor: pointer; +} +.run-cell.clickable:hover { + text-decoration: underline; + text-decoration-color: var(--accent); +} + +/* Scenario breakdown spacing */ +.scenario-breakdown { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 6px; + padding: 1rem 1.2rem; +} diff --git a/packages/hdx-eval/scripts/viewer/server.js b/packages/hdx-eval/scripts/viewer/server.js new file mode 100644 index 0000000000..aaefe0db45 --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/server.js @@ -0,0 +1,183 @@ +#!/usr/bin/env node +// HDX Eval Run Viewer — tiny stdlib HTTP server. +// Browses runs/ produced by hdx-eval: batches → scenarios → cells → run files, +// serving trajectory + grading JSON to a single-page UI. + +const http = require('node:http'); +const fs = require('node:fs'); +const path = require('node:path'); +const url = require('node:url'); + +const RUNS_DIR = path.resolve(__dirname, '..', '..', 'runs'); +const PUBLIC_DIR = path.join(__dirname, 'public'); +const PORT = Number(process.env.PORT) || 5176; + +const MIME = { + '.html': 'text/html; charset=utf-8', + '.js': 'application/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', +}; + +function sendJson(res, status, body) { + const buf = Buffer.from(JSON.stringify(body)); + res.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': buf.length, + 'cache-control': 'no-store', + }); + res.end(buf); +} + +function sendText(res, status, text, type = 'text/plain; charset=utf-8') { + const buf = Buffer.from(text); + res.writeHead(status, { 'content-type': type, 'content-length': buf.length }); + res.end(buf); +} + +function safeJoin(root, ...parts) { + const p = path.resolve(root, ...parts); + if (!p.startsWith(path.resolve(root))) throw new Error('path escapes root'); + return p; +} + +function readJsonSafe(p) { + try { + return JSON.parse(fs.readFileSync(p, 'utf8')); + } catch { + return null; + } +} + +function listBatches() { + if (!fs.existsSync(RUNS_DIR)) return []; + return fs + .readdirSync(RUNS_DIR, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .sort() + .reverse(); +} + +function listCells(batch) { + const batchDir = safeJoin(RUNS_DIR, batch); + if (!fs.existsSync(batchDir)) return null; + const scenarios = fs + .readdirSync(batchDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + const out = []; + for (const scenario of scenarios) { + const scenarioDir = path.join(batchDir, scenario); + const mcps = fs + .readdirSync(scenarioDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + for (const mcp of mcps) { + const cellDir = path.join(scenarioDir, mcp); + const files = fs.readdirSync(cellDir); + const runIdxs = new Set(); + for (const f of files) { + const m = /^(\d+)\.json$/.exec(f); + if (m) runIdxs.add(Number(m[1])); + } + const runs = [...runIdxs].sort((a, b) => a - b).map((i) => { + const grade = readJsonSafe(path.join(cellDir, `${i}.grade.json`)); + const traj = readJsonSafe(path.join(cellDir, `${i}.json`)); + return { + idx: i, + combinedScore: grade?.combinedScore ?? null, + programmaticScore: grade?.programmatic?.score ?? null, + judgeScore: grade?.judge?.weightedScore ?? null, + termination: traj?.termination ?? null, + durationMs: traj?.durationMs ?? null, + toolCalls: traj?.toolCalls?.length ?? null, + toolErrors: grade?.toolErrors?.errors ?? null, + }; + }); + out.push({ scenario, mcp, runs }); + } + } + return out; +} + +const ROUTES = [ + [ + /^\/api\/batches$/, + (_m, _q, res) => { + const batches = listBatches().map((name) => { + const summary = readJsonSafe(path.join(RUNS_DIR, name, '_summary.json')); + return { + name, + generatedAt: summary?.generatedAt ?? null, + scenarios: summary?.scenarios?.map((s) => s.scenario) ?? null, + }; + }); + sendJson(res, 200, { batches }); + }, + ], + [ + /^\/api\/batches\/([^/]+)$/, + (m, _q, res) => { + const batch = decodeURIComponent(m[1]); + const summary = readJsonSafe(safeJoin(RUNS_DIR, batch, '_summary.json')); + const cells = listCells(batch); + if (!cells) return sendJson(res, 404, { error: 'batch not found' }); + sendJson(res, 200, { batch, summary, cells }); + }, + ], + [ + /^\/api\/batches\/([^/]+)\/runs\/([^/]+)\/([^/]+)\/(\d+)$/, + (m, _q, res) => { + const [, batch, scenario, mcp, idx] = m.map((x, i) => + i === 0 ? x : decodeURIComponent(x), + ); + const base = safeJoin(RUNS_DIR, batch, scenario, mcp); + const trajectory = readJsonSafe(path.join(base, `${idx}.json`)); + const grade = readJsonSafe(path.join(base, `${idx}.grade.json`)); + if (!trajectory) return sendJson(res, 404, { error: 'run not found' }); + sendJson(res, 200, { trajectory, grade }); + }, + ], +]; + +function handleApi(req, res) { + const parsed = url.parse(req.url, true); + for (const [re, fn] of ROUTES) { + const m = re.exec(parsed.pathname); + if (m) return fn(m, parsed.query, res); + } + sendJson(res, 404, { error: 'not found' }); +} + +function handleStatic(req, res) { + let pathname = url.parse(req.url).pathname; + if (pathname === '/') pathname = '/index.html'; + let filePath; + try { + filePath = safeJoin(PUBLIC_DIR, '.' + pathname); + } catch { + return sendText(res, 400, 'bad path'); + } + fs.readFile(filePath, (err, data) => { + if (err) return sendText(res, 404, 'not found'); + const ext = path.extname(filePath).toLowerCase(); + res.writeHead(200, { + 'content-type': MIME[ext] || 'application/octet-stream', + 'content-length': data.length, + 'cache-control': 'no-store', + }); + res.end(data); + }); +} + +const server = http.createServer((req, res) => { + if (req.url.startsWith('/api/')) return handleApi(req, res); + handleStatic(req, res); +}); + +server.listen(PORT, '127.0.0.1', () => { + console.log(`hdx-eval viewer: http://localhost:${PORT}`); + console.log(` runs dir: ${RUNS_DIR}`); +}); diff --git a/packages/hdx-eval/src/__tests__/aggregate.test.ts b/packages/hdx-eval/src/__tests__/aggregate.test.ts new file mode 100644 index 0000000000..9227de4aa4 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/aggregate.test.ts @@ -0,0 +1,264 @@ +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; +import { buildAggregate, type GradedRunPair } from '../reports/aggregate'; + +function pair(args: { + scenario: string; + mcp: string; + i: number; + programmaticScore: number; + judgeScore: number; + toolCalls: number; + outputTokens: number; + durationMs: number; +}): GradedRunPair { + const run: RunRecord = { + schemaVersion: 1, + runId: `${args.scenario}-${args.mcp}-${args.i}`, + scenario: args.scenario, + mcp: args.mcp, + model: 'claude-sonnet-4-6', + runIndex: args.i, + seed: 42, + startedAt: '2026-05-09T07:50:00.000Z', + endedAt: '2026-05-09T07:51:00.000Z', + durationMs: args.durationMs, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: Array.from({ length: args.toolCalls }, (_, k) => ({ + name: `t${k}`, + input: null, + output: null, + isError: false, + startedAt: '', + endedAt: null, + durationMs: null, + })), + messages: [], + finalAnswer: 'a', + tokens: { + input: 100, + output: args.outputTokens, + cacheCreation: 0, + cacheRead: 0, + }, + totalCostUsd: 0.01, + stderr: '', + }; + const grade: GradeRecord = { + schemaVersion: 2, + runId: run.runId, + scenario: run.scenario, + mcp: run.mcp, + programmatic: { + hits: [ + { + id: 'check-a', + weight: 1, + matched: args.programmaticScore >= 0.5, + satisfied: args.programmaticScore >= 0.5, + }, + { + id: 'check-b', + weight: 1, + matched: args.programmaticScore >= 1, + satisfied: args.programmaticScore >= 1, + }, + ], + score: args.programmaticScore, + }, + judge: { + model: 'claude-opus-4-7', + scores: { + correctness: { score: Math.round(args.judgeScore * 5), rationale: '' }, + }, + weightedScore: args.judgeScore, + rawResponse: '', + durationMs: 1000, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + }, + toolErrors: { + total: args.toolCalls, + errors: 0, + rate: 0, + penalty: 0, + samples: [], + }, + combinedScore: 0.4 * args.programmaticScore + 0.6 * args.judgeScore, + gradedAt: '', + judgeModel: 'claude-opus-4-7', + }; + return { run, grade }; +} + +describe('buildAggregate', () => { + it('groups by scenario and computes per-cell means with baseline delta', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'error-root-cause', + mcp: 'clickhouse', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 7, + outputTokens: 1500, + durationMs: 30_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 13, + outputTokens: 4000, + durationMs: 60_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 1, + programmaticScore: 0.8, + judgeScore: 0.8, + toolCalls: 11, + outputTokens: 3500, + durationMs: 50_000, + }), + ]; + // Baseline defaults to first MCP alphabetically: 'clickhouse'. + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.scenarios).toHaveLength(1); + expect(summary.baseline).toBe('clickhouse'); + expect(summary.mcpOrder).toEqual(['clickhouse', 'hyperdx']); + const sc = summary.scenarios[0]; + const h = sc.cells['hyperdx']!; + const c = sc.cells['clickhouse']!; + expect(h.n).toBe(2); + expect(c.n).toBe(1); + expect(h.toolCalls.mean).toBeCloseTo(12, 5); + expect(c.toolCalls.mean).toBeCloseTo(7, 5); + expect(h.programmatic.mean).toBeCloseTo(0.9, 5); + expect(h.judge.weightedMean).toBeCloseTo(0.85, 5); + expect(h.combinedScore.mean).toBeCloseTo(0.87, 5); + // Delta is hyperdx (challenger) - clickhouse (baseline) + expect(sc.deltas['hyperdx'].toolCalls).toBeCloseTo(5, 5); + expect(sc.deltas['hyperdx'].combinedScore!).toBeCloseTo(-0.13, 5); + // No delta for baseline itself + expect(sc.deltas['clickhouse']).toBeUndefined(); + }); + + it('supports explicit baseline selection', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 13, + outputTokens: 4000, + durationMs: 60_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'clickhouse', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 7, + outputTokens: 1500, + durationMs: 30_000, + }), + ]; + const summary = buildAggregate({ + batchDir: '/tmp/x', + pairs, + baseline: 'hyperdx', + }); + expect(summary.baseline).toBe('hyperdx'); + const sc = summary.scenarios[0]; + // Delta should be clickhouse (challenger) - hyperdx (baseline) + expect(sc.deltas['clickhouse']).toBeDefined(); + expect(sc.deltas['hyperdx']).toBeUndefined(); + }); + + it('produces an ordered, multi-scenario summary', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'noisy-signals', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 9, + outputTokens: 3200, + durationMs: 50_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 13, + outputTokens: 4000, + durationMs: 60_000, + }), + ]; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.scenarios.map(s => s.scenario)).toEqual([ + 'error-root-cause', + 'noisy-signals', + ]); + }); + + it('handles 3+ MCPs with baseline + challengers model', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'error-root-cause', + mcp: 'alpha', + i: 0, + programmaticScore: 0.8, + judgeScore: 0.8, + toolCalls: 10, + outputTokens: 3000, + durationMs: 40_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'beta', + i: 0, + programmaticScore: 0.9, + judgeScore: 0.9, + toolCalls: 12, + outputTokens: 3500, + durationMs: 50_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'gamma', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 8, + outputTokens: 2000, + durationMs: 25_000, + }), + ]; + const summary = buildAggregate({ + batchDir: '/tmp/x', + pairs, + baseline: 'alpha', + }); + expect(summary.mcpOrder).toEqual(['alpha', 'beta', 'gamma']); + const sc = summary.scenarios[0]; + // Challengers: beta and gamma + expect(sc.deltas['alpha']).toBeUndefined(); + expect(sc.deltas['beta']).toBeDefined(); + expect(sc.deltas['gamma']).toBeDefined(); + expect(sc.deltas['gamma'].toolCalls).toBeCloseTo(-2, 5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/blind.test.ts b/packages/hdx-eval/src/__tests__/blind.test.ts new file mode 100644 index 0000000000..1b73984301 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/blind.test.ts @@ -0,0 +1,102 @@ +import { + blindAnswer, + type BlindingEntry, + buildBlindingEntries, +} from '../grading/blind'; +import type { McpDefinition } from '../harness/types'; + +const hyperdxDef: McpDefinition = { + type: 'http', + url: 'http://localhost/mcp', + toolPattern: 'mcp__hyperdx__*', + label: 'HyperDX', + brandTerms: ['HyperDX', 'hyperdx'], +}; + +const clickhouseDef: McpDefinition = { + type: 'stdio', + command: 'uv', + toolPattern: 'mcp__clickhouse__*', + label: 'ClickHouse MCP', + brandTerms: ['ClickHouse MCP', 'clickhouse'], +}; + +const entries: BlindingEntry[] = buildBlindingEntries([ + { kind: 'hyperdx', def: hyperdxDef }, + { kind: 'clickhouse', def: clickhouseDef }, +]); + +describe('blindAnswer', () => { + it('returns text unchanged when no entries are provided', () => { + const text = 'I called mcp__hyperdx__hyperdx_query to find...'; + expect(blindAnswer(text)).toBe(text); + expect(blindAnswer(text, [])).toBe(text); + }); + + it('redacts mcp__hyperdx__ tool prefix', () => { + const out = blindAnswer( + 'I called mcp__hyperdx__hyperdx_query to find...', + entries, + ); + expect(out).not.toContain('mcp__hyperdx__'); + expect(out).toContain('mcp__redacted__'); + }); + + it('redacts mcp__clickhouse__ tool prefix', () => { + const out = blindAnswer( + 'Used mcp__clickhouse__run_query to look up...', + entries, + ); + expect(out).not.toContain('mcp__clickhouse__'); + expect(out).toContain('mcp__redacted__'); + }); + + it('replaces HyperDX brand mentions with anonymous label', () => { + const out = blindAnswer('Using HyperDX I confirmed the issue...', entries); + expect(out).toContain('MCP A'); + expect(out).not.toContain('HyperDX'); + }); + + it('replaces ClickHouse MCP brand mentions with anonymous label', () => { + const out = blindAnswer('Using ClickHouse MCP I confirmed...', entries); + expect(out).toContain('MCP B'); + expect(out).not.toContain('ClickHouse MCP'); + }); + + it('preserves data references that are part of the answer', () => { + const answer = + 'Root cause: payment-service db.payment.connect timeout. ' + + 'Affected users hit checkout-api 5xx.'; + const out = blindAnswer(answer, entries); + expect(out).toContain('payment-service'); + expect(out).toContain('db.payment.connect'); + expect(out).toContain('checkout-api'); + }); + + it('assigns sequential labels: MCP A, MCP B, MCP C, ...', () => { + const threeEntries = buildBlindingEntries([ + { kind: 'alpha', def: { ...hyperdxDef, brandTerms: ['AlphaMCP'] } }, + { kind: 'beta', def: { ...clickhouseDef, brandTerms: ['BetaMCP'] } }, + { + kind: 'gamma', + def: { + ...hyperdxDef, + toolPattern: 'mcp__gamma__*', + brandTerms: ['GammaMCP'], + }, + }, + ]); + expect(threeEntries[0].anonLabel).toBe('MCP A'); + expect(threeEntries[1].anonLabel).toBe('MCP B'); + expect(threeEntries[2].anonLabel).toBe('MCP C'); + + const out = blindAnswer( + 'Used AlphaMCP and GammaMCP but not BetaMCP.', + threeEntries, + ); + expect(out).toContain('MCP A'); + expect(out).toContain('MCP C'); + expect(out).toContain('MCP B'); + expect(out).not.toMatch(/AlphaMCP|BetaMCP|GammaMCP/); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/error-root-cause.test.ts b/packages/hdx-eval/src/__tests__/error-root-cause.test.ts new file mode 100644 index 0000000000..2f107fb29b --- /dev/null +++ b/packages/hdx-eval/src/__tests__/error-root-cause.test.ts @@ -0,0 +1,228 @@ +import { mulberry32 } from '../rng/seeded'; +import { errorRootCauseScenario } from '../scenarios/error-root-cause/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-08T12:00:00.000Z'); +// Tests run at 1% volume (full v5 ≈ 6M base traces / 12M logs). +// Pattern shares scale linearly; planted anomaly + distractors stay fixed. +const TEST_VOLUME_FACTOR = 0.01; + +function run(seed: number) { + return collectScenario( + errorRootCauseScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('error-root-cause scenario', () => { + // Generate once per run() to avoid 5x cost across tests. + const result = run(42); + + it('produces high-volume background + distractors + anomaly at 1%', () => { + // Full v5: 6M base × ~1.6 spans + distractors → ~9.6M+ spans. + // At 1%: 60K × ~1.6 + ~465 distractors → ~96K traces. + expect(result.traces.length).toBeGreaterThan(90_000); + expect(result.traces.length).toBeLessThan(110_000); + }); + + it('produces ~120K templated background logs at 1% (12M at full scale)', () => { + expect(result.logs.length).toBeGreaterThanOrEqual(100_000); + expect(result.logs.length).toBeLessThanOrEqual(140_000); + }); + + it('uses 25 services (high cardinality)', () => { + const services = new Set(result.traces.map(t => t.serviceName)); + expect(services.size).toBeGreaterThanOrEqual(20); + }); + + it('attaches region and tenant.id attributes to background traces', () => { + const sample = result.traces.slice(0, 1000); + const withRegion = sample.filter(t => t.spanAttributes['cloud.region']); + expect(withRegion.length).toBeGreaterThan(500); + const regions = new Set( + sample.map(t => t.spanAttributes['cloud.region']).filter(Boolean), + ); + expect(regions.size).toBeGreaterThanOrEqual(3); + }); + + it('is deterministic for a fixed seed and now', () => { + const b = run(42); + expect(b.traces.length).toBe(result.traces.length); + expect(b.logs.length).toBe(result.logs.length); + // Cheap structural check — sampling spans by index for byte-equality is too slow at 100x volume. + expect(result.traces[0].spanId).toBe(b.traces[0].spanId); + expect(result.traces[1000].spanId).toBe(b.traces[1000].spanId); + expect(result.logs[5000].body).toBe(b.logs[5000].body); + }); + + it('plants exactly 8 payment-service db.payment.connect error spans in the anomaly window (small relative to decoy bursts)', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const dbErrors = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'db.payment.connect' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ); + expect(dbErrors.length).toBe(8); + }); + + it('uses 3 timeout message variants on the planted db.payment.connect spans', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const dbErrors = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'db.payment.connect' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ); + const variants = new Set(dbErrors.map(t => t.statusMessage)); + expect(variants.size).toBe(3); + expect( + [...variants].every(v => v.toLowerCase().includes('db-payment')), + ).toBe(true); + }); + + it('propagates the error up to checkout-api root spans in the anomaly window', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const checkoutErrors = result.traces.filter( + t => + t.serviceName === 'checkout-api' && + t.spanName === 'POST /api/checkout' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ); + expect(checkoutErrors.length).toBe(8); + }); + + it('emits correlated payment-service error logs with the same TraceId', () => { + const erroringTraceIds = new Set( + result.traces + .filter( + t => + t.serviceName === 'payment-service' && + t.statusCode === 'STATUS_CODE_ERROR', + ) + .map(t => t.traceId), + ); + const correlated = result.logs.filter( + l => + l.serviceName === 'payment-service' && + l.severityText === 'ERROR' && + l.body.includes('db-payment') && + l.traceId && + erroringTraceIds.has(l.traceId), + ); + expect(correlated.length).toBe(8); + }); + + it('plants the api-gateway TLS distractor at T-25min (not in current incident window)', () => { + const distractorStart = NOW_MS - 25 * 60 * 1000; + const distractorEnd = distractorStart + 5 * 60 * 1000; + const tls = result.traces.filter( + t => + t.serviceName === 'api-gateway' && + t.spanAttributes['error.type'] === 'TLSHandshakeError' && + t.timestampMs >= distractorStart && + t.timestampMs <= distractorEnd, + ); + expect(tls.length).toBeGreaterThanOrEqual(40); + // Should NOT be in the current incident window (last 10 min) + const recent = tls.filter(t => t.timestampMs >= NOW_MS - 10 * 60 * 1000); + expect(recent.length).toBe(0); + }); + + it('plants the auth-service rate-limit distractor at T-40min (resolved before incident)', () => { + const auth = result.traces.filter( + t => + t.serviceName === 'auth-service' && + t.spanAttributes['error.type'] === 'RateLimitExceeded', + ); + expect(auth.length).toBeGreaterThanOrEqual(15); + const recent = auth.filter(t => t.timestampMs >= NOW_MS - 35 * 60 * 1000); + expect(recent.length).toBe(0); + }); + + it('plants slow-but-OK search-service queries throughout the hour (not errors)', () => { + const slowSearch = result.traces.filter( + t => + t.serviceName === 'search-service' && + t.spanName === 'search.query' && + t.statusCode === 'STATUS_CODE_OK', + ); + expect(slowSearch.length).toBeGreaterThanOrEqual(70); + }); + + it('keeps payment-service free of DB-cascade errors outside the anomaly window', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const preAnomalyCascade = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'db.payment.connect' && + t.timestampMs < anomalyStart, + ); + expect(preAnomalyCascade.length).toBe(0); + }); + + it('plants 200 background payment.charge declines distributed across the hour (operational, not infra)', () => { + const declines = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'payment.charge' && + t.spanAttributes['error.type'] === 'PaymentDeclined', + ); + expect(declines.length).toBe(200); + const reasons = new Set(declines.map(t => t.statusMessage)); + expect(reasons.size).toBe(5); + }); + + it('plants 80 concurrent cdn origin-fetch errors in last 5 min, separate from checkout traces (dominates raw 5xx counts)', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const cdn = result.traces.filter( + t => + t.serviceName === 'cdn' && + t.spanAttributes['error.type'] === 'OriginUnreachableError' && + t.timestampMs >= anomalyStart, + ); + expect(cdn.length).toBe(80); + const checkoutCascadeTraceIds = new Set( + result.traces + .filter( + t => + t.serviceName === 'checkout-api' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ) + .map(t => t.traceId), + ); + const overlap = cdn.filter(t => checkoutCascadeTraceIds.has(t.traceId)); + expect(overlap.length).toBe(0); + }); + + it('plants 25 concurrent notification-service smtp errors in last 5 min, separate from checkout traces', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const smtp = result.traces.filter( + t => + t.serviceName === 'notification-service' && + t.spanAttributes['error.type'] === 'SMTPConnectionError' && + t.timestampMs >= anomalyStart, + ); + expect(smtp.length).toBe(25); + // None of these should share traceids with the planted checkout cascade. + const checkoutCascadeTraceIds = new Set( + result.traces + .filter( + t => + t.serviceName === 'checkout-api' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ) + .map(t => t.traceId), + ); + const overlap = smtp.filter(t => checkoutCascadeTraceIds.has(t.traceId)); + expect(overlap.length).toBe(0); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/grade.test.ts b/packages/hdx-eval/src/__tests__/grade.test.ts new file mode 100644 index 0000000000..f8a035167a --- /dev/null +++ b/packages/hdx-eval/src/__tests__/grade.test.ts @@ -0,0 +1,169 @@ +/** + * Tests for the tool-error penalty applied by gradeBatch. We can't easily + * test the full pipeline (it touches the LLM judge), so instead these tests + * cover the pure computeToolErrorStats helper by re-implementing the call + * pattern: build a synthetic RunRecord, write it to a temp batch dir, run + * gradeBatch with --no-judge, and read back the grade JSON. + */ + +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { gradeBatch } from '../grading/grade'; +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; + +function buildRun(args: { + scenario: string; + toolCalls: Array<{ name: string; isError: boolean; output?: string }>; + finalAnswer: string; +}): RunRecord { + return { + schemaVersion: 1, + runId: `${args.scenario}-test-0`, + scenario: args.scenario, + mcp: 'hyperdx', + model: 'claude-sonnet-4-6', + runIndex: 0, + seed: 42, + startedAt: '2026-05-10T00:00:00.000Z', + endedAt: '2026-05-10T00:01:00.000Z', + durationMs: 60_000, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: args.toolCalls.map(c => ({ + name: c.name, + input: null, + output: c.output ?? null, + isError: c.isError, + startedAt: '', + endedAt: null, + durationMs: null, + })), + messages: [], + finalAnswer: args.finalAnswer, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + totalCostUsd: 0, + stderr: '', + }; +} + +function writeBatch( + batchDir: string, + scenario: string, + mcp: string, + run: RunRecord, +): void { + const dir = join(batchDir, scenario, mcp); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, '0.json'), JSON.stringify(run, null, 2)); +} + +describe('gradeBatch tool-error penalty', () => { + const tmpRoot = join('/tmp', `hdx-eval-grade-test-${Date.now()}`); + // Answer that hits every positive check in the error-root-cause rubric: + // service, error.type, full db hostname, specific db span, cascade name, + // ruled-out distractor — without triggering any negative blame-pattern. + const ANSWER = + 'Root cause: payment-service ConnectionTimeoutError on db.payment.connect ' + + 'reaching db-payment.internal — DB connection timeout — cascading into ' + + 'checkout-api 5xx errors. Ruled out concurrent SMTP and CDN bursts ' + + '(separate trace trees, no checkout parent).'; + + afterAll(() => { + rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it('records zero penalty when all tool calls succeeded', async () => { + const batchDir = join(tmpRoot, 'ok'); + writeBatch( + batchDir, + 'error-root-cause', + 'hyperdx', + buildRun({ + scenario: 'error-root-cause', + toolCalls: [ + { name: 'mcp__hyperdx__hyperdx_list_sources', isError: false }, + { name: 'mcp__hyperdx__hyperdx_query', isError: false }, + ], + finalAnswer: ANSWER, + }), + ); + const result = await gradeBatch(batchDir, { skipJudge: true }); + expect(result.graded).toHaveLength(1); + const grade = result.graded[0]; + expect(grade.toolErrors.total).toBe(2); + expect(grade.toolErrors.errors).toBe(0); + expect(grade.toolErrors.rate).toBe(0); + expect(grade.toolErrors.penalty).toBe(0); + }); + + it('penalizes runs where tool calls failed', async () => { + const batchDir = join(tmpRoot, 'failed'); + writeBatch( + batchDir, + 'error-root-cause', + 'hyperdx', + buildRun({ + scenario: 'error-root-cause', + toolCalls: [ + { + name: 'mcp__hyperdx__hyperdx_list_sources', + isError: true, + output: 'Maximum call stack size exceeded', + }, + { + name: 'mcp__hyperdx__hyperdx_list_sources', + isError: true, + output: 'Maximum call stack size exceeded', + }, + { + name: 'mcp__hyperdx__hyperdx_query', + isError: true, + output: 'sourceId must be a 24-char hex ObjectId', + }, + { name: 'mcp__hyperdx__hyperdx_query', isError: false }, + ], + finalAnswer: ANSWER, + }), + ); + const result = await gradeBatch(batchDir, { skipJudge: true }); + const grade = result.graded[0]; + expect(grade.toolErrors.total).toBe(4); + expect(grade.toolErrors.errors).toBe(3); + expect(grade.toolErrors.rate).toBeCloseTo(0.75, 5); + // Penalty is capped at MAX_ERROR_PENALTY (0.2) + expect(grade.toolErrors.penalty).toBeCloseTo(0.2, 5); + expect(grade.toolErrors.samples).toHaveLength(3); + expect(grade.toolErrors.samples[0].name).toMatch(/list_sources/); + }); + + it('subtracts the penalty from combinedScore (and clamps to [0,1])', async () => { + const batchDir = join(tmpRoot, 'clamped'); + // Programmatic rubric will hit, but every tool call fails. + writeBatch( + batchDir, + 'error-root-cause', + 'hyperdx', + buildRun({ + scenario: 'error-root-cause', + toolCalls: Array.from({ length: 5 }, () => ({ + name: 'mcp__hyperdx__hyperdx_list_sources', + isError: true, + output: 'fail', + })), + finalAnswer: ANSWER, + }), + ); + const result = await gradeBatch(batchDir, { skipJudge: true }); + const grade = result.graded[0]; + // Programmatic score = 1.0 (answer hits every positive check). Without + // judge, raw combined is programmatic.score. Penalty is 0.2. Final: + // 1.0 - 0.2 = 0.8. + expect(grade.programmatic.score).toBeCloseTo(1, 5); + expect(grade.combinedScore).toBeCloseTo(0.8, 5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/judgePrompt.test.ts b/packages/hdx-eval/src/__tests__/judgePrompt.test.ts new file mode 100644 index 0000000000..2012176aa1 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/judgePrompt.test.ts @@ -0,0 +1,62 @@ +import { + buildJudgeSystem, + buildJudgeUser, + formatGroundTruthFacts, +} from '../grading/judgePrompt'; +import { loadScenarioRubric } from '../grading/rubric'; +import { getScenario } from '../scenarios'; + +describe('buildJudgeSystem', () => { + it('includes the scenario name and every criterion in the rubric', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const sys = buildJudgeSystem('error-root-cause', rubric); + expect(sys).toContain('SCENARIO: error-root-cause'); + for (const c of rubric.judge.criteria) { + expect(sys).toContain(`"${c.id}"`); + expect(sys).toContain(c.description); + } + }); + + it('instructs strict JSON output', () => { + const rubric = loadScenarioRubric('latency-spike'); + const sys = buildJudgeSystem('latency-spike', rubric); + expect(sys).toMatch(/STRICT JSON/); + expect(sys).toMatch(/"scores"/); + }); + + it('asks the judge to ignore tool choice', () => { + const rubric = loadScenarioRubric('noisy-signals'); + const sys = buildJudgeSystem('noisy-signals', rubric); + expect(sys.toLowerCase()).toContain('tool choice'); + }); +}); + +describe('buildJudgeUser', () => { + it('includes the scenario question, ground truth, and candidate answer', () => { + const user = buildJudgeUser({ + scenarioPrompt: 'Why is checkout failing?', + groundTruthFacts: '- root cause: payment-service', + candidateAnswer: 'It is the payment service.', + }); + expect(user).toContain('Why is checkout failing?'); + expect(user).toContain('root cause: payment-service'); + expect(user).toContain('It is the payment service.'); + }); +}); + +describe('formatGroundTruthFacts', () => { + it('renders nested expected fields as bullets', () => { + const scenario = getScenario('error-root-cause'); + const out = formatGroundTruthFacts(scenario.groundTruth); + expect(out).toContain('rootCauseService: payment-service'); + expect(out).toContain('errorBodyPattern: db-payment'); + }); + + it('handles arrays/objects via JSON for non-scalar values', () => { + const scenario = getScenario('noisy-signals'); + const out = formatGroundTruthFacts(scenario.groundTruth); + // compositeCells is an array — it gets JSON-stringified + expect(out).toContain('compositeCells:'); + expect(out).toContain('inventory'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/latency-spike.test.ts b/packages/hdx-eval/src/__tests__/latency-spike.test.ts new file mode 100644 index 0000000000..38f147266f --- /dev/null +++ b/packages/hdx-eval/src/__tests__/latency-spike.test.ts @@ -0,0 +1,200 @@ +import { mulberry32 } from '../rng/seeded'; +import { latencySpikeScenario } from '../scenarios/latency-spike/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-08T12:00:00.000Z'); +// Tests run at 1% volume to keep memory tractable (full v5 ≈ 6M base +// traces / ~12M+ spans). Pattern shares scale linearly so structural +// invariants still hold at 1%. +const TEST_VOLUME_FACTOR = 0.01; + +function run(seed: number) { + return collectScenario( + latencySpikeScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +function nsToMs(ns: number): number { + return ns / 1_000_000; +} + +describe('latency-spike scenario', () => { + const result = run(42); + + it('produces high-volume trace data at 1% (~60K planted + ~40K background = ~200K spans)', () => { + // Full v6: 6M planted × ~2 spans + 4M background × 2 spans + extras + // → ~20M+ spans. At 1%: ~120K + ~80K + extras → ~200-215K spans. + expect(result.traces.length).toBeGreaterThan(195_000); + expect(result.traces.length).toBeLessThan(220_000); + }); + + it('uses 12 api-server endpoints and 5 regions', () => { + const sample = result.traces + .filter(t => t.serviceName === 'api-server') + .slice(0, 5000); + const endpoints = new Set(sample.map(t => t.spanName)); + const regions = new Set(sample.map(t => t.spanAttributes['cloud.region'])); + expect(endpoints.size).toBe(12); + expect(regions.size).toBe(5); + const tenantIds = new Set(sample.map(t => t.spanAttributes['tenant.id'])); + expect(tenantIds.size).toBeGreaterThan(2000); + }); + + it('plants production-shaped (ServiceName, SpanName) cardinality', () => { + // Background traffic should produce hundreds of distinct + // (Service, SpanName) combinations so naive + // `GROUP BY ServiceName, SpanName` returns too many rows to scan. + const combos = new Set( + result.traces.map(t => `${t.serviceName}::${t.spanName}`), + ); + expect(combos.size).toBeGreaterThan(800); + // Background should yield at least 80 distinct service names beyond + // the 4-service planted/decoy pool. + const services = new Set(result.traces.map(t => t.serviceName)); + expect(services.size).toBeGreaterThan(80); + }); + + it('is deterministic for a fixed seed and now', () => { + const b = run(42); + expect(b.traces.length).toBe(result.traces.length); + expect(result.traces[0].spanId).toBe(b.traces[0].spanId); + expect(result.traces[10_000].spanId).toBe(b.traces[10_000].spanId); + }); + + it('produces enterprise spans in the anomaly window with ~80% slow / ~20% cache-hit', () => { + const anomalyStart = NOW_MS - 10 * 60 * 1000; + const enterpriseInWindow = result.traces.filter( + t => + t.spanName === 'GET /api/orders/search' && + t.timestampMs >= anomalyStart && + t.spanAttributes['tenant.tier'] === 'enterprise', + ); + expect(enterpriseInWindow.length).toBeGreaterThan(20); + const slow = enterpriseInWindow.filter(t => nsToMs(t.durationNs) >= 1500); + const fast = enterpriseInWindow.filter(t => nsToMs(t.durationNs) < 100); + const slowFraction = slow.length / enterpriseInWindow.length; + expect(slowFraction).toBeGreaterThan(0.65); + expect(slowFraction).toBeLessThan(0.95); + expect(fast.length).toBeGreaterThan(0); // cache-hit subset exists + }); + + it('keeps enterprise traffic on /api/orders/search fast outside the anomaly window', () => { + const anomalyStart = NOW_MS - 10 * 60 * 1000; + const enterprisePreAnomaly = result.traces.filter( + t => + t.spanName === 'GET /api/orders/search' && + t.timestampMs < anomalyStart && + t.spanAttributes['tenant.tier'] === 'enterprise' && + // Exclude distractors that legitimately add latency + !t.spanAttributes['runtime.event'], + ); + if (enterprisePreAnomaly.length > 50) { + const tail = enterprisePreAnomaly + .map(t => nsToMs(t.durationNs)) + .sort((a, b) => b - a)[0]; + expect(tail).toBeLessThan(1500); + } + }); + + it('plants GC-pause distractors uniformly across tiers (not enterprise-concentrated)', () => { + const gcPauses = result.traces.filter( + t => t.spanAttributes['runtime.event'] === 'gc_pause', + ); + expect(gcPauses.length).toBeGreaterThan(50); + const enterprisePauseFraction = + gcPauses.filter(t => t.spanAttributes['tenant.tier'] === 'enterprise') + .length / gcPauses.length; + // Enterprise tier weight is 5%; GC pauses should match that distribution. + expect(enterprisePauseFraction).toBeLessThan(0.15); + }); + + it('plants cold-start distractors only at the start of the hour', () => { + const coldStarts = result.traces.filter( + t => t.spanAttributes['runtime.event'] === 'cold_start', + ); + expect(coldStarts.length).toBeGreaterThan(20); + const inLast10Min = coldStarts.filter( + t => t.timestampMs >= NOW_MS - 10 * 60 * 1000, + ); + expect(inLast10Min.length).toBe(0); + }); + + it('emits cache.lookup distractor spans on a fraction of traces', () => { + const cacheLookups = result.traces.filter( + t => t.spanName === 'cache.lookup', + ); + expect(cacheLookups.length).toBeGreaterThan(500); + }); + + it('includes GET /api/orders/search among the endpoints', () => { + const endpoints = new Set( + result.traces + .filter(t => t.serviceName === 'api-server') + .map(t => t.spanName), + ); + expect(endpoints.has('GET /api/orders/search')).toBe(true); + }); + + it('every parent span has a child query (db or elasticsearch) with the same TraceId', () => { + const parents = result.traces.filter(t => t.serviceName === 'api-server'); + const childTraceIds = new Set( + result.traces + .filter( + t => + t.spanName === 'database.query' || + t.spanName === 'elasticsearch.query', + ) + .map(t => t.traceId), + ); + let missing = 0; + for (let i = 0; i < parents.length; i += Math.ceil(parents.length / 1000)) { + if (!childTraceIds.has(parents[i].traceId)) missing++; + } + expect(missing).toBe(0); + }); + + it('plants concurrent products/search hot-shard slowness in same window, all tiers', () => { + const anomalyStart = NOW_MS - 10 * 60 * 1000; + const productsHotShard = result.traces.filter( + t => + t.spanName === 'elasticsearch.query' && + t.spanAttributes['elasticsearch.shard.hot'] === 'true' && + t.timestampMs >= anomalyStart, + ); + expect(productsHotShard.length).toBeGreaterThan(50); + // Affects all tiers — collect tiers via parent traceId + const productsHotTraceIds = new Set(productsHotShard.map(t => t.traceId)); + const parents = result.traces.filter( + t => + t.spanName === 'GET /api/products/search' && + productsHotTraceIds.has(t.traceId), + ); + const tiers = new Set(parents.map(t => t.spanAttributes['tenant.tier'])); + expect(tiers.size).toBeGreaterThanOrEqual(2); + }); + + it('feature_flag.experiment_a is correlated with enterprise but also enabled on free/pro', () => { + const sample = result.traces + .filter(t => t.serviceName === 'api-server') + .slice(0, 5000); + const enterprise = sample.filter( + t => t.spanAttributes['tenant.tier'] === 'enterprise', + ); + const free = sample.filter(t => t.spanAttributes['tenant.tier'] === 'free'); + const enterpriseOnRate = + enterprise.filter( + t => t.spanAttributes['feature_flag.experiment_a'] === 'enabled', + ).length / Math.max(enterprise.length, 1); + const freeOnRate = + free.filter( + t => t.spanAttributes['feature_flag.experiment_a'] === 'enabled', + ).length / Math.max(free.length, 1); + expect(enterpriseOnRate).toBeGreaterThan(0.6); + expect(freeOnRate).toBeGreaterThan(0.15); + expect(freeOnRate).toBeLessThan(0.5); // confounder, not perfectly correlated + }); +}); diff --git a/packages/hdx-eval/src/__tests__/markdown.test.ts b/packages/hdx-eval/src/__tests__/markdown.test.ts new file mode 100644 index 0000000000..1ed1fe60c7 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/markdown.test.ts @@ -0,0 +1,116 @@ +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; +import { buildAggregate, type GradedRunPair } from '../reports/aggregate'; +import { renderMarkdownReport } from '../reports/markdown'; + +function pair(scenario: string, mcp: string, i: number): GradedRunPair { + const run: RunRecord = { + schemaVersion: 1, + runId: `${scenario}-${mcp}-${i}`, + scenario, + mcp, + model: 'claude-sonnet-4-6', + runIndex: i, + seed: 42, + startedAt: '2026-05-09T07:50:00.000Z', + endedAt: '2026-05-09T07:51:00.000Z', + durationMs: 60_000, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: [], + messages: [], + finalAnswer: 'a', + tokens: { input: 50, output: 1000, cacheCreation: 0, cacheRead: 5000 }, + totalCostUsd: 0.01, + stderr: '', + }; + const grade: GradeRecord = { + schemaVersion: 2, + runId: run.runId, + scenario, + mcp, + programmatic: { + hits: [{ id: 'names_root', weight: 1, matched: true, satisfied: true }], + score: 1, + }, + judge: { + model: 'claude-opus-4-7', + scores: { + correctness: { score: 5, rationale: '' }, + completeness: { score: 4, rationale: '' }, + }, + weightedScore: 0.9, + rawResponse: '', + durationMs: 1000, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + }, + toolErrors: { total: 1, errors: 0, rate: 0, penalty: 0, samples: [] }, + combinedScore: 0.94, + gradedAt: '', + judgeModel: 'claude-opus-4-7', + }; + return { run, grade }; +} + +describe('renderMarkdownReport', () => { + const summary = buildAggregate({ + batchDir: '/tmp/2026-05-09T07-50-58-566Z', + pairs: [ + pair('error-root-cause', 'hyperdx', 0), + pair('error-root-cause', 'clickhouse', 0), + pair('latency-spike', 'hyperdx', 0), + pair('latency-spike', 'clickhouse', 0), + ], + }); + const md = renderMarkdownReport(summary); + + it('renders a top-line verdict table', () => { + expect(md).toContain('### Top-line verdict'); + // Dynamic column headers with MCP names + expect(md).toContain('clickhouse'); + expect(md).toContain('hyperdx'); + }); + + it('emits a section per scenario', () => { + expect(md).toContain('## error-root-cause'); + expect(md).toContain('## latency-spike'); + }); + + it('includes all summary metrics in each scenario table', () => { + expect(md).toContain('Combined score'); + expect(md).toContain('Programmatic score'); + expect(md).toContain('Judge mean (weighted)'); + expect(md).toContain('Tool calls (mean)'); + expect(md).toContain('Output tokens (mean)'); + expect(md).toContain('Wall clock (s, mean)'); + expect(md).toContain('Termination'); + }); + + it('formats deltas with explicit sign', () => { + // hyperdx and clickhouse have identical fixtures → delta is 0 + expect(md).toMatch(/\+0%|\+0\.0/); + }); + + it('emits judge per-criterion breakdown when judge data is present', () => { + expect(md).toContain('Judge per-criterion (mean 0–5)'); + expect(md).toContain('correctness'); + expect(md).toContain('completeness'); + }); + + it('emits programmatic per-check breakdown', () => { + expect(md).toContain('Programmatic per-check (pass rate)'); + expect(md).toContain('names_root'); + }); + + it('uses the batch basename in the title', () => { + expect(md.split('\n')[0]).toContain('2026-05-09T07-50-58-566Z'); + }); + + it('shows MCPs and baseline in the header', () => { + expect(md).toContain('MCPs: clickhouse, hyperdx'); + expect(md).toContain('Baseline: clickhouse'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/mcpConfig.test.ts b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts new file mode 100644 index 0000000000..5c5dc38144 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts @@ -0,0 +1,189 @@ +import { allowedToolsPattern, buildMcpConfig } from '../harness/mcpConfig'; +import { + buildSettings, + DENIED_BUILT_IN_TOOLS, + deniedToolsFor, +} from '../harness/settingsFile'; +import type { McpDefinition } from '../harness/types'; +import { + configMcpNames, + enabledMcpNames, + type EvalConfig, +} from '../hyperdx/config'; + +const hyperdxDef: McpDefinition = { + type: 'http', + url: 'http://localhost:30196/mcp', + headers: { Authorization: 'Bearer test-access-key' }, + toolPattern: 'mcp__hyperdx__*', + label: 'HyperDX', + brandTerms: ['HyperDX', 'hyperdx'], + deniedTools: [ + 'mcp__hyperdx__hyperdx_delete_dashboard', + 'mcp__hyperdx__hyperdx_get_dashboard', + 'mcp__hyperdx__hyperdx_save_dashboard', + 'mcp__hyperdx__hyperdx_query_tile', + 'mcp__hyperdx__hyperdx_get_saved_search', + 'mcp__hyperdx__hyperdx_save_saved_search', + 'mcp__hyperdx__hyperdx_get_alert', + 'mcp__hyperdx__hyperdx_get_webhook', + 'mcp__hyperdx__hyperdx_save_alert', + ], +}; + +const clickhouseDef: McpDefinition = { + type: 'stdio', + command: 'uv', + args: [ + 'run', + '--with', + 'mcp-clickhouse==0.3.0', + '--python', + '3.10', + 'mcp-clickhouse', + ], + env: { + CLICKHOUSE_HOST: 'localhost', + CLICKHOUSE_PORT: '30596', + CLICKHOUSE_USER: 'default', + CLICKHOUSE_PASSWORD: '', + CLICKHOUSE_SECURE: 'false', + CLICKHOUSE_VERIFY: 'false', + CLICKHOUSE_SEND_RECEIVE_TIMEOUT: '10', + }, + toolPattern: 'mcp__clickhouse__*', + label: 'ClickHouse MCP', + brandTerms: ['ClickHouse MCP', 'clickhouse'], +}; + +describe('buildMcpConfig', () => { + it('emits HyperDX HTTP transport with bearer token header', () => { + const json = buildMcpConfig(hyperdxDef, 'hyperdx'); + expect(json).toEqual({ + mcpServers: { + hyperdx: { + type: 'http', + url: 'http://localhost:30196/mcp', + headers: { Authorization: 'Bearer test-access-key' }, + }, + }, + }); + }); + + it('emits ClickHouse stdio transport via uv with pinned version + env', () => { + const json = buildMcpConfig(clickhouseDef, 'clickhouse') as { + mcpServers: { + clickhouse: { + command: string; + args: string[]; + env: Record; + }; + }; + }; + expect(json.mcpServers.clickhouse.command).toBe('uv'); + expect(json.mcpServers.clickhouse.args).toEqual([ + 'run', + '--with', + 'mcp-clickhouse==0.3.0', + '--python', + '3.10', + 'mcp-clickhouse', + ]); + expect(json.mcpServers.clickhouse.env.CLICKHOUSE_HOST).toBe('localhost'); + expect(json.mcpServers.clickhouse.env.CLICKHOUSE_PORT).toBe('30596'); + expect(json.mcpServers.clickhouse.env.CLICKHOUSE_SECURE).toBe('false'); + }); + + it('uses the MCP key name as the server key (supports arbitrary names)', () => { + const json = buildMcpConfig(hyperdxDef, 'my-custom-mcp'); + expect(json).toHaveProperty('mcpServers.my-custom-mcp'); + }); +}); + +describe('allowedToolsPattern', () => { + it('returns the toolPattern from the definition', () => { + expect(allowedToolsPattern(hyperdxDef)).toBe('mcp__hyperdx__*'); + expect(allowedToolsPattern(clickhouseDef)).toBe('mcp__clickhouse__*'); + }); +}); + +describe('buildSettings', () => { + it("allowlists only the run condition's MCP tools when no tempdir given", () => { + expect(buildSettings(hyperdxDef)).toEqual({ + permissions: { + allow: ['mcp__hyperdx__*'], + deny: [...deniedToolsFor('baseline', hyperdxDef)], + }, + }); + expect(buildSettings(clickhouseDef)).toEqual({ + permissions: { + allow: ['mcp__clickhouse__*'], + deny: [...DENIED_BUILT_IN_TOOLS], + }, + }); + }); + + it('adds a scoped Read allow rule when tempdir is provided', () => { + const settings = buildSettings( + hyperdxDef, + 'baseline', + '/tmp/hdx-eval-abc', + ) as { + permissions: { allow: string[] }; + }; + expect(settings.permissions.allow).toEqual([ + 'mcp__hyperdx__*', + 'Read(/tmp/hdx-eval-abc/*)', + ]); + }); + + it('denies the built-in filesystem/shell tools so the agent cannot access the source tree', () => { + const settings = buildSettings(hyperdxDef) as { + permissions: { deny: string[] }; + }; + expect(settings.permissions.deny).toEqual( + expect.arrayContaining(['Bash', 'Glob', 'Grep', 'Write', 'Edit']), + ); + // Read is NOT denied — it is allowed with a path scope via the allow list + expect(settings.permissions.deny).not.toContain('Read'); + }); + + it('includes per-MCP denied tools from the definition', () => { + const settings = buildSettings(hyperdxDef) as { + permissions: { deny: string[] }; + }; + expect(settings.permissions.deny).toContain( + 'mcp__hyperdx__hyperdx_delete_dashboard', + ); + }); + + it('does not include per-MCP denied tools for MCPs without deniedTools', () => { + const settings = buildSettings(clickhouseDef) as { + permissions: { deny: string[] }; + }; + // Should only have built-in tools denied, no MCP-specific ones + expect(settings.permissions.deny).toEqual([...DENIED_BUILT_IN_TOOLS]); + }); +}); + +describe('enabledMcpNames', () => { + const cfg: EvalConfig = { + mcps: { + alpha: { ...hyperdxDef, label: 'Alpha' }, + beta: { ...clickhouseDef, label: 'Beta', enabled: true }, + gamma: { ...hyperdxDef, label: 'Gamma', enabled: false }, + }, + }; + + it('returns all MCP names from configMcpNames regardless of enabled', () => { + expect(configMcpNames(cfg).sort()).toEqual(['alpha', 'beta', 'gamma']); + }); + + it('excludes MCPs where enabled is explicitly false', () => { + expect(enabledMcpNames(cfg).sort()).toEqual(['alpha', 'beta']); + }); + + it('treats missing enabled field as true', () => { + expect(enabledMcpNames(cfg)).toContain('alpha'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/noisy-signals.test.ts b/packages/hdx-eval/src/__tests__/noisy-signals.test.ts new file mode 100644 index 0000000000..8014e3ce9d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/noisy-signals.test.ts @@ -0,0 +1,273 @@ +import { mulberry32 } from '../rng/seeded'; +import { noisySignalsScenario } from '../scenarios/noisy-signals/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-08T12:00:00.000Z'); +// Tests run at 1% volume (15M → 150K) to keep memory reasonable. Pattern +// shares are preserved by the volumeFactor scaling. +const TEST_VOLUME_FACTOR = 0.01; +const TARGET_TOTAL = 16_000_000; +const SCALED_TOTAL = TARGET_TOTAL * TEST_VOLUME_FACTOR; // ~160K + +function run(seed: number) { + return collectScenario( + noisySignalsScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('noisy-signals scenario v6 (composite cells)', () => { + const result = run(42); + + it(`emits ~${SCALED_TOTAL.toLocaleString()} logs at volumeFactor ${TEST_VOLUME_FACTOR} (target ${TARGET_TOTAL.toLocaleString()} at full scale)`, () => { + // Allow 5% slack since each phase rounds independently. + expect(result.logs.length).toBeGreaterThan(SCALED_TOTAL * 0.95); + expect(result.logs.length).toBeLessThan(SCALED_TOTAL * 1.05); + }); + + it('is deterministic for a fixed seed and now', () => { + const b = run(42); + expect(b.logs.length).toBe(result.logs.length); + for (const i of [0, 1000, result.logs.length - 1]) { + expect(result.logs[i].body).toBe(b.logs[i].body); + expect(result.logs[i].timestampMs).toBe(b.logs[i].timestampMs); + } + }); + + // ─── Per-pattern shares ──────────────────────────────────────────────── + + function shareByEvent(eventName: string): number { + return ( + result.logs.filter(l => l.logAttributes['event.name'] === eventName) + .length / result.logs.length + ); + } + + it('cell 1: notification-service × DEBUG carries BOTH cache.hit (~6.7%) AND notification.delivery (~6.7%)', () => { + expect(shareByEvent('cache.hit')).toBeCloseTo(0.067, 1); + expect(shareByEvent('notification.delivery')).toBeCloseTo(0.067, 1); + }); + + it('cell 2: billing-service × INFO carries BOTH subscription metric-as-log (~6.7%) AND legit financial events (~6.7%)', () => { + expect(shareByEvent('subscription.metric.calculated')).toBeCloseTo( + 0.067, + 1, + ); + const billingEvents = [ + 'invoice.charged', + 'invoice.refunded', + 'subscription.activated', + 'subscription.cancelled', + 'payment.failed', + ]; + const share = + result.logs.filter(l => + billingEvents.includes(l.logAttributes['event.name'] ?? ''), + ).length / result.logs.length; + expect(share).toBeCloseTo(0.067, 1); + }); + + it('cell 3: worker × ERROR carries BOTH NonFatalRetryableError caught (~2.7%) AND job.failed.permanent (~2.7%)', () => { + expect(shareByEvent('job.caught_exception')).toBeCloseTo(0.027, 1); + expect(shareByEvent('job.failed.permanent')).toBeCloseTo(0.027, 1); + }); + + it('cell 4: 4-service × DEBUG carries BOTH health.check (~13.3%) AND legit per-service ops debug (~13.3%)', () => { + expect(shareByEvent('health.check')).toBeCloseTo(0.133, 1); + const opsEvents = [ + 'inventory.stock_lookup', + 'pricing.calculation', + 'shipping.rate.quote', + 'recommendation.score', + ]; + const share = + result.logs.filter(l => + opsEvents.includes(l.logAttributes['event.name'] ?? ''), + ).length / result.logs.length; + expect(share).toBeCloseTo(0.133, 1); + }); + + it('cell 5: frontend-proxy × INFO carries BOTH lb.health.probe (~6.7%) AND proxy.access (~6.7%)', () => { + expect(shareByEvent('lb.health.probe')).toBeCloseTo(0.067, 1); + expect(shareByEvent('proxy.access')).toBeCloseTo(0.067, 1); + }); + + it('cell 6: frontend × INFO carries BOTH console.log_dump (~5.3%) AND page.render (~5.3%)', () => { + expect(shareByEvent('frontend.console.log_dump')).toBeCloseTo(0.053, 1); + expect(shareByEvent('page.render')).toBeCloseTo(0.053, 1); + }); + + it('cell 7: search-service × INFO carries BOTH search.cache.miss (~4%) AND index.shard.flush (~4%)', () => { + expect(shareByEvent('search.cache.miss')).toBeCloseTo(0.04, 1); + expect(shareByEvent('index.shard.flush')).toBeCloseTo(0.04, 1); + }); + + // ─── The whole point: GROUP BY (Service, Severity) does NOT separate + // noise from load-bearing — the cells are composite. ─────────────── + + it('inside notification-service × DEBUG, cache.hit and notification.delivery are within 30% of each other in volume', () => { + const cell = result.logs.filter( + l => + l.serviceName === 'notification-service' && l.severityText === 'DEBUG', + ); + const cacheHits = cell.filter( + l => l.logAttributes['event.name'] === 'cache.hit', + ).length; + const deliveries = cell.filter( + l => l.logAttributes['event.name'] === 'notification.delivery', + ).length; + const ratio = + Math.min(cacheHits, deliveries) / Math.max(cacheHits, deliveries); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside billing-service × INFO, subscription metric-as-log and financial events are within 30% of each other', () => { + const cell = result.logs.filter( + l => l.serviceName === 'billing-service' && l.severityText === 'INFO', + ); + const subs = cell.filter( + l => l.logAttributes['event.name'] === 'subscription.metric.calculated', + ).length; + const billingEventsList = [ + 'invoice.charged', + 'invoice.refunded', + 'subscription.activated', + 'subscription.cancelled', + 'payment.failed', + ]; + const fin = cell.filter(l => + billingEventsList.includes(l.logAttributes['event.name'] ?? ''), + ).length; + const ratio = Math.min(subs, fin) / Math.max(subs, fin); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside worker × ERROR, caught-exception and permanent-failure are within 30% of each other', () => { + const cell = result.logs.filter( + l => l.serviceName === 'worker' && l.severityText === 'ERROR', + ); + const caught = cell.filter( + l => l.logAttributes['event.name'] === 'job.caught_exception', + ).length; + const perm = cell.filter( + l => l.logAttributes['event.name'] === 'job.failed.permanent', + ).length; + const ratio = Math.min(caught, perm) / Math.max(caught, perm); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside frontend-proxy × INFO, lb.health.probe and proxy.access are within 30% of each other', () => { + const cell = result.logs.filter( + l => l.serviceName === 'frontend-proxy' && l.severityText === 'INFO', + ); + const probes = cell.filter( + l => l.logAttributes['event.name'] === 'lb.health.probe', + ).length; + const access = cell.filter( + l => l.logAttributes['event.name'] === 'proxy.access', + ).length; + const ratio = Math.min(probes, access) / Math.max(probes, access); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside inventory-service × DEBUG, heartbeats and stock_lookups are within 30% of each other (sample cell)', () => { + const cell = result.logs.filter( + l => l.serviceName === 'inventory-service' && l.severityText === 'DEBUG', + ); + const heart = cell.filter( + l => l.logAttributes['event.name'] === 'health.check', + ).length; + const ops = cell.filter( + l => l.logAttributes['event.name'] === 'inventory.stock_lookup', + ).length; + const ratio = Math.min(heart, ops) / Math.max(heart, ops); + expect(ratio).toBeGreaterThan(0.7); + }); + + // ─── Body cardinality + attribute discoverability ────────────────────── + + it('heartbeat / cache.hit / notification.delivery — each has variable bodies (>50% unique in 500-row sample)', () => { + for (const ev of ['health.check', 'cache.hit', 'notification.delivery']) { + const sample = result.logs + .filter(l => l.logAttributes['event.name'] === ev) + .slice(0, 500); + const unique = new Set(sample.map(l => l.body)); + expect(unique.size / sample.length).toBeGreaterThan(0.5); + } + }); + + it('subscription metric-as-log keeps metric.kind=gauge attribute (the conversion signal)', () => { + const subs = result.logs.filter( + l => l.logAttributes['metric.name'] === 'subscription_mrr', + ); + expect(subs.length).toBeGreaterThan(0); + expect(subs[0].logAttributes['metric.kind']).toBe('gauge'); + }); + + it('NonFatalRetryableError caught-exception has stable error.kind attr; permanent has different error.kind', () => { + const caught = result.logs.filter( + l => + l.logAttributes['event.name'] === 'job.caught_exception' && + l.logAttributes['error.kind'] === 'NonFatalRetryableError', + ); + expect(caught.length).toBeGreaterThan(0); + const perm = result.logs.filter( + l => l.logAttributes['event.name'] === 'job.failed.permanent', + ); + expect(perm.length).toBeGreaterThan(0); + // The two patterns share serviceName + severityText but must be + // distinguishable by event.name (and have different error.kind). + const permKinds = new Set(perm.map(l => l.logAttributes['error.kind'])); + expect(permKinds.has('NonFatalRetryableError')).toBe(false); + }); + + it('lb.health.probe uses kube-probe user-agent (the signal that it is a probe)', () => { + const probes = result.logs.filter( + l => l.logAttributes['event.name'] === 'lb.health.probe', + ); + expect(probes.length).toBeGreaterThan(0); + expect(probes[0].logAttributes['user_agent.original']).toContain( + 'kube-probe', + ); + }); + + it('background variety (~9.3%) emits many distinct body templates across many services', () => { + const bg = result.logs.filter( + l => l.logAttributes['event.name'] === 'app.background', + ); + expect(bg.length / result.logs.length).toBeCloseTo(0.093, 1); + const services = new Set(bg.slice(0, 5000).map(l => l.serviceName)); + expect(services.size).toBeGreaterThan(8); + const bodies = new Set(bg.slice(0, 5000).map(l => l.body)); + expect(bodies.size).toBeGreaterThan(500); + }); + + it('severity is messy (lowercase + alias variants present in _severity_raw attr)', () => { + const rawSevs = new Set( + result.logs + .map(l => l.logAttributes._severity_raw) + .filter(Boolean) as string[], + ); + expect(rawSevs.has('info')).toBe(true); + expect(rawSevs.size).toBeGreaterThanOrEqual(3); + }); + + it('every log has rich resource attributes (k8s + service.* keys)', () => { + const sample = result.logs.slice(0, 500); + for (const l of sample) { + expect(l.resourceAttributes['service.name']).toBeDefined(); + expect(l.resourceAttributes['k8s.pod.name']).toBeDefined(); + expect(l.resourceAttributes['cloud.region']).toBeDefined(); + } + }); + + it('overall body cardinality is high (>50% unique in a 5K sample)', () => { + const sample = result.logs.slice(0, 5000); + const unique = new Set(sample.map(l => l.body)); + const ratio = unique.size / sample.length; + expect(ratio).toBeGreaterThan(0.5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/programmatic.test.ts b/packages/hdx-eval/src/__tests__/programmatic.test.ts new file mode 100644 index 0000000000..a4c1eecfdb --- /dev/null +++ b/packages/hdx-eval/src/__tests__/programmatic.test.ts @@ -0,0 +1,140 @@ +import { runProgrammaticChecks } from '../grading/programmatic'; +import { loadScenarioRubric } from '../grading/rubric'; + +describe('runProgrammaticChecks', () => { + it('hits all checks when answer mentions every required fact', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, + { id: 'b', weight: 2, pattern: 'bar' }, + ]; + const result = runProgrammaticChecks('foo and bar are present', checks); + expect(result.score).toBeCloseTo(1, 5); + expect(result.hits.every(h => h.matched)).toBe(true); + }); + + it('weights hits proportionally', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, // matches → +1 + { id: 'b', weight: 3, pattern: 'BAR', flags: '' }, // case-sensitive miss + ]; + const result = runProgrammaticChecks('only foo here', checks); + // 1 / (1+3) = 0.25 + expect(result.score).toBeCloseTo(0.25, 5); + }); + + it('defaults flags to case-insensitive', () => { + const checks = [{ id: 'a', weight: 1, pattern: 'PAYMENT-SERVICE' }]; + const result = runProgrammaticChecks( + 'mentions payment-service here', + checks, + ); + expect(result.hits[0].matched).toBe(true); + }); + + it('throws on invalid regex', () => { + expect(() => + runProgrammaticChecks('any', [{ id: 'a', weight: 1, pattern: '(' }]), + ).toThrow(); + }); + + it('returns 0 score for empty checks list', () => { + expect(runProgrammaticChecks('anything', []).score).toBe(0); + }); + + it('error-root-cause rubric scores 100% on a strong answer that hits every multi-criterion check', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const answer = + 'Root cause: payment-service ConnectionTimeoutError on db.payment.connect ' + + 'reaching db-payment.internal — DB connection timeout — cascading into ' + + 'checkout-api 5xx errors. Ruled out concurrent SMTP and CDN origin bursts ' + + '(separate trace trees, no checkout parent) and the historical TLS / ' + + 'rate-limit incidents.'; + const result = runProgrammaticChecks(answer, rubric.programmatic); + expect(result.score).toBeCloseTo(1, 5); + }); + + it('error-root-cause rubric drops below saturation when the answer is generic (no error.type / db host / distractor rule-out)', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const genericAnswer = + 'Root cause is payment-service connection timeout to db-payment, ' + + 'cascading into checkout-api 5xx errors.'; + const result = runProgrammaticChecks(genericAnswer, rubric.programmatic); + // Pre-tightening, this answer scored 1.0. Now it misses + // names_db_host_fqdn (full db-payment.internal), names_specific_db_span + // (db.payment.connect), and ruled_out_a_distractor — should land + // measurably below 1.0. + expect(result.score).toBeLessThan(0.9); + expect(result.score).toBeGreaterThan(0.5); + }); + + it('error-root-cause rubric scores noticeably worse on an irrelevant answer than a correct one', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const irrelevant = runProgrammaticChecks( + 'I think the database needs more memory.', + rubric.programmatic, + ); + const correct = runProgrammaticChecks( + 'Root cause is payment-service connection timeout to db-payment, ' + + 'cascading into checkout-api 5xx errors.', + rubric.programmatic, + ); + // Irrelevant answers vacuously satisfy the negative checks (they don't + // blame anything) but miss every positive — leaves a clear gap below the + // correct answer. + expect(correct.score - irrelevant.score).toBeGreaterThan(0.3); + expect(irrelevant.score).toBeLessThan(correct.score); + }); + + it('negative check is satisfied when the pattern does NOT match', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, // positive, miss + { id: 'b', weight: 1, pattern: 'baz', negative: true }, // negative, miss → satisfied + ]; + const result = runProgrammaticChecks('only bar here', checks); + expect(result.hits[0]).toMatchObject({ matched: false, satisfied: false }); + expect(result.hits[1]).toMatchObject({ + matched: false, + satisfied: true, + negative: true, + }); + // 1 / (1+1) = 0.5 + expect(result.score).toBeCloseTo(0.5, 5); + }); + + it('negative check is unsatisfied when the pattern DOES match', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, // positive, hit + { id: 'b', weight: 2, pattern: 'BAR', negative: true }, // negative, hit → unsatisfied + ]; + const result = runProgrammaticChecks('foo and bar', checks); + expect(result.hits[1]).toMatchObject({ + matched: true, + satisfied: false, + negative: true, + }); + // 1 / (1+2) = 0.333 + expect(result.score).toBeCloseTo(1 / 3, 5); + }); + + it('error-root-cause rubric penalizes blaming a distractor as root cause', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const goodAnswer = + 'Root cause: payment-service ConnectionTimeoutError on db.payment.connect ' + + 'reaching db-payment.internal — DB connection timeout — cascading into ' + + 'checkout-api 5xx. Ruled out concurrent SMTP and CDN bursts ' + + '(separate trace trees) and historical TLS / rate-limit incidents.'; + const goodResult = runProgrammaticChecks(goodAnswer, rubric.programmatic); + + const wrongBlameAnswer = + 'Root cause is the SMTP connection refused error on notification-service. ' + + 'payment-service db-payment connection timeout into checkout-api also seen.'; + const wrongResult = runProgrammaticChecks( + wrongBlameAnswer, + rubric.programmatic, + ); + + expect(goodResult.score).toBeGreaterThan(0.95); + // Wrong blame should drop score below 1 even though all positive checks still hit. + expect(wrongResult.score).toBeLessThan(goodResult.score); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/runRun.test.ts b/packages/hdx-eval/src/__tests__/runRun.test.ts new file mode 100644 index 0000000000..9d0d85621c --- /dev/null +++ b/packages/hdx-eval/src/__tests__/runRun.test.ts @@ -0,0 +1,138 @@ +import { assembleRecord } from '../harness/runRun'; +import type { ParsedEvent } from '../harness/streamParser'; + +function event(raw: object): ParsedEvent | null { + // Round-trip through the JSON parser so we exercise the same code path + // the runtime would. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { parseStreamLine } = require('../harness/streamParser'); + return parseStreamLine(JSON.stringify(raw)); +} + +function evs(...raws: object[]): ParsedEvent[] { + return raws.map(r => event(r)).filter((e): e is ParsedEvent => e !== null); +} + +const baseInput = { + scenario: 'error-root-cause', + agentPrompt: 'find root cause', + systemPromptAppend: 'sys', + mcp: 'hyperdx' as const, + model: 'claude-sonnet-4-6', + runIndex: 0, + seed: 42, + startedAtIso: '2026-05-09T10:00:00.000Z', + endedAtIso: '2026-05-09T10:01:30.000Z', + durationMs: 90_000, + stderr: '', +}; + +describe('assembleRecord', () => { + it('builds a successful run with one tool call and a final answer', () => { + const events = evs( + { + type: 'system', + subtype: 'init', + session_id: 's1', + tools: [{ name: 'mcp__hyperdx__hyperdx_query' }], + }, + { + type: 'assistant', + message: { + content: [ + { type: 'text', text: 'looking up' }, + { + type: 'tool_use', + id: 'tu_1', + name: 'mcp__hyperdx__hyperdx_query', + input: { sql: 'SELECT 1' }, + }, + ], + usage: { input_tokens: 100, output_tokens: 25 }, + }, + }, + { + type: 'user', + message: { + content: [ + { type: 'tool_result', tool_use_id: 'tu_1', content: 'rows...' }, + ], + }, + }, + { + type: 'assistant', + message: { + content: [{ type: 'text', text: 'Root cause is payment-service.' }], + }, + }, + { + type: 'result', + subtype: 'success', + is_error: false, + duration_ms: 90_000, + total_cost_usd: 0.05, + result: 'Root cause is payment-service.', + usage: { + input_tokens: 5000, + output_tokens: 200, + cache_read_input_tokens: 1500, + }, + }, + ); + + const record = assembleRecord({ + ...baseInput, + events, + exitCode: 0, + timedOut: false, + }); + expect(record.termination).toBe('final_answer'); + expect(record.toolCalls).toHaveLength(1); + expect(record.toolCalls[0].name).toBe('mcp__hyperdx__hyperdx_query'); + expect(record.toolCalls[0].output).toBe('rows...'); + expect(record.tokens.input).toBe(5000); + expect(record.tokens.cacheRead).toBe(1500); + expect(record.totalCostUsd).toBe(0.05); + expect(record.finalAnswer).toBe('Root cause is payment-service.'); + expect(record.tools.map(t => t.name)).toEqual([ + 'mcp__hyperdx__hyperdx_query', + ]); + }); + + it('marks termination as max_turns when result subtype indicates it', () => { + const events = evs({ + type: 'result', + subtype: 'error_max_turns', + is_error: true, + result: '', + usage: { input_tokens: 10, output_tokens: 0 }, + }); + const record = assembleRecord({ + ...baseInput, + events, + exitCode: 0, + timedOut: false, + }); + expect(record.termination).toBe('max_turns'); + }); + + it('marks termination as timeout when killed', () => { + const record = assembleRecord({ + ...baseInput, + events: [], + exitCode: null, + timedOut: true, + }); + expect(record.termination).toBe('timeout'); + }); + + it('marks termination as error on non-zero exit with no answer', () => { + const record = assembleRecord({ + ...baseInput, + events: [], + exitCode: 1, + timedOut: false, + }); + expect(record.termination).toBe('error'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/seeded.test.ts b/packages/hdx-eval/src/__tests__/seeded.test.ts new file mode 100644 index 0000000000..fcd191c4bb --- /dev/null +++ b/packages/hdx-eval/src/__tests__/seeded.test.ts @@ -0,0 +1,66 @@ +import { mulberry32 } from '../rng/seeded'; + +describe('mulberry32', () => { + it('produces an identical sequence for the same seed', () => { + const a = mulberry32(42); + const b = mulberry32(42); + const seqA = Array.from({ length: 10 }, () => a.next()); + const seqB = Array.from({ length: 10 }, () => b.next()); + expect(seqA).toEqual(seqB); + }); + + it('produces a different sequence for a different seed', () => { + const a = mulberry32(42); + const b = mulberry32(43); + const seqA = Array.from({ length: 10 }, () => a.next()); + const seqB = Array.from({ length: 10 }, () => b.next()); + expect(seqA).not.toEqual(seqB); + }); + + it('range() stays in [lo, hi)', () => { + const rng = mulberry32(1); + for (let i = 0; i < 1000; i++) { + const v = rng.range(5, 10); + expect(v).toBeGreaterThanOrEqual(5); + expect(v).toBeLessThan(10); + } + }); + + it('intRange() stays in [lo, hi) and returns integers', () => { + const rng = mulberry32(2); + for (let i = 0; i < 1000; i++) { + const v = rng.intRange(0, 5); + expect(Number.isInteger(v)).toBe(true); + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThan(5); + } + }); + + it('pick() returns a deterministic element for a fixed seed', () => { + const items = ['a', 'b', 'c', 'd'] as const; + const v1 = mulberry32(99).pick(items); + const v2 = mulberry32(99).pick(items); + expect(v1).toBe(v2); + }); + + it('weightedPick() approximates the weight distribution', () => { + const rng = mulberry32(7); + const counts = { a: 0, b: 0 }; + const N = 5000; + for (let i = 0; i < N; i++) { + const v = rng.weightedPick<'a' | 'b'>([ + { value: 'a', weight: 90 }, + { value: 'b', weight: 10 }, + ]); + counts[v]++; + } + const ratioA = counts.a / N; + expect(ratioA).toBeGreaterThan(0.85); + expect(ratioA).toBeLessThan(0.94); + }); + + it('hex() returns the requested byte length as hex', () => { + expect(mulberry32(1).hex(8)).toMatch(/^[0-9a-f]{16}$/); + expect(mulberry32(1).hex(16)).toMatch(/^[0-9a-f]{32}$/); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/segmented-regression.test.ts b/packages/hdx-eval/src/__tests__/segmented-regression.test.ts new file mode 100644 index 0000000000..ee9816a021 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/segmented-regression.test.ts @@ -0,0 +1,157 @@ +import { mulberry32 } from '../rng/seeded'; +import { segmentedRegressionScenario } from '../scenarios/segmented-regression/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-10T20:00:00.000Z'); +// 1% volume keeps the test cheap (~60K traces) while preserving the +// (tier, cache) cross-tab signal — distractors are still at fixed counts. +const TEST_VOLUME_FACTOR = 0.01; +const ANOMALY_WINDOW_MS = 10 * 60 * 1000; + +function run(seed: number) { + return collectScenario( + segmentedRegressionScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('segmented-regression scenario', () => { + const result = run(42); + + it('produces ~60K api-server traces at 1% volume', () => { + const apiTraces = result.traces.filter(t => t.serviceName === 'api-server'); + expect(apiTraces.length).toBeGreaterThan(50_000); + expect(apiTraces.length).toBeLessThan(70_000); + }); + + it('plants exactly 220 recommendation-service distractor errors', () => { + const rec = result.traces.filter( + t => t.serviceName === 'recommendation-service', + ); + expect(rec.length).toBe(220); + for (const r of rec) { + expect(r.statusCode).toBe('STATUS_CODE_ERROR'); + expect(r.statusMessage).toMatch(/upstream cf-feed unreachable/); + } + }); + + it('is deterministic for a fixed seed', () => { + const b = run(42); + expect(b.traces.length).toBe(result.traces.length); + expect(b.logs.length).toBe(result.logs.length); + expect(b.traces[0].spanId).toBe(result.traces[0].spanId); + expect(b.traces[5000].spanId).toBe(result.traces[5000].spanId); + }); + + describe('the planted intersection bug — enterprise × cache.hit=false', () => { + const anomalyStart = NOW_MS - ANOMALY_WINDOW_MS; + const apiAnomaly = result.traces.filter( + t => t.serviceName === 'api-server' && t.timestampMs >= anomalyStart, + ); + const bySegment = ( + tier: string, + hit: 'true' | 'false', + ): { total: number; errors: number } => { + const slice = apiAnomaly.filter( + t => + t.spanAttributes['tenant.tier'] === tier && + t.spanAttributes['cache.hit'] === hit, + ); + const errors = slice.filter( + t => t.statusCode === 'STATUS_CODE_ERROR', + ).length; + return { total: slice.length, errors }; + }; + + it('elevates error rate for enterprise × cache-miss to ~12% (16x baseline)', () => { + const seg = bySegment('enterprise', 'false'); + expect(seg.total).toBeGreaterThan(100); // enough sample at 1% volume + const rate = seg.errors / seg.total; + expect(rate).toBeGreaterThan(0.05); // well above baseline + expect(rate).toBeLessThan(0.2); // not so dominant it's obvious + }); + + it('leaves OTHER segments at the 0.5% baseline error rate', () => { + for (const [tier, hit] of [ + ['enterprise', 'true'], + ['pro', 'false'], + ['pro', 'true'], + ['free', 'false'], + ['free', 'true'], + ] as const) { + const seg = bySegment(tier, hit); + if (seg.total < 30) continue; // skip cells without enough sample + const rate = seg.errors / seg.total; + expect(rate).toBeLessThan(0.04); // close to baseline 0.5% + } + }); + + it('uses the distinctive FallbackHandlerError body for the planted errors', () => { + const fallbackErrors = apiAnomaly.filter( + t => + t.spanAttributes['error.type'] === 'FallbackHandlerError' && + t.statusCode === 'STATUS_CODE_ERROR', + ); + // Every fallback error must be enterprise × miss. + for (const f of fallbackErrors) { + expect(f.spanAttributes['tenant.tier']).toBe('enterprise'); + expect(f.spanAttributes['cache.hit']).toBe('false'); + } + // Body has the schema-mismatch signature. + expect(fallbackErrors[0].statusMessage).toMatch( + /schema mismatch in enterprise_v2_response/, + ); + }); + + it('emits correlated ERROR logs for each fallback error (same traceId)', () => { + const fallbackTraceIds = new Set( + apiAnomaly + .filter( + t => t.spanAttributes['error.type'] === 'FallbackHandlerError', + ) + .map(t => t.traceId), + ); + const fallbackLogs = result.logs.filter( + l => l.traceId && fallbackTraceIds.has(l.traceId), + ); + // Every planted error must have a log row attached. + expect(fallbackLogs.length).toBe(fallbackTraceIds.size); + expect(fallbackLogs[0].body).toMatch(/schema mismatch/); + }); + }); + + it('single-axis aggregates dilute the signal vs the cross-tab', () => { + const anomalyStart = NOW_MS - ANOMALY_WINDOW_MS; + const apiAnomaly = result.traces.filter( + t => t.serviceName === 'api-server' && t.timestampMs >= anomalyStart, + ); + const rateBy = ( + predicate: (t: (typeof apiAnomaly)[number]) => boolean, + ): number => { + const slice = apiAnomaly.filter(predicate); + if (slice.length === 0) return 0; + return ( + slice.filter(t => t.statusCode === 'STATUS_CODE_ERROR').length / + slice.length + ); + }; + const enterpriseRate = rateBy( + t => t.spanAttributes['tenant.tier'] === 'enterprise', + ); + const cacheMissRate = rateBy( + t => t.spanAttributes['cache.hit'] === 'false', + ); + const intersectionRate = rateBy( + t => + t.spanAttributes['tenant.tier'] === 'enterprise' && + t.spanAttributes['cache.hit'] === 'false', + ); + // Each single-axis rate is materially lower than the cross-tab — proves + // the agent has to combine them to find the actual signal. + expect(intersectionRate).toBeGreaterThan(enterpriseRate * 1.5); + expect(intersectionRate).toBeGreaterThan(cacheMissRate * 1.5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/service-health-check.test.ts b/packages/hdx-eval/src/__tests__/service-health-check.test.ts new file mode 100644 index 0000000000..cd34dd933d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/service-health-check.test.ts @@ -0,0 +1,208 @@ +import { mulberry32 } from '../rng/seeded'; +import { serviceHealthCheckScenario } from '../scenarios/service-health-check/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-10T20:00:00.000Z'); + +// 1% volume keeps the test cheap (~120K traces, ~240K logs) while preserving +// the planted-signal invariants. The blip count (~250 errors / minute) is +// derived from baseline traffic density × 1% volume, so it stays +// proportional to TEST_VOLUME_FACTOR. +const TEST_VOLUME_FACTOR = 0.01; + +const FOUR_HOURS_MS = 4 * 60 * 60 * 1000; +const ONE_HOUR_MS = 60 * 60 * 1000; +const REPORT_START_MS = NOW_MS - ONE_HOUR_MS; +const BASELINE_START_MS = NOW_MS - FOUR_HOURS_MS; +const BLIP_START_MS = REPORT_START_MS + 3 * 60 * 1000; +const BLIP_END_MS = BLIP_START_MS + 60 * 1000; +const NEW_TEMPLATE_START_MS = NOW_MS - 70 * 60 * 1000; + +function run(seed: number) { + return collectScenario( + serviceHealthCheckScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('service-health-check scenario', () => { + const result = run(42); + const apiTraces = result.traces; + const apiLogs = result.logs; + + it('produces traces and logs over the 4-hour window', () => { + // 12M × 0.01 = 120K traces target; 24M × 0.01 = 240K logs. + expect(apiTraces.length).toBeGreaterThan(100_000); + expect(apiTraces.length).toBeLessThan(140_000); + expect(apiLogs.length).toBeGreaterThan(220_000); + expect(apiLogs.length).toBeLessThan(260_000); + // All rows belong to api-server. + expect(apiTraces.every(t => t.serviceName === 'api-server')).toBe(true); + expect(apiLogs.every(l => l.serviceName === 'api-server')).toBe(true); + }); + + it('covers the full 4-hour window with the report window at the end', () => { + let minTs = Infinity; + let maxTs = -Infinity; + for (const t of apiTraces) { + if (t.timestampMs < minTs) minTs = t.timestampMs; + if (t.timestampMs > maxTs) maxTs = t.timestampMs; + } + // First sample within 1 min of baseline start; last within 1 min of now. + expect(minTs - BASELINE_START_MS).toBeLessThan(60_000); + expect(NOW_MS - maxTs).toBeLessThan(60_000); + }); + + it('is deterministic for a fixed seed', () => { + const b = run(42); + expect(b.traces.length).toBe(apiTraces.length); + expect(b.logs.length).toBe(apiLogs.length); + expect(b.traces[100].spanId).toBe(apiTraces[100].spanId); + expect(b.logs[100].body).toBe(apiLogs[100].body); + }); + + describe('baseline characteristics', () => { + it('error rate is ~0.5% outside the blip window', () => { + const nonBlip = apiTraces.filter( + t => !(t.timestampMs >= BLIP_START_MS && t.timestampMs < BLIP_END_MS), + ); + const errors = nonBlip.filter( + t => t.statusCode === 'STATUS_CODE_ERROR', + ).length; + const rate = errors / nonBlip.length; + expect(rate).toBeGreaterThan(0.002); + expect(rate).toBeLessThan(0.008); + }); + + it('log Body cardinality is high — naive GROUP BY Body is useless', () => { + // Sample 5K logs and check unique-body ratio. >= 90% unique means + // GROUP BY Body returns ~1 row per log row, so the agent must use + // hyperdx_log_patterns (Drain) to find templates. + const sample = apiLogs.slice(0, 5000); + const uniqueBodies = new Set(sample.map(l => l.body)); + expect(uniqueBodies.size / sample.length).toBeGreaterThan(0.9); + }); + }); + + describe('planted signal: new log template', () => { + it('feature-flag template appears ONLY in the planted window (~last 70 min)', () => { + const planted = apiLogs.filter(l => + /feature_flag\.new_recommendation_engine/.test(l.body), + ); + expect(planted.length).toBeGreaterThan(0); + // None in the baseline (the first 170 minutes of the dataset). + const baselinePlanted = planted.filter( + l => l.timestampMs < NEW_TEMPLATE_START_MS, + ); + expect(baselinePlanted.length).toBe(0); + // Approximately 0.1% of total log volume. At 1% volume that's ~240 + // logs; relax to a generous band to absorb sampling noise. + expect(planted.length).toBeGreaterThan(50); + expect(planted.length).toBeLessThan(600); + }); + }); + + describe('planted signal: v2 endpoint rollout', () => { + it('GET /api/v2/products appears ONLY in the report window and ramps up', () => { + const v2 = apiTraces.filter(t => t.spanName === 'GET /api/v2/products'); + expect(v2.length).toBeGreaterThan(0); + // None in the baseline. + const baselineV2 = v2.filter(t => t.timestampMs < REPORT_START_MS); + expect(baselineV2.length).toBe(0); + // Ramp check: more v2 in the second half of the report window than + // the first half. (The ramp is linear so 2nd half should be ~3× 1st.) + const halfMs = REPORT_START_MS + ONE_HOUR_MS / 2; + const firstHalf = v2.filter(t => t.timestampMs < halfMs).length; + const secondHalf = v2.filter(t => t.timestampMs >= halfMs).length; + expect(secondHalf).toBeGreaterThan(firstHalf); + }); + }); + + describe('planted signal: latency drift', () => { + it('p99 in the last 10 min of report > p99 in the first 10 min of report', () => { + const firstMinMs = REPORT_START_MS; + const firstMaxMs = REPORT_START_MS + 10 * 60 * 1000; + const lastMinMs = NOW_MS - 10 * 60 * 1000; + const firstSlice = apiTraces + .filter(t => t.timestampMs >= firstMinMs && t.timestampMs < firstMaxMs) + .map(t => t.durationNs / 1e6); + const lastSlice = apiTraces + .filter(t => t.timestampMs >= lastMinMs && t.timestampMs <= NOW_MS) + .map(t => t.durationNs / 1e6); + const p99 = (arr: number[]): number => { + const sorted = [...arr].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length * 0.99)] ?? 0; + }; + const p99First = p99(firstSlice); + const p99Last = p99(lastSlice); + // Drift should be visible: last p99 > first p99. At low volume the + // tail is noisy so we only require a positive direction with a + // moderate floor. + expect(p99Last).toBeGreaterThan(p99First); + expect(p99Last - p99First).toBeGreaterThan(10); + }); + }); + + describe('planted signal: brief error blip', () => { + it('blip minute has ~5% error rate, ~10× baseline', () => { + const blip = apiTraces.filter( + t => t.timestampMs >= BLIP_START_MS && t.timestampMs < BLIP_END_MS, + ); + expect(blip.length).toBeGreaterThan(100); + const blipErrors = blip.filter( + t => t.statusCode === 'STATUS_CODE_ERROR', + ).length; + const blipRate = blipErrors / blip.length; + expect(blipRate).toBeGreaterThan(0.025); + expect(blipRate).toBeLessThan(0.1); + }); + + it('all blip errors use the "upstream connect timeout" message', () => { + const blipErrors = apiTraces.filter( + t => + t.timestampMs >= BLIP_START_MS && + t.timestampMs < BLIP_END_MS && + t.statusCode === 'STATUS_CODE_ERROR', + ); + const timeoutErrors = blipErrors.filter( + t => t.statusMessage === 'upstream connect timeout', + ); + expect(timeoutErrors.length).toBe(blipErrors.length); + }); + }); + + describe('distractor: recurring batch-sync spikes', () => { + it('appears in BOTH the baseline window and the report window', () => { + const batch = apiTraces.filter( + t => t.spanName === 'POST /api/internal/batch-sync', + ); + expect(batch.length).toBeGreaterThan(0); + const inBaseline = batch.filter( + t => t.timestampMs < REPORT_START_MS, + ).length; + const inReport = batch.filter( + t => t.timestampMs >= REPORT_START_MS, + ).length; + // 3 hours of baseline vs 1 hour of report → roughly 3:1 ratio. + expect(inBaseline).toBeGreaterThan(inReport); // present in both + expect(inReport).toBeGreaterThan(0); + }); + + it('repeats on a ~15-minute cadence (16 spike-windows across 4 hours)', () => { + const batch = apiTraces.filter( + t => t.spanName === 'POST /api/internal/batch-sync', + ); + // Bucket batch-sync spans by 15-minute cycle and count occupied buckets. + // We expect 16 buckets to each have at least some spans (one per cycle). + const occupied = new Set(); + for (const t of batch) { + const offset = t.timestampMs - BASELINE_START_MS; + occupied.add(Math.floor(offset / (15 * 60 * 1000))); + } + expect(occupied.size).toBeGreaterThanOrEqual(15); + }); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/streamParser.test.ts b/packages/hdx-eval/src/__tests__/streamParser.test.ts new file mode 100644 index 0000000000..4d20b9587d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/streamParser.test.ts @@ -0,0 +1,127 @@ +import { chunkToLines, parseStreamLine } from '../harness/streamParser'; + +describe('parseStreamLine', () => { + it('returns null for blank/invalid input', () => { + expect(parseStreamLine('')).toBeNull(); + expect(parseStreamLine(' ')).toBeNull(); + expect(parseStreamLine('not json')).toBeNull(); + expect(parseStreamLine('null')).toBeNull(); + }); + + it('parses a system init event', () => { + const line = JSON.stringify({ + type: 'system', + subtype: 'init', + session_id: 'abc-123', + model: 'claude-sonnet-4-6', + tools: [ + { name: 'mcp__hyperdx__hyperdx_query', description: 'Query' }, + 'plain_string_tool', + ], + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('system_init'); + if (ev?.kind !== 'system_init') throw new Error('type narrow'); + expect(ev.sessionId).toBe('abc-123'); + expect(ev.model).toBe('claude-sonnet-4-6'); + expect(ev.tools).toHaveLength(2); + expect(ev.tools?.[0]?.name).toBe('mcp__hyperdx__hyperdx_query'); + expect(ev.tools?.[1]?.name).toBe('plain_string_tool'); + }); + + it('parses an assistant message with text + tool_use blocks', () => { + const line = JSON.stringify({ + type: 'assistant', + message: { + role: 'assistant', + content: [ + { type: 'text', text: 'Looking up sources' }, + { + type: 'tool_use', + id: 'tu_1', + name: 'mcp__hyperdx__hyperdx_list_sources', + input: {}, + }, + ], + usage: { input_tokens: 100, output_tokens: 50 }, + }, + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('assistant_message'); + if (ev?.kind !== 'assistant_message') throw new Error('type narrow'); + expect(ev.content).toHaveLength(2); + expect(ev.usage?.input_tokens).toBe(100); + }); + + it('parses a user (tool_result) event', () => { + const line = JSON.stringify({ + type: 'user', + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tu_1', + content: 'OK', + }, + ], + }, + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('user_message'); + if (ev?.kind !== 'user_message') throw new Error('type narrow'); + expect(ev.content).toHaveLength(1); + }); + + it('parses a final result event', () => { + const line = JSON.stringify({ + type: 'result', + subtype: 'success', + is_error: false, + duration_ms: 12345, + total_cost_usd: 0.0123, + result: 'Root cause is payment-service.', + usage: { + input_tokens: 5000, + output_tokens: 200, + cache_read_input_tokens: 1000, + }, + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('result'); + if (ev?.kind !== 'result') throw new Error('type narrow'); + expect(ev.subtype).toBe('success'); + expect(ev.isError).toBe(false); + expect(ev.totalCostUsd).toBe(0.0123); + expect(ev.usage?.cache_read_input_tokens).toBe(1000); + }); + + it('classifies an unknown event type without throwing', () => { + const line = JSON.stringify({ type: 'futuristic_event', payload: 42 }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('unknown'); + }); +}); + +describe('chunkToLines', () => { + it('splits a complete buffer into lines with empty remainder', () => { + const { events, remainder } = chunkToLines('', 'a\nb\nc\n'); + expect(events).toEqual(['a', 'b', 'c']); + expect(remainder).toBe(''); + }); + + it('preserves the trailing partial line as remainder', () => { + const { events, remainder } = chunkToLines('', 'a\nb\npart'); + expect(events).toEqual(['a', 'b']); + expect(remainder).toBe('part'); + }); + + it('combines a leftover buffer with the new chunk', () => { + const r1 = chunkToLines('', '{"a":'); + expect(r1.events).toHaveLength(0); + expect(r1.remainder).toBe('{"a":'); + const r2 = chunkToLines(r1.remainder, '1}\n{"b":2}\n'); + expect(r2.events).toEqual(['{"a":1}', '{"b":2}']); + expect(r2.remainder).toBe(''); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts new file mode 100644 index 0000000000..948f4ca5be --- /dev/null +++ b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts @@ -0,0 +1,40 @@ +import { buildSystemPrompt } from '../harness/systemPrompt'; + +describe('buildSystemPrompt', () => { + it('includes scenario-specific table names', () => { + const p = buildSystemPrompt('error-root-cause'); + expect(p).toContain('default.eval_error_root_cause_otel_traces'); + expect(p).toContain('default.eval_error_root_cause_otel_logs'); + }); + + it('includes generic tool-environment guidance', () => { + const p1 = buildSystemPrompt('latency-spike'); + const p2 = buildSystemPrompt('noisy-signals'); + // Both prompts should explain tool environment and Read availability + for (const p of [p1, p2]) { + expect(p).toContain('TOOL ENVIRONMENT'); + expect(p).toContain('Read tool'); + expect(p).toContain('oversized tool responses'); + } + }); + + it('asks for service, operation, root cause, and ruled-out section', () => { + const p = buildSystemPrompt('error-root-cause'); + expect(p.toLowerCase()).toContain('service'); + expect(p.toLowerCase()).toContain('operation'); + expect(p.toLowerCase()).toContain('root cause'); + expect(p.toLowerCase()).toContain("what's not the cause"); + }); + + it('includes anchor time when provided', () => { + const p = buildSystemPrompt('error-root-cause', '2026-01-15T10:00:00.000Z'); + expect(p).toContain('FIXED CURRENT TIME'); + expect(p).toContain('2026-01-15T10:00:00.000Z'); + }); + + it('includes hypothesis playbook when variant is hypothesis', () => { + const p = buildSystemPrompt('error-root-cause', undefined, 'hypothesis'); + expect(p).toContain('INVESTIGATION GUIDANCE'); + expect(p).toContain('hypotheses'); + }); +}); diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts new file mode 100644 index 0000000000..a256fbdcbf --- /dev/null +++ b/packages/hdx-eval/src/cli.ts @@ -0,0 +1,894 @@ +#!/usr/bin/env node +import { Command } from 'commander'; +import fs from 'fs'; +import { resolve } from 'path'; + +import { createEvalClient, defaultClickHouseUrl } from './clickhouse/client'; +import { + dropScenarioTables, + scenarioIsSeeded, + scenarioTables, +} from './clickhouse/schema'; +import { buildBlindingEntries } from './grading/blind'; +import { + gradeBatch, + type GradeBatchOptions, + resolveBatchDir, +} from './grading/grade'; +import { runCell } from './harness/runRun'; +import type { McpKind, PromptVariant } from './harness/types'; +import { + configExists, + configMcpNames, + configPath, + enabledMcpNames, + ensureAnchorTime, + getMcpDefinition, + readConfig, +} from './hyperdx/config'; +import { runCheck, runSetup } from './hyperdx/setup'; +import { writeBatchSummary } from './reports/store'; +import { instrumentBatch, summarizeTimingRecord } from './runs/instrument'; +import { batchDirName } from './runs/path'; +import { writeRun } from './runs/store'; +import { listBatches, listRunsInBatch, readRun } from './runs/store'; +import { getScenario, SCENARIO_NAMES, SCENARIOS } from './scenarios'; +import { type SeedProgress, seedScenario } from './scenarios/seedScenario'; + +if (!process.env.ANTHROPIC_API_KEY && process.env.AI_API_KEY) { + process.env.ANTHROPIC_API_KEY = process.env.AI_API_KEY; +} + +function formatCount(n: number): string { + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`; + return String(n); +} + +function logSeedProgress(prefix: string, startMs: number) { + return (p: SeedProgress) => { + const elapsed = ((Date.now() - startMs) / 1000).toFixed(0); + const total = p.tracesInserted + p.logsInserted; + process.stdout.write( + `\r${prefix}${formatCount(total)} rows (${formatCount(p.tracesInserted)} traces, ${formatCount(p.logsInserted)} logs) · ${elapsed}s`, + ); + }; +} + +type GlobalOpts = { + chUrl?: string; + chUser?: string; + chPassword?: string; +}; + +function buildClient(opts: GlobalOpts) { + return createEvalClient({ + url: opts.chUrl ?? defaultClickHouseUrl(), + username: opts.chUser, + password: opts.chPassword, + }); +} + +function buildClientFromConfig( + cfg: import('./hyperdx/config').EvalConfig, + globals: GlobalOpts, +) { + return createEvalClient({ + url: + globals.chUrl ?? + (cfg.clickhouse + ? `http://${cfg.clickhouse.host}:${cfg.clickhouse.port}` + : defaultClickHouseUrl()), + username: globals.chUser ?? cfg.clickhouse?.user, + password: globals.chPassword ?? cfg.clickhouse?.password, + }); +} + +function defaultApiUrl(): string { + if (process.env.HDX_EVAL_API_URL) return process.env.HDX_EVAL_API_URL; + if (process.env.HYPERDX_API_PORT) { + return `http://localhost:${process.env.HYPERDX_API_PORT}`; + } + if (process.env.HDX_DEV_API_PORT) { + return `http://localhost:${process.env.HDX_DEV_API_PORT}`; + } + return 'http://localhost:8000'; +} + +const program = new Command(); + +program + .name('hdx-eval') + .description( + 'HyperDX eval framework — synthetic telemetry generators + agent harness', + ) + .option('--ch-url ', 'ClickHouse HTTP URL') + .option('--ch-user ', 'ClickHouse username', 'default') + .option('--ch-password ', 'ClickHouse password', ''); + +program + .command('list') + .description('List available scenarios') + .action(() => { + for (const name of SCENARIO_NAMES) { + const s = SCENARIOS[name]; + const tables = scenarioTables(name); + console.log(`\n${name}`); + console.log(` prompt: ${s.agentPrompt}`); + console.log(` description: ${s.description}`); + console.log(` tables: default.${tables.traces}, default.${tables.logs}`); + } + console.log(); + }); + +program + .command('seed ') + .description('Seed ClickHouse with synthetic data for the given scenario') + .option('--seed ', 'PRNG seed', '42') + .option('--now ', 'Anchor "now" to a fixed ISO timestamp') + .option( + '--volume-factor ', + 'Scale background row counts (e.g. 0.1 for 10% of full volume)', + ) + .action( + async ( + scenarioName: string, + cmdOpts: { seed: string; now?: string; volumeFactor?: string }, + ) => { + const opts = program.opts(); + const seedNum = Number(cmdOpts.seed); + if (!Number.isFinite(seedNum)) { + throw new Error(`--seed must be a number, got: ${cmdOpts.seed}`); + } + const nowMs = cmdOpts.now ? Date.parse(cmdOpts.now) : Date.now(); + if (cmdOpts.now && !Number.isFinite(nowMs)) { + throw new Error( + `--now must be a valid ISO timestamp, got: ${cmdOpts.now}`, + ); + } + let volumeFactor: number | undefined; + if (cmdOpts.volumeFactor !== undefined) { + volumeFactor = Number(cmdOpts.volumeFactor); + if (!Number.isFinite(volumeFactor) || volumeFactor <= 0) { + throw new Error( + `--volume-factor must be a positive number, got: ${cmdOpts.volumeFactor}`, + ); + } + } + + const scenario = getScenario(scenarioName); + console.log(`Seeding scenario "${scenario.name}"`); + console.log(` seed: ${seedNum}`); + console.log(` now: ${new Date(nowMs).toISOString()}`); + if (volumeFactor !== undefined) { + console.log(` volumeFactor: ${volumeFactor}`); + } + + const client = buildClient(opts); + try { + const seedStart = Date.now(); + const result = await seedScenario({ + client, + scenarioName: scenario.name, + seed: seedNum, + nowMs, + volumeFactor, + onProgress: logSeedProgress(' seeding: ', seedStart), + }); + const seedSecs = ((Date.now() - seedStart) / 1000).toFixed(1); + process.stdout.write('\n'); + console.log( + `Inserted ${result.tracesInserted} trace rows → default.${result.tables.traces}`, + ); + console.log( + `Inserted ${result.logsInserted} log rows → default.${result.tables.logs}`, + ); + console.log(`Done in ${seedSecs}s`); + } finally { + await client.close(); + } + }, + ); + +program + .command('drop ') + .description('Drop the eval tables for the given scenario') + .action(async (scenarioName: string) => { + const opts = program.opts(); + getScenario(scenarioName); + const client = buildClient(opts); + try { + await dropScenarioTables(client, scenarioName); + const tables = scenarioTables(scenarioName); + console.log( + `Dropped default.${tables.traces} and default.${tables.logs}`, + ); + } finally { + await client.close(); + } + }); + +program + .command('setup-hyperdx') + .description( + 'One-time setup: register/login + create Connection + per-scenario Sources', + ) + .option('--api-url ', 'HyperDX API URL', defaultApiUrl()) + .option('--email ', 'Account email', 'eval@local.test') + .option('--password ', 'Account password', 'EvalPass123!#') + .option( + '--creds-file ', + 'JSON file with {"email":..., "password":...} (overrides --email/--password)', + ) + .option('--check', 'Validate existing config instead of creating') + .option( + '--reset-sources', + 'Delete and recreate all eval-* Sources (e.g. to apply new querySettings)', + ) + .action( + async (cmdOpts: { + apiUrl: string; + email: string; + password: string; + credsFile?: string; + check?: boolean; + resetSources?: boolean; + }) => { + const opts = program.opts(); + let email = cmdOpts.email; + let password = cmdOpts.password; + if (cmdOpts.credsFile) { + const expanded = cmdOpts.credsFile.replace( + /^~(?=$|\/|\\)/, + process.env.HOME ?? '~', + ); + const raw = JSON.parse(fs.readFileSync(expanded, 'utf8')); + if (typeof raw.email !== 'string' || typeof raw.password !== 'string') { + throw new Error( + `--creds-file must be a JSON object with string fields "email" and "password"`, + ); + } + email = raw.email; + password = raw.password; + } + if (cmdOpts.check) { + const result = await runCheck(); + console.log( + 'Config: ', + result.configOk ? 'OK' : 'MISSING/INVALID', + ); + console.log( + 'HyperDX MCP: ', + result.mcpReachable ? 'OK' : 'UNREACHABLE', + ); + console.log( + 'ClickHouse: ', + result.clickhouseReachable ? 'OK' : 'UNREACHABLE', + ); + console.log( + 'uv installed: ', + result.uvAvailable ? 'OK' : 'MISSING', + ); + if (result.errors.length > 0) { + console.log('\nErrors:'); + for (const e of result.errors) console.log(' -', e); + process.exit(1); + } + return; + } + + const chUrl = opts.chUrl ?? defaultClickHouseUrl(); + const url = new URL(chUrl); + console.log(`Setting up HyperDX eval account at ${cmdOpts.apiUrl}`); + const result = await runSetup({ + apiUrl: cmdOpts.apiUrl, + email, + password, + clickhouse: { + host: url.hostname, + port: url.port || '8123', + user: opts.chUser ?? 'default', + password: opts.chPassword ?? '', + }, + resetSources: cmdOpts.resetSources ?? false, + }); + console.log(`Wrote ${result.configPath}`); + console.log( + ` connection: ${result.created.connection ? 'created' : 'already existed'}`, + ); + if (result.created.sources.length === 0) { + console.log(' sources: all already existed'); + } else { + console.log(' sources: created', result.created.sources.join(', ')); + } + }, + ); + +program + .command('run ') + .description( + 'Run Claude Code as the agent against one or more MCPs and capture trajectories', + ) + .option( + '--mcp ', + 'Comma-separated MCP names from config, or "all" (default: all)', + 'all', + ) + .option( + '--baseline ', + 'MCP to use as baseline in reports (default: first MCP in list)', + ) + .option('--runs ', 'Number of runs per (scenario,MCP) cell', '3') + .option('--model ', 'Model ID to pass to Claude Code', 'claude-opus-4-6') + // Lower than the previous 25 — tightens the budget so sloppy agents that + // make 20+ exploratory calls can't paper over correctness with volume. + // Override with --max-turns if a specific scenario needs more. + .option('--max-turns ', 'Max tool-use turns', '15') + .option('--seed ', 'PRNG seed for re-seeding', '42') + .option( + '--timeout ', + 'Per-run wall-clock timeout in milliseconds', + '300000', + ) + .option( + '--reseed', + 'Re-seed ClickHouse data before running (default: skip reseed and reuse ' + + 'existing data). Use after changing seed parameters or when data is stale.', + ) + .option( + '--anchor-time ', + 'Override the saved "now" anchor with a specific ISO timestamp. ' + + 'Saved to eval.config.json for future runs.', + ) + .option( + '--live', + 'Ignore the saved anchor time and use wall-clock "now". ' + + 'The agent will NOT receive a FIXED CURRENT TIME system prompt block. ' + + 'Implies --reseed since data must be seeded to the current time.', + ) + .option( + '--concurrency ', + 'Run up to N cells concurrently within this scenario. Default 1 ' + + '(sequential). Each cell is one (mcp, runIndex) pair.', + '1', + ) + .option( + '--prompt-variant ', + 'Prompt variant: "baseline" (default) or "hypothesis" (forces the ' + + 'agent to enumerate 2-4 hypotheses up front and spawn one Task ' + + 'subagent per hypothesis to investigate in parallel). The hypothesis ' + + 'variant also allows the Task built-in tool; the baseline variant ' + + 'denies it.', + 'baseline', + ) + .option('--no-grade', 'Skip automatic grading after runs complete') + .option('--no-report', 'Skip automatic report generation after grading') + .option( + '--judge-model ', + 'Judge model ID (used when auto-grading)', + 'claude-opus-4-7', + ) + .option( + '--no-judge', + 'Run programmatic checks only during auto-grading (skip LLM judge)', + ) + .action( + async ( + scenarioName: string, + cmdOpts: { + mcp: string; + baseline?: string; + runs: string; + model: string; + maxTurns: string; + seed: string; + timeout: string; + reseed?: true; + anchorTime?: string; + live?: true; + concurrency: string; + promptVariant: string; + grade: boolean; + report: boolean; + judgeModel: string; + judge: boolean; + }, + ) => { + const opts = program.opts(); + const scenario = getScenario(scenarioName); + const apiKeyEnv = process.env.ANTHROPIC_API_KEY; + if (!apiKeyEnv) { + throw new Error( + 'ANTHROPIC_API_KEY is not set. Export it before running `hdx-eval run`.', + ); + } + const apiKey: string = apiKeyEnv; + if (!configExists()) { + throw new Error( + `Eval config missing. Run \`hdx-eval setup-hyperdx\` first. Expected: ${configPath()}`, + ); + } + const config = readConfig(); + + const mcpKinds: McpKind[] = parseMcpFlag(cmdOpts.mcp, config); + // Validate that all requested MCPs exist in config. + for (const mcp of mcpKinds) { + getMcpDefinition(config, mcp); + } + const baseline = cmdOpts.baseline ?? mcpKinds[0]; + if (cmdOpts.baseline && !mcpKinds.includes(cmdOpts.baseline)) { + throw new Error( + `--baseline "${cmdOpts.baseline}" is not in the MCP list: ${mcpKinds.join(', ')}`, + ); + } + const promptVariant: PromptVariant = parsePromptVariant( + cmdOpts.promptVariant, + ); + const runs = Number(cmdOpts.runs); + const maxTurns = Number(cmdOpts.maxTurns); + const timeoutMs = Number(cmdOpts.timeout); + const seedNum = Number(cmdOpts.seed); + const concurrency = Number(cmdOpts.concurrency); + for (const [k, v] of Object.entries({ + runs, + maxTurns, + timeoutMs, + seedNum, + concurrency, + })) { + if (!Number.isFinite(v) || v <= 0) { + throw new Error( + `--${k.replace(/Num$/, '')} must be a positive number`, + ); + } + } + + // ── Anchor-time resolution ─────────────────────────────────── + // Default: read from config (persisted). First run auto-generates. + // --anchor-time : override + save to config. + // --live: ignore saved anchor, use wall-clock now (no FIXED CURRENT + // TIME in system prompt), and force reseed. + let anchorTimeIso: string | undefined; + let anchorMs: number; + if (cmdOpts.live) { + if (cmdOpts.anchorTime) { + throw new Error('--live and --anchor-time are mutually exclusive.'); + } + anchorMs = Date.now(); + anchorTimeIso = undefined; // no anchor injection when running live + } else { + if (cmdOpts.anchorTime) { + const parsed = Date.parse(cmdOpts.anchorTime); + if (!Number.isFinite(parsed)) { + throw new Error( + `--anchor-time must be a valid ISO timestamp, got: ${cmdOpts.anchorTime}`, + ); + } + } + const anchor = ensureAnchorTime(config, cmdOpts.anchorTime); + anchorTimeIso = anchor.anchorTimeIso; + anchorMs = anchor.anchorMs; + } + + // ── Re-seed ─────────────────────────────────────────────────── + // Default: skip reseed IF data already exists. Auto-seeds on first + // run when scenario tables are empty or missing. + // --reseed: force re-seed even if data exists. + // --live: always reseed (data must match wall-clock now). + const forceReseed = cmdOpts.reseed === true || cmdOpts.live === true; + let shouldReseed = forceReseed; + if (!forceReseed) { + const checkClient = buildClientFromConfig(config, opts); + try { + const seeded = await scenarioIsSeeded(checkClient, scenario.name); + if (!seeded) { + console.log(`No data found for ${scenario.name} — auto-seeding...`); + shouldReseed = true; + } + } finally { + await checkClient.close(); + } + } + if (shouldReseed) { + console.log( + `Seeding ${scenario.name} (seed=${seedNum}, now=${new Date(anchorMs).toISOString()})...`, + ); + const seedStart = Date.now(); + const client = buildClientFromConfig(config, opts); + try { + const r = await seedScenario({ + client, + scenarioName: scenario.name, + seed: seedNum, + nowMs: anchorMs, + onProgress: logSeedProgress(' ', seedStart), + }); + const seedSecs = ((Date.now() - seedStart) / 1000).toFixed(1); + process.stdout.write('\n'); + console.log( + `Seeded ${scenario.name}: ${formatCount(r.tracesInserted)} traces, ${formatCount(r.logsInserted)} logs in ${seedSecs}s`, + ); + } finally { + await client.close(); + } + } + + const batchDir = batchDirName(); + console.log(`\nBatch: runs/${batchDir}`); + console.log(`Scenario: ${scenario.name}`); + console.log(`MCPs: ${mcpKinds.join(', ')}`); + console.log( + `Runs/cell: ${runs}, model: ${cmdOpts.model}, max-turns: ${maxTurns}, concurrency: ${concurrency}, prompt-variant: ${promptVariant}\n`, + ); + + type SummaryRow = { + mcp: McpKind; + i: number; + toolCalls: number; + inputTokens: number; + outputTokens: number; + durationS: number; + termination: string; + path: string; + }; + + // Flatten the (mcp, runIndex) matrix into a single queue and pull from it + // with a worker pool of size `concurrency`. + const cells: Array<{ mcp: McpKind; i: number }> = []; + for (const mcp of mcpKinds) { + for (let i = 0; i < runs; i++) cells.push({ mcp, i }); + } + + const summary: SummaryRow[] = []; + let cursor = 0; + const errors: Array<{ mcp: string; i: number; error: string }> = []; + async function worker(workerId: number): Promise { + while (true) { + const idx = cursor++; + if (idx >= cells.length) return; + const { mcp, i } = cells[idx]; + const label = concurrency > 1 ? `[w${workerId}] ` : ' '; + try { + process.stdout.write( + `${label}${mcp} run ${i + 1}/${runs}... starting\n`, + ); + const startedMs = Date.now(); + const record = await runCell({ + config, + scenario: scenario.name, + agentPrompt: scenario.agentPrompt, + mcp, + model: cmdOpts.model, + maxTurns, + timeoutMs, + runIndex: i, + seed: seedNum, + apiKey, + anchorTimeIso, + promptVariant, + }); + const path = writeRun({ record, batchDir }); + const seconds = ((Date.now() - startedMs) / 1000).toFixed(1); + console.log( + `${label}${mcp} run ${i + 1}/${runs}: ${record.termination} · ${record.toolCalls.length} tool calls · ${record.tokens.input}+${record.tokens.output} tok · ${seconds}s`, + ); + summary.push({ + mcp, + i, + toolCalls: record.toolCalls.length, + inputTokens: record.tokens.input, + outputTokens: record.tokens.output, + durationS: Math.round((Date.now() - startedMs) / 100) / 10, + termination: record.termination, + path, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error( + `${label}${mcp} run ${i + 1}/${runs}: FAILED — ${msg}`, + ); + errors.push({ mcp, i, error: msg }); + } + } + } + const workerCount = Math.min(concurrency, cells.length); + await Promise.all( + Array.from({ length: workerCount }, (_, w) => worker(w + 1)), + ); + + if (errors.length > 0) { + console.error( + `\n${errors.length} run(s) failed:\n` + + errors.map(e => ` ${e.mcp} #${e.i}: ${e.error}`).join('\n'), + ); + } + console.log('\nResults written under runs/' + batchDir); + // Sort by (mcp, i) for stable summary output even with parallel workers. + summary.sort((a, b) => a.mcp.localeCompare(b.mcp) || a.i - b.i); + for (const row of summary) { + console.log( + ` ${row.mcp.padEnd(10)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, + ); + } + + // ─── Auto-grade ────────────────────────────────────────────── + if (cmdOpts.grade !== false) { + console.log('\n--- Auto-grading ---'); + // Build blinding entries from the MCPs we ran against. + const blindingEntries = buildBlindingEntries( + mcpKinds.map(k => ({ + kind: k, + def: getMcpDefinition(config, k), + })), + ); + const gradeOpts: GradeBatchOptions = { + judgeModel: cmdOpts.judgeModel, + skipJudge: cmdOpts.judge === false, + blindingEntries, + }; + const gradeResult = await gradeBatch(batchDir, gradeOpts); + console.log( + `\nGraded ${gradeResult.graded.length} run${gradeResult.graded.length === 1 ? '' : 's'}; ${gradeResult.errors.length} error${gradeResult.errors.length === 1 ? '' : 's'}.`, + ); + if (gradeResult.errors.length > 0) { + for (const e of gradeResult.errors) + console.log(' -', e.runPath, e.error); + } + + // ─── Auto-report ──────────────────────────────────────────── + if (cmdOpts.report !== false && gradeResult.graded.length > 0) { + console.log('\n--- Auto-report ---'); + const resolvedDir = resolveBatchDir(batchDir); + const outPath = `${resolvedDir}/_summary.md`; + const reportResult = writeBatchSummary( + resolvedDir, + outPath, + baseline, + ); + console.log(`Wrote ${reportResult.mdPath}`); + console.log(`Wrote ${reportResult.jsonPath}`); + console.log( + `Aggregated ${reportResult.pairsCount} run${reportResult.pairsCount === 1 ? '' : 's'}.`, + ); + } + } + }, + ); + +program + .command('runs-list') + .description('List recorded run batches') + .action(() => { + const batches = listBatches(); + if (batches.length === 0) { + console.log('No runs recorded yet.'); + return; + } + for (const b of batches) { + const files = listRunsInBatch(b); + console.log( + `${b} (${files.length} run${files.length === 1 ? '' : 's'})`, + ); + } + }); + +program + .command('runs-show ') + .description('Pretty-print a run record file') + .option( + '--queries', + 'Print every tool call with its full input (SQL / builder spec) and any matching query_log rows from the timing sidecar', + ) + .option( + '--final-answer', + 'Print the final answer (omitted by default to keep --queries output focused)', + ) + .action( + (path: string, cmdOpts: { queries?: boolean; finalAnswer?: boolean }) => { + const r = readRun(path); + console.log(`Run ${r.runId}`); + console.log(` scenario: ${r.scenario}`); + console.log(` mcp: ${r.mcp}`); + console.log(` model: ${r.model}`); + console.log( + ` termination: ${r.termination} (exit ${r.exitCode ?? '?'})`, + ); + console.log(` duration: ${(r.durationMs / 1000).toFixed(1)}s`); + console.log( + ` tokens: in=${r.tokens.input} out=${r.tokens.output} cacheRead=${r.tokens.cacheRead}`, + ); + console.log(` tool calls: ${r.toolCalls.length}`); + + if (cmdOpts.queries) { + const timingPath = path.replace(/\.json$/, '.timing.json'); + const timing = fs.existsSync(timingPath) + ? (JSON.parse(fs.readFileSync(timingPath, 'utf8')) as { + toolCalls: Array<{ + index: number; + serverQueries: Array<{ + queryDurationMs: number; + readRows: number; + queryPreview: string; + }>; + }>; + } | null) + : null; + console.log('\n--- Tool calls ---'); + r.toolCalls.forEach((c, i) => { + const inp = c.input as Record; + let summary: string; + // Generic tool display — detect query-like tools by input shape. + if (typeof inp?.query === 'string') { + summary = `query=${String(inp.query).replace(/\s+/g, ' ').trim()}`; + } else if (typeof inp?.sql === 'string') { + summary = `[sql] ${String(inp.sql).replace(/\s+/g, ' ').trim()}`; + } else if (inp?.displayType) { + summary = `[${inp.displayType ?? '?'}] where=${inp?.where ?? '—'} groupBy=${inp?.groupBy ?? '—'} select=${JSON.stringify(inp?.select ?? null)}`; + } else { + summary = JSON.stringify(inp).slice(0, 200); + } + console.log(`\n#${i + 1} ${c.name}`); + console.log(` ${summary}`); + const t = timing?.toolCalls?.[i]; + if (t && t.serverQueries.length > 0) { + for (const q of t.serverQueries) { + console.log( + ` └─ server: ${q.queryDurationMs}ms, read=${q.readRows} ${q.queryPreview.replace(/\s+/g, ' ').slice(0, 140)}`, + ); + } + } + }); + } + + if (cmdOpts.finalAnswer) { + console.log(`\n--- Final answer ---\n${r.finalAnswer}\n`); + } + }, + ); + +program + .command('grade ') + .description( + 'Grade trajectories: programmatic checks + LLM-as-judge (Opus 4.7 by default)', + ) + .option('--judge-model ', 'Judge model ID', 'claude-opus-4-7') + .option('--rerun-judge', 'Re-call judge even if a grade JSON already exists') + .option('--no-judge', 'Run programmatic checks only (cheap regrade)') + .action( + async ( + batch: string, + cmdOpts: { judgeModel: string; rerunJudge?: boolean; judge: boolean }, + ) => { + const dir = resolveBatchDir(batch); + console.log(`Grading batch at ${dir}`); + // Build blinding entries from config if available. + let blindingEntries; + if (configExists()) { + try { + const cfg = readConfig(); + blindingEntries = buildBlindingEntries( + configMcpNames(cfg).map(k => ({ + kind: k, + def: getMcpDefinition(cfg, k), + })), + ); + } catch { + // Config may be stale — grade without blinding. + } + } + const summary = await gradeBatch(dir, { + judgeModel: cmdOpts.judgeModel, + rerunJudge: cmdOpts.rerunJudge ?? false, + skipJudge: cmdOpts.judge === false, + blindingEntries, + }); + console.log( + `\nGraded ${summary.graded.length} run${summary.graded.length === 1 ? '' : 's'}; ${summary.errors.length} error${summary.errors.length === 1 ? '' : 's'}.`, + ); + if (summary.errors.length > 0) { + for (const e of summary.errors) console.log(' -', e.runPath, e.error); + process.exit(1); + } + }, + ); + +program + .command('runs-instrument ') + .description( + 'Enrich each run with per-tool-call query_log timings from ClickHouse', + ) + .action(async (batch: string) => { + const opts = program.opts(); + const cfgPath = configExists() ? configPath() : null; + let chUrl = opts.chUrl; + let chUser = opts.chUser; + let chPassword = opts.chPassword; + if (!chUrl && cfgPath) { + const cfg = readConfig(); + if (cfg.clickhouse) { + chUrl = `http://${cfg.clickhouse.host}:${cfg.clickhouse.port}`; + chUser = chUser ?? cfg.clickhouse.user; + chPassword = chPassword ?? cfg.clickhouse.password; + } + } + chUrl = chUrl ?? defaultClickHouseUrl(); + const records = await instrumentBatch(batch, { + clickhouseUrl: chUrl, + username: chUser, + password: chPassword, + }); + for (const r of records) { + console.log(' ' + summarizeTimingRecord(r)); + } + console.log(`\nWrote ${records.length} *.timing.json sidecars.`); + }); + +program + .command('report ') + .description('Render markdown + JSON summary from grade JSONs in the batch') + .option('--out ', 'Output markdown path (default: /_summary.md)') + .option( + '--baseline ', + 'MCP to use as baseline for delta computation (default: first MCP found)', + ) + .action( + async (batch: string, cmdOpts: { out?: string; baseline?: string }) => { + const dir = resolveBatchDir(batch); + const out = resolve(cmdOpts.out ?? `${dir}/_summary.md`); + const result = writeBatchSummary(dir, out, cmdOpts.baseline); + console.log(`Wrote ${result.mdPath}`); + console.log(`Wrote ${result.jsonPath}`); + console.log( + `Aggregated ${result.pairsCount} run${result.pairsCount === 1 ? '' : 's'}.`, + ); + }, + ); + +function parseMcpFlag( + v: string, + config: import('./hyperdx/config').EvalConfig, +): McpKind[] { + // "all" returns only enabled MCPs (where enabled !== false). + if (v === 'all') { + const enabled = enabledMcpNames(config); + if (enabled.length === 0) { + throw new Error( + 'No enabled MCPs in config. Set "enabled": true on at least one MCP, ' + + 'or name MCPs explicitly with --mcp.', + ); + } + return enabled; + } + // Explicit names bypass the enabled flag — you can run a disabled MCP + // by naming it directly. + const names = v + .split(',') + .map(s => s.trim()) + .filter(Boolean); + if (names.length === 0) { + throw new Error( + `--mcp must be comma-separated MCP names or "all", got: ${v}`, + ); + } + const available = configMcpNames(config); + for (const name of names) { + if (!available.includes(name)) { + throw new Error( + `--mcp: "${name}" not found in config. Available: ${available.join(', ')}`, + ); + } + } + return names; +} + +function parsePromptVariant(v: string): PromptVariant { + if (v === 'baseline' || v === 'hypothesis') return v; + throw new Error( + `--prompt-variant must be "baseline" or "hypothesis", got: ${v}`, + ); +} + +program.parseAsync(process.argv).catch(err => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/packages/hdx-eval/src/clickhouse/client.ts b/packages/hdx-eval/src/clickhouse/client.ts new file mode 100644 index 0000000000..50364f45b4 --- /dev/null +++ b/packages/hdx-eval/src/clickhouse/client.ts @@ -0,0 +1,35 @@ +import { type ClickHouseClient, createClient } from '@clickhouse/client'; +import ms from 'ms'; + +export type EvalClickHouseConfig = { + url: string; + username?: string; + password?: string; +}; + +export function createEvalClient( + cfg: EvalClickHouseConfig, + application = 'hdx-eval', +): ClickHouseClient { + return createClient({ + url: cfg.url, + username: cfg.username ?? 'default', + password: cfg.password ?? '', + application, + request_timeout: ms('2m'), + compression: { request: true, response: false }, + clickhouse_settings: { + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + max_execution_time: ms('2m') / 1000, + }, + }); +} + +export function defaultClickHouseUrl(): string { + if (process.env.HDX_EVAL_CH_URL) return process.env.HDX_EVAL_CH_URL; + if (process.env.HDX_DEV_CH_HTTP_PORT) { + return `http://localhost:${process.env.HDX_DEV_CH_HTTP_PORT}`; + } + return 'http://localhost:8123'; +} diff --git a/packages/hdx-eval/src/clickhouse/insert.ts b/packages/hdx-eval/src/clickhouse/insert.ts new file mode 100644 index 0000000000..8d9019256e --- /dev/null +++ b/packages/hdx-eval/src/clickhouse/insert.ts @@ -0,0 +1,106 @@ +import type { ClickHouseClient } from '@clickhouse/client'; + +import type { LogRow, TraceRow } from '../generators/types'; +import { EVAL_DATABASE } from './schema'; + +const BATCH_SIZE = 100_000; + +function msToDateTime64(ms: number): string { + const date = new Date(ms); + const iso = date.toISOString(); + const base = iso.slice(0, 19).replace('T', ' '); + const msPart = iso.slice(20, 23); + return `${base}.${msPart}000000`; +} + +function chunk(rows: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < rows.length; i += size) { + out.push(rows.slice(i, i + size)); + } + return out; +} + +function traceRowToCHObject(r: TraceRow): Record { + return { + Timestamp: msToDateTime64(r.timestampMs), + TraceId: r.traceId, + SpanId: r.spanId, + ParentSpanId: r.parentSpanId, + TraceState: '', + SpanName: r.spanName, + SpanKind: r.spanKind, + ServiceName: r.serviceName, + ResourceAttributes: r.resourceAttributes, + ScopeName: '', + ScopeVersion: '', + SpanAttributes: r.spanAttributes, + Duration: String(r.durationNs), + StatusCode: r.statusCode, + StatusMessage: r.statusMessage, + 'Events.Timestamp': [], + 'Events.Name': [], + 'Events.Attributes': [], + 'Links.TraceId': [], + 'Links.SpanId': [], + 'Links.TraceState': [], + 'Links.Attributes': [], + }; +} + +function logRowToCHObject(r: LogRow): Record { + return { + Timestamp: msToDateTime64(r.timestampMs), + TraceId: r.traceId ?? '', + SpanId: r.spanId ?? '', + TraceFlags: 0, + SeverityText: r.severityText, + SeverityNumber: r.severityNumber, + ServiceName: r.serviceName, + Body: r.body, + ResourceSchemaUrl: '', + ResourceAttributes: r.resourceAttributes, + ScopeSchemaUrl: '', + ScopeName: '', + ScopeVersion: '', + ScopeAttributes: {}, + LogAttributes: r.logAttributes, + EventName: '', + }; +} + +export async function insertTraceRows( + client: ClickHouseClient, + table: string, + rows: TraceRow[], +): Promise { + if (rows.length === 0) return 0; + let inserted = 0; + for (const batch of chunk(rows, BATCH_SIZE)) { + await client.insert({ + table: `${EVAL_DATABASE}.${table}`, + values: batch.map(traceRowToCHObject), + format: 'JSONEachRow', + }); + inserted += batch.length; + } + return inserted; +} + +export async function insertLogRows( + client: ClickHouseClient, + table: string, + rows: LogRow[], +): Promise { + if (rows.length === 0) return 0; + let inserted = 0; + for (const batch of chunk(rows, BATCH_SIZE)) { + await client.insert({ + table: `${EVAL_DATABASE}.${table}`, + values: batch.map(logRowToCHObject), + format: 'JSONEachRow', + }); + inserted += batch.length; + } + return inserted; +} diff --git a/packages/hdx-eval/src/clickhouse/schema.ts b/packages/hdx-eval/src/clickhouse/schema.ts new file mode 100644 index 0000000000..27decf3d36 --- /dev/null +++ b/packages/hdx-eval/src/clickhouse/schema.ts @@ -0,0 +1,368 @@ +import type { ClickHouseClient } from '@clickhouse/client'; + +export const EVAL_DATABASE = 'default'; +const SOURCE_TRACES_TABLE = 'otel_traces'; +const SOURCE_LOGS_TABLE = 'otel_logs'; + +export type ScenarioTables = { + traces: string; + logs: string; + tracesKvRollup: string; + tracesKeyRollup: string; + logsKvRollup: string; + logsKeyRollup: string; +}; + +function scenarioSlug(scenario: string): string { + if (!/^[a-z0-9_-]+$/.test(scenario)) { + throw new Error( + `Invalid scenario name "${scenario}": must match /^[a-z0-9_-]+$/`, + ); + } + return scenario.replace(/-/g, '_'); +} + +export function scenarioTables(scenario: string): ScenarioTables { + const slug = scenarioSlug(scenario); + return { + traces: `eval_${slug}_otel_traces`, + logs: `eval_${slug}_otel_logs`, + tracesKvRollup: `eval_${slug}_otel_traces_kv_rollup_15m`, + tracesKeyRollup: `eval_${slug}_otel_traces_key_rollup_15m`, + logsKvRollup: `eval_${slug}_otel_logs_kv_rollup_15m`, + logsKeyRollup: `eval_${slug}_otel_logs_key_rollup_15m`, + }; +} + +async function tableExists( + client: ClickHouseClient, + database: string, + table: string, +): Promise { + const rs = await client.query({ + query: `SELECT 1 FROM system.tables WHERE database = {db:String} AND name = {tbl:String}`, + query_params: { db: database, tbl: table }, + format: 'JSONEachRow', + }); + const rows = await rs.json(); + return rows.length > 0; +} + +async function assertSourceTablesExist( + client: ClickHouseClient, +): Promise { + for (const t of [SOURCE_TRACES_TABLE, SOURCE_LOGS_TABLE]) { + if (!(await tableExists(client, EVAL_DATABASE, t))) { + throw new Error( + `Required source table ${EVAL_DATABASE}.${t} does not exist. ` + + `Start the dev stack (\`yarn dev\`) so the OTel collector can ` + + `create the standard otel_* tables before seeding eval data.`, + ); + } + } +} + +export async function ensureScenarioTables( + client: ClickHouseClient, + scenario: string, +): Promise { + await assertSourceTablesExist(client); + const tables = scenarioTables(scenario); + + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${EVAL_DATABASE}.${tables.traces} AS ${EVAL_DATABASE}.${SOURCE_TRACES_TABLE}`, + }); + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${EVAL_DATABASE}.${tables.logs} AS ${EVAL_DATABASE}.${SOURCE_LOGS_TABLE}`, + }); + + // The source tables inherit a 1-day TTL from HyperDX migrations. Eval data + // is anchored to a chosen `now` (often historical) and we control its + // lifecycle via the seed/drop CLI, so strip TTL on the eval copies. Without + // this, seeding with `--now` >1 day in the past silently drops the part. + // Idempotent: subsequent runs error with "no TTL to remove" — ignore. + await removeTtlIfPresent(client, tables.traces); + await removeTtlIfPresent(client, tables.logs); + + // Create KV/Key rollup tables and materialized views so the HyperDX MCP + // can use fast metadata discovery instead of scanning the full raw tables. + // MVs are created *before* seeding so they capture inserts as they happen. + await ensureRollupTables(client, tables); + + return tables; +} + +async function removeTtlIfPresent( + client: ClickHouseClient, + table: string, +): Promise { + try { + await client.command({ + query: `ALTER TABLE ${EVAL_DATABASE}.${table} REMOVE TTL`, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/cannot remove|doesn't have any table TTL/i.test(msg)) return; + throw err; + } +} + +/** + * Return `true` when the scenario's traces table exists and has at least one + * row. A quick, cheap check to decide whether an auto-seed is needed before + * a `run`. + */ +export async function scenarioIsSeeded( + client: ClickHouseClient, + scenario: string, +): Promise { + const tables = scenarioTables(scenario); + if (!(await tableExists(client, EVAL_DATABASE, tables.traces))) { + return false; + } + const rs = await client.query({ + query: `SELECT 1 FROM ${EVAL_DATABASE}.${tables.traces} LIMIT 1`, + format: 'JSONEachRow', + }); + const rows = await rs.json(); + return rows.length > 0; +} + +export async function truncateScenarioTables( + client: ClickHouseClient, + scenario: string, +): Promise { + const tables = scenarioTables(scenario); + // Truncate rollup tables alongside raw tables. + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.tracesKvRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.tracesKeyRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logsKvRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logsKeyRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.traces}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logs}`, + }); +} + +export async function dropScenarioTables( + client: ClickHouseClient, + scenario: string, +): Promise { + const tables = scenarioTables(scenario); + // Drop MVs first (they depend on the tables), then rollup tables, then raw. + await dropRollupTables(client, tables); + await client.command({ + query: `DROP TABLE IF EXISTS ${EVAL_DATABASE}.${tables.traces}`, + }); + await client.command({ + query: `DROP TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logs}`, + }); +} + +// --------------------------------------------------------------------------- +// Rollup tables & materialized views +// --------------------------------------------------------------------------- + +const ROLLUP_TABLE_DDL = `( + \`Timestamp\` DateTime, + \`ColumnIdentifier\` LowCardinality(String), + \`Key\` LowCardinality(String), + \`Value\` String, + \`count\` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp, Value) +SETTINGS index_granularity = 8192`; + +const KEY_ROLLUP_TABLE_DDL = `( + \`Timestamp\` DateTime, + \`ColumnIdentifier\` LowCardinality(String), + \`Key\` LowCardinality(String), + \`count\` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp) +SETTINGS index_granularity = 8192`; + +function mvName(rollupTable: string): string { + return `${rollupTable}_mv`; +} + +function keyMvName(keyRollupTable: string): string { + return `${keyRollupTable}_mv`; +} + +/** Build the KV rollup MV SELECT for a traces table. */ +function tracesKvMvSelect(db: string, rawTable: string): string { + return ` +WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'SpanAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN SpanAttributes AS entry + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ServiceName', CAST(ServiceName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'SpanName', CAST(SpanName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'SpanKind', CAST(SpanKind AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'StatusCode', CAST(StatusCode AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeName', CAST(ScopeName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeVersion', CAST(ScopeVersion AS String) FROM ${db}.${rawTable} +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value`; +} + +/** Build the KV rollup MV SELECT for a logs table. */ +function logsKvMvSelect(db: string, rawTable: string): string { + return ` +WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'LogAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN LogAttributes AS entry + UNION ALL + SELECT + 'ScopeAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN ScopeAttributes AS entry + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'SeverityText', CAST(SeverityText AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ServiceName', CAST(ServiceName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeName', CAST(ScopeName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeVersion', CAST(ScopeVersion AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ResourceSchemaUrl', CAST(ResourceSchemaUrl AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeSchemaUrl', CAST(ScopeSchemaUrl AS String) FROM ${db}.${rawTable} +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value`; +} + +/** Key rollup MV SELECT (shared for both traces and logs). */ +function keyRollupMvSelect(db: string, kvRollupTable: string): string { + return ` +SELECT + Timestamp, + ColumnIdentifier, + Key, + sum(count) as count +FROM ${db}.${kvRollupTable} +GROUP BY ColumnIdentifier, Key, Timestamp`; +} + +async function ensureRollupTables( + client: ClickHouseClient, + tables: ScenarioTables, +): Promise { + const db = EVAL_DATABASE; + + // -- Traces rollups -- + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.tracesKvRollup} ${ROLLUP_TABLE_DDL}`, + }); + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.tracesKeyRollup} ${KEY_ROLLUP_TABLE_DDL}`, + }); + // KV rollup MV: raw traces → kv rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${mvName(tables.tracesKvRollup)} TO ${db}.${tables.tracesKvRollup} AS ${tracesKvMvSelect(db, tables.traces)}`, + }); + // Key rollup MV: kv rollup → key rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${keyMvName(tables.tracesKeyRollup)} TO ${db}.${tables.tracesKeyRollup} AS ${keyRollupMvSelect(db, tables.tracesKvRollup)}`, + }); + + // -- Logs rollups -- + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.logsKvRollup} ${ROLLUP_TABLE_DDL}`, + }); + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.logsKeyRollup} ${KEY_ROLLUP_TABLE_DDL}`, + }); + // KV rollup MV: raw logs → kv rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${mvName(tables.logsKvRollup)} TO ${db}.${tables.logsKvRollup} AS ${logsKvMvSelect(db, tables.logs)}`, + }); + // Key rollup MV: kv rollup → key rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${keyMvName(tables.logsKeyRollup)} TO ${db}.${tables.logsKeyRollup} AS ${keyRollupMvSelect(db, tables.logsKvRollup)}`, + }); +} + +async function dropRollupTables( + client: ClickHouseClient, + tables: ScenarioTables, +): Promise { + const db = EVAL_DATABASE; + // Drop MVs first (they reference the tables). + for (const mv of [ + mvName(tables.tracesKvRollup), + keyMvName(tables.tracesKeyRollup), + mvName(tables.logsKvRollup), + keyMvName(tables.logsKeyRollup), + ]) { + await client.command({ + query: `DROP VIEW IF EXISTS ${db}.${mv}`, + }); + } + for (const tbl of [ + tables.tracesKvRollup, + tables.tracesKeyRollup, + tables.logsKvRollup, + tables.logsKeyRollup, + ]) { + await client.command({ + query: `DROP TABLE IF EXISTS ${db}.${tbl}`, + }); + } +} diff --git a/packages/hdx-eval/src/generators/logs.ts b/packages/hdx-eval/src/generators/logs.ts new file mode 100644 index 0000000000..1e5b6b25da --- /dev/null +++ b/packages/hdx-eval/src/generators/logs.ts @@ -0,0 +1,30 @@ +import type { LogRow } from './types'; +import { SEVERITY_NUMBER } from './types'; + +export type LogInput = { + timestampMs: number; + serviceName: string; + severityText: LogRow['severityText']; + body: string; + traceId?: string; + spanId?: string; + resourceAttributes?: Record; + logAttributes?: Record; +}; + +export function makeLog(input: LogInput): LogRow { + return { + timestampMs: input.timestampMs, + traceId: input.traceId, + spanId: input.spanId, + serviceName: input.serviceName, + severityText: input.severityText, + severityNumber: SEVERITY_NUMBER[input.severityText], + body: input.body, + resourceAttributes: { + 'service.name': input.serviceName, + ...(input.resourceAttributes ?? {}), + }, + logAttributes: input.logAttributes ?? {}, + }; +} diff --git a/packages/hdx-eval/src/generators/templates.ts b/packages/hdx-eval/src/generators/templates.ts new file mode 100644 index 0000000000..3acf65973a --- /dev/null +++ b/packages/hdx-eval/src/generators/templates.ts @@ -0,0 +1,1162 @@ +/** + * Realistic body/attribute generators inspired by the public ClickHouse + * otel_v2 demo dataset (~68M log rows, ~55% unique bodies). Each helper + * produces a body string with embedded dynamic values plus an attribute + * map — the goal is high body cardinality with stable identifiable + * templates. + */ +import type { SeededRng } from '../rng/seeded'; + +// ─── Severity ──────────────────────────────────────────────────────────── +// +// Real-world OTel collectors aggregate logs from many runtimes and emit +// inconsistent severity strings. Mirror that mess so agents have to handle +// case folding and aliasing. + +const SEVERITY_DISTRIBUTION: { value: string; weight: number }[] = [ + { value: 'info', weight: 60 }, + { value: 'INFO', weight: 8 }, + { value: 'information', weight: 8 }, + { value: 'debug', weight: 5 }, + { value: 'DEBUG', weight: 2 }, + { value: 'warn', weight: 8 }, + { value: 'WARN', weight: 1.5 }, + { value: 'warning', weight: 1 }, + { value: 'error', weight: 3 }, + { value: 'ERROR', weight: 1.5 }, + { value: 'fatal', weight: 0.4 }, + { value: 'trace', weight: 1.6 }, +]; + +const SEVERITY_NUMBER_BY_TEXT: Record = { + trace: 1, + TRACE: 1, + debug: 5, + DEBUG: 5, + info: 9, + INFO: 9, + information: 9, + warn: 13, + WARN: 13, + warning: 13, + error: 17, + ERROR: 17, + fatal: 21, + FATAL: 21, +}; + +function pickSeverity(rng: SeededRng): { + text: string; + number: number; +} { + const text = rng.weightedPick(SEVERITY_DISTRIBUTION); + return { text, number: SEVERITY_NUMBER_BY_TEXT[text] ?? 9 }; +} + +/** + * Pick a severity but bias toward a target band ('info'|'warn'|'error'). + * Useful when emitting service-level chatter where the level is determined + * by the event itself but should still vary case/alias. + */ +export function pickSeverityIn( + rng: SeededRng, + band: 'info' | 'debug' | 'warn' | 'error' | 'trace', +): { text: string; number: number } { + const families: Record = { + info: [ + { value: 'info', weight: 70 }, + { value: 'INFO', weight: 15 }, + { value: 'information', weight: 15 }, + ], + debug: [ + { value: 'debug', weight: 75 }, + { value: 'DEBUG', weight: 25 }, + ], + warn: [ + { value: 'warn', weight: 70 }, + { value: 'WARN', weight: 15 }, + { value: 'warning', weight: 15 }, + ], + error: [ + { value: 'error', weight: 70 }, + { value: 'ERROR', weight: 25 }, + { value: 'fatal', weight: 5 }, + ], + trace: [{ value: 'trace', weight: 100 }], + }; + const text = rng.weightedPick(families[band]); + return { text, number: SEVERITY_NUMBER_BY_TEXT[text] ?? 9 }; +} + +/** + * Normalize messy OTel severity text (case variants, aliases) into the + * canonical uppercase form used by ClickHouse schema columns. + */ +export function normalizeSeverityText( + raw: string, +): 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' { + const u = raw.toUpperCase(); + if (u.startsWith('WARN')) return 'WARN'; + if (u.startsWith('ERR') || u === 'FATAL') return 'ERROR'; + if (u.startsWith('DEB')) return 'DEBUG'; + if (u === 'TRACE') return 'TRACE'; + return 'INFO'; +} + +/** + * Spread `count` timestamps linearly across a time window. + * `marginMs` is subtracted from the window end to avoid landing exactly + * on the boundary. + */ +export function spreadTimestamp( + i: number, + count: number, + startMs: number, + windowMs: number, + marginMs = 30_000, +): number { + return count > 1 + ? startMs + (i / (count - 1)) * (windowMs - marginMs) + : startMs; +} + +// ─── Resource attributes (k8s/OTel-style) ──────────────────────────────── + +const NAMESPACES = [ + 'production', + 'staging', + 'platform', + 'observability', + 'edge', + 'data-plane', +]; +const NODES = [ + 'gke-prod-pool-0a1b', + 'gke-prod-pool-2c3d', + 'gke-prod-pool-4e5f', + 'gke-prod-pool-6g7h', +]; +const SERVICE_VERSIONS = ['1.42.3', '1.42.4', '1.43.0', '1.43.1', '2.0.0-rc1']; + +function buildResourceAttrs(args: { + rng: SeededRng; + serviceName: string; + region?: string; +}): Record { + const { rng, serviceName, region } = args; + const podShortId = rng.hex(5); + const podHash = rng.hex(8); + const ns = rng.pick(NAMESPACES); + const node = rng.pick(NODES); + return { + 'service.name': serviceName, + 'service.namespace': ns, + 'service.version': rng.pick(SERVICE_VERSIONS), + 'service.instance.id': `${serviceName}-${podHash}-${podShortId}`, + 'k8s.namespace.name': ns, + 'k8s.pod.name': `${serviceName}-${podHash}-${podShortId}`, + 'k8s.pod.uid': uuidv4(rng), + 'k8s.node.name': node, + 'k8s.deployment.name': serviceName, + 'k8s.container.name': serviceName, + 'host.name': node, + 'cloud.region': + region ?? rng.pick(['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1']), + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.language': rng.pick([ + 'go', + 'java', + 'nodejs', + 'python', + 'rust', + ]), + 'telemetry.sdk.version': rng.pick(['1.27.0', '1.28.0', '1.29.0']), + 'deployment.environment.name': ns, + }; +} + +/** + * Pre-build a small pool of resource-attribute objects per service so we can + * sample at row-emit time without paying ~30 attribute-construction lookups + * per span. Mirrors reality (small fixed set of pods/replicas per service). + */ +export function buildResourcePool(args: { + rng: SeededRng; + services: readonly string[]; + instancesPerService?: number; + region?: string; +}): Record[]> { + const { rng, services, region } = args; + const n = args.instancesPerService ?? 24; + const out: Record[]> = {}; + for (const svc of services) { + out[svc] = []; + for (let i = 0; i < n; i++) { + out[svc].push(buildResourceAttrs({ rng, serviceName: svc, region })); + } + } + return out; +} + +export function pickResource( + rng: SeededRng, + pool: Record[]>, + serviceName: string, +): Record { + const list = pool[serviceName]; + if (!list || list.length === 0) { + return buildResourceAttrs({ rng, serviceName }); + } + return rng.pick(list); +} + +// ─── Body templates ────────────────────────────────────────────────────── + +export function uuidv4(rng: SeededRng): string { + // Deterministic v4-shaped UUID (no real randomness — we use seeded RNG). + const h = rng.hex(16); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-4${h.slice(13, 16)}-${ + '89ab'[rng.intRange(0, 4)] + }${h.slice(17, 20)}-${h.slice(20, 32)}`; +} + +const HTTP_METHODS = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH']; +const HTTP_PATHS = [ + '/api/cart', + '/api/checkout', + '/api/orders', + '/api/orders/{id}', + '/api/products', + '/api/products/{id}', + '/api/search', + '/api/recommendations', + '/api/users/{id}', + '/api/inventory/{sku}', + '/api/health', + '/api/auth/refresh', +]; +const USER_AGENTS = [ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36', + 'python-requests/2.32.3', + 'curl/8.6.0', + 'Mozilla/5.0 (compatible; CheckpointBot/1.0)', +]; + +/** Envoy-style access log body — high cardinality (timestamps, IDs, sizes). */ +export function envoyAccessLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const ts = new Date(nowMs).toISOString(); + const method = rng.pick(HTTP_METHODS); + const path = rng.pick(HTTP_PATHS); + const status = rng.weightedPick([ + { value: 200, weight: 90 }, + { value: 304, weight: 3 }, + { value: 404, weight: 3 }, + { value: 401, weight: 1 }, + { value: 500, weight: 2 }, + { value: 503, weight: 1 }, + ]); + const reqBytes = rng.intRange(0, 4096); + const respBytes = rng.intRange(0, 32768); + const totalMs = rng.intRange(2, 800); + const upstreamMs = rng.intRange(1, totalMs); + const reqId = uuidv4(rng); + const ua = rng.pick(USER_AGENTS); + const upstreamIp = `10.${rng.intRange(0, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`; + const sourceIp = `34.${rng.intRange(100, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`; + const body = + `[${ts}] "${method} ${path} HTTP/1.1" ${status} - via_upstream - "-" ` + + `${reqBytes} ${respBytes} ${totalMs} ${upstreamMs} "-" "${ua}" ` + + `"${reqId}" "frontend-proxy:8080" "${upstreamIp}:80" frontend-proxy ` + + `${sourceIp}:0 ${upstreamIp}:80 - - default`; + return { + body, + attrs: { + 'event.name': 'proxy.access', + 'http.method': method, + 'url.template': path, + 'url.path': path, + 'url.full': `https://example.com${path}`, + 'http.status_code': String(status), + 'user_agent.original': ua, + 'net.peer_address': upstreamIp, + 'source.address': sourceIp, + 'server.address': 'frontend-proxy', + 'upstream.host': `${upstreamIp}:80`, + 'upstream.cluster': 'default', + 'destination.address': `${upstreamIp}:80`, + 'http.request.body.size': String(reqBytes), + 'http.response.body.size': String(respBytes), + 'http.duration_ms': String(totalMs), + request_id: reqId, + 'log.iostream': 'stdout', + 'log.file.path': '/var/log/envoy/access.log', + logtag: 'F', + }, + }; +} + +/** logfmt body (popular Go/Python convention). */ +function logfmtBody(args: { + rng: SeededRng; + nowMs: number; + level: string; + msg: string; + fields: Record; +}): string { + const ts = new Date(args.nowMs).toISOString(); + const parts = [`time="${ts}"`, `level=${args.level}`, `msg="${args.msg}"`]; + for (const [k, v] of Object.entries(args.fields)) { + if (typeof v === 'string' && /[\s"=]/.test(v)) { + parts.push(`${k}="${v.replace(/"/g, '\\"')}"`); + } else { + parts.push(`${k}=${v}`); + } + } + return parts.join(' '); +} + +const BRANDS = [ + 'Nikon', + 'Canon', + 'Sony', + 'Vixen', + 'Bushnell', + 'Orion', + 'Sky-Watcher', + 'Bresser', + 'Leica', + 'Celestron', + 'Meade', +]; +const CATEGORIES = [ + 'binoculars', + 'telescopes', + 'eyepieces', + 'mounts', + 'tripods', + 'filters', + 'cameras', + 'accessories', + 'books', + 'travel', + 'assembly', +]; + +/** Product catalog "Product found" log — logfmt with rich attrs. */ +export function catalogLog(args: { + rng: SeededRng; + nowMs: number; + level?: string; +}): { body: string; attrs: Record } { + const { rng, nowMs } = args; + const level = args.level ?? 'info'; + const productId = rng.hex(5).toUpperCase().slice(0, 10); + const brand = rng.pick(BRANDS); + const category = rng.pick(CATEGORIES); + const cacheHit = rng.next() < 0.78; + const inStock = rng.next() < 0.82; + const featured = rng.next() < 0.12; + const onSale = rng.next() < 0.18; + const price = rng.intRange(2999, 89999); + const fields = { + 'catalog.product_id': productId, + 'catalog.brand': brand, + 'catalog.cache_hit': cacheHit, + 'catalog.category': category, + 'catalog.in_stock': inStock, + 'catalog.is_featured': featured, + 'catalog.is_on_sale': onSale, + 'catalog.price_cents': price, + 'catalog.lookup_ms': rng.intRange(1, 80), + }; + return { + body: logfmtBody({ + rng, + nowMs, + level, + msg: 'Product found', + fields, + }), + attrs: { + 'event.name': 'catalog.lookup', + 'code.function.name': 'GetProduct', + 'code.line.number': String(rng.intRange(40, 320)), + 'code.file.path': '/app/internal/catalog/server.go', + 'product.id': productId, + 'product.brand': brand, + 'product.category': category, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** JSON-event body, common for app-tier services. */ +function jsonEventBody(args: { + rng: SeededRng; + fields: Record; +}): string { + return JSON.stringify(args.fields); +} + +const PAGES = [ + '/home', + '/product', + '/cart', + '/checkout', + '/profile', + '/orders', + '/recommendations', + '/search', +]; + +/** Frontend page render timing. Keep — it's RUM data. */ +export function pageRenderLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const path = rng.pick(PAGES); + const renderMs = rng.intRange(40, 1200); + const ttfb = rng.intRange(20, renderMs / 2); + const fcp = rng.intRange(ttfb, renderMs); + const lcp = rng.intRange(fcp, renderMs); + const userId = `u${rng.intRange(1, 1_000_000)}`; + const sessionId = uuidv4(rng); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'page.render', + fields: { + path, + render_ms: renderMs, + ttfb_ms: ttfb, + fcp_ms: fcp, + lcp_ms: lcp, + user_id: userId, + session_id: sessionId, + viewport: rng.pick(['1920x1080', '1366x768', '375x812', '1280x800']), + }, + }), + attrs: { + 'event.name': 'page.render', + 'http.route': path, + 'render.ms': String(renderMs), + 'rum.session_id': sessionId, + 'user.id': userId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** search-service indexing progress. KEEP — operational signal. */ +export function indexingLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const docs = rng.intRange(1, 5000); + const shard = rng.intRange(0, 32); + const lagMs = rng.intRange(50, 1500); + const seq = rng.intRange(1_000_000, 9_999_999); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'index.shard.flush', + fields: { + 'search.docs_indexed': docs, + 'search.shard': shard, + 'search.lag_ms': lagMs, + 'search.seq': seq, + 'search.bytes_written': rng.intRange(2048, 524288), + }, + }), + attrs: { + 'event.name': 'index.shard.flush', + 'search.shard': String(shard), + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +const ANALYTICS_EVENTS = [ + 'product.view', + 'cart.add', + 'cart.remove', + 'checkout.start', + 'checkout.complete', + 'search.perform', + 'search.click', + 'recommendation.click', + 'page.scroll', + 'video.play', + 'wishlist.add', +]; + +/** Analytics event log — JSON, KEEP (product signal). */ +export function analyticsEventLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const event = rng.pick(ANALYTICS_EVENTS); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + const productId = rng.hex(5).toUpperCase().slice(0, 10); + const fields: Record = { + 'event.name': event, + user_id: userId, + session_id: sessionId, + timestamp: new Date(nowMs).toISOString(), + referrer: rng.weightedPick([ + { value: 'organic', weight: 50 }, + { value: 'paid', weight: 20 }, + { value: 'direct', weight: 25 }, + { value: 'email', weight: 5 }, + ]), + device: rng.pick(['desktop', 'mobile', 'tablet']), + }; + if ( + event.startsWith('product.') || + event.startsWith('cart.') || + event === 'checkout.start' + ) { + fields.product_id = productId; + fields.price_cents = rng.intRange(999, 99999); + } + if (event === 'search.perform') { + fields.query = rng.pick([ + 'nikon binoculars', + 'celestron telescope', + 'tripod', + 'eyepiece 1.25', + 'star chart', + ]); + fields.results_count = rng.intRange(0, 250); + } + return { + body: jsonEventBody({ rng, fields }), + attrs: { + 'event.name': event, + 'user.id': userId, + 'rum.session_id': sessionId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** Health check / heartbeat — DROP. High volume but body has dynamic mem/cpu/uptime. */ +export function heartbeatLog(args: { + rng: SeededRng; + nowMs: number; + serviceName: string; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs } = args; + const uptimeS = rng.intRange(60, 8 * 24 * 3600); + const cpu = rng.range(0.5, 78).toFixed(2); + const mem = rng.intRange(128, 4096); + const queue = rng.intRange(0, 50); + const inflight = rng.intRange(0, 200); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'debug', + msg: 'health.check', + fields: { + component: 'heartbeat', + status: 'ok', + 'infra.uptime_sec': uptimeS, + 'infra.cpu_pct': cpu, + 'infra.memory_usage_mb': mem, + 'infra.queue_depth': queue, + 'infra.inflight_requests': inflight, + }, + }), + attrs: { + 'event.name': 'health.check', + 'infra.cpu_pct': cpu, + 'infra.memory_usage_mb': String(mem), + 'infra.queue_depth': String(queue), + 'log.iostream': 'stdout', + logtag: 'F', + component: 'heartbeat', + }, + level: 'debug', + }; +} + +/** Cache hit chatter — DROP. JSON with high-cardinality keys. */ +export function cacheHitLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const keyKind = rng.pick([ + 'user_profile', + 'session', + 'product', + 'campaign', + 'feature_flag', + 'rate_limit', + 'auth_token', + ]); + const keyId = rng.intRange(1, 10_000_000); + const key = `${keyKind}:${keyId}`; + const layer = rng.pick(['memcached', 'redis', 'in-memory']); + const node = rng.pick(['cache-0', 'cache-1', 'cache-2', 'cache-3']); + const fields = { + event: 'cache.hit', + key, + layer, + node, + 'cache.ttl_remaining_sec': rng.intRange(1, 86400), + 'cache.value_size_bytes': rng.intRange(8, 65536), + timestamp: new Date(nowMs).toISOString(), + }; + return { + body: jsonEventBody({ rng, fields }), + attrs: { + 'event.name': 'cache.hit', + 'cache.layer': layer, + 'cache.key': key, + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'debug', + }; +} + +const STACK_FILES = [ + '/app/worker.js', + '/app/runner.js', + '/app/lib/retry.js', + '/app/lib/queue.js', + '/app/handlers/job.js', +]; + +/** Recurring caught exception — DROP. Same exception class, varying frames. */ +export function caughtExceptionLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const cause = rng.pick([ + 'transient blip', + 'connection reset', + 'upstream backoff', + 'queue full', + 'rate limited downstream', + 'response timeout', + ]); + const frames = Array.from({ length: rng.intRange(3, 6) }, () => { + const file = rng.pick(STACK_FILES); + const line = rng.intRange(20, 480); + const fn = rng.pick([ + 'handleJob', + 'run', + 'process', + 'enqueue', + 'flush', + 'tick', + 'consume', + ]); + return ` at Worker.${fn} (${file}:${line}:${rng.intRange(2, 80)})`; + }); + const body = [ + `NonFatalRetryableError: ${cause} (caught)`, + ...frames, + ` at processTicksAndRejections (node:internal/process/task_queues:96:5)`, + ].join('\n'); + return { + body, + attrs: { + 'event.name': 'job.caught_exception', + 'error.kind': 'NonFatalRetryableError', + 'error.cause': cause, + handled: 'true', + 'code.function.name': 'handleJob', + 'log.iostream': 'stderr', + logtag: 'F', + // Make this look enough like a real error to fool naive severity-based filters + // but the cardinality is constrained — agent should recognize the pattern. + _ts: new Date(nowMs).toISOString(), + }, + level: 'error', + }; +} + +/** Subscription metric-as-log — DROP, convert to gauge metric. */ +export function subscriptionMetricLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs } = args; + const tenant = rng.intRange(1, 50_000); + const mrrCents = rng.intRange(0, 1_000_000); + const seats = rng.intRange(1, 5000); + const trialDaysLeft = rng.intRange(0, 30); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'subscription.metric.calculated', + fields: { + 'tenant.id': tenant, + 'metric.name': 'subscription_mrr', + 'metric.kind': 'gauge', + mrr_cents: mrrCents, + seats, + trial_days_left: trialDaysLeft, + plan: rng.pick(['starter', 'team', 'business', 'enterprise']), + currency: 'USD', + }, + }), + attrs: { + 'event.name': 'subscription.metric.calculated', + 'metric.name': 'subscription_mrr', + 'metric.kind': 'gauge', + 'tenant.id': String(tenant), + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'info', + }; +} + +const BACKGROUND_BODIES_TEMPLATES = [ + (rng: SeededRng) => + `Connection pool stats: idle=${rng.intRange(0, 10)} active=${rng.intRange(0, 30)} waiting=${rng.intRange(0, 5)}`, + (rng: SeededRng) => + `Refunded order ${uuidv4(rng)} amount=${(rng.intRange(100, 99999) / 100).toFixed(2)} reason="${rng.pick(['customer request', 'duplicate charge', 'fraud review', 'pricing error'])}"`, + (rng: SeededRng) => + `Webhook delivered to https://hooks.example.com/${rng.hex(8)} status=${rng.pick([200, 201, 204, 502, 503])} retry=${rng.intRange(0, 3)}`, + (rng: SeededRng) => + `OAuth token refreshed for client_id=${rng.hex(12)} grant_type=refresh_token expires_in=${rng.intRange(300, 7200)}`, + (rng: SeededRng) => + `Slow query detected duration_ms=${rng.intRange(500, 4500)} table=${rng.pick(['orders', 'users', 'invoices', 'sessions'])} rows_scanned=${rng.intRange(10_000, 5_000_000)}`, + (rng: SeededRng) => + `Kafka producer flushed batch_size=${rng.intRange(50, 500)} bytes=${rng.intRange(1024, 1048576)} topic=${rng.pick(['orders', 'audit', 'analytics', 'notifications'])} partition=${rng.intRange(0, 32)}`, + (rng: SeededRng) => + `Migration step completed name="${rng.pick(['add_index_orders_tenant', 'backfill_users_email_lower', 'drop_legacy_sessions'])}" rows_affected=${rng.intRange(0, 10_000_000)} duration_s=${rng.intRange(1, 300)}`, + (rng: SeededRng) => + `CSRF token mismatch on POST /checkout origin=${rng.pick(['https://example.com', 'https://api.example.com', 'null'])} ip=${rng.intRange(1, 254)}.${rng.intRange(0, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`, + (rng: SeededRng) => + `JWT verification failed reason="${rng.pick(['expired', 'invalid signature', 'unknown kid', 'malformed'])}" sub=user-${rng.intRange(1, 999999)}`, + (rng: SeededRng) => + `GDPR deletion request processed user_id=u${rng.intRange(1, 9_999_999)} records_purged=${rng.intRange(0, 50_000)} duration_ms=${rng.intRange(50, 8000)}`, + (rng: SeededRng) => + `Circuit breaker tripped service="${rng.pick(['payment-gateway', 'fraud-detection', 'shipping-api', 'tax-service'])}" failures_in_window=${rng.intRange(5, 200)} cooldown_s=${rng.intRange(15, 300)}`, + (rng: SeededRng) => + `Feature flag evaluated flag="${rng.pick(['new_checkout_flow', 'fast_search', 'price_experiment_b', 'reco_v2'])}" tenant=${rng.intRange(1, 50000)} value=${rng.next() < 0.5 ? 'true' : 'false'}`, + (rng: SeededRng) => + `Health probe response upstream="${rng.pick(['payment-service', 'inventory-service', 'cart-service'])}" code=${rng.pick([200, 200, 200, 503, 504])} latency_ms=${rng.intRange(2, 500)}`, +]; + +/** Random varied background log. */ +export function backgroundLog(args: { rng: SeededRng; nowMs: number }): string { + const tmpl = args.rng.pick(BACKGROUND_BODIES_TEMPLATES); + return tmpl(args.rng); +} + +// ─── Composite-cell helpers ───────────────────────────────────────────── +// +// These helpers exist so that each (ServiceName, SeverityText) cell can +// contain BOTH a noisy pattern and a legitimate load-bearing pattern at +// comparable volumes. That prevents a `GROUP BY ServiceName, SeverityText` +// agent from cheaply identifying noise without looking at event.name / +// body templates. + +/** + * Push-notification delivery record — KEEP. + * Lives in `notification-service × DEBUG` next to the cache.hit chatter. + * Used by support / compliance to confirm whether a user received a push. + */ +export function notificationDeliveryLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs } = args; + const channel = rng.pick(['apns', 'fcm', 'webpush', 'sms']); + const template = rng.pick([ + 'order_shipped', + 'order_delivered', + 'cart_abandoned', + 'price_drop', + 'back_in_stock', + 'security_alert', + 'subscription_expiring', + ]); + const userId = `u${rng.intRange(1, 9_000_000)}`; + const deviceId = uuidv4(rng); + const messageId = uuidv4(rng); + const status = rng.weightedPick([ + { value: 'delivered', weight: 88 }, + { value: 'queued', weight: 5 }, + { value: 'throttled', weight: 4 }, + { value: 'unregistered', weight: 3 }, + ]); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'debug', + msg: 'notification.delivery', + fields: { + 'notification.channel': channel, + 'notification.template': template, + 'notification.message_id': messageId, + 'notification.status': status, + 'user.id': userId, + 'device.id': deviceId, + 'notification.delivery_ms': rng.intRange(40, 2200), + 'notification.retry_count': rng.intRange(0, 3), + }, + }), + attrs: { + 'event.name': 'notification.delivery', + 'notification.channel': channel, + 'notification.status': status, + 'notification.template': template, + 'user.id': userId, + 'message.id': messageId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'debug', + }; +} + +/** + * Billing financial events — KEEP. + * Lives in `billing-service × INFO` next to the subscription_mrr metric-as-log. + * These are audit/finance-critical records. + */ +export function billingEventLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const event = rng.weightedPick< + | 'invoice.charged' + | 'invoice.refunded' + | 'subscription.activated' + | 'subscription.cancelled' + | 'payment.failed' + >([ + { value: 'invoice.charged', weight: 55 }, + { value: 'invoice.refunded', weight: 8 }, + { value: 'subscription.activated', weight: 18 }, + { value: 'subscription.cancelled', weight: 9 }, + { value: 'payment.failed', weight: 10 }, + ]); + const tenant = rng.intRange(1, 50_000); + const invoiceId = `inv_${rng.hex(12)}`; + const customerId = `cus_${rng.hex(10)}`; + const amountCents = rng.intRange(500, 999_999); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: event, + fields: { + 'tenant.id': tenant, + 'billing.invoice_id': invoiceId, + 'billing.customer_id': customerId, + 'billing.amount_cents': amountCents, + 'billing.currency': 'USD', + 'billing.processor': rng.pick(['stripe', 'braintree', 'adyen']), + 'billing.payment_method': rng.pick([ + 'card', + 'wallet', + 'ach', + 'invoice', + ]), + }, + }), + attrs: { + 'event.name': event, + 'tenant.id': String(tenant), + 'billing.invoice_id': invoiceId, + 'billing.customer_id': customerId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'info', + }; +} + +/** + * Real, unhandled worker failure — KEEP. + * Lives in `worker × ERROR` next to the NonFatalRetryableError chatter. + * These are the errors that actually represent jobs that gave up. + */ +export function jobFailedPermanentLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record; level: string } { + const { rng } = args; + const jobType = rng.pick([ + 'image_resize', + 'email_send', + 'webhook_dispatch', + 'report_export', + 'invoice_generate', + 'data_purge', + ]); + const jobId = uuidv4(rng); + const cause = rng.pick([ + 'JobPoisonPillError: input checksum mismatch', + 'JobTimeoutError: exceeded 300s deadline', + 'JobDataLossError: source row deleted mid-flight', + 'JobIntegrityError: foreign key constraint failed after retries', + ]); + const attempts = rng.intRange(5, 12); + const body = [ + `job.failed.permanent type=${jobType} id=${jobId} attempts=${attempts}`, + `${cause}`, + ` at Worker.handleJob (/app/worker.js:${rng.intRange(20, 480)}:${rng.intRange(2, 80)})`, + ` at Worker.process (/app/runner.js:${rng.intRange(20, 480)}:${rng.intRange(2, 80)})`, + ].join('\n'); + return { + body, + attrs: { + 'event.name': 'job.failed.permanent', + 'job.type': jobType, + 'job.id': jobId, + 'job.attempts': String(attempts), + 'error.kind': cause.split(':')[0], + handled: 'false', + 'log.iostream': 'stderr', + logtag: 'F', + }, + level: 'error', + }; +} + +/** + * Per-service operational debug — KEEP. + * Lives in inventory/pricing/shipping/recommendation × DEBUG next to the + * health.check heartbeats. Each service emits a distinct event.name. + */ +export function serviceOpsDebugLog(args: { + rng: SeededRng; + nowMs: number; + serviceName: + | 'inventory-service' + | 'pricing-service' + | 'shipping-service' + | 'recommendation-service'; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs, serviceName } = args; + let msg: string; + let extra: Record; + let eventName: string; + switch (serviceName) { + case 'inventory-service': { + eventName = 'inventory.stock_lookup'; + msg = 'inventory.stock_lookup'; + const sku = `sku-${rng.hex(8)}`; + extra = { + 'inventory.sku': sku, + 'inventory.warehouse': rng.pick(['us-east', 'us-west', 'eu', 'apac']), + 'inventory.on_hand': rng.intRange(0, 1200), + 'inventory.reserved': rng.intRange(0, 300), + 'inventory.lookup_ms': rng.intRange(2, 90), + }; + break; + } + case 'pricing-service': { + eventName = 'pricing.calculation'; + msg = 'pricing.calculation'; + extra = { + 'pricing.product_id': `prd-${rng.hex(8)}`, + 'pricing.tier': rng.pick(['retail', 'wholesale', 'partner']), + 'pricing.list_cents': rng.intRange(199, 199999), + 'pricing.discount_pct': rng.intRange(0, 60), + 'pricing.tax_jurisdiction': rng.pick(['US-CA', 'US-NY', 'EU-DE', 'JP']), + 'pricing.calc_ms': rng.intRange(1, 35), + }; + break; + } + case 'shipping-service': { + eventName = 'shipping.rate.quote'; + msg = 'shipping.rate.quote'; + extra = { + 'shipping.origin': rng.pick(['LAX', 'JFK', 'DFW', 'SEA', 'ATL']), + 'shipping.dest_zip': String(rng.intRange(10000, 99999)), + 'shipping.carrier': rng.pick(['ups', 'fedex', 'usps', 'dhl']), + 'shipping.weight_g': rng.intRange(50, 18000), + 'shipping.rate_cents': rng.intRange(299, 4999), + 'shipping.eta_days': rng.intRange(1, 9), + }; + break; + } + case 'recommendation-service': { + eventName = 'recommendation.score'; + msg = 'recommendation.score'; + extra = { + 'reco.model': rng.pick(['als_v3', 'content_v2', 'hybrid_v1']), + 'reco.user_id': `u${rng.intRange(1, 8_000_000)}`, + 'reco.candidates': rng.intRange(10, 500), + 'reco.returned': rng.intRange(5, 25), + 'reco.score_ms': rng.intRange(8, 220), + }; + break; + } + } + return { + body: logfmtBody({ rng, nowMs, level: 'debug', msg, fields: extra }), + attrs: { + 'event.name': eventName, + 'log.iostream': 'stdout', + logtag: 'F', + // Surface the most identifying field as an attribute too so + // attribute-grouping agents can find this pattern. + ...Object.fromEntries( + Object.entries(extra) + .filter(([, v]) => typeof v !== 'number') + .map(([k, v]) => [k, String(v)]), + ), + }, + level: 'debug', + }; +} + +/** + * Load-balancer health-probe spam — DROP (or filter at the LB layer). + * Lives in `frontend-proxy × INFO` next to the real Envoy access logs. + * Looks like an access log to a naive grouping but represents internal + * `/healthz` traffic that adds noise without diagnostic value. + */ +export function upstreamHealthProbeLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record } { + const { rng, nowMs } = args; + const ts = new Date(nowMs).toISOString(); + const probe = rng.pick(['/healthz', '/-/ready', '/healthcheck', '/lb/probe']); + const upstreamMs = rng.intRange(0, 8); + const sourceIp = `10.${rng.intRange(0, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`; + return { + body: + `[${ts}] "GET ${probe} HTTP/1.1" 200 - via_upstream - "-" ` + + `0 4 ${upstreamMs} ${upstreamMs} "-" "kube-probe/1.30" ` + + `"-" "frontend-proxy:8080" "127.0.0.1:80" frontend-proxy ` + + `${sourceIp}:0 127.0.0.1:80 - - default`, + attrs: { + 'event.name': 'lb.health.probe', + 'http.method': 'GET', + 'url.path': probe, + 'http.status_code': '200', + 'user_agent.original': 'kube-probe/1.30', + 'http.duration_ms': String(upstreamMs), + 'source.address': sourceIp, + 'server.address': 'frontend-proxy', + 'lb.probe_kind': probe.replace(/^\//, ''), + 'log.iostream': 'stdout', + 'log.file.path': '/var/log/envoy/access.log', + logtag: 'F', + }, + }; +} + +/** + * Client-side console.log dumps flushed to the backend — DROP. + * Lives in `frontend × INFO` next to the page.render RUM logs. Looks like + * a frontend log but is just chatty debug-style console output never used + * for anything. + */ +export function consoleLogDumpLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const lines = rng.intRange(2, 6); + const components = ['Cart', 'Header', 'ProductCard', 'Footer', 'Search']; + const messages = [ + 'render', + 'state hydrated', + 'effect ran', + 'memoize hit', + 'prop drilled', + 'click handler attached', + 'noop', + ]; + const dump: string[] = []; + for (let i = 0; i < lines; i++) { + const comp = rng.pick(components); + const msg = rng.pick(messages); + dump.push( + `[console.log] ${comp}.${msg} ts=${nowMs - rng.intRange(0, 300)}`, + ); + } + const sessionId = uuidv4(rng); + return { + body: `console.log_dump session=${sessionId}\n` + dump.join('\n'), + attrs: { + 'event.name': 'frontend.console.log_dump', + 'rum.session_id': sessionId, + 'frontend.line_count': String(lines), + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** + * Verbose cache-miss event from search-service — DROP. + * Lives in `search-service × INFO` next to the legit index.shard.flush + * KEEP logs. Looks like a normal INFO log but is chatty per-query. + */ +export function searchCacheMissVerboseLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record } { + const { rng, nowMs } = args; + const query = `q-${rng.hex(10)}`; + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'search.cache.miss', + fields: { + 'search.query_hash': query, + 'search.cache_layer': rng.pick(['l1', 'l2', 'edge']), + 'search.shard': rng.intRange(0, 32), + 'search.miss_reason': rng.pick([ + 'ttl_expired', + 'evicted', + 'not_present', + 'invalidated', + ]), + 'search.fallback_ms': rng.intRange(5, 90), + }, + }), + attrs: { + 'event.name': 'search.cache.miss', + 'search.query_hash': query, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} diff --git a/packages/hdx-eval/src/generators/traces.ts b/packages/hdx-eval/src/generators/traces.ts new file mode 100644 index 0000000000..f433f888f2 --- /dev/null +++ b/packages/hdx-eval/src/generators/traces.ts @@ -0,0 +1,50 @@ +import type { SeededRng } from '../rng/seeded'; +import type { TraceRow } from './types'; + +export function newTraceId(rng: SeededRng): string { + return rng.hex(16); +} + +export function newSpanId(rng: SeededRng): string { + return rng.hex(8); +} + +export type SpanInput = { + rng: SeededRng; + timestampMs: number; + traceId: string; + spanId: string; + parentSpanId?: string; + spanName: string; + spanKind?: TraceRow['spanKind']; + serviceName: string; + durationNs: number; + statusCode?: TraceRow['statusCode']; + statusMessage?: string; + resourceAttributes?: Record; + spanAttributes?: Record; +}; + +export function makeSpan(input: SpanInput): TraceRow { + return { + timestampMs: input.timestampMs, + traceId: input.traceId, + spanId: input.spanId, + parentSpanId: input.parentSpanId ?? '', + spanName: input.spanName, + spanKind: input.spanKind ?? 'SPAN_KIND_INTERNAL', + serviceName: input.serviceName, + durationNs: input.durationNs, + statusCode: input.statusCode ?? 'STATUS_CODE_OK', + statusMessage: input.statusMessage ?? '', + resourceAttributes: { + 'service.name': input.serviceName, + ...(input.resourceAttributes ?? {}), + }, + spanAttributes: input.spanAttributes ?? {}, + }; +} + +export function msToNs(ms: number): number { + return Math.floor(ms * 1_000_000); +} diff --git a/packages/hdx-eval/src/generators/types.ts b/packages/hdx-eval/src/generators/types.ts new file mode 100644 index 0000000000..d0d70ee780 --- /dev/null +++ b/packages/hdx-eval/src/generators/types.ts @@ -0,0 +1,39 @@ +export type TraceRow = { + timestampMs: number; + traceId: string; + spanId: string; + parentSpanId: string; + spanName: string; + spanKind: + | 'SPAN_KIND_SERVER' + | 'SPAN_KIND_CLIENT' + | 'SPAN_KIND_INTERNAL' + | 'SPAN_KIND_PRODUCER' + | 'SPAN_KIND_CONSUMER'; + serviceName: string; + durationNs: number; + statusCode: 'STATUS_CODE_OK' | 'STATUS_CODE_ERROR' | 'STATUS_CODE_UNSET'; + statusMessage: string; + resourceAttributes: Record; + spanAttributes: Record; +}; + +export type LogRow = { + timestampMs: number; + traceId?: string; + spanId?: string; + serviceName: string; + severityText: 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; + severityNumber: number; + body: string; + resourceAttributes: Record; + logAttributes: Record; +}; + +export const SEVERITY_NUMBER: Record = { + TRACE: 1, + DEBUG: 5, + INFO: 9, + WARN: 13, + ERROR: 17, +}; diff --git a/packages/hdx-eval/src/grading/blind.ts b/packages/hdx-eval/src/grading/blind.ts new file mode 100644 index 0000000000..082da589db --- /dev/null +++ b/packages/hdx-eval/src/grading/blind.ts @@ -0,0 +1,63 @@ +/** + * Blind candidate answers before showing them to the judge so the judge + * can't tell which MCP the agent used. We replace MCP-identifying tool + * names and brand mentions with neutral placeholders. We do NOT touch + * data references like `payment-service` or `database.query` — those + * are part of the answer being graded. + * + * The blinding rules are derived from each MCP's `brandTerms` and + * `toolPattern` in the config, so adding a new MCP automatically + * gets its identity redacted. + */ + +import type { McpDefinition, McpKind } from '../harness/types'; + +function escapeRegex(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +export type BlindingEntry = { + kind: McpKind; + def: McpDefinition; + /** Anonymized label: "MCP A", "MCP B", etc. */ + anonLabel: string; +}; + +/** + * Build blinding entries from a list of MCP names + definitions. + * Assigns sequential anonymous labels: MCP A, MCP B, MCP C, ... + */ +export function buildBlindingEntries( + mcps: Array<{ kind: McpKind; def: McpDefinition }>, +): BlindingEntry[] { + return mcps.map((m, i) => ({ + ...m, + anonLabel: `MCP ${String.fromCharCode(65 + i)}`, + })); +} + +/** + * Blind an answer string by replacing MCP-identifying tool prefixes + * and brand terms with anonymous labels. + * + * @param text The candidate's final answer text. + * @param entries Blinding entries built from `buildBlindingEntries()`. + * If omitted, no blinding is performed (passthrough). + */ +export function blindAnswer(text: string, entries?: BlindingEntry[]): string { + if (!entries || entries.length === 0) return text; + + let result = text; + for (const { kind, def, anonLabel } of entries) { + // Replace tool prefix: `mcp____` → `mcp__redacted__` + const prefixRx = new RegExp(`\\bmcp__${escapeRegex(kind)}__`, 'gi'); + result = result.replace(prefixRx, 'mcp__redacted__'); + + // Replace brand terms with the anonymous label. + for (const term of def.brandTerms ?? []) { + const termRx = new RegExp(`\\b${escapeRegex(term)}\\b`, 'gi'); + result = result.replace(termRx, anonLabel); + } + } + return result; +} diff --git a/packages/hdx-eval/src/grading/grade.ts b/packages/hdx-eval/src/grading/grade.ts new file mode 100644 index 0000000000..71c2463af7 --- /dev/null +++ b/packages/hdx-eval/src/grading/grade.ts @@ -0,0 +1,268 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; + +import type { RunRecord } from '../harness/types'; +import { runsRoot } from '../runs/path'; +import { readRun } from '../runs/store'; +import { getScenario, SCENARIO_NAMES } from '../scenarios'; +import type { BlindingEntry } from './blind'; +import { judgeTrajectory } from './judge'; +import { runProgrammaticChecks } from './programmatic'; +import { loadScenarioRubric } from './rubric'; +import { + COMBINED_SCORE_JUDGE_WEIGHT, + COMBINED_SCORE_PROGRAMMATIC_WEIGHT, + type GradeRecord, + type JudgeResult, + MAX_ERROR_PENALTY, + type ToolErrorStats, +} from './types'; + +// Infrastructure-error patterns that come from the MCP/CH server itself +// rather than from agent mistakes. We exclude these from the tool-error +// penalty so a server hiccup doesn't get charged against the agent. +const INFRA_ERROR_PATTERNS: RegExp[] = [ + /too many simultaneous queries/i, + /stream has been already consumed/i, + /rate.?limit/i, + /429/, + /503\b/, + /504\b/, + /econn(refused|reset|aborted)/i, + /socket hang up/i, + /timeout.{0,20}(connecting|while waiting)/i, +]; + +function isInfraError(text: string): boolean { + return INFRA_ERROR_PATTERNS.some(rx => rx.test(text)); +} + +function computeToolErrorStats(record: RunRecord): ToolErrorStats { + const total = record.toolCalls.length; + const errored = record.toolCalls.filter(c => c.isError); + // Separate agent-attributable errors from infrastructure failures (rate + // limits, server-side bugs, transient 5xx). Only agent errors feed the + // penalty — the agent has no control over server hiccups. + const agentErrored = errored.filter(c => { + const text = + typeof c.output === 'string' ? c.output : JSON.stringify(c.output ?? ''); + return !isInfraError(text); + }); + const errors = agentErrored.length; + const rate = total > 0 ? errors / total : 0; + // Linear penalty up to MAX_ERROR_PENALTY when ALL tool calls fail. + // Mathematically: penalty = min(rate, MAX_ERROR_PENALTY). + // We intentionally use rate (not errors/maxTurns) so a run that calls + // 1 tool which fails (rate=1.0) is penalized as harshly as a run that + // fails 20 of 20 tool calls — both are equally broken sessions. + const penalty = Math.min(rate, MAX_ERROR_PENALTY); + const samples = agentErrored.slice(0, 3).map(c => { + const text = + typeof c.output === 'string' ? c.output : JSON.stringify(c.output ?? ''); + return { + name: c.name, + preview: text.length > 160 ? text.slice(0, 160) + '…' : text, + }; + }); + return { total, errors, rate, penalty, samples }; +} + +export type GradeBatchOptions = { + judgeModel?: string; + rerunJudge?: boolean; + skipJudge?: boolean; + apiKey?: string; + /** Blinding entries for anonymizing MCP identity during judging. */ + blindingEntries?: BlindingEntry[]; +}; + +export type GradeBatchSummary = { + batchDir: string; + graded: GradeRecord[]; + errors: { runPath: string; error: string }[]; +}; + +export async function gradeBatch( + batchDir: string, + opts: GradeBatchOptions = {}, +): Promise { + const resolved = resolveBatchDir(batchDir); + if (!existsSync(resolved)) { + throw new Error(`Batch directory does not exist: ${resolved}`); + } + const runFiles = listRunFiles(resolved); + + // Decide whether we'll actually need the judge: skipJudge wins, otherwise + // we need it if any run is missing a cached judge OR rerun was requested. + const needsJudge = + !opts.skipJudge && + runFiles.some(p => { + if (opts.rerunJudge) return true; + const existing = readExistingGrade(gradeFilePath(p)); + return !existing?.judge; + }); + + if (needsJudge && !opts.apiKey && !process.env.ANTHROPIC_API_KEY) { + throw new Error( + 'ANTHROPIC_API_KEY is not set; pass --no-judge to skip the LLM judge ' + + 'or supply a key via env.', + ); + } + + const client = needsJudge + ? new Anthropic({ + apiKey: opts.apiKey ?? process.env.ANTHROPIC_API_KEY, + }) + : undefined; + const graded: GradeRecord[] = []; + const errors: { runPath: string; error: string }[] = []; + + for (const runPath of runFiles) { + try { + const record = readRun(runPath); + const gradePath = gradeFilePath(runPath); + const existing = readExistingGrade(gradePath); + + const grade = await gradeOne({ + record, + client, + existing, + opts, + }); + writeGradeFile(gradePath, grade); + graded.push(grade); + const judgeBit = grade.judge + ? `judge=${(grade.judge.weightedScore * 5).toFixed(1)}/5` + : 'judge=skipped'; + const errBit = grade.toolErrors.errors + ? ` errs=${grade.toolErrors.errors}/${grade.toolErrors.total}` + + (grade.toolErrors.penalty > 0 + ? ` (-${(grade.toolErrors.penalty * 100).toFixed(0)}pp)` + : '') + : ''; + console.log( + ` ${grade.scenario}/${grade.mcp}/${grade.runId.split('-').slice(-1)[0]} prog=${( + grade.programmatic.score * 100 + ).toFixed( + 0, + )}% ${judgeBit} combined=${(grade.combinedScore * 100).toFixed(0)}%${errBit}`, + ); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + errors.push({ runPath, error: msg }); + console.error(` ${runPath}: ${msg}`); + } + } + + return { batchDir: resolved, graded, errors }; +} + +async function gradeOne(args: { + record: RunRecord; + client?: Anthropic; + existing: GradeRecord | null; + opts: GradeBatchOptions; +}): Promise { + const { record, client, existing, opts } = args; + const scenario = getScenario(record.scenario); + const rubric = loadScenarioRubric(record.scenario); + + const programmatic = runProgrammaticChecks( + record.finalAnswer, + rubric.programmatic, + ); + + let judge: JudgeResult | null = existing?.judge ?? null; + if (!opts.skipJudge && client && (!judge || opts.rerunJudge)) { + judge = await judgeTrajectory({ + scenarioName: scenario.name, + scenarioPrompt: scenario.agentPrompt, + groundTruth: scenario.groundTruth, + rubric, + finalAnswer: record.finalAnswer, + judgeModel: opts.judgeModel, + client, + blindingEntries: opts.blindingEntries, + }); + } + + const judgeError = judge && 'error' in judge && judge.error; + const judgeScore = judge?.weightedScore ?? 0; + const rawCombined = + judge && !judgeError + ? COMBINED_SCORE_PROGRAMMATIC_WEIGHT * programmatic.score + + COMBINED_SCORE_JUDGE_WEIGHT * judgeScore + : programmatic.score; + const toolErrors = computeToolErrorStats(record); + // Apply the tool-error penalty AFTER scoring the answer. Clamp to [0,1]. + const combinedScore = Math.max( + 0, + Math.min(1, rawCombined - toolErrors.penalty), + ); + + return { + schemaVersion: 2, + runId: record.runId, + scenario: record.scenario, + mcp: record.mcp, + programmatic, + judge, + toolErrors, + combinedScore, + gradedAt: new Date().toISOString(), + judgeModel: judge?.model ?? opts.judgeModel ?? 'skipped', + }; +} + +function listRunFiles(batchDir: string): string[] { + const out: string[] = []; + for (const scenario of safeReaddir(batchDir)) { + if (!SCENARIO_NAMES.includes(scenario)) continue; + const sceneDir = join(batchDir, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const file of safeReaddir(mcpDir)) { + if (file.endsWith('.grade.json')) continue; + if (file.endsWith('.timing.json')) continue; + if (file.endsWith('.json')) out.push(join(mcpDir, file)); + } + } + } + return out.sort(); +} + +function safeReaddir(path: string): string[] { + try { + return readdirSync(path); + } catch { + return []; + } +} + +function readExistingGrade(path: string): GradeRecord | null { + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, 'utf8')) as GradeRecord; + } catch { + return null; + } +} + +function writeGradeFile(path: string, grade: GradeRecord): void { + writeFileSync(path, JSON.stringify(grade, null, 2) + '\n', 'utf8'); +} + +function gradeFilePath(runPath: string): string { + return runPath.replace(/\.json$/, '.grade.json'); +} + +export function resolveBatchDir(input: string): string { + // Accept absolute path, relative path, or just a batch basename. + if (existsSync(input)) return input; + const root = runsRoot(); + const candidate = join(root, input); + if (existsSync(candidate)) return candidate; + // Allow user to drop the bare basename even if not yet present. + return dirname(input) === '.' ? candidate : input; +} diff --git a/packages/hdx-eval/src/grading/judge.ts b/packages/hdx-eval/src/grading/judge.ts new file mode 100644 index 0000000000..7bcb076417 --- /dev/null +++ b/packages/hdx-eval/src/grading/judge.ts @@ -0,0 +1,259 @@ +import Anthropic from '@anthropic-ai/sdk'; + +import { blindAnswer, type BlindingEntry } from './blind'; +import { + buildJudgeSystem, + buildJudgeUser, + formatGroundTruthFacts, +} from './judgePrompt'; +import type { JudgeCriterionScore, JudgeResult, Rubric } from './types'; + +const DEFAULT_JUDGE_MODEL = 'claude-opus-4-7'; +const MAX_TOKENS = 1500; + +export type JudgeOptions = { + scenarioName: string; + scenarioPrompt: string; + groundTruth: unknown; + rubric: Rubric; + finalAnswer: string; + judgeModel?: string; + client?: Anthropic; + /** Blinding entries for anonymizing MCP identity in the answer. */ + blindingEntries?: BlindingEntry[]; +}; + +export async function judgeTrajectory( + opts: JudgeOptions, +): Promise { + const model = opts.judgeModel ?? DEFAULT_JUDGE_MODEL; + const client = + opts.client ?? new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); + + const systemPrompt = buildJudgeSystem(opts.scenarioName, opts.rubric); + const userPrompt = buildJudgeUser({ + scenarioPrompt: opts.scenarioPrompt, + groundTruthFacts: formatGroundTruthFacts(opts.groundTruth), + candidateAnswer: blindAnswer(opts.finalAnswer, opts.blindingEntries), + }); + + const startedMs = Date.now(); + const result = await callJudgeWithRetry({ + client, + model, + systemPrompt, + userPrompt, + }); + const durationMs = Date.now() - startedMs; + + if (result.error) { + return { + model, + scores: {}, + weightedScore: 0, + rawResponse: result.rawResponse, + durationMs, + tokens: result.tokens, + error: result.error, + }; + } + + const validated = validateAndScore(result.parsed, opts.rubric); + if (validated.error) { + return { + model, + scores: validated.scores, + weightedScore: 0, + rawResponse: result.rawResponse, + durationMs, + tokens: result.tokens, + error: validated.error, + }; + } + + return { + model, + scores: validated.scores, + weightedScore: validated.weightedScore, + rawResponse: result.rawResponse, + durationMs, + tokens: result.tokens, + }; +} + +type JudgeApiOutcome = { + parsed: unknown; + rawResponse: string; + tokens: JudgeResult['tokens']; + error?: string; +}; + +async function callJudgeWithRetry(args: { + client: Anthropic; + model: string; + systemPrompt: string; + userPrompt: string; +}): Promise { + const first = await callJudgeOnce(args); + if (!first.error) return first; + + // Retry once with a stricter reminder appended. + const retry = await callJudgeOnce({ + ...args, + userPrompt: + args.userPrompt + + '\n\nReturn STRICT JSON only — no Markdown fences, no prose.', + }); + // Always accumulate tokens from both attempts so cost tracking is accurate. + retry.tokens = sumTokens(first.tokens, retry.tokens); + return retry; +} + +async function callJudgeOnce(args: { + client: Anthropic; + model: string; + systemPrompt: string; + userPrompt: string; +}): Promise { + let response: Anthropic.Message; + try { + response = await args.client.messages.create({ + model: args.model, + max_tokens: MAX_TOKENS, + system: [ + { + type: 'text', + text: args.systemPrompt, + cache_control: { type: 'ephemeral' }, + }, + ], + messages: [{ role: 'user', content: args.userPrompt }], + }); + } catch (err) { + return { + parsed: null, + rawResponse: '', + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + error: + err instanceof Anthropic.APIError + ? `${err.constructor.name} (${err.status}): ${err.message}` + : err instanceof Error + ? err.message + : String(err), + }; + } + + const text = response.content + .filter((b): b is Anthropic.TextBlock => b.type === 'text') + .map(b => b.text) + .join('\n'); + + const tokens = { + input: response.usage.input_tokens, + output: response.usage.output_tokens, + cacheCreation: response.usage.cache_creation_input_tokens ?? 0, + cacheRead: response.usage.cache_read_input_tokens ?? 0, + }; + + const parsed = tryParseJson(text); + if (parsed === null) { + return { + parsed: null, + rawResponse: text, + tokens, + error: 'judge response was not valid JSON', + }; + } + return { parsed, rawResponse: text, tokens }; +} + +function tryParseJson(text: string): unknown { + const trimmed = text.trim(); + // Direct parse first. + try { + return JSON.parse(trimmed); + } catch { + // fall through + } + // Extract the first {...} block in case the model wrapped JSON in prose. + const start = trimmed.indexOf('{'); + const end = trimmed.lastIndexOf('}'); + if (start === -1 || end <= start) return null; + try { + return JSON.parse(trimmed.slice(start, end + 1)); + } catch { + return null; + } +} + +function validateAndScore( + parsed: unknown, + rubric: Rubric, +): { + scores: Record; + weightedScore: number; + error?: string; +} { + if (!parsed || typeof parsed !== 'object') { + return { scores: {}, weightedScore: 0, error: 'judge JSON not an object' }; + } + const obj = parsed as Record; + const rawScores = obj.scores; + if (!rawScores || typeof rawScores !== 'object') { + return { + scores: {}, + weightedScore: 0, + error: 'judge response missing `scores` object', + }; + } + const scoresMap = rawScores as Record; + + const scores: Record = {}; + let weightedSum = 0; + let totalWeight = 0; + const missing: string[] = []; + + for (const criterion of rubric.judge.criteria) { + const entry = scoresMap[criterion.id]; + if (!entry || typeof entry !== 'object') { + missing.push(criterion.id); + continue; + } + const e = entry as Record; + const rawScore = typeof e.score === 'number' ? e.score : Number(e.score); + if (!Number.isFinite(rawScore)) { + missing.push(criterion.id); + continue; + } + const clamped = Math.max(0, Math.min(5, Math.round(rawScore))); + scores[criterion.id] = { + score: clamped, + rationale: typeof e.rationale === 'string' ? e.rationale : '', + }; + weightedSum += clamped * criterion.weight; + totalWeight += criterion.weight; + } + + if (missing.length > 0) { + return { + scores, + weightedScore: 0, + error: `judge omitted criteria: ${missing.join(', ')}`, + }; + } + + const weightedScore = totalWeight === 0 ? 0 : weightedSum / (5 * totalWeight); + return { scores, weightedScore }; +} + +function sumTokens( + a: JudgeResult['tokens'], + b: JudgeResult['tokens'], +): JudgeResult['tokens'] { + return { + input: a.input + b.input, + output: a.output + b.output, + cacheCreation: a.cacheCreation + b.cacheCreation, + cacheRead: a.cacheRead + b.cacheRead, + }; +} diff --git a/packages/hdx-eval/src/grading/judgePrompt.ts b/packages/hdx-eval/src/grading/judgePrompt.ts new file mode 100644 index 0000000000..082ca8639c --- /dev/null +++ b/packages/hdx-eval/src/grading/judgePrompt.ts @@ -0,0 +1,77 @@ +import type { Rubric } from './types'; + +const SYSTEM_PREAMBLE = `You are evaluating an SRE investigation. You will receive: +- the scenario question (what the candidate was asked) +- the ground-truth facts (the planted answer the candidate did NOT see) +- a rubric with weighted criteria +- the candidate's final answer (anonymized — tool names and product brands have been redacted so you cannot tell which tool the candidate used) + +For each rubric criterion, output an integer score from 0 to 5 plus a one-sentence rationale. Do not consider tool choice, query syntax, or implementation details — score only the quality of the candidate's final answer relative to the ground truth. + +Return STRICT JSON of shape: +{ "scores": { "": { "score": N, "rationale": "..." } } } +No prose outside the JSON. Include every criterion id from the rubric.`; + +export function buildJudgeSystem(scenarioName: string, rubric: Rubric): string { + const criteriaSection = rubric.judge.criteria + .map(c => `- "${c.id}" (weight ${c.weight}): ${c.description}`) + .join('\n'); + return [ + SYSTEM_PREAMBLE, + '', + `SCENARIO: ${scenarioName}`, + '', + 'RUBRIC CRITERIA:', + criteriaSection, + ].join('\n'); +} + +export function buildJudgeUser(args: { + scenarioPrompt: string; + groundTruthFacts: string; + candidateAnswer: string; +}): string { + return [ + 'SCENARIO QUESTION:', + args.scenarioPrompt, + '', + 'GROUND-TRUTH FACTS (internal only — the candidate did not see these):', + args.groundTruthFacts, + '', + 'CANDIDATE FINAL ANSWER (anonymized):', + args.candidateAnswer, + ].join('\n'); +} + +/** + * Renders the `expected` and `anomalyAttributes` portions of a scenario + * ground-truth.json as a human-readable bulleted summary for the judge. + * Strips internal-only fields (rubric, agentPrompt, scenario name). + */ +export function formatGroundTruthFacts(groundTruth: unknown): string { + if (!groundTruth || typeof groundTruth !== 'object') return '(none)'; + const gt = groundTruth as Record; + const lines: string[] = []; + const expected = gt.expected; + if (expected && typeof expected === 'object') { + lines.push('Expected:'); + for (const [k, v] of Object.entries(expected)) { + lines.push(` - ${k}: ${formatValue(v)}`); + } + } + const anomaly = gt.anomalyAttributes; + if (anomaly && typeof anomaly === 'object') { + lines.push('Anomaly attributes:'); + for (const [k, v] of Object.entries(anomaly)) { + lines.push(` - ${k}: ${formatValue(v)}`); + } + } + return lines.join('\n'); +} + +function formatValue(v: unknown): string { + if (v === null || v === undefined) return 'null'; + if (typeof v === 'string') return v; + if (typeof v === 'number' || typeof v === 'boolean') return String(v); + return JSON.stringify(v); +} diff --git a/packages/hdx-eval/src/grading/programmatic.ts b/packages/hdx-eval/src/grading/programmatic.ts new file mode 100644 index 0000000000..769081d1ae --- /dev/null +++ b/packages/hdx-eval/src/grading/programmatic.ts @@ -0,0 +1,42 @@ +import type { + ProgrammaticCheck, + ProgrammaticHit, + ProgrammaticResult, +} from './types'; + +export function runProgrammaticChecks( + answer: string, + checks: ProgrammaticCheck[], +): ProgrammaticResult { + const hits: ProgrammaticHit[] = []; + let totalWeight = 0; + let hitWeight = 0; + + for (const check of checks) { + totalWeight += check.weight; + const flags = check.flags ?? 'i'; + let regex: RegExp; + try { + regex = new RegExp(check.pattern, flags); + } catch (err) { + throw new Error( + `Invalid regex in programmatic check '${check.id}': ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + const matched = regex.test(answer); + const satisfied = check.negative ? !matched : matched; + if (satisfied) hitWeight += check.weight; + hits.push({ + id: check.id, + weight: check.weight, + matched, + satisfied, + negative: check.negative, + }); + } + + const score = totalWeight === 0 ? 0 : hitWeight / totalWeight; + return { hits, score }; +} diff --git a/packages/hdx-eval/src/grading/rubric.ts b/packages/hdx-eval/src/grading/rubric.ts new file mode 100644 index 0000000000..3d19dfe2e8 --- /dev/null +++ b/packages/hdx-eval/src/grading/rubric.ts @@ -0,0 +1,96 @@ +import { getScenario } from '../scenarios'; +import type { ProgrammaticCheck, Rubric } from './types'; + +export function loadScenarioRubric(scenarioName: string): Rubric { + const scenario = getScenario(scenarioName); + const gt = scenario.groundTruth as { rubric?: unknown }; + if (!gt || typeof gt !== 'object' || !gt.rubric) { + throw new Error( + `Scenario '${scenarioName}' has no \`rubric\` block in ground-truth.json`, + ); + } + return validateRubric(gt.rubric, scenarioName); +} + +/** + * Hydrate a compact tuple `[id, weight, pattern, negative?]` into a + * ProgrammaticCheck object. The `flags` field always defaults to `"i"`. + */ +function hydrateCheck(entry: unknown, scenarioName: string): ProgrammaticCheck { + if (Array.isArray(entry)) { + const [id, weight, pattern, negative] = entry; + if ( + typeof id !== 'string' || + typeof weight !== 'number' || + typeof pattern !== 'string' + ) { + throw new Error( + `rubric.programmatic for '${scenarioName}': tuple must be [string, number, string, boolean?]`, + ); + } + return { + id, + weight, + pattern, + flags: 'i', + ...(negative === true ? { negative: true } : {}), + }; + } + if (!entry || typeof entry !== 'object') { + throw new Error( + `rubric.programmatic for '${scenarioName}' has a non-object/non-array entry`, + ); + } + const check = entry as Record; + if (typeof check.id !== 'string' || typeof check.pattern !== 'string') { + throw new Error( + `rubric.programmatic for '${scenarioName}': each check needs string 'id' and 'pattern'`, + ); + } + if (typeof check.weight !== 'number' || check.weight <= 0) { + throw new Error( + `rubric.programmatic for '${scenarioName}': check '${check.id}' weight must be a positive number`, + ); + } + return entry as ProgrammaticCheck; +} + +function validateRubric(raw: unknown, scenarioName: string): Rubric { + if (!raw || typeof raw !== 'object') { + throw new Error(`rubric for '${scenarioName}' must be an object`); + } + const obj = raw as Record; + if (!Array.isArray(obj.programmatic)) { + throw new Error( + `rubric.programmatic for '${scenarioName}' must be an array`, + ); + } + const programmatic = (obj.programmatic as unknown[]).map(c => + hydrateCheck(c, scenarioName), + ); + const judge = obj.judge as { criteria?: unknown } | undefined; + if (!judge || !Array.isArray(judge.criteria) || judge.criteria.length === 0) { + throw new Error( + `rubric.judge.criteria for '${scenarioName}' must be a non-empty array`, + ); + } + for (const c of judge.criteria as unknown[]) { + if (!c || typeof c !== 'object') { + throw new Error( + `rubric.judge.criteria for '${scenarioName}' has a non-object entry`, + ); + } + const crit = c as Record; + if (typeof crit.id !== 'string' || typeof crit.description !== 'string') { + throw new Error( + `rubric.judge.criteria for '${scenarioName}': each criterion needs string 'id' and 'description'`, + ); + } + if (typeof crit.weight !== 'number' || crit.weight <= 0) { + throw new Error( + `rubric.judge.criteria for '${scenarioName}': criterion '${crit.id}' weight must be a positive number`, + ); + } + } + return { programmatic, judge: judge as Rubric['judge'] }; +} diff --git a/packages/hdx-eval/src/grading/types.ts b/packages/hdx-eval/src/grading/types.ts new file mode 100644 index 0000000000..80e15cec58 --- /dev/null +++ b/packages/hdx-eval/src/grading/types.ts @@ -0,0 +1,100 @@ +import type { McpKind } from '../harness/types'; + +export type ProgrammaticCheck = { + id: string; + weight: number; + pattern: string; + flags?: string; + // When true, the check is satisfied when the pattern does NOT match the + // answer. Used to penalize wrong attributions / blamed distractors. + negative?: boolean; +}; + +export type JudgeCriterion = { + id: string; + weight: number; + description: string; +}; + +export type Rubric = { + programmatic: ProgrammaticCheck[]; + judge: { criteria: JudgeCriterion[] }; +}; + +export type ProgrammaticHit = { + id: string; + weight: number; + matched: boolean; + satisfied: boolean; + negative?: boolean; +}; + +export type ProgrammaticResult = { + hits: ProgrammaticHit[]; + score: number; // 0..1 +}; + +export type JudgeCriterionScore = { + score: number; // 0..5 + rationale: string; +}; + +export type JudgeResult = { + model: string; + scores: Record; + weightedScore: number; // 0..1, sum(score*weight) / (5*sum(weight)) + rawResponse: string; + durationMs: number; + tokens: { + input: number; + output: number; + cacheCreation: number; + cacheRead: number; + }; + error?: string; +}; + +/** + * Per-run tool-error stats — computed from the saved RunRecord, not the + * agent's answer. Bad runs (lots of failed tool calls) get a combined-score + * penalty even if the final answer happens to be correct. + */ +export type ToolErrorStats = { + /** Total tool calls observed. */ + total: number; + /** Tool calls flagged isError by the harness/MCP. */ + errors: number; + /** errors / total (0 when total = 0). */ + rate: number; + /** Penalty applied to combinedScore: clamp(rate, 0, MAX_ERROR_PENALTY). */ + penalty: number; + /** First few error tool names + a short snippet of the error text. */ + samples: Array<{ name: string; preview: string }>; +}; + +export type GradeRecord = { + schemaVersion: 2; + runId: string; + scenario: string; + mcp: McpKind; + programmatic: ProgrammaticResult; + judge: JudgeResult | null; + toolErrors: ToolErrorStats; + /** + * combinedScore = clamp01( + * PROGRAMMATIC_WEIGHT * programmatic + JUDGE_WEIGHT * judge + * - toolErrors.penalty + * ) + * The penalty is at most MAX_ERROR_PENALTY (0.2) — a run can't be reduced + * to zero from a high error rate alone, but spamming bad tool calls makes + * an otherwise-good answer measurably worse. + */ + combinedScore: number; + gradedAt: string; + judgeModel: string; +}; + +export const COMBINED_SCORE_PROGRAMMATIC_WEIGHT = 0.4; +export const COMBINED_SCORE_JUDGE_WEIGHT = 0.6; +/** Maximum penalty applied for tool errors (subtracted from combined score). */ +export const MAX_ERROR_PENALTY = 0.2; diff --git a/packages/hdx-eval/src/harness/claudeSpawn.ts b/packages/hdx-eval/src/harness/claudeSpawn.ts new file mode 100644 index 0000000000..94d1166d7e --- /dev/null +++ b/packages/hdx-eval/src/harness/claudeSpawn.ts @@ -0,0 +1,185 @@ +import { spawn } from 'child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import type { EvalConfig } from '../hyperdx/config'; +import { allowedToolsPattern, buildMcpConfig } from './mcpConfig'; +import { buildSettings, deniedToolsFor } from './settingsFile'; +import { + chunkToLines, + type ParsedEvent, + parseStreamLine, +} from './streamParser'; +import type { McpDefinition, McpKind, PromptVariant } from './types'; + +export type SpawnOptions = { + config: EvalConfig; + scenario: string; + mcp: McpKind; + /** The resolved MCP definition from the config. */ + mcpDef: McpDefinition; + model: string; + maxTurns: number; + timeoutMs: number; + agentPrompt: string; + systemPromptAppend: string; + apiKey: string; + /** Prompt variant — affects which built-in tools are denied. The + * `hypothesis` variant allows the Task tool so the agent can spawn + * subagents. Default 'baseline'. */ + promptVariant?: PromptVariant; +}; + +export type SpawnResult = { + events: ParsedEvent[]; + stderr: string; + exitCode: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; + argv: string[]; + tempdir: string; +}; + +export async function runClaude(opts: SpawnOptions): Promise { + const tempdir = mkdtempSync(join(tmpdir(), 'hdx-eval-')); + const mcpConfigPath = join(tempdir, 'mcp-config.json'); + const settingsPath = join(tempdir, 'settings.json'); + + const mcpDef = opts.mcpDef; + writeFileSync( + mcpConfigPath, + JSON.stringify(buildMcpConfig(mcpDef, opts.mcp), null, 2), + ); + const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; + writeFileSync( + settingsPath, + JSON.stringify(buildSettings(mcpDef, promptVariant, tempdir), null, 2), + ); + + const argv = [ + '-p', + '--mcp-config', + mcpConfigPath, + '--allowedTools', + `${allowedToolsPattern(mcpDef)},Read(${tempdir}/*)`, + '--disallowedTools', + deniedToolsFor(promptVariant, mcpDef).join(','), + '--dangerously-skip-permissions', + '--setting-sources', + 'local', + '--settings', + settingsPath, + '--append-system-prompt', + opts.systemPromptAppend, + '--output-format', + 'stream-json', + '--verbose', + '--model', + opts.model, + '--max-turns', + String(opts.maxTurns), + opts.agentPrompt, + ]; + + // Run claude with cwd=tempdir (a fresh isolated dir). The agent can't + // Bash/Read/Glob into the repo to peek at ground-truth, prior eval runs, + // or the eval config — the only state visible is the mcp-config + settings + // files we just wrote here. + // + // detached: true gives claude its own process group so we can kill the + // entire tree (claude + MCP servers it spawns) on SIGKILL escalation. + // SIGTERM is still sent to claude only — it handles graceful shutdown of + // its children. The SIGKILL escalation uses process.kill(-pid) to reap + // any orphaned MCP server processes that survived SIGTERM. + const proc = spawn('claude', argv, { + env: { ...process.env, ANTHROPIC_API_KEY: opts.apiKey }, + cwd: tempdir, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + + const events: ParsedEvent[] = []; + let stdoutBuffer = ''; + let stderr = ''; + let timedOut = false; + + const stdoutDone = new Promise(resolve => { + proc.stdout.setEncoding('utf8'); + proc.stdout.on('data', (chunk: string) => { + const { events: lines, remainder } = chunkToLines(stdoutBuffer, chunk); + stdoutBuffer = remainder; + for (const line of lines) { + const parsed = parseStreamLine(line); + if (parsed) events.push(parsed); + } + }); + proc.stdout.on('end', () => { + if (stdoutBuffer.trim()) { + const parsed = parseStreamLine(stdoutBuffer); + if (parsed) events.push(parsed); + } + resolve(); + }); + }); + + proc.stderr.setEncoding('utf8'); + proc.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + + let escalationTimer: ReturnType | undefined; + const timeout = setTimeout(() => { + timedOut = true; + try { + proc.kill('SIGTERM'); + } catch { + // Process may not have started (ENOENT) — ignore. + } + escalationTimer = setTimeout(() => { + try { + // Kill the entire process group (-pid) to reap any orphaned MCP + // server children that didn't exit after SIGTERM. + if (proc.pid) process.kill(-proc.pid, 'SIGKILL'); + } catch { + // Process group already exited — ignore. + } + }, 5_000); + escalationTimer.unref(); + }, opts.timeoutMs); + + try { + const exitInfo = await new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve, reject) => { + proc.on('error', err => { + clearTimeout(timeout); + if (escalationTimer) clearTimeout(escalationTimer); + reject(err); + }); + proc.on('exit', (code, signal) => resolve({ code, signal })); + }); + clearTimeout(timeout); + if (escalationTimer) clearTimeout(escalationTimer); + await stdoutDone; + + return { + events, + stderr, + exitCode: exitInfo.code, + signal: exitInfo.signal, + timedOut, + argv, + tempdir, + }; + } finally { + // Clean up the temp directory to avoid leaking MCP configs with API keys. + // This runs even if spawn() rejects (e.g. ENOENT when claude is not on PATH). + try { + rmSync(tempdir, { recursive: true, force: true }); + } catch { + // Best effort — don't fail the run if cleanup fails. + } + } +} diff --git a/packages/hdx-eval/src/harness/mcpConfig.ts b/packages/hdx-eval/src/harness/mcpConfig.ts new file mode 100644 index 0000000000..f047ab5f03 --- /dev/null +++ b/packages/hdx-eval/src/harness/mcpConfig.ts @@ -0,0 +1,45 @@ +import type { McpDefinition, McpKind } from './types'; + +// Simulate a production observability backend: queries that scan the whole +// table or don't use filters/indexes time out. Forces agents to write +// efficient queries instead of `SELECT * FROM huge_table`. +export const QUERY_TIMEOUT_SECONDS = 10; + +/** + * Build the Claude Code MCP config JSON for a single MCP definition. + * The server name inside `mcpServers` is set to the MCP's config key + * so that tool names match the expected `mcp____*` pattern. + */ +export function buildMcpConfig( + def: McpDefinition, + kind: McpKind, +): Record { + if (def.type === 'http') { + return { + mcpServers: { + [kind]: { + type: 'http', + url: def.url, + ...(def.headers ? { headers: def.headers } : {}), + }, + }, + }; + } + // stdio transport + return { + mcpServers: { + [kind]: { + command: def.command, + ...(def.args ? { args: def.args } : {}), + ...(def.env ? { env: def.env } : {}), + }, + }, + }; +} + +/** + * Return the tool glob pattern for a given MCP definition. + */ +export function allowedToolsPattern(def: McpDefinition): string { + return def.toolPattern; +} diff --git a/packages/hdx-eval/src/harness/runRun.ts b/packages/hdx-eval/src/harness/runRun.ts new file mode 100644 index 0000000000..06fd4a455c --- /dev/null +++ b/packages/hdx-eval/src/harness/runRun.ts @@ -0,0 +1,235 @@ +import { type EvalConfig, getMcpDefinition } from '../hyperdx/config'; +import { runClaude } from './claudeSpawn'; +import type { ParsedEvent } from './streamParser'; +import { buildSystemPrompt } from './systemPrompt'; +import type { + McpKind, + PromptVariant, + RunRecord, + Termination, + ToolCallRecord, +} from './types'; + +export type RunCellOptions = { + config: EvalConfig; + scenario: string; + agentPrompt: string; + mcp: McpKind; + model: string; + maxTurns: number; + timeoutMs: number; + runIndex: number; + seed: number; + apiKey: string; + /** When set, the agent's system prompt declares this ISO timestamp as the + * fixed "now" for any relative-time reference in the user prompt. */ + anchorTimeIso?: string; + /** Prompt variant — 'baseline' (default) or 'hypothesis' (forces + * hypothesis-enumeration + parallel Task subagents). */ + promptVariant?: PromptVariant; +}; + +export async function runCell(opts: RunCellOptions): Promise { + const startedAtIso = new Date().toISOString(); + const startedAtMs = Date.now(); + const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; + const mcpDef = getMcpDefinition(opts.config, opts.mcp); + const systemPromptAppend = buildSystemPrompt( + opts.scenario, + opts.anchorTimeIso, + promptVariant, + opts.maxTurns, + ); + + const result = await runClaude({ + config: opts.config, + scenario: opts.scenario, + mcp: opts.mcp, + mcpDef, + model: opts.model, + maxTurns: opts.maxTurns, + timeoutMs: opts.timeoutMs, + agentPrompt: opts.agentPrompt, + systemPromptAppend, + apiKey: opts.apiKey, + promptVariant, + }); + + const endedAtIso = new Date().toISOString(); + const endedAtMs = Date.now(); + + return assembleRecord({ + events: result.events, + stderr: result.stderr, + exitCode: result.exitCode, + timedOut: result.timedOut, + scenario: opts.scenario, + agentPrompt: opts.agentPrompt, + systemPromptAppend, + mcp: opts.mcp, + model: opts.model, + runIndex: opts.runIndex, + seed: opts.seed, + startedAtIso, + endedAtIso, + durationMs: endedAtMs - startedAtMs, + }); +} + +type AssembleInput = { + events: ParsedEvent[]; + stderr: string; + exitCode: number | null; + timedOut: boolean; + scenario: string; + agentPrompt: string; + systemPromptAppend: string; + mcp: McpKind; + model: string; + runIndex: number; + seed: number; + startedAtIso: string; + endedAtIso: string; + durationMs: number; +}; + +export function assembleRecord(input: AssembleInput): RunRecord { + const toolCalls: ToolCallRecord[] = []; + const callsById = new Map(); + const tools: { name: string; description?: string }[] = []; + let lastAssistantText = ''; + let resultUsage: + | { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; + } + | undefined; + let resultText: string | undefined; + let resultSubtype: string | undefined; + let resultIsError = false; + let totalCostUsd: number | null = null; + + for (const ev of input.events) { + if (ev.kind === 'system_init') { + if (ev.tools) tools.push(...ev.tools); + } else if (ev.kind === 'assistant_message') { + const ts = new Date().toISOString(); + for (const block of ev.content) { + if (!block || typeof block !== 'object') continue; + const b = block as Record; + if (b.type === 'text' && typeof b.text === 'string') { + lastAssistantText = b.text; + } else if (b.type === 'tool_use') { + const id = typeof b.id === 'string' ? b.id : null; + if (!id) continue; + const call: ToolCallRecord = { + name: typeof b.name === 'string' ? b.name : 'unknown', + input: b.input ?? null, + output: null, + isError: false, + startedAt: ts, + endedAt: null, + durationMs: null, + }; + callsById.set(id, call); + toolCalls.push(call); + } + } + } else if (ev.kind === 'user_message') { + const ts = new Date().toISOString(); + for (const block of ev.content) { + if (!block || typeof block !== 'object') continue; + const b = block as Record; + if (b.type !== 'tool_result') continue; + const id = typeof b.tool_use_id === 'string' ? b.tool_use_id : null; + if (!id) continue; + const call = callsById.get(id); + if (!call) continue; + call.output = stringifyToolResult(b.content); + call.isError = b.is_error === true; + call.endedAt = ts; + const startMs = Date.parse(call.startedAt); + const endMs = Date.parse(ts); + if (Number.isFinite(startMs) && Number.isFinite(endMs)) { + call.durationMs = Math.max(0, endMs - startMs); + } + } + } else if (ev.kind === 'result') { + resultUsage = ev.usage; + resultText = ev.resultText; + resultSubtype = ev.subtype; + resultIsError = ev.isError; + if (typeof ev.totalCostUsd === 'number') totalCostUsd = ev.totalCostUsd; + } + } + + const finalAnswer = resultText ?? lastAssistantText; + const termination = determineTermination({ + timedOut: input.timedOut, + exitCode: input.exitCode, + resultSubtype, + resultIsError, + hasFinalAnswer: !!finalAnswer, + }); + + return { + schemaVersion: 1, + runId: `${input.startedAtIso}-${input.scenario}-${input.mcp}-${input.runIndex}`, + scenario: input.scenario, + mcp: input.mcp, + model: input.model, + runIndex: input.runIndex, + seed: input.seed, + startedAt: input.startedAtIso, + endedAt: input.endedAtIso, + durationMs: input.durationMs, + agentPrompt: input.agentPrompt, + systemPromptAppend: input.systemPromptAppend, + termination, + exitCode: input.exitCode, + tools, + toolCalls, + messages: input.events.map(e => e.raw), + finalAnswer, + tokens: { + input: resultUsage?.input_tokens ?? 0, + output: resultUsage?.output_tokens ?? 0, + cacheCreation: resultUsage?.cache_creation_input_tokens ?? 0, + cacheRead: resultUsage?.cache_read_input_tokens ?? 0, + }, + totalCostUsd, + stderr: input.stderr, + }; +} + +function stringifyToolResult(content: unknown): string { + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .map(c => { + if (!c || typeof c !== 'object') return JSON.stringify(c); + const o = c as Record; + if (typeof o.text === 'string') return o.text; + return JSON.stringify(o); + }) + .join('\n'); + } + return JSON.stringify(content); +} + +function determineTermination(args: { + timedOut: boolean; + exitCode: number | null; + resultSubtype?: string; + resultIsError: boolean; + hasFinalAnswer: boolean; +}): Termination { + if (args.timedOut) return 'timeout'; + if (args.resultSubtype === 'error_max_turns') return 'max_turns'; + if (args.resultIsError) return 'error'; + if (args.exitCode !== null && args.exitCode !== 0) return 'error'; + if (args.hasFinalAnswer) return 'final_answer'; + return 'error'; +} diff --git a/packages/hdx-eval/src/harness/settingsFile.ts b/packages/hdx-eval/src/harness/settingsFile.ts new file mode 100644 index 0000000000..5e3a510f58 --- /dev/null +++ b/packages/hdx-eval/src/harness/settingsFile.ts @@ -0,0 +1,79 @@ +import { allowedToolsPattern } from './mcpConfig'; +import type { McpDefinition, PromptVariant } from './types'; + +/** + * Filesystem / shell / built-in tools we never want the eval agent to use. + * The agent's working directory is a fresh tempdir (no answer key) — but + * Bash/Glob/Grep would still let it shell out to /home/ and read prior + * runs or the source tree. Lock it to the MCP tools only. + * + * Read is NOT denied here — it is allowed with a path scope restricted to + * the agent's tempdir so the agent can recover oversized tool responses + * that Claude Code saves to disk. The scoped allow rule is added in + * buildSettings() using the concrete tempdir path. + * + * `Task` (Claude Code's subagent-spawning tool) is denied by default but + * ALLOWED in the `hypothesis` prompt variant — that variant explicitly + * asks the agent to spawn parallel subagents to investigate multiple + * hypotheses concurrently. + */ +const DENIED_BUILT_IN_TOOLS_BASE = [ + 'Bash', + 'Write', + 'Edit', + 'MultiEdit', + 'NotebookEdit', + 'Glob', + 'Grep', + 'WebFetch', + 'WebSearch', + 'TodoWrite', +] as const; + +/** Retained for backward-compatibility with the existing claudeSpawn import. */ +export const DENIED_BUILT_IN_TOOLS = [ + ...DENIED_BUILT_IN_TOOLS_BASE, + 'Task', +] as const; + +/** + * Build the list of denied tools for a given MCP and prompt variant. + * Per-MCP denied tools come from `McpDefinition.deniedTools`. + */ +export function deniedToolsFor( + variant: PromptVariant, + def?: McpDefinition, +): readonly string[] { + const builtIn = + variant === 'hypothesis' + ? DENIED_BUILT_IN_TOOLS_BASE + : DENIED_BUILT_IN_TOOLS; + + const mcpDenied = def?.deniedTools ?? []; + if (mcpDenied.length > 0) { + return [...builtIn, ...mcpDenied]; + } + return builtIn; +} + +export function buildSettings( + def: McpDefinition, + variant: PromptVariant = 'baseline', + tempdir?: string, +): Record { + const allow: string[] = [allowedToolsPattern(def)]; + + // Allow Read scoped to the agent's tempdir so it can recover oversized + // tool responses that Claude Code saves to disk. The tempdir is an + // empty mkdtemp directory — no ground truth, source, or prior runs. + if (tempdir) { + allow.push(`Read(${tempdir}/*)`); + } + + return { + permissions: { + allow, + deny: [...deniedToolsFor(variant, def)], + }, + }; +} diff --git a/packages/hdx-eval/src/harness/streamParser.ts b/packages/hdx-eval/src/harness/streamParser.ts new file mode 100644 index 0000000000..bed4669d9c --- /dev/null +++ b/packages/hdx-eval/src/harness/streamParser.ts @@ -0,0 +1,156 @@ +/** + * Parser for `claude -p --output-format stream-json` event lines. + * Tolerant of unknown event shapes — passes through as Unknown. + */ + +export type Usage = { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; +}; + +export type SystemInitEvent = { + kind: 'system_init'; + sessionId?: string; + model?: string; + tools?: { name: string; description?: string }[]; + raw: unknown; +}; + +export type AssistantMessageEvent = { + kind: 'assistant_message'; + content: unknown[]; + usage?: Usage; + raw: unknown; +}; + +export type UserMessageEvent = { + kind: 'user_message'; + content: unknown[]; + raw: unknown; +}; + +export type ResultEvent = { + kind: 'result'; + subtype?: string; + isError: boolean; + durationMs?: number; + totalCostUsd?: number; + usage?: Usage; + resultText?: string; + raw: unknown; +}; + +export type UnknownEvent = { + kind: 'unknown'; + type?: string; + raw: unknown; +}; + +export type ParsedEvent = + | SystemInitEvent + | AssistantMessageEvent + | UserMessageEvent + | ResultEvent + | UnknownEvent; + +export function parseStreamLine(line: string): ParsedEvent | null { + const trimmed = line.trim(); + if (!trimmed) return null; + let raw: unknown; + try { + raw = JSON.parse(trimmed); + } catch { + return null; + } + if (!raw || typeof raw !== 'object') return null; + const obj = raw as Record; + const type = typeof obj.type === 'string' ? obj.type : undefined; + + if (type === 'system' && obj.subtype === 'init') { + return { + kind: 'system_init', + sessionId: + typeof obj.session_id === 'string' ? obj.session_id : undefined, + model: typeof obj.model === 'string' ? obj.model : undefined, + tools: Array.isArray(obj.tools) + ? (obj.tools as unknown[]) + .map(toToolDescriptor) + .filter((t): t is { name: string } => t !== null) + : undefined, + raw, + }; + } + + if (type === 'assistant') { + const message = (obj.message ?? {}) as Record; + return { + kind: 'assistant_message', + content: Array.isArray(message.content) + ? (message.content as unknown[]) + : [], + usage: message.usage as Usage | undefined, + raw, + }; + } + + if (type === 'user') { + const message = (obj.message ?? {}) as Record; + return { + kind: 'user_message', + content: Array.isArray(message.content) + ? (message.content as unknown[]) + : [], + raw, + }; + } + + if (type === 'result') { + return { + kind: 'result', + subtype: typeof obj.subtype === 'string' ? obj.subtype : undefined, + isError: obj.is_error === true, + durationMs: + typeof obj.duration_ms === 'number' ? obj.duration_ms : undefined, + totalCostUsd: + typeof obj.total_cost_usd === 'number' ? obj.total_cost_usd : undefined, + usage: obj.usage as Usage | undefined, + resultText: typeof obj.result === 'string' ? obj.result : undefined, + raw, + }; + } + + return { kind: 'unknown', type, raw }; +} + +function toToolDescriptor( + v: unknown, +): { name: string; description?: string } | null { + if (typeof v === 'string') return { name: v }; + if (v && typeof v === 'object') { + const obj = v as Record; + if (typeof obj.name === 'string') { + return { + name: obj.name, + description: + typeof obj.description === 'string' ? obj.description : undefined, + }; + } + } + return null; +} + +/** + * Splits a chunk of streamed bytes into complete JSON-line events plus a + * remainder. Preserves the trailing partial line for the next chunk. + */ +export function chunkToLines( + buffer: string, + chunk: string, +): { events: string[]; remainder: string } { + const combined = buffer + chunk; + const parts = combined.split('\n'); + const remainder = parts.pop() ?? ''; + return { events: parts.filter(p => p.length > 0), remainder }; +} diff --git a/packages/hdx-eval/src/harness/systemPrompt.ts b/packages/hdx-eval/src/harness/systemPrompt.ts new file mode 100644 index 0000000000..48b15666dd --- /dev/null +++ b/packages/hdx-eval/src/harness/systemPrompt.ts @@ -0,0 +1,129 @@ +import { scenarioTables } from '../clickhouse/schema'; +import type { PromptVariant } from './types'; + +export function buildSystemPrompt( + scenario: string, + anchorTimeIso?: string, + variant: PromptVariant = 'baseline', + maxTurns?: number, +): string { + const { traces, logs } = scenarioTables(scenario); + const sharedSchema = ''; + // `These follow the standard OpenTelemetry ClickHouse schema: + // - traces have Timestamp DateTime64(9), TraceId, SpanId, ParentSpanId, + // ServiceName, SpanName, SpanKind, Duration (nanoseconds), StatusCode, + // StatusMessage, plus Map(String,String) columns ResourceAttributes and + // SpanAttributes. + // - logs have Timestamp DateTime64(9), TraceId, SpanId, ServiceName, + // SeverityText, SeverityNumber, Body, plus Map(String,String) columns + // ResourceAttributes and LogAttributes.`; + + // When the harness anchors a run to a fixed past time, the agent must use + // that anchor as "now" for any relative window in the user's prompt. Without + // this, the model uses today's date and queries an empty future window. + const anchorBlock = anchorTimeIso + ? `\nFIXED CURRENT TIME: ${anchorTimeIso} +All "now", "recently", "in the last N minutes/hours" references in the user's +prompt are anchored to this timestamp. When you query, use absolute ISO +timestamps relative to this anchor — do NOT use today's wall-clock date.\n` + : ''; + + // Hypothesis-variant playbook — encourages hypothesis enumeration and + // (optionally) parallel subagent investigation. Goal: break the + // "commit to the first thing you see" anchoring failure we observe on + // incident scenarios, without paying the subagent overhead when a + // straightforward investigation will do. + const playbookBlock = + variant === 'hypothesis' + ? ` + +INVESTIGATION GUIDANCE: + +1. Before investigating, briefly list the candidate hypotheses about what + could be causing what the user is asking about. List as few or as many + as the problem warrants — one if it's obvious, several if it's + ambiguous. Each hypothesis should be a specific testable claim + ("X is caused by Y in component Z"). Consider whether what looks like + one problem might actually be two or more independent issues happening + at once — if that's plausible, include "multiple independent causes" + as one of the hypotheses worth investigating. + +2. Decide whether to investigate sequentially yourself or to fan out via + parallel Task subagents. The built-in Task tool is available — use it + when you have several plausible hypotheses that you'd otherwise have + to investigate one at a time, or when the question is ambiguous enough + that you'd benefit from having multiple framings explored + independently. Skip it when the answer is likely a single short query + chain. Subagents inherit the same MCP tools you have. If you spawn + subagents, send them in the same response so they run concurrently and + give each a focused prompt that says what would confirm or refute the + hypothesis and what unexpected findings to surface. + +3. Synthesize what you (and your subagents) found. Watch specifically + for: (a) two independent things going wrong at once that look like + one — the loudest signal can mask a quieter co-occurring issue, + (b) subagent unanimous agreement that's actually all four latching + onto the same loud signal rather than independent confirmation, + (c) evidence that refutes the framing entirely and points somewhere + you didn't list. + +4. Then write the final answer. + +The goal is to avoid both common failure modes: committing to the first +plausible story (anchoring) AND ignoring the obvious in favor of busywork +(over-investigation). Spend the budget where it actually pays. +` + : ''; + + return `You are an SRE answering an operational question using +observability data. The OpenTelemetry data lives in ClickHouse: + +- Traces: default.${traces} +- Logs: default.${logs} +${anchorBlock} +${sharedSchema} + +${playbookBlock} +TOOL ENVIRONMENT: Only MCP query tools and the Read tool are available. +There are NO shell, write, or search tools (Bash, Write, Edit, Glob, Grep, +etc.). The Read tool is restricted to your working directory — you can only +use it to read files that were saved there by oversized tool responses. +If a tool response is too large and gets saved to a file, use Read to +retrieve its contents. Do NOT ToolSearch for Bash or other file tools. +If the saved file is still too large for Read, re-run the query with +narrower filters or smaller limits. + +TURN BUDGET: You have a limited number of tool calls. Manage them wisely: +- After ~${maxTurns ?? 15} tool calls, if you have identified a primary root cause with + supporting evidence, write your final answer. You can note additional + areas worth investigating without exhaustively querying them. +- Do not keep exploring tangential signals once you have a strong causal + chain for the user's reported symptom. +- High log/error volume alone does not indicate relevance. Before + investigating a service further, verify it is in the causal chain of + the user's reported symptom — not just noisy in the same time window. + +Investigate the user's question. In your final answer, identify: +- the specific service(s) and operation(s) involved +- the root cause (or, for cleanup-style questions, the top concrete actions) +- supporting evidence from the data (counts, time ranges, sample messages) +- a brief "What's not the cause" section that explicitly names any + investigated signals you ruled out and why — this helps the reader + trust your conclusion + +Stay open to the possibility that what looks like one issue is actually +multiple independent issues — when two components look similarly degraded, +confirm they share the same underlying cause before declaring one. + +Before finalizing a root cause, briefly check whether your explanation +fully accounts for the data you've seen. Note any datapoints that don't +fit, and decide whether the gap is benign noise or evidence that the +real story is more complex. + +When the user's question is about understanding why something is slow +or what's gone wrong, drill into the cause rather than stopping at the +symptom. Use tools that aggregate the underlying contributors rather +than reading individual events one by one. + +Be concise. No filler — the reader is technical.`; +} diff --git a/packages/hdx-eval/src/harness/types.ts b/packages/hdx-eval/src/harness/types.ts new file mode 100644 index 0000000000..004ef9c443 --- /dev/null +++ b/packages/hdx-eval/src/harness/types.ts @@ -0,0 +1,94 @@ +/** + * MCP identifier — a free-form string key matching an entry in the + * `mcps` section of `eval.config.json`. No longer restricted to a + * fixed union; any config-defined name is valid. + */ +export type McpKind = string; + +/** + * Transport configuration for an HTTP-based MCP server (e.g. HyperDX MCP). + */ +export type HttpMcpTransport = { + type: 'http'; + url: string; + headers?: Record; +}; + +/** + * Transport configuration for a stdio-based MCP server (e.g. mcp-clickhouse). + */ +export type StdioMcpTransport = { + type: 'stdio'; + command: string; + args?: string[]; + env?: Record; +}; + +/** + * A single MCP definition in the eval config. Fully specifies how to + * connect to the MCP, which tools it exposes, and how to blind its + * identity for fair judging. + */ +export type McpDefinition = (HttpMcpTransport | StdioMcpTransport) & { + /** Glob pattern for allowed tools, e.g. `mcp__hyperdx__*`. */ + toolPattern: string; + /** Human-readable label for reports and CLI output. */ + label: string; + /** Brand terms to redact when blinding answers for the LLM judge. */ + brandTerms?: string[]; + /** MCP tools to deny (e.g. non-investigation tools). */ + deniedTools?: string[]; + /** Whether this MCP is included when `--mcp all` is used. Default: true. + * Explicitly naming a disabled MCP via `--mcp name` still works. */ + enabled?: boolean; +}; + +/** + * Prompt variants for the system-prompt A/B. + * - `baseline`: the default investigative prompt, no subagent affordance. + * - `hypothesis`: hard playbook that asks the agent to enumerate 2–4 + * hypotheses up front, spawn a Task subagent per hypothesis to + * investigate in parallel, then synthesize. + */ +export type PromptVariant = 'baseline' | 'hypothesis'; + +export type Termination = 'final_answer' | 'max_turns' | 'timeout' | 'error'; + +export type ToolCallRecord = { + name: string; + input: unknown; + output: string | null; + isError: boolean; + startedAt: string; + endedAt: string | null; + durationMs: number | null; +}; + +export type RunRecord = { + schemaVersion: 1; + runId: string; + scenario: string; + mcp: McpKind; + model: string; + runIndex: number; + seed: number; + startedAt: string; + endedAt: string; + durationMs: number; + agentPrompt: string; + systemPromptAppend: string; + termination: Termination; + exitCode: number | null; + tools: { name: string; description?: string }[]; + toolCalls: ToolCallRecord[]; + messages: unknown[]; + finalAnswer: string; + tokens: { + input: number; + output: number; + cacheCreation: number; + cacheRead: number; + }; + totalCostUsd: number | null; + stderr: string; +}; diff --git a/packages/hdx-eval/src/hyperdx/api.ts b/packages/hdx-eval/src/hyperdx/api.ts new file mode 100644 index 0000000000..5a75dd3f8f --- /dev/null +++ b/packages/hdx-eval/src/hyperdx/api.ts @@ -0,0 +1,203 @@ +type CookieJar = Map; + +type HyperdxConnection = { + _id: string; + name: string; + host: string; + username: string; + password?: string; +}; + +export type HyperdxSource = { + _id: string; + id: string; + name: string; + kind: 'log' | 'trace' | 'session' | 'metric'; + connection: string; + from: { databaseName: string; tableName: string }; + timestampValueExpression: string; + [key: string]: unknown; +}; + +type MeResponse = { + id: string; + email: string; + accessKey: string; + team: { _id: string; name: string }; +}; + +export class HyperdxApiClient { + private cookies: CookieJar = new Map(); + + constructor(private readonly apiUrl: string) {} + + private url(path: string): string { + return `${this.apiUrl.replace(/\/$/, '')}${path}`; + } + + private cookieHeader(): string { + return Array.from(this.cookies.entries()) + .map(([k, v]) => `${k}=${v}`) + .join('; '); + } + + private absorbSetCookie(res: Response): void { + // Node's fetch exposes Set-Cookie via getSetCookie() (Node 22+). + const headers = res.headers as Headers & { + getSetCookie?: () => string[]; + }; + const list = headers.getSetCookie?.() ?? []; + for (const raw of list) { + const [pair] = raw.split(';'); + const eq = pair.indexOf('='); + if (eq <= 0) continue; + const name = pair.slice(0, eq).trim(); + const value = pair.slice(eq + 1).trim(); + this.cookies.set(name, value); + } + } + + private async request( + method: string, + path: string, + body?: unknown, + ): Promise<{ status: number; body: T }> { + const headers: Record = { + Accept: 'application/json', + }; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + const cookie = this.cookieHeader(); + if (cookie) headers['Cookie'] = cookie; + + const res = await fetch(this.url(path), { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + redirect: 'manual', + signal: AbortSignal.timeout(30_000), + }); + this.absorbSetCookie(res); + + const text = await res.text(); + let parsed: unknown = text; + if (text && (res.headers.get('content-type') ?? '').includes('json')) { + try { + parsed = JSON.parse(text); + } catch { + // keep as text + } + } + + if (res.status >= 400) { + const snippet = + typeof parsed === 'string' + ? parsed.slice(0, 500) + : JSON.stringify(parsed).slice(0, 500); + throw new Error( + `HyperDX API ${method} ${path} → ${res.status}: ${snippet}`, + ); + } + return { status: res.status, body: parsed as T }; + } + + async ping(): Promise { + try { + const res = await fetch(this.url('/'), { + method: 'GET', + signal: AbortSignal.timeout(10_000), + }); + return res.status < 500; + } catch { + return false; + } + } + + async register( + email: string, + password: string, + ): Promise<'created' | 'exists'> { + const res = await fetch(this.url('/register/password'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password, confirmPassword: password }), + redirect: 'manual', + signal: AbortSignal.timeout(30_000), + }); + this.absorbSetCookie(res); + if (res.status === 200) return 'created'; + // Existing accounts return 400 with { error: 'invalid' }; treat as 'exists' + // and let login retry verify the credentials match. + return 'exists'; + } + + async login(email: string, password: string): Promise { + const res = await fetch(this.url('/login/password'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password }), + redirect: 'manual', + signal: AbortSignal.timeout(30_000), + }); + this.absorbSetCookie(res); + // Passport login redirects to /login?err=authFail on bad credentials. + // The redirect itself is 3xx (not 4xx), so we have to inspect the + // Location header to detect failure. + const location = res.headers.get('location') ?? ''; + if (/err=/.test(location)) { + throw new Error( + `HyperDX login failed: passport returned auth-failure redirect to ${location}`, + ); + } + if (res.status >= 400) { + const text = await res.text(); + throw new Error( + `HyperDX login failed (${res.status}): ${text.slice(0, 300)}`, + ); + } + } + + async me(): Promise { + const { body } = await this.request('GET', '/me'); + return body; + } + + async listConnections(): Promise { + const { body } = await this.request( + 'GET', + '/connections', + ); + return body; + } + + async createConnection(input: { + name: string; + host: string; + username: string; + password?: string; + }): Promise<{ id: string }> { + const { body } = await this.request<{ id: string }>( + 'POST', + '/connections', + input, + ); + return body; + } + + async listSources(): Promise { + const { body } = await this.request('GET', '/sources'); + return body; + } + + async createSource(input: Record): Promise { + const { body } = await this.request( + 'POST', + '/sources', + input, + ); + return body; + } + + async deleteSource(id: string): Promise { + await this.request('DELETE', `/sources/${id}`); + } +} diff --git a/packages/hdx-eval/src/hyperdx/config.ts b/packages/hdx-eval/src/hyperdx/config.ts new file mode 100644 index 0000000000..2a147be880 --- /dev/null +++ b/packages/hdx-eval/src/hyperdx/config.ts @@ -0,0 +1,185 @@ +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { resolve } from 'path'; + +import type { McpDefinition, McpKind } from '../harness/types'; + +export type ScenarioSourceIds = { + tracesSourceId: string; + logsSourceId: string; +}; + +/** + * Configuration for the HyperDX API — only needed by the `setup-hyperdx` + * command to create Connections and Sources. Not required for running evals. + */ +export type HyperdxApiConfig = { + apiUrl: string; + accessKey: string; + connectionId: string; +}; + +export type EvalConfig = { + /** Registry of MCP definitions keyed by a free-form name. */ + mcps: Record; + /** Per-scenario HyperDX Source IDs (only needed when an MCP points at + * HyperDX and the Sources must exist). */ + scenarios?: Record; + /** HyperDX API config — only needed for `setup-hyperdx`. */ + hyperdxApi?: HyperdxApiConfig; + /** ClickHouse connection details — used by seed/drop/instrument commands + * and as the default for stdio MCP env vars. */ + clickhouse?: { + host: string; + port: string; + user: string; + password: string; + }; + /** + * Fixed "now" anchor (ISO 8601) for both seed and agent system prompt. + * Defaults to the current time on first run and is persisted so subsequent + * runs reuse the same anchor. Override with `--anchor-time` on the CLI, or + * pass `--live` to ignore the saved value and use wall-clock time. + */ + anchorTime?: string; +}; + +const CONFIG_FILENAME = 'eval.config.json'; + +export function configPath(): string { + // Resolve relative to the package root regardless of cwd. + // dist/cli.js → dist/ → packageRoot. src/cli.ts via tsx → src/ → packageRoot. + return resolve(__dirname, '..', '..', CONFIG_FILENAME); +} + +export function configExists(path: string = configPath()): boolean { + return existsSync(path); +} + +export function readConfig(path: string = configPath()): EvalConfig { + if (!existsSync(path)) { + throw new Error( + `Eval config not found at ${path}. Run \`hdx-eval setup-hyperdx\` first.`, + ); + } + const raw = JSON.parse(readFileSync(path, 'utf8')); + return validateConfig(raw, path); +} + +export function writeConfig( + config: EvalConfig, + path: string = configPath(), +): void { + validateConfig(config, path); + writeFileSync(path, JSON.stringify(config, null, 2) + '\n', 'utf8'); +} + +/** Return all MCP names defined in the config. */ +export function configMcpNames(config: EvalConfig): McpKind[] { + return Object.keys(config.mcps); +} + +/** Return only the enabled MCP names (where `enabled` is not `false`). */ +export function enabledMcpNames(config: EvalConfig): McpKind[] { + return Object.entries(config.mcps) + .filter(([, def]) => def.enabled !== false) + .map(([name]) => name); +} + +/** Get a single MCP definition by name, or throw. */ +export function getMcpDefinition( + config: EvalConfig, + name: McpKind, +): McpDefinition { + const def = config.mcps[name]; + if (!def) { + const available = configMcpNames(config).join(', '); + throw new Error( + `MCP "${name}" not found in config. Available: ${available}`, + ); + } + return def; +} + +/** + * Return the config's saved `anchorTime`, creating and persisting a default + * (current wall-clock time) when none exists yet. This makes anchor-time + * "sticky" — the first `run` freezes the anchor, and later runs reuse it + * automatically. + * + * Pass `overrideIso` to force a specific value (e.g. from `--anchor-time`). + * The override is saved back to the config file. + */ +export function ensureAnchorTime( + config: EvalConfig, + overrideIso?: string, +): { anchorTimeIso: string; anchorMs: number } { + let iso = overrideIso ?? config.anchorTime; + if (!iso) { + iso = new Date().toISOString(); + } + const ms = Date.parse(iso); + if (!Number.isFinite(ms)) { + throw new Error(`Invalid anchorTime value: ${iso}`); + } + // Normalise and persist if it changed. + const normalised = new Date(ms).toISOString(); + if (config.anchorTime !== normalised) { + config.anchorTime = normalised; + writeConfig(config); + } + return { anchorTimeIso: normalised, anchorMs: ms }; +} + +function validateConfig(raw: unknown, path: string): EvalConfig { + if (!raw || typeof raw !== 'object') { + throw new Error(`Eval config at ${path} is not an object.`); + } + const obj = raw as Record; + + // Require the `mcps` section. + const mcps = obj.mcps as Record | undefined; + if (!mcps || typeof mcps !== 'object') { + throw new Error( + `Eval config at ${path} missing 'mcps' section. ` + + `Re-run \`hdx-eval setup-hyperdx\` to generate a new config.`, + ); + } + + // Validate each MCP definition. + for (const [name, def] of Object.entries(mcps)) { + if (!def || typeof def !== 'object') { + throw new Error(`Eval config 'mcps.${name}' must be an object`); + } + const d = def as Record; + if (d.type !== 'http' && d.type !== 'stdio') { + throw new Error( + `Eval config 'mcps.${name}.type' must be "http" or "stdio"`, + ); + } + if (d.type === 'http') { + if (typeof d.url !== 'string' || !d.url) { + throw new Error( + `Eval config 'mcps.${name}.url' must be a non-empty string`, + ); + } + } else { + if (typeof d.command !== 'string' || !d.command) { + throw new Error( + `Eval config 'mcps.${name}.command' must be a non-empty string`, + ); + } + } + if (typeof d.toolPattern !== 'string' || !d.toolPattern) { + throw new Error( + `Eval config 'mcps.${name}.toolPattern' must be a non-empty string`, + ); + } + if (typeof d.label !== 'string' || !d.label) { + throw new Error( + `Eval config 'mcps.${name}.label' must be a non-empty string`, + ); + } + } + + return raw as EvalConfig; +} diff --git a/packages/hdx-eval/src/hyperdx/setup.ts b/packages/hdx-eval/src/hyperdx/setup.ts new file mode 100644 index 0000000000..7450a37855 --- /dev/null +++ b/packages/hdx-eval/src/hyperdx/setup.ts @@ -0,0 +1,374 @@ +import { scenarioTables } from '../clickhouse/schema'; +import { QUERY_TIMEOUT_SECONDS } from '../harness/mcpConfig'; +import type { McpDefinition } from '../harness/types'; +import { SCENARIO_NAMES } from '../scenarios'; +import { HyperdxApiClient, type HyperdxSource } from './api'; +import { + configPath, + type EvalConfig, + readConfig as readEvalConfig, + writeConfig, +} from './config'; + +const EVAL_CONNECTION_NAME = 'hdx-eval-clickhouse'; +const TRACE_DEFAULT_SELECT = + 'Timestamp, ServiceName, SpanName, Duration, StatusCode'; +const LOG_DEFAULT_SELECT = 'Timestamp, ServiceName, SeverityText, Body'; + +export type SetupOptions = { + apiUrl: string; + email: string; + password: string; + clickhouse: { + host: string; + port: string; + user: string; + password: string; + }; + resetSources?: boolean; +}; + +export type SetupResult = { + configPath: string; + config: EvalConfig; + created: { connection: boolean; sources: string[] }; +}; + +export async function runSetup(opts: SetupOptions): Promise { + const api = new HyperdxApiClient(opts.apiUrl); + + // 1. Register (idempotent — accepts "already exists") then login. + await api.register(opts.email, opts.password); + await api.login(opts.email, opts.password); + + // 2. Fetch user accessKey for MCP Bearer auth. + const me = await api.me(); + + // 3. Ensure eval Connection exists pointing at the local ClickHouse. + const connections = await api.listConnections(); + const chHost = `http://${opts.clickhouse.host}:${opts.clickhouse.port}`; + let connection = connections.find(c => c.name === EVAL_CONNECTION_NAME); + let createdConnection = false; + if (!connection) { + const { id } = await api.createConnection({ + name: EVAL_CONNECTION_NAME, + host: chHost, + username: opts.clickhouse.user, + password: opts.clickhouse.password, + }); + connection = { + _id: id, + name: EVAL_CONNECTION_NAME, + host: chHost, + username: opts.clickhouse.user, + }; + createdConnection = true; + } + + // 4. Ensure one Trace Source + one Log Source per scenario. + let sources = await api.listSources(); + if (opts.resetSources) { + const evalSources = sources.filter(s => s.name.startsWith('eval-')); + for (const s of evalSources) { + await api.deleteSource(sourceId(s)); + } + sources = await api.listSources(); + } + const sourcesByName = new Map(sources.map(s => [s.name, s])); + const created: string[] = []; + const scenarioIds: Record< + string, + { tracesSourceId: string; logsSourceId: string } + > = {}; + + for (const name of SCENARIO_NAMES) { + const tables = scenarioTables(name); + const traceName = `eval-${name}-traces`; + const logName = `eval-${name}-logs`; + + let traceSource = sourcesByName.get(traceName); + if (!traceSource) { + traceSource = await api.createSource( + buildTraceSourceBody(traceName, connection._id, tables.traces, { + kvRollupTable: tables.tracesKvRollup, + keyRollupTable: tables.tracesKeyRollup, + }), + ); + created.push(traceName); + } + + let logSource = sourcesByName.get(logName); + if (!logSource) { + logSource = await api.createSource( + buildLogSourceBody(logName, connection._id, tables.logs, { + kvRollupTable: tables.logsKvRollup, + keyRollupTable: tables.logsKeyRollup, + }), + ); + created.push(logName); + } + + scenarioIds[name] = { + tracesSourceId: sourceId(traceSource), + logsSourceId: sourceId(logSource), + }; + } + + const mcpUrl = `${opts.apiUrl.replace(/\/$/, '')}/mcp`; + + const hyperdxMcp: McpDefinition = { + type: 'http', + url: mcpUrl, + headers: { Authorization: `Bearer ${me.accessKey}` }, + toolPattern: 'mcp__hyperdx__*', + label: 'HyperDX', + brandTerms: ['HyperDX', 'hyperdx'], + deniedTools: [ + 'mcp__hyperdx__hyperdx_delete_dashboard', + 'mcp__hyperdx__hyperdx_get_dashboard', + 'mcp__hyperdx__hyperdx_save_dashboard', + 'mcp__hyperdx__hyperdx_query_tile', + 'mcp__hyperdx__hyperdx_get_saved_search', + 'mcp__hyperdx__hyperdx_save_saved_search', + 'mcp__hyperdx__hyperdx_get_alert', + 'mcp__hyperdx__hyperdx_get_webhook', + 'mcp__hyperdx__hyperdx_save_alert', + ], + }; + + const clickhouseMcp: McpDefinition = { + type: 'stdio', + command: 'uv', + args: [ + 'run', + '--with', + 'mcp-clickhouse==0.3.0', + '--python', + '3.10', + 'mcp-clickhouse', + ], + env: { + CLICKHOUSE_HOST: opts.clickhouse.host, + CLICKHOUSE_PORT: opts.clickhouse.port, + CLICKHOUSE_USER: opts.clickhouse.user, + CLICKHOUSE_PASSWORD: opts.clickhouse.password, + CLICKHOUSE_SECURE: 'false', + CLICKHOUSE_VERIFY: 'false', + CLICKHOUSE_SEND_RECEIVE_TIMEOUT: String(QUERY_TIMEOUT_SECONDS), + }, + toolPattern: 'mcp__clickhouse__*', + label: 'ClickHouse MCP', + brandTerms: ['ClickHouse MCP', 'clickhouse'], + }; + + const config: EvalConfig = { + mcps: { + hyperdx: hyperdxMcp, + clickhouse: clickhouseMcp, + }, + scenarios: scenarioIds, + hyperdxApi: { + apiUrl: opts.apiUrl, + accessKey: me.accessKey, + connectionId: connection._id, + }, + clickhouse: opts.clickhouse, + }; + writeConfig(config); + + return { + configPath: configPath(), + config, + created: { connection: createdConnection, sources: created }, + }; +} + +function sourceId(source: HyperdxSource): string { + return source.id ?? source._id; +} + +// Apply a per-query max_execution_time to every Source so the HyperDX MCP +// faces the same query timeout as the ClickHouse MCP. Simulates a busy +// production cluster where queries that scan everything time out. +const EVAL_QUERY_SETTINGS = [ + { setting: 'max_execution_time', value: String(QUERY_TIMEOUT_SECONDS) }, +]; + +function buildTraceSourceBody( + name: string, + connectionId: string, + tableName: string, + rollup: { kvRollupTable: string; keyRollupTable: string }, +): Record { + return { + name, + kind: 'trace', + connection: connectionId, + from: { databaseName: 'default', tableName }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: TRACE_DEFAULT_SELECT, + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + serviceNameExpression: 'ServiceName', + resourceAttributesExpression: 'ResourceAttributes', + eventAttributesExpression: 'SpanAttributes', + implicitColumnExpression: 'SpanName', + querySettings: EVAL_QUERY_SETTINGS, + metadataMaterializedViews: { + keyRollupTable: rollup.keyRollupTable, + kvRollupTable: rollup.kvRollupTable, + granularity: '15 minute', + }, + }; +} + +function buildLogSourceBody( + name: string, + connectionId: string, + tableName: string, + rollup: { kvRollupTable: string; keyRollupTable: string }, +): Record { + return { + name, + kind: 'log', + connection: connectionId, + from: { databaseName: 'default', tableName }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: LOG_DEFAULT_SELECT, + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + bodyExpression: 'Body', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + resourceAttributesExpression: 'ResourceAttributes', + eventAttributesExpression: 'LogAttributes', + implicitColumnExpression: 'Body', + querySettings: EVAL_QUERY_SETTINGS, + metadataMaterializedViews: { + keyRollupTable: rollup.keyRollupTable, + kvRollupTable: rollup.kvRollupTable, + granularity: '15 minute', + }, + }; +} + +export type CheckResult = { + configOk: boolean; + mcpReachable: boolean; + clickhouseReachable: boolean; + uvAvailable: boolean; + errors: string[]; +}; + +export async function runCheck( + apiBearerCheck: ( + mcpUrl: string, + accessKey: string, + ) => Promise = defaultMcpCheck, + uvCheck: () => Promise = defaultUvCheck, +): Promise { + const errors: string[] = []; + let configOk = false; + let cfg: EvalConfig | null = null; + try { + cfg = readEvalConfig(); + configOk = true; + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + + let mcpReachable = false; + let chReachable = false; + if (cfg) { + // Check the HyperDX MCP if there's an http-type MCP in the config. + const httpMcp = Object.values(cfg.mcps).find(m => m.type === 'http'); + if (httpMcp && httpMcp.type === 'http') { + const accessKey = + httpMcp.headers?.Authorization?.replace('Bearer ', '') ?? ''; + mcpReachable = await apiBearerCheck(httpMcp.url, accessKey); + if (!mcpReachable) { + errors.push( + `HyperDX MCP at ${httpMcp.url} did not respond as expected`, + ); + } + } + if (cfg.clickhouse) { + try { + const chUrl = `http://${cfg.clickhouse.host}:${cfg.clickhouse.port}/ping`; + const res = await fetch(chUrl, { + signal: AbortSignal.timeout(10_000), + }); + chReachable = res.ok; + if (!chReachable) errors.push(`ClickHouse ping → ${res.status}`); + } catch (e) { + errors.push( + `ClickHouse not reachable: ${e instanceof Error ? e.message : e}`, + ); + } + } + } + + const uvAvailable = await uvCheck(); + if (!uvAvailable) { + errors.push( + `\`uv\` not found on PATH. Install via \`brew install uv\` or follow https://docs.astral.sh/uv/`, + ); + } + + return { + configOk, + mcpReachable, + clickhouseReachable: chReachable, + uvAvailable, + errors, + }; +} + +async function defaultMcpCheck( + mcpUrl: string, + accessKey: string, +): Promise { + try { + // The MCP HTTP transport responds to a GET with 405 (method not allowed) + // when not given a JSON-RPC body, but it must accept the bearer token. + // POST a minimal initialize message instead. + const res = await fetch(mcpUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${accessKey}`, + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + }, + signal: AbortSignal.timeout(10_000), + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'hdx-eval-check', version: '0.1.0' }, + }, + }), + }); + return res.ok || res.status === 200 || res.status === 202; + } catch { + return false; + } +} + +async function defaultUvCheck(): Promise { + const { spawn } = await import('child_process'); + return new Promise(resolve => { + const proc = spawn('uv', ['--version'], { stdio: 'ignore' }); + proc.on('error', () => resolve(false)); + proc.on('exit', code => resolve(code === 0)); + }); +} diff --git a/packages/hdx-eval/src/index.ts b/packages/hdx-eval/src/index.ts new file mode 100644 index 0000000000..6e5c8f0aae --- /dev/null +++ b/packages/hdx-eval/src/index.ts @@ -0,0 +1,10 @@ +export { scenarioTables } from './clickhouse/schema'; +export type { LogRow, TraceRow } from './generators/types'; +export type { SeededRng } from './rng/seeded'; +export { mulberry32 } from './rng/seeded'; +export { getScenario, SCENARIO_NAMES, SCENARIOS } from './scenarios'; +export type { + GenerateContext, + Scenario, + ScenarioBatch, +} from './scenarios/types'; diff --git a/packages/hdx-eval/src/reports/aggregate.ts b/packages/hdx-eval/src/reports/aggregate.ts new file mode 100644 index 0000000000..ee86152123 --- /dev/null +++ b/packages/hdx-eval/src/reports/aggregate.ts @@ -0,0 +1,235 @@ +import type { GradeRecord } from '../grading/types'; +import type { McpKind, RunRecord } from '../harness/types'; + +export type CellSummary = { + scenario: string; + mcp: McpKind; + n: number; + programmatic: { mean: number; perCheck: Record }; + judge: { + weightedMean: number; + perCriterion: Record; + judged: number; + }; + combinedScore: { mean: number }; + toolCalls: { mean: number }; + toolErrors: { mean: number; rateMean: number; penaltyMean: number }; + tokens: { + input: number; + output: number; + cacheRead: number; + cacheCreation: number; + }; + durationMs: { mean: number }; + termination: Record; +}; + +export type DeltaSummary = { + combinedScore: number | null; + programmaticScore: number | null; + judgeWeightedMean: number | null; + toolCalls: number | null; + outputTokens: number | null; + durationMs: number | null; +}; + +export type ScenarioSummary = { + scenario: string; + /** All MCP cells keyed by MCP name. */ + cells: Record; + /** Which MCP is the baseline (first MCP, or explicitly set). */ + baseline?: McpKind; + /** Per-challenger deltas vs the baseline. Keyed by challenger MCP name. */ + deltas: Record; +}; + +export type BatchSummary = { + batchDir: string; + generatedAt: string; + /** The baseline MCP used for delta computation. */ + baseline?: McpKind; + /** Ordered list of MCP names as they appear in the report columns. */ + mcpOrder: McpKind[]; + scenarios: ScenarioSummary[]; +}; + +export type GradedRunPair = { + run: RunRecord; + grade: GradeRecord; +}; + +export function buildAggregate(args: { + batchDir: string; + pairs: GradedRunPair[]; + /** Explicit baseline MCP. If not set, the first MCP encountered is used. */ + baseline?: McpKind; +}): BatchSummary { + const byScenario = new Map(); + // Track all MCP names in insertion order. + const mcpSet = new Set(); + for (const p of args.pairs) { + if (!byScenario.has(p.run.scenario)) byScenario.set(p.run.scenario, []); + byScenario.get(p.run.scenario)!.push(p); + mcpSet.add(p.run.mcp); + } + const mcpOrder = [...mcpSet].sort(); + const baseline = args.baseline ?? mcpOrder[0]; + + const scenarios: ScenarioSummary[] = []; + for (const [name, ps] of byScenario.entries()) { + const byMcp = new Map(); + for (const p of ps) { + if (!byMcp.has(p.run.mcp)) byMcp.set(p.run.mcp, []); + byMcp.get(p.run.mcp)!.push(p); + } + const cells: Record = {}; + for (const [mcp, list] of byMcp.entries()) { + cells[mcp] = buildCellSummary(name, mcp, list); + } + + // Compute deltas: each non-baseline MCP vs the baseline. + const deltas: Record = {}; + const baselineCell = cells[baseline]; + for (const mcp of Object.keys(cells)) { + if (mcp === baseline) continue; + deltas[mcp] = computeDelta(cells[mcp], baselineCell); + } + + scenarios.push({ + scenario: name, + cells, + baseline, + deltas, + }); + } + + scenarios.sort((a, b) => a.scenario.localeCompare(b.scenario)); + return { + batchDir: args.batchDir, + generatedAt: new Date().toISOString(), + baseline, + mcpOrder, + scenarios, + }; +} + +function buildCellSummary( + scenario: string, + mcp: McpKind, + pairs: GradedRunPair[], +): CellSummary { + const n = pairs.length; + const programmaticScore = mean(pairs.map(p => p.grade.programmatic.score)); + + const judgeable = pairs.filter(p => p.grade.judge); + const judgeMean = + judgeable.length === 0 + ? 0 + : mean(judgeable.map(p => p.grade.judge!.weightedScore)); + + const combinedMean = mean(pairs.map(p => p.grade.combinedScore)); + const toolCallsMean = mean(pairs.map(p => p.run.toolCalls.length)); + const toolErrorsMean = mean(pairs.map(p => p.grade.toolErrors?.errors ?? 0)); + const toolErrorRateMean = mean(pairs.map(p => p.grade.toolErrors?.rate ?? 0)); + const toolErrorPenaltyMean = mean( + pairs.map(p => p.grade.toolErrors?.penalty ?? 0), + ); + const durationMean = mean(pairs.map(p => p.run.durationMs)); + + const tokens = { + input: mean(pairs.map(p => p.run.tokens.input)), + output: mean(pairs.map(p => p.run.tokens.output)), + cacheRead: mean(pairs.map(p => p.run.tokens.cacheRead)), + cacheCreation: mean(pairs.map(p => p.run.tokens.cacheCreation)), + }; + + const perCheck: Record = {}; + if (pairs.length > 0) { + const checkIds = new Set(); + for (const p of pairs) + for (const h of p.grade.programmatic.hits) checkIds.add(h.id); + for (const id of checkIds) { + const satisfiedHits = pairs.map( + p => + p.grade.programmatic.hits.find(h => h.id === id)?.satisfied ?? false, + ); + perCheck[id] = + satisfiedHits.filter(Boolean).length / satisfiedHits.length; + } + } + + const perCriterion: Record = {}; + if (judgeable.length > 0) { + const critIds = new Set(); + for (const p of judgeable) + for (const k of Object.keys(p.grade.judge!.scores)) critIds.add(k); + for (const id of critIds) { + const scores = judgeable + .map(p => p.grade.judge!.scores[id]?.score) + .filter((s): s is number => typeof s === 'number'); + if (scores.length > 0) perCriterion[id] = mean(scores); + } + } + + const termination: Record = {}; + for (const p of pairs) { + termination[p.run.termination] = (termination[p.run.termination] ?? 0) + 1; + } + + return { + scenario, + mcp, + n, + programmatic: { mean: programmaticScore, perCheck }, + judge: { + weightedMean: judgeMean, + perCriterion, + judged: judgeable.length, + }, + combinedScore: { mean: combinedMean }, + toolCalls: { mean: toolCallsMean }, + toolErrors: { + mean: toolErrorsMean, + rateMean: toolErrorRateMean, + penaltyMean: toolErrorPenaltyMean, + }, + tokens, + durationMs: { mean: durationMean }, + termination, + }; +} + +/** + * Compute the delta between a challenger and the baseline. + * Delta = challenger − baseline. + */ +function computeDelta( + challenger: CellSummary | undefined, + baseline: CellSummary | undefined, +): DeltaSummary { + if (!challenger || !baseline) { + return { + combinedScore: null, + programmaticScore: null, + judgeWeightedMean: null, + toolCalls: null, + outputTokens: null, + durationMs: null, + }; + } + return { + combinedScore: challenger.combinedScore.mean - baseline.combinedScore.mean, + programmaticScore: + challenger.programmatic.mean - baseline.programmatic.mean, + judgeWeightedMean: + challenger.judge.weightedMean - baseline.judge.weightedMean, + toolCalls: challenger.toolCalls.mean - baseline.toolCalls.mean, + outputTokens: challenger.tokens.output - baseline.tokens.output, + durationMs: challenger.durationMs.mean - baseline.durationMs.mean, + }; +} + +function mean(xs: number[]): number { + if (xs.length === 0) return 0; + return xs.reduce((s, x) => s + x, 0) / xs.length; +} diff --git a/packages/hdx-eval/src/reports/markdown.ts b/packages/hdx-eval/src/reports/markdown.ts new file mode 100644 index 0000000000..4aabedb9fc --- /dev/null +++ b/packages/hdx-eval/src/reports/markdown.ts @@ -0,0 +1,264 @@ +import type { McpKind } from '../harness/types'; +import type { + BatchSummary, + CellSummary, + DeltaSummary, + ScenarioSummary, +} from './aggregate'; + +export function renderMarkdownReport(summary: BatchSummary): string { + const lines: string[] = []; + lines.push(`# Eval Batch — ${basename(summary.batchDir)}`); + lines.push(''); + lines.push(`Generated: ${summary.generatedAt}`); + if (summary.baseline) { + lines.push(`Baseline: ${summary.baseline}`); + } + lines.push(`MCPs: ${summary.mcpOrder.join(', ')}`); + lines.push(''); + lines.push(renderTopVerdict(summary)); + lines.push(''); + + for (const scenario of summary.scenarios) { + lines.push(`## ${scenario.scenario}`); + lines.push(''); + lines.push(renderScenarioTable(scenario, summary.mcpOrder)); + lines.push(''); + const judgeBreakdown = renderJudgeBreakdown(scenario, summary.mcpOrder); + if (judgeBreakdown) { + lines.push(judgeBreakdown); + lines.push(''); + } + const programmaticBreakdown = renderProgrammaticBreakdown( + scenario, + summary.mcpOrder, + ); + if (programmaticBreakdown) { + lines.push(programmaticBreakdown); + lines.push(''); + } + } + return lines.join('\n'); +} + +function renderTopVerdict(summary: BatchSummary): string { + const mcps = summary.mcpOrder; + const baseline = summary.baseline; + // Header: | Scenario | MCP1 | MCP2 | ... | Δ₁ | Δ₂ | ... + const mcpHeaders = mcps.map(m => m).join(' | '); + const challengers = mcps.filter(m => m !== baseline); + const deltaHeaders = + challengers.length > 0 + ? ' | ' + challengers.map(m => `Δ (${m})`).join(' | ') + : ''; + const header = `| Scenario | ${mcpHeaders}${deltaHeaders} |`; + const sep = '|---' + '|---'.repeat(mcps.length + challengers.length) + '|'; + + const rows = [header, sep]; + for (const s of summary.scenarios) { + const mcpCells = mcps.map(m => { + const c = s.cells[m]; + return c ? pct(c.combinedScore.mean) : '—'; + }); + const deltaCells = challengers.map(m => { + const d = s.deltas[m]; + return d?.combinedScore !== null && d?.combinedScore !== undefined + ? signedPct(d.combinedScore) + : '—'; + }); + const allCells = [...mcpCells, ...deltaCells]; + rows.push(`| ${s.scenario} | ${allCells.join(' | ')} |`); + } + return ['### Top-line verdict', '', ...rows].join('\n'); +} + +function renderScenarioTable( + scenario: ScenarioSummary, + mcpOrder: McpKind[], +): string { + const baseline = scenario.baseline; + const challengers = mcpOrder.filter(m => m !== baseline); + const hasDelta = challengers.length > 0; + + // Build dynamic headers: | Metric | MCP1 | MCP2 | ... | Δ₁ | ... + const mcpHeaders = mcpOrder.map(m => m).join(' | '); + const deltaHeaders = hasDelta + ? ' | ' + challengers.map(m => `Δ (${m})`).join(' | ') + : ''; + const header = `| Metric | ${mcpHeaders}${deltaHeaders} |`; + const sep = + '|---' + '|---'.repeat(mcpOrder.length + challengers.length) + '|'; + + const rows = [header, sep]; + + const cellFor = (m: McpKind): CellSummary | undefined => scenario.cells[m]; + const deltaFor = (m: McpKind): DeltaSummary | undefined => scenario.deltas[m]; + + function addRow( + label: string, + valueFn: (c: CellSummary | undefined) => string, + deltaFn?: (d: DeltaSummary | undefined) => string, + ) { + const mcpCells = mcpOrder.map(m => valueFn(cellFor(m))); + const deltaCells = hasDelta + ? challengers.map(m => (deltaFn ? deltaFn(deltaFor(m)) : '—')) + : []; + rows.push(`| ${label} | ${[...mcpCells, ...deltaCells].join(' | ')} |`); + } + + addRow( + 'Combined score', + c => val(c?.combinedScore.mean, pct), + d => signedPct(d?.combinedScore), + ); + addRow( + 'Programmatic score', + c => val(c?.programmatic.mean, pct), + d => signedPct(d?.programmaticScore), + ); + addRow( + 'Judge mean (weighted)', + c => val(c?.judge.weightedMean, pct), + d => signedPct(d?.judgeWeightedMean), + ); + addRow( + 'Tool calls (mean)', + c => val(c?.toolCalls.mean, oneDecimal), + d => signedNumber(d?.toolCalls, 1), + ); + addRow('Tool errors (mean)', c => val(c?.toolErrors?.mean, oneDecimal)); + addRow('Tool-error penalty', c => + val(c?.toolErrors?.penaltyMean, x => `${(x * 100).toFixed(0)}pp`), + ); + addRow( + 'Output tokens (mean)', + c => val(c?.tokens.output, intStr), + d => signedNumber(d?.outputTokens, 0), + ); + addRow('Cache reads (mean)', c => val(c?.tokens.cacheRead, intStr)); + addRow( + 'Wall clock (s, mean)', + c => val(c?.durationMs.mean, secondsFromMs), + d => + d?.durationMs !== null && d?.durationMs !== undefined + ? signedNumber(d.durationMs / 1000, 1) + : '—', + ); + addRow('Termination', c => (c ? terminationBreakdown(c.termination) : '—')); + addRow('N', c => (c ? String(c.n) : '—')); + + return rows.join('\n'); +} + +function renderJudgeBreakdown( + scenario: ScenarioSummary, + mcpOrder: McpKind[], +): string | null { + const allCriteria = new Set(); + for (const cell of Object.values(scenario.cells)) { + if (!cell) continue; + for (const id of Object.keys(cell.judge.perCriterion)) allCriteria.add(id); + } + if (allCriteria.size === 0) return null; + + const mcpHeaders = mcpOrder.join(' | '); + const rows = [ + '#### Judge per-criterion (mean 0–5)', + '', + `| Criterion | ${mcpHeaders} |`, + '|---' + '|---'.repeat(mcpOrder.length) + '|', + ]; + for (const id of [...allCriteria].sort()) { + const mcpCells = mcpOrder.map(m => { + const v = scenario.cells[m]?.judge.perCriterion[id]; + return fmtScore(v); + }); + rows.push(`| ${id} | ${mcpCells.join(' | ')} |`); + } + return rows.join('\n'); +} + +function renderProgrammaticBreakdown( + scenario: ScenarioSummary, + mcpOrder: McpKind[], +): string | null { + const allChecks = new Set(); + for (const cell of Object.values(scenario.cells)) { + if (!cell) continue; + for (const id of Object.keys(cell.programmatic.perCheck)) allChecks.add(id); + } + if (allChecks.size === 0) return null; + + const mcpHeaders = mcpOrder.join(' | '); + const rows = [ + '#### Programmatic per-check (pass rate)', + '', + 'Pass rate = positive checks matched + negative checks not matched.', + '', + `| Check | ${mcpHeaders} |`, + '|---' + '|---'.repeat(mcpOrder.length) + '|', + ]; + for (const id of [...allChecks].sort()) { + const mcpCells = mcpOrder.map(m => { + const v = scenario.cells[m]?.programmatic.perCheck[id]; + return fmtRate(v); + }); + const isNegative = id.startsWith('false_'); + const label = isNegative ? `${id} (neg)` : id; + rows.push(`| ${label} | ${mcpCells.join(' | ')} |`); + } + return rows.join('\n'); +} + +function val(v: T | undefined, fmt: (x: T) => string): string { + return v === undefined ? '—' : fmt(v); +} + +function pct(x: number | undefined): string { + if (x === undefined) return '—'; + return `${(x * 100).toFixed(0)}%`; +} + +function signedPct(x: number | null | undefined): string { + if (x === null || x === undefined) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${(x * 100).toFixed(0)}%`; +} + +function oneDecimal(x: number): string { + return x.toFixed(1); +} + +function intStr(x: number): string { + return Math.round(x).toString(); +} + +function secondsFromMs(x: number): string { + return (x / 1000).toFixed(1); +} + +function signedNumber(x: number | null | undefined, decimals: number): string { + if (x === null || x === undefined) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${x.toFixed(decimals)}`; +} + +function fmtScore(x: number | undefined): string { + if (x === undefined) return '—'; + return `${x.toFixed(2)}/5`; +} + +function fmtRate(x: number | undefined): string { + if (x === undefined) return '—'; + return `${(x * 100).toFixed(0)}%`; +} + +function terminationBreakdown(rec: Record): string { + const entries = Object.entries(rec).sort((a, b) => b[1] - a[1]); + return entries.map(([k, v]) => `${k}: ${v}`).join(', '); +} + +function basename(path: string): string { + const parts = path.split(/[\\/]/); + return parts[parts.length - 1] || path; +} diff --git a/packages/hdx-eval/src/reports/store.ts b/packages/hdx-eval/src/reports/store.ts new file mode 100644 index 0000000000..ebc5e41aa5 --- /dev/null +++ b/packages/hdx-eval/src/reports/store.ts @@ -0,0 +1,60 @@ +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import type { GradeRecord } from '../grading/types'; +import type { McpKind, RunRecord } from '../harness/types'; +import { SCENARIO_NAMES } from '../scenarios'; +import { buildAggregate, type GradedRunPair } from './aggregate'; +import { renderMarkdownReport } from './markdown'; + +function loadGradedPairs(batchDir: string): GradedRunPair[] { + const pairs: GradedRunPair[] = []; + for (const scenario of safeReaddir(batchDir)) { + if (!SCENARIO_NAMES.includes(scenario)) continue; + const sceneDir = join(batchDir, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const file of safeReaddir(mcpDir)) { + if (!file.endsWith('.json')) continue; + if (file.endsWith('.grade.json')) continue; + const runPath = join(mcpDir, file); + const gradePath = runPath.replace(/\.json$/, '.grade.json'); + if (!existsSync(gradePath)) continue; + try { + const run = JSON.parse(readFileSync(runPath, 'utf8')) as RunRecord; + const grade = JSON.parse( + readFileSync(gradePath, 'utf8'), + ) as GradeRecord; + pairs.push({ run, grade }); + } catch { + // skip malformed + } + } + } + } + return pairs; +} + +export function writeBatchSummary( + batchDir: string, + outPath: string, + baseline?: McpKind, +): { jsonPath: string; mdPath: string; pairsCount: number } { + const pairs = loadGradedPairs(batchDir); + const summary = buildAggregate({ batchDir, pairs, baseline }); + + const jsonPath = outPath.replace(/\.md$/, '.json'); + writeFileSync(jsonPath, JSON.stringify(summary, null, 2) + '\n', 'utf8'); + const md = renderMarkdownReport(summary); + writeFileSync(outPath, md, 'utf8'); + + return { jsonPath, mdPath: outPath, pairsCount: pairs.length }; +} + +function safeReaddir(path: string): string[] { + try { + return readdirSync(path); + } catch { + return []; + } +} diff --git a/packages/hdx-eval/src/rng/seeded.ts b/packages/hdx-eval/src/rng/seeded.ts new file mode 100644 index 0000000000..9dac8fa9a4 --- /dev/null +++ b/packages/hdx-eval/src/rng/seeded.ts @@ -0,0 +1,57 @@ +export type SeededRng = { + next(): number; + range(minInclusive: number, maxExclusive: number): number; + intRange(minInclusive: number, maxExclusive: number): number; + pick(items: readonly T[]): T; + weightedPick(items: readonly { value: T; weight: number }[]): T; + hex(bytes: number): string; +}; + +export function mulberry32(seed: number): SeededRng { + let state = seed >>> 0; + + const next = (): number => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + + const range = (lo: number, hi: number): number => lo + next() * (hi - lo); + + const intRange = (lo: number, hi: number): number => + Math.floor(range(lo, hi)); + + const pick = (items: readonly T[]): T => { + if (items.length === 0) { + throw new Error('pick() called with empty array'); + } + return items[intRange(0, items.length)]; + }; + + const weightedPick = ( + items: readonly { value: T; weight: number }[], + ): T => { + const total = items.reduce((s, i) => s + i.weight, 0); + if (total <= 0) { + throw new Error('weightedPick() requires positive total weight'); + } + let roll = next() * total; + for (const item of items) { + roll -= item.weight; + if (roll < 0) return item.value; + } + return items[items.length - 1].value; + }; + + const hex = (bytes: number): string => { + let out = ''; + for (let i = 0; i < bytes; i++) { + out += intRange(0, 256).toString(16).padStart(2, '0'); + } + return out; + }; + + return { next, range, intRange, pick, weightedPick, hex }; +} diff --git a/packages/hdx-eval/src/runs/instrument.ts b/packages/hdx-eval/src/runs/instrument.ts new file mode 100644 index 0000000000..fd9b24f1a3 --- /dev/null +++ b/packages/hdx-eval/src/runs/instrument.ts @@ -0,0 +1,327 @@ +import type { ClickHouseClient } from '@clickhouse/client'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { createEvalClient, defaultClickHouseUrl } from '../clickhouse/client'; +import { scenarioTables } from '../clickhouse/schema'; +import type { RunRecord } from '../harness/types'; +import { SCENARIO_NAMES } from '../scenarios'; +import { runsRoot } from './path'; + +const QUERY_LOG_BUFFER_MS = 5_000; + +export type ServerQuery = { + eventTime: string; + queryDurationMs: number; + readRows: number; + resultRows: number; + memoryUsageBytes: number; + queryPreview: string; +}; + +export type ToolCallTiming = { + index: number; + name: string; + wallStartTs: string; + wallEndTs: string; + wallMs: number; + serverQueries: ServerQuery[]; + serverMs: number; + inferenceMs: number; +}; + +export type TimingRecord = { + schemaVersion: 1; + runId: string; + scenario: string; + mcp: string; + toolCalls: ToolCallTiming[]; + totalWallMs: number; + totalServerMs: number; + totalInferenceMs: number; + unmatchedQueriesCount: number; +}; + +type QueryLogRow = { + event_time_microseconds: string; + query_duration_ms: number | string; + read_rows: number | string; + result_rows: number | string; + memory_usage: number | string; + query: string; +}; + +export type InstrumentOptions = { + clickhouseUrl?: string; + username?: string; + password?: string; +}; + +/** + * Walk each run's `messages` array to derive per-tool-call wall-clock + * windows from user-message timestamps (which are present) plus + * tool_use ids in the preceding assistant message. + */ +function extractToolCallWindows(record: RunRecord): Array<{ + index: number; + name: string; + startTs: string; + endTs: string; +}> { + const out: Array<{ + index: number; + name: string; + startTs: string; + endTs: string; + }> = []; + let prevTs: string | null = record.startedAt; + let lastToolUseName: string | null = null; + let idx = 0; + for (const ev of record.messages as Array>) { + const t = ev?.type; + if (t === 'system' && ev.subtype === 'init') { + const stamp = ev.timestamp; + if (typeof stamp === 'string') prevTs = stamp; + } else if (t === 'assistant') { + const msg = (ev.message as Record) ?? {}; + const content = Array.isArray(msg.content) + ? (msg.content as unknown[]) + : []; + for (const block of content) { + if ( + block && + typeof block === 'object' && + (block as Record).type === 'tool_use' + ) { + const name = (block as Record).name; + if (typeof name === 'string') lastToolUseName = name; + } + } + } else if (t === 'user') { + const ts = ev.timestamp; + if (typeof ts !== 'string') continue; + const msg = (ev.message as Record) ?? {}; + const content = Array.isArray(msg.content) + ? (msg.content as unknown[]) + : []; + const hasToolResult = content.some( + b => + b && + typeof b === 'object' && + (b as Record).type === 'tool_result', + ); + if (!hasToolResult) continue; + out.push({ + index: idx++, + name: lastToolUseName ?? 'unknown', + startTs: prevTs ?? record.startedAt, + endTs: ts, + }); + prevTs = ts; + } + } + return out; +} + +async function fetchQueryLog( + client: ClickHouseClient, + startTs: string, + endTs: string, + tablePatterns: string[], +): Promise { + const startMs = Date.parse(startTs) - QUERY_LOG_BUFFER_MS; + const endMs = Date.parse(endTs) + QUERY_LOG_BUFFER_MS; + const startStr = new Date(startMs) + .toISOString() + .slice(0, 19) + .replace('T', ' '); + const endStr = new Date(endMs).toISOString().slice(0, 19).replace('T', ' '); + // Match queries that touch any of the given table patterns. + const orClauses = tablePatterns + .map( + p => + `query LIKE '%${p.replace(/'/g, "''").replace(/_/g, '\\_').replace(/%/g, '\\%')}%' ESCAPE '\\'`, + ) + .join(' OR '); + // Restrict to SELECT-shaped queries — agents don't issue INSERTs, but the + // re-seed step right before the run does, and we don't want those in the + // per-call attribution. + const sql = ` + SELECT + toString(event_time_microseconds) AS event_time_microseconds, + query_duration_ms, + read_rows, + result_rows, + memory_usage, + substring(query, 1, 240) AS query + FROM system.query_log + WHERE type = 'QueryFinish' + AND event_time >= toDateTime('${startStr}') + AND event_time <= toDateTime('${endStr}') + AND query_kind = 'Select' + AND (${orClauses}) + ORDER BY event_time_microseconds + SETTINGS max_execution_time = 30 + `; + const rs = await client.query({ query: sql, format: 'JSONEachRow' }); + return (await rs.json()) as QueryLogRow[]; +} + +function matchQueriesToCalls( + windows: ReturnType, + queries: QueryLogRow[], +): { calls: ToolCallTiming[]; unmatched: number } { + const callTimings: ToolCallTiming[] = windows.map(w => ({ + index: w.index, + name: w.name, + wallStartTs: w.startTs, + wallEndTs: w.endTs, + wallMs: Math.max(0, Date.parse(w.endTs) - Date.parse(w.startTs)), + serverQueries: [], + serverMs: 0, + inferenceMs: 0, + })); + + let unmatched = 0; + for (const q of queries) { + // ClickHouse returns toString(DateTime64(6)) as 'YYYY-MM-DD HH:MM:SS.uuuuuu' + // (UTC, space-separated). Convert to ISO so Date.parse() accepts it. + const isoForm = q.event_time_microseconds.replace(' ', 'T') + 'Z'; + const ts = Date.parse(isoForm); + if (!Number.isFinite(ts)) { + unmatched++; + continue; + } + // Walk calls in reverse: prefer the LATEST call whose start <= ts. This + // assigns queries to the call that issued them even when there's a small + // network/processing lag between query event_time and the user-message + // timestamp that closes the window. + let owner: ToolCallTiming | undefined; + for (let i = callTimings.length - 1; i >= 0; i--) { + const c = callTimings[i]; + const start = Date.parse(c.wallStartTs); + const end = Date.parse(c.wallEndTs); + if (ts >= start && ts <= end + QUERY_LOG_BUFFER_MS) { + owner = c; + break; + } + } + if (!owner) { + unmatched++; + continue; + } + const durMs = Number(q.query_duration_ms); + owner.serverQueries.push({ + eventTime: q.event_time_microseconds, + queryDurationMs: Number.isFinite(durMs) ? durMs : 0, + readRows: Number(q.read_rows), + resultRows: Number(q.result_rows), + memoryUsageBytes: Number(q.memory_usage), + queryPreview: q.query, + }); + owner.serverMs += Number.isFinite(durMs) ? durMs : 0; + } + + for (const c of callTimings) { + c.inferenceMs = Math.max(0, c.wallMs - c.serverMs); + } + return { calls: callTimings, unmatched }; +} + +async function instrumentRun(args: { + runPath: string; + client: ClickHouseClient; +}): Promise { + const record = JSON.parse(readFileSync(args.runPath, 'utf8')) as RunRecord; + const windows = extractToolCallWindows(record); + + // Match query_log on either the eval-specific tables OR system.tables + // calls (mcp-clickhouse does list_tables internally). + const tables = scenarioTables(record.scenario); + const tablePatterns = [tables.traces, tables.logs, 'system.tables']; + + const queries = await fetchQueryLog( + args.client, + record.startedAt, + record.endedAt, + tablePatterns, + ); + const matched = matchQueriesToCalls(windows, queries); + + const totalWall = matched.calls.reduce((s, c) => s + c.wallMs, 0); + const totalServer = matched.calls.reduce((s, c) => s + c.serverMs, 0); + + const timing: TimingRecord = { + schemaVersion: 1, + runId: record.runId, + scenario: record.scenario, + mcp: record.mcp, + toolCalls: matched.calls, + totalWallMs: totalWall, + totalServerMs: totalServer, + totalInferenceMs: Math.max(0, totalWall - totalServer), + unmatchedQueriesCount: matched.unmatched, + }; + const sidecarPath = args.runPath.replace(/\.json$/, '.timing.json'); + writeFileSync(sidecarPath, JSON.stringify(timing, null, 2) + '\n', 'utf8'); + return timing; +} + +export async function instrumentBatch( + batchDir: string, + opts: InstrumentOptions = {}, +): Promise { + const resolved = existsSync(batchDir) ? batchDir : join(runsRoot(), batchDir); + if (!existsSync(resolved)) { + throw new Error(`Batch not found: ${resolved}`); + } + const client = createEvalClient({ + url: opts.clickhouseUrl ?? defaultClickHouseUrl(), + username: opts.username, + password: opts.password, + }); + try { + const out: TimingRecord[] = []; + for (const scenario of safeReaddir(resolved)) { + if (!SCENARIO_NAMES.includes(scenario)) continue; + const sceneDir = join(resolved, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const file of safeReaddir(mcpDir)) { + if (!file.endsWith('.json')) continue; + if (file.endsWith('.grade.json') || file.endsWith('.timing.json')) { + continue; + } + const timing = await instrumentRun({ + runPath: join(mcpDir, file), + client, + }); + out.push(timing); + } + } + } + return out; + } finally { + await client.close(); + } +} + +function safeReaddir(path: string): string[] { + try { + return readdirSync(path); + } catch { + return []; + } +} + +export function summarizeTimingRecord(t: TimingRecord): string { + const wall = (t.totalWallMs / 1000).toFixed(1); + const server = (t.totalServerMs / 1000).toFixed(2); + const inference = (t.totalInferenceMs / 1000).toFixed(1); + const serverPct = + t.totalWallMs > 0 ? Math.round((t.totalServerMs / t.totalWallMs) * 100) : 0; + return ( + `${t.scenario}/${t.mcp}: wall=${wall}s server=${server}s (${serverPct}%) ` + + `inference=${inference}s calls=${t.toolCalls.length} unmatched=${t.unmatchedQueriesCount}` + ); +} diff --git a/packages/hdx-eval/src/runs/path.ts b/packages/hdx-eval/src/runs/path.ts new file mode 100644 index 0000000000..b69c48a8f6 --- /dev/null +++ b/packages/hdx-eval/src/runs/path.ts @@ -0,0 +1,26 @@ +import { resolve } from 'path'; + +import type { McpKind } from '../harness/types'; + +export function runsRoot(): string { + return resolve(__dirname, '..', '..', 'runs'); +} + +export function batchDirName(date: Date = new Date()): string { + return date.toISOString().replace(/[:.]/g, '-'); +} + +export function runFilePath(args: { + batchDir: string; + scenario: string; + mcp: McpKind; + runIndex: number; +}): string { + return resolve( + runsRoot(), + args.batchDir, + args.scenario, + args.mcp, + `${args.runIndex}.json`, + ); +} diff --git a/packages/hdx-eval/src/runs/store.ts b/packages/hdx-eval/src/runs/store.ts new file mode 100644 index 0000000000..8f58eb3499 --- /dev/null +++ b/packages/hdx-eval/src/runs/store.ts @@ -0,0 +1,64 @@ +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; + +import type { RunRecord } from '../harness/types'; +import { runFilePath, runsRoot } from './path'; + +export function writeRun(args: { + record: RunRecord; + batchDir: string; +}): string { + const path = runFilePath({ + batchDir: args.batchDir, + scenario: args.record.scenario, + mcp: args.record.mcp, + runIndex: args.record.runIndex, + }); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, JSON.stringify(args.record, null, 2) + '\n', 'utf8'); + return path; +} + +export function readRun(path: string): RunRecord { + return JSON.parse(readFileSync(path, 'utf8')) as RunRecord; +} + +export function listBatches(): string[] { + try { + return readdirSync(runsRoot()).sort(); + } catch { + return []; + } +} + +export function listRunsInBatch(batchDir: string): string[] { + const root = join(runsRoot(), batchDir); + const out: string[] = []; + try { + for (const scenario of readdirSync(root)) { + const sceneDir = join(root, scenario); + try { + for (const mcp of readdirSync(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + try { + for (const file of readdirSync(mcpDir)) { + if ( + file.endsWith('.json') && + !file.endsWith('.grade.json') && + !file.endsWith('.timing.json') + ) + out.push(join(mcpDir, file)); + } + } catch { + // skip + } + } + } catch { + // skip + } + } + } catch { + // none + } + return out.sort(); +} diff --git a/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts b/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts new file mode 100644 index 0000000000..dcc1a60b49 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts @@ -0,0 +1,736 @@ +import { makeLog } from '../../generators/logs'; +import { + analyticsEventLog, + backgroundLog, + buildResourcePool, + catalogLog, + envoyAccessLog, + heartbeatLog, + normalizeSeverityText, + pageRenderLog, + pickResource, + pickSeverityIn, + spreadTimestamp, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +const BACKGROUND_SERVICES = [ + 'checkout-api', + 'payment-service', + 'cart-service', + 'inventory-service', + 'notification-service', + 'search-service', + 'recommendation-service', + 'frontend', + 'api-gateway', + 'database', + 'auth-service', + 'billing-service', + 'analytics-service', + 'image-service', + 'cdn', + 'session-service', + 'pricing-service', + 'review-service', + 'shipping-service', + 'tax-service', + 'email-service', + 'feature-flags', + 'rum-collector', + 'webhook-relay', + 'metrics-pipeline', +] as const; + +const REGIONS = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1'] as const; + +const BACKGROUND_OPS: Record = { + 'checkout-api': ['POST /api/checkout', 'GET /api/cart', 'GET /api/order'], + 'payment-service': ['payment.charge', 'payment.refund', 'payment.lookup'], + 'cart-service': ['cart.add', 'cart.list', 'cart.remove'], + 'inventory-service': ['inventory.check', 'inventory.reserve'], + 'notification-service': ['notify.email', 'notify.sms', 'notify.push'], + 'search-service': ['search.autocomplete', 'search.click'], + 'recommendation-service': ['rec.compute', 'rec.fetch'], + frontend: ['GET /home', 'GET /product', 'GET /checkout', 'GET /account'], + 'api-gateway': ['route.dispatch', 'route.auth'], + database: ['database.query', 'database.tx'], + 'auth-service': ['auth.login', 'auth.refresh', 'auth.verify'], + 'billing-service': ['billing.invoice', 'billing.subscription'], + 'analytics-service': ['analytics.event', 'analytics.batch'], + 'image-service': ['image.resize', 'image.fetch'], + cdn: ['cdn.purge', 'cdn.fetch'], + 'session-service': ['session.create', 'session.validate', 'session.expire'], + 'pricing-service': ['pricing.compute', 'pricing.lookup'], + 'review-service': ['review.list', 'review.create'], + 'shipping-service': ['shipping.quote', 'shipping.track'], + 'tax-service': ['tax.compute'], + 'email-service': ['email.send', 'email.queue'], + 'feature-flags': ['flag.evaluate', 'flag.list'], + 'rum-collector': ['rum.ingest'], + 'webhook-relay': ['webhook.dispatch', 'webhook.retry'], + 'metrics-pipeline': ['metrics.ingest', 'metrics.flush'], +}; + +// v6 — 6M base background traces (~12M+ spans w/ child) + 12M logs. +// FAILING_TRACE_COUNT is intentionally low so a naive +// `WHERE StatusCode='ERROR'` query returns the planted failures as a small +// minority of the total recent error count (most of which is decoy traffic). +const BACKGROUND_TRACE_COUNT = 6_000_000; +const BACKGROUND_LOG_COUNT = 12_000_000; +const FAILING_TRACE_COUNT = 8; + +const ANOMALY_WINDOW_MS = 5 * 60 * 1000; // last 5 min +const HISTORY_WINDOW_MS = 60 * 60 * 1000; + +const TLS_SPIKE_OFFSET_FROM_NOW_MS = 25 * 60 * 1000; // T-25min +const TLS_SPIKE_DURATION_MS = 5 * 60 * 1000; +const TLS_SPIKE_COUNT = 50; + +const AUTH_RATE_LIMIT_OFFSET_FROM_NOW_MS = 40 * 60 * 1000; // T-40min +const AUTH_RATE_LIMIT_DURATION_MS = 2 * 60 * 1000; +const AUTH_RATE_LIMIT_COUNT = 20; + +const SEARCH_SLOW_COUNT = 80; + +// Background "operational" payment.charge errors throughout the hour — these +// are NOT the planted infra incident. The decline reasons rotate so an agent +// blanket-grepping payment-service errors will get a noisy result that mixes +// legitimate user-level failures with the actual root cause. +const BACKGROUND_PAYMENT_ERROR_COUNT = 200; +const PAYMENT_DECLINE_REASONS = [ + 'card declined', + 'insufficient funds', + 'CVV mismatch', + 'expired card', + 'payment method not supported', +] as const; + +// Concurrent decoy: notification-service smtp errors in the SAME 5-min window +// as the planted cascade. Looks like a similar service-can't-reach-dependency +// pattern, but isn't part of any checkout-api trace — different traceids. +const SMTP_DECOY_COUNT = 25; + +// Concurrent decoy: cdn origin-fetch failures in the SAME 5-min window. Same +// shape as the smtp decoy (separate trace tree, no checkout-api parent) but +// at higher count, so a status-code-only query in the recent window returns +// far more cdn rows than planted-cascade rows. Forces the agent to do +// causal/cascade analysis instead of leaning on raw error counts. +const CDN_ORIGIN_DECOY_COUNT = 80; + +// Three variants of the planted timeout message — same root cause, resists +// naive exact-string matching. Counts sum to FAILING_TRACE_COUNT. +const TIMEOUT_VARIANTS = [ + { count: 3, body: 'connection timeout to db-payment' }, + { count: 3, body: 'timeout connecting to db-payment.internal' }, + { count: 2, body: 'db-payment unreachable after 5 retries' }, +] as const; + +type Rng = GenerateContext['rng']; + +function generateTlsSpike(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - TLS_SPIKE_OFFSET_FROM_NOW_MS; + for (let i = 0; i < TLS_SPIKE_COUNT; i++) { + const t = spreadTimestamp( + i, + TLS_SPIKE_COUNT, + startMs, + TLS_SPIKE_DURATION_MS, + 1000, + ); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'route.dispatch', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-gateway', + durationNs: msToNs(rng.range(50, 300)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'tls handshake timeout: peer reset', + spanAttributes: { + 'http.method': 'GET', + 'http.status_code': '525', + 'error.type': 'TLSHandshakeError', + 'tls.version': '1.3', + }, + }), + ); + } + return traces; +} + +function generateAuthRateLimitBlip(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - AUTH_RATE_LIMIT_OFFSET_FROM_NOW_MS; + for (let i = 0; i < AUTH_RATE_LIMIT_COUNT; i++) { + const t = spreadTimestamp( + i, + AUTH_RATE_LIMIT_COUNT, + startMs, + AUTH_RATE_LIMIT_DURATION_MS, + 1000, + ); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'auth.refresh', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'auth-service', + durationNs: msToNs(rng.range(8, 25)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'rate limit exceeded: 1000 req/min on /v2/token', + spanAttributes: { + 'http.method': 'POST', + 'http.status_code': '429', + 'error.type': 'RateLimitExceeded', + }, + }), + ); + } + return traces; +} + +function generateSearchSlowQueries(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + // Slow but successful — distractor for latency-style queries. + for (let i = 0; i < SEARCH_SLOW_COUNT; i++) { + const t = spreadTimestamp(i, SEARCH_SLOW_COUNT, startMs, HISTORY_WINDOW_MS); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'search.query', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'search-service', + durationNs: msToNs(rng.range(1500, 3000)), + statusCode: 'STATUS_CODE_OK', + spanAttributes: { + 'http.method': 'GET', + 'db.system': 'elasticsearch', + 'db.statement': + "SELECT * FROM products WHERE description LIKE '%query%'", + 'search.results_count': String(rng.intRange(0, 1000)), + }, + }), + ); + } + return traces; +} + +function generateBackgroundPaymentErrors(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + for (let i = 0; i < BACKGROUND_PAYMENT_ERROR_COUNT; i++) { + const t = spreadTimestamp( + i, + BACKGROUND_PAYMENT_ERROR_COUNT, + startMs, + HISTORY_WINDOW_MS, + ); + const reason = rng.pick(PAYMENT_DECLINE_REASONS); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'payment.charge', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'payment-service', + durationNs: msToNs(rng.range(50, 250)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: reason, + spanAttributes: { + 'rpc.system': 'grpc', + 'payment.amount_cents': String(rng.intRange(100, 50000)), + 'payment.method': rng.pick(['card', 'ach', 'wallet']), + 'error.type': 'PaymentDeclined', + 'decline.reason': reason, + }, + }), + ); + } + return traces; +} + +function generateSmtpDecoy( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const onsetMs = nowMs - ANOMALY_WINDOW_MS; + for (let i = 0; i < SMTP_DECOY_COUNT; i++) { + const t = spreadTimestamp( + i, + SMTP_DECOY_COUNT, + onsetMs, + ANOMALY_WINDOW_MS, + 5_000, + ); + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: 'notify.email', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'notification-service', + durationNs: msToNs(rng.range(800, 2000)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'smtp connection refused: smtp.internal:587', + spanAttributes: { + 'net.peer.name': 'smtp.internal', + 'net.peer.port': '587', + 'error.type': 'SMTPConnectionError', + 'email.recipient_count': '1', + }, + }), + ); + logs.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'notification-service', + severityText: 'ERROR', + body: 'smtp connection refused: smtp.internal:587', + traceId, + spanId, + logAttributes: { + 'error.kind': 'SMTPConnectionError', + component: 'notification-service', + }, + }), + ); + } + return { traces, logs }; +} + +function generateCdnOriginDecoy( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const onsetMs = nowMs - ANOMALY_WINDOW_MS; + for (let i = 0; i < CDN_ORIGIN_DECOY_COUNT; i++) { + const t = spreadTimestamp( + i, + CDN_ORIGIN_DECOY_COUNT, + onsetMs, + ANOMALY_WINDOW_MS, + 5_000, + ); + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: 'cdn.cache.fetch', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'cdn', + durationNs: msToNs(rng.range(150, 600)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'origin connection refused: origin-static.internal:443', + spanAttributes: { + 'http.method': 'GET', + 'http.status_code': '502', + 'net.peer.name': 'origin-static.internal', + 'net.peer.port': '443', + 'error.type': 'OriginUnreachableError', + 'cdn.edge_pop': rng.pick([ + 'edge-iad', + 'edge-sfo', + 'edge-fra', + 'edge-syd', + ]), + }, + }), + ); + logs.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'cdn', + severityText: 'ERROR', + body: 'origin connection refused: origin-static.internal:443', + traceId, + spanId, + logAttributes: { + 'error.kind': 'OriginUnreachableError', + component: 'cdn', + 'origin.host': 'origin-static.internal', + }, + }), + ); + } + return { traces, logs }; +} + +function generateAnomaly( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const onsetMs = nowMs - ANOMALY_WINDOW_MS; + const slots: { body: string }[] = []; + for (const v of TIMEOUT_VARIANTS) { + for (let k = 0; k < v.count; k++) slots.push({ body: v.body }); + } + // Shuffle slots deterministically using the seeded RNG. + for (let i = slots.length - 1; i > 0; i--) { + const j = rng.intRange(0, i + 1); + [slots[i], slots[j]] = [slots[j], slots[i]]; + } + + for (let i = 0; i < FAILING_TRACE_COUNT; i++) { + const t = spreadTimestamp( + i, + FAILING_TRACE_COUNT, + onsetMs, + ANOMALY_WINDOW_MS, + 5_000, + ); + const traceId = newTraceId(rng); + const variantBody = slots[i].body; + + const rootSpanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: rootSpanId, + spanName: 'POST /api/checkout', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'checkout-api', + durationNs: msToNs(rng.range(2200, 3200)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'payment_service_unavailable', + spanAttributes: { + 'http.method': 'POST', + 'http.route': '/api/checkout', + 'http.status_code': '500', + 'user.id': `user-${rng.intRange(1000, 9999)}`, + }, + }), + ); + + const midSpanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 5), + traceId, + spanId: midSpanId, + parentSpanId: rootSpanId, + spanName: 'payment.charge', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'payment-service', + durationNs: msToNs(rng.range(2000, 2900)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'db connection timeout', + spanAttributes: { + 'rpc.system': 'grpc', + 'payment.amount_cents': String(rng.intRange(500, 50000)), + 'error.type': 'ConnectionTimeoutError', + }, + }), + ); + + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(5, 15), + traceId, + spanId: newSpanId(rng), + parentSpanId: midSpanId, + spanName: 'db.payment.connect', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'payment-service', + durationNs: msToNs(rng.range(1900, 2500)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: variantBody, + spanAttributes: { + 'db.system': 'postgres', + 'db.name': 'payments', + 'db.statement': 'SELECT 1', + 'net.peer.name': 'db-payment.internal', + 'error.type': 'ConnectionTimeoutError', + }, + }), + ); + + logs.push( + makeLog({ + timestampMs: t + rng.intRange(10, 50), + serviceName: 'payment-service', + severityText: 'ERROR', + body: `${variantBody} (trace_id=${traceId})`, + traceId, + spanId: midSpanId, + logAttributes: { + 'error.kind': 'DBTimeout', + component: 'payment-service', + 'db.host': 'db-payment.internal', + }, + }), + ); + } + return { traces, logs }; +} + +export const errorRootCauseScenario: Scenario = { + name: 'error-root-cause', + agentPrompt: groundTruth.agentPrompt, + description: + 'payment-service DB connection timeout cascading into checkout-api 5xx, buried in 12M+ background spans + 12M templated log bodies + 5 distractor anomalies', + *generate(ctx): Iterable { + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const bgTraceCount = Math.max( + 50, + Math.round(BACKGROUND_TRACE_COUNT * factor), + ); + const bgLogCount = Math.max(50, Math.round(BACKGROUND_LOG_COUNT * factor)); + + const resourcePool = buildResourcePool({ + rng: ctx.rng, + services: BACKGROUND_SERVICES, + instancesPerService: 16, + }); + + yield* streamBackground( + ctx.rng, + ctx.nowMs, + bgTraceCount, + bgLogCount, + batchSize, + resourcePool, + ); + + // Distractors and the planted anomaly stay at fixed counts so the + // structural invariants remain identical regardless of volumeFactor. + const tls = generateTlsSpike(ctx.rng, ctx.nowMs); + const auth = generateAuthRateLimitBlip(ctx.rng, ctx.nowMs); + const slow = generateSearchSlowQueries(ctx.rng, ctx.nowMs); + const bgPayment = generateBackgroundPaymentErrors(ctx.rng, ctx.nowMs); + const smtp = generateSmtpDecoy(ctx.rng, ctx.nowMs); + const cdn = generateCdnOriginDecoy(ctx.rng, ctx.nowMs); + const anomaly = generateAnomaly(ctx.rng, ctx.nowMs); + yield { + traces: [ + ...tls, + ...auth, + ...slow, + ...bgPayment, + ...smtp.traces, + ...cdn.traces, + ...anomaly.traces, + ], + logs: [...smtp.logs, ...cdn.logs, ...anomaly.logs], + }; + }, + groundTruth, +}; + +type LogTemplateKind = + | 'page_render' + | 'envoy_access' + | 'analytics' + | 'catalog' + | 'heartbeat' + | 'background'; + +const LOG_TEMPLATE_MIX: { value: LogTemplateKind; weight: number }[] = [ + { value: 'page_render', weight: 25 }, + { value: 'envoy_access', weight: 15 }, + { value: 'analytics', weight: 15 }, + { value: 'catalog', weight: 15 }, + { value: 'heartbeat', weight: 10 }, + { value: 'background', weight: 20 }, +]; + +const LOG_TEMPLATE_SERVICE: Record = { + page_render: 'frontend', + envoy_access: 'api-gateway', + analytics: 'analytics-service', + catalog: 'image-service', + heartbeat: 'inventory-service', + background: 'session-service', +}; + +function* streamBackground( + rng: Rng, + nowMs: number, + traceCount: number, + logCount: number, + batchSize: number, + resourcePool: Record[]>, +) { + const bufTraces: TraceRow[] = []; + const bufLogs: LogRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + + for (let i = 0; i < traceCount; i++) { + const t = spreadTimestamp( + i, + traceCount, + startMs, + HISTORY_WINDOW_MS, + 60_000, + ); + const traceId = newTraceId(rng); + const parentService = rng.pick(BACKGROUND_SERVICES); + const parentOp = rng.pick(BACKGROUND_OPS[parentService]); + const parentSpanId = newSpanId(rng); + const isError = parentService !== 'payment-service' && rng.next() < 0.005; + const region = rng.pick(REGIONS); + const tenantId = `tenant-${rng.intRange(1, 5000)}`; + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + bufTraces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: parentSpanId, + spanName: parentOp, + spanKind: 'SPAN_KIND_SERVER', + serviceName: parentService, + durationNs: msToNs(rng.range(15, 200)), + statusCode: isError ? 'STATUS_CODE_ERROR' : 'STATUS_CODE_OK', + resourceAttributes: pickResource(rng, resourcePool, parentService), + spanAttributes: { + 'http.method': 'GET', + 'http.status_code': isError ? '500' : '200', + 'http.route': `/internal/${parentOp.replace(/[^a-z0-9]+/gi, '/')}`, + 'cloud.region': region, + 'tenant.id': tenantId, + 'user.id': userId, + 'session.id': sessionId, + 'request.id': requestId, + 'rpc.system': 'http', + }, + }), + ); + if (rng.next() < 0.6) { + const candidates = BACKGROUND_SERVICES.filter(s => s !== parentService); + const childService = rng.pick(candidates); + const childOp = rng.pick(BACKGROUND_OPS[childService]); + bufTraces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 10), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: childOp, + spanKind: 'SPAN_KIND_CLIENT', + serviceName: childService, + durationNs: msToNs(rng.range(5, 80)), + statusCode: 'STATUS_CODE_OK', + resourceAttributes: pickResource(rng, resourcePool, childService), + spanAttributes: { + 'rpc.system': 'http', + 'cloud.region': region, + 'request.id': requestId, + 'user.id': userId, + 'net.peer.name': `${childService}.svc.cluster.local`, + }, + }), + ); + } + if (bufTraces.length >= batchSize) { + yield { traces: bufTraces.splice(0, bufTraces.length), logs: [] }; + } + } + if (bufTraces.length) { + yield { traces: bufTraces.splice(0, bufTraces.length), logs: [] }; + } + + for (let i = 0; i < logCount; i++) { + const t = spreadTimestamp(i, logCount, startMs, HISTORY_WINDOW_MS, 60_000); + const kind = rng.weightedPick(LOG_TEMPLATE_MIX); + const service = LOG_TEMPLATE_SERVICE[kind]; + const sev = pickSeverityIn( + rng, + kind === 'heartbeat' + ? 'debug' + : rng.weightedPick<'info' | 'warn' | 'error'>([ + { value: 'info', weight: 88 }, + { value: 'warn', weight: 9 }, + { value: 'error', weight: 3 }, + ]), + ); + let body: string; + let attrs: Record; + if (kind === 'page_render') { + const tmpl = pageRenderLog({ rng, nowMs: t }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'envoy_access') { + const tmpl = envoyAccessLog({ rng, nowMs: t }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'analytics') { + const tmpl = analyticsEventLog({ rng, nowMs: t }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'catalog') { + const tmpl = catalogLog({ rng, nowMs: t, level: sev.text.toLowerCase() }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'heartbeat') { + const tmpl = heartbeatLog({ rng, nowMs: t, serviceName: service }); + body = tmpl.body; + attrs = tmpl.attrs; + } else { + body = backgroundLog({ rng, nowMs: t }); + attrs = { + 'event.name': 'app.background', + 'log.iostream': 'stdout', + logtag: 'F', + }; + } + bufLogs.push( + makeLog({ + timestampMs: t, + serviceName: service, + severityText: normalizeSeverityText(sev.text), + body, + resourceAttributes: pickResource(rng, resourcePool, service), + logAttributes: { ...attrs, _severity_raw: sev.text }, + }), + ); + if (bufLogs.length >= batchSize) { + yield { traces: [], logs: bufLogs.splice(0, bufLogs.length) }; + } + } + if (bufLogs.length) yield { traces: [], logs: bufLogs }; +} diff --git a/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json b/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json new file mode 100644 index 0000000000..8b9fa25a9b --- /dev/null +++ b/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json @@ -0,0 +1,173 @@ +{ + "scenario": "error-root-cause", + "agentPrompt": "Checkout requests are failing for some users in the last 10 minutes. Find the root cause.", + "expected": { + "rootCauseService": "payment-service", + "rootCauseOperation": "payment.charge", + "downstreamFailure": "db.payment.connect", + "errorBodyPattern": "db-payment", + "errorMessageVariants": [ + "connection timeout to db-payment", + "timeout connecting to db-payment.internal", + "db-payment unreachable after 5 retries" + ], + "userFacingService": "checkout-api", + "userFacingOperation": "POST /api/checkout", + "onsetMinutesAgo": 5, + "approxFailingTraces": 8 + }, + "distractors": [ + { + "kind": "api-gateway TLS handshake spike", + "service": "api-gateway", + "windowMinutesAgo": [25, 20], + "approxCount": 50, + "errorMessage": "tls handshake timeout: peer reset", + "whyNotRootCause": "Resolved 20 minutes ago; unrelated to current checkout failures." + }, + { + "kind": "auth-service rate-limit blip", + "service": "auth-service", + "windowMinutesAgo": [40, 38], + "approxCount": 20, + "errorMessage": "rate limit exceeded: 1000 req/min on /v2/token", + "whyNotRootCause": "Resolved 38 minutes ago; not in the current incident window." + }, + { + "kind": "search-service slow queries", + "service": "search-service", + "windowMinutesAgo": "throughout-hour", + "approxCount": 80, + "whyNotRootCause": "Status OK, not errors. Slow but successful. Unrelated to checkout cascade." + }, + { + "kind": "background payment.charge declines", + "service": "payment-service", + "spanName": "payment.charge", + "windowMinutesAgo": "throughout-hour", + "approxCount": 200, + "errorTypes": [ + "card declined", + "insufficient funds", + "CVV mismatch", + "expired card", + "payment method not supported" + ], + "whyNotRootCause": "Operational user-level declines, not infrastructure failures. error.type='PaymentDeclined' (not ConnectionTimeoutError). Distributed throughout the hour, not concentrated in the recent window." + }, + { + "kind": "notification-service smtp failure (concurrent)", + "service": "notification-service", + "spanName": "notify.email", + "windowMinutesAgo": [5, 0], + "approxCount": 25, + "errorMessage": "smtp connection refused: smtp.internal:587", + "errorType": "SMTPConnectionError", + "whyNotRootCause": "Concurrent but separate incident — these traces have no checkout-api parent and don't appear in any checkout trace tree. Different dependency (smtp.internal vs db-payment.internal)." + }, + { + "kind": "cdn origin-fetch burst (concurrent)", + "service": "cdn", + "spanName": "cdn.cache.fetch", + "windowMinutesAgo": [5, 0], + "approxCount": 80, + "errorMessage": "origin connection refused: origin-static.internal:443", + "errorType": "OriginUnreachableError", + "whyNotRootCause": "Concurrent recent 5xx spike, but on cdn traces with no checkout-api parent. Looks superficially similar to the planted cascade (service-can't-reach-dependency) but lives in entirely separate trace trees and has no link to payment-service or checkout-api. Status-code-only queries will return these alongside the planted failures." + } + ], + "anomalyAttributes": { + "spanAttributes": { + "db.system": "postgres", + "db.name": "payments", + "error.type": "ConnectionTimeoutError", + "net.peer.name": "db-payment.internal" + }, + "logAttributes": { + "error.kind": "DBTimeout", + "component": "payment-service" + } + }, + "rubric": { + "programmatic": [ + ["names_root_cause_service", 2, "payment-service"], + [ + "names_underlying_failure", + 2, + "(connection timeout|db connection|connect.*timeout|timeout.*connect|unreachable)" + ], + ["names_user_facing_service", 1, "checkout(-api)?"], + ["names_db_target", 1, "db-payment"], + [ + "names_error_type", + 2, + "(ConnectionTimeoutError|connection.?timeout|DBTimeout|error\\.type[^a-z]+(connection|db))" + ], + ["names_db_host_fqdn", 1, "db-payment\\.internal"], + [ + "names_specific_db_span", + 1, + "(db\\.payment\\.connect|payment\\.charge)" + ], + [ + "ruled_out_a_distractor", + 2, + "(rul(ed|ing) out|not (the )?(root )?cause|concurrent (but )?(separate|unrelated)|distractor|separate (incident|trace tree|burst))[\\s\\S]{0,250}?\\b(tls|api-gateway|auth-service|rate.?limit|smtp|notification-service|cdn|origin|card declin|insufficient funds|search-service)" + ], + [ + "false_blame_tls_handshake", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(tls handshake|api-gateway[^.\\n]{0,40}?(tls|handshake|peer reset))", + true + ], + [ + "false_blame_auth_rate_limit", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(rate.?limit|auth-service[^.\\n]{0,40}?(rate.?limit|429|throttl))", + true + ], + [ + "false_blame_smtp", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(smtp|notification-service[^.\\n]{0,40}?(smtp|email))", + true + ], + [ + "false_blame_payment_decline", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(card declined|insufficient funds|cvv mismatch|expired card|payment.?declined)", + true + ], + [ + "false_blame_cdn_origin", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(cdn|origin[^.\\n]{0,40}?(refused|unreachable|timeout)|cache.?miss[^.\\n]{0,40}?upstream)", + true + ] + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify payment-service as the root-cause service AND the DB connection timeout to db-payment as the underlying failure? The error message has 3 variants — recognizing them as one cause counts." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did it explain the full failure chain — payment-service db.payment.connect timeout → payment.charge fails → checkout-api 5xx — rather than just naming one layer?" + }, + { + "id": "calibration", + "weight": 2, + "description": "Did the answer correctly distinguish the planted timeout cascade from the distractors? Specifically: (a) Resolved historical incidents — api-gateway TLS spike at T-25min, auth-service rate-limit blip at T-40min — are NOT the current cause. (b) search-service slow queries are OK status, not errors. (c) payment-service payment.charge declines (card declined/insufficient funds) are operational user-level failures, NOT the infrastructure incident. (d) notification-service smtp errors and cdn origin-fetch errors are CONCURRENT bursts in the same window but live on separate traces with no checkout-api parent — not the cause of checkout failures. Status-code-only queries will surface all of these together; the agent must trace the causal chain rather than rely on raw error counts. Mistaking any of these for the root cause is a calibration failure." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and free of filler or unnecessary speculation?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/index.ts b/packages/hdx-eval/src/scenarios/index.ts new file mode 100644 index 0000000000..6387da339e --- /dev/null +++ b/packages/hdx-eval/src/scenarios/index.ts @@ -0,0 +1,26 @@ +import { errorRootCauseScenario } from './error-root-cause/generate'; +import { latencySpikeScenario } from './latency-spike/generate'; +import { noisySignalsScenario } from './noisy-signals/generate'; +import { segmentedRegressionScenario } from './segmented-regression/generate'; +import { serviceHealthCheckScenario } from './service-health-check/generate'; +import type { Scenario } from './types'; + +export const SCENARIOS: Record = { + [errorRootCauseScenario.name]: errorRootCauseScenario, + [latencySpikeScenario.name]: latencySpikeScenario, + [noisySignalsScenario.name]: noisySignalsScenario, + [segmentedRegressionScenario.name]: segmentedRegressionScenario, + [serviceHealthCheckScenario.name]: serviceHealthCheckScenario, +}; + +export const SCENARIO_NAMES = Object.keys(SCENARIOS); + +export function getScenario(name: string): Scenario { + const s = SCENARIOS[name]; + if (!s) { + throw new Error( + `Unknown scenario: ${name}. Known: ${SCENARIO_NAMES.join(', ')}`, + ); + } + return s; +} diff --git a/packages/hdx-eval/src/scenarios/latency-spike/generate.ts b/packages/hdx-eval/src/scenarios/latency-spike/generate.ts new file mode 100644 index 0000000000..42d1700da3 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/latency-spike/generate.ts @@ -0,0 +1,554 @@ +import { + buildResourcePool, + pickResource, + spreadTimestamp, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +// v6 — 6M base traces × ~2 child spans + 4M background traces × 2 spans +// + extras → ~20M+ total spans. Background traffic plants ~100 services +// × ~15 operations each so the (ServiceName, SpanName) cardinality is +// production-shaped (~1500 distinct combos) and naive +// `GROUP BY ServiceName, SpanName` returns too many rows to scan. +const TOTAL_TRACES = 6_000_000; +const BACKGROUND_TRACES = 4_000_000; +const HISTORY_WINDOW_MS = 60 * 60 * 1000; +const ANOMALY_WINDOW_MS = 10 * 60 * 1000; + +const ENDPOINTS = [ + { name: 'GET /api/orders/search', weight: 12 }, + { name: 'GET /api/orders/{id}', weight: 18 }, + { name: 'POST /api/orders', weight: 6 }, + { name: 'GET /api/products/search', weight: 10 }, + { name: 'GET /api/products/{id}', weight: 12 }, + { name: 'GET /api/inventory/{sku}', weight: 8 }, + { name: 'POST /api/cart/items', weight: 8 }, + { name: 'GET /api/cart/{id}', weight: 8 }, + { name: 'GET /api/users/{id}', weight: 6 }, + { name: 'POST /api/checkout', weight: 4 }, + { name: 'GET /api/recommendations', weight: 4 }, + { name: 'GET /api/health', weight: 4 }, +]; + +const REGIONS = [ + 'us-east-1', + 'us-west-2', + 'eu-west-1', + 'ap-south-1', + 'sa-east-1', +] as const; +const TENANT_RANGE = 5_000; // was 200 + +const TENANT_TIERS = [ + { value: 'free', weight: 60 }, + { value: 'pro', weight: 35 }, + { value: 'enterprise', weight: 5 }, +] as const; + +// Normal-traffic latency distribution. Each tuple is (lo_ms, hi_ms, weight). +const NORMAL_LATENCY_BUCKETS = [ + { lo: 25, hi: 80, weight: 65 }, + { lo: 80, hi: 180, weight: 25 }, + { lo: 180, hi: 350, weight: 8 }, + { lo: 350, hi: 600, weight: 2 }, +]; + +// Distractors +const GC_PAUSE_RATE = 0.003; // 0.3% of all traces — uniform across tiers/endpoints +const CACHE_LOOKUP_RATE = 0.03; // 3% of traces add an extra cache.lookup child span (200-400ms) + +const COLD_START_WINDOW_END_MS = 55 * 60 * 1000; // T-55min +const COLD_START_WINDOW_START_MS = 60 * 60 * 1000; // T-60min +const COLD_START_RATE_IN_WINDOW = 0.4; // 40% of traces in the early window are cold-start + +// Anomaly +const ENTERPRISE_CACHE_HIT_RATE = 0.2; // 20% of enterprise window requests still fast (cache hit) + +// Concurrent decoy: GET /api/products/search has its own slowness in the +// same window, all tiers, caused by an elasticsearch hot shard. Real, but +// unrelated to the orders/search × enterprise × missing-index issue. +const PRODUCTS_SEARCH_HOT_SHARD_RATE = 0.5; // 50% of products/search calls in window are slow + +// Confounder: feature_flag.experiment_a is enabled at very different rates +// across tiers — naive correlation analysis would suggest the flag causes +// the slowness, but the actual cause is the missing index on tenant_id. +const FEATURE_FLAG_RATE_BY_TIER: Record = { + enterprise: 0.8, + pro: 0.3, + free: 0.3, +}; + +type Rng = GenerateContext['rng']; + +function pickNormalLatencyMs(rng: Rng): number { + const bucket = rng.weightedPick( + NORMAL_LATENCY_BUCKETS.map(b => ({ value: b, weight: b.weight })), + ); + return rng.range(bucket.lo, bucket.hi); +} + +const SERVICE_POOL_LIST = [ + 'api-server', + 'database', + 'search-index', + 'cache', +] as const; + +// ─── Background traffic — production-shaped cardinality ───────────────────── +// +// In production environments a single application emits traces from dozens of +// services with thousands of distinct (ServiceName, SpanName) combinations. +// To prevent agents from trivially solving the eval via +// `GROUP BY ServiceName, SpanName ORDER BY p99 DESC LIMIT 20`, we plant a +// pool of ~100 background services × ~15 operations each into the trace +// stream alongside the focal incident pattern. Background traces have +// normal-distribution latency (no spikes) and zero overlap with the +// planted/decoy span names, so they don't confound the rubric — they just +// raise the noise floor of group-by enumerations. + +const BG_SERVICE_PREFIXES = [ + 'auth', + 'billing', + 'payments', + 'notifications', + 'emails', + 'sms', + 'user-profile', + 'user-prefs', + 'user-events', + 'analytics', + 'metrics', + 'tracing', + 'logging', + 'audit', + 'compliance', + 'gdpr', + 'pii-scan', + 'feature-flag', + 'config', + 'secrets', + 'kms', + 'queue', + 'worker', + 'cron', + 'scheduler', + 'retry', + 'ml-inference', + 'recommendation', + 'personalization', + 'fraud', + 'risk', + 'rate-limiter', + 'throttle', + 'cdn', + 'image-resizer', + 'thumbnailer', + 'media', + 'webhook', + 'integration', + 'partner-api', + 'graph', + 'recommender', + 'feed', + 'timeline', + 'shipping', + 'tax', + 'currency', + 'i18n', + 'inventory-sync', + 'price-sync', + 'catalog-sync', +] as const; +const BG_SERVICE_SUFFIXES = ['v1', 'v2', 'worker', 'api', 'svc'] as const; +const BG_OPERATIONS = [ + 'GET /health', + 'GET /ready', + 'GET /metrics', + 'GET /v1/get', + 'POST /v1/create', + 'PUT /v1/update', + 'DELETE /v1/delete', + 'GET /v1/list', + 'GET /v1/search', + 'POST /v1/batch', + 'http.send', + 'http.receive', + 'grpc.call', + 'rpc.invoke', + 'db.query', + 'db.write', + 'db.connect', + 'cache.get', + 'cache.set', + 'cache.evict', + 'queue.publish', + 'queue.consume', + 'queue.ack', + 'compute', + 'transform', + 'serialize', + 'validate', + 'render', + 'enrich', + 'normalize', + 'aggregate', +] as const; +const BG_CHILD_OPERATIONS = [ + 'db.select', + 'db.insert', + 'db.update', + 'cache.lookup', + 'cache.store', + 'http.dependency', + 'grpc.dependency', + 'queue.fetch', + 'queue.send', + 'compute.task', +] as const; +const BG_SERVICE_COUNT = 100; +const BG_OPS_PER_SERVICE = 15; + +type BgService = { + name: string; + operations: readonly string[]; + childOps: readonly string[]; +}; + +function buildBackgroundServices(rng: Rng): BgService[] { + const services: BgService[] = []; + const used = new Set(); + // Cap at BG_SERVICE_COUNT distinct names. + let attempts = 0; + while (services.length < BG_SERVICE_COUNT && attempts < 10_000) { + attempts++; + const prefix = rng.pick(BG_SERVICE_PREFIXES); + const suffix = rng.pick(BG_SERVICE_SUFFIXES); + const name = `${prefix}-${suffix}`; + if (used.has(name)) continue; + used.add(name); + // Each service emits ~BG_OPS_PER_SERVICE distinct operations. + const ops: string[] = []; + const opsUsed = new Set(); + for (let i = 0; i < BG_OPS_PER_SERVICE; i++) { + const o = rng.pick(BG_OPERATIONS); + if (!opsUsed.has(o)) { + opsUsed.add(o); + ops.push(o); + } + } + services.push({ + name, + operations: ops, + childOps: rng.pick([ + BG_CHILD_OPERATIONS.slice(0, 3), + BG_CHILD_OPERATIONS.slice(2, 6), + BG_CHILD_OPERATIONS.slice(5, 10), + ]), + }); + } + return services; +} + +export const latencySpikeScenario: Scenario = { + name: 'latency-spike', + agentPrompt: groundTruth.agentPrompt, + description: + 'api-server p99 spike on /api/orders/search for enterprise tenants (missing-index DB query). 12M+ spans across 12 endpoints, 5 regions, 5K tenants, with GC-pause / cold-start / cache-lookup / hot-shard distractors.', + *generate(ctx): Iterable { + const { rng, nowMs } = ctx; + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const totalTraces = Math.max(50, Math.round(TOTAL_TRACES * factor)); + const resourcePool = buildResourcePool({ + rng, + services: SERVICE_POOL_LIST, + instancesPerService: 24, + }); + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + const anomalyStartMs = nowMs - ANOMALY_WINDOW_MS; + + for (let i = 0; i < totalTraces; i++) { + const t = spreadTimestamp(i, totalTraces, startMs, HISTORY_WINDOW_MS); + const traceId = newTraceId(rng); + const endpoint = rng.weightedPick( + ENDPOINTS.map(e => ({ value: e.name, weight: e.weight })), + ); + const tenantTier = rng.weightedPick( + TENANT_TIERS.map(t => ({ value: t.value, weight: t.weight })), + ); + const tenantId = `${tenantTier}-${rng.intRange(1, TENANT_RANGE)}`; + const region = rng.pick(REGIONS); + + const inAnomalyWindow = t >= anomalyStartMs; + const isAffectedSegment = + inAnomalyWindow && + endpoint === 'GET /api/orders/search' && + tenantTier === 'enterprise'; + const isCacheHit = + isAffectedSegment && rng.next() < ENTERPRISE_CACHE_HIT_RATE; + const isSlowAnomaly = isAffectedSegment && !isCacheHit; + + // Concurrent decoy: products/search hot-shard slowness in the SAME + // window, all tiers. A real incident on a different endpoint — + // calibration test for not conflating with the orders/search anomaly. + const isProductsSearchHotShard = + inAnomalyWindow && + endpoint === 'GET /api/products/search' && + rng.next() < PRODUCTS_SEARCH_HOT_SHARD_RATE; + + // Distractors only fire when not the planted anomaly nor the + // products/search decoy (so duration profiles don't overlap). + const inColdStartWindow = + t >= nowMs - COLD_START_WINDOW_START_MS && + t <= nowMs - COLD_START_WINDOW_END_MS; + const isColdStart = + !isAffectedSegment && + !isProductsSearchHotShard && + inColdStartWindow && + rng.next() < COLD_START_RATE_IN_WINDOW; + const isGcPause = + !isAffectedSegment && + !isProductsSearchHotShard && + !isColdStart && + rng.next() < GC_PAUSE_RATE; + + // Feature-flag confounder — correlated with tier but not the cause. + const flagRate = FEATURE_FLAG_RATE_BY_TIER[tenantTier] ?? 0.3; + const flagOn = rng.next() < flagRate; + + let dbDurationMs: number; + let parentDurationMs: number; + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + const route = endpoint.replace(/^[A-Z]+ /, ''); + const queryParam = + endpoint.includes('search') || endpoint.includes('?') + ? `?q=${rng.hex(4)}&page=${rng.intRange(1, 50)}&limit=${rng.intRange(10, 100)}` + : ''; + const parentAttrs: Record = { + 'http.method': endpoint.startsWith('POST') ? 'POST' : 'GET', + 'http.route': route, + 'http.target': `${route}${queryParam}`, + 'url.path': route, + 'url.full': `https://api.example.com${route}${queryParam}`, + 'url.scheme': 'https', + 'http.status_code': '200', + 'http.request.body.size': String(rng.intRange(0, 4096)), + 'http.response.body.size': String(rng.intRange(0, 32768)), + 'tenant.id': tenantId, + 'tenant.tier': tenantTier, + 'user.id': userId, + 'session.id': sessionId, + 'cloud.region': region, + 'feature_flag.experiment_a': flagOn ? 'enabled' : 'disabled', + 'request.id': requestId, + 'net.peer.name': 'api-server.svc.cluster.local', + 'rpc.system': 'http', + }; + const childAttrs: Record = { + 'db.system': 'postgres', + 'db.name': 'orders', + 'tenant.id': tenantId, + 'tenant.tier': tenantTier, + 'user.id': userId, + 'request.id': requestId, + 'net.peer.name': 'db-orders-primary.internal', + 'net.peer.port': '5432', + }; + let childSpanName = 'database.query'; + let childServiceName = 'database'; + + if (isSlowAnomaly) { + dbDurationMs = rng.range(1500, 2500); + parentDurationMs = dbDurationMs + rng.range(15, 35); + childAttrs['db.statement'] = 'SELECT * FROM orders WHERE tenant_id = ?'; + } else if (isProductsSearchHotShard) { + dbDurationMs = rng.range(1500, 2500); + parentDurationMs = dbDurationMs + rng.range(15, 35); + childServiceName = 'search-index'; + childSpanName = 'elasticsearch.query'; + childAttrs['db.system'] = 'elasticsearch'; + childAttrs['db.name'] = 'products'; + childAttrs['db.statement'] = 'POST /products/_search'; + childAttrs['elasticsearch.shard.hot'] = 'true'; + childAttrs['elasticsearch.shard.id'] = '17'; + } else if (isCacheHit) { + // Enterprise cache-hit subset — still fast despite being in window. + dbDurationMs = rng.range(2, 8); + parentDurationMs = rng.range(8, 30); + childAttrs['cache.hit'] = 'true'; + childAttrs['db.statement'] = + 'SELECT * FROM orders WHERE tenant_id = ? -- cached'; + } else if (isGcPause) { + dbDurationMs = rng.range(20, 60); + parentDurationMs = rng.range(1500, 2200); + parentAttrs['runtime.event'] = 'gc_pause'; + parentAttrs['runtime.gc.duration_ms'] = String( + Math.round(parentDurationMs - dbDurationMs), + ); + } else if (isColdStart) { + dbDurationMs = rng.range(20, 60); + parentDurationMs = rng.range(2000, 3000); + parentAttrs['runtime.event'] = 'cold_start'; + parentAttrs['runtime.cold_start_ms'] = String( + Math.round(parentDurationMs - dbDurationMs), + ); + } else { + dbDurationMs = rng.range(5, 50); + parentDurationMs = pickNormalLatencyMs(rng); + // Endpoint-appropriate db statement + if (endpoint.startsWith('GET /api/products')) { + childAttrs['db.name'] = 'products'; + childAttrs['db.statement'] = 'SELECT * FROM products WHERE id = ?'; + } else if (endpoint.startsWith('GET /api/inventory')) { + childAttrs['db.name'] = 'inventory'; + childAttrs['db.statement'] = 'SELECT * FROM inventory WHERE sku = ?'; + } else { + childAttrs['db.statement'] = 'SELECT * FROM orders WHERE id = ?'; + } + } + + const parentSpanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: parentSpanId, + spanName: endpoint, + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-server', + durationNs: msToNs(parentDurationMs), + resourceAttributes: pickResource(rng, resourcePool, 'api-server'), + spanAttributes: parentAttrs, + }), + ); + + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 5), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: childSpanName, + spanKind: 'SPAN_KIND_CLIENT', + serviceName: childServiceName, + durationNs: msToNs(dbDurationMs), + resourceAttributes: pickResource(rng, resourcePool, childServiceName), + spanAttributes: childAttrs, + }), + ); + + // 3% of traces also get a cache.lookup child (200-400ms) — adds tail + // latency that an agent might mistake for the spike. + if (!isAffectedSegment && rng.next() < CACHE_LOOKUP_RATE) { + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(2, 8), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: 'cache.lookup', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'cache', + durationNs: msToNs(rng.range(200, 400)), + resourceAttributes: pickResource(rng, resourcePool, 'cache'), + spanAttributes: { + 'cache.system': 'redis', + 'cache.key': `cache:${rng.hex(8)}`, + 'cache.hit': rng.next() < 0.7 ? 'true' : 'false', + 'request.id': requestId, + 'net.peer.name': 'redis-cache.internal', + 'net.peer.port': '6379', + }, + }), + ); + } + // Stream traces in batches to avoid OOM at high volume. + if (traces.length >= batchSize) { + yield { traces: traces.splice(0, traces.length), logs: [] }; + } + } + + // ── Background traffic ──────────────────────────────────────────────── + // Production-shaped cardinality: ~100 services × ~15 operations each. + // Normal latency, no slow tails, no overlap with planted span names. + // Builds a pool from the same rng so the result is deterministic for a + // given seed. + const bgServices = buildBackgroundServices(rng); + const bgResourcePool = buildResourcePool({ + rng, + services: bgServices.map(s => s.name), + instancesPerService: 4, + }); + const bgTotal = Math.max(0, Math.round(BACKGROUND_TRACES * factor)); + for (let i = 0; i < bgTotal; i++) { + const t = spreadTimestamp(i, bgTotal, startMs, HISTORY_WINDOW_MS); + const svc = bgServices[i % bgServices.length]; + const op = rng.pick(svc.operations); + const childOp = rng.pick(svc.childOps); + const traceId = newTraceId(rng); + const parentSpanId = newSpanId(rng); + const parentDurationMs = pickNormalLatencyMs(rng); + const childDurationMs = Math.round( + parentDurationMs * rng.range(0.1, 0.7), + ); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: parentSpanId, + spanName: op, + spanKind: 'SPAN_KIND_SERVER', + serviceName: svc.name, + durationNs: msToNs(parentDurationMs), + resourceAttributes: pickResource(rng, bgResourcePool, svc.name), + spanAttributes: { + 'http.status_code': '200', + 'rpc.system': op.includes('grpc') ? 'grpc' : 'http', + }, + }), + ); + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 5), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: childOp, + spanKind: 'SPAN_KIND_CLIENT', + serviceName: svc.name, + durationNs: msToNs(childDurationMs), + resourceAttributes: pickResource(rng, bgResourcePool, svc.name), + spanAttributes: { + 'net.peer.name': `${svc.name}.internal`, + }, + }), + ); + if (traces.length >= batchSize) { + yield { traces: traces.splice(0, traces.length), logs: [] }; + } + } + + if (traces.length || logs.length) { + yield { traces, logs }; + } + }, + groundTruth, +}; diff --git a/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json b/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json new file mode 100644 index 0000000000..bd2b93bced --- /dev/null +++ b/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json @@ -0,0 +1,136 @@ +{ + "scenario": "latency-spike", + "agentPrompt": "p99 latency on api-server has jumped in the last 15 minutes. What's slow and why?", + "expected": { + "affectedService": "api-server", + "affectedOperation": "GET /api/orders/search", + "slowChildService": "database", + "slowChildOperation": "database.query", + "segmentAttribute": "tenant.tier", + "segmentValue": "enterprise", + "onsetMinutesAgo": 10, + "baselineP99Ms": 600, + "anomalyP99Ms": 2200, + "cacheHitRateInWindow": 0.2, + "endpointsInWorld": [ + "GET /api/orders/search", + "GET /api/orders/{id}", + "POST /api/orders", + "GET /api/products/search", + "GET /api/inventory/{sku}" + ] + }, + "distractors": [ + { + "kind": "GC pause distractor", + "approxRate": 0.003, + "marker": "runtime.event=gc_pause", + "durationMs": [1500, 2200], + "whyNotRootCause": "Uniform across tenants and endpoints — not concentrated on enterprise GET /api/orders/search." + }, + { + "kind": "Cold-start spike", + "windowMinutesAgo": [60, 55], + "marker": "runtime.event=cold_start", + "durationMs": [2000, 3000], + "whyNotRootCause": "Bounded to the first 5 minutes of the hour; not present in the recent window." + }, + { + "kind": "cache.lookup latency tail", + "approxRate": 0.03, + "spanName": "cache.lookup", + "durationMs": [200, 400], + "whyNotRootCause": "Adds tail latency but cap is well below the 1500ms+ spike." + }, + { + "kind": "products/search elasticsearch hot shard (concurrent)", + "endpoint": "GET /api/products/search", + "approxRate": 0.5, + "marker": "elasticsearch.shard.hot=true", + "windowMinutesAgo": [10, 0], + "durationMs": [1500, 2500], + "whyNotRootCause": "Real but unrelated incident on a different endpoint with a different root cause (hot shard, not missing index). Affects all tiers, not enterprise-specific." + }, + { + "kind": "feature_flag.experiment_a confounder", + "marker": "feature_flag.experiment_a=enabled", + "tierCorrelation": { "enterprise": 0.8, "pro": 0.3, "free": 0.3 }, + "whyNotRootCause": "Highly correlated with enterprise tier (because experiment was rolled out there first), so naive correlation analysis would suggest the flag causes the slowness. The flag is also enabled on many pro/free traces that are NOT slow — so it's a correlation, not a cause. The actual cause is the missing index on tenant_id." + } + ], + "anomalyAttributes": { + "spanAttributes": { + "db.system": "postgres", + "db.name": "orders", + "db.statement": "SELECT * FROM orders WHERE tenant_id = ?" + }, + "rootCauseHint": "missing index on orders.tenant_id" + }, + "rubric": { + "programmatic": [ + ["names_affected_service", 1, "api-server"], + ["names_affected_endpoint", 1, "(/api/orders/search|orders/search)"], + ["names_slow_child", 2, "(database\\.query|SELECT.*FROM\\s+orders)"], + ["names_segment", 2, "(enterprise|tenant\\.tier)"], + [ + "names_index_or_query_root_cause", + 1, + "(index|tenant_id|missing.*index)" + ], + [ + "false_blame_hot_shard", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is|main issue)[^.\\n]{0,80}?\\b(hot.shard|elasticsearch.shard.hot|shard.{0,30}?hot)", + true + ], + [ + "false_blame_products_search", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bproducts.search", + true + ], + [ + "false_blame_feature_flag", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(feature.flag|experiment_a)", + true + ], + [ + "false_blame_gc_pause", + 1, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bgc.pause", + true + ], + [ + "false_blame_cold_start", + 1, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcold.start", + true + ] + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify GET /api/orders/search on api-server as affected AND localize the slow path to the database.query child for enterprise tenants? With 5 endpoints in play, naming the wrong endpoint is a correctness failure." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did it surface the segment (tenant.tier=enterprise), describe how p99 changes (only for enterprise on /api/orders/search, other tiers and endpoints unchanged), and propose a plausible root cause (missing index on tenant_id or equivalent)?" + }, + { + "id": "calibration", + "weight": 2, + "description": "Did the answer avoid mistaking distractors for the cause? Specifically: (a) GC pauses are uniform across tiers and would not concentrate on enterprise. (b) cold starts only appear in the first 5 minutes of the window. (c) cache.lookup latency caps at 400ms, well below the 1500ms+ spike. (d) GET /api/products/search has its own concurrent slowness (hot shard, all tiers) — that is a SEPARATE incident from the enterprise orders/search issue, not the same cause. (e) feature_flag.experiment_a is correlated with enterprise tier (~80%) but is also enabled on many free/pro traces that are NOT slow — it is a confounder, not a cause. Blaming the flag, conflating products/search with orders/search, or claiming a single root cause for both endpoint spikes is a calibration failure." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and free of filler?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts b/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts new file mode 100644 index 0000000000..7014841cae --- /dev/null +++ b/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts @@ -0,0 +1,415 @@ +import { makeLog } from '../../generators/logs'; +import { + analyticsEventLog, + backgroundLog, + billingEventLog, + buildResourcePool, + cacheHitLog, + catalogLog, + caughtExceptionLog, + consoleLogDumpLog, + envoyAccessLog, + heartbeatLog, + indexingLog, + jobFailedPermanentLog, + normalizeSeverityText, + notificationDeliveryLog, + pageRenderLog, + pickResource, + pickSeverityIn, + searchCacheMissVerboseLog, + serviceOpsDebugLog, + spreadTimestamp, + subscriptionMetricLog, + upstreamHealthProbeLog, +} from '../../generators/templates'; +import type { LogRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +const HISTORY_WINDOW_MS = 60 * 60 * 1000; +const DEFAULT_BATCH_SIZE = 10_000; + +// Composite-cell design: every (ServiceName, SeverityText) cell that holds a +// noisy pattern also holds a high-volume LEGITIMATE pattern of comparable +// size. `GROUP BY ServiceName, SeverityText` no longer identifies noise — +// the agent has to look at event.name / body templates. +// +// Total ≈ 16M logs. +// +// cell NOISE LOAD-BEARING +// notification-service × DEBUG cache.hit notification.delivery +// billing-service × INFO subscription_mrr-log invoice / refund / sub events +// worker × ERROR NonFatalRetryableError job.failed.permanent +// {inv,pri,ship,rec}-svc × DEBUG health.check per-svc ops debug +// frontend-proxy × INFO lb.health.probe envoy access +// frontend × INFO console.log_dump page.render +// search-service × INFO search.cache.miss index.shard.flush +const DEFAULTS = { + // Composite cell 1 (notification-service × DEBUG) + CACHE_HIT: 1_000_000, // ~6.7% NOISE + NOTIFICATION_DELIVERY: 1_000_000, // ~6.7% LOAD-BEARING + + // Composite cell 2 (billing-service × INFO) + SUBSCRIPTION: 1_000_000, // ~6.7% NOISE (metric-as-log) + BILLING_EVENTS: 1_000_000, // ~6.7% LOAD-BEARING (invoice/refund/sub) + + // Composite cell 3 (worker × ERROR) + STACKTRACE: 400_000, // ~2.7% NOISE (caught) + JOB_FAILED: 400_000, // ~2.7% LOAD-BEARING (permanent) + + // Composite cell 4 ({inv,pri,ship,rec}-service × DEBUG) + HEARTBEAT: 2_000_000, // ~13.3% NOISE across 4 services + SERVICE_OPS: 2_000_000, // ~13.3% LOAD-BEARING across same 4 services + + // Composite cell 5 (frontend-proxy × INFO) + LB_PROBE: 1_000_000, // ~6.7% NOISE + ACCESS_LOG: 1_000_000, // ~6.7% LOAD-BEARING + + // Composite cell 6 (frontend × INFO) + CONSOLE_DUMP: 800_000, // ~5.3% NOISE + PAGE_RENDER: 800_000, // ~5.3% LOAD-BEARING + + // Composite cell 7 (search-service × INFO) + SEARCH_MISS: 600_000, // ~4% NOISE + INDEXING: 600_000, // ~4% LOAD-BEARING + + // Single-pattern cells (analytics is load-bearing-only; catalog + bg + // contribute variety so the cells above aren't the only thing in the + // table). + ANALYTICS: 600_000, // ~4% LOAD-BEARING (no noise twin) + CATALOG_LOOKUP: 400_000, // ~2.7% mixed-info background + BACKGROUND_VARIED: 1_400_000, // ~9.3% genuine variety +}; + +type Rng = GenerateContext['rng']; + +function spread(i: number, total: number, startMs: number): number { + return spreadTimestamp(i, total, startMs, HISTORY_WINDOW_MS); +} + +// ─── Generic log-phase generator ────────────────────────────────────────── +// Most phases follow the same skeleton: loop N times, spread timestamps, +// call a template function, push a log row, yield in batches. This generic +// eliminates the per-phase boilerplate. + +type TemplateResult = { body: string; attrs: Record }; + +function* streamLogPhase(opts: { + rng: Rng; + startMs: number; + count: number; + batchSize: number; + serviceName: string | ((i: number) => string); + severityText: LogRow['severityText']; + template: (rng: Rng, t: number, svc: string) => TemplateResult; + extraAttrs?: (rng: Rng, t: number) => Record; + resourcePool: Record[]>; +}): Iterable { + const { + rng, + startMs, + count, + batchSize, + severityText, + template, + extraAttrs, + resourcePool, + } = opts; + const resolveSvc = + typeof opts.serviceName === 'function' + ? opts.serviceName + : () => opts.serviceName as string; + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const svc = resolveSvc(i); + const tmpl = template(rng, t, svc); + buf.push( + makeLog({ + timestampMs: t, + serviceName: svc, + severityText, + body: tmpl.body, + resourceAttributes: pickResource(rng, resourcePool, svc), + logAttributes: extraAttrs + ? { ...tmpl.attrs, ...extraAttrs(rng, t) } + : tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +export const noisySignalsScenario: Scenario = { + name: 'noisy-signals', + agentPrompt: groundTruth.agentPrompt, + description: + '~16M logs across composite (service, severity) cells. Each cell that contains a noise pattern (drop/throttle candidate) also contains a high-volume LOAD-BEARING pattern of comparable size. Grouping by service+severity alone cannot identify noise — the agent must inspect event.name / body templates to disambiguate keep-vs-drop within each cell.', + *generate(ctx): Iterable { + const { rng, nowMs } = ctx; + const startMs = nowMs - HISTORY_WINDOW_MS; + const batchSize = ctx.batchSize ?? DEFAULT_BATCH_SIZE; + const factor = ctx.volumeFactor ?? 1; + const counts = scaleCounts(DEFAULTS, factor); + + const FOUR_SERVICES = [ + 'inventory-service', + 'pricing-service', + 'shipping-service', + 'recommendation-service', + ] as const; + + // All services used across phases — pre-build a resource pool so we + // pick from ~24 pre-built objects per service instead of constructing + // a fresh 18-key resource-attributes object on every row. + const ALL_SERVICES = [ + 'notification-service', + 'billing-service', + 'worker', + ...FOUR_SERVICES, + 'frontend-proxy', + 'frontend', + 'search-service', + 'analytics-service', + 'product-catalog', + // Background varied services + 'auth-service', + 'tax-service', + 'fraud-detection', + 'kafka', + 'payment-service', + 'cart-service', + 'session-service', + 'webhook-relay', + 'feature-flags', + 'image-service', + ] as const; + const resourcePool = buildResourcePool({ + rng, + services: ALL_SERVICES, + }); + + const base = { rng, startMs, batchSize, resourcePool }; + + // Composite cell 1: notification-service × DEBUG + yield* streamLogPhase({ + ...base, + count: counts.CACHE_HIT, + serviceName: 'notification-service', + severityText: 'DEBUG', + template: (r, t) => cacheHitLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.NOTIFICATION_DELIVERY, + serviceName: 'notification-service', + severityText: 'DEBUG', + template: (r, t) => notificationDeliveryLog({ rng: r, nowMs: t }), + }); + // Composite cell 2: billing-service × INFO + yield* streamLogPhase({ + ...base, + count: counts.SUBSCRIPTION, + serviceName: 'billing-service', + severityText: 'INFO', + template: (r, t) => subscriptionMetricLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.BILLING_EVENTS, + serviceName: 'billing-service', + severityText: 'INFO', + template: (r, t) => billingEventLog({ rng: r, nowMs: t }), + }); + // Composite cell 3: worker × ERROR + yield* streamLogPhase({ + ...base, + count: counts.STACKTRACE, + serviceName: 'worker', + severityText: 'ERROR', + template: (r, t) => caughtExceptionLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.JOB_FAILED, + serviceName: 'worker', + severityText: 'ERROR', + template: (r, t) => jobFailedPermanentLog({ rng: r, nowMs: t }), + }); + // Composite cell 4: 4-service × DEBUG + yield* streamLogPhase({ + ...base, + count: counts.HEARTBEAT, + serviceName: (i: number) => FOUR_SERVICES[i % FOUR_SERVICES.length], + severityText: 'DEBUG', + template: (r, t, svc) => + heartbeatLog({ rng: r, nowMs: t, serviceName: svc }), + }); + yield* streamLogPhase({ + ...base, + count: counts.SERVICE_OPS, + serviceName: (i: number) => FOUR_SERVICES[i % FOUR_SERVICES.length], + severityText: 'DEBUG', + template: (r, t, svc) => + serviceOpsDebugLog({ + rng: r, + nowMs: t, + serviceName: svc as (typeof FOUR_SERVICES)[number], + }), + }); + // Composite cell 5: frontend-proxy × INFO + yield* streamLogPhase({ + ...base, + count: counts.LB_PROBE, + serviceName: 'frontend-proxy', + severityText: 'INFO', + template: (r, t) => upstreamHealthProbeLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.ACCESS_LOG, + serviceName: 'frontend-proxy', + severityText: 'INFO', + template: (r, t) => envoyAccessLog({ rng: r, nowMs: t }), + extraAttrs: r => ({ _severity_raw: pickSeverityIn(r, 'info').text }), + }); + // Composite cell 6: frontend × INFO + yield* streamLogPhase({ + ...base, + count: counts.CONSOLE_DUMP, + serviceName: 'frontend', + severityText: 'INFO', + template: (r, t) => consoleLogDumpLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.PAGE_RENDER, + serviceName: 'frontend', + severityText: 'INFO', + template: (r, t) => pageRenderLog({ rng: r, nowMs: t }), + }); + // Composite cell 7: search-service × INFO + yield* streamLogPhase({ + ...base, + count: counts.SEARCH_MISS, + serviceName: 'search-service', + severityText: 'INFO', + template: (r, t) => searchCacheMissVerboseLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.INDEXING, + serviceName: 'search-service', + severityText: 'INFO', + template: (r, t) => indexingLog({ rng: r, nowMs: t }), + }); + // Single-pattern cells + yield* streamLogPhase({ + ...base, + count: counts.ANALYTICS, + serviceName: 'analytics-service', + severityText: 'INFO', + template: (r, t) => analyticsEventLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.CATALOG_LOOKUP, + serviceName: 'product-catalog', + severityText: 'INFO', + template: (r, t) => { + const sev = pickSeverityIn(r, 'info'); + const tmpl = catalogLog({ + rng: r, + nowMs: t, + level: sev.text.toLowerCase(), + }); + return { + body: tmpl.body, + attrs: { ...tmpl.attrs, _severity_raw: sev.text }, + }; + }, + }); + // Background varied — custom logic, uses phaseBackgroundVaried + yield* phaseBackgroundVaried( + rng, + startMs, + counts.BACKGROUND_VARIED, + batchSize, + resourcePool, + ); + }, + groundTruth, +}; + +function scaleCounts>( + defaults: T, + factor: number, +): T { + const out = {} as T; + for (const k of Object.keys(defaults) as (keyof T)[]) { + (out[k] as number) = Math.max(1, Math.round(defaults[k] * factor)); + } + return out; +} + +function* phaseBackgroundVaried( + rng: Rng, + startMs: number, + count: number, + batchSize: number, + resourcePool: Record[]>, +): Iterable { + const buf: LogRow[] = []; + const services = [ + 'auth-service', + 'shipping-service', + 'tax-service', + 'fraud-detection', + 'recommendation-service', + 'kafka', + 'payment-service', + 'cart-service', + 'session-service', + 'webhook-relay', + 'feature-flags', + 'image-service', + ]; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const svc = services[i % services.length]; + const sev = rng.weightedPick<'info' | 'warn' | 'error' | 'debug' | 'trace'>( + [ + { value: 'info', weight: 78 }, + { value: 'warn', weight: 14 }, + { value: 'error', weight: 5 }, + { value: 'debug', weight: 2 }, + { value: 'trace', weight: 1 }, + ], + ); + const sevPick = pickSeverityIn(rng, sev); + const body = backgroundLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: svc, + severityText: normalizeSeverityText(sevPick.text), + body, + resourceAttributes: pickResource(rng, resourcePool, svc), + logAttributes: { + 'event.name': 'app.background', + 'log.iostream': 'stdout', + logtag: 'F', + _severity_raw: sevPick.text, + }, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} diff --git a/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json b/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json new file mode 100644 index 0000000000..0046e311d2 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json @@ -0,0 +1,247 @@ +{ + "scenario": "noisy-signals", + "agentPrompt": "We want to cut log ingest cost. What are the top noisy signals we should drop or throttle?", + "expected": { + "totalLogVolume": 16000000, + "designNote": "Every (ServiceName, SeverityText) cell that contains a noise pattern also contains a high-volume LOAD-BEARING pattern of comparable size. The agent has to inspect event.name / body templates to disambiguate noise from legitimate logs WITHIN each cell — grouping by service+severity alone is insufficient. A blanket drop on any of these cells would destroy load-bearing data.", + "compositeCells": [ + { + "cell": "notification-service × DEBUG", + "noise": { + "eventName": "cache.hit", + "approxCount": 1000000, + "remediation": "drop debug-level cache hit logs" + }, + "loadBearing": { + "eventName": "notification.delivery", + "approxCount": 1000000, + "whyKeep": "Push delivery audit trail — required for support and compliance." + } + }, + { + "cell": "billing-service × INFO", + "noise": { + "eventName": "subscription.metric.calculated", + "metricName": "subscription_mrr", + "approxCount": 1000000, + "remediation": "convert to a gauge metric (subscription_mrr); stop emitting as a log" + }, + "loadBearing": { + "eventName": "invoice.charged / invoice.refunded / subscription.activated / subscription.cancelled / payment.failed", + "approxCount": 1000000, + "whyKeep": "Financial / audit events — required for compliance and revenue recognition." + } + }, + { + "cell": "worker × ERROR", + "noise": { + "eventName": "job.caught_exception", + "errorKind": "NonFatalRetryableError", + "approxCount": 400000, + "remediation": "downgrade severity or throttle — caught/handled error" + }, + "loadBearing": { + "eventName": "job.failed.permanent", + "approxCount": 400000, + "whyKeep": "Real, unhandled failures — must be kept for incident response." + } + }, + { + "cell": "{inventory,pricing,shipping,recommendation}-service × DEBUG", + "noise": { + "eventName": "health.check", + "approxCount": 2000000, + "remediation": "drop entirely; replace with a metric" + }, + "loadBearing": { + "eventName": "inventory.stock_lookup / pricing.calculation / shipping.rate.quote / recommendation.score", + "approxCount": 2000000, + "whyKeep": "Operational debug used by SREs to diagnose hot-path slowness in each service." + } + }, + { + "cell": "frontend-proxy × INFO", + "noise": { + "eventName": "lb.health.probe", + "approxCount": 1000000, + "remediation": "filter out kube-probe / LB internal health probes at the proxy" + }, + "loadBearing": { + "eventName": "proxy.access", + "approxCount": 1000000, + "whyKeep": "Envoy access logs — request-level auditing and debugging." + } + }, + { + "cell": "frontend × INFO", + "noise": { + "eventName": "frontend.console.log_dump", + "approxCount": 800000, + "remediation": "stop flushing client-side console.log dumps to backend logs" + }, + "loadBearing": { + "eventName": "page.render", + "approxCount": 800000, + "whyKeep": "RUM data — frontend performance signal." + } + }, + { + "cell": "search-service × INFO", + "noise": { + "eventName": "search.cache.miss", + "approxCount": 600000, + "remediation": "drop verbose per-query cache-miss logs; replace with a counter" + }, + "loadBearing": { + "eventName": "index.shard.flush", + "approxCount": 600000, + "whyKeep": "Indexer health signal — needed to track pipeline lag." + } + } + ], + "loadBearingOnlyCells": [ + { + "cell": "analytics-service × INFO", + "eventName": "product.view / cart.add / checkout.* / search.* / etc.", + "approxCount": 600000, + "whyKeep": "Product analytics feeding business dashboards." + } + ] + }, + "rubric": { + "programmatic": [ + ["names_heartbeat_pattern", 2, "(health\\.check|heartbeat)"], + ["names_cache_hit_pattern", 2, "cache\\.?hit"], + [ + "names_subscription_metric_pattern", + 2, + "(subscription_mrr|metric\\.name=subscription|metric.{0,15}calculated|mrr)" + ], + [ + "names_stacktrace_pattern", + 2, + "(NonFatalRetryableError|job\\.caught_exception|caught.{0,8}exception)" + ], + [ + "names_lb_probe_pattern", + 2, + "(lb\\.health\\.probe|kube[- ]?probe|/healthz|health.?probe|LB.{0,20}probe|internal.{0,10}health.?check)" + ], + [ + "names_console_dump_pattern", + 2, + "(console\\.log_dump|console\\.log dump|console.{0,10}dump|client.?side.{0,15}console|frontend.{0,10}console)" + ], + [ + "names_search_miss_pattern", + 1, + "(search\\.cache\\.miss|cache.?miss.{0,30}(search|verbose|per.?query)|verbose.{0,20}cache.?miss)" + ], + [ + "names_keep_notification_delivery", + 1, + "(notification\\.delivery|push.{0,10}delivery|delivery.{0,15}(audit|record|trail))" + ], + [ + "names_keep_billing_events", + 1, + "(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|billing.{0,15}(events?|financial|audit))" + ], + [ + "names_keep_job_failed_permanent", + 1, + "(job\\.failed\\.permanent|permanent.{0,15}(failure|fail)|unhandled.{0,20}(failure|error|job))" + ], + [ + "false_blanket_drop_notification_debug", + 3, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bnotification-service[^.\\n]{0,40}?\\bdebug\\b(?![^.\\n]*?(cache\\.?hit|cache.?lookup))", + true + ], + [ + "false_blanket_drop_billing_info", + 3, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bbilling-service[^.\\n]{0,40}?\\binfo\\b(?![^.\\n]*?(subscription|metric|mrr))", + true + ], + [ + "false_blanket_drop_worker_errors", + 3, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bworker[^.\\n]{0,40}?\\berror\\b(?![^.\\n]*?(NonFatalRetryable|caught|retryable|handled))", + true + ], + [ + "false_drop_envoy_access", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(envoy.{0,20}?access|proxy\\.access|access.?log)\\b", + true + ], + [ + "false_drop_page_render", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(page\\.render|rum|frontend.{0,15}render)", + true + ], + [ + "false_drop_indexing", + 1, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(index\\.shard\\.flush|indexing.?progress|indexer.?health)", + true + ], + [ + "false_drop_analytics_events", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(analytics-service|product\\.view|cart\\.add|checkout\\.[a-z]+|event tracking|product analytics)", + true + ], + [ + "false_drop_notification_delivery", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(notification\\.delivery|push.{0,10}delivery|delivery.?(audit|record|trail))", + true + ], + [ + "false_drop_billing_events", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|financial.?event)", + true + ], + [ + "false_drop_job_failed_permanent", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(job\\.failed\\.permanent|permanent.{0,15}(failure|fail))", + true + ], + [ + "false_drop_service_ops_debug", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(inventory\\.stock_lookup|pricing\\.calculation|shipping\\.rate\\.quote|recommendation\\.score)", + true + ] + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify the SEVEN noise patterns and name them by event.name or body template? (1) health.check heartbeats across inventory/pricing/shipping/recommendation services × DEBUG, (2) notification-service cache.hit chatter, (3) billing-service subscription_mrr metric-as-log, (4) worker NonFatalRetryableError caught-exception, (5) frontend-proxy lb.health.probe internal probes, (6) frontend console.log_dump client-side dumps, (7) search-service search.cache.miss verbose. Missing the billing metric-as-log (most subtle — it's INFO not DEBUG) is a correctness failure. Missing pattern 5/6/7 (the new mixed-cell noises) is also a correctness failure." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did it propose a concrete remediation for each noise pattern (drop, throttle, sample, downgrade severity, or convert to metric)? The billing-service one specifically should be flagged as 'convert to a gauge metric'." + }, + { + "id": "calibration", + "weight": 3, + "description": "CRITICAL — every (ServiceName, SeverityText) cell that contains a noise pattern ALSO contains a high-volume LOAD-BEARING pattern of comparable size. The agent must DISTINGUISH noise from legitimate within each cell. Specifically: notification-service DEBUG has cache.hit (drop) AND notification.delivery (KEEP, push delivery audit); billing-service INFO has subscription_mrr-as-log (drop/convert) AND invoice/refund/subscription financial events (KEEP); worker ERROR has NonFatalRetryableError (drop/downgrade) AND job.failed.permanent (KEEP, real failures); inventory/pricing/shipping/recommendation × DEBUG each have heartbeats (drop) AND legit per-service ops debug (KEEP); frontend-proxy INFO has lb.health.probe (drop/filter) AND envoy access (KEEP); frontend INFO has console.log_dump (drop) AND page.render (KEEP, RUM); search-service INFO has search.cache.miss (drop) AND index.shard.flush (KEEP). A blanket 'drop notification-service DEBUG' or 'drop worker ERROR' recommendation that does not name the specific event.name to drop is a calibration failure — it would destroy load-bearing data. The recommendation must be event.name-specific, not service-and-severity-specific." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and ranked clearly?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/seedScenario.ts b/packages/hdx-eval/src/scenarios/seedScenario.ts new file mode 100644 index 0000000000..3ba13c913f --- /dev/null +++ b/packages/hdx-eval/src/scenarios/seedScenario.ts @@ -0,0 +1,58 @@ +import type { ClickHouseClient } from '@clickhouse/client'; + +import { insertLogRows, insertTraceRows } from '../clickhouse/insert'; +import { + ensureScenarioTables, + truncateScenarioTables, +} from '../clickhouse/schema'; +import { mulberry32 } from '../rng/seeded'; +import { getScenario } from './index'; + +export type SeedProgress = { + tracesInserted: number; + logsInserted: number; + batchesCompleted: number; +}; + +export type SeedResult = { + tracesInserted: number; + logsInserted: number; + tables: { traces: string; logs: string }; +}; + +export async function seedScenario(args: { + client: ClickHouseClient; + scenarioName: string; + seed: number; + nowMs: number; + volumeFactor?: number; + onProgress?: (progress: SeedProgress) => void; +}): Promise { + const scenario = getScenario(args.scenarioName); + const tables = await ensureScenarioTables(args.client, scenario.name); + await truncateScenarioTables(args.client, scenario.name); + const rng = mulberry32(args.seed); + + let tracesInserted = 0; + let logsInserted = 0; + let batchesCompleted = 0; + for (const batch of scenario.generate({ + rng, + nowMs: args.nowMs, + volumeFactor: args.volumeFactor, + })) { + if (batch.traces.length > 0) { + tracesInserted += await insertTraceRows( + args.client, + tables.traces, + batch.traces, + ); + } + if (batch.logs.length > 0) { + logsInserted += await insertLogRows(args.client, tables.logs, batch.logs); + } + batchesCompleted++; + args.onProgress?.({ tracesInserted, logsInserted, batchesCompleted }); + } + return { tracesInserted, logsInserted, tables }; +} diff --git a/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts b/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts new file mode 100644 index 0000000000..084a205018 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts @@ -0,0 +1,299 @@ +/** + * segmented-regression scenario. + * + * Story: api-server error rate has gone up in the last 10 minutes, but only + * for the intersection of `tenant.tier=enterprise` AND `cache.hit=false`. + * A newly deployed enterprise-only fallback handler has a schema-mismatch + * bug; it only fires when both conditions hold. Single-axis aggregates + * (group-by tier only, or group-by cache state only) dilute the signal + * because most enterprise traffic is cache-hit (normal) and most cache-miss + * traffic is non-enterprise (normal). + * + * Distractors: + * - Background 0.5% error rate everywhere (uniform noise). + * - Concurrent recommendation-service 502 burst (different service — + * real but unrelated incident, tempting because of the timing overlap). + * + * What a successful agent does: + * 1. Spots the elevated error rate on api-server in the last 10 min. + * 2. Notices that aggregating by tenant.tier alone OR by cache.hit alone + * shows only a modest bump. + * 3. Cross-tabs on (tenant.tier, cache.hit), sees enterprise × miss at + * ~12% vs ~0.5% baseline elsewhere. + * 4. Identifies the schema-mismatch / FallbackHandlerError body pattern. + * 5. Calls out that recommendation-service is on different traces and + * not the cause. + */ +import { makeLog } from '../../generators/logs'; +import { + buildResourcePool, + pickResource, + spreadTimestamp, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +// ─── Volumes ─────────────────────────────────────────────────────────────── + +const TOTAL_API_SERVER_TRACES = 6_000_000; +const HISTORY_WINDOW_MS = 60 * 60 * 1000; +const ANOMALY_WINDOW_MS = 10 * 60 * 1000; + +// Concurrent distractor burst on a different service. +const REC_DECOY_COUNT = 220; +const REC_DECOY_WINDOW_MS = 12 * 60 * 1000; + +// ─── Distribution knobs ─────────────────────────────────────────────────── + +const TENANT_TIERS = [ + { value: 'free', weight: 60 }, + { value: 'pro', weight: 30 }, + { value: 'enterprise', weight: 10 }, +] as const; + +const CACHE_HIT_RATE = 0.7; // 70% cache hits, 30% misses + +const REGIONS = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1'] as const; + +const ENDPOINTS = [ + 'GET /api/products/{id}', + 'GET /api/products/search', + 'POST /api/cart/items', + 'GET /api/cart/{id}', + 'GET /api/users/{id}', + 'GET /api/orders/{id}', + 'POST /api/orders', + 'POST /api/checkout', + 'GET /api/recommendations', + 'GET /api/health', +] as const; + +// Background error rate everywhere (0.5%) — pure noise, no signal. +const BASELINE_ERROR_RATE = 0.005; + +// Anomaly: enterprise × cache.hit=false in the LAST 10 min errors at 12%. +// All other (tier, hit) combinations stay at BASELINE_ERROR_RATE. +const ANOMALY_ERROR_RATE = 0.12; + +// Generic background error messages — heterogeneous so simple body grouping +// doesn't immediately reveal the anomaly pattern. +const BACKGROUND_ERROR_BODIES = [ + 'upstream connect timeout', + 'context deadline exceeded', + 'temporary unavailable: retry', + 'rate limited by upstream', + 'failed to read response body', + 'invalid request: missing header', +] as const; + +// ─── Generator ──────────────────────────────────────────────────────────── + +type Rng = GenerateContext['rng']; + +// Streaming generator: yield batches of `batchSize` rows as we go instead of +// buffering all 6M traces in memory. Without this, full-volume seed runs OOM +// the Node heap even with --max-old-space-size=8192. +function* streamBackgroundRows( + rng: Rng, + nowMs: number, + totalTraces: number, + batchSize: number, +): Generator { + const startMs = nowMs - HISTORY_WINDOW_MS; + const anomalyStartMs = nowMs - ANOMALY_WINDOW_MS; + let tracesBuf: TraceRow[] = []; + let logsBuf: LogRow[] = []; + + const resourcePool = buildResourcePool({ + rng, + services: ['api-server'], + instancesPerService: 24, + }); + + for (let i = 0; i < totalTraces; i++) { + const t = spreadTimestamp(i, totalTraces, startMs, HISTORY_WINDOW_MS); + + const tenantTier = rng.weightedPick( + TENANT_TIERS.map(tt => ({ value: tt.value, weight: tt.weight })), + ); + const tenantId = `${tenantTier}-${rng.intRange(1, 5_000)}`; + const cacheHit = rng.next() < CACHE_HIT_RATE; + const region = rng.pick(REGIONS); + const endpoint = rng.pick(ENDPOINTS); + const inAnomalyWindow = t >= anomalyStartMs; + const isAffectedSegment = + inAnomalyWindow && tenantTier === 'enterprise' && !cacheHit; + const errorRate = isAffectedSegment + ? ANOMALY_ERROR_RATE + : BASELINE_ERROR_RATE; + const isError = rng.next() < errorRate; + + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + const resource = pickResource(rng, resourcePool, 'api-server'); + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const route = endpoint.replace(/^[A-Z]+ /, ''); + const httpMethod = endpoint.startsWith('POST') ? 'POST' : 'GET'; + + const spanAttributes: Record = { + 'http.method': httpMethod, + 'http.route': route, + 'http.target': route, + 'http.status_code': isError ? '500' : '200', + 'tenant.id': tenantId, + 'tenant.tier': tenantTier, + 'user.id': userId, + 'cloud.region': region, + 'cache.hit': cacheHit ? 'true' : 'false', + 'request.id': requestId, + }; + + let statusCode: TraceRow['statusCode'] = 'STATUS_CODE_OK'; + let statusMessage = ''; + if (isError) { + statusCode = 'STATUS_CODE_ERROR'; + if (isAffectedSegment) { + statusMessage = + 'fallback handler returned 500: schema mismatch in enterprise_v2_response'; + spanAttributes['error.type'] = 'FallbackHandlerError'; + spanAttributes['fallback.path'] = 'enterprise_v2'; + } else { + statusMessage = rng.pick(BACKGROUND_ERROR_BODIES); + spanAttributes['error.type'] = 'BackgroundError'; + } + } + + tracesBuf.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: endpoint, + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-server', + durationNs: msToNs(rng.range(20, 250)), + statusCode, + statusMessage, + resourceAttributes: resource, + spanAttributes, + }), + ); + + // Emit a correlated ERROR log for affected-segment failures so an agent + // searching logs can find the distinctive body too. + if (isError && isAffectedSegment) { + logsBuf.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'api-server', + severityText: 'ERROR', + body: `fallback handler returned 500: schema mismatch in enterprise_v2_response tenant=${tenantId} request_id=${requestId}`, + traceId, + spanId, + logAttributes: { + 'error.kind': 'FallbackHandlerError', + 'tenant.tier': 'enterprise', + 'cache.hit': 'false', + }, + }), + ); + } + + if (tracesBuf.length >= batchSize) { + yield { traces: tracesBuf, logs: logsBuf }; + tracesBuf = []; + logsBuf = []; + } + } + if (tracesBuf.length > 0 || logsBuf.length > 0) { + yield { traces: tracesBuf, logs: logsBuf }; + } +} + +function buildRecommendationDecoy( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const startMs = nowMs - REC_DECOY_WINDOW_MS; + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + for (let i = 0; i < REC_DECOY_COUNT; i++) { + const t = spreadTimestamp( + i, + REC_DECOY_COUNT, + startMs, + REC_DECOY_WINDOW_MS, + 5_000, + ); + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: 'rec.compute', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'recommendation-service', + durationNs: msToNs(rng.range(400, 1200)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'rec.compute timed out: upstream cf-feed unreachable', + spanAttributes: { + 'http.status_code': '502', + 'error.type': 'UpstreamTimeout', + 'upstream.host': 'cf-feed.internal', + }, + }), + ); + logs.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'recommendation-service', + severityText: 'ERROR', + body: 'rec.compute timed out: upstream cf-feed.internal unreachable after 1500ms', + traceId, + spanId, + logAttributes: { + 'error.kind': 'UpstreamTimeout', + component: 'recommendation-service', + }, + }), + ); + } + return { traces, logs }; +} + +export const segmentedRegressionScenario: Scenario = { + name: 'segmented-regression', + agentPrompt: groundTruth.agentPrompt, + description: + '6M api-server traces over an hour. Last 10 min: enterprise × cache-miss errors at ~12% (schema mismatch in a newly deployed fallback). All other (tier, cache) combinations stay at ~0.5% baseline. Single-axis aggregates dilute the signal — the agent must cross-tab. Concurrent recommendation-service 502 burst as distractor.', + *generate(ctx): Iterable { + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const totalTraces = Math.max( + 50, + Math.round(TOTAL_API_SERVER_TRACES * factor), + ); + + // Stream the background api-server traces in batches so we don't buffer + // 6M rows in memory. + yield* streamBackgroundRows(ctx.rng, ctx.nowMs, totalTraces, batchSize); + // Decoy is small (220 rows) — fine to build in memory. + const decoy = buildRecommendationDecoy(ctx.rng, ctx.nowMs); + if (decoy.traces.length > 0) yield { traces: decoy.traces, logs: [] }; + if (decoy.logs.length > 0) yield { traces: [], logs: decoy.logs }; + }, + groundTruth, +}; diff --git a/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json b/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json new file mode 100644 index 0000000000..59169bee16 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json @@ -0,0 +1,121 @@ +{ + "scenario": "segmented-regression", + "agentPrompt": "API error rate has gone up in the last 10 minutes for some users. Find the segment(s) where the regression is concentrated and what's going wrong.", + "expected": { + "affectedService": "api-server", + "segmentAttributeA": "tenant.tier", + "segmentValueA": "enterprise", + "segmentAttributeB": "cache.hit", + "segmentValueB": "false", + "anomalyWindowMinutesAgo": 10, + "errorRateInSegment": 0.12, + "errorRateBaseline": 0.005, + "errorBodyPattern": "schema mismatch in enterprise_v2_response", + "errorType": "FallbackHandlerError", + "rootCauseHint": "newly deployed enterprise-only cache-miss fallback path has a schema mismatch — only fires when tier=enterprise AND cache.hit=false" + }, + "distractors": [ + { + "kind": "background uniform 5xx", + "approxRate": 0.005, + "whyNotRootCause": "Uniform 0.5% error rate across all segments and time — normal background noise, NOT the regression." + }, + { + "kind": "concurrent recommendation-service incident", + "service": "recommendation-service", + "windowMinutesAgo": [12, 0], + "approxCount": 220, + "errorMessage": "rec.compute timed out: upstream cf-feed unreachable", + "whyNotRootCause": "Concurrent unrelated 502 burst on a different service. Tempting because it overlaps in time, but the prompt is about api-server (the customer-facing tier) and the affected traces have no recommendation-service span." + }, + { + "kind": "non-anomaly enterprise errors", + "approxRate": 0.005, + "tierFilter": "enterprise", + "whyNotRootCause": "Enterprise tier has the same 0.5% baseline error rate as other tiers OUTSIDE the cache-miss path. Blaming enterprise on its own (without the cache axis) is incomplete — enterprise + cache.hit=true errors at baseline." + }, + { + "kind": "non-anomaly cache-miss errors", + "approxRate": 0.005, + "cacheFilter": "false", + "whyNotRootCause": "Cache misses on non-enterprise tiers error at the same 0.5% baseline. Blaming cache misses on their own (without the tier axis) is incomplete — pro/free + cache.hit=false errors at baseline." + } + ], + "anomalyAttributes": { + "spanAttributes": { + "tenant.tier": "enterprise", + "cache.hit": "false", + "http.status_code": "500", + "error.type": "FallbackHandlerError" + } + }, + "rubric": { + "programmatic": [ + ["names_affected_service", 1, "api-server"], + ["names_tenant_segment", 2, "(tenant\\.?tier|enterprise)"], + [ + "names_cache_segment", + 2, + "(cache\\.?(hit|miss)|cache hit:?\\s*false|cache miss|miss(es)?)" + ], + [ + "names_both_axes_combined", + 3, + "(enterprise[^.\\n]{0,80}?(cache|miss)|cache[^.\\n]{0,40}?miss[^.\\n]{0,80}?enterprise|tier\\s*=\\s*['\"]?enterprise['\"]?[^.\\n]{0,80}?cache|enterprise[^.\\n]{0,40}?AND[^.\\n]{0,40}?(cache|hit\\s*=\\s*false|miss))" + ], + [ + "names_error_pattern", + 1, + "(schema mismatch|enterprise_v2|FallbackHandlerError|fallback (handler|path))" + ], + [ + "false_blame_global", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(global|api-server (alone|generally)|all (tiers|users|traffic)|service[- ]wide)", + true + ], + [ + "false_blame_tier_only", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\benterprise[^.\\n]{0,40}?(tier|customers|tenants?)(?![^.\\n]{0,200}?(cache|miss|hit\\s*=\\s*false))", + true + ], + [ + "false_blame_cache_only", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcache[ \\-]?(miss(es)?|lookup|hit\\s*=\\s*false)(?![^.\\n]{0,200}?(enterprise|tier))", + true + ], + [ + "false_blame_recommendation", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\brecommendation-service", + true + ] + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify the affected segment as the INTERSECTION of `tenant.tier=enterprise` AND `cache.hit=false` on `api-server`? Identifying only ONE axis (just the tier OR just the cache state) is a correctness failure — the bug only fires on the intersection, and quoting an aggregate over a single axis (which dilutes the signal) misses the actual problem." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did the answer (a) note the rough magnitude (e.g. ~12% error rate in the affected segment vs ~0.5% baseline), (b) reference the specific error signature (schema mismatch / enterprise_v2 / FallbackHandlerError or the fallback handler path), and (c) explain why neither single-axis aggregation reveals the issue?" + }, + { + "id": "calibration", + "weight": 2, + "description": "Did the answer rule out the distractors? Specifically: (a) the concurrent recommendation-service incident is on a different service and unrelated. (b) Enterprise-tier alone is NOT the cause — enterprise + cache.hit=true errors at the baseline rate. (c) Cache misses alone are NOT the cause — pro/free + cache.hit=false error at baseline. (d) The global api-server error rate is dominated by baseline noise; the regression is invisible without segmenting. Mistaking any of these for the root cause is a calibration failure." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and free of filler or unnecessary speculation?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/service-health-check/generate.ts b/packages/hdx-eval/src/scenarios/service-health-check/generate.ts new file mode 100644 index 0000000000..c6b30cc7fb --- /dev/null +++ b/packages/hdx-eval/src/scenarios/service-health-check/generate.ts @@ -0,0 +1,431 @@ +/** + * service-health-check scenario. + * + * Peace-time SLI report on `api-server`. The dataset is healthy — no + * incident — but the report window contains four genuinely-novel-but- + * non-critical signals the agent should call out, plus a couple of + * recurring/baseline patterns it should NOT escalate. + * + * Dataset spans 4 hours ending at `nowMs`: + * - Baseline window: nowMs-240min .. nowMs-60min (steady, no planted signals) + * - Report window: nowMs-60min .. nowMs (what the prompt covers) + * + * Planted novel signals (in the report window): + * 1. New log template "feature_flag.new_recommendation_engine: shadow eval + * succeeded …" at ~0.1% of total log volume, starting 70 min before + * nowMs. Absent from the baseline. + * 2. `GET /api/v2/products` endpoint ramps from 0% → ~4% of products + * traffic across the report window. Absent from the baseline. + * 3. p99 latency drifts ~340ms → ~380ms over the report window. Still + * well under the stated 500ms SLO. Steady ~340ms in the baseline. + * 4. 60-second error blip at reportStart+3min: ~5% error rate (~250 + * errors), all "upstream connect timeout". Outside that minute the + * error rate is the baseline 0.5%. + * + * Distractors (in BOTH baseline and report window — should NOT be flagged): + * 5. POST /api/internal/batch-sync 30-second burst every 15 minutes. + * Recurring; an agent comparing to baseline can see it's routine. + * 6. Constant 0.5% background error rate uniformly across endpoints + * and time. NOT a regression. + * + * Hardness: + * - Subtle signals (0.1% / 30ms / 4%) all below typical chart thresholds. + * - Log Body has per-row IDs/UUIDs/timestamps so `GROUP BY Body` returns + * millions of distinct rows — only hyperdx_log_patterns (Drain) gives + * a coherent view. + * - Strong calibration penalty: false-alarming on the baseline noise or + * escalating the brief blip costs as much as missing a real finding. + * - Baseline comparison required: distractor #5 is visible in the report + * window only as a "spike"; an agent that queries only the report + * window may flag it as novel. + */ +import { makeLog } from '../../generators/logs'; +import { + buildResourcePool, + pickResource, + spreadTimestamp, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +// ─── Volumes ───────────────────────────────────────────────────────────────── + +const TOTAL_TRACES = 12_000_000; // 50K/min × 240 min +const TOTAL_LOGS = 24_000_000; // 100K/min × 240 min + +const BASELINE_WINDOW_MS = 4 * 60 * 60 * 1000; // 4 hours total +const REPORT_WINDOW_MS = 60 * 60 * 1000; // last hour + +// ─── Distractor: recurring batch-sync spike ───────────────────────────────── +const BATCH_SPIKE_PERIOD_MS = 15 * 60 * 1000; // every 15 min +const BATCH_SPIKE_DURATION_MS = 30 * 1000; // 30s burst +// During a spike window, this fraction of traffic is batch-sync. Tuned so +// the spike adds ~5K extra batch-sync calls per cycle on top of baseline. +const BATCH_SPIKE_SHARE = 0.4; + +// ─── Planted #1: new log template ────────────────────────────────────────── +// Appears starting 70 min before nowMs (10 min before the report window). +const NEW_TEMPLATE_LEAD_MS = 70 * 60 * 1000; +const NEW_TEMPLATE_VOLUME_SHARE = 0.001; // 0.1% of TOTAL log volume + +// ─── Planted #2: v2 endpoint ramp ───────────────────────────────────────── +// Linear ramp from 0% at reportStart to 4% of products traffic by nowMs. +const V2_PRODUCTS_MAX_SHARE = 0.04; + +// ─── Planted #3: latency drift in the report window ─────────────────────── +// Scale baseline latency by 1.0 at reportStart, 1.12 at nowMs. +// Baseline p99 ~340ms → end-of-window p99 ~380ms. +const LATENCY_DRIFT_MAX = 0.12; + +// ─── Planted #4: brief 60-second error blip ─────────────────────────────── +const BLIP_OFFSET_FROM_REPORT_START_MS = 3 * 60 * 1000; // T+3min +const BLIP_DURATION_MS = 60 * 1000; // 60s +const BLIP_ERROR_RATE = 0.05; // 5% (vs 0.5% baseline) + +// ─── Baseline distribution ────────────────────────────────────────────────── + +const BASELINE_ERROR_RATE = 0.005; // 0.5% pure background noise + +// Endpoint mix (products is intentionally large so v2 has room to ramp +// visibly within "4% of products traffic"). +const ENDPOINTS = [ + { spanName: 'GET /api/v1/products', method: 'GET', weight: 35 }, + { spanName: 'GET /api/cart/{id}', method: 'GET', weight: 25 }, + { spanName: 'POST /api/checkout', method: 'POST', weight: 20 }, + { spanName: 'GET /api/users/{id}', method: 'GET', weight: 20 }, +] as const; + +const REGIONS = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1'] as const; +const TABLES = [ + 'orders', + 'users', + 'products', + 'sessions', + 'inventory', +] as const; +const UPSTREAMS = [ + 'cf-feed.internal', + 'recsys.internal', + 'auth.internal', + 'inventory.internal', +] as const; + +// ─── Latency ──────────────────────────────────────────────────────────────── + +type Rng = GenerateContext['rng']; + +/** + * Piecewise distribution chosen so baseline (scale=1) has p50 ~80ms and + * p99 ~340ms. `scale` multiplies everything — used to drift latency + * during the report window. + */ +function sampleLatencyMs(rng: Rng, scale: number): number { + const r = rng.next(); + let base: number; + if (r < 0.5) + base = 30 + rng.next() * 70; // 30..100 (p50 ~65) + else if (r < 0.9) + base = 60 + rng.next() * 140; // 60..200 (p90 ~190) + else if (r < 0.99) + base = 180 + rng.next() * 120; // 180..300 (p99 entry) + else base = 280 + rng.next() * 80; // 280..360 (p99 tail) + return base * scale; +} + +function latencyScaleAt( + t: number, + reportStartMs: number, + nowMs: number, +): number { + if (t < reportStartMs) return 1; + const frac = Math.min(1, (t - reportStartMs) / (nowMs - reportStartMs)); + return 1 + frac * LATENCY_DRIFT_MAX; +} + +// ─── Log body templates ───────────────────────────────────────────────────── +// Every body embeds per-row dynamic values so naive `GROUP BY Body` +// returns ~1 row per log entry. Drain clusters them into the templates +// below. + +function buildAccessBody(rng: Rng): string { + const reqId = uuidv4(rng); + const method = rng.pick(['GET', 'GET', 'GET', 'POST'] as const); + const path = `/api/v1/${rng.pick(['products', 'cart', 'users', 'checkout'] as const)}/${rng.intRange(1, 100_000)}`; + const status = rng.weightedPick([ + { value: 200, weight: 90 }, + { value: 404, weight: 5 }, + { value: 500, weight: 3 }, + { value: 401, weight: 2 }, + ]); + const latency = rng.intRange(20, 400); + return `req_id=${reqId} ${method} ${path} -> ${status} in ${latency}ms`; +} + +function buildCacheBody(rng: Rng): string { + const key = `${rng.pick(['user', 'product', 'session'] as const)}:${rng.intRange(1, 1_000_000)}`; + const hit = rng.next() < 0.85 ? 'true' : 'false'; + const region = rng.pick(REGIONS); + return `cache.lookup key=${key} hit=${hit} region=${region}`; +} + +function buildAuthBody(rng: Rng): string { + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + const ttl = rng.intRange(60, 3600); + return `auth.verified user=${userId} session=${sessionId} ttl=${ttl}s`; +} + +function buildRetryBody(rng: Rng): string { + const upstream = rng.pick(UPSTREAMS); + const retry = rng.intRange(1, 4); + const latency = rng.intRange(100, 1500); + const tenant = `t-${rng.intRange(1, 50_000)}`; + return `upstream=${upstream} retry=${retry} latency_ms=${latency} tenant=${tenant}`; +} + +function buildSlowQueryBody(rng: Rng): string { + const duration = rng.intRange(200, 1500); + const table = rng.pick(TABLES); + const rows = rng.intRange(1000, 1_000_000); + return `slow.query duration_ms=${duration} table=${table} rows_scanned=${rows}`; +} + +function buildNewTemplateBody(rng: Rng): string { + const corrId = uuidv4(rng); + const latency = rng.intRange(15, 90); + return `feature_flag.new_recommendation_engine: shadow eval succeeded correlation_id=${corrId} latency_ms=${latency}`; +} + +// Weighted pick for the baseline log template. Add up to 100. +type LogTemplateKind = 'access' | 'cache' | 'auth' | 'retry' | 'slow'; +const LOG_TEMPLATES: Array<{ + value: { kind: LogTemplateKind; severity: 'INFO' | 'WARN' }; + weight: number; +}> = [ + { value: { kind: 'access', severity: 'INFO' }, weight: 60 }, + { value: { kind: 'cache', severity: 'INFO' }, weight: 20 }, + { value: { kind: 'auth', severity: 'INFO' }, weight: 10 }, + { value: { kind: 'retry', severity: 'INFO' }, weight: 5 }, + { value: { kind: 'slow', severity: 'WARN' }, weight: 5 }, +]; + +function buildBaselineLogBody(rng: Rng): { + body: string; + severityText: 'INFO' | 'WARN'; +} { + const pick = rng.weightedPick(LOG_TEMPLATES); + switch (pick.kind) { + case 'access': + return { body: buildAccessBody(rng), severityText: pick.severity }; + case 'cache': + return { body: buildCacheBody(rng), severityText: pick.severity }; + case 'auth': + return { body: buildAuthBody(rng), severityText: pick.severity }; + case 'retry': + return { body: buildRetryBody(rng), severityText: pick.severity }; + case 'slow': + return { body: buildSlowQueryBody(rng), severityText: pick.severity }; + } +} + +// ─── Streaming generators ────────────────────────────────────────────────── + +function* streamTraces( + rng: Rng, + nowMs: number, + totalTraces: number, + batchSize: number, +): Generator { + const baselineStartMs = nowMs - BASELINE_WINDOW_MS; + const reportStartMs = nowMs - REPORT_WINDOW_MS; + const blipStartMs = reportStartMs + BLIP_OFFSET_FROM_REPORT_START_MS; + const blipEndMs = blipStartMs + BLIP_DURATION_MS; + + const resourcePool = buildResourcePool({ + rng, + services: ['api-server'], + instancesPerService: 24, + }); + + let buf: TraceRow[] = []; + for (let i = 0; i < totalTraces; i++) { + const t = spreadTimestamp( + i, + totalTraces, + baselineStartMs, + BASELINE_WINDOW_MS, + ); + const inReportWindow = t >= reportStartMs; + const cycleOffset = (t - baselineStartMs) % BATCH_SPIKE_PERIOD_MS; + const inBatchSpike = cycleOffset < BATCH_SPIKE_DURATION_MS; + const inBlip = t >= blipStartMs && t < blipEndMs; + + // Endpoint selection + let spanName: string; + let httpMethod: string; + if (inBatchSpike && rng.next() < BATCH_SPIKE_SHARE) { + spanName = 'POST /api/internal/batch-sync'; + httpMethod = 'POST'; + } else { + const endpoint = rng.weightedPick( + ENDPOINTS.map(e => ({ value: e, weight: e.weight })), + ); + spanName = endpoint.spanName; + httpMethod = endpoint.method; + // Planted #2: v2 endpoint ramp. Only inside the report window, and + // only for the products endpoint. + if ( + inReportWindow && + spanName === 'GET /api/v1/products' && + rng.next() < + ((t - reportStartMs) / (nowMs - reportStartMs)) * + V2_PRODUCTS_MAX_SHARE + ) { + spanName = 'GET /api/v2/products'; + } + } + + const errorRate = inBlip ? BLIP_ERROR_RATE : BASELINE_ERROR_RATE; + const isError = rng.next() < errorRate; + + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + const resource = pickResource(rng, resourcePool, 'api-server'); + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const region = rng.pick(REGIONS); + + let statusCode: TraceRow['statusCode'] = 'STATUS_CODE_OK'; + let statusMessage = ''; + if (isError) { + statusCode = 'STATUS_CODE_ERROR'; + statusMessage = inBlip + ? 'upstream connect timeout' + : rng.pick([ + 'connection reset by peer', + 'context deadline exceeded', + 'temporary unavailable: retry', + 'rate limited by upstream', + ] as const); + } + + const durationMs = sampleLatencyMs( + rng, + latencyScaleAt(t, reportStartMs, nowMs), + ); + + const spanAttributes: Record = { + 'http.method': httpMethod, + 'http.route': spanName.replace(/^[A-Z]+ /, ''), + 'http.status_code': isError ? '500' : '200', + 'user.id': userId, + 'cloud.region': region, + 'request.id': requestId, + }; + if (inBlip) spanAttributes['error.type'] = 'UpstreamTimeout'; + + buf.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName, + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-server', + durationNs: msToNs(durationMs), + statusCode, + statusMessage, + resourceAttributes: resource, + spanAttributes, + }), + ); + + if (buf.length >= batchSize) { + yield { traces: buf, logs: [] }; + buf = []; + } + } + if (buf.length > 0) yield { traces: buf, logs: [] }; +} + +function* streamLogs( + rng: Rng, + nowMs: number, + totalLogs: number, + batchSize: number, +): Generator { + const baselineStartMs = nowMs - BASELINE_WINDOW_MS; + const newTemplateStartMs = nowMs - NEW_TEMPLATE_LEAD_MS; + + // Within the planted window, what fraction of rows are the planted + // template? Calibrated so the planted rows are 0.1% of TOTAL log volume. + const plantedWindowMs = NEW_TEMPLATE_LEAD_MS; + const plantedWindowShareOfTotal = plantedWindowMs / BASELINE_WINDOW_MS; + const plantedShareWithinWindow = + plantedWindowShareOfTotal > 0 + ? NEW_TEMPLATE_VOLUME_SHARE / plantedWindowShareOfTotal + : 0; + + let buf: LogRow[] = []; + for (let i = 0; i < totalLogs; i++) { + const t = spreadTimestamp( + i, + totalLogs, + baselineStartMs, + BASELINE_WINDOW_MS, + ); + + let body: string; + let severityText: 'INFO' | 'WARN'; + if (t >= newTemplateStartMs && rng.next() < plantedShareWithinWindow) { + body = buildNewTemplateBody(rng); + severityText = 'INFO'; + } else { + const baseline = buildBaselineLogBody(rng); + body = baseline.body; + severityText = baseline.severityText; + } + + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'api-server', + severityText, + body, + logAttributes: {}, + }), + ); + + if (buf.length >= batchSize) { + yield { traces: [], logs: buf }; + buf = []; + } + } + if (buf.length > 0) yield { traces: [], logs: buf }; +} + +export const serviceHealthCheckScenario: Scenario = { + name: 'service-health-check', + agentPrompt: groundTruth.agentPrompt, + description: + '4-hour dataset on api-server (12M traces + 24M logs). 3-hour steady baseline + 1-hour report window. Healthy service. Four subtle novel signals (new log template, v2 endpoint rollout, p99 latency drift within SLO, brief 60s error blip) plus two distractors (recurring batch-sync spikes, baseline 0.5% error noise). Tests calibrated SLI reporting without false alarms.', + *generate(ctx): Iterable { + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const totalTraces = Math.max(200, Math.round(TOTAL_TRACES * factor)); + const totalLogs = Math.max(400, Math.round(TOTAL_LOGS * factor)); + + yield* streamTraces(ctx.rng, ctx.nowMs, totalTraces, batchSize); + yield* streamLogs(ctx.rng, ctx.nowMs, totalLogs, batchSize); + }, + groundTruth, +}; diff --git a/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json b/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json new file mode 100644 index 0000000000..1024569f27 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json @@ -0,0 +1,164 @@ +{ + "scenario": "service-health-check", + "agentPrompt": "Routine status check on `api-server` over the last hour. Summarize key SLIs (traffic, error rate, latency) and call out anything novel or worth a closer look. Don't escalate routine variance. Be concise.", + "expected": { + "service": "api-server", + "reportWindowMinutes": 60, + "baselineWindowMinutes": 180, + "sliBaselines": { + "trafficPerMinute": 50000, + "errorRate": 0.005, + "latencyP50Ms": 80, + "latencyP99StartMs": 340, + "latencyP99EndMs": 380, + "latencySloMs": 500 + }, + "noveltiesToReport": [ + { + "kind": "new log template", + "approxVolumeShare": 0.001, + "bodyPattern": "feature_flag.new_recommendation_engine: shadow eval succeeded …", + "framing": "new pattern observed, looks like a feature-flag shadow-eval rollout — informational" + }, + { + "kind": "traffic shift", + "endpoint": "GET /api/v2/products", + "approxEndOfWindowShare": 0.04, + "framing": "v2 rollout in progress, ramps to ~4% of products traffic" + }, + { + "kind": "latency drift within SLO", + "fromP99Ms": 340, + "toP99Ms": 380, + "sloMs": 500, + "framing": "p99 trending up modestly across the hour, still well under SLO" + }, + { + "kind": "brief error blip", + "offsetIntoReportWindowMinutes": 3, + "durationSeconds": 60, + "approxErrorCount": 250, + "errorMessage": "upstream connect timeout", + "framing": "brief 1-minute spike of upstream timeouts, recovered immediately — transient, not a pattern" + } + ], + "distractorsToNotEscalate": [ + { + "kind": "recurring batch-sync spike", + "endpoint": "POST /api/internal/batch-sync", + "periodMinutes": 15, + "durationSeconds": 30, + "whyNotEscalate": "Visible throughout the 3-hour baseline; recurring batch job, not novel." + }, + { + "kind": "baseline 0.5% error rate", + "whyNotEscalate": "Constant across the full 4-hour dataset, uniformly distributed. Background noise, not a regression." + } + ] + }, + "rubric": { + "programmatic": [ + ["names_service", 1, "api-server"], + [ + "reports_traffic_rate", + 1, + "(requests?/min|requests/s|req/min|traffic|throughput|call volume|qps|rps)" + ], + [ + "reports_error_rate", + 1, + "(error rate|errors?/min|0\\.5\\s?%?|~?0\\.5|baseline.{0,15}error)" + ], + [ + "reports_latency_metrics", + 1, + "(p50|p95|p99|latency)[\\s\\S]{0,200}?\\d+(\\.\\d+)?\\s?(ms|millisecond)" + ], + [ + "notes_new_log_pattern", + 2, + "(shadow.?eval|new[_\\s]+recommendation|recommendation.engine|feature.?flag)" + ], + [ + "notes_v2_endpoint", + 2, + "(/api/v2/products|/v2/|v2\\s+(endpoint|rollout|rolling))" + ], + [ + "notes_latency_drift", + 1, + "(p50|p95|p99|latency)[\\s\\S]{0,60}?\\b(trend|drift|trending|creep|creeping|rising|rising|climb|increasing|monotonic)" + ], + [ + "notes_brief_blip", + 1, + "(brief|transient|1.?(min(ute)?|second)|60.?second|short.lived).{0,80}(spike|blip|errors?|timeouts?)" + ], + [ + "frames_as_healthy", + 1, + "(healthy|stable|within SLO|no concerns|operating normally|nominal|steady)" + ], + [ + "false_alarm_incident", + 2, + "\\b(incident|outage|service.degradation|degraded|critical|severe|emergency)\\b", + true + ], + [ + "false_alarm_paging", + 2, + "\\b(page (the|on-call)|on-call|escalat(e|ion)|wake.{0,5}up|urgent(ly)?)\\b", + true + ], + [ + "false_alarm_blip_as_pattern", + 2, + "\\b(sustained|ongoing|persistent|recurring|continuing)\\b[^.\\n]{0,60}(error|timeout)", + true + ], + [ + "false_alarm_baseline_errors", + 2, + "error rate[^.\\n]{0,60}\\b(elevated|high|concerning|problem(atic)?|investigate)\\b", + true + ], + [ + "false_alarm_slo_breach", + 2, + "(SLO[^.\\n]{0,20}(breach|violation|exceed)|latency[^.\\n]{0,30}(critical|breach|violation))", + true + ], + [ + "false_flag_batch_job", + 1, + "batch.?sync[^.\\n]{0,80}\\b(novel|unusual|new|anomal(y|ous)|concern|investigate)\\b", + true + ] + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer identify ALL FOUR planted signals? (a) new log template (feature-flag shadow eval rollout), (b) GET /api/v2/products traffic rollout, (c) p99 latency drift within SLO (~340 → ~380ms), (d) brief 60-second upstream-timeout error blip. Each missing item is a correctness hit. False positives (claiming a signal that isn't there — e.g. saying the batch-sync recurrence is new, or that the 0.5% baseline error rate is elevated) are also correctness hits." + }, + { + "id": "completeness", + "weight": 2, + "description": "Does the answer cover all four SLI dimensions of a routine status check: traffic rate, error rate, latency (p50/p99), and a section on novel observations? Missing a dimension is a completeness hit." + }, + { + "id": "calibration", + "weight": 3, + "description": "Are magnitudes and framing appropriate for a healthy service? Specifically: the p99 latency drift must be described as a small trend within SLO (NOT as a breach, problem, or risk); the brief 60-second blip must be described as transient (NOT as a pattern or ongoing); the 0.5% baseline error rate must be described as baseline / background / nominal (NOT as elevated or concerning); the recurring batch-sync spikes must be acknowledged as recurring/expected (NOT as novel). BONUS for explicitly comparing the report window against the 3-hour baseline to verify novelty vs recurrence." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Is the answer a structured status report rather than an exhaustive dump? It should fit on a screen, lead with the headline (service is healthy), then SLI numbers, then 'noted observations' for the four novelties — without long stack-by-stack drill-downs." + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/types.ts b/packages/hdx-eval/src/scenarios/types.ts new file mode 100644 index 0000000000..fadf9ae8ea --- /dev/null +++ b/packages/hdx-eval/src/scenarios/types.ts @@ -0,0 +1,47 @@ +import type { LogRow, TraceRow } from '../generators/types'; +import type { SeededRng } from '../rng/seeded'; + +export type GenerateContext = { + rng: SeededRng; + nowMs: number; + /** + * Test-only override. Scenarios should multiply background counts by this + * factor. Planted anomaly counts (the diagnostic signal) stay fixed so + * structural invariants still hold at low volume. Default 1. + */ + volumeFactor?: number; + /** + * Rows per batch for streaming generation. Default 10_000. Tests can pass + * a larger value to reduce overhead when collecting. + */ + batchSize?: number; +}; + +export type ScenarioBatch = { + traces: TraceRow[]; + logs: LogRow[]; +}; + +export type Scenario = { + name: string; + agentPrompt: string; + description: string; + /** + * Yields batches of rows for insertion. Implementations should respect + * `ctx.batchSize` (cap per-yield row count) and `ctx.volumeFactor` + * (scale background volumes — planted anomalies stay constant). + */ + generate(ctx: GenerateContext): Iterable; + groundTruth: Record; +}; + +/** Helper for tests: drain an iterable into one combined batch. */ +export function collectScenario(iter: Iterable): ScenarioBatch { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + for (const b of iter) { + if (b.traces.length) traces.push(...b.traces); + if (b.logs.length) logs.push(...b.logs); + } + return { traces, logs }; +} diff --git a/packages/hdx-eval/tsconfig.build.json b/packages/hdx-eval/tsconfig.build.json new file mode 100644 index 0000000000..a9862d7ad9 --- /dev/null +++ b/packages/hdx-eval/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["node_modules", "dist", "**/__tests__/**"] +} diff --git a/packages/hdx-eval/tsconfig.json b/packages/hdx-eval/tsconfig.json new file mode 100644 index 0000000000..48370bd1d7 --- /dev/null +++ b/packages/hdx-eval/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "CommonJS", + "moduleResolution": "Node", + "baseUrl": "./src", + "paths": { + "@/*": ["./*"] + }, + "types": ["jest", "node"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "isolatedModules": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "strict": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +} diff --git a/yarn.lock b/yarn.lock index a5a567ff49..1d1459d40b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -111,6 +111,22 @@ __metadata: languageName: node linkType: hard +"@anthropic-ai/sdk@npm:^0.92.0": + version: 0.92.0 + resolution: "@anthropic-ai/sdk@npm:0.92.0" + dependencies: + json-schema-to-ts: "npm:^3.1.1" + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + bin: + anthropic-ai-sdk: bin/cli + checksum: 10c0/a06f2ebc4dce6e3ee36e3cc938b9fb952a6eebaa3b4478351770d0bf264aa3cb141a5a6854cdc93d1693218415d626338f66d0c446e1d1fa2402a02dcd890610 + languageName: node + linkType: hard + "@apidevtools/json-schema-ref-parser@npm:^9.0.6": version: 9.1.2 resolution: "@apidevtools/json-schema-ref-parser@npm:9.1.2" @@ -2609,6 +2625,13 @@ __metadata: languageName: node linkType: hard +"@babel/runtime@npm:^7.18.3": + version: 7.29.2 + resolution: "@babel/runtime@npm:7.29.2" + checksum: 10c0/30b80a0140d16467792e1bbeb06f655b0dab70407da38dfac7fedae9c859f9ae9d846ef14ad77bd3814c064295fe9b1bc551f1541ea14646ae9f22b71a8bc17a + languageName: node + linkType: hard + "@babel/runtime@npm:^7.18.6": version: 7.24.0 resolution: "@babel/runtime@npm:7.24.0" @@ -4611,6 +4634,28 @@ __metadata: languageName: unknown linkType: soft +"@hyperdx/hdx-eval@workspace:packages/hdx-eval": + version: 0.0.0-use.local + resolution: "@hyperdx/hdx-eval@workspace:packages/hdx-eval" + dependencies: + "@anthropic-ai/sdk": "npm:^0.92.0" + "@clickhouse/client": "npm:^1.12.1" + "@dotenvx/dotenvx": "npm:^1.51.1" + "@types/jest": "npm:^29.5.14" + "@types/ms": "npm:^0.7.31" + "@types/node": "npm:^22.15.18" + commander: "npm:^12.1.0" + jest: "npm:^30.2.0" + ms: "npm:^2.1.3" + rimraf: "npm:^4.4.1" + ts-jest: "npm:^29.4.5" + tsx: "npm:^4.19.0" + typescript: "npm:^5.9.3" + bin: + hdx-eval: ./dist/cli.js + languageName: unknown + linkType: soft + "@hyperdx/instrumentation-exception@npm:^0.1.0": version: 0.1.0 resolution: "@hyperdx/instrumentation-exception@npm:0.1.0" @@ -19704,6 +19749,16 @@ __metadata: languageName: node linkType: hard +"json-schema-to-ts@npm:^3.1.1": + version: 3.1.1 + resolution: "json-schema-to-ts@npm:3.1.1" + dependencies: + "@babel/runtime": "npm:^7.18.3" + ts-algebra: "npm:^2.0.0" + checksum: 10c0/609bae04aa5e860a11b6d30ccf41445fae1c7f66fb600c1d170257cf33aa468aa9d03aa046428c3688aff0ff450c2b0c76584b66fa4a5d0da8e33799e4c439a6 + languageName: node + linkType: hard + "json-schema-traverse@npm:^0.4.1": version: 0.4.1 resolution: "json-schema-traverse@npm:0.4.1" @@ -27249,6 +27304,13 @@ __metadata: languageName: node linkType: hard +"ts-algebra@npm:^2.0.0": + version: 2.0.0 + resolution: "ts-algebra@npm:2.0.0" + checksum: 10c0/4ae93bec1bada635bba425854eec323dad50b6ffe86bc04ad2d7f9ce3fb129d673dcf483e19a6e70d07a3a9083e6a0a7f4e004bb8d2164cddc60cc9540ba187f + languageName: node + linkType: hard + "ts-api-utils@npm:^2.1.0": version: 2.1.0 resolution: "ts-api-utils@npm:2.1.0" From 7e7159a5e87dca0aef0b571bf61e35d64f84b412 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 10 Jun 2026 07:57:48 -0600 Subject: [PATCH 038/728] fix(mcp): improve error hints and improve readonly mode for query safety settings (#2433) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Improves two error hints that fire when agents hit ClickHouse errors during investigations: - **DateTime64 cast errors** now recommend `parseDateTime64BestEffort()` (works on both DateTime and DateTime64 columns) instead of `toDateTime64('...', 9)` (DateTime64-only) - **Unknown column/identifier errors** now hint to call `describe_source` before retrying MCP query tools sent `readonly=1` alongside `max_execution_time` and `max_result_rows`, but ClickHouse's readonly mode 1 rejects all setting changes — so the safety limits were silently ignored. This switches to `readonly=2` (blocks writes, allows setting overrides) and adds an integration test that catches the regression. ### Eval results Ran the full 5-scenario eval suite (5 runs each, feature vs main) with synchronized seed data. Overall **+3% improvement** across the board: | Scenario | Feature | Main | Δ | |---|---|---|---| | error-root-cause | 99% | 95% | +4% | | latency-spike | 83% | 70% | **+14%** | | noisy-signals | 66% | 60% | +6% | | segmented-regression | 67% | 79% | -12% | | service-health-check | 59% | 56% | +3% | The latency-spike scenario saw the largest gain (+14%) — the improved DateTime64 hint helps agents recover from timestamp cast errors in one retry instead of 2-3. The segmented-regression dip is within variance (judge scores are tied at 78%; the gap is entirely from tool-error penalty noise). ### Related issues - [HDX-4473](https://linear.app/clickhouse/issue/HDX-4473/improve-datetime64-error-hint-to-recommend-universal-safe) — Improve DateTime64 error hint to recommend parseDateTime64BestEffort - [HDX-4474](https://linear.app/clickhouse/issue/HDX-4474/strengthen-describesource-nudge-and-add-unknown-column-error-hint) — Add unknown-column error hint (nudge portion deferred — eval showed proactive nudges hurt turn budget) - [HDX-4472](https://linear.app/clickhouse/issue/HDX-4472/fix-per-select-where-timestamp-example-to-use) — Fix per-select where timestamp example (deferred — eval showed inline warnings increase agent verbosity) - [HDX-4475](https://linear.app/clickhouse/issue/HDX-4475/restructure-sql-tool-timestamp-docs-with-clear-macro-hierarchy-and) — Restructure SQL tool timestamp docs (deferred — eval showed restructured docs increase SQL usage without improving scores) ### Other Fixes `deniedTools` in the eval setup template to use correct `clickstack_*` tool name prefix (the old `hyperdx_*` names didn't match any tools, so no tools were actually denied). --- .changeset/mcp-error-hints-readonly.md | 16 ++++++++++++ packages/api/src/mcp/__tests__/query.test.ts | 4 +-- .../api/src/mcp/__tests__/queryTool.test.ts | 25 +++++++++++++++++-- packages/api/src/mcp/tools/query/helpers.ts | 24 +++++++++++++----- packages/hdx-eval/src/hyperdx/setup.ts | 21 +++++++++------- 5 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 .changeset/mcp-error-hints-readonly.md diff --git a/.changeset/mcp-error-hints-readonly.md b/.changeset/mcp-error-hints-readonly.md new file mode 100644 index 0000000000..095316d290 --- /dev/null +++ b/.changeset/mcp-error-hints-readonly.md @@ -0,0 +1,16 @@ +--- +'@hyperdx/api': patch +--- + +fix(mcp): improve error hints and fix readonly mode for query safety settings + +Switch MCP ClickHouse safety settings from readonly=1 to readonly=2 so +max_execution_time and max_result_rows are actually applied (readonly=1 +silently rejects all setting changes). + +Improve DateTime64 cast error hint to recommend parseDateTime64BestEffort() +which works on both DateTime and DateTime64 columns, replacing +toDateTime64() which only works on DateTime64. + +Add error hint for unknown column/identifier errors directing agents to +call describe_source before retrying. diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index 1a9c7ac55c..0307b96813 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -188,7 +188,7 @@ describe('errorHint', () => { "Cannot convert string '2025-01-01T00:00:00Z' to type DateTime64(9)", ); expect(hint).not.toBeNull(); - expect(hint).toContain('toDateTime64'); + expect(hint).toContain('parseDateTime64BestEffort'); }); it('should match DateTime64 parse errors', () => { @@ -196,7 +196,7 @@ describe('errorHint', () => { "Cannot parse string '2025-01-01' as type DateTime64", ); expect(hint).not.toBeNull(); - expect(hint).toContain('toDateTime64'); + expect(hint).toContain('parseDateTime64BestEffort'); }); it('should match AS alias syntax errors with word boundary', () => { diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index a63571f1d7..e7ff5281b9 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -782,7 +782,7 @@ describe('MCP Query Tools', () => { expect(parsed.result?.data?.[0]?.value).toBe(100000); }); - it('should propagate readonly=1 to ClickHouse', async () => { + it('should propagate readonly=2 to ClickHouse', async () => { const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), sql: "SELECT getSetting('readonly') AS value", @@ -790,7 +790,28 @@ describe('MCP Query Tools', () => { expect(result.isError).toBeFalsy(); const parsed = JSON.parse(getFirstText(result)); - expect(parsed.result?.data?.[0]?.value).toBe(1); + expect(parsed.result?.data?.[0]?.value).toBe(2); + }); + + it('should apply all safety settings together without readonly conflicts', async () => { + // readonly=1 rejects setting changes, so max_execution_time and + // max_result_rows would be silently ignored. readonly=2 allows + // setting changes while still blocking writes. This test verifies + // all three settings are applied in a single query. + const result = await callTool(client, 'clickstack_sql', { + connectionId: connection._id.toString(), + sql: `SELECT + getSetting('readonly') AS readonly_mode, + getSetting('max_execution_time') AS max_exec_time, + getSetting('max_result_rows') AS max_rows`, + }); + + expect(result.isError).toBeFalsy(); + const parsed = JSON.parse(getFirstText(result)); + const row = parsed.result?.data?.[0]; + expect(row?.readonly_mode).toBe(2); + expect(row?.max_exec_time).toBe(30); + expect(row?.max_rows).toBe(100000); }); }); }); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index c6782a4227..5a339bf7de 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -63,11 +63,13 @@ export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; // ─── Safety limits ─────────────────────────────────────────────────────────── -/** ClickHouse settings applied to all MCP query-tool executions. */ +/** ClickHouse settings applied to all MCP query-tool executions. + * readonly=2 so max_execution_time and max_result_rows can be set + * (readonly=1 rejects all setting changes). */ const MCP_CLICKHOUSE_SETTINGS = { max_execution_time: 30, max_result_rows: '100000', - readonly: 1, + readonly: 2, } as const; /** @@ -451,10 +453,10 @@ export function errorHint(msg: string): string | null { /Cannot (convert|parse) string .* (to|as) (type )?DateTime64/i.test(msg) ) { return ( - "Wrap ISO timestamps with `toDateTime64('YYYY-MM-DD HH:MM:SS', 9)` " + - 'or use the supplied macros: `$__timeFilter(Timestamp)` (the easiest), ' + - '`{startDateMilliseconds:Int64}`, `{endDateMilliseconds:Int64}`. ' + - 'Bare ISO 8601 strings will NOT auto-cast to DateTime64.' + "Wrap ISO timestamps with `parseDateTime64BestEffort('YYYY-MM-DDTHH:MM:SSZ')` — " + + 'this works for both DateTime and DateTime64 columns. For the sql tool, prefer ' + + '`$__timeFilter(Timestamp)` which handles casting automatically. ' + + 'Bare ISO 8601 strings will NOT auto-cast to DateTime/DateTime64.' ); } if (/Syntax error.*\bAS\b/.test(msg)) { @@ -479,5 +481,15 @@ export function errorHint(msg: string): string | null { 'Add a LIMIT, narrow the time range, or add filters to reduce the result set.' ); } + if ( + /Unknown (expression|identifier)|UNKNOWN_IDENTIFIER|Missing columns/i.test( + msg, + ) + ) { + return ( + 'Call clickstack_describe_source to discover available columns and ' + + 'map attribute keys before retrying.' + ); + } return null; } diff --git a/packages/hdx-eval/src/hyperdx/setup.ts b/packages/hdx-eval/src/hyperdx/setup.ts index 7450a37855..091a9252ff 100644 --- a/packages/hdx-eval/src/hyperdx/setup.ts +++ b/packages/hdx-eval/src/hyperdx/setup.ts @@ -124,15 +124,18 @@ export async function runSetup(opts: SetupOptions): Promise { label: 'HyperDX', brandTerms: ['HyperDX', 'hyperdx'], deniedTools: [ - 'mcp__hyperdx__hyperdx_delete_dashboard', - 'mcp__hyperdx__hyperdx_get_dashboard', - 'mcp__hyperdx__hyperdx_save_dashboard', - 'mcp__hyperdx__hyperdx_query_tile', - 'mcp__hyperdx__hyperdx_get_saved_search', - 'mcp__hyperdx__hyperdx_save_saved_search', - 'mcp__hyperdx__hyperdx_get_alert', - 'mcp__hyperdx__hyperdx_get_webhook', - 'mcp__hyperdx__hyperdx_save_alert', + 'mcp__hyperdx__clickstack_delete_dashboard', + 'mcp__hyperdx__clickstack_get_dashboard', + 'mcp__hyperdx__clickstack_get_dashboard_tile', + 'mcp__hyperdx__clickstack_save_dashboard', + 'mcp__hyperdx__clickstack_patch_dashboard', + 'mcp__hyperdx__clickstack_search_dashboards', + 'mcp__hyperdx__clickstack_query_tile', + 'mcp__hyperdx__clickstack_get_saved_search', + 'mcp__hyperdx__clickstack_save_saved_search', + 'mcp__hyperdx__clickstack_get_alert', + 'mcp__hyperdx__clickstack_get_webhook', + 'mcp__hyperdx__clickstack_save_alert', ], }; From 81e524c2c4a588717cdf07668e62e261700fb047 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 10 Jun 2026 07:36:48 -0700 Subject: [PATCH 039/728] fix(charts): cap group-by time-series to top-N series to prevent OOM (#2429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary High-cardinality group-by time charts could pull hundreds of thousands of series into a single tile — only ~60 were ever drawn (`HARD_LINES_LIMIT`), but every series was still fetched, JSON-parsed, and zero-filled into a dense buckets×series matrix in memory, OOMing the browser tab. This adds an opt-in query-level `seriesLimit`: when set on a group-by + granularity chart, `renderChartConfig` emits a CTE (`__hdx_series_limit`) that keeps only the top-N series ranked by max value in any bucket and restricts the outer query to those groups (mirroring the existing `aggFn=increase` TopGroups path). The time-chart display path (`convertToTimeChartConfig`) sets this cap, while alerts and other `renderChartConfig` consumers leave it unset so their series evaluation is unchanged. The cap is also **configurable per team**: a new `seriesLimit` team setting (alongside the existing ClickHouse client settings) lets teams tune it, defaulting to 100 when unset and floored at 1 (no upper bound, so teams can intentionally fetch more series). The default and the rendered-line cap (`HARD_LINES_LIMIT`) move together; a team value only governs how many series are *fetched*, with any surplus available in the series selector. Verified end-to-end against real ClickHouse (50 series → top 5) plus unit tests; all existing SQL snapshots are unchanged. ### How to test on Vercel preview **Preview routes:** /team **Steps:** 1. Open /team. 2. Scroll to the "ClickHouse Client Settings" section. 3. Verify a "Time Chart Series Limit" setting is listed with a default of 100. ### References - Linear Issue: HDX-4499 - Related PRs: --- .changeset/time-chart-series-limit.md | 7 + packages/api/src/models/team.ts | 1 + packages/app/src/ChartUtils.tsx | 7 + packages/app/src/HDXMultiSeriesTimeChart.tsx | 8 +- packages/app/src/__tests__/ChartUtils.test.ts | 31 ++ packages/app/src/components/DBTimeChart.tsx | 7 +- .../src/components/SearchTotalCountChart.tsx | 8 +- .../TeamSettings/TeamQueryConfigSection.tsx | 11 + packages/app/src/defaults.ts | 1 + .../__tests__/queryChartConfig.int.test.ts | 289 ++++++++++++++++++ .../src/__tests__/renderChartConfig.test.ts | 158 ++++++++++ .../src/core/renderChartConfig.ts | 106 ++++++- packages/common-utils/src/types.ts | 3 + 13 files changed, 626 insertions(+), 11 deletions(-) create mode 100644 .changeset/time-chart-series-limit.md diff --git a/.changeset/time-chart-series-limit.md b/.changeset/time-chart-series-limit.md new file mode 100644 index 0000000000..27aae22282 --- /dev/null +++ b/.changeset/time-chart-series-limit.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +"@hyperdx/api": patch +--- + +feat(charts): cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys. The cap defaults to 100 (the number of series rendered) and is configurable per team via a new "Time Chart Series Limit" setting; series beyond the cap remain available in the series selector. diff --git a/packages/api/src/models/team.ts b/packages/api/src/models/team.ts index 4d6b620162..15e92ec72c 100644 --- a/packages/api/src/models/team.ts +++ b/packages/api/src/models/team.ts @@ -42,6 +42,7 @@ export default mongoose.model( fieldMetadataDisabled: Boolean, parallelizeWhenPossible: Boolean, filterKeysFetchLimit: Number, + seriesLimit: Number, }, { timestamps: true, diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index 544c6762d5..b284670182 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -37,6 +37,7 @@ import { notifications } from '@mantine/notifications'; import DateRangeIndicator from './components/charts/DateRangeIndicator'; import { MVOptimizationExplanationResult } from './hooks/useMVOptimizationExplanation'; +import { DEFAULT_SERIES_LIMIT } from './defaults'; import { getMetricNameSql } from './otelSemanticConventions'; import { AggFn, TableChartSeries, TimeChartSeries } from './types'; import { NumberFormat } from './types'; @@ -103,9 +104,14 @@ function getTimeChartDateRange( : getAlignedDateRange(dateRange, granularity); } +export const MAX_TIME_CHART_SERIES = DEFAULT_SERIES_LIMIT; + export function convertToTimeChartConfig( config: ChartConfigWithDateRange, + teamSeriesLimit?: number, ): ChartConfigWithDateRange { + const seriesLimit = Math.max(1, teamSeriesLimit ?? MAX_TIME_CHART_SERIES); + const granularity = getTimeChartGranularity( config.granularity, config.dateRange, @@ -133,6 +139,7 @@ export function convertToTimeChartConfig( dateRangeEndInclusive, granularity, limit: { limit: 100000 }, + seriesLimit, } : { ...config, diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index 1a08dc1785..d97285eda4 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -29,7 +29,11 @@ import { ChartTooltipContainer, ChartTooltipItem, } from './components/charts/ChartTooltip'; -import { LineData, toStartOfInterval } from './ChartUtils'; +import { + LineData, + MAX_TIME_CHART_SERIES, + toStartOfInterval, +} from './ChartUtils'; import { FormatTime, useFormatTime } from './useFormatTime'; import styles from '../styles/HDXLineChart.module.scss'; @@ -317,7 +321,7 @@ const LegendRenderer = memo<{ ); }); -export const HARD_LINES_LIMIT = 60; +export const HARD_LINES_LIMIT = MAX_TIME_CHART_SERIES; const StackedBarWithOverlap = (props: BarProps) => { const { x, y, width, height, fill } = props; diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index 2d3d5834ed..ab9acf58e7 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -11,6 +11,7 @@ import { formatResponseForPieChart, formatResponseForTimeChart, } from '@/ChartUtils'; +import { DEFAULT_SERIES_LIMIT } from '@/defaults'; import { COLORS } from '@/utils'; // Anchor info/error to concrete hexes rather than `getChartColorInfo()` / @@ -806,6 +807,36 @@ describe('ChartUtils', () => { expect(granularityFromFunction).toBe('5 minute'); }); + + const seriesLimitConfig = { + granularity: '5 minute', + dateRange: [ + new Date('2025-11-26T00:00:00Z'), + new Date('2025-11-27T00:00:00Z'), + ], + } as BuilderChartConfigWithDateRange; + + // seriesLimit lives on the builder member of the ChartConfigWithDateRange + // union, so narrow the result before reading it. + const seriesLimitOf = (teamSeriesLimit?: number) => + ( + convertToTimeChartConfig( + seriesLimitConfig, + teamSeriesLimit, + ) as BuilderChartConfigWithDateRange + ).seriesLimit; + + it('defaults seriesLimit to DEFAULT_SERIES_LIMIT when no team value is given', () => { + expect(seriesLimitOf()).toBe(DEFAULT_SERIES_LIMIT); + }); + + it('uses the team seriesLimit when provided', () => { + expect(seriesLimitOf(5)).toBe(5); + }); + + it('passes a large team seriesLimit through unbounded', () => { + expect(seriesLimitOf(100000)).toBe(100000); + }); }); describe('convertToNumberChartConfig', () => { diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index c0f2b26616..9ae6f1a54c 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -287,9 +287,11 @@ function DBTimeChartComponent({ fillNulls, } = useTimeChartSettings(config); + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const queriedConfig = useMemo( - () => convertToTimeChartConfig(config), - [config], + () => convertToTimeChartConfig(config, me?.team?.seriesLimit), + [config, me?.team?.seriesLimit], ); // Determine whether the config can be optimized with an MV, to determine whether @@ -299,7 +301,6 @@ function DBTimeChartComponent({ const { data: mvOptimizationData } = useMVOptimizationExplanation(builderQueriedConfig); - const { data: me, isLoading: isLoadingMe } = api.useMe(); const { data, isLoading, isError, error, isPlaceholderData, isSuccess } = useQueriedChartConfig(queriedConfig, { placeholderData: (prev: any) => prev, diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index edceee9741..a6be03f89d 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -35,13 +35,13 @@ export function useSearchTotalCount( enableParallelQueries?: boolean; } = {}, ) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + // queriedConfig, queryKey, and enableQueryChunking match DBTimeChart so that react query can de-dupe these queries. const queriedConfig = useMemo( - () => convertToTimeChartConfig(config), - [config], + () => convertToTimeChartConfig(config, me?.team?.seriesLimit), + [config, me?.team?.seriesLimit], ); - - const { data: me, isLoading: isLoadingMe } = api.useMe(); const { data: totalCountData, isLoading, diff --git a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx index de850d2d6f..2cfc66c9b4 100644 --- a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx +++ b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx @@ -24,6 +24,7 @@ import { DEFAULT_FILTER_KEYS_FETCH_LIMIT_WITH_MVS, DEFAULT_QUERY_TIMEOUT, DEFAULT_SEARCH_ROW_LIMIT, + DEFAULT_SERIES_LIMIT, } from '@/defaults'; import { useBrandDisplayName } from '@/theme/ThemeProvider'; @@ -343,6 +344,16 @@ export default function TeamQueryConfigSection() { type="boolean" displayValue={value => (value ? 'Enabled' : 'Disabled')} /> + diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 55bf3dafc2..c3537c5911 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -5,6 +5,7 @@ export const DEFAULT_SEARCH_ROW_LIMIT = 200; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 20; export const DEFAULT_FILTER_KEYS_FETCH_LIMIT_WITH_MVS = 100; +export const DEFAULT_SERIES_LIMIT = 100; export function searchChartConfigDefaults( team: any | undefined | null, diff --git a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts index 65e09e8d42..fc50abd854 100644 --- a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts +++ b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts @@ -172,4 +172,293 @@ describe('queryChartConfig Integration Tests', () => { expect(metaNames).toContain('__hdx_time_bucket'); expect(metaNames.indexOf('__hdx_time_bucket')).toBeGreaterThanOrEqual(3); }); + + // End-to-end: the cap CTE is valid SQL and restricts a high-cardinality + // group-by to the top N by max value in any bucket. + it('caps high-cardinality group-by series to the top N via seriesLimit', async () => { + const SERIES_TABLE = 'logs_series_limit_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${SERIES_TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + ServiceName String CODEC(ZSTD(1)), + Value Float64 CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (ServiceName, Timestamp)`, + }); + + // 50 distinct series; Value == index so the top 5 by value are svc-45..49. + const rows = Array.from({ length: 50 }, (_, i) => ({ + Timestamp: '2025-04-15 00:10:00', + ServiceName: `svc-${String(i).padStart(2, '0')}`, + Value: i, + })); + await client.insert({ + table: `${DATABASE}.${SERIES_TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: SERIES_TABLE }, + select: [ + { + aggFn: 'max', + aggCondition: '', + aggConditionLanguage: 'sql', + valueExpression: 'Value', + }, + ], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + seriesLimit: 5, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + // Without the cap this would be 50 distinct services. + const services = new Set( + (result.data as Array<{ ServiceName: string }>).map(r => r.ServiceName), + ); + expect(services.size).toBeLessThanOrEqual(5); + expect([...services].sort()).toEqual([ + 'svc-45', + 'svc-46', + 'svc-47', + 'svc-48', + 'svc-49', + ]); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${SERIES_TABLE}`, + }); + } + }); + + // Regression: a comma-separated string group-by (with a Map access) must split + // per-column (not emit toString(col1, col2)); empty-string groups are kept. + it('handles a multi-column string group-by (with Map access) under seriesLimit', async () => { + const TABLE = 'logs_string_gb_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + LogAttributes Map(String, String) CODEC(ZSTD(1)), + ServiceName String CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (ServiceName, Timestamp)`, + }); + + const ts = '2025-04-15 00:10:00'; + const rows = [ + // capA/svc1 (5 rows), capB/svc2 (3 rows) — the two non-empty series. + ...Array.from({ length: 5 }, () => ({ + Timestamp: ts, + LogAttributes: { 'agentToServer.capabilities': 'capA' }, + ServiceName: 'svc1', + })), + ...Array.from({ length: 3 }, () => ({ + Timestamp: ts, + LogAttributes: { 'agentToServer.capabilities': 'capB' }, + ServiceName: 'svc2', + })), + // Missing capability key (Map access -> '') for svc3 — largest by count. + // Empty-string groups are kept, so this ranks #1 and survives the cap. + ...Array.from({ length: 10 }, () => ({ + Timestamp: ts, + LogAttributes: {}, + ServiceName: 'svc3', + })), + ]; + await client.insert({ + table: `${DATABASE}.${TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + // Comma-separated string group-by — the shape that previously errored. + groupBy: "LogAttributes['agentToServer.capabilities'],ServiceName", + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + // Cap to 2 so the ranking is observable: by count svc3 (10) > svc1 (5) + // > svc2 (3), so the top 2 are svc3 and svc1; svc2 is dropped. + seriesLimit: 2, + }; + + // The query must execute without a ClickHouse error (the original bug). + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const services = new Set( + (result.data as Array<{ ServiceName: string }>).map(r => r.ServiceName), + ); + // svc3 (empty capability) is kept and ranks #1; the cap drops svc2. + expect([...services].sort()).toEqual(['svc1', 'svc3']); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE}`, + }); + } + }); + + // NULL group components are dropped from the ranking; otherwise a NULL group + // could take a slot the NULL-unsafe outer `tuple() IN (...)` can never fill. + it('excludes NULL group components from the series cap', async () => { + const TABLE = 'logs_nullable_gb_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + Region Nullable(String) CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (Timestamp)`, + }); + + const ts = '2025-04-15 00:10:00'; + const rows = [ + // 'us' has 5 rows; the NULL-region group has 10 (the largest by count). + ...Array.from({ length: 5 }, () => ({ Timestamp: ts, Region: 'us' })), + ...Array.from({ length: 10 }, () => ({ Timestamp: ts, Region: null })), + ]; + await client.insert({ + table: `${DATABASE}.${TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: [{ aggCondition: '', valueExpression: 'Region' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + // Only one slot: without the NULL filter the (larger) NULL group would + // claim it and then match nothing, yielding an empty chart. + seriesLimit: 1, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const regions = (result.data as Array<{ Region: string | null }>).map( + r => r.Region, + ); + // 'us' (top non-null) takes the slot; the NULL group is excluded. + expect(new Set(regions)).toEqual(new Set(['us'])); + expect(regions).not.toContain(null); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE}`, + }); + } + }); + + // Multi-column array group-by with an alias: the 2-column tuple()/IN executes + // (alias stripped in the CTE — a leaked `AS "reg"` there is a syntax error) + // and the alias is preserved as the output column. + it('handles a multi-column array group-by with an alias under seriesLimit', async () => { + const TABLE = 'logs_array_alias_gb_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + Region String CODEC(ZSTD(1)), + ServiceName String CODEC(ZSTD(1)), + Value Float64 CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (Timestamp)`, + }); + + const ts = '2025-04-15 00:10:00'; + const rows = [ + { Timestamp: ts, Region: 'us', ServiceName: 'svc1', Value: 10 }, + { Timestamp: ts, Region: 'eu', ServiceName: 'svc2', Value: 5 }, + { Timestamp: ts, Region: 'ap', ServiceName: 'svc3', Value: 1 }, + ]; + await client.insert({ + table: `${DATABASE}.${TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE }, + select: [ + { + aggFn: 'max', + aggCondition: '', + aggConditionLanguage: 'sql', + valueExpression: 'Value', + }, + ], + groupBy: [ + { aggCondition: '', valueExpression: 'Region', alias: 'reg' }, + { aggCondition: '', valueExpression: 'ServiceName' }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + // Top 2 by max(Value): (us,svc1)=10 and (eu,svc2)=5; (ap,svc3)=1 dropped. + seriesLimit: 2, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const services = new Set( + (result.data as Array<{ ServiceName: string }>).map(r => r.ServiceName), + ); + expect(services).toEqual(new Set(['svc1', 'svc2'])); + // The alias survives in the output even though it is stripped in the CTE. + expect(result.meta?.some(m => m.name === 'reg')).toBe(true); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE}`, + }); + } + }); }); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 9816c0f935..13a588900c 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -469,6 +469,164 @@ describe('renderChartConfig', () => { expect(actual).not.toContain('TopGroups'); }); + describe('seriesLimit (group-by series cap)', () => { + const baseLogsConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'logs' }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-13')], + granularity: '5 minute', + }; + + it('restricts to the top N group-by series via a CTE when seriesLimit is set', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { ...baseLogsConfig, seriesLimit: 60 }, + mockMetadata, + querySettings, + ), + ); + // A ranking CTE keeps the top N groups by max value in any bucket. + expect(sql).toContain('__hdx_series_limit'); + expect(sql).toMatch(/ORDER\s+BY\s+max\(`__hdx_series_rank`\)\s+DESC/); + expect(sql).toContain('LIMIT 60'); + // The outer query is restricted to those groups via an IN subquery. + expect(sql).toMatch( + /tuple\(ServiceName\)\s+IN\s*\(\s*SELECT\s+`group`\s+FROM\s+`__hdx_series_limit`\)/, + ); + // Groups with a NULL component are excluded; empty-string groups are kept + // (no `!= ''` check). + expect(sql).toMatch(/ServiceName\s+IS\s+NOT\s+NULL/); + expect(sql).not.toMatch(/toString\(ServiceName\)\s*!=\s*''/); + }); + + it('does not emit a series-limit CTE when seriesLimit is unset (e.g. alert evaluation)', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig(baseLogsConfig, mockMetadata, querySettings), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + + it('does not emit a series-limit CTE without a group-by', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { ...baseLogsConfig, groupBy: undefined, seriesLimit: 60 }, + mockMetadata, + querySettings, + ), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + + it('does not emit a series-limit CTE without granularity', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { ...baseLogsConfig, granularity: undefined, seriesLimit: 60 }, + mockMetadata, + querySettings, + ), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + + it('packs a multi-column group-by into a tuple for the series-limit CTE', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...baseLogsConfig, + groupBy: [ + { aggCondition: '', valueExpression: 'ServiceName' }, + { aggCondition: '', valueExpression: 'TraceId' }, + ], + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('__hdx_series_limit'); + expect(sql).toMatch(/tuple\(\s*ServiceName\s*,\s*TraceId\s*\)/); + }); + + it('strips group-by aliases inside the series-limit CTE tuple and null filter', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...baseLogsConfig, + groupBy: [ + { + aggCondition: '', + valueExpression: 'ServiceName', + alias: 'svc', + }, + ], + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('__hdx_series_limit'); + // tuple() and `IS NOT NULL` must use the bare expression, not `ServiceName + // AS "svc"` (which would be invalid SQL there). + expect(sql).toMatch( + /tuple\(ServiceName\)\s+IN\s*\(\s*SELECT\s+`group`\s+FROM\s+`__hdx_series_limit`\)/, + ); + expect(sql).not.toContain('tuple(ServiceName AS'); + expect(sql).not.toMatch(/ServiceName\s+AS\s+"svc"\s+IS\s+NOT\s+NULL/); + }); + + it('splits a comma-separated string group-by into per-column null checks', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...baseLogsConfig, + groupBy: "LogAttributes['agentToServer.capabilities'],ServiceName", + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('__hdx_series_limit'); + // Each column gets its own NULL check, split on the top-level comma — not + // the comma inside Map['...']. + expect(sql).toMatch( + /LogAttributes\[['"]agentToServer\.capabilities['"]\]\s+IS\s+NOT\s+NULL/, + ); + expect(sql).toMatch(/ServiceName\s+IS\s+NOT\s+NULL/); + // Regression: must NOT emit a two-argument toString of both columns (the + // original bug that prompted the split). + expect(sql).not.toMatch(/toString\([^)]*,/); + // Both columns are packed into the tuple for the IN predicate. + expect(sql).toMatch( + /tuple\(\s*LogAttributes\[['"]agentToServer\.capabilities['"]\]\s*,\s*ServiceName\s*\)\s+IN\s*\(\s*SELECT\s+`group`\s+FROM\s+`__hdx_series_limit`\)/, + ); + }); + + it('does not emit a series-limit CTE for a metric source', async () => { + // Metric configs are rewritten to query a Bucketed CTE (no real source + // table to re-scan), so the cap is gated off even with seriesLimit set. + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...gaugeConfiguration, + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + }); + it('should throw when aggFn=increase is used on a non-Sum metric', async () => { const config: ChartConfigWithOptDateRange = { displayType: DisplayType.Line, diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 1ae4198c1b..c25e985890 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -1210,6 +1210,94 @@ async function renderGroupBy( ); } +async function renderSeriesLimitCte( + chartConfig: BuilderChartConfigWithOptDateRangeEx, + metadata: Metadata, + { + from, + where, + groupBy, + }: { from: ChSql; where: ChSql; groupBy: ChSql | undefined }, +): Promise<{ cte: ChSql; predicate: ChSql } | undefined> { + const { seriesLimit } = chartConfig; + if ( + seriesLimit == null || + !isUsingGroupBy(chartConfig) || + !isUsingGranularity(chartConfig) || + chartConfig.selectGroupBy === false || + // Skip CTE/metric sources (no real table to re-scan) and string selects. + !chartConfig.from?.databaseName || + !chartConfig.from?.tableName || + !Array.isArray(chartConfig.select) || + chartConfig.select.length === 0 || + groupBy == null + ) { + return undefined; + } + + // One ChSql per group-by column (groupBy may be an array or a comma-separated + // string). splitAndTrimWithBracket respects []/()/quotes so it won't split + // inside Map['a,b']; the per-column null filter below needs them separated. + let groupByCols: ChSql[]; + if (typeof chartConfig.groupBy === 'string') { + groupByCols = splitAndTrimWithBracket(chartConfig.groupBy).map( + col => chSql`${{ UNSAFE_RAW_SQL: col }}`, + ); + } else { + // Strip aliases: these go inside tuple(...)/`IS NOT NULL`, where an + // `AS "alias"` suffix is a syntax error (unlike the outer GROUP BY). + const rendered = await renderSelectList( + chartConfig.groupBy.map(col => ({ ...col, alias: undefined })), + chartConfig, + metadata, + ); + groupByCols = Array.isArray(rendered) ? rendered : [rendered]; + } + const groupByTuple = concatChSql(',', groupByCols); + + // Rank by the chart's first aggregate (alias stripped — we add our own). + const firstSelect = chartConfig.select[0]; + const rankSelectList = + typeof firstSelect === 'string' + ? firstSelect + : [{ ...firstSelect, alias: undefined }]; + const rankRendered = await renderSelectList( + rankSelectList, + chartConfig, + metadata, + ); + const rankValue = Array.isArray(rankRendered) + ? rankRendered[0] + : rankRendered; + + // Drop NULL components only (no-op on non-nullable columns). + const groupByNotNullFilter = concatChSql( + ' AND ', + groupByCols.map(g => chSql`${g} IS NOT NULL`), + ); + const innerWhere = where.sql + ? concatChSql(' AND ', where, groupByNotNullFilter) + : groupByNotNullFilter; + + // Per-(group, bucket) aggregate, then max per group, keeping the top N. + const cte = chSql`\`__hdx_series_limit\` AS ( + SELECT \`group\` + FROM ( + SELECT tuple(${groupByTuple}) AS \`group\`, ${rankValue} AS \`__hdx_series_rank\` + FROM ${from} + WHERE ${innerWhere} + GROUP BY ${groupBy} + ) + GROUP BY \`group\` + ORDER BY max(\`__hdx_series_rank\`) DESC, \`group\` + LIMIT ${{ Int32: seriesLimit }} + )`; + + const predicate = chSql`tuple(${groupByTuple}) IN (SELECT \`group\` FROM \`__hdx_series_limit\`)`; + + return { cte, predicate }; +} + async function renderHaving( chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, @@ -2065,10 +2153,10 @@ export async function renderChartConfig( : undefined), }; - const withClauses = await renderWith(chartConfig, metadata, querySettings); + let withClauses = await renderWith(chartConfig, metadata, querySettings); const select = await renderSelect(chartConfig, metadata); const from = renderFrom(chartConfig); - const where = await renderWhere(chartConfig, metadata); + let where = await renderWhere(chartConfig, metadata); const groupBy = await renderGroupBy(chartConfig, metadata); const having = await renderHaving(chartConfig, metadata); const orderBy = renderOrderBy(chartConfig); @@ -2076,6 +2164,20 @@ export async function renderChartConfig( const limit = renderLimit(chartConfig); const settings = renderSettings(chartConfig, querySettings); + const seriesCap = await renderSeriesLimitCte(chartConfig, metadata, { + from, + where, + groupBy, + }); + if (seriesCap) { + withClauses = withClauses + ? concatChSql(',', withClauses, seriesCap.cte) + : seriesCap.cte; + where = where.sql + ? concatChSql(' AND ', where, seriesCap.predicate) + : seriesCap.predicate; + } + return concatChSql(' ', [ chSql`${withClauses?.sql ? chSql`WITH ${withClauses}` : ''}`, chSql`SELECT ${select}`, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 95c0f157df..dc38e71b2f 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -275,6 +275,7 @@ export const SelectSQLStatementSchema = z.object({ havingLanguage: SearchConditionLanguageSchema.optional(), orderBy: SortSpecificationListSchema.optional(), limit: LimitSchema.optional(), + seriesLimit: z.number().int().positive().optional(), }); export type SQLInterval = z.infer; @@ -1522,6 +1523,7 @@ export const TeamClickHouseSettingsSchema = z.object({ metadataMaxRowsToRead: z.number().optional(), parallelizeWhenPossible: z.boolean().optional(), filterKeysFetchLimit: z.number().optional(), + seriesLimit: z.number().int().positive().optional(), }); /** Accepts null to unset (reset to default) a setting. */ @@ -1532,6 +1534,7 @@ export const TeamClickHouseSettingsUpdateSchema = z.object({ metadataMaxRowsToRead: z.number().nullish(), parallelizeWhenPossible: z.boolean().nullish(), filterKeysFetchLimit: z.number().nullish(), + seriesLimit: z.number().int().positive().nullish(), }); export type TeamClickHouseSettingsUpdate = z.infer< typeof TeamClickHouseSettingsUpdateSchema From 213077568855c968dab1f826164e3af98c0814d1 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Wed, 10 Jun 2026 08:14:38 -0700 Subject: [PATCH 040/728] fix(row-panel): emit Map['key'] subscript for Map columns in mergePath (#2357) ## Summary `mergePath` in `packages/app/src/utils.ts` converted numeric path segments to 1-based array subscripts (`[N+1]`) regardless of whether the parent column was a Map or an Array. On a `Map(String, String)` column this produced SQL like `LogAttributes[2]`, which ClickHouse rejects with `Illegal types of arguments: Map(String, String), UInt8 for function arrayElement`. The grid row "expand" view failed for any row whose attribute path included a numeric-looking key under a Map column. The function now accepts a `mapColumns` argument alongside `jsonColumns`. For Map-typed parents, all sub-keys render as string-key subscripts (`Map['1']`) regardless of whether the key looks numeric. Array columns keep their existing 1-based array-index behavior. ## Confirmed impact I traced the original report back to production session data. Two sessions in the 10-day window covered by HDX-4369 hit the exact ClickHouse error above. Both stack traces resolve to the ClickHouse query client in the chunked `_app-*.js` bundle, consistent with the bug class: an autocomplete / sidebar interaction on the search page produces a query that subscripts a Map column with a numeric token. Reproduction patterns observed: - Search view with no `where` clause set, user clicks an attribute in the filter sidebar; autocomplete fetch fires and ClickHouse rejects with the `arrayElement` type error. - Search view filtered by `TraceId = '...'` with a Map-bracket field in the selected columns; same error. Low absolute frequency in the sampled window, but the failure class matches a broader pattern of user-reported "grid expand keeps failing" symptoms, so the cheap fix is worth shipping. Full per-session detail (timestamps, user identities, session IDs, source IDs, exact URLs) is on the Linear ticket. Adjacent failures in the same RUM window look superficially similar but are NOT addressed by this PR: two cases of subscripting a String column (e.g. `SomeStringCol['...']`) and one case of subscripting a JSON column. Different bug class, different fix; flagging for context. ## What this PR changes `mergePath` now accepts a third `mapColumns` argument. The three direct call sites thread it through: - `useAutoCompleteOptions` derives `mapColumns` from the field list via the new `deriveMapColumnsFromFields` helper, matching on the canonical `JSDataType.Map` so wrapped Map types (`LowCardinality(Map(...))`, `Nullable(Map(...))`) are detected too. - `DBRowJsonViewer` accepts an optional `mapColumns` prop, threaded by `DBRowDataPanel` and `DBRowOverviewPanel` from the result-set metadata via a new `getMapColumnNames` helper. - `DBSearchPageFilters` uses a new `useMapColumns` hook (mirrors `useJsonColumns`). The same-bug-class sibling in `DBSearchPageFilters/utils.ts` (`toClickHouseKeyExpression`, the "Load more" SQL path) is fixed in this PR by threading the parsed base name as `mapColumns` into `mergePath`. `parseMapFieldName` already proves the base column is a Map. `mergePath`'s Map and string-key branches now escape backslash and single-quote before interpolating into single-quoted SQL strings, so a Map sub-key like `it's` produces `Map['it\'s']` instead of malformed SQL. The deep-review pass on the previous revision flagged a residual leak through `buildJSONExtractQuery`. When a Map column sub-value is itself a JSON-parseable string, `HyperJson` promotes the child context to `isInParsedJson=true` with `parsedJsonRootPath=[MapCol, key]`. The inner `mergePath` call in `buildJSONExtractQuery` did not receive `mapColumns`, so a numeric sub-key fell through the default branch and re-introduced `Map[N+1]`. The latest commit threads `mapColumns` through `buildJSONExtractQuery` and its four call sites in `DBRowJsonViewer` (Add to Filters, Search, Chart, Toggle Column). ## Explained test case for the reviewer The shortest path from "this PR's claim" to "this PR's fix" is one test in `packages/app/src/__tests__/utils.test.ts`: ```ts it("emits Map['1'] for a numeric segment under a Map column", () => { expect(mergePath(['LogAttributes', '1'], [], ['LogAttributes'])) .toBe("LogAttributes['1']"); }); ``` On `main` this test fails with `LogAttributes[2]`. That's the exact subscript that produced the ClickHouse error in the production sessions. After this PR the test passes because `mergePath` consults `mapColumns` (the third argument) and renders the string-key subscript instead of the array-index subscript. The same shape applies to the other layers: ```ts // DBSearchPageFilters/utils.test.ts: the "Load more" SQL path expect(toClickHouseKeyExpression('LogAttributes.1', 'String')) .toBe("LogAttributes['1'] AS \"LogAttributes.1\""); // DBRowJsonViewer.test.tsx: the JSON-extract path through a Map column expect(buildJSONExtractQuery( ['LogAttributes', '1', 'foo'], ['LogAttributes', '1'], [], 'JSONExtractString', ['LogAttributes'], )).toBe("JSONExtractString(LogAttributes['1'], 'foo')"); ``` Each one is a regression test for a specific surface where a Map column meets a numeric-looking sub-key. If the threading drops at any call site, the corresponding test trips. ## Test plan - [x] New tests in `packages/app/src/__tests__/utils.test.ts` covering: - The failing repro from the issue body: `mergePath(['LogAttributes', '1'], [], ['LogAttributes'])` returns `LogAttributes['1']`, not `LogAttributes[2]` - Non-numeric Map sub-key (unchanged behavior) - Multi-segment Map path - Array column with numeric key (inverse: unchanged behavior) - JSON-vs-Map precedence - Default (Array / unknown column) cases unchanged - SQL escaping for single quote, backslash, and both combined (Map and default branches) - [x] New tests in `packages/app/src/components/DBSearchPageFilters/utils.test.ts` covering: - Numeric-looking Map sub-key on the "Load more" path: `LogAttributes.1` -> `LogAttributes['1']` - Multi-segment property path starting with a numeric segment - [x] New tests in `packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx` covering `deriveMapColumnsFromFields`: - Top-level Map columns - Wrapped Map types via `jsType` (`LowCardinality(Map(...))`, `Nullable(Map(...))`) - Nested fields excluded - Non-Map columns excluded - Undefined and empty inputs - [x] New tests in `packages/app/src/components/DBRowJsonViewer.test.tsx` covering `buildJSONExtractQuery` with `mapColumns`: - Numeric Map sub-key holding parsed JSON: emits `JSONExtractString(LogAttributes['1'], 'foo')` - Deeply nested numeric Map sub-key - Non-numeric Map sub-key (regression) - Empty `mapColumns` falls back to array-index (regression, covers Array(JSON)) - [x] `yarn jest` for affected files (208 passed) - [x] `make ci-lint` clean - [x] `yarn tsc --noEmit` clean - [x] Reproduction confirmed against the production session data referenced in HDX-4369 ## Deferred follow-up `packages/api/src/controllers/ai.ts` has three `mergePath` calls with the same bug class on the AI Summary / AI Assistant backend path. Fixing it crosses into a second package and would push this PR to Tier 3, so I split it out: #2367 (tracked as HDX-4402). Fixes HDX-4369. --- .../hdx-4369-mergepath-map-subscript.md | 23 ++++ packages/app/src/__tests__/utils.test.ts | 111 ++++++++++++++++++ .../app/src/components/DBRowDataPanel.tsx | 22 +++- .../src/components/DBRowJsonViewer.test.tsx | 52 ++++++++ .../app/src/components/DBRowJsonViewer.tsx | 24 +++- .../app/src/components/DBRowOverviewPanel.tsx | 9 +- .../src/components/DBSearchPageFilters.tsx | 5 +- .../DBSearchPageFilters/utils.test.ts | 17 +++ .../components/DBSearchPageFilters/utils.ts | 12 +- .../__tests__/DBRowDataPanel.test.ts | 43 ++++++- .../__tests__/useAutoCompleteOptions.test.tsx | 81 ++++++++++++- .../app/src/hooks/useAutoCompleteOptions.tsx | 31 ++++- packages/app/src/hooks/useMetadata.tsx | 29 +++++ packages/app/src/utils.ts | 71 ++++++++--- 14 files changed, 501 insertions(+), 29 deletions(-) create mode 100644 .changeset/hdx-4369-mergepath-map-subscript.md diff --git a/.changeset/hdx-4369-mergepath-map-subscript.md b/.changeset/hdx-4369-mergepath-map-subscript.md new file mode 100644 index 0000000000..00c07cd9ba --- /dev/null +++ b/.changeset/hdx-4369-mergepath-map-subscript.md @@ -0,0 +1,23 @@ +--- +"@hyperdx/app": patch +--- + +fix(row-panel): mergePath now emits string-key subscripts for Map columns, +preventing a crash when expanding rows with numeric-looking attribute keys + +`mergePath` converted numeric path segments to 1-based array subscripts +(`[N+1]`) regardless of whether the parent column was a Map or an Array. +On a `Map(String, String)` column this produced SQL like `LogAttributes[2]`, +which ClickHouse rejects with `Illegal types of arguments: +Map(String, String), UInt8 for function arrayElement`. The grid row +"expand" view failed for any row whose attribute path included a +numeric-looking key under a Map column. + +`mergePath` now accepts a `mapColumns` argument alongside `jsonColumns`. +For Map-typed parents, sub-keys always render as string subscripts +(`Map['1']`) regardless of whether the key looks numeric. The three +callers (`useAutoCompleteOptions`, `DBRowJsonViewer` via the row panels, +`DBSearchPageFilters`) now thread Map-column names from the source +schema. A new `useMapColumns` hook mirrors `useJsonColumns`. + +Fixes HDX-4369. diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index 63b77d78ea..15b46d4632 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -20,6 +20,7 @@ import { getColorFromCSSToken, getMetricTableName, mapKeyBy, + mergePath, orderByStringToSortingState, parseTimestampToMs, resolveConditionalColor, @@ -1212,6 +1213,116 @@ describe('formatDurationMsCompact', () => { }); }); +describe('mergePath', () => { + describe('default (Array / unknown column)', () => { + it('returns the bare key for a single-segment path', () => { + expect(mergePath(['Body'])).toBe('Body'); + }); + + it('numeric sub-segment becomes 1-based array index', () => { + // ClickHouse arrays are 1-based but flattened data uses 0-based indices. + expect(mergePath(['SomeArray', '0'])).toBe('SomeArray[1]'); + expect(mergePath(['SomeArray', '4'])).toBe('SomeArray[5]'); + }); + + it('non-numeric sub-segment becomes string-key subscript', () => { + expect(mergePath(['SomeColumn', 'service.name'])).toBe( + "SomeColumn['service.name']", + ); + }); + + it('mixed numeric and string segments chain', () => { + expect(mergePath(['Outer', '1', 'inner'])).toBe("Outer[2]['inner']"); + }); + }); + + describe('JSON column', () => { + it('emits dotted backtick-quoted accessor', () => { + expect(mergePath(['BodyJson', 'service', 'name'], ['BodyJson'])).toBe( + 'BodyJson.`service`.`name`', + ); + }); + }); + + describe('Map column (HDX-4369)', () => { + // Failing reproducer from the issue body: on a Map(String, String), a + // numeric-looking sub-key must NOT collapse into array-index syntax. + // ClickHouse rejects `LogAttributes[2]` against a Map column with + // "Illegal types of arguments: Map(String, String), UInt8 for function + // arrayElement". The fix adds a `mapColumns` parameter that forces the + // bracketed string-key form regardless of whether the key parses as a + // non-negative integer. + it('numeric sub-key on a Map renders as string subscript, not array index', () => { + const result = mergePath(['LogAttributes', '1'], [], ['LogAttributes']); + expect(result).not.toBe('LogAttributes[2]'); + expect(result).not.toMatch(/\[\d+\]$/); + expect(result).toBe("LogAttributes['1']"); + }); + + it('non-numeric Map sub-key keeps string subscript (unchanged)', () => { + expect( + mergePath(['LogAttributes', 'service.name'], [], ['LogAttributes']), + ).toBe("LogAttributes['service.name']"); + }); + + it('multi-segment Map path chains string subscripts', () => { + expect( + mergePath(['LogAttributes', '1', 'foo'], [], ['LogAttributes']), + ).toBe("LogAttributes['1']['foo']"); + }); + + it('Array column with numeric key still uses array-index syntax', () => { + // Inverse case: keep existing behavior for non-Map parents. + expect(mergePath(['SomeArray', '1'], [], ['LogAttributes'])).toBe( + 'SomeArray[2]', + ); + }); + + it('JSON column wins over Map column when both lists contain the key', () => { + // Caller can't currently configure the same column as both; the order + // is deterministic if they did. + expect(mergePath(['Body', '1'], ['Body'], ['Body'])).toBe('Body.`1`'); + }); + }); + + describe('SQL escaping of single quotes and backslashes', () => { + // Keys can contain user-controlled characters (Map sub-keys carry + // arbitrary text). An unescaped single quote produces malformed SQL like + // `Map['it's']`, which ClickHouse parses as the broken token sequence + // `Map['it']s']`. Backslash must escape first so the quote-escape + // backslash is not itself doubled. + it('escapes single quotes in Map sub-keys', () => { + expect(mergePath(['LogAttributes', "it's"], [], ['LogAttributes'])).toBe( + "LogAttributes['it\\'s']", + ); + }); + + it('escapes backslashes in Map sub-keys', () => { + expect( + mergePath(['LogAttributes', 'back\\slash'], [], ['LogAttributes']), + ).toBe("LogAttributes['back\\\\slash']"); + }); + + it('escapes a key containing both a backslash and a quote', () => { + expect( + mergePath(['LogAttributes', "a\\b'c"], [], ['LogAttributes']), + ).toBe("LogAttributes['a\\\\b\\'c']"); + }); + + it('escapes single quotes in default-branch string subscripts', () => { + // The default Array / unknown column branch also takes string-key + // subscripts when the segment is non-numeric. Same escape applies. + expect(mergePath(['SomeColumn', "it's"])).toBe("SomeColumn['it\\'s']"); + }); + + it('leaves numeric segments untouched in the default branch', () => { + // Numeric path collapses to bracketed integer index; escape is a + // no-op because Number.isInteger(asNumber) succeeds. Sanity check. + expect(mergePath(['SomeArray', '0'])).toBe('SomeArray[1]'); + }); + }); +}); + describe('getColorFromCSSToken', () => { afterEach(() => { jest.restoreAllMocks(); diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index ef2bd8ed1c..5ca3a6194e 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -228,6 +228,21 @@ export function getJSONColumnNames(meta: ResponseJSON['meta'] | undefined) { ); } +// Returns the names of Map-typed columns in the result metadata. Used by +// `mergePath` to keep numeric-looking sub-keys on a Map(String, ...) from +// collapsing into ClickHouse array-index syntax (`Map[2]`), which the +// server rejects with +// `Illegal types of arguments: Map(String, ...), UInt8 for function +// arrayElement`. HDX-4369. +export function getMapColumnNames(meta: ResponseJSON['meta'] | undefined) { + return ( + meta + // Match both `Map(K, V)` and the bare `Map` (rare; defensive). + ?.filter(m => m.type === 'Map' || m.type.startsWith('Map(')) + .map(m => m.name) ?? [] + ); +} + export function RowDataPanel({ source, rowId, @@ -250,11 +265,16 @@ export function RowDataPanel({ }, [data]); const jsonColumns = getJSONColumnNames(data?.meta); + const mapColumns = getMapColumnNames(data?.meta); return (
- +
); diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index f0a1165d6b..64a9c3ae13 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -294,5 +294,57 @@ describe('DBRowJsonViewer', () => { ), ).toBe("JSONExtractString(LogAttributes.`config`, 'host')"); }); + + // HDX-4369. HyperJson promotes a Map sub-value that is itself a + // JSON-parseable string to `isInParsedJson=true` with + // parsedJsonRootPath=[MapCol, key] (see HyperJson.tsx:227-234). When that + // key is numeric, the inner `mergePath` used to emit `MapCol[N+1]` array + // syntax, which ClickHouse rejects with "Illegal types of arguments: + // Map(String, String), UInt8 for function arrayElement". Threading + // `mapColumns` keeps the Map[\'1\'] subscript. + it("emits Map['1'] for Map column with numeric sub-key holding parsed JSON", () => { + expect( + buildJSONExtractQuery( + ['LogAttributes', '1', 'foo'], + ['LogAttributes', '1'], + [], // jsonColumns + 'JSONExtractString', + ['LogAttributes'], // mapColumns + ), + ).toBe("JSONExtractString(LogAttributes['1'], 'foo')"); + }); + + it("emits Map['42'] for deeply nested Map column with numeric sub-key holding parsed JSON", () => { + expect( + buildJSONExtractQuery( + ['LogAttributes', '42', 'bar', 'baz'], + ['LogAttributes', '42'], + [], + 'JSONExtractString', + ['LogAttributes'], + ), + ).toBe("JSONExtractString(LogAttributes['42'], 'bar', 'baz')"); + }); + + it('keeps non-numeric Map sub-key unchanged when mapColumns is threaded', () => { + expect( + buildJSONExtractQuery( + ['LogAttributes', 'config', 'host'], + ['LogAttributes', 'config'], + [], + 'JSONExtractString', + ['LogAttributes'], + ), + ).toBe("JSONExtractString(LogAttributes['config'], 'host')"); + }); + + it('falls back to array index when mapColumns is empty (unchanged behavior)', () => { + // Without mapColumns, a numeric segment still gets the array-index + // treatment. This pins the pre-HDX-4369 default for the non-Map case + // (e.g. an Array(JSON) column whose element holds a parsed JSON value). + expect( + buildJSONExtractQuery(['SomeArray', '0', 'id'], ['SomeArray', '0']), + ).toBe("JSONExtractString(SomeArray[1], 'id')"); + }); }); }); diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index fd9b4c61b0..82bbb11e10 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -44,13 +44,20 @@ export function buildJSONExtractQuery( parsedJsonRootPath: string[], jsonColumns: string[] = [], jsonExtractFn: JSONExtractFn = 'JSONExtractString', + mapColumns: string[] = [], ): string | null { const nestedPath = keyPath.slice(parsedJsonRootPath.length); if (nestedPath.length === 0) { return null; // No nested path to extract } - const baseColumn = mergePath(parsedJsonRootPath, jsonColumns); + // `parsedJsonRootPath[0]` is the column the parsed-JSON view is anchored on. + // It can be a JSON column (auto-detected by ClickHouse JSON type) OR a Map + // column whose sub-value is a JSON-parseable string (HyperJson promotes those + // to `isInParsedJson=true`, see HyperJson.tsx:227). Thread `mapColumns` so a + // numeric-looking Map sub-key renders as `Map['1']` instead of the array + // `Map[2]`. See HDX-4369. + const baseColumn = mergePath(parsedJsonRootPath, jsonColumns, mapColumns); const jsonPathArgs = nestedPath.map(p => `'${p}'`).join(', '); return `${jsonExtractFn}(${baseColumn}, ${jsonPathArgs})`; } @@ -332,9 +339,14 @@ function HyperJsonMenu({ rowData }: { rowData: any }) { export function DBRowJsonViewer({ data, jsonColumns, + mapColumns, }: { data: any; jsonColumns?: string[]; + // Map column names from the result-set metadata. Threaded into + // `mergePath` so numeric-looking sub-keys on a Map render as + // `Map['key']` instead of the array `Map[N+1]`. HDX-4369. + mapColumns?: string[]; }) { const { onPropertyAddClick, @@ -369,7 +381,7 @@ export function DBRowJsonViewer({ const getLineActions = useCallback( ({ keyPath, value, isInParsedJson, parsedJsonRootPath }) => { const actions: LineAction[] = []; - const fieldPath = mergePath(keyPath, jsonColumns); + const fieldPath = mergePath(keyPath, jsonColumns, mapColumns); const isJsonColumn = keyPath.length > 0 && jsonColumns?.includes(keyPath[0]); @@ -395,6 +407,8 @@ export function DBRowJsonViewer({ keyPath, parsedJsonRootPath, jsonColumns, + 'JSONExtractString', + mapColumns, ); if (jsonQuery) { filterFieldPath = jsonQuery; @@ -443,6 +457,7 @@ export function DBRowJsonViewer({ parsedJsonRootPath, jsonColumns, jsonExtractFn, + mapColumns, ); if (jsonQuery) { @@ -486,6 +501,8 @@ export function DBRowJsonViewer({ keyPath, parsedJsonRootPath, jsonColumns, + 'JSONExtractString', + mapColumns, ); if (jsonQuery) { chartFieldPath = jsonQuery; @@ -513,6 +530,8 @@ export function DBRowJsonViewer({ keyPath, parsedJsonRootPath, jsonColumns, + 'JSONExtractString', + mapColumns, ); if (jsonQuery) { columnFieldPath = jsonQuery; @@ -609,6 +628,7 @@ export function DBRowJsonViewer({ rowData, toggleColumn, jsonColumns, + mapColumns, ], ); diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx index 1c5c3c5710..2d8560fff9 100644 --- a/packages/app/src/components/DBRowOverviewPanel.tsx +++ b/packages/app/src/components/DBRowOverviewPanel.tsx @@ -8,7 +8,11 @@ import { WithClause } from '@/hooks/useRowWhere'; import { getEventBody } from '@/source'; import { getHighlightedAttributesFromData } from '@/utils/highlightedAttributes'; -import { getJSONColumnNames, useRowData } from './DBRowDataPanel'; +import { + getJSONColumnNames, + getMapColumnNames, + useRowData, +} from './DBRowDataPanel'; import { DBRowJsonViewer } from './DBRowJsonViewer'; import { RowSidePanelContext } from './DBRowSidePanel'; import DBRowSidePanelHeader from './DBRowSidePanelHeader'; @@ -52,6 +56,7 @@ export function RowOverviewPanel({ }, [source, data]); const jsonColumns = getJSONColumnNames(data?.meta); + const mapColumns = getMapColumnNames(data?.meta); const eventAttributesExpr = source.kind === SourceKind.Log || source.kind === SourceKind.Trace @@ -282,6 +287,7 @@ export function RowOverviewPanel({ @@ -300,6 +306,7 @@ export function RowOverviewPanel({ diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index ba5de2b41c..dfbc7024e0 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -64,6 +64,7 @@ import { useGetKeyValues, useGetValuesDistribution, useJsonColumns, + useMapColumns, useTableMetadata, } from '@/hooks/useMetadata'; import { useMetadataWithSettings } from '@/hooks/useMetadata'; @@ -1141,6 +1142,7 @@ const DBSearchPageFiltersComponent = ({ const { data: source } = useSource({ id: sourceId }); const sourceTableConnection = tcFromSource(source); const { data: jsonColumns } = useJsonColumns(sourceTableConnection); + const { data: mapColumns } = useMapColumns(sourceTableConnection); const filterMode = showAllValues ? ('all' as const) : ('exact' as const); const hasMVs = !!sourceTableConnection.metadataMVs; @@ -1237,7 +1239,7 @@ const DBSearchPageFiltersComponent = ({ .map(({ path, type }) => { return { type, - path: mergePath(path, jsonColumns ?? []), + path: mergePath(path, jsonColumns ?? [], mapColumns ?? []), isMapSubField: path.length > 1, }; }) @@ -1259,6 +1261,7 @@ const DBSearchPageFiltersComponent = ({ }, [ allFields, jsonColumns, + mapColumns, filterState, showMoreFields, isFieldPinned, diff --git a/packages/app/src/components/DBSearchPageFilters/utils.test.ts b/packages/app/src/components/DBSearchPageFilters/utils.test.ts index c851f8d026..56f07094f7 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.test.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.test.ts @@ -141,4 +141,21 @@ describe('toClickHouseKeyExpression', () => { it('leaves plain column names unchanged', () => { expect(toClickHouseKeyExpression('Timestamp')).toBe('Timestamp'); }); + + // HDX-4369: parseMapFieldName proves the base is a Map, so a numeric- + // looking sub-key must NOT collapse into array-index syntax. Without + // mergePath's third argument the result was `LogAttributes[2]`, which + // ClickHouse rejects with "Illegal types of arguments: Map(String, + // String), UInt8 for function arrayElement" on the "Load more" path. + it('rewrites a numeric-looking map sub-key to bracket form', () => { + expect(toClickHouseKeyExpression('LogAttributes.1')).toBe( + "LogAttributes['1']", + ); + }); + + it('preserves a multi-segment property path that starts with a numeric segment', () => { + expect(toClickHouseKeyExpression('LogAttributes.42.foo')).toBe( + "LogAttributes['42.foo']", + ); + }); }); diff --git a/packages/app/src/components/DBSearchPageFilters/utils.ts b/packages/app/src/components/DBSearchPageFilters/utils.ts index 7ccdd44874..c8558c8166 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.ts @@ -150,6 +150,12 @@ export function getFilterStateEntry( // caller (e.g. "Load more" via metadata.getKeyValues), since `setFilterValue` // normalizes Map sub-keys to dot form which ClickHouse cannot resolve as map // access. +// +// `parseMapFieldName` already guarantees `parsed.baseName` is a Map (its only +// callers are the dot-form facet keys that originate from Map columns), so +// `mergePath` must treat it as one. Without the third argument, a numeric- +// looking sub-key like `LogAttributes.1` collapses into the Array branch and +// emits the illegal `LogAttributes[2]`. HDX-4369. export function toClickHouseKeyExpression(key: string): string { if ( key.includes("['") || @@ -161,5 +167,9 @@ export function toClickHouseKeyExpression(key: string): string { } const parsed = parseMapFieldName(key); if (!parsed) return key; - return mergePath([parsed.baseName, parsed.propertyPath]); + return mergePath( + [parsed.baseName, parsed.propertyPath], + [], + [parsed.baseName], + ); } diff --git a/packages/app/src/components/__tests__/DBRowDataPanel.test.ts b/packages/app/src/components/__tests__/DBRowDataPanel.test.ts index 348c84aa70..513653646f 100644 --- a/packages/app/src/components/__tests__/DBRowDataPanel.test.ts +++ b/packages/app/src/components/__tests__/DBRowDataPanel.test.ts @@ -1,4 +1,4 @@ -import { getJSONColumnNames } from '../DBRowDataPanel'; +import { getJSONColumnNames, getMapColumnNames } from '../DBRowDataPanel'; describe('DBRowDataPanel', () => { describe('getJSONColumnNames', () => { @@ -12,4 +12,45 @@ describe('DBRowDataPanel', () => { expect(result).toEqual(['col2', 'col3']); }); }); + + // Regression test for the OSS #2357 conflict-resolution merge. The + // composed result wraps `Event Attributes` in a length check from + // origin/main AND passes `mapColumns={mapColumns}` through to the + // DBRowJsonViewer from HEAD. Both branches are wired through + // `getMapColumnNames`, which is the symbol the resolution + // introduces from HEAD and that origin/main otherwise lacks. A + // regression in either compose direction would either drop the + // helper or change its semantics; this test pins both. + describe('getMapColumnNames', () => { + it('returns Map column names', () => { + const meta = [ + { name: 'col1', type: 'String' }, + { name: 'LogAttributes', type: 'Map(String, String)' }, + { name: 'ResourceAttributes', type: 'Map(String, String)' }, + { name: 'col4', type: 'JSON' }, + ]; + expect(getMapColumnNames(meta)).toEqual([ + 'LogAttributes', + 'ResourceAttributes', + ]); + }); + + it('matches the bare Map type as well as Map(K, V)', () => { + const meta = [ + { name: 'bareMap', type: 'Map' }, + { name: 'typedMap', type: 'Map(String, UInt8)' }, + { name: 'notMap', type: 'String' }, + ]; + expect(getMapColumnNames(meta)).toEqual(['bareMap', 'typedMap']); + }); + + it('returns an empty array when meta is undefined', () => { + expect(getMapColumnNames(undefined)).toEqual([]); + }); + + it('does not classify JSON columns as Map columns', () => { + const meta = [{ name: 'BodyJson', type: 'JSON' }]; + expect(getMapColumnNames(meta)).toEqual([]); + }); + }); }); diff --git a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx index 67b88ac86b..ce8cebe785 100644 --- a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx +++ b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx @@ -3,8 +3,11 @@ import { Field } from '@hyperdx/common-utils/dist/core/metadata'; import { renderHook } from '@testing-library/react'; import { LuceneLanguageFormatter } from '../../components/SearchInput/SearchInputV2'; -import { useAutoCompleteOptions } from '../useAutoCompleteOptions'; -import { tokenizeAtCursor } from '../useAutoCompleteOptions'; +import { + deriveMapColumnsFromFields, + tokenizeAtCursor, + useAutoCompleteOptions, +} from '../useAutoCompleteOptions'; import { useGetKeyValues, useMultipleAllFields } from '../useMetadata'; // Mock dependencies @@ -364,3 +367,77 @@ describe('tokenizeAtCursor', () => { }); }); }); + +// HDX-4369: pins the threading from "field list" -> "mapColumns" inside +// useAutoCompleteOptions. The hook uses the derived array as the third +// argument to mergePath when computing `searchKeys`, so a regression here +// silently re-introduces the illegal `Map[N+1]` SQL. +describe('deriveMapColumnsFromFields', () => { + it('returns top-level Map column names', () => { + const fields: Field[] = [ + { path: ['LogAttributes'], jsType: JSDataType.Map, type: 'map' }, + { path: ['ResourceAttributes'], jsType: JSDataType.Map, type: 'map' }, + { + path: ['ServiceName'], + jsType: JSDataType.String, + type: 'String', + }, + ]; + expect(deriveMapColumnsFromFields(fields)).toEqual([ + 'LogAttributes', + 'ResourceAttributes', + ]); + }); + + it('matches wrapped Map types via the canonical jsType', () => { + // convertCHDataTypeToJSType peels off LowCardinality(...) and + // Nullable(...) before classifying, so jsType is the canonical signal. + // A raw-string check on f.type would miss these wrappers and silently + // fall through to the array-index path in mergePath. + const fields: Field[] = [ + { + path: ['LowCardMap'], + jsType: JSDataType.Map, + type: 'LowCardinality(Map(String, String))', + }, + { + path: ['NullableMap'], + jsType: JSDataType.Map, + type: 'Nullable(Map(String, UInt8))', + }, + ]; + expect(deriveMapColumnsFromFields(fields)).toEqual([ + 'LowCardMap', + 'NullableMap', + ]); + }); + + it('excludes nested fields (path.length > 1)', () => { + // Sub-keys under a Map (e.g. ResourceAttributes.service.name) are not + // themselves Map-typed parents; including them would change mergePath's + // semantics for the outer column. + const fields: Field[] = [ + { path: ['ResourceAttributes'], jsType: JSDataType.Map, type: 'Map' }, + { + path: ['ResourceAttributes', 'service.name'], + jsType: JSDataType.String, + type: 'String', + }, + ]; + expect(deriveMapColumnsFromFields(fields)).toEqual(['ResourceAttributes']); + }); + + it('excludes non-Map columns even when path.length === 1', () => { + const fields: Field[] = [ + { path: ['BodyJson'], jsType: JSDataType.JSON, type: 'JSON' }, + { path: ['Timestamp'], jsType: JSDataType.Date, type: 'DateTime64(9)' }, + { path: ['Body'], jsType: JSDataType.String, type: 'String' }, + ]; + expect(deriveMapColumnsFromFields(fields)).toEqual([]); + }); + + it('handles undefined and empty inputs without throwing', () => { + expect(deriveMapColumnsFromFields(undefined)).toEqual([]); + expect(deriveMapColumnsFromFields([])).toEqual([]); + }); +}); diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx index 1e1b10ad17..0dbde75e0e 100644 --- a/packages/app/src/hooks/useAutoCompleteOptions.tsx +++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx @@ -1,4 +1,5 @@ import { useMemo } from 'react'; +import { JSDataType } from '@hyperdx/common-utils/dist/clickhouse'; import { Field, parseKeyPath, @@ -15,6 +16,24 @@ import { import { useSource } from '@/source'; import { mergePath, toArray, useDebounce } from '@/utils'; +// Derive top-level Map column names from a fields list. Matches on the +// canonical `JSDataType.Map` rather than the raw ClickHouse type string so +// wrapped Map types (e.g. `LowCardinality(Map(...))`, `Nullable(Map(...))`) +// are detected too: `convertCHDataTypeToJSType` unwraps those wrappers before +// classifying. Top-level only (path.length === 1) since nested Map sub-keys +// surface as deeper path segments and are not themselves Map-typed parents. +// +// Exported separately so a regression in `useAutoCompleteOptions`'s Map +// derivation trips a unit test: dropping or breaking this filter would +// silently re-introduce the array-index emission HDX-4369 fixed. +export function deriveMapColumnsFromFields( + fields: readonly Field[] | undefined, +): string[] { + return (fields ?? []) + .filter(f => f.path.length === 1 && f.jsType === JSDataType.Map) + .map(f => f.path[0]); +} + export type TokenInfo = { /** The full token at the cursor position */ token: string; @@ -234,9 +253,17 @@ export function useAutoCompleteOptions( [tcs, effectiveDateRange, source?.timestampValueExpression], ); + // Map columns from the field list, so a path like `['LogAttributes', '1']` + // on a Map(String, ...) renders as `LogAttributes['1']` instead of the + // illegal array `LogAttributes[2]`. HDX-4369. + const mapColumns = useMemo( + () => deriveMapColumnsFromFields(fields), + [fields], + ); + const searchKeys = useMemo( - () => (searchField ? [mergePath(searchField.path)] : []), - [searchField], + () => (searchField ? [mergePath(searchField.path, [], mapColumns)] : []), + [searchField, mapColumns], ); const metadataMVs = tcs[0]?.metadataMVs; diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 2f170d980f..b2dbdeb34c 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -126,6 +126,35 @@ export function useJsonColumns( }); } +// Mirrors `useJsonColumns` for Map-typed columns. Used by `mergePath` +// callers (notably `DBSearchPageFilters`) so numeric-looking sub-keys on a +// Map render as `Map['key']` instead of the illegal array `Map[N+1]`. +// HDX-4369. +export function useMapColumns( + tableConnection: TableConnection | undefined, + options?: Partial>, +) { + const metadata = useMetadataWithSettings(); + return useQuery({ + queryKey: ['useMetadata.useMapColumns', tableConnection], + queryFn: async () => { + if (!tableConnection) return []; + const columns = await metadata.getColumns(tableConnection); + return ( + filterColumnMetaByType(columns, [JSDataType.Map])?.map( + column => column.name, + ) ?? [] + ); + }, + enabled: + tableConnection && + !!tableConnection.databaseName && + !!tableConnection.tableName && + !!tableConnection.connectionId, + ...options, + }); +} + export function useMultipleAllFields( tableConnections: TableConnection[], options?: Partial> & { diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index ab027059eb..fadce48ef6 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -1144,28 +1144,63 @@ export const formatUptime = (seconds: number) => { // FIXME: eventually we want to separate metric name into two fields // Date formatting -export const mergePath = (path: string[], jsonColumns: string[] = []) => { +// +// `mergePath` rebuilds a column-access expression for a nested path that the +// row table flattened during display. It has three column-type modes: +// +// - JSON column: dotted backtick-quoted accessor, e.g. `Body.\`key\`` +// - Map column: bracketed string-key subscript, e.g. `LogAttributes['1']` +// - Array column (default): numeric segments get 1-based array subscripts +// (`Array[N+1]`); string segments get bracketed string-key subscripts. +// +// Without the Map-column branch, a Map(String, String) like `LogAttributes` +// with a numeric-looking key (`"1"`) collapses into the array branch and +// ClickHouse rejects the resulting `LogAttributes[2]` with +// `Illegal types of arguments: Map(String, String), UInt8 for function +// arrayElement`. HDX-4369. +// Escape backslash and single-quote inside a key before interpolating it +// into a single-quoted SQL string. Keys can contain user-controlled +// characters (Map sub-keys, JSON field names from the row data) and an +// unescaped quote produces malformed SQL. +const escapeSqlSingleQuoted = (v: string): string => + v.replace(/\\/g, '\\\\').replace(/'/g, "\\'"); + +export const mergePath = ( + path: string[], + jsonColumns: string[] = [], + mapColumns: string[] = [], +) => { const [key, ...rest] = path; if (rest.length === 0) { return key; } - return jsonColumns.includes(key) - ? `${key}.${rest - .map(v => - v - .split('.') - .map(v => (v.startsWith('`') && v.endsWith('`') ? v : `\`${v}\``)) - .join('.'), - ) - .join('.')}` - : `${key}${rest - .map(v => { - const asNumber = Number(v); - const isArrayIndex = Number.isInteger(asNumber) && asNumber >= 0; - // ClickHouse arrays are 1-based, but flattened data uses 0-based indices - return isArrayIndex ? `[${asNumber + 1}]` : `['${v}']`; - }) - .join('')}`; + if (jsonColumns.includes(key)) { + return `${key}.${rest + .map(v => + v + .split('.') + .map(v => (v.startsWith('`') && v.endsWith('`') ? v : `\`${v}\``)) + .join('.'), + ) + .join('.')}`; + } + if (mapColumns.includes(key)) { + // Map columns always take string-key subscripts; ClickHouse's Map index + // operator is keyed by the Map's key type, not by integer position. A + // numeric-looking sub-key like `"1"` on a Map(String, ...) must still + // render as `Map['1']`. + return `${key}${rest.map(v => `['${escapeSqlSingleQuoted(v)}']`).join('')}`; + } + return `${key}${rest + .map(v => { + const asNumber = Number(v); + const isArrayIndex = Number.isInteger(asNumber) && asNumber >= 0; + // ClickHouse arrays are 1-based, but flattened data uses 0-based indices + return isArrayIndex + ? `[${asNumber + 1}]` + : `['${escapeSqlSingleQuoted(v)}']`; + }) + .join('')}`; }; const _useTry = (fn: () => T): [null | Error | unknown, null | T] => { From a7586176017d1bc55b1fa9102133300f606359a0 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 10 Jun 2026 12:30:15 -0400 Subject: [PATCH 041/728] fix: use cloud versions for usable ALIAS columns (#2431) ## Summary The `direct_read` map column use in queries is a different version for the ClickHouse Cloud version instead of OSS. We need to support that properly ### References - Linear Issue: - Related PRs: --- .../src/__tests__/clickhouseVersion.test.ts | 81 +++++++++++++++++++ .../src/__tests__/metadata.test.ts | 77 ++++++++++++++++++ .../src/__tests__/renderChartConfig.test.ts | 1 + .../src/core/clickhouseVersion.ts | 29 ++++++- packages/common-utils/src/core/metadata.ts | 38 +++++++++ packages/common-utils/src/queryParser.ts | 5 +- 6 files changed, 228 insertions(+), 3 deletions(-) diff --git a/packages/common-utils/src/__tests__/clickhouseVersion.test.ts b/packages/common-utils/src/__tests__/clickhouseVersion.test.ts index 3232cfdec6..e364335de4 100644 --- a/packages/common-utils/src/__tests__/clickhouseVersion.test.ts +++ b/packages/common-utils/src/__tests__/clickhouseVersion.test.ts @@ -173,4 +173,85 @@ describe('supportsDirectReadMap', () => { it('returns false when the version is undefined', () => { expect(supportsDirectReadMap(undefined)).toBe(false); }); + + describe('ClickHouse Cloud (isCloud=true)', () => { + describe('26.2 cloud backport branch (min 26.2.1.307)', () => { + it.each([ + [[26, 2, 1, 307], true], + [[26, 2, 1, 308], true], + [[26, 2, 2, 0], true], + [[26, 2, 99, 99], true], + [[26, 2, 1, 306], false], + [[26, 2, 1, 0], false], + [[26, 2, 0, 99], false], + [[26, 2, 0, 0], false], + ])('%j → %s', (version, expected) => { + expect(supportsDirectReadMap(version, true)).toBe(expected); + }); + }); + + describe('26.3 cloud branch (always supported)', () => { + it.each([ + [[26, 3, 0, 0], true], + [[26, 3, 0, 1], true], + [[26, 3, 1, 0], true], + [[26, 3, 12, 2], true], + [[26, 3, 99, 99], true], + ])('%j → %s', (version, expected) => { + expect(supportsDirectReadMap(version, true)).toBe(expected); + }); + }); + + describe('26.4 cloud backport branch (min 26.4.1.1740)', () => { + it.each([ + [[26, 4, 1, 1740], true], + [[26, 4, 1, 1741], true], + [[26, 4, 2, 0], true], + [[26, 4, 99, 99], true], + [[26, 4, 1, 1739], false], + [[26, 4, 1, 0], false], + [[26, 4, 0, 99], false], + [[26, 4, 0, 0], false], + ])('%j → %s', (version, expected) => { + expect(supportsDirectReadMap(version, true)).toBe(expected); + }); + }); + + describe('26.5+ cloud baseline (always supported)', () => { + it.each([ + [[26, 5, 0, 0], true], + [[26, 6, 0, 0], true], + [[27, 0, 0, 0], true], + ])('%j → %s', (version, expected) => { + expect(supportsDirectReadMap(version, true)).toBe(expected); + }); + }); + + describe('unsupported cloud branches', () => { + it.each([ + [[26, 1, 99, 99], false], + [[26, 0, 0, 0], false], + [[25, 12, 0, 0], false], + [[24, 0, 0, 0], false], + ])('%j → %s', (version, expected) => { + expect(supportsDirectReadMap(version, true)).toBe(expected); + }); + }); + + it('returns false when the version is undefined', () => { + expect(supportsDirectReadMap(undefined, true)).toBe(false); + }); + + it.each([ + [[26, 3, 0, 0]], + [[26, 2, 1, 307]], + [[26, 4, 1, 1740]], + ])( + 'cloud cutoffs differ from self-hosted on %j (supported on cloud, not self-hosted)', + version => { + expect(supportsDirectReadMap(version, false)).toBe(false); + expect(supportsDirectReadMap(version, true)).toBe(true); + }, + ); + }); }); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index bbd40781ef..2006bf3131 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -309,6 +309,83 @@ describe('Metadata', () => { }); }); + describe('isClickHouseCloud', () => { + beforeEach(() => { + mockCache.getOrFetch.mockImplementation((key, queryFn) => queryFn()); + }); + + it('returns true when SharedMergeTree is registered in system.table_engines', async () => { + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + data: [{ is_cloud: true }], + }), + }); + + const result = await metadata.isClickHouseCloud({ + connectionId: 'test_connection', + }); + + expect(result).toBe(true); + }); + + it('returns false when SharedMergeTree is absent from system.table_engines', async () => { + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + data: [], + }), + }); + + const result = await metadata.isClickHouseCloud({ + connectionId: 'test_connection', + }); + + expect(result).toBe(false); + }); + + it('re-probes after a transient failure instead of caching false', async () => { + const realCache = new MetadataCache(); + const realMetadata = new Metadata(mockClickhouseClient, realCache); + + (mockClickhouseClient.query as jest.Mock) + .mockRejectedValueOnce(new Error('connection refused')) + .mockResolvedValueOnce({ + json: jest.fn().mockResolvedValue({ data: [{ is_cloud: true }] }), + }); + + const first = await realMetadata.isClickHouseCloud({ + connectionId: 'test_connection', + }); + expect(first).toBe(false); + + const second = await realMetadata.isClickHouseCloud({ + connectionId: 'test_connection', + }); + expect(second).toBe(true); + + expect(mockClickhouseClient.query).toHaveBeenCalledTimes(2); + }); + + it('caches a successful negative result and does not re-query', async () => { + const realCache = new MetadataCache(); + const realMetadata = new Metadata(mockClickhouseClient, realCache); + + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: jest.fn().mockResolvedValue({ data: [] }), + }); + + const first = await realMetadata.isClickHouseCloud({ + connectionId: 'test_connection', + }); + const second = await realMetadata.isClickHouseCloud({ + connectionId: 'test_connection', + }); + + expect(first).toBe(false); + expect(second).toBe(false); + expect(mockClickhouseClient.query).toHaveBeenCalledTimes(1); + }); + }); + describe('getSkipIndices', () => { beforeEach(() => { mockCache.getOrFetch.mockImplementation((key, queryFn) => queryFn()); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 13a588900c..96e26dc805 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -48,6 +48,7 @@ describe('renderChartConfig', () => { .mockResolvedValue({ primary_key: 'timestamp' }), getSkipIndices: jest.fn().mockResolvedValue([]), getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), } as unknown as jest.Mocked; }); diff --git a/packages/common-utils/src/core/clickhouseVersion.ts b/packages/common-utils/src/core/clickhouseVersion.ts index 0f781a2b10..7fe2fbb428 100644 --- a/packages/common-utils/src/core/clickhouseVersion.ts +++ b/packages/common-utils/src/core/clickhouseVersion.ts @@ -84,6 +84,25 @@ const DIRECT_READ_MAP_BACKPORT_MINS: ReadonlyArray = [ [26, 4, 3, 37], ]; +/** + * Per-branch minimum versions required for the direct_read map column + * optimization on ClickHouse Cloud. Cloud has its own release cadence and + * received the backport at different cutoffs than self-hosted: + * + * - 26.2 branch → first available at 26.2.1.307 + * - 26.3 branch → automatically supported (any 26.3.x on Cloud) + * - 26.4 branch → first available at 26.4.1.1740 + * - 26.5+ → always supported (inherits `DIRECT_READ_MAP_BASELINE`) + * + * The 26.3 entry sits at the floor of the branch so every 26.3.x.y version + * passes the per-branch comparison. + */ +const DIRECT_READ_MAP_CLOUD_BACKPORT_MINS: ReadonlyArray = [ + [26, 2, 1, 307], + [26, 3, 0, 0], + [26, 4, 1, 1740], +]; + /** * First release where direct_read map support shipped unconditionally. Any * server with major.minor at or above this is considered supported, even if @@ -95,9 +114,13 @@ const DIRECT_READ_MAP_BASELINE: ClickHouseVersion = [26, 5, 0, 0]; * Returns true when the connected ClickHouse server supports the direct_read * map column optimization. Returns false when the version is undefined or * predates every known backport. + * + * When `isCloud` is true, uses the ClickHouse Cloud-specific backport cutoffs + * (which differ from self-hosted because Cloud ships its own release stream). */ export function supportsDirectReadMap( version: ClickHouseVersion | undefined, + isCloud = false, ): boolean { if (!version) return false; @@ -105,8 +128,12 @@ export function supportsDirectReadMap( return true; } + const branchMins = isCloud + ? DIRECT_READ_MAP_CLOUD_BACKPORT_MINS + : DIRECT_READ_MAP_BACKPORT_MINS; + const [vMajor, vMinor] = version; - const branchMin = DIRECT_READ_MAP_BACKPORT_MINS.find( + const branchMin = branchMins.find( ([major, minor]) => major === vMajor && minor === vMinor, ); if (!branchMin) return false; diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index d6e2b10e34..5e80ad7f4a 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -934,6 +934,44 @@ export class Metadata { }); } + /** + * Returns true when the connected server is ClickHouse Cloud, detected by + * checking whether `SharedMergeTree` is registered in `system.table_engines`. + * The SharedMergeTree engine is compiled into Cloud builds only, so its + * presence in the engine registry is a reliable Cloud signal that does not + * depend on any user table existing. + * + * Result is cached per connection — Cloud-ness is a server property. + */ + async isClickHouseCloud({ + connectionId, + }: { + connectionId: string; + }): Promise { + const result = await this.cache.getOrFetch( + `${connectionId}.isClickHouseCloud`, + async () => { + try { + const query = + "SELECT count() > 0 AS is_cloud FROM system.table_engines WHERE name = 'SharedMergeTree'"; + const json = await this.clickhouseClient + .query<'JSON'>({ + connectionId, + query, + clickhouse_settings: this.getClickHouseSettings(), + shouldSkipApplySettings: true, + }) + .then(res => res.json<{ is_cloud: boolean }>()); + return json.data.length > 0 && json.data[0].is_cloud; + } catch (e) { + console.warn('Error detecting ClickHouse Cloud:', e); + return undefined; + } + }, + ); + return result ?? false; + } + /** * Returns the parsed ClickHouse server version (from `SELECT version()`). * Returns undefined when the query fails or the value cannot be parsed; the diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 595a334dda..2a500c6b1c 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -1103,15 +1103,16 @@ export async function buildKvItemsLookup({ }): Promise { const lookup: KvItemsLookup = new Map(); try { - const [serverVersion, columns, skipIndices] = await Promise.all([ + const [serverVersion, columns, skipIndices, isCloud] = await Promise.all([ metadata.getServerVersion({ connectionId }), metadata.getColumns({ databaseName, tableName, connectionId }), metadata .getSkipIndices({ databaseName, tableName, connectionId }) .catch(() => [] as SkipIndexMetadata[]), + metadata.isClickHouseCloud({ connectionId }).catch(() => false), ]); - const directReadSupported = supportsDirectReadMap(serverVersion); + const directReadSupported = supportsDirectReadMap(serverVersion, isCloud); const kvItemsCandidates = columns.filter( c => From f126d5b1d338d428cc4515cf187b0a857249e390 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Wed, 10 Jun 2026 10:08:34 -0700 Subject: [PATCH 042/728] feat(api): support number-tile color in the external dashboards API (#2428) Number tiles let you set a static color and ordered conditional color rules in the editor, but the v2 external dashboards API carried neither field, so dashboards authored through `POST /api/v2/dashboards` could not set them. A raw SQL number tile that set a color in the UI also lost it on a GET then PUT round-trip, because the external conversion never emitted `color`. This adds number-tile color to the external API so the authoring surfaces match. Closes #2359 (that issue predates the conditional rules and only mentioned the static color; this covers both). Builds on #2265 (static color) and #2386 (conditional rules). ## What - Builder number chart config gains `color` (a palette token) and `colorRules` (ordered conditional rules, last match wins, max 10). - Raw SQL number chart config gains `color`. `colorRules` is intentionally left off the raw SQL variant: the editor shows the rules section for raw SQL number tiles, but its save path drops them, so persisted raw SQL configs never carry rules and the API should not invent capability the UI lacks. - Both directions of the internal/external conversion round-trip the new fields. - The OpenAPI spec adds a `ChartPaletteToken` enum and a `ColorCondition` schema, referenced from the number config. `openapi.json` is regenerated. ## Why the palette-token handling looks the way it does Colors persist as palette tokens, not hex, so they reflow across light and dark themes. The canonical tokens were renamed to hue names (`chart-blue`, `chart-red`, and so on) recently, and the older numeric tokens (`chart-1` through `chart-10`) can still sit in stored dashboards. So: - On input, the API accepts hue-named tokens only (the same enum the schema validates). A legacy numeric token in a hand-written payload is rejected with a clear enum error. - On output, a stored legacy token is normalized to its hue name via the shared `resolveChartPaletteToken` helper, matching the app's fetch-time normalizer. Older dashboards keep returning a valid token and keep rendering. I reused the existing `ChartPaletteTokenSchema` and `ColorConditionSchema` from `common-utils` rather than re-declaring them in the external schema (the file already imports leaf enums and the OnClick schemas the same way). That keeps the external surface from drifting from what the editor persists. ## Test plan - [x] `yarn workspace @hyperdx/api tsc --noEmit` - [x] `yarn workspace @hyperdx/api lint` - [x] `yarn workspace @hyperdx/api lint:openapi` - [x] `make dev-int FILE=external-api/__tests__/dashboards.test.ts` (111 passed) - Round-trip: builder `color` plus `colorRules` covering every operator family, and raw SQL `color`. - Rejections: non-palette token, legacy numeric token on input, more than 10 rules, `between` without a two-number tuple, a numeric operator with a string value, an invalid regex, and an over-length label. - Backward compat: a tile with neither field; a stored legacy token normalized to its hue name on read, for both builder and raw SQL number tiles. - The two "round-trip all supported fields" tests (POST and PUT) now include `color` and `colorRules` on the number tile. ## What is not in this PR - MCP `hyperdx_save_dashboard` parity for the same fields. That is the next PR in this series and reuses this conversion. - Customer docs for the number-tile color options (tracked in `ClickHouse/clickhouse-docs#6298`). --- .changeset/number-tile-color-external-api.md | 5 + packages/api/openapi.json | 184 +++++++++ .../external-api/__tests__/dashboards.test.ts | 376 ++++++++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 133 +++++++ .../external-api/v2/utils/dashboards.ts | 60 +++ packages/api/src/utils/zod.ts | 23 ++ packages/common-utils/src/types.ts | 130 +++--- 7 files changed, 863 insertions(+), 48 deletions(-) create mode 100644 .changeset/number-tile-color-external-api.md diff --git a/.changeset/number-tile-color-external-api.md b/.changeset/number-tile-color-external-api.md new file mode 100644 index 0000000000..3739b66c4f --- /dev/null +++ b/.changeset/number-tile-color-external-api.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +Support number-tile color authoring through the external dashboards API. The v2 REST API and OpenAPI spec now accept `color` (a palette token) and `colorRules` (ordered conditional color rules, last match wins) on builder number tiles, and `color` on raw SQL number tiles, matching what the in-product number-tile editor persists. Color rules accept the numeric and equality operators the editor offers (`gt`, `gte`, `lt`, `lte`, `between`, `eq`, `neq`). Existing dashboards keep working: tiles saved before the palette was renamed to hue names are normalized to the current token names on read. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 9bcad76117..d043230d7e 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -779,6 +779,174 @@ } } }, + "ChartPaletteToken": { + "type": "string", + "enum": [ + "chart-blue", + "chart-orange", + "chart-red", + "chart-cyan", + "chart-green", + "chart-pink", + "chart-purple", + "chart-light-blue", + "chart-brown", + "chart-gray", + "chart-success", + "chart-warning", + "chart-error" + ], + "description": "Palette token used to color a number tile. Tokens reflow across light and dark themes, so raw hex values are not accepted.\n", + "example": "chart-red" + }, + "NumericColorCondition": { + "type": "object", + "required": [ + "operator", + "value", + "color" + ], + "description": "Color rule comparing the displayed value against a single numeric bound.", + "properties": { + "operator": { + "type": "string", + "enum": [ + "gt", + "gte", + "lt", + "lte" + ], + "description": "Numeric comparison operator.", + "example": "gt" + }, + "value": { + "type": "number", + "description": "Numeric bound the displayed value is compared against. Only finite numbers are accepted (Infinity and NaN are rejected).\n", + "example": 100 + }, + "color": { + "$ref": "#/components/schemas/ChartPaletteToken", + "description": "Color applied when the rule matches." + }, + "label": { + "type": "string", + "maxLength": 40, + "description": "Optional label describing the rule.", + "example": "High" + } + } + }, + "BetweenColorCondition": { + "type": "object", + "required": [ + "operator", + "value", + "color" + ], + "description": "Color rule matching when the displayed value falls within an inclusive range.", + "properties": { + "operator": { + "type": "string", + "enum": [ + "between" + ], + "description": "Range comparison operator.", + "example": "between" + }, + "value": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "number" + }, + "description": "Inclusive [min, max] range. Both bounds must be finite numbers.\n", + "example": [ + 100, + 500 + ] + }, + "color": { + "$ref": "#/components/schemas/ChartPaletteToken", + "description": "Color applied when the rule matches." + }, + "label": { + "type": "string", + "maxLength": 40, + "description": "Optional label describing the rule.", + "example": "Warning" + } + } + }, + "EqualityColorCondition": { + "type": "object", + "required": [ + "operator", + "value", + "color" + ], + "description": "Color rule matching when the displayed value equals (eq) or does not equal (neq) a number or string.", + "properties": { + "operator": { + "type": "string", + "enum": [ + "eq", + "neq" + ], + "description": "Equality comparison operator.", + "example": "eq" + }, + "value": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "string", + "maxLength": 200 + } + ], + "description": "A finite number, or a string up to 200 characters, to compare for equality.\n", + "example": "OK" + }, + "color": { + "$ref": "#/components/schemas/ChartPaletteToken", + "description": "Color applied when the rule matches." + }, + "label": { + "type": "string", + "maxLength": 40, + "description": "Optional label describing the rule.", + "example": "Healthy" + } + } + }, + "NumberTileColorCondition": { + "description": "A single conditional color rule for a number tile. Rules are evaluated in order and the last matching rule wins. When no rule matches, the static color applies, then the default text color. The number-tile editor surfaces numeric and equality operators only.\n", + "oneOf": [ + { + "$ref": "#/components/schemas/NumericColorCondition" + }, + { + "$ref": "#/components/schemas/BetweenColorCondition" + }, + { + "$ref": "#/components/schemas/EqualityColorCondition" + } + ], + "discriminator": { + "propertyName": "operator", + "mapping": { + "gt": "#/components/schemas/NumericColorCondition", + "gte": "#/components/schemas/NumericColorCondition", + "lt": "#/components/schemas/NumericColorCondition", + "lte": "#/components/schemas/NumericColorCondition", + "between": "#/components/schemas/BetweenColorCondition", + "eq": "#/components/schemas/EqualityColorCondition", + "neq": "#/components/schemas/EqualityColorCondition" + } + } + }, "TimeChartSeries": { "type": "object", "required": [ @@ -1417,6 +1585,18 @@ "numberFormat": { "$ref": "#/components/schemas/NumberFormat", "description": "Number formatting options for displayed values." + }, + "color": { + "$ref": "#/components/schemas/ChartPaletteToken", + "description": "Optional static color applied to the displayed number." + }, + "colorRules": { + "type": "array", + "maxItems": 10, + "description": "Ordered conditional color rules evaluated against the displayed value (last match wins). Falls back to color, then the default text color when no rule matches.\n", + "items": { + "$ref": "#/components/schemas/NumberTileColorCondition" + } } } }, @@ -1772,6 +1952,10 @@ ], "description": "Display as a single big-number chart.", "example": "number" + }, + "color": { + "$ref": "#/components/schemas/ChartPaletteToken", + "description": "Optional static color applied to the displayed number. Raw SQL number tiles do not support conditional colorRules.\n" } } } diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts index 7aa2799d6a..cdf07957bc 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts @@ -2352,6 +2352,26 @@ describe('External API v2 Dashboards - new format', () => { thousandSeparated: true, average: true, }, + color: 'chart-green', + colorRules: [ + { + operator: 'gt', + value: 1000, + color: 'chart-warning', + label: 'Slow', + }, + { + operator: 'between', + value: [200, 1000], + color: 'chart-blue', + }, + { + operator: 'gte', + value: 5000, + color: 'chart-error', + label: 'Critical', + }, + ], }, }; @@ -2765,6 +2785,8 @@ describe('External API v2 Dashboards - new format', () => { sqlTemplate, sourceId, numberFormat: { output: 'currency', currencySymbol: '$' }, + // Raw SQL number tiles carry the static tile color (no colorRules). + color: 'chart-purple', }, }; @@ -3857,6 +3879,26 @@ describe('External API v2 Dashboards - new format', () => { thousandSeparated: true, average: true, }, + color: 'chart-green', + colorRules: [ + { + operator: 'gt', + value: 1000, + color: 'chart-warning', + label: 'Slow', + }, + { + operator: 'between', + value: [200, 1000], + color: 'chart-blue', + }, + { + operator: 'gte', + value: 5000, + color: 'chart-error', + label: 'Critical', + }, + ], }, }; @@ -4027,6 +4069,8 @@ describe('External API v2 Dashboards - new format', () => { sqlTemplate, sourceId, numberFormat: { output: 'currency', currencySymbol: '$' }, + // Raw SQL number tiles carry the static tile color (no colorRules). + color: 'chart-purple', }, }; @@ -4535,6 +4579,338 @@ describe('External API v2 Dashboards - new format', () => { }); }); + describe('Number tile color (HDX-1360)', () => { + // Minimal builder number tile; callers supply color / colorRules. The + // payload is sent through `.send()` (untyped) so negative tests can post + // intentionally invalid values without tripping the compile-time schema. + const numberTile = (config: Record) => ({ + name: 'Number', + x: 0, + y: 0, + w: 3, + h: 3, + config: { + displayType: 'number', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count', where: '' }], + ...config, + }, + }); + + const postTile = (config: Record) => + authRequest('post', BASE_URL).send({ + name: 'Number color dashboard', + tiles: [numberTile(config)], + tags: [], + }); + + const rawSqlNumberTile = (config: Record) => ({ + name: 'Number Raw SQL', + x: 0, + y: 0, + w: 3, + h: 3, + config: { + configType: 'sql', + displayType: 'number', + connectionId: connection._id.toString(), + sqlTemplate: 'SELECT count() FROM otel_logs WHERE {timeFilter}', + sourceId: traceSource._id.toString(), + ...config, + }, + }); + + // ── Positive: one per UI input ────────────────────────────────────── + + it('round-trips a builder number tile with a static color', async () => { + const create = await postTile({ color: 'chart-red' }).expect(200); + expect(create.body.data.tiles[0].config.color).toBe('chart-red'); + + const get = await authRequest( + 'get', + `${BASE_URL}/${create.body.data.id}`, + ).expect(200); + expect(get.body.data.tiles[0].config.color).toBe('chart-red'); + }); + + it('round-trips colorRules covering each operator family', async () => { + const colorRules = [ + { operator: 'gt', value: 1000, color: 'chart-warning', label: 'Slow' }, + { + operator: 'gte', + value: 5000, + color: 'chart-error', + label: 'Critical', + }, + { operator: 'lt', value: 0, color: 'chart-gray' }, + { operator: 'lte', value: 10, color: 'chart-purple' }, + { operator: 'between', value: [200, 1000], color: 'chart-blue' }, + { operator: 'eq', value: 0, color: 'chart-cyan' }, + { operator: 'neq', value: 'OK', color: 'chart-success' }, + ]; + const create = await postTile({ colorRules }).expect(200); + expect(create.body.data.tiles[0].config.colorRules).toEqual(colorRules); + + const get = await authRequest( + 'get', + `${BASE_URL}/${create.body.data.id}`, + ).expect(200); + expect(get.body.data.tiles[0].config.colorRules).toEqual(colorRules); + }); + + it('round-trips a raw SQL number tile with a static color', async () => { + const create = await authRequest('post', BASE_URL) + .send({ + name: 'Raw SQL number color', + tiles: [rawSqlNumberTile({ color: 'chart-blue' })], + tags: [], + }) + .expect(200); + expect(create.body.data.tiles[0].config.color).toBe('chart-blue'); + + const get = await authRequest( + 'get', + `${BASE_URL}/${create.body.data.id}`, + ).expect(200); + expect(get.body.data.tiles[0].config.color).toBe('chart-blue'); + }); + + it('round-trips color and colorRules through an update (PUT)', async () => { + const created = await postTile({}).expect(200); + const dashboardId = created.body.data.id; + const tile = created.body.data.tiles[0]; + + const colorRules = [ + { + operator: 'gte', + value: 5000, + color: 'chart-error', + label: 'Critical', + }, + { operator: 'between', value: [200, 1000], color: 'chart-blue' }, + ]; + const update = await authRequest('put', `${BASE_URL}/${dashboardId}`) + .send({ + name: 'Number color dashboard', + tiles: [ + { + ...tile, + config: { ...tile.config, color: 'chart-red', colorRules }, + }, + ], + tags: [], + }) + .expect(200); + expect(update.body.data.tiles[0].config).toMatchObject({ + color: 'chart-red', + colorRules, + }); + + const get = await authRequest('get', `${BASE_URL}/${dashboardId}`).expect( + 200, + ); + expect(get.body.data.tiles[0].config).toMatchObject({ + color: 'chart-red', + colorRules, + }); + }); + + it('strips colorRules from a raw SQL number tile, keeping color', async () => { + const create = await authRequest('post', BASE_URL) + .send({ + name: 'Raw SQL colorRules', + tiles: [ + rawSqlNumberTile({ + color: 'chart-blue', + colorRules: [{ operator: 'gt', value: 1, color: 'chart-red' }], + }), + ], + tags: [], + }) + .expect(200); + expect(create.body.data.tiles[0].config.color).toBe('chart-blue'); + expect(create.body.data.tiles[0].config.colorRules).toBeUndefined(); + }); + + // ── Negative: one per schema rejection rule ───────────────────────── + + it('rejects a static color that is not a palette token', async () => { + const res = await postTile({ color: 'red' }).expect(400); + expect(res.body.message).toContain('tiles.0.config.color'); + await postTile({ color: 'chart-99' }).expect(400); + await postTile({ color: '#ff0000' }).expect(400); + }); + + it('rejects a legacy numeric palette token on input', async () => { + // chart-1..chart-10 were renamed to hue names; the input enum is + // strict hue-only, so a legacy token in a hand-written payload is + // rejected. Legacy tokens are normalized on read, never accepted on + // write. + await postTile({ color: 'chart-1' }).expect(400); + }); + + it('rejects more than 10 colorRules', async () => { + const colorRules = Array.from({ length: 11 }, (_, i) => ({ + operator: 'gt', + value: i, + color: 'chart-blue', + })); + const res = await postTile({ colorRules }).expect(400); + expect(res.body.message).toContain('tiles.0.config.colorRules'); + }); + + it('rejects a between rule whose value is not a two-number tuple', async () => { + await postTile({ + colorRules: [{ operator: 'between', value: 100, color: 'chart-blue' }], + }).expect(400); + }); + + it('rejects a numeric operator rule with a string value', async () => { + await postTile({ + colorRules: [{ operator: 'gt', value: 'high', color: 'chart-blue' }], + }).expect(400); + }); + + it('rejects operators the number-tile editor never emits', async () => { + for (const operator of ['contains', 'startsWith', 'endsWith', 'regex']) { + const res = await postTile({ + colorRules: [{ operator, value: 'error', color: 'chart-blue' }], + }).expect(400); + expect(res.body.message).toContain('tiles.0.config.colorRules'); + } + }); + + it('rejects a per-rule color that is not a palette token', async () => { + const res = await postTile({ + colorRules: [{ operator: 'gt', value: 1, color: 'red' }], + }).expect(400); + expect(res.body.message).toContain('tiles.0.config.colorRules'); + // Legacy numeric tokens are normalized on read, never accepted on write. + await postTile({ + colorRules: [{ operator: 'gt', value: 1, color: 'chart-1' }], + }).expect(400); + }); + + it('rejects a rule label longer than 40 characters', async () => { + await postTile({ + colorRules: [ + { + operator: 'gt', + value: 1, + color: 'chart-blue', + label: 'x'.repeat(41), + }, + ], + }).expect(400); + }); + + // ── Backward compatibility: existing dashboards keep working ──────── + + it('round-trips a number tile with neither color nor colorRules', async () => { + const create = await postTile({}).expect(200); + expect(create.body.data.tiles[0].config.color).toBeUndefined(); + expect(create.body.data.tiles[0].config.colorRules).toBeUndefined(); + }); + + it('normalizes a legacy numeric token on a builder number tile to its hue name on read', async () => { + const create = await postTile({ color: 'chart-green' }).expect(200); + const dashboardId = create.body.data.id; + + // Simulate a tile saved during the #2265 window by writing a legacy + // numeric token directly to Mongo (the `tiles` field is `Mixed`, so + // this bypasses the create-path enum). + await Dashboard.updateOne( + { _id: dashboardId }, + { $set: { 'tiles.0.config.color': 'chart-1' } }, + ); + + const get = await authRequest('get', `${BASE_URL}/${dashboardId}`).expect( + 200, + ); + // chart-1 maps to chart-green (LEGACY_CHART_PALETTE_TOKEN_MAP). + expect(get.body.data.tiles[0].config.color).toBe('chart-green'); + }); + + it('normalizes legacy colorRule colors and drops unresolvable ones on read', async () => { + const create = await postTile({ + colorRules: [{ operator: 'gt', value: 1, color: 'chart-green' }], + }).expect(200); + const dashboardId = create.body.data.id; + + // Direct Mongo write: a legacy numeric token (normalized to its hue + // name on read) and an unrecognized token (dropped on read so the + // response stays within the palette-token enum). Neither is reachable + // through the validated create path. + await Dashboard.updateOne( + { _id: dashboardId }, + { + $set: { + 'tiles.0.config.colorRules': [ + { operator: 'gt', value: 1, color: 'chart-1' }, + { operator: 'gt', value: 2, color: 'not-a-token' }, + ], + }, + }, + ); + + const get = await authRequest('get', `${BASE_URL}/${dashboardId}`).expect( + 200, + ); + // chart-1 maps to chart-green; the unresolvable rule is dropped. + expect(get.body.data.tiles[0].config.colorRules).toEqual([ + { operator: 'gt', value: 1, color: 'chart-green' }, + ]); + }); + + it('omits colorRules when every stored rule color is unresolvable on read', async () => { + const create = await postTile({ + colorRules: [{ operator: 'gt', value: 1, color: 'chart-green' }], + }).expect(200); + const dashboardId = create.body.data.id; + + // Direct Mongo write of an unresolvable token (not reachable via the + // validated create path); the only rule drops, so the field is omitted + // rather than returned as an empty array. + await Dashboard.updateOne( + { _id: dashboardId }, + { + $set: { + 'tiles.0.config.colorRules': [ + { operator: 'gt', value: 1, color: 'not-a-token' }, + ], + }, + }, + ); + + const get = await authRequest('get', `${BASE_URL}/${dashboardId}`).expect( + 200, + ); + expect(get.body.data.tiles[0].config.colorRules).toBeUndefined(); + }); + + it('normalizes a legacy numeric token on a raw SQL number tile to its hue name on read', async () => { + const create = await authRequest('post', BASE_URL) + .send({ + name: 'Raw SQL legacy color', + tiles: [rawSqlNumberTile({ color: 'chart-blue' })], + tags: [], + }) + .expect(200); + const dashboardId = create.body.data.id; + + await Dashboard.updateOne( + { _id: dashboardId }, + { $set: { 'tiles.0.config.color': 'chart-4' } }, + ); + + const get = await authRequest('get', `${BASE_URL}/${dashboardId}`).expect( + 200, + ); + // chart-4 maps to chart-red. + expect(get.body.data.tiles[0].config.color).toBe('chart-red'); + }); + }); + describe('Containers and tabs', () => { const buildTile = ( sourceId: string, diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 9075e15388..345ea97bf5 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -122,6 +122,122 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: Custom unit label. * example: "ms" * + * ChartPaletteToken: + * type: string + * enum: [chart-blue, chart-orange, chart-red, chart-cyan, chart-green, chart-pink, chart-purple, chart-light-blue, chart-brown, chart-gray, chart-success, chart-warning, chart-error] + * description: > + * Palette token used to color a number tile. Tokens reflow across + * light and dark themes, so raw hex values are not accepted. + * example: "chart-red" + * NumericColorCondition: + * type: object + * required: + * - operator + * - value + * - color + * description: Color rule comparing the displayed value against a single numeric bound. + * properties: + * operator: + * type: string + * enum: [gt, gte, lt, lte] + * description: Numeric comparison operator. + * example: "gt" + * value: + * type: number + * description: > + * Numeric bound the displayed value is compared against. Only + * finite numbers are accepted (Infinity and NaN are rejected). + * example: 100 + * color: + * $ref: '#/components/schemas/ChartPaletteToken' + * description: Color applied when the rule matches. + * label: + * type: string + * maxLength: 40 + * description: Optional label describing the rule. + * example: "High" + * BetweenColorCondition: + * type: object + * required: + * - operator + * - value + * - color + * description: Color rule matching when the displayed value falls within an inclusive range. + * properties: + * operator: + * type: string + * enum: [between] + * description: Range comparison operator. + * example: "between" + * value: + * type: array + * minItems: 2 + * maxItems: 2 + * items: + * type: number + * description: > + * Inclusive [min, max] range. Both bounds must be finite numbers. + * example: [100, 500] + * color: + * $ref: '#/components/schemas/ChartPaletteToken' + * description: Color applied when the rule matches. + * label: + * type: string + * maxLength: 40 + * description: Optional label describing the rule. + * example: "Warning" + * EqualityColorCondition: + * type: object + * required: + * - operator + * - value + * - color + * description: Color rule matching when the displayed value equals (eq) or does not equal (neq) a number or string. + * properties: + * operator: + * type: string + * enum: [eq, neq] + * description: Equality comparison operator. + * example: "eq" + * value: + * oneOf: + * - type: number + * - type: string + * maxLength: 200 + * description: > + * A finite number, or a string up to 200 characters, to compare + * for equality. + * example: "OK" + * color: + * $ref: '#/components/schemas/ChartPaletteToken' + * description: Color applied when the rule matches. + * label: + * type: string + * maxLength: 40 + * description: Optional label describing the rule. + * example: "Healthy" + * NumberTileColorCondition: + * description: > + * A single conditional color rule for a number tile. Rules are + * evaluated in order and the last matching rule wins. When no rule + * matches, the static color applies, then the default text color. + * The number-tile editor surfaces numeric and equality operators + * only. + * oneOf: + * - $ref: '#/components/schemas/NumericColorCondition' + * - $ref: '#/components/schemas/BetweenColorCondition' + * - $ref: '#/components/schemas/EqualityColorCondition' + * discriminator: + * propertyName: operator + * mapping: + * gt: '#/components/schemas/NumericColorCondition' + * gte: '#/components/schemas/NumericColorCondition' + * lt: '#/components/schemas/NumericColorCondition' + * lte: '#/components/schemas/NumericColorCondition' + * between: '#/components/schemas/BetweenColorCondition' + * eq: '#/components/schemas/EqualityColorCondition' + * neq: '#/components/schemas/EqualityColorCondition' + * * TimeChartSeries: * type: object * required: @@ -627,6 +743,18 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * numberFormat: * $ref: '#/components/schemas/NumberFormat' * description: Number formatting options for displayed values. + * color: + * $ref: '#/components/schemas/ChartPaletteToken' + * description: Optional static color applied to the displayed number. + * colorRules: + * type: array + * maxItems: 10 + * description: > + * Ordered conditional color rules evaluated against the displayed + * value (last match wins). Falls back to color, then the default + * text color when no rule matches. + * items: + * $ref: '#/components/schemas/NumberTileColorCondition' * * PieBuilderChartConfig: * type: object @@ -902,6 +1030,11 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * enum: [number] * description: Display as a single big-number chart. * example: "number" + * color: + * $ref: '#/components/schemas/ChartPaletteToken' + * description: > + * Optional static color applied to the displayed number. Raw + * SQL number tiles do not support conditional colorRules. * * PieRawSqlChartConfig: * description: Raw SQL configuration for a pie chart. diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index a8506a3e6b..81b0191f4e 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -15,6 +15,8 @@ import { import { AggregateFunctionSchema, BuilderSavedChartConfig, + ChartPaletteToken, + ColorCondition, DASHBOARD_MAX_CONTAINERS, DashboardContainer, DashboardContainerSchema, @@ -23,7 +25,10 @@ import { isOnClickDashboardById, isOnClickSearchById, isTraceSource, + NumberTileColorCondition, + NumberTileColorConditionSchema, RawSqlSavedChartConfig, + resolveChartPaletteToken, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; import { SearchConditionLanguageSchema as whereLanguageSchema } from '@hyperdx/common-utils/dist/types'; @@ -148,6 +153,36 @@ const convertToExternalSelectItem = ( }; }; +// Normalize the per-rule palette colors of a number tile's `colorRules` +// for the API response. `colorRules` shipped after the hue rename so +// stored rules already hold hue-named tokens; running each through +// `resolveChartPaletteToken` keeps the static `color` (which can hold a +// legacy `chart-1`..`chart-10` token from before the rename) and the rule +// colors on a single normalization path. A rule whose color cannot be +// resolved (reachable only via a direct DB write, since the input schema +// validates rule colors) is dropped, mirroring how the static `color` +// field omits an unresolvable token, so the response always stays within +// the palette-token enum instead of leaking an unknown string. When no +// rule survives (or the stored array is empty) the field is omitted +// entirely rather than emitted as `[]`, matching the static `color` omit. +const toExternalColorRules = ( + colorRules: ColorCondition[] | undefined, +): NumberTileColorCondition[] | undefined => { + if (!colorRules) return undefined; + const resolved = colorRules.flatMap((rule): NumberTileColorCondition[] => { + const color = resolveChartPaletteToken(rule.color); + if (!color) return []; + // Re-validate against the number-tile subset so the response stays + // within the documented operator set: a string-match or regex rule + // (reachable only via a direct DB write, since neither the editor nor + // the input schema produces one on a number tile) is dropped, just + // like an unresolvable color token. + const parsed = NumberTileColorConditionSchema.safeParse({ ...rule, color }); + return parsed.success ? [parsed.data] : []; + }); + return resolved.length > 0 ? resolved : undefined; +}; + const convertToExternalTileChartConfig = ( config: SavedChartConfig, ): ExternalDashboardTileConfig | undefined => { @@ -195,6 +230,10 @@ const convertToExternalTileChartConfig = ( sqlTemplate: config.sqlTemplate, sourceId: config.source, numberFormat: config.numberFormat, + // Raw SQL number tiles carry the static tile color too (no + // colorRules; see the schema). Normalize a legacy token saved + // before the hue rename to its hue name on output. + color: resolveChartPaletteToken(config.color), }; case DisplayType.Pie: return { @@ -276,6 +315,16 @@ const convertToExternalTileChartConfig = ( ? [convertToExternalSelectItem(config.select[0])] : [DEFAULT_SELECT_ITEM], numberFormat: config.numberFormat, + // Normalize stored palette tokens on the way out. A static `color` + // saved before the hue rename holds a legacy `chart-1`..`chart-10` + // token in Mongo (the `tiles` field is `Mixed`), so map it to the + // hue name via `resolveChartPaletteToken`, matching the app's + // fetch-time `normalizeDashboardTileColors`. `colorRules` colors go + // through the same path (see `toExternalColorRules`). An absent or + // unrecognized token resolves to undefined and is omitted from the + // response, keeping it within the palette-token enum. + color: resolveChartPaletteToken(config.color), + colorRules: toExternalColorRules(config.colorRules), }; case DisplayType.Pie: return { @@ -576,6 +625,12 @@ export function convertToInternalTileConfig( externalConfig.displayType === 'table' ? externalConfig.onClick : undefined, + // Only the raw SQL number variant carries `color`; table and pie + // do not expose it. `_.omitBy(_.isNil)` below drops it when absent. + color: + externalConfig.displayType === 'number' + ? externalConfig.color + : undefined, } satisfies RawSqlSavedChartConfig; break; default: @@ -635,6 +690,11 @@ export function convertToInternalTileConfig( source: externalConfig.sourceId, where: '', numberFormat: externalConfig.numberFormat, + // The input schema validates these as hue-only palette tokens, + // so pass them through directly; `_.omitBy(_.isNil)` below drops + // them when absent. + color: externalConfig.color, + colorRules: externalConfig.colorRules, name, } satisfies BuilderSavedChartConfig; break; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index d098e4d44e..d271c5ef4c 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -3,11 +3,13 @@ import { AggregateFunctionSchema, alertNoteSchema, AlertThresholdType, + ChartPaletteTokenSchema, DASHBOARD_CONTAINER_ID_MAX, DASHBOARD_MAX_TILES, DashboardFilterSchema, MetricsDataType, NumberFormatSchema, + NumberTileColorConditionSchema, OnClickDashboardSchema, OnClickSearchSchema, scheduleStartAtSchema, @@ -301,6 +303,14 @@ const externalDashboardTableRawSqlChartConfigSchema = const externalDashboardNumberRawSqlChartConfigSchema = externalDashboardRawSqlChartConfigBaseSchema.extend({ displayType: z.literal('number'), + // Raw SQL number tiles expose the same static tile color as builder + // number tiles: the editor gates the picker on displayType, not + // configType (`ChartDisplaySettingsDrawer`). `colorRules` is + // intentionally omitted here because the editor's save path + // (`convertFormStateToSavedChartConfig`) picks `color` but not + // `colorRules` for raw SQL configs, so persisted raw SQL number tiles + // never carry rules. + color: ChartPaletteTokenSchema.optional(), }); const externalDashboardPieRawSqlChartConfigSchema = @@ -313,6 +323,19 @@ const externalDashboardNumberChartConfigSchema = z.object({ sourceId: objectIdSchema, select: z.array(externalDashboardSelectItemSchema).length(1), numberFormat: NumberFormatSchema.optional(), + // Number-tile color authoring. Mirrors the internal + // `SharedChartSettingsSchema` fields (common-utils types.ts), which the + // editor gates to number tiles (`ChartDisplaySettingsDrawer`: + // `showTileColor = displayType === DisplayType.Number`). `color` is a + // hue-named palette token; `colorRules` are ordered conditional rules + // (last match wins), capped at 10 to match the editor. `colorRules` uses + // `NumberTileColorConditionSchema` (numeric and equality operators only), + // not the full `ColorConditionSchema`, so the API cannot accept the + // string-match or regex rules the number-tile editor never emits. Both + // schemas are imported from common-utils so the external surface cannot + // drift from what the UI persists. + color: ChartPaletteTokenSchema.optional(), + colorRules: z.array(NumberTileColorConditionSchema).max(10).optional(), }); const externalDashboardPieChartConfigSchema = z.object({ diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dc38e71b2f..28d38ec500 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1029,60 +1029,94 @@ export const ChartPaletteTokenSchema = z.enum(CHART_PALETTE_TOKENS); * Lives in common-utils so both the app and a future external-API parity * PR can import it. */ +// Numeric ordered operators (gt | gte | lt | lte). +const numericOrderedColorCondition = z.object({ + operator: z.enum(['gt', 'gte', 'lt', 'lte']), + value: z.number().finite(), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), +}); + +const betweenColorCondition = z.object({ + operator: z.literal('between'), + value: z.tuple([z.number().finite(), z.number().finite()]), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), +}); + +// Equality against a number or a string value. +const equalityColorCondition = z.object({ + operator: z.enum(['eq', 'neq']), + value: z.union([z.number().finite(), z.string().max(200)]), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), +}); + +// String-match operators, kept at the schema level only for a future +// table-tile slice (see the doc comment above). The number-tile editor +// never emits these. +const stringMatchColorCondition = z.object({ + operator: z.enum(['contains', 'startsWith', 'endsWith']), + value: z.string().min(1).max(200), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), +}); + +const regexColorCondition = z.object({ + operator: z.literal('regex'), + value: z + .string() + .min(1) + .max(500) + .refine( + v => { + try { + new RegExp(v); + return true; + } catch { + return false; + } + }, + { message: 'Invalid regex pattern' }, + ), + color: ChartPaletteTokenSchema, + label: z.string().max(40).optional(), +}); + export const ColorConditionSchema = z.discriminatedUnion('operator', [ - // Numeric ordered operators - z.object({ - operator: z.enum(['gt', 'gte', 'lt', 'lte']), - value: z.number().finite(), - color: ChartPaletteTokenSchema, - label: z.string().max(40).optional(), - }), - z.object({ - operator: z.literal('between'), - value: z.tuple([z.number().finite(), z.number().finite()]), - color: ChartPaletteTokenSchema, - label: z.string().max(40).optional(), - }), - // Equality (number OR string) - z.object({ - operator: z.enum(['eq', 'neq']), - value: z.union([z.number().finite(), z.string().max(200)]), - color: ChartPaletteTokenSchema, - label: z.string().max(40).optional(), - }), - // String operators (allowed at schema level for future table-tile reuse) - z.object({ - operator: z.enum(['contains', 'startsWith', 'endsWith']), - value: z.string().min(1).max(200), - color: ChartPaletteTokenSchema, - label: z.string().max(40).optional(), - }), - z.object({ - operator: z.literal('regex'), - value: z - .string() - .min(1) - .max(500) - .refine( - v => { - try { - new RegExp(v); - return true; - } catch { - return false; - } - }, - { message: 'Invalid regex pattern' }, - ), - color: ChartPaletteTokenSchema, - label: z.string().max(40).optional(), - }), + numericOrderedColorCondition, + betweenColorCondition, + equalityColorCondition, + stringMatchColorCondition, + regexColorCondition, ]); export type ColorCondition = z.infer; +/** + * The subset of color-rule operators the number-tile editor actually + * emits (`ColorRulesEditor.tsx` OPERATOR_OPTIONS: gt, gte, lt, lte, + * between, eq, neq). The external dashboards API and the MCP dashboard + * tool validate number-tile `colorRules` against this schema rather than + * the full `ColorConditionSchema`, so the authoring surface cannot accept + * the string-match or regex rules the UI can never produce (a stored + * regex would be compiled and evaluated at render time). Keep the operator + * set in sync with the editor's options. + */ +export const NumberTileColorConditionSchema = z.discriminatedUnion('operator', [ + numericOrderedColorCondition, + betweenColorCondition, + equalityColorCondition, +]); + +export type NumberTileColorCondition = z.infer< + typeof NumberTileColorConditionSchema +>; + // When making changes here, consider if they need to be made to the external API -// schema as well (packages/api/src/utils/zod.ts). +// as well: the Zod schema (packages/api/src/utils/zod.ts) and the hand-written +// OpenAPI JSDoc (packages/api/src/routers/external-api/v2/dashboards.ts), which +// duplicates this shape for the generated spec. /** * Schema describing settings which are shared between Raw SQL * chart configs and Structured ChartBuilder chart configs From 99f33d40190991611fc668aead23d83473c0b899 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 03:36:34 +0000 Subject: [PATCH 043/728] chore(deps): bump shell-quote from 1.8.3 to 1.8.4 (#2430) Bumps [shell-quote](https://github.com/ljharb/shell-quote) from 1.8.3 to 1.8.4.
Changelog

Sourced from shell-quote's changelog.

v1.8.4 - 2026-05-22

Commits

  • [Fix] quote: validate object-token shapes 4378a6e
  • [Dev Deps] update @ljharb/eslint-config, auto-changelog, eslint, npmignore 22ebec0
  • [Tests] increase coverage 9f3caa3
  • [readme] replace runkit CI badge with shields.io check-runs badge 3344a04
  • [Dev Deps] update @ljharb/eslint-config 699c511
Commits
  • ff166e2 v1.8.4
  • 4378a6e [Fix] quote: validate object-token shapes
  • 22ebec0 [Dev Deps] update @ljharb/eslint-config, auto-changelog, eslint, `npmig...
  • 9f3caa3 [Tests] increase coverage
  • 3344a04 [readme] replace runkit CI badge with shields.io check-runs badge
  • 699c511 [Dev Deps] update @ljharb/eslint-config
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=shell-quote&package-manager=npm_and_yarn&previous-version=1.8.3&new-version=1.8.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/hyperdxio/hyperdx/network/alerts).
--- yarn.lock | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/yarn.lock b/yarn.lock index 1d1459d40b..ad0209f169 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25633,17 +25633,10 @@ __metadata: languageName: node linkType: hard -"shell-quote@npm:^1.6.1": - version: 1.8.3 - resolution: "shell-quote@npm:1.8.3" - checksum: 10c0/bee87c34e1e986cfb4c30846b8e6327d18874f10b535699866f368ade11ea4ee45433d97bf5eada22c4320c27df79c3a6a7eb1bf3ecfc47f2c997d9e5e2672fd - languageName: node - linkType: hard - -"shell-quote@npm:^1.8.1": - version: 1.8.2 - resolution: "shell-quote@npm:1.8.2" - checksum: 10c0/85fdd44f2ad76e723d34eb72c753f04d847ab64e9f1f10677e3f518d0e5b0752a176fd805297b30bb8c3a1556ebe6e77d2288dbd7b7b0110c7e941e9e9c20ce1 +"shell-quote@npm:^1.6.1, shell-quote@npm:^1.8.1": + version: 1.8.4 + resolution: "shell-quote@npm:1.8.4" + checksum: 10c0/86c93678bc394cb81f5ddcdc87df9c95d279ef9652775cd1cd1eed361404169a8d8cbaacaeed232ab09919e36ee1e5363863570390d78571f8c22b7f6312fb40 languageName: node linkType: hard From 8aad6d6d6c66e2eb6b041f241268f82d675edfb1 Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:58:39 +0200 Subject: [PATCH 044/728] fix: unknown lucene field falls through in search (#2422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary ### Why When a Lucene search field couldn't be resolved to a real column, the `CustomSchemaSQLSerializerV2` fall-through emitted it **verbatim as a raw SQL identifier** (`queryParser.ts`, the old `// It might be an alias, let's just try the column` branch). That had two problems: - **Typos / non-existent fields** produced SQL like `WHERE myTypo = '...'`, which ClickHouse rejects with a confusing `Unknown identifier` error instead of simply returning no rows. - The fall-through was the *only* thing making legitimate **SELECT-alias** references work (ClickHouse resolves SELECT aliases in `WHERE`), but it couldn't tell an alias apart from a typo — both were emitted blindly. - **Why `WITH`-clause aliases matter — saved-search alerts**: a SAVED_SEARCH alert's query selects `count()`, *not* the saved search's `select`, so the saved search's select aliases are injected as expression `WITH` clauses by `computeAliasWithClauses` (in the alert task — unchanged here). Including those in the alias set keeps a Lucene alert `WHERE` such as `body:wrong` (where the saved search declares `toString(Body) AS body`) resolving to the alias instead of collapsing to `(1 = 0)`. Without this, gating the fall-through would have silently regressed saved-search alerts that reference a select alias in their `WHERE` — they'd stop firing. This change makes that distinction explicit: a field that matches a known SELECT alias is emitted as a bare identifier; anything genuinely unknown resolves to the no-match predicate `(1 = 0)`. ### What changed - **Gate the fall-through on known aliases** (`queryParser.ts`): `CustomSchemaConfig` / `CustomSchemaSQLSerializerV2` gain an optional `selectAliases: Set`. The resolver now returns the bare identifier only when `selectAliases.has(field)`; otherwise it returns `found: false`, which renders as `(1 = 0)`. - **Collect aliases from the chart config** (`renderChartConfig.ts`): new `extractSelectAliases({ selectLists, withClauses })` helper gathers the identifiers a Lucene `WHERE` may legally reference, from three sources: - *array-form select* (`{ valueExpression, alias }[]`) — reads `alias` directly; - *string-form select* (raw SQL, e.g. a source's `defaultTableSelectExpression` such as `'Timestamp, ServiceName as service, Body'`) — parsed via the existing `chSqlToAliasMap()` helper to recover declared aliases. This matters because the default search/events view uses string-form selects, so without it the default view would lose alias resolution; - *expression-form `WITH` clauses* — contributes a clause's name only when it declares an expression alias (`isSubquery === false`, i.e. `WITH (expr) AS ident`). Subquery CTEs (`WITH ident AS (subquery)`) are excluded, since they name a table-like source rather than a column usable in `WHERE`. The set is threaded through `renderWhereExpressionStr` into the serializer. - **Null-safety fix** (`queryParser.ts`): the exact-match branch now treats a `null`/`undefined` materialized-columns lookup the same as the existing catch path (`?? new Map()`), so a missing lookup proceeds with no materialized columns instead of throwing on `.entries()`. ### Behavior change | Lucene field | Before | After | |---|---|---| | Real column (`ServiceName:foo`) | resolves | resolves (unchanged) | | SELECT alias (`Content:foo` where `Body AS Content`) | raw identifier (worked by luck) | resolves explicitly | | Unknown / typo (`myTypo:foo`) | raw identifier → ClickHouse error | `(1 = 0)` → no rows | Example that now works end-to-end: `SELECT Body AS Content … WHERE Content = '…'` ### How to test on Vercel preview **Preview routes:** `/search` **Steps:** 1. Go to the search page. 2. Try some of the search examples from the **Behavior change** section above. 3. Ensure alerts fire correctly. ### References - Linear Issue: Closes HDX-4367 --- .changeset/healthy-pans-grab.md | 5 + .../src/__tests__/queryParser.test.ts | 45 ++ .../src/__tests__/renderChartConfig.test.ts | 395 ++++++++++++++++++ .../src/core/renderChartConfig.ts | 81 ++++ packages/common-utils/src/queryParser.ts | 24 +- 5 files changed, 543 insertions(+), 7 deletions(-) create mode 100644 .changeset/healthy-pans-grab.md diff --git a/.changeset/healthy-pans-grab.md b/.changeset/healthy-pans-grab.md new file mode 100644 index 0000000000..df79b08a65 --- /dev/null +++ b/.changeset/healthy-pans-grab.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": patch +--- + +fix: unknown lucene field falls through in search diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index a2afc2118f..d735d52f2a 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -35,6 +35,8 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { return { name: 'SeverityNumber', type: 'UInt8' }; } else if (column === 'foo') { return { name: 'foo', type: 'String' }; + } else if (column === 'bar') { + return { name: 'bar', type: 'String' }; } else if (column === 'MaterializedExample') { return { name: 'MaterializedExample', type: 'String' }; } else { @@ -494,6 +496,49 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { expect(actualSql).toBe(expectedSql); }); + describe('CustomSchemaSQLSerializerV2: select alias resolution', () => { + // ClickHouse resolves SELECT aliases in WHERE, so a Lucene field matching a + // known alias must resolve to a bare identifier. A field that is neither a + // column nor an alias is genuinely unknown and must not be injected as raw + // SQL (which ClickHouse would reject as an unknown identifier). + const aliasSerializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression: 'Body', + selectAliases: new Set(['Content']), + }); + + it('resolves a known select alias as a column reference', async () => { + const res = await aliasSerializer.getColumnForField('Content', {}); + expect(res).toEqual( + expect.objectContaining({ found: true, column: 'Content' }), + ); + }); + + it('renders a select alias reference as a bare identifier', async () => { + const sql = await new SearchQueryBuilder( + 'Content:foo', + aliasSerializer, + ).build(); + expect(sql).toBe("((Content ILIKE '%foo%'))"); + }); + + it('returns not-found for an unknown, non-alias field', async () => { + const res = await serializer.getColumnForField('NotAColumn', {}); + expect(res).toEqual(expect.objectContaining({ found: false })); + }); + + it('renders an unknown, non-alias field as the no-match predicate', async () => { + const sql = await new SearchQueryBuilder( + 'NotAColumn:foo', + serializer, + ).build(); + expect(sql).toBe('((1 = 0))'); + }); + }); + describe('CustomSchemaSQLSerializerV2: implicit column falls back to bodyExpression', () => { // Symmetric counterpart to the one-way fallback in // `getEventBody` (packages/app/src/source.ts): a log source admin who diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 96e26dc805..8acf84b011 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -31,6 +31,7 @@ describe('renderChartConfig', () => { { name: 'value', type: 'Float64' }, { name: 'TraceId', type: 'String' }, { name: 'ServiceName', type: 'String' }, + { name: 'severity', type: 'String' }, ]; mockMetadata = { getColumns: jest.fn().mockResolvedValue([ @@ -2252,6 +2253,400 @@ describe('renderChartConfig', () => { }); }); + describe('SELECT alias references in a Lucene WHERE', () => { + // ClickHouse resolves SELECT aliases in WHERE. A Lucene WHERE that + // references a SELECT alias (here `Body AS Content`) must resolve the alias + // to a bare identifier rather than rendering the no-match predicate. + it('resolves a Lucene WHERE reference to a SELECT alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [{ valueExpression: 'Body', alias: 'Content' }], + where: 'Content:"swagger"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + }); + + it('renders an unknown, non-alias field in a Lucene WHERE as the no-match predicate', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [{ valueExpression: 'Body', alias: 'Content' }], + where: 'NotAColumn:"swagger"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain('(1 = 0)'); + expect(actual).not.toContain('NotAColumn'); + }); + + // String-form select lists (e.g. a source's defaultTableSelectExpression) + // declare aliases as raw SQL. These are parsed to recover the aliases so + // alias references in a Lucene WHERE still resolve. + it('resolves a Lucene WHERE reference to an alias in a string-form select', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: 'Body AS Content', + where: 'Content:"swagger"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + }); + + it('resolves an alias from a realistic default-view string select', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: 'Timestamp, ServiceName as service, Body', + where: 'service:"prod"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("service = 'prod'"); + }); + + it('renders an unknown field as the no-match predicate when a string select declares no matching alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: 'Timestamp, Body', + where: 'NotAColumn:"x"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain('(1 = 0)'); + expect(actual).not.toContain('NotAColumn'); + }); + + // SAVED_SEARCH alerts select count() while injecting the saved search's + // select aliases as expression-form WITH clauses (isSubquery: false). A + // Lucene WHERE referencing such an alias must resolve to the bare + // identifier, not the no-match predicate. + it('resolves a Lucene WHERE reference to an expression-form WITH alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + with: [ + { + name: 'Content', + sql: chSql`toString(Body)`, + isSubquery: false, + }, + ], + select: [{ aggFn: 'count', valueExpression: '' }], + where: 'Content:"swagger"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + expect(actual).not.toContain('(1 = 0)'); + }); + + // Subquery CTEs (isSubquery: true) name a table-like source, not a column, + // so an alias reference to one in a Lucene WHERE is still unknown and must + // render the no-match predicate. + it('does not treat a subquery WITH alias as a column reference', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + with: [ + { + name: 'sub', + sql: chSql`SELECT 1`, + isSubquery: true, + }, + ], + select: [{ aggFn: 'count', valueExpression: '' }], + where: 'sub:"x"', + whereLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain('(1 = 0)'); + }); + }); + + describe('SELECT alias references in other Lucene call sites', () => { + // The main WHERE clause is covered above; these exercise the remaining + // Lucene-capable call sites in renderChartConfig (filters array, + // per-aggregate conditions, and HAVING), all of which must resolve SELECT + // aliases identically to the main WHERE. + + it('resolves a Lucene filter reference to a SELECT alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [{ valueExpression: 'Body', alias: 'Content' }], + where: '', + filters: [{ type: 'lucene', condition: 'Content:"swagger"' }], + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + }); + + it('renders an unknown, non-alias field in a Lucene filter as the no-match predicate', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [{ valueExpression: 'Body', alias: 'Content' }], + where: '', + filters: [{ type: 'lucene', condition: 'NotAColumn:"swagger"' }], + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain('(1 = 0)'); + expect(actual).not.toContain('NotAColumn'); + }); + + // A per-aggregate Lucene condition is pushed into the WHERE clause (when + // every select has one) and also rendered as the aggregate's `...If(...)` + // predicate. Both paths must resolve the alias. The alias here is an + // expression-form WITH clause to avoid a select referencing its own alias. + it('resolves a Lucene per-aggregate condition to a SELECT alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + with: [ + { + name: 'Content', + sql: chSql`toString(Body)`, + isSubquery: false, + }, + ], + select: [ + { + aggFn: 'count', + valueExpression: '', + aggCondition: 'Content:"swagger"', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + expect(actual).not.toContain('(1 = 0)'); + }); + + it('renders an unknown per-aggregate Lucene field as the no-match predicate', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [ + { + aggFn: 'count', + valueExpression: '', + aggCondition: 'NotAColumn:"swagger"', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain('(1 = 0)'); + expect(actual).not.toContain('NotAColumn'); + }); + + // A Lucene value expression (aggFn omitted, valueExpressionLanguage: lucene) + // is rendered as a SELECT column expression and must resolve aliases too. + it('resolves a Lucene value expression reference to a SELECT alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + with: [ + { + name: 'Content', + sql: chSql`toString(Body)`, + isSubquery: false, + }, + ], + select: [ + { + valueExpression: 'Content:"swagger"', + valueExpressionLanguage: 'lucene', + alias: 'matched', + }, + ], + where: '', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + expect(actual).not.toContain('(1 = 0)'); + }); + + // ClickHouse resolves SELECT aliases in HAVING, so a Lucene HAVING that + // references an alias must resolve it rather than render the no-match + // predicate. + it('resolves a Lucene HAVING reference to a SELECT alias', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [{ valueExpression: 'Body', alias: 'Content' }], + where: '', + having: 'Content:"swagger"', + havingLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain("Content = 'swagger'"); + }); + + it('renders an unknown, non-alias field in a Lucene HAVING as the no-match predicate', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [{ valueExpression: 'Body', alias: 'Content' }], + where: '', + having: 'NotAColumn:"swagger"', + havingLanguage: 'lucene', + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain('(1 = 0)'); + expect(actual).not.toContain('NotAColumn'); + }); + }); + describe('SETTINGS clause', () => { const config: ChartConfigWithOptDateRangeEx = { displayType: DisplayType.Table, diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index c25e985890..75cc63fc33 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -5,6 +5,7 @@ import SqlString from 'sqlstring'; import { ChSql, chSql, + chSqlToAliasMap, concatChSql, convertCHDataTypeToJSType, JSDataType, @@ -689,6 +690,11 @@ async function renderSelectList( const isRatio = isRatioChartConfig(selectList, chartConfig); + const selectAliases = extractSelectAliases({ + selectLists: [chartConfig.select, chartConfig.groupBy], + withClauses: chartConfig.with, + }); + const selectsSQL = await Promise.all( selectList.map(async select => { const whereClause = await renderWhereExpression({ @@ -702,6 +708,7 @@ async function renderSelectList( metadata, connectionId: chartConfig.connection, with: chartConfig.with, + selectAliases, }); let expr: ChSql; @@ -719,6 +726,7 @@ async function renderSelectList( metadata, connectionId: chartConfig.connection, with: chartConfig.with, + selectAliases, }) : chSql`${{ UNSAFE_RAW_SQL: select.valueExpression }}`; } else if ( @@ -978,6 +986,64 @@ function renderFrom({ ); } +/** + * Collects the aliases a Lucene WHERE may legally reference as bare + * identifiers: SELECT-list aliases plus expression-form WITH clauses. + * + * Array-form select lists expose `alias` directly. Raw-string lists (e.g. a + * source's `defaultTableSelectExpression` like + * 'Timestamp, ServiceName as service, Body') are parsed via `chSqlToAliasMap` + * to recover their declared aliases. + * + * WITH clauses only contribute when they declare an expression alias + * (`isSubquery === false`, i.e. `WITH (expr) AS ident`). Subquery CTEs + * (`WITH ident AS (subquery)`) name a table-like source, not a column usable in + * WHERE, so they're excluded. SAVED_SEARCH alerts depend on this: the alert + * query selects count() while the saved search's select aliases are injected as + * expression WITH clauses (see `computeAliasWithClauses` in the alert task). + */ +function extractSelectAliases({ + selectLists, + withClauses, +}: { + selectLists: (SelectList | undefined)[]; + withClauses?: BuilderChartConfigWithDateRange['with']; +}): Set { + const aliases: string[] = []; + + for (const list of selectLists) { + if (list == null) { + continue; + } + + if (typeof list === 'string') { + // Wrap the select-list string in a parse scaffold (mirrors the + // `SELECT * FROM \`t\`` scaffold used below). chSqlToAliasMap swallows + // parse failures and returns {}, so an unparseable list contributes no + // aliases rather than throwing. + const aliasMap = chSqlToAliasMap( + chSql`SELECT ${{ UNSAFE_RAW_SQL: list }} FROM t`, + ); + aliases.push(...Object.keys(aliasMap)); + continue; + } + + for (const col of list) { + if (col.alias && col.alias.trim() !== '') { + aliases.push(col.alias); + } + } + } + + for (const clause of withClauses ?? []) { + if (clause.isSubquery === false && clause.name.trim() !== '') { + aliases.push(clause.name); + } + } + + return new Set(aliases); +} + async function renderWhereExpressionStr({ condition, language, @@ -988,6 +1054,7 @@ async function renderWhereExpressionStr({ useTextIndexForImplicitColumn, connectionId, with: withClauses, + selectAliases, }: { condition: SearchCondition; language: SearchConditionLanguage; @@ -998,6 +1065,7 @@ async function renderWhereExpressionStr({ useTextIndexForImplicitColumn?: BuilderChartConfigWithDateRange['useTextIndexForImplicitColumn']; connectionId: string; with?: BuilderChartConfigWithDateRange['with']; + selectAliases?: Set; }): Promise { let _condition = condition; if (language === 'lucene') { @@ -1009,6 +1077,7 @@ async function renderWhereExpressionStr({ bodyExpression, useTextIndexForImplicitColumn, connectionId: connectionId, + selectAliases, }); const builder = new SearchQueryBuilder(condition, serializer); _condition = await builder.build(); @@ -1058,6 +1127,11 @@ async function renderWhere( chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { + const selectAliases = extractSelectAliases({ + selectLists: [chartConfig.select, chartConfig.groupBy], + withClauses: chartConfig.with, + }); + let whereSearchCondition: ChSql | [] = []; if (isNonEmptyWhereExpr(chartConfig.where)) { whereSearchCondition = wrapChSqlIfNotEmpty( @@ -1072,6 +1146,7 @@ async function renderWhere( metadata, connectionId: chartConfig.connection, with: chartConfig.with, + selectAliases, }), '(', ')', @@ -1100,6 +1175,7 @@ async function renderWhere( metadata, connectionId: chartConfig.connection, with: chartConfig.with, + selectAliases, }); } return null; @@ -1148,6 +1224,7 @@ async function renderWhere( metadata, connectionId: chartConfig.connection, with: chartConfig.with, + selectAliases, }), '(', ')', @@ -1316,6 +1393,10 @@ async function renderHaving( metadata, connectionId: chartConfig.connection, with: chartConfig.with, + selectAliases: extractSelectAliases({ + selectLists: [chartConfig.select, chartConfig.groupBy], + withClauses: chartConfig.with, + }), }); } diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 2a500c6b1c..64c9a7cca3 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -762,6 +762,7 @@ export type CustomSchemaConfig = { * metadata; otherwise, it forces the chosen behavior. */ useTextIndexForImplicitColumn?: UseTextIndex; + selectAliases?: Set; }; function renderArrayFieldExpression({ @@ -1166,6 +1167,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { private skipIndicesPromise?: Promise; private enableTextIndexPromise?: Promise; private kvItemsLookupPromise?: Promise; + private selectAliases: Set | undefined; constructor({ metadata, @@ -1175,6 +1177,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { implicitColumnExpression, bodyExpression, useTextIndexForImplicitColumn, + selectAliases, }: { metadata: Metadata } & CustomSchemaConfig) { super(); this.metadata = metadata; @@ -1185,6 +1188,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { this.connectionId = connectionId; this.useTextIndexForImplicitColumn = useTextIndexForImplicitColumn ?? UseTextIndex.Auto; + this.selectAliases = selectAliases; // Pre-fetch skip indices for potential bloom filter optimization this.skipIndicesPromise = this.metadata @@ -1473,13 +1477,14 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { }; let materializedColumns: Map; try { - // This won't work for CTEs + // This won't work for CTEs. Treat a missing lookup (null/undefined) the + // same as the catch path below: proceed with no materialized columns. materializedColumns = - await this.metadata.getMaterializedColumnsLookupTable({ + (await this.metadata.getMaterializedColumnsLookupTable({ databaseName: this.databaseName, tableName: this.tableName, connectionId: this.connectionId, - }); + })) ?? new Map(); } catch (e) { console.debug('Error in getMaterializedColumnsLookupTable', e); materializedColumns = new Map(); @@ -1586,14 +1591,19 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { throw new Error('Unsupported column type for prefix match'); } - // It might be an alias, let's just try the column - // TODO: Verify aliases + if (this.selectAliases?.has(field)) { + return { + found: true, + columnExpression: field, + columnType: 'Unknown', + }; + } + return { - found: true, + found: false, columnExpression: field, columnType: 'Unknown', }; - // throw new Error(`Column not found: ${field}`); } private isLowerExpression(expr: string): boolean { From 60de7305250cb71be8daacd901215ab8858bb01d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:46:08 +0000 Subject: [PATCH 045/728] chore(deps): bump axios from 1.15.2 to 1.17.0 (#2441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [axios](https://github.com/axios/axios) from 1.15.2 to 1.17.0. Sourced from axios's releases.

v1.17.0 — June 1, 2026

This release adds Node HTTP zstd decompression, hardens config and release workflows, and fixes authentication, header, proxy, and type-handling regressions.

🔒 Security Fixes

  • Config Hardening: Guarded socketPath, params, and paramsSerializer reads with own-property checks to prevent inherited prototype values from affecting request behavior, including SSRF-sensitive paths. (#10901, #10922)
  • Release Publishing: Switched the publish workflow to npm staged publishing for safer, auditable package releases with provenance. (#10926)

🚀 New Features

  • HTTP Compression: Added Node HTTP adapter support for zstd response decompression, with transitional.advertiseZstdAcceptEncoding controlling whether zstd is advertised in Accept-Encoding. (#6792, #10920)

🐛 Bug Fixes

  • Authentication Handling: Restored Basic auth on same-origin Node redirects while continuing to strip credentials cross-origin, and aligned the fetch adapter with HTTP adapter behavior for URL-embedded Basic auth. (#10929, #10896)
  • Proxy TLS: Preserved user httpsAgent TLS options when tunneling HTTPS requests through HTTP CONNECT proxies. (#10957)
  • React Native FormData: Cleared default Content-Type for React Native FormData so multipart boundaries can be generated correctly. (#10898)
  • Headers: Silently skipped empty or whitespace-only header names instead of throwing, matching parsed-header behavior and avoiding React Native response crashes. (#10875)
  • Request Data Merging: Preserved enumerable symbol keys when cloning plain request data through axios merge logic. (#10812)
  • Bundler Compatibility: Converted resolveConfig from an arrow default export to a named function export to avoid webpack and Babel transform interop failures. (#10891)
  • Types: Corrected AxiosHeaders.toJSON() return types and updated CommonJS isCancel typings to narrow to CanceledError<T>. (#10956, #10952)
  • Build Tooling: Avoided emitting a null Authorization header from the GitHub build helper when GITHUB_TOKEN is unset. (#10931)

🔧 Maintenance & Chores

  • HTTP/2 Internals: Extracted Http2Sessions into its own helper module and added direct unit coverage for session pooling, timeout, and cleanup behavior. (#10861)
  • Package Publishing: Reduced published package size by switching to a files allowlist and dropping unneeded unminified bundle source maps. (#10939)
  • CI and Release Automation: Added bundle-size reporting, moved reports to the job summary, fixed bundle-size comparison coverage, added Node 26 to the matrix, pinned npm for staged publishing, and prepared the 1.17.0 release. (#10907, #10911, #10916, #10927, #10935, #10983)
  • Developer Workflow: Added a dev container and iterated on OpenSpec workflow files before removing them from the release branch. (#10925, #10914, #10958)
  • Documentation and Policy: Updated disclosure, contributor, collaboration, threat-model, advanced docs, README badges, release notes, moderator configuration, and project metadata. (#10890, #10889, #10921, #10945, #10905, #10933, #10915, #10887, #10955)
  • Dependencies: Bumped Babel tooling, Commitlint, ESLint, Rollup, Globals, Vitest, Playwright, fs-extra, qs, docs dependencies, and GitHub Actions dependencies including actions/dependency-review-action and zizmorcore/zizmor-action. (#10871, #10879, #10918, #10919, #10934, #10947, #10954, #10960)

🌟 New Contributors

We are thrilled to welcome our new contributors. Thank you for helping improve axios:

Full Changelog

... (truncated)

Changelog

Sourced from axios's changelog.

v1.17.0 — June 1, 2026

This release adds Node HTTP zstd decompression, hardens config and release workflows, and fixes authentication, header, proxy, and type-handling regressions.

🔒 Security Fixes

  • Config Hardening: Guarded socketPath, params, and paramsSerializer reads with own-property checks to prevent inherited prototype values from affecting request behavior, including SSRF-sensitive paths. (#10901, #10922)
  • Release Publishing: Switched the publish workflow to npm staged publishing for safer, auditable package releases with provenance. (#10926)

🚀 New Features

  • HTTP Compression: Added Node HTTP adapter support for zstd response decompression, with transitional.advertiseZstdAcceptEncoding controlling whether zstd is advertised in Accept-Encoding. (#6792, #10920)

🐛 Bug Fixes

  • Authentication Handling: Restored Basic auth on same-origin Node redirects while continuing to strip credentials cross-origin, and aligned the fetch adapter with HTTP adapter behavior for URL-embedded Basic auth. (#10929, #10896)
  • Proxy TLS: Preserved user httpsAgent TLS options when tunneling HTTPS requests through HTTP CONNECT proxies. (#10957)
  • React Native FormData: Cleared default Content-Type for React Native FormData so multipart boundaries can be generated correctly. (#10898)
  • Headers: Silently skipped empty or whitespace-only header names instead of throwing, matching parsed-header behavior and avoiding React Native response crashes. (#10875)
  • Request Data Merging: Preserved enumerable symbol keys when cloning plain request data through axios merge logic. (#10812)
  • Bundler Compatibility: Converted resolveConfig from an arrow default export to a named function export to avoid webpack and Babel transform interop failures. (#10891)
  • Types: Corrected AxiosHeaders.toJSON() return types and updated CommonJS isCancel typings to narrow to CanceledError<T>. (#10956, #10952)
  • Build Tooling: Avoided emitting a null Authorization header from the GitHub build helper when GITHUB_TOKEN is unset. (#10931)

🔧 Maintenance & Chores

  • HTTP/2 Internals: Extracted Http2Sessions into its own helper module and added direct unit coverage for session pooling, timeout, and cleanup behavior. (#10861)
  • Package Publishing: Reduced published package size by switching to a files allowlist and dropping unneeded unminified bundle source maps. (#10939)
  • CI and Release Automation: Added bundle-size reporting, moved reports to the job summary, fixed bundle-size comparison coverage, added Node 26 to the matrix, pinned npm for staged publishing, and prepared the 1.17.0 release. (#10907, #10911, #10916, #10927, #10935, #10983)
  • Developer Workflow: Added a dev container and iterated on OpenSpec workflow files before removing them from the release branch. (#10925, #10914, #10958)
  • Documentation and Policy: Updated disclosure, contributor, collaboration, threat-model, advanced docs, README badges, release notes, moderator configuration, and project metadata. (#10890, #10889, #10921, #10945, #10905, #10933, #10915, #10887, #10955)
  • Dependencies: Bumped Babel tooling, Commitlint, ESLint, Rollup, Globals, Vitest, Playwright, fs-extra, qs, docs dependencies, and GitHub Actions dependencies including actions/dependency-review-action and zizmorcore/zizmor-action. (#10871, #10879, #10918, #10919, #10934, #10947, #10954, #10960)

🌟 New Contributors

We are thrilled to welcome our new contributors. Thank you for helping improve axios:

Full Changelog

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=axios&package-manager=npm_and_yarn&previous-version=1.15.2&new-version=1.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/hyperdxio/hyperdx/network/alerts).
--- yarn.lock | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index ad0209f169..ecc5d7d547 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11247,6 +11247,15 @@ __metadata: languageName: node linkType: hard +"agent-base@npm:6": + version: 6.0.2 + resolution: "agent-base@npm:6.0.2" + dependencies: + debug: "npm:4" + checksum: 10c0/dc4f757e40b5f3e3d674bc9beb4f1048f4ee83af189bae39be99f57bf1f48dde166a8b0a5342a84b5944ee8e6ed1e5a9d801858f4ad44764e84957122fe46261 + languageName: node + linkType: hard + "agent-base@npm:^7.1.0, agent-base@npm:^7.1.1, agent-base@npm:^7.1.2": version: 7.1.4 resolution: "agent-base@npm:7.1.4" @@ -11869,13 +11878,14 @@ __metadata: linkType: hard "axios@npm:^1.13.5, axios@npm:^1.8.3": - version: 1.15.2 - resolution: "axios@npm:1.15.2" + version: 1.17.0 + resolution: "axios@npm:1.17.0" dependencies: - follow-redirects: "npm:^1.15.11" + follow-redirects: "npm:^1.16.0" form-data: "npm:^4.0.5" + https-proxy-agent: "npm:^5.0.1" proxy-from-env: "npm:^2.1.0" - checksum: 10c0/4eeae0feeaa7fdc1ef24f81f8b378fdadedf4aebdd6bf224484675160f8744cf17b9b0d1c215279979940f7e8ce463beffa2f713099612e428eac238515c81d5 + checksum: 10c0/c4fa19ff3a3a63bde48beec03ad816b133b9a6385cccffffe172577ab18c6a70e299280d57f12c80c867fe25df41f92cb91d3a8258708a6d2be3e9e085f92650 languageName: node linkType: hard @@ -16583,7 +16593,7 @@ __metadata: languageName: node linkType: hard -"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.15.11": +"follow-redirects@npm:^1.0.0": version: 1.15.11 resolution: "follow-redirects@npm:1.15.11" peerDependenciesMeta: @@ -16593,6 +16603,16 @@ __metadata: languageName: node linkType: hard +"follow-redirects@npm:^1.16.0": + version: 1.16.0 + resolution: "follow-redirects@npm:1.16.0" + peerDependenciesMeta: + debug: + optional: true + checksum: 10c0/a1e2900163e6f1b4d1ed5c221b607f41decbab65534c63fe7e287e40a5d552a6496e7d9d7d976fa4ba77b4c51c11e5e9f683f10b43011ea11e442ff128d0e181 + languageName: node + linkType: hard + "for-each@npm:^0.3.3, for-each@npm:^0.3.5": version: 0.3.5 resolution: "for-each@npm:0.3.5" @@ -17743,6 +17763,16 @@ __metadata: languageName: node linkType: hard +"https-proxy-agent@npm:^5.0.1": + version: 5.0.1 + resolution: "https-proxy-agent@npm:5.0.1" + dependencies: + agent-base: "npm:6" + debug: "npm:4" + checksum: 10c0/6dd639f03434003577c62b27cafdb864784ef19b2de430d8ae2a1d45e31c4fd60719e5637b44db1a88a046934307da7089e03d6089ec3ddacc1189d8de8897d1 + languageName: node + linkType: hard + "https-proxy-agent@npm:^7.0.1, https-proxy-agent@npm:^7.0.6": version: 7.0.6 resolution: "https-proxy-agent@npm:7.0.6" From 712ba11c5d623298149dafd8d1c77eed4c34f446 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 11 Jun 2026 09:51:21 -0400 Subject: [PATCH 046/728] fix: Navigate to the dashboard listing page after deleting a dashboard (#2444) ## Summary When deleting a dashboard from the dashboard detail page (via the three-dot menu), the user was incorrectly navigated to the temporary dashboard page (`/dashboards`). This changes the post-delete navigation to redirect to the dashboard listing page (`/dashboards/list`) instead, which is consistent with the breadcrumbs and app navigation. ### How to test on Vercel preview Create a dashboard, then delete it and note that you are now navigated to the dashboards listing page. ### References - Linear Issue: Closes HDX-4529 Linear Issue: [HDX-4529](https://linear.app/clickhouse/issue/HDX-4529/navigate-to-the-dashboard-listing-page-after-deleting-a-dashboard)
Open in Web Open in Cursor 
Co-authored-by: Cursor Agent <199161495+cursoragent@users.noreply.github.com> --- .changeset/giant-waves-argue.md | 5 ++++ packages/app/src/DBDashboardPage.tsx | 2 +- .../app/tests/e2e/features/dashboard.spec.ts | 30 +++++++++++++++++++ .../tests/e2e/page-objects/DashboardPage.ts | 9 ++++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 .changeset/giant-waves-argue.md diff --git a/.changeset/giant-waves-argue.md b/.changeset/giant-waves-argue.md new file mode 100644 index 0000000000..50974a7279 --- /dev/null +++ b/.changeset/giant-waves-argue.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: Navigate to the dashboard listing page after deleting a dashboard diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index ec89bb5384..b92504fa9d 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -2415,7 +2415,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { onClick={() => deleteDashboard.mutate(dashboard?.id ?? '', { onSuccess: () => { - router.push('/dashboards'); + router.push('/dashboards/list'); }, }) } diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 50945f86b8..34eb579ca8 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -1487,4 +1487,34 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { }); }, ); + + test( + 'should navigate to the dashboard listing page after deleting a dashboard', + { tag: '@full-stack' }, + async ({ page }) => { + const ts = Date.now(); + const uniqueName = `E2E Delete Nav Dashboard ${ts}`; + + await test.step('Create a new saved dashboard', async () => { + await dashboardsListPage.goto(); + await dashboardsListPage.createNewDashboard(); + await dashboardPage.editDashboardName(uniqueName); + }); + + await test.step('Delete the dashboard via the dashboard menu', async () => { + await dashboardPage.deleteDashboard(); + }); + + await test.step('Verify navigation to the dashboards listing page', async () => { + await expect(page).toHaveURL(/\/dashboards\/list/); + await expect(dashboardsListPage.pageContainer).toBeVisible(); + }); + + await test.step('Verify the deleted dashboard is not listed', async () => { + await expect( + dashboardsListPage.getDashboardCard(uniqueName), + ).toBeHidden(); + }); + }, + ); }); diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index 4bbf8b532b..77c2a7f69b 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -84,6 +84,7 @@ export class DashboardPage { private readonly confirmCancelButton: Locator; private readonly confirmConfirmButton: Locator; private readonly dashboardMenuButton: Locator; + private readonly deleteDashboardMenuItem: Locator; private readonly saveDefaultQueryAndFiltersMenuItem: Locator; private readonly removeDefaultQueryAndFiltersMenuItem: Locator; private readonly exportDashboardMenuItem: Locator; @@ -132,6 +133,9 @@ export class DashboardPage { this.confirmCancelButton = page.getByTestId('confirm-cancel-button'); this.confirmConfirmButton = page.getByTestId('confirm-confirm-button'); this.dashboardMenuButton = page.getByTestId('dashboard-menu-button'); + this.deleteDashboardMenuItem = page.getByRole('menuitem', { + name: 'Delete Dashboard', + }); this.saveDefaultQueryAndFiltersMenuItem = page.getByTestId( 'save-default-query-filters-menu-item', ); @@ -636,6 +640,11 @@ export class DashboardPage { await this.removeDefaultQueryAndFiltersMenuItem.click(); } + async deleteDashboard() { + await this.dashboardMenuButton.click(); + await this.deleteDashboardMenuItem.click(); + } + /** * Export the current dashboard via the dashboard menu and return the parsed * JSON that was downloaded. The export triggers a client-side anchor download From 9a7e392a6e2fb48dcebc9d86a440c4d81a35d58f Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 11 Jun 2026 10:44:43 -0400 Subject: [PATCH 047/728] fix: Add missing numberFormats, compareToPreviousPeriod fields to MCP Schemas (#2445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR adds a few missing display properties to the MCP schemas, so that the agent can use them, and so that they're not stripped when the agent edits an existing dashboard. - Tile-level number formats - Number format on Raw SQL Charts - Compare to Previous Period on line charts ### Screenshots or video The agent can now create and update dashboards leveraging all of these fields: Screenshot 2026-06-11 at 8 17 48 AM Screenshot 2026-06-11 at 8 25 37 AM Screenshot 2026-06-11 at 8 27 25 AM Screenshot 2026-06-11 at 8 21 18 AM ### How to test on Vercel preview This can be tested locally by connecting to MCP and requesting the agent creates and updates dashboards with these properties. ### References - Linear Issue: Closes HDX-4519 Fixes #2435 - Related PRs: --- .changeset/curvy-foxes-give.md | 5 + .../dashboards/saveDashboard.test.ts | 209 ++++++++++++++++++ .../api/src/mcp/tools/dashboards/schemas.ts | 178 +++++++++------ 3 files changed, 322 insertions(+), 70 deletions(-) create mode 100644 .changeset/curvy-foxes-give.md diff --git a/.changeset/curvy-foxes-give.md b/.changeset/curvy-foxes-give.md new file mode 100644 index 0000000000..84f55a7ee6 --- /dev/null +++ b/.changeset/curvy-foxes-give.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +fix: Add missing numberFormats, compareToPreviousPeriod fields to MCP Schemas diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts index 5b58b03e9f..36e359e0bf 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts @@ -371,6 +371,113 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { expect(output.tiles).toHaveLength(1); }); + it('should preserve display fields on raw SQL tiles through save, get, update, and re-get', async () => { + const connectionId = ctx.connection._id.toString(); + const sqlConfig = { + configType: 'sql' as const, + displayType: 'line' as const, + connectionId, + sqlTemplate: + 'SELECT $__timeInterval(Timestamp) AS ts, avg(Duration) AS v FROM otel_traces WHERE $__timeFilter(Timestamp) GROUP BY ts ORDER BY ts LIMIT 1000', + numberFormat: { + output: 'percent' as const, + mantissa: 2, + thousandSeparated: true, + }, + compareToPreviousPeriod: true, + fitYAxisToData: true, + }; + const sqlNumberConfig = { + configType: 'sql' as const, + displayType: 'number' as const, + connectionId, + sqlTemplate: 'SELECT 0.99 AS value LIMIT 1', + numberFormat: { output: 'percent' as const, mantissa: 2 }, + }; + + const saveResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'SQL NumberFormat Round-Trip', + tiles: [ + { name: 'CPU %', config: sqlConfig }, + { name: 'SLO', config: sqlNumberConfig }, + { + name: 'Other Tile', + config: { + configType: 'sql' as const, + displayType: 'table' as const, + connectionId, + sqlTemplate: 'SELECT 1 AS value LIMIT 1', + }, + }, + ], + }, + ); + expect(saveResult.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(saveResult)); + const savedSqlTile = saved.tiles.find( + (t: { name: string }) => t.name === 'CPU %', + ); + expect(savedSqlTile.config).toMatchObject(sqlConfig); + const savedSqlNumberTile = saved.tiles.find( + (t: { name: string }) => t.name === 'SLO', + ); + expect(savedSqlNumberTile.config).toMatchObject(sqlNumberConfig); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { id: saved.id }, + ); + expect(getResult.isError).toBeFalsy(); + const fetched = JSON.parse(getFirstText(getResult)); + const fetchedSqlTile = fetched.tiles.find( + (t: { name: string }) => t.name === 'CPU %', + ); + expect(fetchedSqlTile.config).toMatchObject(sqlConfig); + + // Update a DIFFERENT tile, passing all fetched tiles back — the + // formatted SQL tile must survive the round-trip untouched. + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: saved.id, + name: 'SQL NumberFormat Round-Trip', + tiles: fetched.tiles.map((t: { name: string; config: object }) => + t.name === 'Other Tile' + ? { + ...t, + config: { + ...t.config, + sqlTemplate: 'SELECT 2 AS value LIMIT 1', + }, + } + : t, + ), + }, + ); + expect(updateResult.isError).toBeFalsy(); + + const getAfterUpdate = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { id: saved.id }, + ); + expect(getAfterUpdate.isError).toBeFalsy(); + const refetched = JSON.parse(getFirstText(getAfterUpdate)); + const refetchedSqlTile = refetched.tiles.find( + (t: { name: string }) => t.name === 'CPU %', + ); + expect(refetchedSqlTile.config).toMatchObject(sqlConfig); + const refetchedSqlNumberTile = refetched.tiles.find( + (t: { name: string }) => t.name === 'SLO', + ); + expect(refetchedSqlNumberTile.config).toMatchObject(sqlNumberConfig); + }); + it('should create a dashboard with a heatmap tile on a Trace source', async () => { const sourceId = ctx.traceSource._id.toString(); const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { @@ -606,6 +713,108 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { expect(result.isError).toBe(true); }); + it('should preserve display fields on builder tiles through save, get, update, and re-get', async () => { + const sourceId = ctx.traceSource._id.toString(); + const numberFormat = { + output: 'percent' as const, + mantissa: 2, + thousandSeparated: true, + }; + const lineConfig = { + displayType: 'line' as const, + sourceId, + select: [{ aggFn: 'count' as const, alias: 'Requests' }], + numberFormat, + compareToPreviousPeriod: true, + fitYAxisToData: true, + }; + const barConfig = { + displayType: 'stacked_bar' as const, + sourceId, + select: [{ aggFn: 'count' as const, alias: 'Requests' }], + numberFormat, + }; + const tableConfig = { + displayType: 'table' as const, + sourceId, + select: [{ aggFn: 'count' as const, alias: 'Requests' }], + groupBy: 'SpanName', + numberFormat, + }; + const pieConfig = { + displayType: 'pie' as const, + sourceId, + select: [{ aggFn: 'count' as const, alias: 'Requests' }], + groupBy: 'SpanName', + numberFormat, + }; + const numberConfig = { + displayType: 'number' as const, + sourceId, + select: [{ aggFn: 'count' as const, alias: 'Requests' }], + numberFormat, + }; + const tiles = [ + { name: 'Line', config: lineConfig }, + { name: 'Bar', config: barConfig }, + { name: 'Table', config: tableConfig }, + { name: 'Pie', config: pieConfig }, + { name: 'Number', config: numberConfig }, + ]; + const configByName: Record = { + Line: lineConfig, + Bar: barConfig, + Table: tableConfig, + Pie: pieConfig, + Number: numberConfig, + }; + const assertTiles = (output: { tiles: { name: string }[] }) => { + for (const [name, config] of Object.entries(configByName)) { + const tile = output.tiles.find(t => t.name === name); + expect(tile).toMatchObject({ config }); + } + }; + + const saveResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { name: 'Builder Display Fields', tiles }, + ); + expect(saveResult.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(saveResult)); + assertTiles(saved); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { id: saved.id }, + ); + expect(getResult.isError).toBeFalsy(); + const fetched = JSON.parse(getFirstText(getResult)); + assertTiles(fetched); + + // Update passing all fetched tiles back verbatim — every display + // field must survive a second pass through the MCP input schema. + const updateResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + id: saved.id, + name: 'Builder Display Fields (updated)', + tiles: fetched.tiles, + }, + ); + expect(updateResult.isError).toBeFalsy(); + + const getAfterUpdate = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { id: saved.id }, + ); + expect(getAfterUpdate.isError).toBeFalsy(); + assertTiles(JSON.parse(getFirstText(getAfterUpdate))); + }); + it('should reject heatmap tile on a non-Trace source', async () => { // Create a Log source so the heatmap source-kind gate has // something to reject. The schema accepts the tile shape (the diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index d3a306f426..e1dbad9674 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -16,67 +16,73 @@ import { z } from 'zod'; import { externalQuantileLevelSchema, objectIdSchema } from '@/utils/zod'; // ─── Shared tile schemas for MCP dashboard tools ───────────────────────────── -const mcpNumberFormatSchema = z - .object({ - output: z - .enum([ - 'currency', - 'percent', - 'byte', - 'time', - 'duration', - 'number', - 'data_rate', - 'throughput', - ]) - .describe( - 'Format category. "duration" auto-formats elapsed times as e.g. "1.2s" (use factor for input unit). ' + - '"time" formats clock-style durations. "byte" formats as KB/MB/GB. ' + - '"data_rate" formats as bytes/sec. "throughput" formats as count/sec. ' + - '"currency" prepends a symbol. "percent" appends %.', - ), - mantissa: z - .number() - .int() - .optional() - .describe( - 'Decimal places (0–10). Not used for "time" or "duration" output.', - ), - thousandSeparated: z - .boolean() - .optional() - .describe('Separate thousands (e.g. 1,234,567)'), - average: z - .boolean() - .optional() - .describe('Abbreviate large numbers (e.g. 1.2m)'), - decimalBytes: z - .boolean() - .optional() - .describe( - 'Use decimal base for bytes (1KB = 1000). Only for "byte" output.', - ), - factor: z - .number() - .optional() - .describe( - 'Input unit factor for "time" or "duration" output. ' + - '1 = seconds, 0.001 = milliseconds, 0.000001 = microseconds, 0.000000001 = nanoseconds.', - ), - currencySymbol: z - .string() - .optional() - .describe('Currency symbol (e.g. "$"). Only for "currency" output.'), - unit: z - .string() - .optional() - .describe('Suffix appended to the value (e.g. " req/s")'), - }) - .describe( - 'Controls how the number value is formatted for display. ' + - 'Most useful: { output: "duration", factor: 0.000000001 } to auto-format nanosecond durations, ' + - 'or { output: "number", mantissa: 2, thousandSeparated: true } for clean counts.', - ); + +const seriesLevelNumberFormatDescription = + 'Per-series display formatting, applied to this series only (overrides any tile-level numberFormat). ' + + 'Controls how the series number value(s) are formatted for display. ' + + 'Most useful: { output: "duration", factor: 0.000000001 } to auto-format nanosecond durations, ' + + 'or { output: "number", mantissa: 2, thousandSeparated: true } for clean counts.'; + +const tileLevelNumberFormatDescription = + 'Controls how the number value(s) are formatted for display. Applies to series or numbers without a series-level numberFormat. ' + + 'Most useful: { output: "duration", factor: 0.000000001 } to auto-format nanosecond durations, ' + + 'or { output: "number", mantissa: 2, thousandSeparated: true } for clean counts.'; + +const mcpNumberFormatSchema = z.object({ + output: z + .enum([ + 'currency', + 'percent', + 'byte', + 'time', + 'duration', + 'number', + 'data_rate', + 'throughput', + ]) + .describe( + 'Format category. "duration" auto-formats elapsed times as e.g. "1.2s" (use factor for input unit). ' + + '"time" formats clock-style durations. "byte" formats as KB/MB/GB. ' + + '"data_rate" formats as bytes/sec. "throughput" formats as count/sec. ' + + '"currency" prepends a symbol. "percent" appends %, and divides the value by 100 (0.5 becomes 50%).', + ), + mantissa: z + .number() + .int() + .optional() + .describe( + 'Decimal places (0–10). Not used for "time" or "duration" output.', + ), + thousandSeparated: z + .boolean() + .optional() + .describe('Separate thousands (e.g. 1,234,567)'), + average: z + .boolean() + .optional() + .describe('Abbreviate large numbers (e.g. 1.2m)'), + decimalBytes: z + .boolean() + .optional() + .describe( + 'Use decimal base for bytes (1KB = 1000). Only for "byte" output.', + ), + factor: z + .number() + .optional() + .describe( + 'Input unit factor for "time" or "duration" output. ' + + '1 = seconds, 0.001 = milliseconds, 0.000001 = microseconds, 0.000000001 = nanoseconds.', + ), + currencySymbol: z + .string() + .optional() + .describe('Currency symbol (e.g. "$"). Only for "currency" output.'), + unit: z + .string() + .optional() + .describe('Suffix appended to the value (e.g. " req/s")'), +}); const mcpTileSelectItemSchema = z .object({ @@ -112,11 +118,7 @@ const mcpTileSelectItemSchema = z .describe('Percentile level for aggFn="quantile"'), numberFormat: mcpNumberFormatSchema .optional() - .describe( - 'Per-series display formatting, applied to this series only (overrides any tile-level numberFormat). ' + - 'Example: { output: "duration", factor: 0.000000001 } to render a nanosecond Duration series as human-readable time ' + - 'while leaving sibling count series unformatted.', - ), + .describe(seriesLevelNumberFormatDescription), }) .superRefine((data, ctx) => { if (data.level && data.aggFn !== 'quantile') { @@ -399,6 +401,19 @@ const mcpLineTileSchema = mcpTileLayoutSchema.extend({ .describe( 'Plot as ratio of two metrics (requires exactly 2 select items)', ), + numberFormat: mcpNumberFormatSchema + .optional() + .describe(tileLevelNumberFormatDescription), + compareToPreviousPeriod: z + .boolean() + .optional() + .describe('Overlay the previous period as a dashed comparison series.'), + fitYAxisToData: z + .boolean() + .optional() + .describe( + 'Scale the y-axis to the data range instead of starting at zero.', + ), }), }); @@ -413,6 +428,9 @@ const mcpBarTileSchema = mcpTileLayoutSchema.extend({ fillNulls: z.boolean().optional().default(true), alignDateRangeToGranularity: z.boolean().optional(), asRatio: z.boolean().optional(), + numberFormat: mcpNumberFormatSchema + .optional() + .describe(tileLevelNumberFormatDescription), }), }); @@ -447,6 +465,9 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ 'Render Group By columns on the left side of the table, before the series columns. ' + 'Default false (Group By columns on the right).', ), + numberFormat: mcpNumberFormatSchema + .optional() + .describe(tileLevelNumberFormatDescription), onClick: mcpOnClickSchema.optional(), }), }); @@ -461,10 +482,7 @@ const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ .describe('Exactly one metric to display'), numberFormat: mcpNumberFormatSchema .optional() - .describe( - 'Display formatting for the number value. Example: { output: "duration", factor: 0.000000001 } ' + - 'to auto-format nanosecond durations as human-readable time.', - ), + .describe(tileLevelNumberFormatDescription), }), }); @@ -480,6 +498,9 @@ const mcpPieTileSchema = mcpTileLayoutSchema.extend({ 'Column that defines pie slices. Use PascalCase for top-level columns. ' + "For attributes: SpanAttributes['key'] or ResourceAttributes['key'].", ), + numberFormat: mcpNumberFormatSchema + .optional() + .describe(tileLevelNumberFormatDescription), }), }); @@ -616,6 +637,23 @@ const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ ), fillNulls: z.boolean().optional(), alignDateRangeToGranularity: z.boolean().optional(), + numberFormat: mcpNumberFormatSchema + .optional() + .describe(tileLevelNumberFormatDescription), + compareToPreviousPeriod: z + .boolean() + .optional() + .describe( + 'Overlay the previous period as a dashed comparison series. ' + + 'Valid only when displayType is "line", ignored otherwise.', + ), + fitYAxisToData: z + .boolean() + .optional() + .describe( + 'Scale the y-axis to the data range instead of starting at zero. ' + + 'Valid only when displayType is "line", ignored otherwise.', + ), onClick: mcpOnClickSchema.optional(), }), }); From 60a91e43feab0f0f9f17976b5533cbe0f29c705e Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 11 Jun 2026 08:56:00 -0600 Subject: [PATCH 048/728] fix(mcp): remove max_result_rows from MCP safety settings (#2439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem MCP queries fail with `SETTING_CONSTRAINT_VIOLATION` on ClickHouse connections where a profile constrains `max_result_rows` below our hardcoded default of 100,000 (e.g. ClickHouse Cloud Demo at play-clickstack.clickhouse.com, which caps it at 1,000): ``` Setting max_result_rows shouldn't be greater than 1000. (SETTING_CONSTRAINT_VIOLATION) ``` ## Solution Remove `max_result_rows` from `MCP_CLICKHOUSE_SETTINGS` entirely. The remaining safety settings provide sufficient protection: - **`max_execution_time=30`** — kills runaway queries after 30 seconds regardless of row count - **`readonly=2`** — blocks all write operations (DDL/DML) The `max_result_rows` cap was a defense-in-depth measure to prevent slow queries returning too many rows and to reduce context window usage, but we already have `trimToolResponse` to cap the response size sent to the agent, and the 30s execution timeout bounds what a query can produce. Removing it avoids the need to detect and clamp per-connection constraints, which would require querying `system.settings` metadata and handling edge cases around profile defaults vs constraint ceilings. ## Changes - `helpers.ts` — removed `max_result_rows` from settings constant, updated error hint text - `queryTool.test.ts` — removed `max_result_rows` propagation test, updated combined settings test - `query.test.ts` — updated error hint assertions --- .changeset/remove-mcp-max-result-rows.md | 14 +++++++++++ packages/api/src/mcp/__tests__/query.test.ts | 13 +++++++++-- .../api/src/mcp/__tests__/queryTool.test.ts | 23 ++++--------------- packages/api/src/mcp/tools/query/helpers.ts | 13 ++++++++--- 4 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 .changeset/remove-mcp-max-result-rows.md diff --git a/.changeset/remove-mcp-max-result-rows.md b/.changeset/remove-mcp-max-result-rows.md new file mode 100644 index 0000000000..4c5d65505a --- /dev/null +++ b/.changeset/remove-mcp-max-result-rows.md @@ -0,0 +1,14 @@ +--- +'@hyperdx/api': patch +--- + +fix(mcp): remove max_result_rows from MCP safety settings + +Remove the hardcoded max_result_rows=100000 setting from MCP query +execution. Some ClickHouse connections impose profile constraints that +cap max_result_rows below our default, causing SETTING_CONSTRAINT_VIOLATION +errors. The remaining safety settings (max_execution_time=30, readonly=2) +and trimToolResponse provide sufficient protection. + +Add a SETTING_CONSTRAINT_VIOLATION error hint so constrained settings +surface actionable guidance instead of raw ClickHouse errors. diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index 0307b96813..c523a67999 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -228,14 +228,23 @@ describe('errorHint', () => { it('should match RESULT_IS_TOO_LARGE errors', () => { const hint = errorHint('Code: 396. DB::Exception: RESULT_IS_TOO_LARGE'); expect(hint).not.toBeNull(); - expect(hint).toContain('100,000 rows'); + expect(hint).toContain('too many rows'); expect(hint).toContain('LIMIT'); }); it('should match TOO_MANY_ROWS_OR_BYTES errors', () => { const hint = errorHint('Code: 396. DB::Exception: TOO_MANY_ROWS_OR_BYTES'); expect(hint).not.toBeNull(); - expect(hint).toContain('100,000 rows'); + expect(hint).toContain('too many rows'); + }); + + it('should match SETTING_CONSTRAINT_VIOLATION errors', () => { + const hint = errorHint( + "Setting max_result_rows shouldn't be greater than 1000. (SETTING_CONSTRAINT_VIOLATION)", + ); + expect(hint).not.toBeNull(); + expect(hint).toContain('profile'); + expect(hint).toContain('constraint'); }); it('should return null for unrecognized errors', () => { diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index e7ff5281b9..725e55047c 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -771,17 +771,6 @@ describe('MCP Query Tools', () => { expect(parsed.result?.data?.[0]?.value).toBe(30); }); - it('should propagate max_result_rows=100000 to ClickHouse', async () => { - const result = await callTool(client, 'clickstack_sql', { - connectionId: connection._id.toString(), - sql: "SELECT getSetting('max_result_rows') AS value", - }); - - expect(result.isError).toBeFalsy(); - const parsed = JSON.parse(getFirstText(result)); - expect(parsed.result?.data?.[0]?.value).toBe(100000); - }); - it('should propagate readonly=2 to ClickHouse', async () => { const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), @@ -794,16 +783,15 @@ describe('MCP Query Tools', () => { }); it('should apply all safety settings together without readonly conflicts', async () => { - // readonly=1 rejects setting changes, so max_execution_time and - // max_result_rows would be silently ignored. readonly=2 allows - // setting changes while still blocking writes. This test verifies - // all three settings are applied in a single query. + // readonly=1 rejects setting changes, so max_execution_time + // would be silently ignored. readonly=2 allows setting changes + // while still blocking writes. This test verifies both settings + // are applied in a single query. const result = await callTool(client, 'clickstack_sql', { connectionId: connection._id.toString(), sql: `SELECT getSetting('readonly') AS readonly_mode, - getSetting('max_execution_time') AS max_exec_time, - getSetting('max_result_rows') AS max_rows`, + getSetting('max_execution_time') AS max_exec_time`, }); expect(result.isError).toBeFalsy(); @@ -811,7 +799,6 @@ describe('MCP Query Tools', () => { const row = parsed.result?.data?.[0]; expect(row?.readonly_mode).toBe(2); expect(row?.max_exec_time).toBe(30); - expect(row?.max_rows).toBe(100000); }); }); }); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 5a339bf7de..87f59c362c 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -64,11 +64,10 @@ export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; // ─── Safety limits ─────────────────────────────────────────────────────────── /** ClickHouse settings applied to all MCP query-tool executions. - * readonly=2 so max_execution_time and max_result_rows can be set + * readonly=2 so max_execution_time can be set * (readonly=1 rejects all setting changes). */ const MCP_CLICKHOUSE_SETTINGS = { max_execution_time: 30, - max_result_rows: '100000', readonly: 2, } as const; @@ -477,7 +476,7 @@ export function errorHint(msg: string): string | null { } if (/TOO_MANY_ROWS_OR_BYTES|RESULT_IS_TOO_LARGE/i.test(msg)) { return ( - 'The query returned more than 100,000 rows. ' + + 'The query returned too many rows. ' + 'Add a LIMIT, narrow the time range, or add filters to reduce the result set.' ); } @@ -491,5 +490,13 @@ export function errorHint(msg: string): string | null { 'map attribute keys before retrying.' ); } + if (/SETTING_CONSTRAINT_VIOLATION|shouldn't be greater than/i.test(msg)) { + return ( + 'This ClickHouse connection has a profile that restricts one or more ' + + 'settings to a value lower than requested. This is a server-side ' + + 'constraint — the query cannot override it. Try running the query ' + + 'without the constrained setting, or contact the connection administrator.' + ); + } return null; } From f326ccf88c470af4403b8e414e2378bdd34baa65 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 11 Jun 2026 15:30:56 -0600 Subject: [PATCH 049/728] fix(mcp): clarify orderBy alias quoting and steer event-pattern usage (#2451) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why While testing the MCP server, two friction points surfaced for agents driving the ClickStack tools: 1. **`orderBy` quoting was undocumented.** When sorting by an aliased column whose name contains spaces or special characters (e.g. `P95 Latency`), the query fails unless the alias is quoted. Agents had no way to know this from the schema description. 2. **`clickstack_event_patterns` was under-used.** Agents reached for `clickstack_search` / `clickstack_table` when the real intent was "what kinds of messages/errors exist here". The pattern tool returns frequency-ranked clusters and gives a far better overview, but its description didn't claim that territory. ## What changed - **`orderBy` descriptions** (table tile schema + shared query `orderBySchema`): document that aliases containing spaces or special characters must be quoted, e.g. `'"P95 Latency" DESC'`. - **`clickstack_event_patterns` description**: add explicit "PREFER THIS TOOL over clickstack_search or clickstack_table" guidance for log/error exploration intents, with example phrasings. These are description/schema-text-only tweaks to the MCP tool definitions — no behavioral or query-execution changes. ## Testing Discovered during hands-on testing of the MCP server. `npx lint-staged` (prettier + eslint) passes on all three files. --- .changeset/mcp-orderby-alias-quoting.md | 17 +++++ packages/api/src/mcp/__tests__/query.test.ts | 62 ++++++++++++++++++- .../api/src/mcp/tools/dashboards/schemas.ts | 9 ++- .../api/src/mcp/tools/query/eventPatterns.ts | 6 +- packages/api/src/mcp/tools/query/schemas.ts | 4 +- packages/api/src/mcp/tools/query/table.ts | 52 +++++++++++++--- 6 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 .changeset/mcp-orderby-alias-quoting.md diff --git a/.changeset/mcp-orderby-alias-quoting.md b/.changeset/mcp-orderby-alias-quoting.md new file mode 100644 index 0000000000..8d8ea67b4c --- /dev/null +++ b/.changeset/mcp-orderby-alias-quoting.md @@ -0,0 +1,17 @@ +--- +'@hyperdx/api': patch +--- + +fix(mcp): quote multi-word aliases in orderBy and steer event-pattern usage + +Quote resolved aliases that are not bare identifiers (e.g. `"P95 Latency"`) +in `resolveOrderBy` output, in both the direct alias-match and aggFn-match +paths. Previously an unquoted multi-word alias produced SQL-invalid +`ORDER BY` output. Incoming orderBy values are stripped of surrounding +double-quote/backtick quoting before matching, so agents that already quote +the alias resolve correctly without being double-quoted. + +Also document the alias-quoting requirement in the `orderBy` schema +descriptions, and update the `clickstack_event_patterns` tool description to +steer agents toward it (over `clickstack_search` / `clickstack_table`) when +exploring what messages, errors, or events exist. diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index c523a67999..91796276f3 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -298,9 +298,69 @@ describe('resolveOrderBy', () => { }); it('should prefer alias over synthesized expression', () => { + expect(resolveOrderBy('count', [{ aggFn: 'count', alias: 'Total' }])).toBe( + 'Total', + ); + }); + + it('should quote a multi-word alias resolved from an aggFn match', () => { expect( resolveOrderBy('count', [{ aggFn: 'count', alias: 'Total Rows' }]), - ).toBe('Total Rows'); + ).toBe('"Total Rows"'); + }); + + it('should quote a multi-word alias matched directly', () => { + expect( + resolveOrderBy('P95 Latency', [ + { + aggFn: 'quantile', + valueExpression: 'Duration', + alias: 'P95 Latency', + }, + ]), + ).toBe('"P95 Latency"'); + }); + + it('should quote a multi-word alias and preserve direction', () => { + expect( + resolveOrderBy('p95 latency DESC', [ + { + aggFn: 'quantile', + valueExpression: 'Duration', + alias: 'P95 Latency', + }, + ]), + ).toBe('"P95 Latency" DESC'); + }); + + it('should accept an already-quoted multi-word alias without double-quoting', () => { + expect( + resolveOrderBy('"P95 Latency" DESC', [ + { + aggFn: 'quantile', + valueExpression: 'Duration', + alias: 'P95 Latency', + }, + ]), + ).toBe('"P95 Latency" DESC'); + }); + + it('should normalize a backtick-quoted alias to double quotes', () => { + expect( + resolveOrderBy('`P95 Latency`', [ + { + aggFn: 'quantile', + valueExpression: 'Duration', + alias: 'P95 Latency', + }, + ]), + ).toBe('"P95 Latency"'); + }); + + it('should leave a bare single-word alias unquoted', () => { + expect(resolveOrderBy('Total', [{ aggFn: 'count', alias: 'Total' }])).toBe( + 'Total', + ); }); it('should be case-insensitive for aggFn matching', () => { diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index e1dbad9674..5ad793f61a 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -456,7 +456,14 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ 'from a groupBy: "StatusMessage" table. Mirrors the same field on the REST ' + 'table chart config in `externalDashboardTableChartConfigSchema`.', ), - orderBy: z.string().optional().describe('Sort results by this column'), + orderBy: z + .string() + .optional() + .describe( + 'Sort results by this column. ' + + 'When ordering by an alias that contains spaces or special characters, ' + + `wrap the alias in quotes: e.g. '"P95 Latency" DESC'.`, + ), asRatio: z.boolean().optional(), groupByColumnsOnLeft: z .boolean() diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts index a2a85f3d0f..daf0c4e735 100644 --- a/packages/api/src/mcp/tools/query/eventPatterns.ts +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -73,7 +73,11 @@ export function registerEventPatterns(server: McpServer, context: McpContext) { 'Discover the most common log messages and event patterns. ' + 'Samples random events, clusters them using the Drain algorithm, and returns ' + 'patterns sorted by frequency with estimated counts and time trends.\n\n' + - 'Use this when asked about "top patterns", "common logs", "noisy services", ' + + 'PREFER THIS TOOL over clickstack_search or clickstack_table when the goal is to ' + + 'understand what kinds of messages, errors, or events exist — e.g. "sample logs", ' + + '"what errors are happening", "show me common messages", "what does this service log". ' + + 'It returns frequency-ranked patterns instead of raw rows, giving a much better overview.\n\n' + + 'Also use when asked about "top patterns", "common logs", "noisy services", ' + '"recurring messages", or log noise analysis.\n\n' + 'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' + 'a follow-up clickstack_search call to browse matching raw events.\n\n' + diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 6bcdd91ab4..8777e25ee8 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -175,5 +175,7 @@ export const orderBySchema = z .string() .optional() .describe( - 'Column to sort results by (builder display types only, mainly "table").', + 'Sort results by this column. ' + + 'When ordering by an alias that contains spaces or special characters, ' + + `wrap the alias in quotes: e.g. '"P95 Latency" DESC'.`, ); diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index 41b7e44f0b..701ea9414b 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -66,6 +66,37 @@ const AGG_FN_NAMES: ReadonlySet = new Set( MCP_AGG_FN_OPTIONS.filter(fn => fn !== 'none'), ); +/** A bare ClickHouse identifier: a letter or underscore followed by word chars. + * Anything outside this shape (spaces, punctuation, leading digit) must be + * quoted to be valid in ORDER BY. */ +const BARE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** Strip one layer of surrounding double-quote or backtick quoting, if present. */ +function stripIdentifierQuotes(value: string): string { + const trimmed = value.trim(); + if ( + trimmed.length >= 2 && + ((trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith('`') && trimmed.endsWith('`'))) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +/** + * Quote an alias for use in ORDER BY when it is not a bare identifier. + * Aliases are emitted into SELECT as `AS "alias"`, so we mirror that with + * double-quote identifier quoting (escaping embedded double quotes). Bare + * identifiers are returned unquoted to keep the common case readable. + */ +function quoteAliasForOrderBy(alias: string): string { + if (BARE_IDENTIFIER.test(alias)) { + return alias; + } + return `"${alias.replace(/"/g, '""')}"`; +} + /** * Resolve an orderBy value that matches a bare aggregation function name * (e.g. "count") to something ClickHouse can resolve in ORDER BY. @@ -77,9 +108,11 @@ const AGG_FN_NAMES: ReadonlySet = new Set( * the ClickHouse expression (e.g. `count()`) when no alias is set. * * Resolution order: - * 1. If orderBy matches a select item's alias exactly → keep as-is - * 2. If orderBy matches an aggFn name → use that item's alias if set, - * otherwise synthesize the ClickHouse expression (e.g. `count()`) + * 1. If orderBy matches a select item's alias → emit the canonical alias, + * quoting it when it is not a bare identifier (e.g. `"P95 Latency"`) + * 2. If orderBy matches an aggFn name → use that item's alias if set + * (quoted as needed), otherwise synthesize the ClickHouse expression + * (e.g. `count()`) * 3. Otherwise → pass through unchanged */ /** @internal Exported for testing only. */ @@ -100,15 +133,20 @@ export function resolveOrderBy( const identifier = dirMatch ? dirMatch[1] : orderBy; const direction = dirMatch ? ` ${dirMatch[2].toUpperCase()}` : ''; - const lower = identifier.toLowerCase(); + // Agents may pass the alias already quoted (per the orderBy docs) or bare. + // Strip any surrounding quotes before matching so both forms resolve, then + // re-quote on output based on the canonical alias. + const lower = stripIdentifierQuotes(identifier).toLowerCase(); // Already matches an alias? Return the canonical alias case so // ClickHouse's case-sensitive identifier resolution works correctly. + // Multi-word / special-character aliases are quoted so the emitted + // ORDER BY is valid SQL even when the agent passed the alias unquoted. const aliasMatch = selectItems.find( s => s.alias && s.alias.toLowerCase() === lower, ); - if (aliasMatch) { - return `${aliasMatch.alias}${direction}`; + if (aliasMatch && aliasMatch.alias) { + return `${quoteAliasForOrderBy(aliasMatch.alias)}${direction}`; } // Matches an aggFn name? Resolve to that item's alias or synthesize. @@ -117,7 +155,7 @@ export function resolveOrderBy( if (match) { // Prefer the explicit alias if set if (match.alias) { - return `${match.alias}${direction}`; + return `${quoteAliasForOrderBy(match.alias)}${direction}`; } // Synthesize the ClickHouse expression so ORDER BY works From 7152d2b62eb20d320308c3f9dc41ee4449c4017b Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 11 Jun 2026 19:14:40 -0400 Subject: [PATCH 050/728] feat: use optimistic updates for favorite toggling (#2443) ## Summary When toggling a saved search or dashboard's favorited status, there was a noticeable delay between clicking the star button and seeing the new favorite state. This was because the mutations waited for the network round-trip to complete before invalidating and refetching the favorites query. This PR adds React Query optimistic updates to the `useAddFavorite` and `useRemoveFavorite` mutation hooks so the UI updates instantly ### Screenshots or video Before (with artificial 1s endpoint delay locally): https://github.com/user-attachments/assets/fce0c94f-b3db-4805-bdc0-573b9f230299 After (with artificial 1s endpoint delay locally): https://github.com/user-attachments/assets/bb1db54d-db34-4f7c-a841-3d412ff18e12 ### How to test on Vercel preview To really illustrate the difference, run locally with the following added to the favorites route to create an exxagerated network delay: ``` await new Promise(resolve => setTimeout(resolve, 1000)); ``` ### References - Linear Issue: Closes HDX-4528 --- .changeset/eight-phones-protect.md | 5 +++ packages/app/src/favorites.ts | 66 ++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 .changeset/eight-phones-protect.md diff --git a/.changeset/eight-phones-protect.md b/.changeset/eight-phones-protect.md new file mode 100644 index 0000000000..25334b05f4 --- /dev/null +++ b/.changeset/eight-phones-protect.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: Use optimistic updates for favorites diff --git a/packages/app/src/favorites.ts b/packages/app/src/favorites.ts index e29af18176..79bfcd2a72 100644 --- a/packages/app/src/favorites.ts +++ b/packages/app/src/favorites.ts @@ -23,6 +23,7 @@ export function useFavorites() { return useQuery({ queryKey: ['favorites'], queryFn: fetchFavorites, + staleTime: 5_000, }); } @@ -30,6 +31,8 @@ function useAddFavorite() { const queryClient = useQueryClient(); return useMutation({ + // shared key with useRemoveFavorite to coordinate refetching and optimistic updates + mutationKey: ['favorites'], mutationFn: (data: { resourceType: Favorite['resourceType']; resourceId: string; @@ -42,8 +45,32 @@ function useAddFavorite() { json: data, }).json(); }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['favorites'] }); + onMutate: async data => { + // Cancel any outgoing favorites refetches (so they don't overwrite our optimistic update) + await queryClient.cancelQueries({ queryKey: ['favorites'] }); + + // Get the previous value so that we can roll back if the mutation fails + const previous = queryClient.getQueryData(['favorites']); + + // Optimistically update to the new value + queryClient.setQueryData(['favorites'], old => [ + ...(old ?? []), + { id: `optimistic-${Math.random().toString(36).slice(2)}`, ...data }, + ]); + return { previous }; + }, + onError: (_err, _data, context) => { + if (context !== undefined) { + queryClient.setQueryData(['favorites'], context.previous); + } + }, + onSettled: () => { + // Only refetch once the last in-flight favorites mutation settles, so a + // refetch with partially-committed server state can't clobber the + // still-pending optimistic updates from other mutations. + if (queryClient.isMutating({ mutationKey: ['favorites'] }) === 1) { + queryClient.invalidateQueries({ queryKey: ['favorites'] }); + } }, }); } @@ -52,6 +79,8 @@ function useRemoveFavorite() { const queryClient = useQueryClient(); return useMutation({ + // shared key with useRemoveFavorite to coordinate refetching and optimistic updates + mutationKey: ['favorites'], mutationFn: (data: { resourceType: Favorite['resourceType']; resourceId: string; @@ -72,8 +101,37 @@ function useRemoveFavorite() { method: 'DELETE', }).json(); }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['favorites'] }); + onMutate: async data => { + // Cancel any outgoing favorites refetches (so they don't overwrite our optimistic update) + await queryClient.cancelQueries({ queryKey: ['favorites'] }); + + // Get the previous value so that we can roll back if the mutation fails + const previous = queryClient.getQueryData(['favorites']); + + // Optimistically update to the new value + queryClient.setQueryData(['favorites'], old => + (old ?? []).filter( + f => + !( + f.resourceType === data.resourceType && + f.resourceId === data.resourceId + ), + ), + ); + return { previous }; + }, + onError: (_err, _data, context) => { + if (context !== undefined) { + queryClient.setQueryData(['favorites'], context.previous); + } + }, + onSettled: () => { + // Only refetch once the last in-flight favorites mutation settles, so a + // refetch with partially-committed server state can't clobber the + // still-pending optimistic updates from other mutations. + if (queryClient.isMutating({ mutationKey: ['favorites'] }) === 1) { + queryClient.invalidateQueries({ queryKey: ['favorites'] }); + } }, }); } From b6a4b3b30a49d512dc023ce494c961e70b610363 Mon Sep 17 00:00:00 2001 From: Mike Shi Date: Thu, 11 Jun 2026 21:47:29 -0700 Subject: [PATCH 051/728] feat: lazy-load dashboard tiles based on viewport visibility (#2454) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Dashboard tiles previously issued their ClickHouse queries on mount, regardless of whether they were visible in the viewport. On a dashboard with many tiles, every query fired at once on load — slow and wasteful, since most tiles are below the fold. ## What Gate each tile's data fetching on viewport visibility, modeled on how v1 (`hyperdx-v1-ee`) does it (`useInViewport` + debounce feeding the React Query `enabled` flag). All changes are in `packages/app/src/DBDashboardPage.tsx`: - Added Mantine's `useInViewport()` + `useDebouncedValue(inViewport, 200)` to the `Tile` component. The 200ms debounce handles react-grid-layout briefly mounting every tile before the layout settles. - Visibility is **sticky**: a tile loads the first time it scrolls into the browser window and keeps its data. This is deliberate — the app's global `QueryClient` sets no `staleTime`, so a purely viewport-driven flag would refetch every time a tile scrolled back into view. The latch uses the React-recommended "adjust state during render" pattern (avoids both a cascading-render effect and the repo's no-ref-access-during-render lint rule). - Threaded `enabled={chartEnabled}` into every chart type (`DBTimeChart`, `DBTableChart`, `DBNumberChart`, `DBPieChart`, `DBHeatmapChart` via `HeatmapTile`, `DBSqlRowTableWithSideBar`). They already accepted an `enabled` prop gating `useQueriedChartConfig`. - The fullscreen view always loads (`enabled = true`), since it's always visible. ## Net effect On dashboard load, off-screen tiles no longer issue their ClickHouse queries — only tiles visible in the browser window fetch data. ## Testing - `tsc --noEmit`: clean on the file. - `eslint`: 0 errors (only pre-existing warnings). - `yarn ci:unit` (dashboard suites): **252 passed**. ## Note for reviewers Sticky visibility was chosen over true lazy *unloading* because of the missing `staleTime`. If unload-on-scroll-away is preferred, it's a one-line change (drop the sticky latch). 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .changeset/lazy-load-dashboard-tiles.md | 7 +++++ AGENTS.md | 8 ++++++ packages/app/src/DBDashboardPage.tsx | 38 +++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 .changeset/lazy-load-dashboard-tiles.md diff --git a/.changeset/lazy-load-dashboard-tiles.md b/.changeset/lazy-load-dashboard-tiles.md new file mode 100644 index 0000000000..c1153e9058 --- /dev/null +++ b/.changeset/lazy-load-dashboard-tiles.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/app': minor +--- + +feat: lazy-load dashboard tiles based on viewport visibility + +Dashboard tiles now only run their ClickHouse queries once they scroll into the browser viewport, instead of every tile querying on page load. A tile loads the first time it becomes visible and keeps its data afterward. This significantly reduces the number of queries fired when opening dashboards with many tiles. diff --git a/AGENTS.md b/AGENTS.md index 8cb3ebe993..2fb675c595 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -185,6 +185,14 @@ efficient and accurate: your agent to produce tests before writing implementation code. See the Testing section below for the commands to use. +5. **Ensure a changeset exists before pushing a PR.** Any change to a published + package (`@hyperdx/app`, `@hyperdx/api`, `@hyperdx/otel-collector`, etc.) that + is user-facing or affects behavior must include a changeset in `.changeset/`. + Add one with `yarn changeset` (or create the markdown file by hand following + the format of existing entries), choosing the appropriate semver bump, before + pushing the branch. Skip only for changes that don't warrant a release (docs, + internal tooling, tests, CI). + ## GitHub Action Workflow (when invoked via @claude) When working on issues or PRs through the GitHub Action: diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index b92504fa9d..dbae6dcc13 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -69,7 +69,7 @@ import { Text, Tooltip, } from '@mantine/core'; -import { useHotkeys } from '@mantine/hooks'; +import { useDebouncedValue, useHotkeys, useInViewport } from '@mantine/hooks'; import { notifications } from '@mantine/notifications'; import { IconAlertTriangle, @@ -191,6 +191,7 @@ function HeatmapTile({ queriedConfig, source, dateRange, + enabled = true, }: { keyPrefix: string; chartId: string; @@ -199,6 +200,7 @@ function HeatmapTile({ queriedConfig: BuilderChartConfigWithDateRange; source: TSource | undefined; dateRange: [Date, Date]; + enabled?: boolean; }) { const { heatmapConfig, scaleType } = toHeatmapChartConfig(queriedConfig); @@ -243,6 +245,7 @@ function HeatmapTile({ toolbarPrefix={toolbar} config={heatmapConfig} scaleType={scaleType} + enabled={enabled} showLegend /> {clickPos != null && eventDeltasUrl != null && ( @@ -395,6 +398,26 @@ const Tile = forwardRef( const [isFullscreen, setIsFullscreen] = useState(false); const [isFocused, setIsFocused] = useState(false); + // Lazy loading: only fetch a tile's data once it has scrolled into the + // browser viewport. React Grid Layout mounts every tile up front, so + // without this gating each tile would issue its ClickHouse query + // immediately, regardless of whether it is visible. We debounce the + // viewport signal (RGL briefly renders all tiles before the layout + // settles) and make visibility "sticky" so that a tile keeps its data + // once loaded instead of refetching every time it scrolls back into view. + const { ref: inViewportRef, inViewport } = useInViewport(); + const [debouncedInViewport] = useDebouncedValue(inViewport, 200); + // Latch to true the first time the tile becomes visible and never flip + // back, so a loaded tile keeps its data instead of refetching every time + // it scrolls out of and back into view. Adjusting state during render (the + // React-recommended pattern for deriving state from changing inputs) is + // cheaper than an effect and only fires once, since the condition is false + // after the first visible render. + const [hasBeenVisible, setHasBeenVisible] = useState(false); + if (debouncedInViewport && !hasBeenVisible) { + setHasBeenVisible(true); + } + const { userPreferences: { isUTC }, } = useUserPreferences(); @@ -804,6 +827,10 @@ const Tile = forwardRef( : [hoverToolbar, filterWarning]; const keyPrefix = isFullscreenView ? 'fullscreen' : 'tile'; + // The fullscreen view is always visible, so it should always load. + // In the tile (grid) view, gate data fetching on viewport visibility. + const chartEnabled = isFullscreenView ? true : hasBeenVisible; + // Use the fullscreen-local date range and granularity when rendering // inside the fullscreen modal so that changing them does not affect // the dashboard. @@ -862,6 +889,7 @@ const Tile = forwardRef( toolbarPrefix={toolbar} sourceId={chart.config.source} showDisplaySwitcher={true} + enabled={chartEnabled} config={effectiveQueriedConfig} onTimeRangeSelect={ isFullscreenView @@ -885,6 +913,7 @@ const Tile = forwardRef( key={`${keyPrefix}-${chart.id}`} title={title} toolbarPrefix={toolbar} + enabled={chartEnabled} config={effectiveQueriedConfig} variant="muted" getRowSearchLink={ @@ -906,6 +935,7 @@ const Tile = forwardRef( key={`${keyPrefix}-${chart.id}`} title={title} toolbarPrefix={toolbar} + enabled={chartEnabled} config={effectiveQueriedConfig} /> )} @@ -914,6 +944,7 @@ const Tile = forwardRef( key={`${keyPrefix}-${chart.id}`} title={title} toolbarPrefix={toolbar} + enabled={chartEnabled} config={effectiveQueriedConfig} /> )} @@ -924,6 +955,7 @@ const Tile = forwardRef( chartId={chart.id} title={title} toolbar={toolbar} + enabled={chartEnabled} queriedConfig={effectiveQueriedConfig} source={source} dateRange={effectiveDateRange} @@ -949,7 +981,7 @@ const Tile = forwardRef( > )}
e.stopPropagation()} > From e49228046a0588776319c914ebc5f174d24f60dc Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Fri, 12 Jun 2026 10:50:46 +0200 Subject: [PATCH 052/728] feat: add source field suggestions (#2436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary **Why:** Configuring a source against a custom (non-OTel) table means typing column names from memory into free-text inputs, with no schema inspection, no validation, and no warning when a config will silently break search (e.g. `bodyExpression` set but `implicitColumnExpression` empty — the HDX-4376 trap). This adds the form-side prevention to pair with HDX-4376 (runtime fallback) and HDX-4373 (row-panel cleanup). **What changed:** - **Candidate hints** — after Database + Table are picked, scans columns (cached via `useColumns`) and suggests the likely column under each empty optional-expression field, with one-click apply. Type-aware, name-matched, resolves per source kind (logs vs traces). Suggests, never pre-fills; lists all candidates when ambiguous. - **Live validation** — filled expressions are validated against the real table via a debounced `EXPLAIN` query, showing valid/invalid inline. - **Pairing warnings** — `getSourceConfigPairingWarnings` flags body/implicit-column mismatches (log sources only) in a non-blocking save-time modal with one-click fixes or "Save anyway". ### Screenshots or video https://github.com/user-attachments/assets/d862f021-c81e-4c4f-b3ba-3c6d9872db95 ### How to test on Vercel preview **Preview routes:** `/team` **Steps:** 1. Create a new source, or edit an existing one and check the suggestions for fields are correct. 2. Check the warning modal displays in one of two ways when _Body_ or _Implicit Column Expression_ fields are unset. ### References - Linear Issue: Closes HDX-4374 - Related PRs: https://github.com/hyperdxio/hyperdx/pull/2354 --- .changeset/polite-moons-give.md | 5 + .../Sources/ExpressionValidationStatus.tsx | 45 ++ .../Sources/SourceFieldCandidateHint.tsx | 61 ++ .../app/src/components/Sources/SourceForm.tsx | 723 ++++++++++-------- .../app/src/hooks/useExpressionValidation.ts | 80 ++ .../__tests__/sourceFieldSuggestions.test.ts | 249 ++++++ .../app/src/utils/sourceFieldSuggestions.ts | 249 ++++++ 7 files changed, 1114 insertions(+), 298 deletions(-) create mode 100644 .changeset/polite-moons-give.md create mode 100644 packages/app/src/components/Sources/ExpressionValidationStatus.tsx create mode 100644 packages/app/src/components/Sources/SourceFieldCandidateHint.tsx create mode 100644 packages/app/src/hooks/useExpressionValidation.ts create mode 100644 packages/app/src/utils/__tests__/sourceFieldSuggestions.test.ts create mode 100644 packages/app/src/utils/sourceFieldSuggestions.ts diff --git a/.changeset/polite-moons-give.md b/.changeset/polite-moons-give.md new file mode 100644 index 0000000000..8ee1597d14 --- /dev/null +++ b/.changeset/polite-moons-give.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: add source field suggestions diff --git a/packages/app/src/components/Sources/ExpressionValidationStatus.tsx b/packages/app/src/components/Sources/ExpressionValidationStatus.tsx new file mode 100644 index 0000000000..2e129a1003 --- /dev/null +++ b/packages/app/src/components/Sources/ExpressionValidationStatus.tsx @@ -0,0 +1,45 @@ +import { Box, MantineSpacing, Text } from '@mantine/core'; + +import { ErrorCollapse } from '@/components/Error/ErrorCollapse'; +import { + TableConnectionLike, + useExpressionValidation, +} from '@/hooks/useExpressionValidation'; + +export function ExpressionValidationStatus({ + expression, + tableConnection, + mt = 'xs', +}: { + expression: string; + tableConnection: TableConnectionLike; + mt?: MantineSpacing; +}) { + const { shouldShowResult, isInvalid, isValid, error } = + useExpressionValidation({ expression, tableConnection }); + + if (!shouldShowResult) { + return null; + } + + if (isInvalid) { + return ( + + + + ); + } + + if (isValid) { + return ( + + Expression is valid. + + ); + } + + return null; +} diff --git a/packages/app/src/components/Sources/SourceFieldCandidateHint.tsx b/packages/app/src/components/Sources/SourceFieldCandidateHint.tsx new file mode 100644 index 0000000000..0d5e7a8d7b --- /dev/null +++ b/packages/app/src/components/Sources/SourceFieldCandidateHint.tsx @@ -0,0 +1,61 @@ +import { Anchor, Code, Group, Text } from '@mantine/core'; +import { IconBulb } from '@tabler/icons-react'; + +import { FieldCandidates } from '@/utils/sourceFieldSuggestions'; + +export function SourceFieldCandidateHint({ + candidates, + onApply, +}: { + candidates?: FieldCandidates; + onApply: (value: string) => void; +}) { + if (!candidates) { + return null; + } + + const { canonical, alternates } = candidates; + + if (!canonical && alternates.length === 0) { + return null; + } + + return ( + + + {canonical ? ( + <> + + Detected + + onApply(canonical)}> + {canonical} — apply + + + ) : ( + + Multiple candidates: + + )} + {alternates.length > 0 && ( + <> + {canonical && ( + + Other candidates: + + )} + {alternates.map(name => ( + onApply(name)} + > + {name} + + ))} + + )} + + ); +} diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 6c4043afb1..fe21d2ad87 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -2,6 +2,7 @@ import React, { Fragment, useCallback, useEffect, + useMemo, useRef, useState, } from 'react'; @@ -14,7 +15,10 @@ import { useWatch, } from 'react-hook-form'; import { z } from 'zod'; -import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; +import { + ClickHouseQueryError, + ColumnMetaType, +} from '@hyperdx/common-utils/dist/clickhouse'; import { MetricsDataType, SourceKind, @@ -33,6 +37,7 @@ import { Flex, Grid, Group, + Modal, Radio, Select, Slider, @@ -61,7 +66,8 @@ import { } from '@/config'; import { useConnections } from '@/connection'; import { useExplainQuery } from '@/hooks/useExplainQuery'; -import { useMetadataWithSettings } from '@/hooks/useMetadata'; +import { useExpressionValidation } from '@/hooks/useExpressionValidation'; +import { useColumns, useMetadataWithSettings } from '@/hooks/useMetadata'; import { inferTableSourceConfig, isValidMetricTable, @@ -77,6 +83,12 @@ import { MV_AGGREGATE_FUNCTIONS, MV_GRANULARITY_OPTIONS, } from '@/utils/materializedViews'; +import { + getSourceConfigPairingWarnings, + inferSourceFieldCandidates, + PairingWarning, + SourceFieldKind, +} from '@/utils/sourceFieldSuggestions'; import ConfirmDeleteMenu from '../ConfirmDeleteMenu'; import { ConnectionSelectControlled } from '../ConnectionSelect'; @@ -86,6 +98,9 @@ import { ErrorCollapse } from '../Error/ErrorCollapse'; import { InputControlled } from '../InputControlled'; import SelectControlled from '../SelectControlled'; +import { ExpressionValidationStatus } from './ExpressionValidationStatus'; +import { SourceFieldCandidateHint } from './SourceFieldCandidateHint'; + type CorrelationField = | 'logSourceId' | 'traceSourceId' @@ -169,6 +184,17 @@ const OTEL_CLICKHOUSE_EXPRESSIONS = { resourceAttributesExpression: 'ResourceAttributes', }; +const SOURCE_FIELD_LABELS: Record = { + bodyExpression: 'Body Expression', + implicitColumnExpression: 'Implicit Column Expression', + serviceNameExpression: 'Service Name Expression', + severityTextExpression: 'Log Level Expression', + eventAttributesExpression: 'Event Attributes Expression', + resourceAttributesExpression: 'Resource Attributes Expression', + traceIdExpression: 'Trace Id Expression', + spanIdExpression: 'Span Id Expression', +}; + const CORRELATION_FIELD_MAP: Record< SourceKind, Partial< @@ -261,6 +287,67 @@ function FormRow({ ); } +function ExpressionFormRow({ + control, + setValue, + name, + label, + placeholder, + helpText, + columns, + sourceKind, + tableConnection, +}: { + control: Control; + setValue: UseFormSetValue; + name: SourceFieldKind; + label: string; + placeholder?: string; + helpText?: string; + columns?: ColumnMetaType[]; + sourceKind: SourceKind; + tableConnection: { + databaseName: string; + tableName: string; + connectionId: string; + }; +}) { + const currentValue = useWatch({ control, name }); + const value = typeof currentValue === 'string' ? currentValue : ''; + + const candidates = useMemo( + () => + columns + ? inferSourceFieldCandidates(columns, name, sourceKind) + : undefined, + [columns, name, sourceKind], + ); + + return ( + + + {value.trim() ? ( + + ) : ( + { + setValue(name, applied, { shouldDirty: true }); + }} + /> + )} + + ); +} + type HighlightedAttributeRowProps = Omit & { id: string; index: number; @@ -293,61 +380,18 @@ function HighlightedAttributeRow({ name: `${name}.${index}.alias`, }); - const [explainParams, setExplainParams] = useState<{ - expression: typeof expressionInput; - alias: typeof aliasInput; - }>(); - - const setExplainParamsDebounced = useDebouncedCallback( - (params: typeof explainParams) => { - setExplainParams(params); - }, - 1_000, - ); - - useDidUpdate(() => { - setExplainParamsDebounced({ - expression: expressionInput, - alias: aliasInput, - }); - }, [expressionInput, aliasInput]); - const { - data: explainData, - error: explainError, - isLoading: explainLoading, - } = useExplainQuery( - { - from: { databaseName, tableName }, - connection: connectionId, - select: [ - { - alias: explainParams?.alias, - valueExpression: explainParams?.expression ?? '', - }, - ], - where: '', - }, - - { - enabled: !!explainParams?.expression, - }, - ); - - const runExpression = () => { - setExplainParams({ - expression: expressionInput, - alias: aliasInput, - }); - }; - - const isExpressionValid = !!explainData?.length; - const isExpressionInvalid = explainError instanceof ClickHouseQueryError; - - const shouldShowResult = - explainParams?.expression === expressionInput && - explainParams?.alias === aliasInput && - (isExpressionValid || isExpressionInvalid); + isLoading: isExplainLoading, + validateNow, + shouldShowResult, + isValid, + isInvalid, + error, + } = useExpressionValidation({ + expression: expressionInput, + alias: aliasInput, + tableConnection: { databaseName, tableName, connectionId }, + }); return ( @@ -383,9 +427,9 @@ function HighlightedAttributeRow({ size="xs" variant="subtle" color="gray" - loading={explainLoading} - disabled={!expressionInput || explainLoading} - onClick={runExpression} + loading={isExplainLoading} + disabled={!expressionInput || isExplainLoading} + onClick={validateNow} > @@ -403,15 +447,15 @@ function HighlightedAttributeRow({ {shouldShowResult && ( - {isExpressionValid && ( + {isValid && ( Expression is valid. )} - {isExpressionInvalid && ( + {isInvalid && ( )} @@ -1207,7 +1251,7 @@ function UseTextIndexFormRow({ control }: { control: Control }) { } function LogTableModelForm(props: TableModelProps) { - const { control } = props; + const { control, setValue } = props; const brandName = useBrandDisplayName(); const databaseName = useWatch({ control, @@ -1217,6 +1261,13 @@ function LogTableModelForm(props: TableModelProps) { const tableName = useWatch({ control, name: 'from.tableName' }); const connectionId = useWatch({ control, name: 'connection' }); + const tableConnection = { databaseName, tableName, connectionId }; + const { data: columns } = useColumns({ + databaseName, + tableName, + connectionId, + }); + const [showOptionalFields, setShowOptionalFields] = useState(false); return ( @@ -1283,66 +1334,56 @@ function LogTableModelForm(props: TableModelProps) { }} > - - - - - - - - - - - - - - - + + + + + - - - - - - + + {/* @@ -1410,21 +1447,17 @@ function LogTableModelForm(props: TableModelProps) { placeholder="ServiceName = 'only_this_service'" /> */} - - - + placeholder="Body" + columns={columns} + sourceKind={SourceKind.Log} + tableConnection={tableConnection} + /> - - - - - - + + - - - - - - - - - + + + - - - + placeholder="SpanName" + columns={columns} + sourceKind={SourceKind.Trace} + tableConnection={tableConnection} + /> void; + }>(); + + const applyPairingFix = useCallback( + (warning: PairingWarning) => { + if (!pendingSave || !warning.suggestedFix) { + return; + } + + const { field, value } = warning.suggestedFix; + + setValue(field, value, { shouldDirty: true }); + + const remaining = pendingSave.warnings.filter(w => w !== warning); + + const patched = { ...pendingSave.parsedData, [field]: value }; + + if (remaining.length === 0) { + setPendingSave(undefined); + pendingSave.persist(patched); + } else { + setPendingSave({ + ...pendingSave, + warnings: remaining, + parsedData: patched, + }); + } + }, + [pendingSave, setValue], + ); + const _onCreate = useCallback(() => { clearErrors(); handleSubmit(async data => { @@ -2305,58 +2365,68 @@ export function TableSourceForm({ return; } - createSource.mutate( - { source: parseResult.data }, - { - onSuccess: async newSource => { - // Handle bidirectional linking for new sources - const correlationFields = CORRELATION_FIELD_MAP[newSource.kind]; - if (correlationFields && sources) { - for (const [fieldName, targetConfigs] of Object.entries( - correlationFields, - )) { - const targetSourceId = getCorrelationFieldValue( - newSource, - fieldName as CorrelationField, - ); - if (targetSourceId) { - for (const { targetKind, targetField } of targetConfigs) { - const targetSource = sources.find( - s => s.id === targetSourceId, - ); - if (targetSource && targetSource.kind === targetKind) { - // Only update if the target field is empty to avoid overwriting existing correlations - if ( - !getCorrelationFieldValue(targetSource, targetField) - ) { - await updateSource.mutateAsync({ - source: setCorrelationFieldValue( - targetSource, - targetField, - newSource.id, - ), - }); + const persist = (source: typeof parseResult.data) => + createSource.mutate( + { source }, + { + onSuccess: async newSource => { + // Handle bidirectional linking for new sources + const correlationFields = CORRELATION_FIELD_MAP[newSource.kind]; + if (correlationFields && sources) { + for (const [fieldName, targetConfigs] of Object.entries( + correlationFields, + )) { + const targetSourceId = getCorrelationFieldValue( + newSource, + fieldName as CorrelationField, + ); + if (targetSourceId) { + for (const { targetKind, targetField } of targetConfigs) { + const targetSource = sources.find( + s => s.id === targetSourceId, + ); + if (targetSource && targetSource.kind === targetKind) { + // Only update if the target field is empty to avoid overwriting existing correlations + if ( + !getCorrelationFieldValue(targetSource, targetField) + ) { + await updateSource.mutateAsync({ + source: setCorrelationFieldValue( + targetSource, + targetField, + newSource.id, + ), + }); + } } } } } } - } - onCreate?.(newSource); - notifications.show({ - color: 'green', - message: 'Source created', - }); - }, - onError: error => { - notifications.show({ - color: 'red', - message: `Failed to create source - ${error.message}`, - }); + onCreate?.(newSource); + notifications.show({ + color: 'green', + message: 'Source created', + }); + }, + onError: error => { + notifications.show({ + color: 'red', + message: `Failed to create source - ${error.message}`, + }); + }, }, - }, - ); + ); + + const warnings = getSourceConfigPairingWarnings(data); + + if (warnings.length > 0) { + setPendingSave({ warnings, parsedData: parseResult.data, persist }); + return; + } + + persist(parseResult.data); })(); }, [ clearErrors, @@ -2376,24 +2446,35 @@ export function TableSourceForm({ handleError(parseResult.error, 'save'); return; } - updateSource.mutate( - { source: parseResult.data }, - { - onSuccess: () => { - onSave?.(); - notifications.show({ - color: 'green', - message: 'Source updated', - }); - }, - onError: () => { - notifications.show({ - color: 'red', - message: 'Failed to update source', - }); + + const persist = (source: typeof parseResult.data) => + updateSource.mutate( + { source }, + { + onSuccess: () => { + onSave?.(); + notifications.show({ + color: 'green', + message: 'Source updated', + }); + }, + onError: () => { + notifications.show({ + color: 'red', + message: 'Failed to update source', + }); + }, }, - }, - ); + ); + + const warnings = getSourceConfigPairingWarnings(data); + + if (warnings.length > 0) { + setPendingSave({ warnings, parsedData: parseResult.data, persist }); + return; + } + + persist(parseResult.data); })(); }, [handleSubmit, updateSource, onSave, clearErrors, handleError]); @@ -2588,6 +2669,52 @@ export function TableSourceForm({ )} + setPendingSave(undefined)} + title="Review source configuration" + centered + > + + {pendingSave?.warnings.map(warning => ( + + {warning.message} + {warning.suggestedFix && ( + + )} + + ))} + + + + + +
); } diff --git a/packages/app/src/hooks/useExpressionValidation.ts b/packages/app/src/hooks/useExpressionValidation.ts new file mode 100644 index 0000000000..dfc6f79487 --- /dev/null +++ b/packages/app/src/hooks/useExpressionValidation.ts @@ -0,0 +1,80 @@ +import { useCallback, useState } from 'react'; +import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; +import { useDebouncedCallback, useDidUpdate } from '@mantine/hooks'; + +import { useExplainQuery } from '@/hooks/useExplainQuery'; + +export type TableConnectionLike = { + databaseName: string; + tableName: string; + connectionId: string; +}; + +export function useExpressionValidation({ + expression, + alias, + tableConnection, + debounceMs = 1000, +}: { + expression: string | undefined; + alias?: string; + tableConnection: TableConnectionLike; + debounceMs?: number; +}) { + const [explainParams, setExplainParams] = useState<{ + expression?: string; + alias?: string; + }>(); + + const setExplainParamsDebounced = useDebouncedCallback( + (params: { expression?: string; alias?: string }) => { + setExplainParams(params); + }, + debounceMs, + ); + + useDidUpdate(() => { + setExplainParamsDebounced({ expression, alias }); + }, [expression, alias]); + + const { databaseName, tableName, connectionId } = tableConnection; + + const { data, error, isLoading } = useExplainQuery( + { + from: { databaseName, tableName }, + connection: connectionId, + select: [ + { + alias: explainParams?.alias, + valueExpression: explainParams?.expression ?? '', + }, + ], + where: '', + }, + { + enabled: !!explainParams?.expression, + }, + ); + + const isValid = !!data?.length; + + const isInvalid = error instanceof ClickHouseQueryError; + + const matchesCurrentInput = + explainParams?.expression === expression && explainParams?.alias === alias; + + const shouldShowResult = matchesCurrentInput && (isValid || isInvalid); + + const validateNow = useCallback(() => { + setExplainParams({ expression, alias }); + }, [expression, alias]); + + return { + isValid, + isInvalid, + isLoading, + error, + shouldShowResult, + validateNow, + }; +} diff --git a/packages/app/src/utils/__tests__/sourceFieldSuggestions.test.ts b/packages/app/src/utils/__tests__/sourceFieldSuggestions.test.ts new file mode 100644 index 0000000000..3a82b604fc --- /dev/null +++ b/packages/app/src/utils/__tests__/sourceFieldSuggestions.test.ts @@ -0,0 +1,249 @@ +import { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { + getSourceConfigPairingWarnings, + inferSourceFieldCandidates, +} from '../sourceFieldSuggestions'; + +const col = (name: string, type: string): ColumnMetaType => ({ name, type }); + +describe('inferSourceFieldCandidates', () => { + it('recommends an exact canonical name match', () => { + const columns = [ + col('Timestamp', 'DateTime64(9)'), + col('Body', 'String'), + col('ServiceName', 'LowCardinality(String)'), + ]; + expect( + inferSourceFieldCandidates(columns, 'bodyExpression', SourceKind.Log), + ).toEqual({ canonical: 'Body', alternates: [] }); + }); + + it('matches canonical names case-insensitively', () => { + const columns = [col('traceid', 'String')]; + expect( + inferSourceFieldCandidates(columns, 'traceIdExpression', SourceKind.Log), + ).toEqual({ canonical: 'traceid', alternates: [] }); + }); + + it('keeps columns that differ only by case rather than dropping one', () => { + const columns = [col('Body', 'String'), col('BODY', 'String')]; + expect( + inferSourceFieldCandidates(columns, 'bodyExpression', SourceKind.Log), + ).toEqual({ canonical: 'Body', alternates: ['BODY'] }); + }); + + it('picks the first column in table order as canonical on a case collision', () => { + const columns = [col('BODY', 'String'), col('Body', 'String')]; + expect( + inferSourceFieldCandidates(columns, 'bodyExpression', SourceKind.Log), + ).toEqual({ canonical: 'BODY', alternates: ['Body'] }); + }); + + it('surfaces every case-colliding Map column as an alternate', () => { + const columns = [ + col('Attributes', 'Map(String, String)'), + col('ATTRIBUTES', 'Map(String, String)'), + ]; + expect( + inferSourceFieldCandidates( + columns, + 'eventAttributesExpression', + SourceKind.Log, + ), + ).toEqual({ canonical: 'Attributes', alternates: ['ATTRIBUTES'] }); + }); + + it('lists other name-matched columns as alternates (not every string column)', () => { + const columns = [ + col('Body', 'String'), + col('message', 'String'), + col('msg', 'String'), + col('some_unrelated_string', 'String'), + ]; + expect( + inferSourceFieldCandidates(columns, 'bodyExpression', SourceKind.Log), + ).toEqual({ canonical: 'Body', alternates: ['message', 'msg'] }); + }); + + it('resolves eventAttributesExpression per source kind', () => { + const columns = [ + col('LogAttributes', 'Map(String, String)'), + col('SpanAttributes', 'Map(LowCardinality(String), String)'), + ]; + expect( + inferSourceFieldCandidates( + columns, + 'eventAttributesExpression', + SourceKind.Log, + ).canonical, + ).toBe('LogAttributes'); + expect( + inferSourceFieldCandidates( + columns, + 'eventAttributesExpression', + SourceKind.Trace, + ).canonical, + ).toBe('SpanAttributes'); + }); + + it('recommends none but lists all when multiple unnamed Map columns exist', () => { + const columns = [ + col('vendor_a', 'Map(String, String)'), + col('vendor_b', 'Map(String, String)'), + ]; + expect( + inferSourceFieldCandidates( + columns, + 'resourceAttributesExpression', + SourceKind.Log, + ), + ).toEqual({ canonical: undefined, alternates: ['vendor_a', 'vendor_b'] }); + }); + + it('recommends a lone Map column for an attributes field', () => { + const columns = [col('attrs', 'Map(String, String)')]; + expect( + inferSourceFieldCandidates( + columns, + 'resourceAttributesExpression', + SourceKind.Log, + ), + ).toEqual({ canonical: 'attrs', alternates: [] }); + }); + + it('matches TraceId on UUID / FixedString(16) but not FixedString(8)', () => { + expect( + inferSourceFieldCandidates( + [col('TraceId', 'UUID')], + 'traceIdExpression', + SourceKind.Log, + ).canonical, + ).toBe('TraceId'); + expect( + inferSourceFieldCandidates( + [col('TraceId', 'FixedString(16)')], + 'traceIdExpression', + SourceKind.Log, + ).canonical, + ).toBe('TraceId'); + expect( + inferSourceFieldCandidates( + [col('TraceId', 'FixedString(8)')], + 'traceIdExpression', + SourceKind.Log, + ), + ).toEqual({ canonical: undefined, alternates: [] }); + }); + + it('matches SpanId on FixedString(8)', () => { + expect( + inferSourceFieldCandidates( + [col('SpanId', 'FixedString(8)')], + 'spanIdExpression', + SourceKind.Log, + ).canonical, + ).toBe('SpanId'); + }); + + it('treats LowCardinality(String) and Nullable(String) as stringy', () => { + expect( + inferSourceFieldCandidates( + [col('Body', 'LowCardinality(String)')], + 'bodyExpression', + SourceKind.Log, + ).canonical, + ).toBe('Body'); + expect( + inferSourceFieldCandidates( + [col('Body', 'Nullable(String)')], + 'bodyExpression', + SourceKind.Log, + ).canonical, + ).toBe('Body'); + }); + + it('excludes type-incompatible columns', () => { + expect( + inferSourceFieldCandidates( + [col('Body', 'Int64')], + 'bodyExpression', + SourceKind.Log, + ), + ).toEqual({ canonical: undefined, alternates: [] }); + }); +}); + +describe('getSourceConfigPairingWarnings', () => { + it('warns when Body is set but Implicit Column is empty, suggesting Body as the fix', () => { + const warnings = getSourceConfigPairingWarnings({ + kind: SourceKind.Log, + bodyExpression: 'Body', + implicitColumnExpression: '', + }); + expect(warnings).toHaveLength(1); + expect(warnings[0].field).toBe('implicitColumnExpression'); + expect(warnings[0].suggestedFix).toEqual({ + field: 'implicitColumnExpression', + value: 'Body', + }); + }); + + it('warns when Implicit Column is set but Body is empty, suggesting Implicit as the fix', () => { + const warnings = getSourceConfigPairingWarnings({ + kind: SourceKind.Log, + bodyExpression: '', + implicitColumnExpression: 'Body', + }); + expect(warnings).toHaveLength(1); + expect(warnings[0].field).toBe('bodyExpression'); + expect(warnings[0].suggestedFix).toEqual({ + field: 'bodyExpression', + value: 'Body', + }); + }); + + it('does not warn when both are set', () => { + expect( + getSourceConfigPairingWarnings({ + kind: SourceKind.Log, + bodyExpression: 'Body', + implicitColumnExpression: 'Body', + }), + ).toEqual([]); + }); + + it('does not warn when both are empty', () => { + expect( + getSourceConfigPairingWarnings({ + kind: SourceKind.Log, + bodyExpression: '', + implicitColumnExpression: '', + }), + ).toEqual([]); + expect(getSourceConfigPairingWarnings({ kind: SourceKind.Log })).toEqual( + [], + ); + }); + + it('treats whitespace-only values as empty', () => { + const warnings = getSourceConfigPairingWarnings({ + kind: SourceKind.Log, + bodyExpression: ' ', + implicitColumnExpression: 'Body', + }); + expect(warnings).toHaveLength(1); + expect(warnings[0].field).toBe('bodyExpression'); + }); + + it('returns no warnings for non-log sources even when only one field is set', () => { + const fields = { bodyExpression: 'Body', implicitColumnExpression: '' }; + expect( + getSourceConfigPairingWarnings({ kind: SourceKind.Trace, ...fields }), + ).toEqual([]); + expect( + getSourceConfigPairingWarnings({ kind: SourceKind.Metric, ...fields }), + ).toEqual([]); + }); +}); diff --git a/packages/app/src/utils/sourceFieldSuggestions.ts b/packages/app/src/utils/sourceFieldSuggestions.ts new file mode 100644 index 0000000000..3e690fb027 --- /dev/null +++ b/packages/app/src/utils/sourceFieldSuggestions.ts @@ -0,0 +1,249 @@ +import { + ColumnMetaType, + convertCHDataTypeToJSType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +export type SourceFieldKind = + | 'bodyExpression' + | 'implicitColumnExpression' + | 'serviceNameExpression' + | 'severityTextExpression' + | 'eventAttributesExpression' + | 'resourceAttributesExpression' + | 'traceIdExpression' + | 'spanIdExpression'; + +export type FieldCandidates = { + /** The single column we recommend, when there is a clear winner. */ + canonical?: string; + /** Other type-compatible, name-matched columns. Excludes `canonical`. */ + alternates: string[]; +}; + +type TypePredicate = (chType: string) => boolean; + +/** Peel LowCardinality(...) / Nullable(...) wrappers off a CH type string. */ +function unwrap(chType: string): string { + let t = chType.trim(); + + while (true) { + if (t.startsWith('LowCardinality(') && t.endsWith(')')) { + t = t.slice('LowCardinality('.length, -1).trim(); + } else if (t.startsWith('Nullable(') && t.endsWith(')')) { + t = t.slice('Nullable('.length, -1).trim(); + } else { + break; + } + } + return t; +} + +/** String, LowCardinality(String), Nullable(String), FixedString, Enum, UUID. */ +const isStringy: TypePredicate = t => + convertCHDataTypeToJSType(t) === JSDataType.String; + +/** Map(String, *) or JSON. */ +const isMapLike: TypePredicate = t => { + const js = convertCHDataTypeToJSType(t); + return js === JSDataType.Map || js === JSDataType.JSON; +}; + +/** TraceId is 16 bytes: String / UUID / FixedString(16). */ +const isTraceIdType: TypePredicate = t => { + const u = unwrap(t); + return u === 'String' || u === 'UUID' || u === 'FixedString(16)'; +}; + +/** SpanId is 8 bytes: String / FixedString(8). */ +const isSpanIdType: TypePredicate = t => { + const u = unwrap(t); + return u === 'String' || u === 'FixedString(8)'; +}; + +type FieldRule = { + /** Preferred names, highest-rank first. Matched case-insensitively. */ + canonicalNames: string[]; + /** A column must satisfy this to be considered a candidate at all. */ + typeMatches: TypePredicate; + /** + * 'name-only' — only ever recommend a column whose name matches one of + * `canonicalNames` (don't guess a lone String column as e.g. the Body). + * 'single' — additionally recommend the lone type-compatible column when no + * name matched (safe for Map attribute columns). + */ + recommendStrategy: 'name-only' | 'single'; + /** Per-kind overrides of `canonicalNames` (e.g. Log vs Trace attributes). */ + canonicalNamesByKind?: Partial>; +}; + +const FIELD_RULES: Record = { + bodyExpression: { + canonicalNames: ['Body', 'message', 'msg', 'log', 'LogMessage', 'content'], + typeMatches: isStringy, + recommendStrategy: 'name-only', + }, + implicitColumnExpression: { + canonicalNames: ['Body', 'message', 'msg', 'log', 'LogMessage', 'content'], + canonicalNamesByKind: { + [SourceKind.Trace]: ['SpanName', 'OperationName', 'name'], + }, + typeMatches: isStringy, + recommendStrategy: 'name-only', + }, + serviceNameExpression: { + canonicalNames: ['ServiceName', 'service', 'service.name', 'service_name'], + typeMatches: isStringy, + recommendStrategy: 'name-only', + }, + severityTextExpression: { + canonicalNames: [ + 'SeverityText', + 'severity', + 'level', + 'log_level', + 'loglevel', + ], + typeMatches: isStringy, + recommendStrategy: 'name-only', + }, + eventAttributesExpression: { + canonicalNames: ['LogAttributes', 'Attributes', 'attributes', 'tags'], + canonicalNamesByKind: { + [SourceKind.Trace]: [ + 'SpanAttributes', + 'Attributes', + 'attributes', + 'tags', + ], + }, + typeMatches: isMapLike, + recommendStrategy: 'single', + }, + resourceAttributesExpression: { + canonicalNames: [ + 'ResourceAttributes', + 'resource_attributes', + 'resource.attributes', + 'resource', + 'resources', + ], + typeMatches: isMapLike, + recommendStrategy: 'single', + }, + traceIdExpression: { + canonicalNames: ['TraceId', 'trace_id', 'traceId', 'trace.id'], + typeMatches: isTraceIdType, + recommendStrategy: 'name-only', + }, + spanIdExpression: { + canonicalNames: ['SpanId', 'span_id', 'spanId', 'span.id'], + typeMatches: isSpanIdType, + recommendStrategy: 'name-only', + }, +}; + +/** + * Inspect a table's columns and return the recommended column for a given + * source-config field, plus any other type-compatible candidates. + */ +export function inferSourceFieldCandidates( + columns: ColumnMetaType[], + fieldKind: SourceFieldKind, + sourceKind: SourceKind, +): FieldCandidates { + const rule = FIELD_RULES[fieldKind]; + const canonicalNames = + rule.canonicalNamesByKind?.[sourceKind] ?? rule.canonicalNames; + + // 1. type-compatible columns only + const compatible = columns.filter(c => rule.typeMatches(c.type)); + + // 2. case-insensitive name match, ranked by `canonicalNames` order. + const byLowerName = new Map(); + + for (const c of compatible) { + const key = c.name.toLowerCase(); + + const existing = byLowerName.get(key); + + if (existing) { + existing.push(c.name); + } else { + byLowerName.set(key, [c.name]); + } + } + + const rankedNameMatches = [ + ...new Set( + canonicalNames.flatMap(n => byLowerName.get(n.toLowerCase()) ?? []), + ), + ]; + + if (rule.recommendStrategy === 'single') { + const canonical = + rankedNameMatches[0] ?? + (compatible.length === 1 ? compatible[0].name : undefined); + + return { + canonical, + // surface all type-compatible columns so ambiguous multi-Map schemas present every option + alternates: compatible.map(c => c.name).filter(n => n !== canonical), + }; + } + + // 'name-only': never dump every String column as an alternate + const canonical = rankedNameMatches[0]; + return { + canonical, + alternates: rankedNameMatches.filter(n => n !== canonical), + }; +} + +export type SourceConfigPairingInput = { + kind: SourceKind; + bodyExpression?: string | null; + implicitColumnExpression?: string | null; +}; + +export type PairingWarning = { + field: SourceFieldKind; + message: string; + suggestedFix?: { field: SourceFieldKind; value: string }; +}; + +export function getSourceConfigPairingWarnings( + formValues: SourceConfigPairingInput, +): PairingWarning[] { + // Body / Implicit Column pairing only applies to log sources. + if (formValues.kind !== SourceKind.Log) { + return []; + } + + const warnings: PairingWarning[] = []; + const body = formValues.bodyExpression?.trim(); + const implicit = formValues.implicitColumnExpression?.trim(); + + if (body && !implicit) { + warnings.push({ + field: 'implicitColumnExpression', + message: + 'Body Expression is set but Implicit Column Expression is empty. ' + + 'Bare-text Lucene search will fail on this source. ' + + 'Use the same column as Body Expression?', + suggestedFix: { field: 'implicitColumnExpression', value: body }, + }); + } else if (implicit && !body) { + warnings.push({ + field: 'bodyExpression', + message: + 'Implicit Column Expression is set but Body Expression is empty. ' + + 'Row-panel body display will fall back to Implicit Column Expression. ' + + 'Set Body Expression explicitly?', + suggestedFix: { field: 'bodyExpression', value: implicit }, + }); + } + + return warnings; +} From e1b4769824cd9f0f6b56c044b4ed9dbfe665bb6d Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:01:37 +0200 Subject: [PATCH 053/728] Revert "fix: unknown lucene field falls through in search" (#2447) Reverts hyperdxio/hyperdx#2422 --- .changeset/healthy-pans-grab.md | 5 - .../src/__tests__/queryParser.test.ts | 45 -- .../src/__tests__/renderChartConfig.test.ts | 395 ------------------ .../src/core/renderChartConfig.ts | 81 ---- packages/common-utils/src/queryParser.ts | 24 +- 5 files changed, 7 insertions(+), 543 deletions(-) delete mode 100644 .changeset/healthy-pans-grab.md diff --git a/.changeset/healthy-pans-grab.md b/.changeset/healthy-pans-grab.md deleted file mode 100644 index df79b08a65..0000000000 --- a/.changeset/healthy-pans-grab.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix: unknown lucene field falls through in search diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index d735d52f2a..a2afc2118f 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -35,8 +35,6 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { return { name: 'SeverityNumber', type: 'UInt8' }; } else if (column === 'foo') { return { name: 'foo', type: 'String' }; - } else if (column === 'bar') { - return { name: 'bar', type: 'String' }; } else if (column === 'MaterializedExample') { return { name: 'MaterializedExample', type: 'String' }; } else { @@ -496,49 +494,6 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { expect(actualSql).toBe(expectedSql); }); - describe('CustomSchemaSQLSerializerV2: select alias resolution', () => { - // ClickHouse resolves SELECT aliases in WHERE, so a Lucene field matching a - // known alias must resolve to a bare identifier. A field that is neither a - // column nor an alias is genuinely unknown and must not be injected as raw - // SQL (which ClickHouse would reject as an unknown identifier). - const aliasSerializer = new CustomSchemaSQLSerializerV2({ - metadata, - databaseName, - tableName, - connectionId, - implicitColumnExpression: 'Body', - selectAliases: new Set(['Content']), - }); - - it('resolves a known select alias as a column reference', async () => { - const res = await aliasSerializer.getColumnForField('Content', {}); - expect(res).toEqual( - expect.objectContaining({ found: true, column: 'Content' }), - ); - }); - - it('renders a select alias reference as a bare identifier', async () => { - const sql = await new SearchQueryBuilder( - 'Content:foo', - aliasSerializer, - ).build(); - expect(sql).toBe("((Content ILIKE '%foo%'))"); - }); - - it('returns not-found for an unknown, non-alias field', async () => { - const res = await serializer.getColumnForField('NotAColumn', {}); - expect(res).toEqual(expect.objectContaining({ found: false })); - }); - - it('renders an unknown, non-alias field as the no-match predicate', async () => { - const sql = await new SearchQueryBuilder( - 'NotAColumn:foo', - serializer, - ).build(); - expect(sql).toBe('((1 = 0))'); - }); - }); - describe('CustomSchemaSQLSerializerV2: implicit column falls back to bodyExpression', () => { // Symmetric counterpart to the one-way fallback in // `getEventBody` (packages/app/src/source.ts): a log source admin who diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 8acf84b011..96e26dc805 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -31,7 +31,6 @@ describe('renderChartConfig', () => { { name: 'value', type: 'Float64' }, { name: 'TraceId', type: 'String' }, { name: 'ServiceName', type: 'String' }, - { name: 'severity', type: 'String' }, ]; mockMetadata = { getColumns: jest.fn().mockResolvedValue([ @@ -2253,400 +2252,6 @@ describe('renderChartConfig', () => { }); }); - describe('SELECT alias references in a Lucene WHERE', () => { - // ClickHouse resolves SELECT aliases in WHERE. A Lucene WHERE that - // references a SELECT alias (here `Body AS Content`) must resolve the alias - // to a bare identifier rather than rendering the no-match predicate. - it('resolves a Lucene WHERE reference to a SELECT alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [{ valueExpression: 'Body', alias: 'Content' }], - where: 'Content:"swagger"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - }); - - it('renders an unknown, non-alias field in a Lucene WHERE as the no-match predicate', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [{ valueExpression: 'Body', alias: 'Content' }], - where: 'NotAColumn:"swagger"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain('(1 = 0)'); - expect(actual).not.toContain('NotAColumn'); - }); - - // String-form select lists (e.g. a source's defaultTableSelectExpression) - // declare aliases as raw SQL. These are parsed to recover the aliases so - // alias references in a Lucene WHERE still resolve. - it('resolves a Lucene WHERE reference to an alias in a string-form select', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: 'Body AS Content', - where: 'Content:"swagger"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - }); - - it('resolves an alias from a realistic default-view string select', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: 'Timestamp, ServiceName as service, Body', - where: 'service:"prod"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("service = 'prod'"); - }); - - it('renders an unknown field as the no-match predicate when a string select declares no matching alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: 'Timestamp, Body', - where: 'NotAColumn:"x"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain('(1 = 0)'); - expect(actual).not.toContain('NotAColumn'); - }); - - // SAVED_SEARCH alerts select count() while injecting the saved search's - // select aliases as expression-form WITH clauses (isSubquery: false). A - // Lucene WHERE referencing such an alias must resolve to the bare - // identifier, not the no-match predicate. - it('resolves a Lucene WHERE reference to an expression-form WITH alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - with: [ - { - name: 'Content', - sql: chSql`toString(Body)`, - isSubquery: false, - }, - ], - select: [{ aggFn: 'count', valueExpression: '' }], - where: 'Content:"swagger"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - expect(actual).not.toContain('(1 = 0)'); - }); - - // Subquery CTEs (isSubquery: true) name a table-like source, not a column, - // so an alias reference to one in a Lucene WHERE is still unknown and must - // render the no-match predicate. - it('does not treat a subquery WITH alias as a column reference', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - with: [ - { - name: 'sub', - sql: chSql`SELECT 1`, - isSubquery: true, - }, - ], - select: [{ aggFn: 'count', valueExpression: '' }], - where: 'sub:"x"', - whereLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain('(1 = 0)'); - }); - }); - - describe('SELECT alias references in other Lucene call sites', () => { - // The main WHERE clause is covered above; these exercise the remaining - // Lucene-capable call sites in renderChartConfig (filters array, - // per-aggregate conditions, and HAVING), all of which must resolve SELECT - // aliases identically to the main WHERE. - - it('resolves a Lucene filter reference to a SELECT alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [{ valueExpression: 'Body', alias: 'Content' }], - where: '', - filters: [{ type: 'lucene', condition: 'Content:"swagger"' }], - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - }); - - it('renders an unknown, non-alias field in a Lucene filter as the no-match predicate', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [{ valueExpression: 'Body', alias: 'Content' }], - where: '', - filters: [{ type: 'lucene', condition: 'NotAColumn:"swagger"' }], - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain('(1 = 0)'); - expect(actual).not.toContain('NotAColumn'); - }); - - // A per-aggregate Lucene condition is pushed into the WHERE clause (when - // every select has one) and also rendered as the aggregate's `...If(...)` - // predicate. Both paths must resolve the alias. The alias here is an - // expression-form WITH clause to avoid a select referencing its own alias. - it('resolves a Lucene per-aggregate condition to a SELECT alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - with: [ - { - name: 'Content', - sql: chSql`toString(Body)`, - isSubquery: false, - }, - ], - select: [ - { - aggFn: 'count', - valueExpression: '', - aggCondition: 'Content:"swagger"', - aggConditionLanguage: 'lucene', - }, - ], - where: '', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - expect(actual).not.toContain('(1 = 0)'); - }); - - it('renders an unknown per-aggregate Lucene field as the no-match predicate', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [ - { - aggFn: 'count', - valueExpression: '', - aggCondition: 'NotAColumn:"swagger"', - aggConditionLanguage: 'lucene', - }, - ], - where: '', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain('(1 = 0)'); - expect(actual).not.toContain('NotAColumn'); - }); - - // A Lucene value expression (aggFn omitted, valueExpressionLanguage: lucene) - // is rendered as a SELECT column expression and must resolve aliases too. - it('resolves a Lucene value expression reference to a SELECT alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - with: [ - { - name: 'Content', - sql: chSql`toString(Body)`, - isSubquery: false, - }, - ], - select: [ - { - valueExpression: 'Content:"swagger"', - valueExpressionLanguage: 'lucene', - alias: 'matched', - }, - ], - where: '', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - expect(actual).not.toContain('(1 = 0)'); - }); - - // ClickHouse resolves SELECT aliases in HAVING, so a Lucene HAVING that - // references an alias must resolve it rather than render the no-match - // predicate. - it('resolves a Lucene HAVING reference to a SELECT alias', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [{ valueExpression: 'Body', alias: 'Content' }], - where: '', - having: 'Content:"swagger"', - havingLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain("Content = 'swagger'"); - }); - - it('renders an unknown, non-alias field in a Lucene HAVING as the no-match predicate', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Table, - connection: 'test-connection', - from: { - databaseName: 'default', - tableName: 'logs', - }, - select: [{ valueExpression: 'Body', alias: 'Content' }], - where: '', - having: 'NotAColumn:"swagger"', - havingLanguage: 'lucene', - }; - - const generatedSql = await renderChartConfig( - config, - mockMetadata, - querySettings, - ); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toContain('(1 = 0)'); - expect(actual).not.toContain('NotAColumn'); - }); - }); - describe('SETTINGS clause', () => { const config: ChartConfigWithOptDateRangeEx = { displayType: DisplayType.Table, diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 75cc63fc33..c25e985890 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -5,7 +5,6 @@ import SqlString from 'sqlstring'; import { ChSql, chSql, - chSqlToAliasMap, concatChSql, convertCHDataTypeToJSType, JSDataType, @@ -690,11 +689,6 @@ async function renderSelectList( const isRatio = isRatioChartConfig(selectList, chartConfig); - const selectAliases = extractSelectAliases({ - selectLists: [chartConfig.select, chartConfig.groupBy], - withClauses: chartConfig.with, - }); - const selectsSQL = await Promise.all( selectList.map(async select => { const whereClause = await renderWhereExpression({ @@ -708,7 +702,6 @@ async function renderSelectList( metadata, connectionId: chartConfig.connection, with: chartConfig.with, - selectAliases, }); let expr: ChSql; @@ -726,7 +719,6 @@ async function renderSelectList( metadata, connectionId: chartConfig.connection, with: chartConfig.with, - selectAliases, }) : chSql`${{ UNSAFE_RAW_SQL: select.valueExpression }}`; } else if ( @@ -986,64 +978,6 @@ function renderFrom({ ); } -/** - * Collects the aliases a Lucene WHERE may legally reference as bare - * identifiers: SELECT-list aliases plus expression-form WITH clauses. - * - * Array-form select lists expose `alias` directly. Raw-string lists (e.g. a - * source's `defaultTableSelectExpression` like - * 'Timestamp, ServiceName as service, Body') are parsed via `chSqlToAliasMap` - * to recover their declared aliases. - * - * WITH clauses only contribute when they declare an expression alias - * (`isSubquery === false`, i.e. `WITH (expr) AS ident`). Subquery CTEs - * (`WITH ident AS (subquery)`) name a table-like source, not a column usable in - * WHERE, so they're excluded. SAVED_SEARCH alerts depend on this: the alert - * query selects count() while the saved search's select aliases are injected as - * expression WITH clauses (see `computeAliasWithClauses` in the alert task). - */ -function extractSelectAliases({ - selectLists, - withClauses, -}: { - selectLists: (SelectList | undefined)[]; - withClauses?: BuilderChartConfigWithDateRange['with']; -}): Set { - const aliases: string[] = []; - - for (const list of selectLists) { - if (list == null) { - continue; - } - - if (typeof list === 'string') { - // Wrap the select-list string in a parse scaffold (mirrors the - // `SELECT * FROM \`t\`` scaffold used below). chSqlToAliasMap swallows - // parse failures and returns {}, so an unparseable list contributes no - // aliases rather than throwing. - const aliasMap = chSqlToAliasMap( - chSql`SELECT ${{ UNSAFE_RAW_SQL: list }} FROM t`, - ); - aliases.push(...Object.keys(aliasMap)); - continue; - } - - for (const col of list) { - if (col.alias && col.alias.trim() !== '') { - aliases.push(col.alias); - } - } - } - - for (const clause of withClauses ?? []) { - if (clause.isSubquery === false && clause.name.trim() !== '') { - aliases.push(clause.name); - } - } - - return new Set(aliases); -} - async function renderWhereExpressionStr({ condition, language, @@ -1054,7 +988,6 @@ async function renderWhereExpressionStr({ useTextIndexForImplicitColumn, connectionId, with: withClauses, - selectAliases, }: { condition: SearchCondition; language: SearchConditionLanguage; @@ -1065,7 +998,6 @@ async function renderWhereExpressionStr({ useTextIndexForImplicitColumn?: BuilderChartConfigWithDateRange['useTextIndexForImplicitColumn']; connectionId: string; with?: BuilderChartConfigWithDateRange['with']; - selectAliases?: Set; }): Promise { let _condition = condition; if (language === 'lucene') { @@ -1077,7 +1009,6 @@ async function renderWhereExpressionStr({ bodyExpression, useTextIndexForImplicitColumn, connectionId: connectionId, - selectAliases, }); const builder = new SearchQueryBuilder(condition, serializer); _condition = await builder.build(); @@ -1127,11 +1058,6 @@ async function renderWhere( chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { - const selectAliases = extractSelectAliases({ - selectLists: [chartConfig.select, chartConfig.groupBy], - withClauses: chartConfig.with, - }); - let whereSearchCondition: ChSql | [] = []; if (isNonEmptyWhereExpr(chartConfig.where)) { whereSearchCondition = wrapChSqlIfNotEmpty( @@ -1146,7 +1072,6 @@ async function renderWhere( metadata, connectionId: chartConfig.connection, with: chartConfig.with, - selectAliases, }), '(', ')', @@ -1175,7 +1100,6 @@ async function renderWhere( metadata, connectionId: chartConfig.connection, with: chartConfig.with, - selectAliases, }); } return null; @@ -1224,7 +1148,6 @@ async function renderWhere( metadata, connectionId: chartConfig.connection, with: chartConfig.with, - selectAliases, }), '(', ')', @@ -1393,10 +1316,6 @@ async function renderHaving( metadata, connectionId: chartConfig.connection, with: chartConfig.with, - selectAliases: extractSelectAliases({ - selectLists: [chartConfig.select, chartConfig.groupBy], - withClauses: chartConfig.with, - }), }); } diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 64c9a7cca3..2a500c6b1c 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -762,7 +762,6 @@ export type CustomSchemaConfig = { * metadata; otherwise, it forces the chosen behavior. */ useTextIndexForImplicitColumn?: UseTextIndex; - selectAliases?: Set; }; function renderArrayFieldExpression({ @@ -1167,7 +1166,6 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { private skipIndicesPromise?: Promise; private enableTextIndexPromise?: Promise; private kvItemsLookupPromise?: Promise; - private selectAliases: Set | undefined; constructor({ metadata, @@ -1177,7 +1175,6 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { implicitColumnExpression, bodyExpression, useTextIndexForImplicitColumn, - selectAliases, }: { metadata: Metadata } & CustomSchemaConfig) { super(); this.metadata = metadata; @@ -1188,7 +1185,6 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { this.connectionId = connectionId; this.useTextIndexForImplicitColumn = useTextIndexForImplicitColumn ?? UseTextIndex.Auto; - this.selectAliases = selectAliases; // Pre-fetch skip indices for potential bloom filter optimization this.skipIndicesPromise = this.metadata @@ -1477,14 +1473,13 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { }; let materializedColumns: Map; try { - // This won't work for CTEs. Treat a missing lookup (null/undefined) the - // same as the catch path below: proceed with no materialized columns. + // This won't work for CTEs materializedColumns = - (await this.metadata.getMaterializedColumnsLookupTable({ + await this.metadata.getMaterializedColumnsLookupTable({ databaseName: this.databaseName, tableName: this.tableName, connectionId: this.connectionId, - })) ?? new Map(); + }); } catch (e) { console.debug('Error in getMaterializedColumnsLookupTable', e); materializedColumns = new Map(); @@ -1591,19 +1586,14 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { throw new Error('Unsupported column type for prefix match'); } - if (this.selectAliases?.has(field)) { - return { - found: true, - columnExpression: field, - columnType: 'Unknown', - }; - } - + // It might be an alias, let's just try the column + // TODO: Verify aliases return { - found: false, + found: true, columnExpression: field, columnType: 'Unknown', }; + // throw new Error(`Column not found: ${field}`); } private isLowerExpression(expr: string): boolean { From 6a800318b8be314716f85ebe2084121298d5d933 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 12 Jun 2026 08:49:49 -0600 Subject: [PATCH 054/728] feat(hdx-eval): support multi-model comparison in eval batches (#2438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds multi-model support to `hdx-eval` so a single batch can compare different LLM models across MCP configurations. The `--model` flag now accepts comma-separated model IDs: ```bash yarn eval run --mcp hyperdx --model claude-opus-4-6,claude-haiku-4-5 --runs 3 ``` This creates a matrix of `(mcp, model, runIndex)` cells, enabling apples-to-apples comparison across models within the same batch. ## Why Previously, comparing models required running separate batches and manually collating results. This makes model comparison a first-class feature of the eval framework. ## Key changes - **New directory layout**: `///.json` — legacy `//.json` layout is still supported for backward compatibility - **`ColumnKey` type**: Reports now use `mcp` (single-model) or `mcp/model` (multi-model) as column identifiers throughout - **Centralized helpers**: `isRunJson`, `safeReaddir`, `isModelSubdir` consolidated in `runs/path.ts`, removing duplication from 4 modules - **Viewer**: Routes and UI updated to show `mcp/model` headings and navigate to model-specific runs - **`BatchSummary`**: Gains `columnOrder`, `multiModel` fields (`mcpOrder` kept as deprecated alias) - **Tests**: New test case for multi-model grouping in `aggregate.test.ts` ## Backward compatibility - Existing batches (single-model, legacy layout) continue to work without changes - `mcpOrder` is preserved as a deprecated alias of `columnOrder` for older viewer code - Viewer server handles both old and new API routes --- .changeset/good-parents-check.md | 5 + .../hdx-eval/scripts/viewer/public/app.js | 35 ++++-- packages/hdx-eval/scripts/viewer/server.js | 84 ++++++++++---- .../hdx-eval/src/__tests__/aggregate.test.ts | 47 +++++++- packages/hdx-eval/src/cli.ts | 96 ++++++++++++---- packages/hdx-eval/src/grading/grade.ts | 32 +++--- packages/hdx-eval/src/reports/aggregate.ts | 107 +++++++++++++----- packages/hdx-eval/src/reports/markdown.ts | 81 ++++++------- packages/hdx-eval/src/reports/store.ts | 55 +++++---- packages/hdx-eval/src/runs/instrument.ts | 41 +++---- packages/hdx-eval/src/runs/path.ts | 55 +++++++++ packages/hdx-eval/src/runs/store.ts | 49 ++++---- 12 files changed, 491 insertions(+), 196 deletions(-) create mode 100644 .changeset/good-parents-check.md diff --git a/.changeset/good-parents-check.md b/.changeset/good-parents-check.md new file mode 100644 index 0000000000..e21e03acf8 --- /dev/null +++ b/.changeset/good-parents-check.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/hdx-eval': patch +--- + +Support multi-model comparison in eval batches via comma-separated --model flag diff --git a/packages/hdx-eval/scripts/viewer/public/app.js b/packages/hdx-eval/scripts/viewer/public/app.js index 9efc87e8d9..c57f71ef7e 100644 --- a/packages/hdx-eval/scripts/viewer/public/app.js +++ b/packages/hdx-eval/scripts/viewer/public/app.js @@ -20,7 +20,7 @@ batch: null, cells: [], summary: null, - selectedRun: null, // { scenario, mcp, idx } + selectedRun: null, // { scenario, mcp, model, idx } runData: null, // { trajectory, grade } activeTab: 'trajectory', view: 'summary', // 'summary' | 'run' @@ -177,9 +177,12 @@ showView('summary'); } - async function loadRun(scenario, mcp, idx) { - state.selectedRun = { scenario, mcp, idx }; - const u = `/api/batches/${encodeURIComponent(state.batch)}/runs/${encodeURIComponent(scenario)}/${encodeURIComponent(mcp)}/${idx}`; + async function loadRun(scenario, mcp, model, idx) { + state.selectedRun = { scenario, mcp, model, idx }; + const base = `/api/batches/${encodeURIComponent(state.batch)}/runs/${encodeURIComponent(scenario)}/${encodeURIComponent(mcp)}`; + const u = model + ? `${base}/${encodeURIComponent(model)}/${idx}` + : `${base}/${idx}`; state.runData = await fetchJson(u); renderNav(); showView('run'); @@ -218,6 +221,18 @@ // ----- rendering: sidebar nav ----- + /** True when the current batch has more than one distinct model across cells. */ + function isMultiModel() { + const models = new Set(); + for (const c of state.cells) if (c.model) models.add(c.model); + return models.size > 1; + } + + /** Heading for a cell: matches the columnKeyFor logic in aggregate.ts. */ + function cellHeading(c) { + return c.model && isMultiModel() ? `${c.mcp}/${c.model}` : c.mcp; + } + function renderNav() { const root = $('#nav'); root.innerHTML = ''; @@ -235,16 +250,18 @@ el('div', { class: 'head' }, scenario), ); for (const c of cells) { + const heading = cellHeading(c); const mNode = el( 'div', { class: 'nav-mcp' }, - el('div', { class: 'head' }, c.mcp), + el('div', { class: 'head' }, heading), ); for (const r of c.runs) { const active = state.selectedRun && state.selectedRun.scenario === scenario && state.selectedRun.mcp === c.mcp && + state.selectedRun.model === c.model && state.selectedRun.idx === r.idx; const chip = el( 'span', @@ -263,7 +280,7 @@ 'div', { class: `nav-run ${active ? 'active' : ''}`, - onclick: () => loadRun(scenario, c.mcp, r.idx), + onclick: () => loadRun(scenario, c.mcp, c.model, r.idx), }, el( 'span', @@ -735,7 +752,7 @@ } root.className = 'summary-dashboard'; - const mcpOrder = s.mcpOrder || Object.keys(s.scenarios[0]?.cells || {}).sort(); + const mcpOrder = s.columnOrder || s.mcpOrder || Object.keys(s.scenarios[0]?.cells || {}).sort(); const baseline = s.baseline || mcpOrder[0]; const challengers = mcpOrder.filter((m) => m !== baseline); @@ -1017,7 +1034,7 @@ const maxRuns = Math.max(...scenarioCells.map((c) => c.runs.length)); const runHeader = [el('th', {}, 'Run')]; for (const c of scenarioCells) { - runHeader.push(el('th', {}, c.mcp)); + runHeader.push(el('th', {}, cellHeading(c))); } const runRows = []; for (let i = 0; i < maxRuns; i++) { @@ -1046,7 +1063,7 @@ 'td', { class: 'run-cell clickable', - onclick: () => loadRun(scenario.scenario, c.mcp, r.idx), + onclick: () => loadRun(scenario.scenario, c.mcp, c.model, r.idx), }, chip, ' ', diff --git a/packages/hdx-eval/scripts/viewer/server.js b/packages/hdx-eval/scripts/viewer/server.js index aaefe0db45..d2e138d925 100644 --- a/packages/hdx-eval/scripts/viewer/server.js +++ b/packages/hdx-eval/scripts/viewer/server.js @@ -60,6 +60,38 @@ function listBatches() { .reverse(); } +function isRunJson(f) { + return /^\d+\.json$/.test(f); +} + +function collectRuns(cellDir) { + let files; + try { + files = fs.readdirSync(cellDir); + } catch { + return []; // directory may have been removed between discovery and read + } + const runIdxs = new Set(); + for (const f of files) { + const m = /^(\d+)\.json$/.exec(f); + if (m) runIdxs.add(Number(m[1])); + } + return [...runIdxs].sort((a, b) => a - b).map((i) => { + const grade = readJsonSafe(path.join(cellDir, `${i}.grade.json`)); + const traj = readJsonSafe(path.join(cellDir, `${i}.json`)); + return { + idx: i, + combinedScore: grade?.combinedScore ?? null, + programmaticScore: grade?.programmatic?.score ?? null, + judgeScore: grade?.judge?.weightedScore ?? null, + termination: traj?.termination ?? null, + durationMs: traj?.durationMs ?? null, + toolCalls: traj?.toolCalls?.length ?? null, + toolErrors: grade?.toolErrors?.errors ?? null, + }; + }); +} + function listCells(batch) { const batchDir = safeJoin(RUNS_DIR, batch); if (!fs.existsSync(batchDir)) return null; @@ -75,28 +107,23 @@ function listCells(batch) { .filter((d) => d.isDirectory()) .map((d) => d.name); for (const mcp of mcps) { - const cellDir = path.join(scenarioDir, mcp); - const files = fs.readdirSync(cellDir); - const runIdxs = new Set(); - for (const f of files) { - const m = /^(\d+)\.json$/.exec(f); - if (m) runIdxs.add(Number(m[1])); + const mcpDir = path.join(scenarioDir, mcp); + const entries = fs.readdirSync(mcpDir, { withFileTypes: true }); + + // Legacy layout: //.json + const hasRunFiles = entries.some((e) => !e.isDirectory() && isRunJson(e.name)); + if (hasRunFiles) { + const runs = collectRuns(mcpDir); + if (runs.length > 0) out.push({ scenario, mcp, model: null, runs }); + } + + // New layout: ///.json + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const modelDir = path.join(mcpDir, entry.name); + const runs = collectRuns(modelDir); + if (runs.length > 0) out.push({ scenario, mcp, model: entry.name, runs }); } - const runs = [...runIdxs].sort((a, b) => a - b).map((i) => { - const grade = readJsonSafe(path.join(cellDir, `${i}.grade.json`)); - const traj = readJsonSafe(path.join(cellDir, `${i}.json`)); - return { - idx: i, - combinedScore: grade?.combinedScore ?? null, - programmaticScore: grade?.programmatic?.score ?? null, - judgeScore: grade?.judge?.weightedScore ?? null, - termination: traj?.termination ?? null, - durationMs: traj?.durationMs ?? null, - toolCalls: traj?.toolCalls?.length ?? null, - toolErrors: grade?.toolErrors?.errors ?? null, - }; - }); - out.push({ scenario, mcp, runs }); } } return out; @@ -127,6 +154,21 @@ const ROUTES = [ sendJson(res, 200, { batch, summary, cells }); }, ], + // New layout: /runs/:scenario/:mcp/:model/:idx + [ + /^\/api\/batches\/([^/]+)\/runs\/([^/]+)\/([^/]+)\/([^/]+)\/(\d+)$/, + (m, _q, res) => { + const [, batch, scenario, mcp, model, idx] = m.map((x, i) => + i === 0 ? x : decodeURIComponent(x), + ); + const base = safeJoin(RUNS_DIR, batch, scenario, mcp, model); + const trajectory = readJsonSafe(path.join(base, `${idx}.json`)); + const grade = readJsonSafe(path.join(base, `${idx}.grade.json`)); + if (!trajectory) return sendJson(res, 404, { error: 'run not found' }); + sendJson(res, 200, { trajectory, grade }); + }, + ], + // Legacy layout: /runs/:scenario/:mcp/:idx (no model segment) [ /^\/api\/batches\/([^/]+)\/runs\/([^/]+)\/([^/]+)\/(\d+)$/, (m, _q, res) => { diff --git a/packages/hdx-eval/src/__tests__/aggregate.test.ts b/packages/hdx-eval/src/__tests__/aggregate.test.ts index 9227de4aa4..0935b426be 100644 --- a/packages/hdx-eval/src/__tests__/aggregate.test.ts +++ b/packages/hdx-eval/src/__tests__/aggregate.test.ts @@ -128,11 +128,14 @@ describe('buildAggregate', () => { durationMs: 50_000, }), ]; - // Baseline defaults to first MCP alphabetically: 'clickhouse'. + // Baseline defaults to first column alphabetically: 'clickhouse'. const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); expect(summary.scenarios).toHaveLength(1); expect(summary.baseline).toBe('clickhouse'); + expect(summary.columnOrder).toEqual(['clickhouse', 'hyperdx']); + // Deprecated alias: expect(summary.mcpOrder).toEqual(['clickhouse', 'hyperdx']); + expect(summary.multiModel).toBe(false); const sc = summary.scenarios[0]; const h = sc.cells['hyperdx']!; const c = sc.cells['clickhouse']!; @@ -253,6 +256,7 @@ describe('buildAggregate', () => { pairs, baseline: 'alpha', }); + expect(summary.columnOrder).toEqual(['alpha', 'beta', 'gamma']); expect(summary.mcpOrder).toEqual(['alpha', 'beta', 'gamma']); const sc = summary.scenarios[0]; // Challengers: beta and gamma @@ -261,4 +265,45 @@ describe('buildAggregate', () => { expect(sc.deltas['gamma']).toBeDefined(); expect(sc.deltas['gamma'].toolCalls).toBeCloseTo(-2, 5); }); + + it('groups by (mcp, model) when multiple models are present', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 13, + outputTokens: 4000, + durationMs: 60_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 1, + programmaticScore: 0.8, + judgeScore: 0.8, + toolCalls: 11, + outputTokens: 3500, + durationMs: 50_000, + }), + ]; + // Change the model on the second pair to trigger multi-model grouping. + pairs[1].run.model = 'claude-haiku-4-5'; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.multiModel).toBe(true); + expect(summary.columnOrder).toEqual([ + 'hyperdx/claude-haiku-4-5', + 'hyperdx/claude-sonnet-4-6', + ]); + const sc = summary.scenarios[0]; + expect(sc.cells['hyperdx/claude-sonnet-4-6']).toBeDefined(); + expect(sc.cells['hyperdx/claude-haiku-4-5']).toBeDefined(); + // Each cell should have n=1 + expect(sc.cells['hyperdx/claude-sonnet-4-6'].n).toBe(1); + expect(sc.cells['hyperdx/claude-haiku-4-5'].n).toBe(1); + // Deltas: challenger vs baseline (first alphabetically) + expect(sc.deltas['hyperdx/claude-sonnet-4-6']?.combinedScore).toBeDefined(); + }); }); diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts index a256fbdcbf..76d3825aa3 100644 --- a/packages/hdx-eval/src/cli.ts +++ b/packages/hdx-eval/src/cli.ts @@ -27,6 +27,7 @@ import { readConfig, } from './hyperdx/config'; import { runCheck, runSetup } from './hyperdx/setup'; +import { columnKeyFor } from './reports/aggregate'; import { writeBatchSummary } from './reports/store'; import { instrumentBatch, summarizeTimingRecord } from './runs/instrument'; import { batchDirName } from './runs/path'; @@ -319,7 +320,12 @@ program 'MCP to use as baseline in reports (default: first MCP in list)', ) .option('--runs ', 'Number of runs per (scenario,MCP) cell', '3') - .option('--model ', 'Model ID to pass to Claude Code', 'claude-opus-4-6') + .option( + '--model ', + 'Comma-separated model IDs to pass to Claude Code. When multiple ' + + 'models are given, every (mcp, model) pair is compared in reports.', + 'claude-opus-4-6', + ) // Lower than the previous 25 — tightens the budget so sloppy agents that // make 20+ exploratory calls can't paper over correctness with volume. // Override with --max-turns if a specific scenario needs more. @@ -415,11 +421,22 @@ program for (const mcp of mcpKinds) { getMcpDefinition(config, mcp); } - const baseline = cmdOpts.baseline ?? mcpKinds[0]; - if (cmdOpts.baseline && !mcpKinds.includes(cmdOpts.baseline)) { - throw new Error( - `--baseline "${cmdOpts.baseline}" is not in the MCP list: ${mcpKinds.join(', ')}`, + const models = parseModelFlag(cmdOpts.model); + const multiModel = models.length > 1; + // For baseline resolution: when multi-model, the column key is + // mcp/sanitizedModel; otherwise just mcp. + const firstColumnKey = columnKeyFor(mcpKinds[0], models[0], multiModel); + const baseline = cmdOpts.baseline ?? firstColumnKey; + if (cmdOpts.baseline) { + // Validate: baseline must match one of the column keys. + const allColumnKeys = mcpKinds.flatMap(m => + models.map(mod => columnKeyFor(m, mod, multiModel)), ); + if (!allColumnKeys.includes(cmdOpts.baseline)) { + throw new Error( + `--baseline "${cmdOpts.baseline}" is not in the column list: ${[...new Set(allColumnKeys)].join(', ')}`, + ); + } } const promptVariant: PromptVariant = parsePromptVariant( cmdOpts.promptVariant, @@ -517,12 +534,14 @@ program console.log(`\nBatch: runs/${batchDir}`); console.log(`Scenario: ${scenario.name}`); console.log(`MCPs: ${mcpKinds.join(', ')}`); + console.log(`Models: ${models.join(', ')}`); console.log( - `Runs/cell: ${runs}, model: ${cmdOpts.model}, max-turns: ${maxTurns}, concurrency: ${concurrency}, prompt-variant: ${promptVariant}\n`, + `Runs/cell: ${runs}, max-turns: ${maxTurns}, concurrency: ${concurrency}, prompt-variant: ${promptVariant}\n`, ); type SummaryRow = { mcp: McpKind; + model: string; i: number; toolCalls: number; inputTokens: number; @@ -532,25 +551,33 @@ program path: string; }; - // Flatten the (mcp, runIndex) matrix into a single queue and pull from it - // with a worker pool of size `concurrency`. - const cells: Array<{ mcp: McpKind; i: number }> = []; + // Flatten the (mcp, model, runIndex) matrix into a single queue and + // pull from it with a worker pool of size `concurrency`. + const cells: Array<{ mcp: McpKind; model: string; i: number }> = []; for (const mcp of mcpKinds) { - for (let i = 0; i < runs; i++) cells.push({ mcp, i }); + for (const model of models) { + for (let i = 0; i < runs; i++) cells.push({ mcp, model, i }); + } } const summary: SummaryRow[] = []; let cursor = 0; - const errors: Array<{ mcp: string; i: number; error: string }> = []; + const errors: Array<{ + mcp: string; + model: string; + i: number; + error: string; + }> = []; async function worker(workerId: number): Promise { while (true) { const idx = cursor++; if (idx >= cells.length) return; - const { mcp, i } = cells[idx]; + const { mcp, model, i } = cells[idx]; + const cellLabel = columnKeyFor(mcp, model, multiModel); const label = concurrency > 1 ? `[w${workerId}] ` : ' '; try { process.stdout.write( - `${label}${mcp} run ${i + 1}/${runs}... starting\n`, + `${label}${cellLabel} run ${i + 1}/${runs}... starting\n`, ); const startedMs = Date.now(); const record = await runCell({ @@ -558,7 +585,7 @@ program scenario: scenario.name, agentPrompt: scenario.agentPrompt, mcp, - model: cmdOpts.model, + model, maxTurns, timeoutMs, runIndex: i, @@ -570,10 +597,11 @@ program const path = writeRun({ record, batchDir }); const seconds = ((Date.now() - startedMs) / 1000).toFixed(1); console.log( - `${label}${mcp} run ${i + 1}/${runs}: ${record.termination} · ${record.toolCalls.length} tool calls · ${record.tokens.input}+${record.tokens.output} tok · ${seconds}s`, + `${label}${cellLabel} run ${i + 1}/${runs}: ${record.termination} · ${record.toolCalls.length} tool calls · ${record.tokens.input}+${record.tokens.output} tok · ${seconds}s`, ); summary.push({ mcp, + model, i, toolCalls: record.toolCalls.length, inputTokens: record.tokens.input, @@ -585,9 +613,9 @@ program } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error( - `${label}${mcp} run ${i + 1}/${runs}: FAILED — ${msg}`, + `${label}${cellLabel} run ${i + 1}/${runs}: FAILED — ${msg}`, ); - errors.push({ mcp, i, error: msg }); + errors.push({ mcp, model, i, error: msg }); } } } @@ -599,15 +627,26 @@ program if (errors.length > 0) { console.error( `\n${errors.length} run(s) failed:\n` + - errors.map(e => ` ${e.mcp} #${e.i}: ${e.error}`).join('\n'), + errors + .map(e => { + const cl = columnKeyFor(e.mcp, e.model, multiModel); + return ` ${cl} #${e.i}: ${e.error}`; + }) + .join('\n'), ); } console.log('\nResults written under runs/' + batchDir); - // Sort by (mcp, i) for stable summary output even with parallel workers. - summary.sort((a, b) => a.mcp.localeCompare(b.mcp) || a.i - b.i); + // Sort by (mcp, model, i) for stable summary output. + summary.sort( + (a, b) => + a.mcp.localeCompare(b.mcp) || + a.model.localeCompare(b.model) || + a.i - b.i, + ); for (const row of summary) { + const cl = columnKeyFor(row.mcp, row.model, multiModel); console.log( - ` ${row.mcp.padEnd(10)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, + ` ${cl.padEnd(multiModel ? 30 : 10)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, ); } @@ -881,6 +920,21 @@ function parseMcpFlag( return names; } +function parseModelFlag(v: string): string[] { + const models = [ + ...new Set( + v + .split(',') + .map(s => s.trim()) + .filter(Boolean), + ), + ]; + if (models.length === 0) { + throw new Error(`--model must be a comma-separated list of model IDs`); + } + return models; +} + function parsePromptVariant(v: string): PromptVariant { if (v === 'baseline' || v === 'hypothesis') return v; throw new Error( diff --git a/packages/hdx-eval/src/grading/grade.ts b/packages/hdx-eval/src/grading/grade.ts index 71c2463af7..eaf4af4b9a 100644 --- a/packages/hdx-eval/src/grading/grade.ts +++ b/packages/hdx-eval/src/grading/grade.ts @@ -1,9 +1,9 @@ import Anthropic from '@anthropic-ai/sdk'; -import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; import { dirname, join } from 'path'; import type { RunRecord } from '../harness/types'; -import { runsRoot } from '../runs/path'; +import { isModelSubdir, isRunJson, runsRoot, safeReaddir } from '../runs/path'; import { readRun } from '../runs/store'; import { getScenario, SCENARIO_NAMES } from '../scenarios'; import type { BlindingEntry } from './blind'; @@ -215,6 +215,11 @@ async function gradeOne(args: { }; } +/** + * List run JSON files. Supports both the new + * `///.json` layout and the legacy + * `//.json` layout. + */ function listRunFiles(batchDir: string): string[] { const out: string[] = []; for (const scenario of safeReaddir(batchDir)) { @@ -222,24 +227,23 @@ function listRunFiles(batchDir: string): string[] { const sceneDir = join(batchDir, scenario); for (const mcp of safeReaddir(sceneDir)) { const mcpDir = join(sceneDir, mcp); - for (const file of safeReaddir(mcpDir)) { - if (file.endsWith('.grade.json')) continue; - if (file.endsWith('.timing.json')) continue; - if (file.endsWith('.json')) out.push(join(mcpDir, file)); + for (const entry of safeReaddir(mcpDir)) { + if (isRunJson(entry)) { + // Legacy layout: //.json + out.push(join(mcpDir, entry)); + } else if (isModelSubdir(mcpDir, entry)) { + // New layout: ///.json + const modelDir = join(mcpDir, entry); + for (const file of safeReaddir(modelDir)) { + if (isRunJson(file)) out.push(join(modelDir, file)); + } + } } } } return out.sort(); } -function safeReaddir(path: string): string[] { - try { - return readdirSync(path); - } catch { - return []; - } -} - function readExistingGrade(path: string): GradeRecord | null { if (!existsSync(path)) return null; try { diff --git a/packages/hdx-eval/src/reports/aggregate.ts b/packages/hdx-eval/src/reports/aggregate.ts index ee86152123..cf68fc8af2 100644 --- a/packages/hdx-eval/src/reports/aggregate.ts +++ b/packages/hdx-eval/src/reports/aggregate.ts @@ -1,9 +1,21 @@ import type { GradeRecord } from '../grading/types'; import type { McpKind, RunRecord } from '../harness/types'; +import { modelDirName } from '../runs/path'; + +/** + * A column key uniquely identifies a (mcp, model) combination in reports. + * When a batch has a single model, the column key equals the MCP name + * (backward-compatible). When multiple models are present, the column + * key is `mcp/model`. + */ +export type ColumnKey = string; export type CellSummary = { scenario: string; mcp: McpKind; + model: string; + /** Display key used in report columns. */ + columnKey: ColumnKey; n: number; programmatic: { mean: number; perCheck: Record }; judge: { @@ -35,22 +47,29 @@ export type DeltaSummary = { export type ScenarioSummary = { scenario: string; - /** All MCP cells keyed by MCP name. */ - cells: Record; - /** Which MCP is the baseline (first MCP, or explicitly set). */ - baseline?: McpKind; - /** Per-challenger deltas vs the baseline. Keyed by challenger MCP name. */ - deltas: Record; + /** Cells keyed by column key (mcp or mcp/model). */ + cells: Record; + /** Which column key is the baseline. */ + baseline?: ColumnKey; + /** Per-challenger deltas vs the baseline. Keyed by challenger column key. */ + deltas: Record; }; export type BatchSummary = { batchDir: string; generatedAt: string; - /** The baseline MCP used for delta computation. */ - baseline?: McpKind; - /** Ordered list of MCP names as they appear in the report columns. */ - mcpOrder: McpKind[]; + /** The baseline column key used for delta computation. */ + baseline?: ColumnKey; + /** Ordered list of column keys as they appear in the report columns. */ + columnOrder: ColumnKey[]; + /** Whether this batch compares multiple models. */ + multiModel: boolean; scenarios: ScenarioSummary[]; + /** + * @deprecated Use `columnOrder`. Preserved for backward compatibility + * with older viewer code. + */ + mcpOrder: ColumnKey[]; }; export type GradedRunPair = { @@ -58,41 +77,65 @@ export type GradedRunPair = { grade: GradeRecord; }; +/** + * Build a column key from a (mcp, model) pair. + * When `multiModel` is false the column key is just the mcp name. + * + * The model component is sanitized through `modelDirName` so that the + * column key matches the directory name on disk (which is what the + * viewer reads). Without this, model IDs containing `/`, `:`, or `.` + * would produce different keys in the summary JSON vs the viewer. + */ +export function columnKeyFor( + mcp: McpKind, + model: string, + multiModel: boolean, +): ColumnKey { + return multiModel ? `${mcp}/${modelDirName(model)}` : mcp; +} + export function buildAggregate(args: { batchDir: string; pairs: GradedRunPair[]; - /** Explicit baseline MCP. If not set, the first MCP encountered is used. */ - baseline?: McpKind; + /** Explicit baseline column key. If not set, the first column encountered + * is used. When using a single model, pass just the mcp name. */ + baseline?: ColumnKey; }): BatchSummary { + // Detect whether this batch involves multiple models. + const modelSet = new Set(); + for (const p of args.pairs) modelSet.add(p.run.model); + const multiModel = modelSet.size > 1; + const byScenario = new Map(); - // Track all MCP names in insertion order. - const mcpSet = new Set(); + const columnSet = new Set(); for (const p of args.pairs) { if (!byScenario.has(p.run.scenario)) byScenario.set(p.run.scenario, []); byScenario.get(p.run.scenario)!.push(p); - mcpSet.add(p.run.mcp); + columnSet.add(columnKeyFor(p.run.mcp, p.run.model, multiModel)); } - const mcpOrder = [...mcpSet].sort(); - const baseline = args.baseline ?? mcpOrder[0]; + const columnOrder = [...columnSet].sort(); + const baseline = args.baseline ?? columnOrder[0]; const scenarios: ScenarioSummary[] = []; for (const [name, ps] of byScenario.entries()) { - const byMcp = new Map(); + const byCol = new Map(); for (const p of ps) { - if (!byMcp.has(p.run.mcp)) byMcp.set(p.run.mcp, []); - byMcp.get(p.run.mcp)!.push(p); + const key = columnKeyFor(p.run.mcp, p.run.model, multiModel); + if (!byCol.has(key)) byCol.set(key, []); + byCol.get(key)!.push(p); } - const cells: Record = {}; - for (const [mcp, list] of byMcp.entries()) { - cells[mcp] = buildCellSummary(name, mcp, list); + const cells: Record = {}; + for (const [col, list] of byCol.entries()) { + const first = list[0].run; + cells[col] = buildCellSummary(name, first.mcp, first.model, col, list); } - // Compute deltas: each non-baseline MCP vs the baseline. - const deltas: Record = {}; + // Compute deltas: each non-baseline column vs the baseline. + const deltas: Record = {}; const baselineCell = cells[baseline]; - for (const mcp of Object.keys(cells)) { - if (mcp === baseline) continue; - deltas[mcp] = computeDelta(cells[mcp], baselineCell); + for (const col of Object.keys(cells)) { + if (col === baseline) continue; + deltas[col] = computeDelta(cells[col], baselineCell); } scenarios.push({ @@ -108,7 +151,9 @@ export function buildAggregate(args: { batchDir: args.batchDir, generatedAt: new Date().toISOString(), baseline, - mcpOrder, + columnOrder, + multiModel, + mcpOrder: columnOrder, scenarios, }; } @@ -116,6 +161,8 @@ export function buildAggregate(args: { function buildCellSummary( scenario: string, mcp: McpKind, + model: string, + columnKey: ColumnKey, pairs: GradedRunPair[], ): CellSummary { const n = pairs.length; @@ -179,6 +226,8 @@ function buildCellSummary( return { scenario, mcp, + model, + columnKey, n, programmatic: { mean: programmaticScore, perCheck }, judge: { diff --git a/packages/hdx-eval/src/reports/markdown.ts b/packages/hdx-eval/src/reports/markdown.ts index 4aabedb9fc..4e0c86373b 100644 --- a/packages/hdx-eval/src/reports/markdown.ts +++ b/packages/hdx-eval/src/reports/markdown.ts @@ -1,12 +1,13 @@ -import type { McpKind } from '../harness/types'; import type { BatchSummary, CellSummary, + ColumnKey, DeltaSummary, ScenarioSummary, } from './aggregate'; export function renderMarkdownReport(summary: BatchSummary): string { + const columns = summary.columnOrder; const lines: string[] = []; lines.push(`# Eval Batch — ${basename(summary.batchDir)}`); lines.push(''); @@ -14,24 +15,28 @@ export function renderMarkdownReport(summary: BatchSummary): string { if (summary.baseline) { lines.push(`Baseline: ${summary.baseline}`); } - lines.push(`MCPs: ${summary.mcpOrder.join(', ')}`); + if (summary.multiModel) { + lines.push(`Columns: ${columns.join(', ')} _(mcp/model)_`); + } else { + lines.push(`MCPs: ${columns.join(', ')}`); + } lines.push(''); - lines.push(renderTopVerdict(summary)); + lines.push(renderTopVerdict(summary, columns)); lines.push(''); for (const scenario of summary.scenarios) { lines.push(`## ${scenario.scenario}`); lines.push(''); - lines.push(renderScenarioTable(scenario, summary.mcpOrder)); + lines.push(renderScenarioTable(scenario, columns)); lines.push(''); - const judgeBreakdown = renderJudgeBreakdown(scenario, summary.mcpOrder); + const judgeBreakdown = renderJudgeBreakdown(scenario, columns); if (judgeBreakdown) { lines.push(judgeBreakdown); lines.push(''); } const programmaticBreakdown = renderProgrammaticBreakdown( scenario, - summary.mcpOrder, + columns, ); if (programmaticBreakdown) { lines.push(programmaticBreakdown); @@ -41,22 +46,21 @@ export function renderMarkdownReport(summary: BatchSummary): string { return lines.join('\n'); } -function renderTopVerdict(summary: BatchSummary): string { - const mcps = summary.mcpOrder; +function renderTopVerdict(summary: BatchSummary, columns: ColumnKey[]): string { const baseline = summary.baseline; - // Header: | Scenario | MCP1 | MCP2 | ... | Δ₁ | Δ₂ | ... - const mcpHeaders = mcps.map(m => m).join(' | '); - const challengers = mcps.filter(m => m !== baseline); + // Header: | Scenario | Col1 | Col2 | ... | Δ₁ | Δ₂ | ... + const colHeaders = columns.join(' | '); + const challengers = columns.filter(m => m !== baseline); const deltaHeaders = challengers.length > 0 ? ' | ' + challengers.map(m => `Δ (${m})`).join(' | ') : ''; - const header = `| Scenario | ${mcpHeaders}${deltaHeaders} |`; - const sep = '|---' + '|---'.repeat(mcps.length + challengers.length) + '|'; + const header = `| Scenario | ${colHeaders}${deltaHeaders} |`; + const sep = '|---' + '|---'.repeat(columns.length + challengers.length) + '|'; const rows = [header, sep]; for (const s of summary.scenarios) { - const mcpCells = mcps.map(m => { + const colCells = columns.map(m => { const c = s.cells[m]; return c ? pct(c.combinedScore.mean) : '—'; }); @@ -66,7 +70,7 @@ function renderTopVerdict(summary: BatchSummary): string { ? signedPct(d.combinedScore) : '—'; }); - const allCells = [...mcpCells, ...deltaCells]; + const allCells = [...colCells, ...deltaCells]; rows.push(`| ${s.scenario} | ${allCells.join(' | ')} |`); } return ['### Top-line verdict', '', ...rows].join('\n'); @@ -74,36 +78,37 @@ function renderTopVerdict(summary: BatchSummary): string { function renderScenarioTable( scenario: ScenarioSummary, - mcpOrder: McpKind[], + columns: ColumnKey[], ): string { const baseline = scenario.baseline; - const challengers = mcpOrder.filter(m => m !== baseline); + const challengers = columns.filter(m => m !== baseline); const hasDelta = challengers.length > 0; - // Build dynamic headers: | Metric | MCP1 | MCP2 | ... | Δ₁ | ... - const mcpHeaders = mcpOrder.map(m => m).join(' | '); + // Build dynamic headers: | Metric | Col1 | Col2 | ... | Δ₁ | ... + const colHeaders = columns.join(' | '); const deltaHeaders = hasDelta ? ' | ' + challengers.map(m => `Δ (${m})`).join(' | ') : ''; - const header = `| Metric | ${mcpHeaders}${deltaHeaders} |`; - const sep = - '|---' + '|---'.repeat(mcpOrder.length + challengers.length) + '|'; + const header = `| Metric | ${colHeaders}${deltaHeaders} |`; + const sep = '|---' + '|---'.repeat(columns.length + challengers.length) + '|'; const rows = [header, sep]; - const cellFor = (m: McpKind): CellSummary | undefined => scenario.cells[m]; - const deltaFor = (m: McpKind): DeltaSummary | undefined => scenario.deltas[m]; + const cellFor = (col: ColumnKey): CellSummary | undefined => + scenario.cells[col]; + const deltaFor = (col: ColumnKey): DeltaSummary | undefined => + scenario.deltas[col]; function addRow( label: string, valueFn: (c: CellSummary | undefined) => string, deltaFn?: (d: DeltaSummary | undefined) => string, ) { - const mcpCells = mcpOrder.map(m => valueFn(cellFor(m))); + const colCells = columns.map(m => valueFn(cellFor(m))); const deltaCells = hasDelta ? challengers.map(m => (deltaFn ? deltaFn(deltaFor(m)) : '—')) : []; - rows.push(`| ${label} | ${[...mcpCells, ...deltaCells].join(' | ')} |`); + rows.push(`| ${label} | ${[...colCells, ...deltaCells].join(' | ')} |`); } addRow( @@ -152,7 +157,7 @@ function renderScenarioTable( function renderJudgeBreakdown( scenario: ScenarioSummary, - mcpOrder: McpKind[], + columns: ColumnKey[], ): string | null { const allCriteria = new Set(); for (const cell of Object.values(scenario.cells)) { @@ -161,26 +166,26 @@ function renderJudgeBreakdown( } if (allCriteria.size === 0) return null; - const mcpHeaders = mcpOrder.join(' | '); + const colHeaders = columns.join(' | '); const rows = [ '#### Judge per-criterion (mean 0–5)', '', - `| Criterion | ${mcpHeaders} |`, - '|---' + '|---'.repeat(mcpOrder.length) + '|', + `| Criterion | ${colHeaders} |`, + '|---' + '|---'.repeat(columns.length) + '|', ]; for (const id of [...allCriteria].sort()) { - const mcpCells = mcpOrder.map(m => { + const colCells = columns.map(m => { const v = scenario.cells[m]?.judge.perCriterion[id]; return fmtScore(v); }); - rows.push(`| ${id} | ${mcpCells.join(' | ')} |`); + rows.push(`| ${id} | ${colCells.join(' | ')} |`); } return rows.join('\n'); } function renderProgrammaticBreakdown( scenario: ScenarioSummary, - mcpOrder: McpKind[], + columns: ColumnKey[], ): string | null { const allChecks = new Set(); for (const cell of Object.values(scenario.cells)) { @@ -189,23 +194,23 @@ function renderProgrammaticBreakdown( } if (allChecks.size === 0) return null; - const mcpHeaders = mcpOrder.join(' | '); + const colHeaders = columns.join(' | '); const rows = [ '#### Programmatic per-check (pass rate)', '', 'Pass rate = positive checks matched + negative checks not matched.', '', - `| Check | ${mcpHeaders} |`, - '|---' + '|---'.repeat(mcpOrder.length) + '|', + `| Check | ${colHeaders} |`, + '|---' + '|---'.repeat(columns.length) + '|', ]; for (const id of [...allChecks].sort()) { - const mcpCells = mcpOrder.map(m => { + const colCells = columns.map(m => { const v = scenario.cells[m]?.programmatic.perCheck[id]; return fmtRate(v); }); const isNegative = id.startsWith('false_'); const label = isNegative ? `${id} (neg)` : id; - rows.push(`| ${label} | ${mcpCells.join(' | ')} |`); + rows.push(`| ${label} | ${colCells.join(' | ')} |`); } return rows.join('\n'); } diff --git a/packages/hdx-eval/src/reports/store.ts b/packages/hdx-eval/src/reports/store.ts index ebc5e41aa5..3bee95111f 100644 --- a/packages/hdx-eval/src/reports/store.ts +++ b/packages/hdx-eval/src/reports/store.ts @@ -1,12 +1,37 @@ -import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; import { join } from 'path'; import type { GradeRecord } from '../grading/types'; import type { McpKind, RunRecord } from '../harness/types'; +import { isModelSubdir, isRunJson, safeReaddir } from '../runs/path'; import { SCENARIO_NAMES } from '../scenarios'; import { buildAggregate, type GradedRunPair } from './aggregate'; import { renderMarkdownReport } from './markdown'; +function collectRunGradePairs(dir: string, pairs: GradedRunPair[]): void { + for (const file of safeReaddir(dir)) { + if (isRunJson(file)) { + const runPath = join(dir, file); + const gradePath = runPath.replace(/\.json$/, '.grade.json'); + if (!existsSync(gradePath)) continue; + try { + const run = JSON.parse(readFileSync(runPath, 'utf8')) as RunRecord; + const grade = JSON.parse( + readFileSync(gradePath, 'utf8'), + ) as GradeRecord; + pairs.push({ run, grade }); + } catch { + // skip malformed + } + } + } +} + +/** + * Load graded pairs from a batch. Supports both the new + * `///.json` layout and the legacy + * `//.json` layout. + */ function loadGradedPairs(batchDir: string): GradedRunPair[] { const pairs: GradedRunPair[] = []; for (const scenario of safeReaddir(batchDir)) { @@ -14,20 +39,12 @@ function loadGradedPairs(batchDir: string): GradedRunPair[] { const sceneDir = join(batchDir, scenario); for (const mcp of safeReaddir(sceneDir)) { const mcpDir = join(sceneDir, mcp); - for (const file of safeReaddir(mcpDir)) { - if (!file.endsWith('.json')) continue; - if (file.endsWith('.grade.json')) continue; - const runPath = join(mcpDir, file); - const gradePath = runPath.replace(/\.json$/, '.grade.json'); - if (!existsSync(gradePath)) continue; - try { - const run = JSON.parse(readFileSync(runPath, 'utf8')) as RunRecord; - const grade = JSON.parse( - readFileSync(gradePath, 'utf8'), - ) as GradeRecord; - pairs.push({ run, grade }); - } catch { - // skip malformed + // Collect runs directly in mcpDir (legacy layout). + collectRunGradePairs(mcpDir, pairs); + // Also check subdirectories (new model layout). + for (const entry of safeReaddir(mcpDir)) { + if (isModelSubdir(mcpDir, entry)) { + collectRunGradePairs(join(mcpDir, entry), pairs); } } } @@ -50,11 +67,3 @@ export function writeBatchSummary( return { jsonPath, mdPath: outPath, pairsCount: pairs.length }; } - -function safeReaddir(path: string): string[] { - try { - return readdirSync(path); - } catch { - return []; - } -} diff --git a/packages/hdx-eval/src/runs/instrument.ts b/packages/hdx-eval/src/runs/instrument.ts index fd9b24f1a3..9ac688d2d6 100644 --- a/packages/hdx-eval/src/runs/instrument.ts +++ b/packages/hdx-eval/src/runs/instrument.ts @@ -1,12 +1,12 @@ import type { ClickHouseClient } from '@clickhouse/client'; -import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; import { join } from 'path'; import { createEvalClient, defaultClickHouseUrl } from '../clickhouse/client'; import { scenarioTables } from '../clickhouse/schema'; import type { RunRecord } from '../harness/types'; import { SCENARIO_NAMES } from '../scenarios'; -import { runsRoot } from './path'; +import { isModelSubdir, isRunJson, runsRoot, safeReaddir } from './path'; const QUERY_LOG_BUFFER_MS = 5_000; @@ -287,16 +287,27 @@ export async function instrumentBatch( const sceneDir = join(resolved, scenario); for (const mcp of safeReaddir(sceneDir)) { const mcpDir = join(sceneDir, mcp); - for (const file of safeReaddir(mcpDir)) { - if (!file.endsWith('.json')) continue; - if (file.endsWith('.grade.json') || file.endsWith('.timing.json')) { - continue; + for (const entry of safeReaddir(mcpDir)) { + if (isRunJson(entry)) { + // Legacy layout: //.json + const timing = await instrumentRun({ + runPath: join(mcpDir, entry), + client, + }); + out.push(timing); + } else if (isModelSubdir(mcpDir, entry)) { + // New layout: ///.json + const modelDir = join(mcpDir, entry); + for (const file of safeReaddir(modelDir)) { + if (isRunJson(file)) { + const timing = await instrumentRun({ + runPath: join(modelDir, file), + client, + }); + out.push(timing); + } + } } - const timing = await instrumentRun({ - runPath: join(mcpDir, file), - client, - }); - out.push(timing); } } } @@ -306,14 +317,6 @@ export async function instrumentBatch( } } -function safeReaddir(path: string): string[] { - try { - return readdirSync(path); - } catch { - return []; - } -} - export function summarizeTimingRecord(t: TimingRecord): string { const wall = (t.totalWallMs / 1000).toFixed(1); const server = (t.totalServerMs / 1000).toFixed(2); diff --git a/packages/hdx-eval/src/runs/path.ts b/packages/hdx-eval/src/runs/path.ts index b69c48a8f6..723aa6c9c2 100644 --- a/packages/hdx-eval/src/runs/path.ts +++ b/packages/hdx-eval/src/runs/path.ts @@ -1,3 +1,4 @@ +import { readdirSync, statSync } from 'fs'; import { resolve } from 'path'; import type { McpKind } from '../harness/types'; @@ -10,10 +11,22 @@ export function batchDirName(date: Date = new Date()): string { return date.toISOString().replace(/[:.]/g, '-'); } +/** + * Sanitize a model ID for use as a directory name. Replaces characters + * that are problematic in file paths (`:`, `/`, `\`, `.`) with dashes. + * + * Dots are replaced to avoid confusion with file extensions when listing + * directory entries alongside run files like `0.json`. + */ +export function modelDirName(model: string): string { + return model.replace(/[:/\\.]/g, '-'); +} + export function runFilePath(args: { batchDir: string; scenario: string; mcp: McpKind; + model: string; runIndex: number; }): string { return resolve( @@ -21,6 +34,48 @@ export function runFilePath(args: { args.batchDir, args.scenario, args.mcp, + modelDirName(args.model), `${args.runIndex}.json`, ); } + +// --------------------------------------------------------------------------- +// Shared filesystem helpers — used by store, instrument, grade, reports, and +// the viewer. Centralised here to avoid duplication. +// --------------------------------------------------------------------------- + +/** + * Returns true if `file` is a numeric-indexed run JSON filename (e.g. + * `0.json`, `12.json`). Excludes sidecar files (`.grade.json`, + * `.timing.json`) and any other non-run JSON by requiring the `^\d+\.json$` + * pattern. + */ +export function isRunJson(file: string): boolean { + return /^\d+\.json$/.test(file); +} + +/** + * Like `readdirSync` but returns `[]` on any error (missing dir, permission + * issues, etc.). + */ +export function safeReaddir(dirPath: string): string[] { + try { + return readdirSync(dirPath); + } catch { + return []; + } +} + +/** + * Returns true when `entry` inside `parentDir` is a subdirectory (i.e. a + * model directory in the new layout), as opposed to a file. Uses `stat` + * instead of a heuristic so model names containing dots are handled + * correctly. + */ +export function isModelSubdir(parentDir: string, entry: string): boolean { + try { + return statSync(resolve(parentDir, entry)).isDirectory(); + } catch { + return false; + } +} diff --git a/packages/hdx-eval/src/runs/store.ts b/packages/hdx-eval/src/runs/store.ts index 8f58eb3499..8773a4b16c 100644 --- a/packages/hdx-eval/src/runs/store.ts +++ b/packages/hdx-eval/src/runs/store.ts @@ -2,7 +2,13 @@ import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'; import { dirname, join } from 'path'; import type { RunRecord } from '../harness/types'; -import { runFilePath, runsRoot } from './path'; +import { + isModelSubdir, + isRunJson, + runFilePath, + runsRoot, + safeReaddir, +} from './path'; export function writeRun(args: { record: RunRecord; @@ -12,6 +18,7 @@ export function writeRun(args: { batchDir: args.batchDir, scenario: args.record.scenario, mcp: args.record.mcp, + model: args.record.model, runIndex: args.record.runIndex, }); mkdirSync(dirname(path), { recursive: true }); @@ -31,34 +38,34 @@ export function listBatches(): string[] { } } +/** + * List run JSON files in a batch. Supports both the new + * `///.json` layout and the legacy + * `//.json` layout (for batches created before + * the multi-model feature). + */ export function listRunsInBatch(batchDir: string): string[] { const root = join(runsRoot(), batchDir); const out: string[] = []; - try { - for (const scenario of readdirSync(root)) { - const sceneDir = join(root, scenario); - try { - for (const mcp of readdirSync(sceneDir)) { - const mcpDir = join(sceneDir, mcp); - try { - for (const file of readdirSync(mcpDir)) { - if ( - file.endsWith('.json') && - !file.endsWith('.grade.json') && - !file.endsWith('.timing.json') - ) - out.push(join(mcpDir, file)); + for (const scenario of safeReaddir(root)) { + const sceneDir = join(root, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const entry of safeReaddir(mcpDir)) { + if (isRunJson(entry)) { + // Legacy layout: //.json + out.push(join(mcpDir, entry)); + } else if (isModelSubdir(mcpDir, entry)) { + // New layout: ///.json + const modelDir = join(mcpDir, entry); + for (const file of safeReaddir(modelDir)) { + if (isRunJson(file)) { + out.push(join(modelDir, file)); } - } catch { - // skip } } - } catch { - // skip } } - } catch { - // none } return out.sort(); } From defbe1f92e0d643ef3d840c8bcfdae711f0cbd63 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:18:53 -0400 Subject: [PATCH 055/728] Add raw SQL chart run shortcut (#2458) Co-authored-by: Cursor Agent Co-authored-by: Aaron Knudtson --- .changeset/raw-sql-chart-shortcut.md | 5 +++++ .../ChartEditor/RawSqlChartEditor.tsx | 1 + .../src/components/SQLEditor/SQLEditor.tsx | 22 ++++++++++++++++++- .../SQLEditor/__tests__/SQLEditor.test.tsx | 16 ++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 .changeset/raw-sql-chart-shortcut.md create mode 100644 packages/app/src/components/SQLEditor/__tests__/SQLEditor.test.tsx diff --git a/.changeset/raw-sql-chart-shortcut.md b/.changeset/raw-sql-chart-shortcut.md new file mode 100644 index 0000000000..4fb19b88d7 --- /dev/null +++ b/.changeset/raw-sql-chart-shortcut.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +Add Cmd/Ctrl+Enter support for running raw SQL chart queries from the SQL editor. diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index fb3a441601..badb7389b8 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -262,6 +262,7 @@ export default function RawSqlChartEditor({ placeholder={placeholderSQl} tableConnections={tableConnections} additionalCompletions={additionalCompletions} + onSubmit={onSubmit} />
diff --git a/packages/app/src/components/SQLEditor/SQLEditor.tsx b/packages/app/src/components/SQLEditor/SQLEditor.tsx index 4fef4ad4e2..20bbe3bc1e 100644 --- a/packages/app/src/components/SQLEditor/SQLEditor.tsx +++ b/packages/app/src/components/SQLEditor/SQLEditor.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef } from 'react'; +import { useCallback, useEffect, useMemo, useRef } from 'react'; import { useController, UseControllerProps } from 'react-hook-form'; import { acceptCompletion } from '@codemirror/autocomplete'; import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata'; @@ -7,6 +7,7 @@ import CodeMirror, { Compartment, EditorView, keymap, + Prec, ReactCodeMirrorRef, } from '@uiw/react-codemirror'; @@ -30,8 +31,17 @@ type SQLEditorProps = { additionalCompletions?: SQLCompletion[]; dateRange?: [Date, Date]; timestampValueExpression?: string; + onSubmit?: () => void; }; +export const createRunQueryKeyBinding = (onSubmit: () => void) => ({ + key: 'Mod-Enter', + run: () => { + onSubmit(); + return true; + }, +}); + export default function SQLEditor({ onChange, placeholder, @@ -42,11 +52,20 @@ export default function SQLEditor({ additionalCompletions, dateRange, timestampValueExpression, + onSubmit, }: SQLEditorProps) { const { colorScheme } = useMantineColorScheme(); const ref = useRef(null); const compartmentRef = useRef(new Compartment()); + const runQueryKeymap = useMemo( + () => + onSubmit == null + ? [] + : [Prec.highest(keymap.of([createRunQueryKeyBinding(onSubmit)]))], + [onSubmit], + ); + const { data: fields } = useMultipleAllFields(tableConnections ?? [], { dateRange, timestampValueExpression, @@ -111,6 +130,7 @@ export default function SQLEditor({ upperCaseKeywords: true, }), ), + ...runQueryKeymap, keymap.of([ { key: 'Tab', diff --git a/packages/app/src/components/SQLEditor/__tests__/SQLEditor.test.tsx b/packages/app/src/components/SQLEditor/__tests__/SQLEditor.test.tsx new file mode 100644 index 0000000000..3fd68f392c --- /dev/null +++ b/packages/app/src/components/SQLEditor/__tests__/SQLEditor.test.tsx @@ -0,0 +1,16 @@ +import { createRunQueryKeyBinding } from '../SQLEditor'; + +jest.mock('@/hooks/useMetadata', () => ({ + useMultipleAllFields: jest.fn().mockReturnValue({ data: [] }), +})); + +describe('SQLEditor', () => { + it('creates a Cmd/Ctrl+Enter key binding that submits the query', () => { + const onSubmit = jest.fn(); + const binding = createRunQueryKeyBinding(onSubmit); + + expect(binding.key).toBe('Mod-Enter'); + expect(binding.run()).toBe(true); + expect(onSubmit).toHaveBeenCalledTimes(1); + }); +}); From 31b87816b049a7848aed363472a64a7f5d43c49a Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 12 Jun 2026 20:48:25 -0700 Subject: [PATCH 056/728] feat(chart-explorer): duplicate a series in the chart builder (#2453) Add a Duplicate control to each series row in the chart builder so a near-identical series (for example avg and p95 of the same column, same WHERE) can be built with one click instead of re-entering every field. HDX-4532 ## Summary The chart builder's **Add Series** button always creates a blank series, so a series that differs only slightly from an existing one has to be re-typed field by field. This adds a **Duplicate** button (copy icon) to each series row's action group, next to Move up / Move down / Remove Series. Clicking it inserts a deep copy of that series directly below, carrying the aggregation function, value expression, WHERE, alias, metric type/name, and per-series number format. **Add Series** is unchanged and still produces a blank series. The control mirrors the existing dashboard-tile Duplicate button (`DBDashboardPage`, same `IconCopy`), so it adds no new UI vocabulary, and it matches the per-query duplicate pattern in comparable query builders. ## Changes - `EditTimeChartForm`: add a `duplicateSeries` handler backed by `useFieldArray`'s `insert`, deep-cloning the source row with `structuredClone` so the copy is independent of the original. - `ChartEditorControls`: thread the handler through, and reuse a single `canAddSeries` predicate to gate both **Add Series** and the new button, so Duplicate is hidden for Number / Pie / Heatmap (display types that only allow one series). - `ChartSeriesEditor`: render the Duplicate button. It shows for every row, including a lone series, so a second series can be seeded from the first. ## Why no auto-run Duplicating does not re-run the query, matching the existing Add Series / Remove / Move behavior. The copy is identical until you edit it, so running immediately would just draw a duplicate line. Hit Run after editing the copy. ## Test plan - [x] make ci-lint (5 projects, no errors) - [x] app unit suite (`nx @hyperdx/app:ci:unit`): 1935 pass, including 3 new duplicate-series cases (insert below, save round-trip, copy independence) - [x] knip (no unused exports) - [x] manual on a live stack: chart explorer, duplicate a series, confirm a second identical row appears below - [x] confirmed the button shows on Line/Bar and Table, and is hidden on Number / Pie / Heatmap - [x] light and dark theme [ui-states: allow] the Duplicate button is a static control with no empty/loading/error state of its own. [viewport: allow] the button sits in the existing series-row action group and inherits its layout; no new responsive behavior. ### Screenshots Verified in the chart explorer in both themes. The Duplicate (copy) icon sits in each series row's action group, between Move and Remove Series: - Light, one series: the copy icon appears next to the per-series format icon. - Light, after Duplicate: a second identical `count` series is inserted below, and the As Ratio toggle appears (shown at two series). - Dark: identical layout, confirming theme parity. (before/after PNGs captured during verification) ### What's not in this PR - Copy/paste of a series across different charts (within one chart's series list only). --- .../duplicate-series-in-chart-builder.md | 12 +++ .../ChartEditorControls.tsx | 46 ++++++----- .../DBEditTimeChartForm/ChartSeriesEditor.tsx | 23 +++++- .../DBEditTimeChartForm/EditTimeChartForm.tsx | 20 +++++ .../__tests__/DBEditTimeChartForm.test.tsx | 76 +++++++++++++++++++ .../e2e/components/ChartEditorComponent.ts | 8 ++ 6 files changed, 164 insertions(+), 21 deletions(-) create mode 100644 .changeset/duplicate-series-in-chart-builder.md diff --git a/.changeset/duplicate-series-in-chart-builder.md b/.changeset/duplicate-series-in-chart-builder.md new file mode 100644 index 0000000000..30b38ae6ba --- /dev/null +++ b/.changeset/duplicate-series-in-chart-builder.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': patch +--- + +feat(chart-explorer): duplicate a series in the chart builder + +Add a Duplicate button to each series row in the chart builder that inserts a +copy of that series directly below it, so building a near-identical variant +(for example avg and p95 of the same column) no longer requires re-entering +every field by hand. "Add Series" still creates a blank series. The copy +starts with an empty alias so it does not collide with the original's alias in +the generated SQL. diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx index 62948f7109..6b2e4574a4 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx @@ -48,6 +48,7 @@ type ChartEditorControlsProps = { append: (value: SavedChartConfigWithSelectArray['select'][number]) => void; removeSeries: (index: number) => void; swapSeries: (from: number, to: number) => void; + duplicateSeries: (index: number) => void; tableSource?: TSource; tableConnection: TableConnection; databaseName?: string; @@ -76,6 +77,7 @@ export function ChartEditorControls({ append, removeSeries, swapSeries, + duplicateSeries, tableSource, tableConnection, databaseName, @@ -94,6 +96,10 @@ export function ChartEditorControls({ openDisplaySettings, openHeatmapSettings, }: ChartEditorControlsProps) { + const canAddSeries = + displayType !== DisplayType.Number && + displayType !== DisplayType.Pie && + displayType !== DisplayType.Heatmap; const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = useState(false); @@ -159,6 +165,7 @@ export function ChartEditorControls({ onRemoveSeries={removeSeries} length={fields.length} onSwapSeries={swapSeries} + onDuplicateSeries={duplicateSeries} onSubmit={onSubmit} setValue={setValue} connectionId={tableSource?.connection} @@ -168,6 +175,7 @@ export function ChartEditorControls({ showHaving={ fields.length === 1 && displayType === DisplayType.Table } + showDuplicate={canAddSeries} tableName={tableName ?? ''} tableSource={tableSource} errors={ @@ -239,26 +247,24 @@ export function ChartEditorControls({ - {displayType !== DisplayType.Number && - displayType !== DisplayType.Pie && - displayType !== DisplayType.Heatmap && ( - - )} + {canAddSeries && ( + + )} {fields.length == 2 && displayType !== DisplayType.Number && ( void; onSwapSeries: (from: number, to: number) => void; + onDuplicateSeries: (index: number) => void; onSubmit: () => void; setValue: UseFormSetValue; showGroupBy: boolean; showHaving: boolean; + showDuplicate: boolean; tableName: string; length: number; tableSource?: TSource; @@ -81,10 +88,12 @@ export function ChartSeriesEditor({ namePrefix, onRemoveSeries, onSwapSeries, + onDuplicateSeries, onSubmit, setValue, showGroupBy, showHaving, + showDuplicate, tableName: _tableName, parentRef, length, @@ -245,6 +254,18 @@ export function ChartSeriesEditor({ )} + {showDuplicate && ( + + )} {((index ?? -1) > 0 || length > 1) && ( - )} - + + ))}
otel_metrics_ts
+ metrics_ts
@@ -67,7 +67,7 @@ remote_read default - otel_metrics_ts
+ metrics_ts
 diff --git a/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql b/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql index 50323f3d05..48b8372b44 100644 --- a/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql +++ b/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql @@ -1,4 +1,4 @@ -- +goose Up -CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_ts +CREATE TABLE IF NOT EXISTS ${DATABASE}.metrics_ts ENGINE = TimeSeries SETTINGS allow_experimental_time_series_table = 1; diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 569720cf77..8f0bf68049 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -11,7 +11,7 @@ import { DashboardFilterType, MetricsDataType, NumberTileColorConditionSchema, - SearchConditionLanguageSchema, + SearchConditionTrimmedLanguageSchema, } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; @@ -125,7 +125,8 @@ const mcpTileSelectItemSchema = z .optional() .default('') .describe('Filter in Lucene syntax. Example: "level:error"'), - whereLanguage: SearchConditionLanguageSchema.optional().default('lucene'), + whereLanguage: + SearchConditionTrimmedLanguageSchema.optional().default('lucene'), alias: z .string() .optional() @@ -255,7 +256,7 @@ const mcpOnClickSearchSchema = z 'Use Lucene or SQL syntax matching `whereLanguage`. Prefer `filters` (below) ' + 'for simple equality; filters merge nicely on the destination.', ), - whereLanguage: SearchConditionLanguageSchema.describe( + whereLanguage: SearchConditionTrimmedLanguageSchema.describe( 'Filter language for `whereTemplate` and `filters` ("lucene" or "sql"). ' + 'Optional, but set it explicitly so the destination knows how to parse rendered ' + 'whereTemplate / filter values.', @@ -290,7 +291,7 @@ const mcpOnClickDashboardSchema = z "dashboard's global filter. Useful when the target dashboard exposes a single " + 'global scope rather than per-tile filters.', ), - whereLanguage: SearchConditionLanguageSchema.describe( + whereLanguage: SearchConditionTrimmedLanguageSchema.describe( 'Filter language for `whereTemplate` and `filters` ("lucene" or "sql"). ' + 'Optional, but set it explicitly so the destination knows how to parse rendered ' + 'whereTemplate / filter values.', @@ -621,7 +622,8 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ .describe( 'Row-level filter applied before bucketing. Example: "level:error"', ), - whereLanguage: SearchConditionLanguageSchema.optional().default('lucene'), + whereLanguage: + SearchConditionTrimmedLanguageSchema.optional().default('lucene'), numberFormat: mcpNumberFormatSchema .optional() .describe( @@ -640,7 +642,8 @@ const mcpSearchTileSchema = mcpTileLayoutSchema.extend({ .optional() .default('') .describe('Filter in Lucene syntax. Example: "level:error"'), - whereLanguage: SearchConditionLanguageSchema.optional().default('lucene'), + whereLanguage: + SearchConditionTrimmedLanguageSchema.optional().default('lucene'), select: z .string() .optional() @@ -865,7 +868,7 @@ const mcpDashboardFilterSchema = z .describe( 'Optional WHERE clause scoping the dropdown values (e.g. "level:error" in Lucene).', ), - whereLanguage: SearchConditionLanguageSchema.describe( + whereLanguage: SearchConditionTrimmedLanguageSchema.describe( 'Filter language for `where` ("lucene" or "sql"). Optional, but set it explicitly.', ), appliesToSourceIds: z diff --git a/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts b/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts index d1aecf76b9..dc48dd8499 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts @@ -147,7 +147,7 @@ describe('prometheus router', () => { const { agent, team } = await getLoggedInAgent(server); const conn = await seedClickHouseConnection(team._id); - // ClickHouse path: will likely fail with 400 because otel_metrics_ts + // ClickHouse path: will likely fail with 400 because metrics_ts // is not seeded in the test CH, but the routing decision is what we // care about — fetch must not be called. await agent.get('/v1/prometheus/query_range').query({ @@ -172,6 +172,7 @@ describe('prometheus router', () => { end: '1700000000', step: '1s', connectionId: conn._id.toString(), + table: 'metrics_ts', }) .expect(400); expect(res.body).toMatchObject({ diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index b6532ecede..07f446816f 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -296,7 +296,14 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { const end = parseTimestamp(params.end); const step = parseDuration(params.step ?? '60s'); const database = params.database ?? 'default'; - const table = params.table ?? 'otel_metrics_ts'; + const table = params.table; + if (!table) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: `table parameter required for querying clickhouse via promql`, + }); + } if (step <= 0 || (end - start) / step > PROMETHEUS_MAX_RESOLUTION) { return res.status(400).json({ @@ -414,7 +421,14 @@ const queryHandler: express.RequestHandler = async (req, res) => { backend = 'clickhouse'; const time = params.time ? parseTimestamp(params.time) : undefined; const database = params.database ?? 'default'; - const table = params.table ?? 'otel_metrics_ts'; + const table = params.table; + if (!table) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: `table parameter required for querying clickhouse via promql`, + }); + } const client = new ClickhouseClient({ host: connection.host, @@ -525,7 +539,14 @@ router.get('/label/:name/values', async (req, res) => { backend = 'clickhouse'; const database = params.database ?? 'default'; - const table = params.table ?? 'otel_metrics_ts'; + const table = params.table; + if (!table) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: `table parameter required for querying clickhouse via promql`, + }); + } const client = new ClickhouseClient({ host: connection.host, diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index dbae6dcc13..a35ab57c7c 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -22,6 +22,7 @@ import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata'; import { convertToDashboardTemplate, displayTypeSupportsBuilderAlerts, + displayTypeSupportsPromQLAlerts, displayTypeSupportsRawSqlAlerts, Granularity, } from '@hyperdx/common-utils/dist/core/utils'; @@ -651,9 +652,12 @@ const Tile = forwardRef( const hoverToolbar = useMemo(() => { const isRawSql = isRawSqlSavedChartConfig(chart.config); + const isPromQL = isPromqlSavedChartConfig(chart.config); const displayTypeSupportsAlerts = isRawSql ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) - : displayTypeSupportsBuilderAlerts(chart.config.displayType); + : isPromQL + ? displayTypeSupportsPromQLAlerts(chart.config.displayType) + : displayTypeSupportsBuilderAlerts(chart.config.displayType); return ( setIsSourceSchemaPreviewOpen(true)} isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled(source)} disabled={!!presetSource} + allowedSourceKinds={[ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, + ]} /> )} diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 78f3aa8413..a79c782d5b 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -541,8 +541,8 @@ export const prometheusApi = { end: number; step: string; connectionId: string; - database: string; - table: string; + database?: string; + table?: string; }): Promise => prometheusFetch('v1/prometheus/query_range', { query: params.query, @@ -550,8 +550,8 @@ export const prometheusApi = { end: String(params.end), step: params.step, connectionId: params.connectionId, - database: params.database, - table: params.table, + ...(params.database ? { database: params.database } : {}), + ...(params.table ? { table: params.table } : {}), }), labelValues: (params: { diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index 6c6cab067c..bc3f05d461 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -175,7 +175,8 @@ export default function ChartDisplaySettingsDrawer({ // The series-limit CTE is only emitted for builder group-by time charts; // raw SQL configs author their own LIMIT logic directly. - const showSeriesLimit = isTimeChart && configType !== 'sql'; + const showSeriesLimit = + isTimeChart && configType !== 'sql' && configType !== 'promql'; // Group By column ordering only applies to builder table charts; raw SQL // configs let the user author whatever column order they want directly. diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 36d3e02712..8d05ec1a84 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -69,11 +69,25 @@ export const isRawSqlDisplayType = ( displayType === DisplayType.Pie || displayType === DisplayType.Number; +export const isPromqlDisplayType = ( + displayType: DisplayType | undefined, +): displayType is + | DisplayType.Table + | DisplayType.Line + | DisplayType.StackedBar + | DisplayType.Pie + | DisplayType.Number => + displayType === DisplayType.Table || + displayType === DisplayType.Line || + displayType === DisplayType.StackedBar || + displayType === DisplayType.Pie || + displayType === DisplayType.Number; + export function convertFormStateToSavedChartConfig( form: ChartEditorFormState, source: TSource | undefined, ): SavedChartConfig | undefined { - if (form.configType === 'promql') { + if (form.configType === 'promql' && isPromqlDisplayType(form.displayType)) { const promqlConfig: PromqlSavedChartConfig = { configType: 'promql', ...pick(form, [ @@ -85,8 +99,7 @@ export function convertFormStateToSavedChartConfig( 'compareToPreviousPeriod', 'fillNulls', 'alignDateRangeToGranularity', - 'alert', - 'step', + // 'alert', // TODO: Support alerts on PromQL (HDX-4636) ]), promqlExpression: form.promqlExpression ?? '', connection: form.connection ?? '', @@ -152,7 +165,7 @@ export function convertFormStateToChartConfig( dateRange: ChartConfigWithDateRange['dateRange'], source: TSource | undefined, ): ChartConfigWithDateRange | undefined { - if (form.configType === 'promql') { + if (form.configType === 'promql' && isPromqlDisplayType(form.displayType)) { const promqlConfig: PromqlChartConfig = { configType: 'promql', ...pick(form, [ @@ -163,7 +176,6 @@ export function convertFormStateToChartConfig( 'compareToPreviousPeriod', 'fillNulls', 'alignDateRangeToGranularity', - 'step', ]), promqlExpression: form.promqlExpression ?? '', connection: source?.connection ?? form.connection ?? '', diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index e31ae52c3b..889c5df4c3 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -55,6 +55,7 @@ import { convertFormStateToChartConfig, convertFormStateToSavedChartConfig, convertSavedChartConfigToFormState, + isPromqlDisplayType, isRawSqlDisplayType, validateChartForm, } from '@/components/ChartEditor/utils'; @@ -210,7 +211,8 @@ export default function EditTimeChartForm({ const chartConfigAlert = chartConfig.alert; const isRawSqlInput = configType === 'sql' && isRawSqlDisplayType(displayType); - const isPromqlInput = configType === 'promql'; + const isPromqlInput = + configType === 'promql' && isPromqlDisplayType(displayType); const { data: tableSource } = useSource({ id: sourceId }); const databaseName = tableSource?.from.databaseName; diff --git a/packages/app/src/components/DBEditTimeChartForm/utils.ts b/packages/app/src/components/DBEditTimeChartForm/utils.ts index 883fb01cf0..7a61bd46ec 100644 --- a/packages/app/src/components/DBEditTimeChartForm/utils.ts +++ b/packages/app/src/components/DBEditTimeChartForm/utils.ts @@ -225,7 +225,7 @@ export function buildChartConfigForExplanations({ } : undefined; - if (!config || isRawSqlChartConfig(config) || isPromqlChartConfig(config)) { + if (!config || !isBuilderChartConfig(config)) { return undefined; } diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index e2cb3c3c14..ac2a78c82c 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -338,8 +338,8 @@ export function useQueriedChartConfig( end: endSec, step: stepStr, connectionId: config.connection, - database: config.from?.databaseName ?? 'default', - table: config.from?.tableName ?? 'otel_metrics_ts', + database: config.from?.databaseName, + table: config.from?.tableName, }); if (resp.status !== 'success' || !resp.data) { diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index 710eecb930..0d952644a5 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -26,7 +26,7 @@ type FilterSourceKey = { sourceId: string; metricType?: string; where: string; - whereLanguage: 'sql' | 'lucene' | 'promql'; + whereLanguage: 'sql' | 'lucene'; }; const filterToKey = (filter: DashboardFilter): string => diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index e8053a33cf..8e7e5d4c23 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1202,6 +1202,14 @@ export function displayTypeSupportsBuilderAlerts( ); } +export function displayTypeSupportsPromQLAlerts( + displayType: DisplayType | undefined, +): boolean { + // TODO: Support alerts for PromQL (HDX-4636) + // This looks funky, just doing it to satisfy knip + return displayType ? false : false; +} + export function validateRawSqlForAlert(chartConfig: RawSqlChartConfig): { errors: string[]; warnings: string[]; diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index f9f3dda896..f7ea805eda 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -147,9 +147,15 @@ export const SQLIntervalSchema = z .string() .regex(/^\d+ (second|minute|hour|day)$/); export const SearchConditionSchema = z.string(); -export const SearchConditionLanguageSchema = z - .enum(['sql', 'lucene', 'promql']) - .optional(); +const SearchConditionRequiredLanguageSchema = z.enum([ + 'sql', + 'lucene', + 'promql', +]); +export const SearchConditionLanguageSchema = + SearchConditionRequiredLanguageSchema.optional(); +export const SearchConditionTrimmedLanguageSchema = + SearchConditionRequiredLanguageSchema.exclude(['promql']).optional(); export const AggregateFunctionSchema = z.enum([ 'avg', 'count', @@ -1486,7 +1492,7 @@ export const DashboardFilterSchema = z.object({ source: z.string().min(1), sourceMetricType: z.nativeEnum(MetricsDataType).optional(), where: z.string().optional(), - whereLanguage: SearchConditionLanguageSchema, + whereLanguage: SearchConditionTrimmedLanguageSchema, // Sources this filter applies to. Undefined / missing means the filter // applies to all tiles. appliesToSourceIds: z.array(z.string().min(1)).optional(), diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index cb0877bf2e..f58608657b 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -412,7 +412,7 @@ func removeCompatLogsSchema(tempDir string) error { // removePromqlSchema removes the experimental TimeSeries-engine schema from // the temp directory so it is only created when PromQL support is opted into -// via ENABLE_PROMQL=true. Keeps the experimental engine and otel_metrics_ts +// via ENABLE_PROMQL=true. Keeps the experimental engine and metrics_ts // table out of deployments that have not enabled the feature. func removePromqlSchema(tempDir string) error { promqlPath := filepath.Join(tempDir, "00008_otel_metrics_timeseries.sql") From 5e8af09be5102ebd111243cc5837e44dcc144bd3 Mon Sep 17 00:00:00 2001 From: Vineet Ahirkar Date: Sat, 27 Jun 2026 01:16:45 +0530 Subject: [PATCH 110/728] Migrate development environment to Turbopack (#2385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Transition the local development server from Webpack to Turbopack to significantly improve build performance and hot-reloading speed. Also update global style declarations to ensure compatibility with the new bundler's CSS compilation requirements. ### Screenshots or video N/A — non-UI change ### How to test on Vercel preview N/A — non-UI change ### References - Linear Issue: N/A - Related PRs: N/A --- .changeset/solid-pigs-rhyme.md | 6 ++++ .gitignore | 2 +- packages/app/next.config.mjs | 21 ++++++------- packages/app/package.json | 5 +-- packages/app/pages/api/[...all].ts | 13 ++++---- packages/app/styles/SearchPage.module.scss | 36 ++++++++++------------ packages/app/types/hyperdx-api.d.ts | 6 ++++ 7 files changed, 50 insertions(+), 39 deletions(-) create mode 100644 .changeset/solid-pigs-rhyme.md create mode 100644 packages/app/types/hyperdx-api.d.ts diff --git a/.changeset/solid-pigs-rhyme.md b/.changeset/solid-pigs-rhyme.md new file mode 100644 index 0000000000..344bcb74b8 --- /dev/null +++ b/.changeset/solid-pigs-rhyme.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/app': patch +--- + +Transition the local development server from Webpack to Turbopack to +significantly improve build performance and hot-reloading speed. diff --git a/.gitignore b/.gitignore index e38ead1f21..31c6b70198 100644 --- a/.gitignore +++ b/.gitignore @@ -60,7 +60,7 @@ package-lock.json # build output **/dist **/build -**/tsconfig.tsbuildinfo +**/*.tsbuildinfo # jest coverage report **/coverage diff --git a/packages/app/next.config.mjs b/packages/app/next.config.mjs index fa2ee03efb..ef3a3fc3ee 100644 --- a/packages/app/next.config.mjs +++ b/packages/app/next.config.mjs @@ -37,17 +37,22 @@ const nextConfig = { '@opentelemetry/auto-instrumentations-node', '@hyperdx/node-opentelemetry', '@hyperdx/instrumentation-sentry-node', + // Outside of Vercel preview deployments, the `/api/[...all]` catch-all + // proxies to a separately-deployed API service and never imports the + // `@hyperdx/api` package at runtime. Mark it (and its subpaths) as a + // CommonJS external so production app builds (Docker fullstack image, + // standalone Next output) stay byte-for-byte equivalent to today and + // do not pull in passport-saml, mongoose, AWS SDK, etc. + ...(process.env.HDX_PREVIEW_INLINE_API !== 'true' ? ['@hyperdx/api'] : []), ], typescript: { tsconfigPath: 'tsconfig.build.json', }, - // NOTE: Using Webpack instead of Turbopack (Next.js 16 default) + // Dev uses Turbopack; production build uses Webpack (--webpack). // Reason: Turbopack has CSS module parsing issues with nested :global syntax // used in styles/SearchPage.module.scss and other SCSS files. - // The --webpack flag is added to dev and build scripts in package.json. - // TODO: Re-evaluate when Turbopack CSS module support improves - // Ignore otel pkgs warnings - // https://github.com/open-telemetry/opentelemetry-js/issues/4173#issuecomment-1822938936 + // TODO: single bundler when Turbopack CSS is solid. + // Ignore otel warnings (Webpack): https://github.com/open-telemetry/opentelemetry-js/issues/4173#issuecomment-1822938936 webpack: ( config, { buildId, dev, isServer, defaultLoaders, nextRuntime, webpack }, @@ -55,12 +60,6 @@ const nextConfig = { if (isServer) { config.ignoreWarnings = [{ module: /opentelemetry/ }]; - // Outside of Vercel preview deployments, the `/api/[...all]` catch-all - // proxies to a separately-deployed API service and never imports the - // `@hyperdx/api` package at runtime. Mark it (and its subpaths) as a - // CommonJS external so production app builds (Docker fullstack image, - // standalone Next output) stay byte-for-byte equivalent to today and - // do not pull in passport-saml, mongoose, AWS SDK, etc. if (process.env.HDX_PREVIEW_INLINE_API !== 'true') { config.externals = [ ...(config.externals ?? []), diff --git a/packages/app/package.json b/packages/app/package.json index 13280fc95a..92d9138243 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -6,9 +6,10 @@ "engines": { "node": ">=22.16.0" }, + "//bundler": "dev/dev:local run on Turbopack for fast local iteration; build/build:clickhouse run on Webpack (--webpack) for production parity. The two bundlers can diverge on CSS Modules/loaders/module resolution, so build failures may not reproduce in dev. See next.config.mjs for rationale.", "scripts": { - "dev": "npx dotenv -e .env.development -- next dev --webpack", - "dev:local": "NEXT_PUBLIC_IS_LOCAL_MODE=true npx dotenv -e .env.development -- next dev --webpack", + "dev": "npx dotenv -e .env.development -- next dev --turbopack", + "dev:local": "NEXT_PUBLIC_IS_LOCAL_MODE=true npx dotenv -e .env.development -- next dev --turbopack", "build": "next build --webpack", "build:clickhouse": "NEXT_PUBLIC_THEME=clickstack NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_PUBLIC_CLICKHOUSE_BUILD=true next build --webpack && node scripts/prepare-clickhouse-build-export.js", "run:clickhouse": "test -d out && npx rimraf tmp && mkdir tmp && cp -r out tmp/clickstack && echo 'visit http://localhost:3000/clickstack to start' && npx serve tmp -l 3000 || echo 'run build:clickhouse first'", diff --git a/packages/app/pages/api/[...all].ts b/packages/app/pages/api/[...all].ts index bfc3bd590a..e05f89d272 100644 --- a/packages/app/pages/api/[...all].ts +++ b/packages/app/pages/api/[...all].ts @@ -16,13 +16,14 @@ export const config = { // we proxy `/api/*` to a separately-deployed API service as before. const isInlineApi = process.env.HDX_PREVIEW_INLINE_API === 'true'; -export default (req: NextApiRequest, res: NextApiResponse) => { +export default async (req: NextApiRequest, res: NextApiResponse) => { if (isInlineApi) { - // Lazy require so non-preview production builds — where the webpack - // externals hook in next.config.mjs marks @hyperdx/api as external — - // never attempt to resolve a module that isn't bundled. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const inlineApi = require('@hyperdx/api/build/serverless'); + // Lazy require so non-preview production builds + const inlineApi = await import( + /* webpackIgnore: true */ + /* turbopackIgnore: true */ + '@hyperdx/api/build/serverless' + ); const handler = inlineApi.default ?? inlineApi; return handler(req, res); } diff --git a/packages/app/styles/SearchPage.module.scss b/packages/app/styles/SearchPage.module.scss index 3e74bd07e0..c617de5d10 100644 --- a/packages/app/styles/SearchPage.module.scss +++ b/packages/app/styles/SearchPage.module.scss @@ -14,28 +14,26 @@ overflow: hidden; z-index: 3; // higher z-index to be above other elements - :global { - .mantine-TextInput-wrapper { - background-color: transparent; - } + :global(.mantine-TextInput-wrapper) { + background-color: transparent; + } - .mantine-TextInput-input { - height: 20px; - min-height: 20px; - background-color: transparent; - color: var(--color-text-secondary); - font-size: var(--mantine-font-size-xs); + :global(.mantine-TextInput-input) { + height: 20px; + min-height: 20px; + background-color: transparent; + color: var(--color-text-secondary); + font-size: var(--mantine-font-size-xs); + } - &::placeholder { - color: var(--color-text-muted); - font-weight: bold; - } + :global(.mantine-TextInput-input::placeholder) { + color: var(--color-text-muted); + font-weight: bold; + } - &:focus { - border: none; - outline: none; - } - } + :global(.mantine-TextInput-input:focus) { + border: none; + outline: none; } } diff --git a/packages/app/types/hyperdx-api.d.ts b/packages/app/types/hyperdx-api.d.ts new file mode 100644 index 0000000000..9a63992d6b --- /dev/null +++ b/packages/app/types/hyperdx-api.d.ts @@ -0,0 +1,6 @@ +// The serverless entry point is only used in Vercel preview deployments +// (HDX_PREVIEW_INLINE_API=true). In all other builds the import is skipped +// at runtime via webpackIgnore/turbopackIgnore. This ambient declaration +// keeps TypeScript happy regardless of whether the API package has been +// built. +declare module '@hyperdx/api/build/serverless'; From 634101c338b0a23ef0f138c18da88e9c06f1351c Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 26 Jun 2026 14:07:32 -0600 Subject: [PATCH 111/728] chore: upgrade moduleResolution to NodeNext and simplify clickhouseProxy import (#2526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Upgrades the base `tsconfig.json` from `Node16` to `NodeNext` module resolution and reverts the `http-proxy-middleware` import in `clickhouseProxy.ts` from a lazy dynamic `import()` back to a static top-level import. ## Motivation After bumping `http-proxy-middleware` to v4 (ESM-only) in #2461, a dynamic import with a cached promise was used to keep CJS compatibility. With `moduleResolution: "NodeNext"`, TypeScript can resolve ESM packages via static imports, eliminating the need for the async wrapper pattern. Ref: https://github.com/hyperdxio/hyperdx/pull/2461#discussion_r3429496145 ## Changes - **`tsconfig.base.json`** — `module` and `moduleResolution` from `Node16` → `NodeNext` - **`clickhouseProxy.ts`** — Replace lazy `import()` + cached promise + async wrapper with a direct static `import { createProxyMiddleware }`. No functional behavior change — path rewriting, header injection, CORS handling, and error handling are identical. - **`jest.setup.ts`** — Add global `jest.mock('http-proxy-middleware')` since the ESM-only package can't be loaded by Jest's CJS module loader. No test exercises the proxy directly. ## Testing - `make ci-lint` — pass - `make ci-unit` — pass (all packages) - `make dev-int` — pass (61 suites, 1,387 tests) --- .changeset/nodenext-module-resolution.md | 5 + packages/api/jest.setup.ts | 6 + .../api/src/routers/api/clickhouseProxy.ts | 279 ++++++++---------- tsconfig.base.json | 4 +- 4 files changed, 138 insertions(+), 156 deletions(-) create mode 100644 .changeset/nodenext-module-resolution.md diff --git a/.changeset/nodenext-module-resolution.md b/.changeset/nodenext-module-resolution.md new file mode 100644 index 0000000000..f50ed4c1b0 --- /dev/null +++ b/.changeset/nodenext-module-resolution.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +chore: upgrade moduleResolution to NodeNext and simplify clickhouseProxy static import diff --git a/packages/api/jest.setup.ts b/packages/api/jest.setup.ts index 7549dd086e..05bb790c6a 100644 --- a/packages/api/jest.setup.ts +++ b/packages/api/jest.setup.ts @@ -1,5 +1,11 @@ jest.retryTimes(1, { logErrorsBeforeRetry: true }); +// http-proxy-middleware v4 is ESM-only and Jest's CJS module loader cannot +// load ESM packages. Auto-mock since no test exercises the proxy directly. +jest.mock('http-proxy-middleware', () => ({ + createProxyMiddleware: jest.fn(() => jest.fn()), +})); + // Suppress noisy console output during test runs. // - debug/info: ClickHouse query logging, server startup messages // - warn: expected column-not-found warnings from renderChartConfig on CTE tables diff --git a/packages/api/src/routers/api/clickhouseProxy.ts b/packages/api/src/routers/api/clickhouseProxy.ts index ad0c240686..8b18b481e9 100644 --- a/packages/api/src/routers/api/clickhouseProxy.ts +++ b/packages/api/src/routers/api/clickhouseProxy.ts @@ -1,5 +1,6 @@ import { sanitizeUrl } from '@braintree/sanitize-url'; import express, { RequestHandler, Response } from 'express'; +import { createProxyMiddleware } from 'http-proxy-middleware'; import { performance } from 'perf_hooks'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; @@ -188,165 +189,135 @@ const getConnection: RequestHandler = } }; -// http-proxy-middleware v4 is ESM-only, so we use a dynamic import with a -// cached promise to keep this CJS module compatible and avoid concurrent-init races. -let _proxyPromise: Promise | undefined; +const proxyMiddleware: RequestHandler = + // prettier-ignore-next-line + createProxyMiddleware({ + target: '', // doesn't matter. it should be overridden by the router + changeOrigin: true, + pathFilter: (path, _req) => { + return _req.method === 'GET' || _req.method === 'POST'; + }, + pathRewrite: function (path, req) { + const sanitizedPath = validateAndSanitizePath( + path.replace(/^\/clickhouse-proxy/, ''), + ); -function getProxyMiddleware(): Promise { - if (_proxyPromise) return _proxyPromise; + const parsedUrl = new URL(sanitizedPath, 'http://localhost'); + const { searchParams, pathname } = parsedUrl; + + // Append user email as custom ClickHouse setting for query log annotation if the prefix was set + const hyperdxSettingPrefix = req._hdx_connection?.hyperdxSettingPrefix; + if (hyperdxSettingPrefix) { + const userEmail = req.user?.email; + if (userEmail) { + const userSettingKey = `${hyperdxSettingPrefix}${CUSTOM_SETTING_KEY_SEP}${CUSTOM_SETTING_KEY_USER_SUFFIX}`; + searchParams.set(userSettingKey, userEmail); + } else { + logger.debug('hyperdxSettingPrefix set, no session user found'); + } + } - _proxyPromise = import('http-proxy-middleware').then( - ({ createProxyMiddleware }) => - createProxyMiddleware({ - target: '', // doesn't matter. it should be overridden by the router - changeOrigin: true, - pathFilter: (path, _req) => { - return _req.method === 'GET' || _req.method === 'POST'; - }, - pathRewrite: function (path, req) { - const sanitizedPath = validateAndSanitizePath( - path.replace(/^\/clickhouse-proxy/, ''), + return `${pathname}?${searchParams.toString()}`; + }, + router: _req => { + if (!_req._hdx_connection?.host) { + throw new Error('[createProxyMiddleware] Connection not found'); + } + return _req._hdx_connection.host; + }, + on: { + proxyReq: (proxyReq, _req, res) => { + // set user-agent to the hyperdx version identifier + proxyReq.setHeader('user-agent', `hyperdx ${CODE_VERSION}`); + + if (_req._hdx_connection?.username) { + proxyReq.setHeader( + 'X-ClickHouse-User', + _req._hdx_connection.username, ); - - const parsedUrl = new URL(sanitizedPath, 'http://localhost'); - const { searchParams, pathname } = parsedUrl; - - // Append user email as custom ClickHouse setting for query log annotation if the prefix was set - const hyperdxSettingPrefix = - req._hdx_connection?.hyperdxSettingPrefix; - if (hyperdxSettingPrefix) { - const userEmail = req.user?.email; - if (userEmail) { - const userSettingKey = `${hyperdxSettingPrefix}${CUSTOM_SETTING_KEY_SEP}${CUSTOM_SETTING_KEY_USER_SUFFIX}`; - searchParams.set(userSettingKey, userEmail); - } else { - logger.debug('hyperdxSettingPrefix set, no session user found'); - } + } + // Passwords can be empty + if (_req._hdx_connection?.password) { + proxyReq.setHeader('X-ClickHouse-Key', _req._hdx_connection.password); + } + + if (_req.method !== 'POST') { + console.error(`Unsupported method ${_req.method}`); + return res.sendStatus(405); + } + + let body = _req.body; + if (_req.headers['content-type'] === 'application/json') { + try { + body = JSON.stringify(body); + } catch (e) { + console.error(e); } + } - return `${pathname}?${searchParams.toString()}`; - }, - router: _req => { - if (!_req._hdx_connection?.host) { - throw new Error('[createProxyMiddleware] Connection not found'); - } - return _req._hdx_connection.host; - }, - on: { - proxyReq: (proxyReq, _req, res) => { - // set user-agent to the hyperdx version identifier - proxyReq.setHeader('user-agent', `hyperdx ${CODE_VERSION}`); - - if (_req._hdx_connection?.username) { - proxyReq.setHeader( - 'X-ClickHouse-User', - _req._hdx_connection.username, - ); - } - // Passwords can be empty - if (_req._hdx_connection?.password) { - proxyReq.setHeader( - 'X-ClickHouse-Key', - _req._hdx_connection.password, - ); - } - - if (_req.method !== 'POST') { - console.error(`Unsupported method ${_req.method}`); - return res.sendStatus(405); - } - - let body = _req.body; - if (_req.headers['content-type'] === 'application/json') { - try { - body = JSON.stringify(body); - } catch (e) { - console.error(e); - } - } - - try { - // TODO: Use fixRequestBody after this issue is resolved: https://github.com/chimurai/http-proxy-middleware/issues/1102 - proxyReq.write(body); - } catch (e) { - console.error( - `clickhouseProxy error writing body, body is type ${typeof body}`, - ); - } - }, - proxyRes: (proxyRes, _req, res) => { - const startedAt = (res as Response).locals?.hdxProxyStartedAt; - const statusCode = proxyRes.statusCode ?? 0; - recordOperationOutcome({ - operation: QUERY_PROXY_OPERATION, - // A response (even a 4xx/5xx from ClickHouse) means the proxy hop - // itself worked; outcome reflects whether ClickHouse served it. - outcome: statusCode < 400 ? 'success' : 'error', - durationMs: - typeof startedAt === 'number' - ? performance.now() - startedAt - : 0, - }); - - // since clickhouse v24, the cors headers * will be attached to the response by default - // which will cause the browser to block the response - if (_req.headers['access-control-request-method']) { - proxyRes.headers['access-control-allow-methods'] = - _req.headers['access-control-request-method']; - } - - if (_req.headers['access-control-request-headers']) { - proxyRes.headers['access-control-allow-headers'] = - _req.headers['access-control-request-headers']; - } - - if (_req.headers.origin) { - proxyRes.headers['access-control-allow-origin'] = - _req.headers.origin; - proxyRes.headers['access-control-allow-credentials'] = 'true'; - } - }, - error: (err, _req, _res) => { - const startedAt = (_res as Response).locals?.hdxProxyStartedAt; - recordOperationOutcome({ - operation: QUERY_PROXY_OPERATION, - // No usable response from ClickHouse (connection refused, timeout, - // DNS failure, ...) — a hard availability failure for the proxy. - outcome: 'error', - durationMs: - typeof startedAt === 'number' - ? performance.now() - startedAt - : 0, - }); - console.error('Proxy error:', err); - (_res as Response).writeHead(500, { - 'Content-Type': 'application/json', - }); - _res.end( - JSON.stringify({ - success: false, - error: err.message || 'Failed to connect to ClickHouse server', - }), - ); - }, - }, - // ...(config.IS_DEV && { - // logger: console, - // }), - }), - ); - - return _proxyPromise; -} - -const proxyMiddleware: RequestHandler = async (req, res, next) => { - try { - const middleware = await getProxyMiddleware(); - middleware(req, res, next); - } catch (e) { - next(e); - } -}; + try { + proxyReq.write(body); + } catch (e) { + console.error( + `clickhouseProxy error writing body, body is type ${typeof body}`, + ); + } + }, + proxyRes: (proxyRes, _req, res) => { + const startedAt = (res as Response).locals?.hdxProxyStartedAt; + const statusCode = proxyRes.statusCode ?? 0; + recordOperationOutcome({ + operation: QUERY_PROXY_OPERATION, + // A response (even a 4xx/5xx from ClickHouse) means the proxy hop + // itself worked; outcome reflects whether ClickHouse served it. + outcome: statusCode < 400 ? 'success' : 'error', + durationMs: + typeof startedAt === 'number' ? performance.now() - startedAt : 0, + }); + + // since clickhouse v24, the cors headers * will be attached to the response by default + // which will cause the browser to block the response + if (_req.headers['access-control-request-method']) { + proxyRes.headers['access-control-allow-methods'] = + _req.headers['access-control-request-method']; + } + + if (_req.headers['access-control-request-headers']) { + proxyRes.headers['access-control-allow-headers'] = + _req.headers['access-control-request-headers']; + } + + if (_req.headers.origin) { + proxyRes.headers['access-control-allow-origin'] = _req.headers.origin; + proxyRes.headers['access-control-allow-credentials'] = 'true'; + } + }, + error: (err, _req, _res) => { + const startedAt = (_res as Response).locals?.hdxProxyStartedAt; + recordOperationOutcome({ + operation: QUERY_PROXY_OPERATION, + // No usable response from ClickHouse (connection refused, timeout, + // DNS failure, ...) — a hard availability failure for the proxy. + outcome: 'error', + durationMs: + typeof startedAt === 'number' ? performance.now() - startedAt : 0, + }); + console.error('Proxy error:', err); + (_res as Response).writeHead(500, { + 'Content-Type': 'application/json', + }); + _res.end( + JSON.stringify({ + success: false, + error: err.message || 'Failed to connect to ClickHouse server', + }), + ); + }, + }, + // ...(config.IS_DEV && { + // logger: console, + // }), + }); // Stamp a start time so the proxy callbacks can record query SLO latency. const markProxyStart: RequestHandler = (_req, res, next) => { diff --git a/tsconfig.base.json b/tsconfig.base.json index 999e258069..909a33e5d1 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "module": "Node16", - "moduleResolution": "Node16", + "module": "NodeNext", + "moduleResolution": "NodeNext", "lib": ["ES2022", "dom"], "allowSyntheticDefaultImports": true, "downlevelIteration": true, From 5c46215f833238cf972b3654748a6466d9dedc58 Mon Sep 17 00:00:00 2001 From: Peter Leonov Date: Fri, 26 Jun 2026 23:46:09 +0200 Subject: [PATCH 112/728] Bump @clickhouse/client* to 1.23.0-head and fix type divergence (#2522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Validates HyperDX against the upcoming `@clickhouse/client*` **1.23** release line (which deprecates `@clickhouse/client-common` and has each platform package bundle its own copy of the shared types). Bumps the pins to the published pre-release build `1.23.0-head.fae5998.1` and fixes the resulting type breakage, so the client release can be validated against a green HyperDX CI. Complements the existing `clickhouse-client-forward-compat` changeset (which adjusted imports without bumping the pin); this PR performs the bump and resolves the one remaining incompatibility — the settings types. ## The problem the bump surfaces In 1.23 each platform package (`@clickhouse/client`, `@clickhouse/client-web`) bundles its own copy of the common module. Their `ClickHouseSettings` types reference `SettingsMap` — a class with a `private` member, so TypeScript compares it **nominally** — which makes the bundled copies **mutually unassignable** from `@clickhouse/client-common`'s. The shared `processClickhouseSettings()` helper produces the `client-common` flavor, so feeding it into the per-platform clients' `query()` broke: - **`tsc` (TS2322)** in `cli` and the `common-utils` declaration build. - **`@typescript-eslint/no-unsafe-type-assertion`** on the existing `as ClickHouseSettings` casts in `node.ts` / `browser.ts`. ## The fix The only thing that bridges nominally-distinct cross-package types is an explicit assertion (a structural "neutral" type doesn't help — the shared settings type still transitively references `SettingsMap` via `additional_table_filters` / `additional_result_filter`). So: - **`node.ts` / `browser.ts`** — keep the necessary `as ClickHouseSettings` bridge at the platform-client boundary, guarded with a scoped `no-unsafe-type-assertion` disable (the same "client library type mismatch" pattern already used in these files), pulled onto its own line. - **`cli`** — the proxy client never narrowed `getClient()` to the node client, so its `query()` saw the `node | web` union (an unsatisfiable settings *intersection*). Added the same `getClient()` override `node.ts` already has, plus the guarded bridge. Also preapproves `@clickhouse/*` in `.yarnrc.yml` so the head build (newer than the `npmMinimalAgeGate: 7d` window) installs. No runtime behavior changes. ## Test plan - [x] `@hyperdx/common-utils` declaration build (`ci:build`) succeeds - [x] `tsc --noEmit` clean for `common-utils` and `cli` (no `TS2322`) - [x] `eslint` clean on the changed files (no `no-unsafe-type-assertion`) - [x] `knip` passes - [x] `yarn install` resolves all three packages to `1.23.0-head.fae5998.1` - [ ] Full CI (lint, knip, unit, integration, e2e) Changeset included (`patch` for `@hyperdx/common-utils` and `@hyperdx/cli`). --- .changeset/clickhouse-client-1-23-head.md | 18 +++++++++ .yarnrc.yml | 3 ++ packages/cli/package.json | 2 +- packages/cli/src/api/client.ts | 16 +++++++- packages/common-utils/package.json | 6 +-- .../common-utils/src/clickhouse/browser.ts | 12 ++++-- packages/common-utils/src/clickhouse/node.ts | 12 ++++-- packages/hdx-eval/package.json | 2 +- yarn.lock | 38 +++++++++---------- 9 files changed, 74 insertions(+), 35 deletions(-) create mode 100644 .changeset/clickhouse-client-1-23-head.md diff --git a/.changeset/clickhouse-client-1-23-head.md b/.changeset/clickhouse-client-1-23-head.md new file mode 100644 index 0000000000..448c950d9e --- /dev/null +++ b/.changeset/clickhouse-client-1-23-head.md @@ -0,0 +1,18 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/cli": patch +--- + +Bump `@clickhouse/client*` to `1.23.0-head.fae5998.1` and fix the type +incompatibility it introduces. + +In `@clickhouse/client*` 1.23 each platform package (`@clickhouse/client`, +`@clickhouse/client-web`) bundles its own copy of the shared types, so their +`ClickHouseSettings` types — which reference the nominally-compared `SettingsMap` +class — are no longer the same type as `@clickhouse/client-common`'s. The shared +`processClickhouseSettings()` helper produces the `client-common` flavor, so +assigning it into the per-platform clients' `query()` now requires an explicit +bridge. Guard the existing `as ClickHouseSettings` assertions at those +boundaries (`node.ts`, `browser.ts`, `cli`) with a scoped +`@typescript-eslint/no-unsafe-type-assertion` disable, matching the existing +"client library type mismatch" pattern. No runtime behavior changes. diff --git a/.yarnrc.yml b/.yarnrc.yml index a140ed0343..db3f615757 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -6,3 +6,6 @@ npmMinimalAgeGate: 7d npmPreapprovedPackages: - '@hyperdx/*' + # Allow @clickhouse/client* pre-release (head) builds, which are published + # for release validation and are newer than the npmMinimalAgeGate window. + - '@clickhouse/*' diff --git a/packages/cli/package.json b/packages/cli/package.json index c5b9d99d67..ac29a95407 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -39,7 +39,7 @@ "dev:unit": "NODE_OPTIONS=--experimental-vm-modules jest --watchAll --runInBand" }, "devDependencies": { - "@clickhouse/client": "^1.12.1", + "@clickhouse/client": "1.23.0-head.fae5998.1", "@hyperdx/common-utils": "^0.20.0", "@jest/globals": "^30.2.0", "@types/crypto-js": "^4.2.2", diff --git a/packages/cli/src/api/client.ts b/packages/cli/src/api/client.ts index 91d4fbc73c..2c57cbc99b 100644 --- a/packages/cli/src/api/client.ts +++ b/packages/cli/src/api/client.ts @@ -10,6 +10,7 @@ import { createClient } from '@clickhouse/client'; import type { BaseResultSet, + ClickHouseClient as NodeClickHouseClient, ClickHouseSettings, DataFormat, } from '@clickhouse/client'; @@ -345,6 +346,13 @@ export class ProxyClickhouseClient extends BaseClickhouseClient { // Silence the "Sending Query: ..." debug output from BaseClickhouseClient protected override logDebugQuery(): void {} + // This subclass always builds a node client, so narrow the base class's + // platform-agnostic client type to the node-specific one. + protected getClient(): NodeClickHouseClient { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- subclass always builds a node client + return super.getClient() as NodeClickHouseClient; + } + protected async __query({ query, format = 'JSON' as Format, @@ -357,10 +365,16 @@ export class ProxyClickhouseClient extends BaseClickhouseClient { }: QueryInputs): Promise> { let clickhouseSettings: ClickHouseSettings | undefined; if (!shouldSkipApplySettings) { - clickhouseSettings = await this.processClickhouseSettings({ + const neutralSettings = await this.processClickhouseSettings({ connectionId, externalClickhouseSettings, }); + // processClickhouseSettings returns the shared base class's settings type + // (from @clickhouse/client-common). It is structurally identical to the + // node client's own self-bundled ClickHouseSettings, but the two packages' + // copies are distinct nominal types since 1.23, so bridge explicitly. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- client library type mismatch + clickhouseSettings = neutralSettings as ClickHouseSettings; } // Pass connection ID as HTTP header — the proxy uses this to diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 551601deae..0345b775f6 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -11,9 +11,9 @@ "node": ">=22.16.0" }, "dependencies": { - "@clickhouse/client": "^1.12.1", - "@clickhouse/client-common": "^1.12.1", - "@clickhouse/client-web": "^1.12.1", + "@clickhouse/client": "1.23.0-head.fae5998.1", + "@clickhouse/client-common": "1.23.0-head.fae5998.1", + "@clickhouse/client-web": "1.23.0-head.fae5998.1", "@hyperdx/lucene": "^3.1.1", "date-fns": "^2.28.0", "date-fns-tz": "^2.0.0", diff --git a/packages/common-utils/src/clickhouse/browser.ts b/packages/common-utils/src/clickhouse/browser.ts index 5f358740e8..7e52cec40c 100644 --- a/packages/common-utils/src/clickhouse/browser.ts +++ b/packages/common-utils/src/clickhouse/browser.ts @@ -133,12 +133,16 @@ export class ClickhouseClient extends BaseClickhouseClient { let clickhouseSettings: ClickHouseSettings | undefined; // If this is the settings query, we must not process the clickhouse settings, or else we will infinitely recurse if (!shouldSkipApplySettings) { - // The shared base class produces a platform-neutral settings object; the - // web client expects its own (now self-bundled) ClickHouseSettings type. - clickhouseSettings = (await this.processClickhouseSettings({ + const neutralSettings = await this.processClickhouseSettings({ connectionId, externalClickhouseSettings, - })) as ClickHouseSettings; + }); + // processClickhouseSettings produces @clickhouse/client-common's + // ClickHouseSettings. It is structurally identical to the web client's + // own (self-bundled, since 1.23) ClickHouseSettings, but the two packages' + // copies are distinct nominal types, so bridge explicitly. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- client library type mismatch + clickhouseSettings = neutralSettings as ClickHouseSettings; } const httpHeaders: { [header: string]: string } = { diff --git a/packages/common-utils/src/clickhouse/node.ts b/packages/common-utils/src/clickhouse/node.ts index d8ad383f38..a43e0911b8 100644 --- a/packages/common-utils/src/clickhouse/node.ts +++ b/packages/common-utils/src/clickhouse/node.ts @@ -51,12 +51,16 @@ export class ClickhouseClient extends BaseClickhouseClient { let clickhouseSettings: ClickHouseSettings | undefined; // If this is the settings query, we must not process the clickhouse settings, or else we will infinitely recurse if (!shouldSkipApplySettings) { - // The shared base class produces a platform-neutral settings object; the - // node client expects its own (now self-bundled) ClickHouseSettings type. - clickhouseSettings = (await this.processClickhouseSettings({ + const neutralSettings = await this.processClickhouseSettings({ externalClickhouseSettings, connectionId, - })) as ClickHouseSettings; + }); + // processClickhouseSettings produces @clickhouse/client-common's + // ClickHouseSettings. It is structurally identical to the node client's + // own (self-bundled, since 1.23) ClickHouseSettings, but the two packages' + // copies are distinct nominal types, so bridge explicitly. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- client library type mismatch + clickhouseSettings = neutralSettings as ClickHouseSettings; } // TODO: Custom error handling diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json index 0f3f511df8..1982842236 100644 --- a/packages/hdx-eval/package.json +++ b/packages/hdx-eval/package.json @@ -14,7 +14,7 @@ ], "dependencies": { "@anthropic-ai/sdk": "^0.92.0", - "@clickhouse/client": "^1.12.1", + "@clickhouse/client": "1.23.0-head.fae5998.1", "commander": "^12.1.0", "ms": "^2.1.3" }, diff --git a/yarn.lock b/yarn.lock index 09167ec3af..e0ab5a0eb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3118,28 +3118,24 @@ __metadata: languageName: node linkType: hard -"@clickhouse/client-common@npm:1.12.1, @clickhouse/client-common@npm:^1.12.1": - version: 1.12.1 - resolution: "@clickhouse/client-common@npm:1.12.1" - checksum: 10c0/9f475c8517b164c3676a969b60c8165a4b0558906917dbe97ae35f69f2702a6fc198cbdb2e6d2ec3543c627ec04a7e2fccb41f3114452bf7f4818152c0f98a5d +"@clickhouse/client-common@npm:1.23.0-head.fae5998.1": + version: 1.23.0-head.fae5998.1 + resolution: "@clickhouse/client-common@npm:1.23.0-head.fae5998.1" + checksum: 10c0/8fa132376daa46cb046e7a50c2ef53c61456034aecd36363e2d60bc948265f08e1a521f7835935fb067a0673c70081eed64462709f64c5a10e1249155bc189c9 languageName: node linkType: hard -"@clickhouse/client-web@npm:^1.12.1": - version: 1.12.1 - resolution: "@clickhouse/client-web@npm:1.12.1" - dependencies: - "@clickhouse/client-common": "npm:1.12.1" - checksum: 10c0/3e374288a2e0e4ced37220638629612dc8d6ced66490c2bdac136ddb887d5e108b9992b26d57911d6007bd47eaf7a754caa48e3afdfdb44cbd4f07bec7900995 +"@clickhouse/client-web@npm:1.23.0-head.fae5998.1": + version: 1.23.0-head.fae5998.1 + resolution: "@clickhouse/client-web@npm:1.23.0-head.fae5998.1" + checksum: 10c0/15ca088a6ea2d6e568bda7f68dc74a0216520127859e2dc46c8868135712efdbae409ccf8bc201d730da62d2e9831b124e0310fae8f7e7ea8cb19ca73224b256 languageName: node linkType: hard -"@clickhouse/client@npm:^1.12.1": - version: 1.12.1 - resolution: "@clickhouse/client@npm:1.12.1" - dependencies: - "@clickhouse/client-common": "npm:1.12.1" - checksum: 10c0/4d715accad5ba74248c63fab83469ebeffc16d7cc5050ee7e5f9e7dd48d0e7c4b79517bf5917f2a60599960aee997e5d8341c480d056286f5d9a86530623211f +"@clickhouse/client@npm:1.23.0-head.fae5998.1": + version: 1.23.0-head.fae5998.1 + resolution: "@clickhouse/client@npm:1.23.0-head.fae5998.1" + checksum: 10c0/a13e1c8d350b59bb42882b7b83a1299675e3fa5bcc2b9786b12506e920f9a592edb74ad5de62cceddab95ed12d21e5e4276bc1e0030e6bc2e0d7fd42d65b7c59 languageName: node linkType: hard @@ -4643,7 +4639,7 @@ __metadata: version: 0.0.0-use.local resolution: "@hyperdx/cli@workspace:packages/cli" dependencies: - "@clickhouse/client": "npm:^1.12.1" + "@clickhouse/client": "npm:1.23.0-head.fae5998.1" "@hyperdx/common-utils": "npm:^0.20.0" "@jest/globals": "npm:^30.2.0" "@types/crypto-js": "npm:^4.2.2" @@ -4674,9 +4670,9 @@ __metadata: version: 0.0.0-use.local resolution: "@hyperdx/common-utils@workspace:packages/common-utils" dependencies: - "@clickhouse/client": "npm:^1.12.1" - "@clickhouse/client-common": "npm:^1.12.1" - "@clickhouse/client-web": "npm:^1.12.1" + "@clickhouse/client": "npm:1.23.0-head.fae5998.1" + "@clickhouse/client-common": "npm:1.23.0-head.fae5998.1" + "@clickhouse/client-web": "npm:1.23.0-head.fae5998.1" "@hyperdx/lucene": "npm:^3.1.1" "@types/hyperdx__lucene": "npm:@types/lucene@*" "@types/jest": "npm:^29.5.14" @@ -4706,7 +4702,7 @@ __metadata: resolution: "@hyperdx/hdx-eval@workspace:packages/hdx-eval" dependencies: "@anthropic-ai/sdk": "npm:^0.92.0" - "@clickhouse/client": "npm:^1.12.1" + "@clickhouse/client": "npm:1.23.0-head.fae5998.1" "@dotenvx/dotenvx": "npm:^1.51.1" "@types/jest": "npm:^29.5.14" "@types/ms": "npm:^0.7.31" From c1403a7a7d480d9e199025e0afcedc693b5bbd41 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Mon, 29 Jun 2026 14:59:40 +1000 Subject: [PATCH 113/728] feat(app): support ratio mode on number chart tiles (#2535) Co-authored-by: Jordan Simonovski --- .changeset/ratio-number-tiles.md | 5 ++ .../ChartEditor/__tests__/utils.test.ts | 72 +++++++++++++++++-- .../app/src/components/ChartEditor/utils.ts | 23 +++++- .../ChartEditorControls.tsx | 11 ++- 4 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 .changeset/ratio-number-tiles.md diff --git a/.changeset/ratio-number-tiles.md b/.changeset/ratio-number-tiles.md new file mode 100644 index 0000000000..c1cbb65897 --- /dev/null +++ b/.changeset/ratio-number-tiles.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": minor +--- + +Number chart tiles now support a second series with the "As Ratio" toggle (`series[0] / series[1]`), matching line and bar charts. Combined with a `percent` number format, this renders a percentage (e.g. success/error rate) as a single big number with the trend sparkline behind it. diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index 104d912481..9ca5183393 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -1036,12 +1036,30 @@ describe('validateChartForm', () => { // ── Number / Pie single-series validation ──────────────────────────── - it('errors when Number chart has more than one series', () => { + it('allows a Number chart with two series in ratio mode', () => { const setError = jest.fn(); const errors = validateChartForm( makeForm({ displayType: DisplayType.Number, source: 'source-log', + seriesReturnType: 'ratio', + series: [seriesItem, seriesItem], + }), + logSource, + setError, + ); + expect(errors).not.toContainEqual( + expect.objectContaining({ path: 'series' }), + ); + }); + + it('errors when Number chart has two series without ratio mode', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + displayType: DisplayType.Number, + source: 'source-log', + seriesReturnType: 'column', series: [seriesItem, seriesItem], }), logSource, @@ -1050,7 +1068,48 @@ describe('validateChartForm', () => { expect(errors).toContainEqual( expect.objectContaining({ path: 'series', - message: `Only one series is allowed for ${DisplayType.Number} charts`, + message: + 'Number charts support a single series unless ratio mode (As Ratio) is enabled', + }), + ); + }); + + it('errors when Number chart has two series and seriesReturnType is unset', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + displayType: DisplayType.Number, + source: 'source-log', + series: [seriesItem, seriesItem], + }), + logSource, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'series', + message: + 'Number charts support a single series unless ratio mode (As Ratio) is enabled', + }), + ); + }); + + it('errors when Number chart has more than two series in ratio mode', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + displayType: DisplayType.Number, + source: 'source-log', + seriesReturnType: 'ratio', + series: [seriesItem, seriesItem, seriesItem], + }), + logSource, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'series', + message: 'Number charts support at most two series (ratio mode)', }), ); }); @@ -1271,12 +1330,14 @@ describe('validateChartForm', () => { series: [ { ...seriesItem, aggFn: 'sum', valueExpression: '' }, { ...seriesItem, aggFn: 'avg', valueExpression: '' }, + { ...seriesItem, aggFn: 'avg', valueExpression: '' }, ], }), logSource, setError, ); - // Should have: source error + 2 valueExpression errors + series count error + // Should have: source error + 3 valueExpression errors + series count error + // (Number charts allow up to two series, so three trips the count rule) expect(errors).toContainEqual(expect.objectContaining({ path: 'source' })); expect(errors).toContainEqual( expect.objectContaining({ path: 'series.0.valueExpression' }), @@ -1284,8 +1345,11 @@ describe('validateChartForm', () => { expect(errors).toContainEqual( expect.objectContaining({ path: 'series.1.valueExpression' }), ); + expect(errors).toContainEqual( + expect.objectContaining({ path: 'series.2.valueExpression' }), + ); expect(errors).toContainEqual(expect.objectContaining({ path: 'series' })); - expect(errors).toHaveLength(4); + expect(errors).toHaveLength(5); }); it('calls setError for every accumulated error', () => { diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 8d05ec1a84..fbb7084bcf 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -381,12 +381,11 @@ export const validateChartForm = ( } } - // Validate number, pie, and heatmap charts only have one series + // Validate pie and heatmap charts only have one series if ( !isRawSqlChart && Array.isArray(form.series) && - (form.displayType === DisplayType.Number || - form.displayType === DisplayType.Pie || + (form.displayType === DisplayType.Pie || form.displayType === DisplayType.Heatmap) && form.series.length > 1 ) { @@ -396,6 +395,24 @@ export const validateChartForm = ( }); } + // Number charts allow a second series only for ratio mode (numerator / + // denominator, which can be shown as a percentage via the number format); + // otherwise they show a single value. + if ( + !isRawSqlChart && + Array.isArray(form.series) && + form.displayType === DisplayType.Number && + form.series.length > (form.seriesReturnType === 'ratio' ? 2 : 1) + ) { + errors.push({ + path: `series`, + message: + form.seriesReturnType === 'ratio' + ? 'Number charts support at most two series (ratio mode)' + : 'Number charts support a single series unless ratio mode (As Ratio) is enabled', + }); + } + // Validate heatmap requires a value expression if ( !isRawSqlChart && diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx index 6b2e4574a4..ce28177822 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx @@ -97,9 +97,11 @@ export function ChartEditorControls({ openHeatmapSettings, }: ChartEditorControlsProps) { const canAddSeries = - displayType !== DisplayType.Number && displayType !== DisplayType.Pie && - displayType !== DisplayType.Heatmap; + displayType !== DisplayType.Heatmap && + // Number tiles support up to two series (numerator + denominator for + // ratio mode); Line/Table types remain unbounded. + !(displayType === DisplayType.Number && fields.length >= 2); const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = useState(false); @@ -265,7 +267,10 @@ export function ChartEditorControls({ Add Series )} - {fields.length == 2 && displayType !== DisplayType.Number && ( + {/* Ratio merges exactly two series via divide(); only + Line/StackedBar/Table/Number can reach two series, so gating + on the count alone covers them all (Number included). */} + {fields.length === 2 && ( Date: Mon, 29 Jun 2026 15:02:50 +0200 Subject: [PATCH 114/728] fix: type JSON metadata filter attributes (#2537) ## Summary - Render JSON-backed metadata filter keys as typed ClickHouse JSON subcolumns before using them in `GROUP BY`, `groupUniqArray`, and fallback value lookups. - Keep Map-backed attribute keys on the existing bracket syntax. - Add regression coverage for `getKeyValues`, `getValuesDistribution`, and `getMapValues`. Validation: - `mise exec node@22.16.0 -- corepack yarn workspace @hyperdx/common-utils jest --runInBand src/__tests__/metadata.test.ts` - `mise exec node@22.16.0 -- corepack yarn workspace @hyperdx/common-utils ci:lint` - `mise exec node@22.16.0 -- corepack yarn workspace @hyperdx/common-utils ci:unit` Duplicate/overlap check: I did not find another PR fixing #2482. #2423 touches faceted filter values but is a separate draft feature and does not address JSON typed subcolumn rendering. ### Screenshots or video N/A - non-UI change ### How to test on Vercel preview N/A - non-UI change ### References - Fixes #2482 - Related PRs: #2423 --- .../type-json-metadata-filter-attributes.md | 5 + .../src/__tests__/metadata.test.ts | 201 ++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 167 ++++++++++++++- 3 files changed, 366 insertions(+), 7 deletions(-) create mode 100644 .changeset/type-json-metadata-filter-attributes.md diff --git a/.changeset/type-json-metadata-filter-attributes.md b/.changeset/type-json-metadata-filter-attributes.md new file mode 100644 index 0000000000..0b4174904c --- /dev/null +++ b/.changeset/type-json-metadata-filter-attributes.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/common-utils': patch +--- + +Type JSON metadata filter attribute paths before value sampling. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 4e7c9bc48e..601e132602 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -635,6 +635,107 @@ describe('Metadata', () => { expect(results).toEqual([]); expect(renderChartConfigSpy).not.toHaveBeenCalled(); }); + + it('renders JSON attribute keys as typed subcolumns', async () => { + jest.spyOn(metadata, 'getColumn').mockImplementation(({ column }) => + Promise.resolve( + column === 'ResourceAttributes' + ? ({ + name: 'ResourceAttributes', + type: 'JSON(max_dynamic_types=8, max_dynamic_paths=64)', + } as any) + : undefined, + ), + ); + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getKeyValues({ + chartConfig: mockChartConfig, + keys: ["ResourceAttributes['k8s.namespace.name']"], + limit: 10, + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + expect(actualConfig.with?.[0]).toMatchObject({ + chartConfig: { + select: + 'ResourceAttributes.`k8s`.`namespace`.`name`.:String as param0', + }, + }); + }); + + it('quotes typed-looking bracket JSON keys instead of passing them through', async () => { + jest.spyOn(metadata, 'getColumn').mockImplementation(({ column }) => + Promise.resolve( + column === 'ResourceAttributes' + ? ({ + name: 'ResourceAttributes', + type: 'JSON(max_dynamic_types=8, max_dynamic_paths=64)', + } as any) + : undefined, + ), + ); + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getKeyValues({ + chartConfig: mockChartConfig, + keys: ["ResourceAttributes['foo.:String, count() AS injected']"], + limit: 10, + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + expect(actualConfig.with?.[0]).toMatchObject({ + chartConfig: { + select: + 'ResourceAttributes.`foo`.`:String, count() AS injected`.:String as param0', + }, + }); + }); + + it('keeps map attribute keys in bracket form', async () => { + jest.spyOn(metadata, 'getColumn').mockImplementation(({ column }) => + Promise.resolve( + column === 'LogAttributes' + ? ({ + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + } as any) + : undefined, + ), + ); + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getKeyValues({ + chartConfig: mockChartConfig, + keys: ["LogAttributes['k8s.namespace.name']"], + limit: 10, + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + expect(actualConfig.with?.[0]).toMatchObject({ + chartConfig: { + select: "LogAttributes['k8s.namespace.name'] as param0", + }, + }); + }); }); describe('getValuesDistribution', () => { @@ -777,6 +878,67 @@ describe('Metadata', () => { condition: "ServiceName IN ('clickhouse')", }); }); + + it('renders JSON distribution keys as typed subcolumns', async () => { + jest.spyOn(metadata, 'getColumn').mockImplementation(({ column }) => + Promise.resolve( + column === 'ResourceAttributes' + ? ({ + name: 'ResourceAttributes', + type: 'JSON(max_dynamic_types=8, max_dynamic_paths=64)', + } as any) + : undefined, + ), + ); + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getValuesDistribution({ + chartConfig: mockChartConfig, + key: 'ResourceAttributes.k8s.namespace.name', + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + expect(actualConfig.select).toBe( + 'ResourceAttributes.`k8s`.`namespace`.`name`.:String AS __hdx_value, count() as __hdx_count, __hdx_count / (sum(__hdx_count) OVER ()) * 100 AS __hdx_percentage', + ); + expect(actualConfig.groupBy).toBe('__hdx_value'); + }); + + it('normalizes typed dot-form JSON distribution keys safely', async () => { + jest.spyOn(metadata, 'getColumn').mockImplementation(({ column }) => + Promise.resolve( + column === 'ResourceAttributes' + ? ({ + name: 'ResourceAttributes', + type: 'JSON(max_dynamic_types=8, max_dynamic_paths=64)', + } as any) + : undefined, + ), + ); + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getValuesDistribution({ + chartConfig: mockChartConfig, + key: 'ResourceAttributes.k8s.namespace.name.:String', + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + expect(actualConfig.select).toBe( + 'ResourceAttributes.`k8s`.`namespace`.`name`.:String AS __hdx_value, count() as __hdx_count, __hdx_count / (sum(__hdx_count) OVER ()) * 100 AS __hdx_percentage', + ); + }); }); describe('getMapKeys', () => { @@ -943,6 +1105,10 @@ describe('Metadata', () => { it("emits only the existing value != '' predicate when dateRange not provided", async () => { const md = buildMetadata(); + jest.spyOn(md, 'getColumn').mockResolvedValue({ + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + } as any); (mockClickhouseClient.query as jest.Mock).mockResolvedValueOnce({ json: () => Promise.resolve({ data: [] }), @@ -966,6 +1132,10 @@ describe('Metadata', () => { it('injects the time filter clause when dateRange and timestampValueExpression are provided', async () => { const md = buildMetadata(); + jest.spyOn(md, 'getColumn').mockResolvedValue({ + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + } as any); (mockClickhouseClient.query as jest.Mock).mockResolvedValueOnce({ json: () => Promise.resolve({ data: [] }), @@ -1002,8 +1172,39 @@ describe('Metadata', () => { expect(valuesCall.query).toContain('__TIME_FILTER__'); }); + it('uses typed JSON subcolumns for JSON attribute values', async () => { + const md = buildMetadata(); + jest.spyOn(md, 'getColumn').mockResolvedValue({ + name: 'ResourceAttributes', + type: 'JSON(max_dynamic_types=8, max_dynamic_paths=64)', + } as any); + + (mockClickhouseClient.query as jest.Mock).mockResolvedValueOnce({ + json: () => Promise.resolve({ data: [] }), + }); + + await md.getMapValues({ + databaseName: 'otel', + tableName: 'generic_logs', + column: 'ResourceAttributes', + key: 'k8s.namespace.name', + connectionId: 'conn-1', + }); + + const valuesCall = (mockClickhouseClient.query as jest.Mock).mock + .calls[0][0]; + expect(valuesCall.query).toContain( + 'ResourceAttributes.`k8s`.`namespace`.`name`.:String as value', + ); + expect(valuesCall.query).not.toContain('ResourceAttributes['); + }); + it('caches values distinctly for different dateRange values', async () => { const md = buildMetadata(); + jest.spyOn(md, 'getColumn').mockResolvedValue({ + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + } as any); (mockClickhouseClient.query as jest.Mock) .mockResolvedValueOnce({ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 05ab7b670c..5fc5588dad 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -48,6 +48,51 @@ const inlineNonNegativeInt = (value: number, label: string): string => { return String(value); }; +const unquoteIdentifier = (identifier: string): string => { + if ( + (identifier.startsWith('`') && identifier.endsWith('`')) || + (identifier.startsWith('"') && identifier.endsWith('"')) + ) { + return identifier.slice(1, -1); + } + return identifier; +}; + +const quoteJsonPathSegment = (segment: string): string => { + const unquoted = unquoteIdentifier(segment); + return `\`${unquoted.replace(/`/g, '``')}\``; +}; + +const quoteIdentifierIfNeeded = (identifier: string): string => { + const unquoted = unquoteIdentifier(identifier); + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(unquoted) + ? unquoted + : quoteJsonPathSegment(unquoted); +}; + +const JSON_STRING_TYPE_SUFFIX = '.:String'; + +const renderJsonStringSubcolumn = ( + column: string, + jsonPath: string, + options: { preserveStringTypeSuffix?: boolean } = {}, +): string => { + const columnIdentifier = quoteIdentifierIfNeeded(column); + const untypedJsonPath = + options.preserveStringTypeSuffix && + jsonPath.endsWith(JSON_STRING_TYPE_SUFFIX) + ? jsonPath.slice(0, -JSON_STRING_TYPE_SUFFIX.length) + : jsonPath; + + const path = untypedJsonPath + .split('.') + .filter(Boolean) + .map(quoteJsonPathSegment) + .join('.'); + + return `${columnIdentifier}.${path}${JSON_STRING_TYPE_SUFFIX}`; +}; + export class MetadataCache { private cache = new Map(); private pendingQueries = new Map>(); @@ -164,6 +209,69 @@ export class Metadata { this.cache.set('clickhouse-settings', updatedSettings); } + private async renderMetadataKeyExpression({ + databaseName, + tableName, + connectionId, + keyExpression, + }: { + databaseName: string; + tableName: string; + connectionId: string; + keyExpression: string; + }): Promise { + const directColumn = await this.getColumn({ + databaseName, + tableName, + column: unquoteIdentifier(keyExpression), + connectionId, + }); + if (directColumn != null) { + return quoteIdentifierIfNeeded(keyExpression); + } + + const bracketPath = parseKeyPath(keyExpression); + if (bracketPath.length >= 2) { + const column = unquoteIdentifier(bracketPath[0]); + const columnMeta = await this.getColumn({ + databaseName, + tableName, + column, + connectionId, + }); + + if ( + convertCHDataTypeToJSType(columnMeta?.type ?? '') === JSDataType.JSON + ) { + return renderJsonStringSubcolumn(column, bracketPath[1]); + } + + return keyExpression; + } + + const dotIdx = keyExpression.indexOf('.'); + if (dotIdx === -1 || keyExpression.includes('(')) { + return keyExpression; + } + + const column = unquoteIdentifier(keyExpression.slice(0, dotIdx)); + const jsonPath = keyExpression.slice(dotIdx + 1); + const columnMeta = await this.getColumn({ + databaseName, + tableName, + column, + connectionId, + }); + + if (convertCHDataTypeToJSType(columnMeta?.type ?? '') === JSDataType.JSON) { + return renderJsonStringSubcolumn(column, jsonPath, { + preserveStringTypeSuffix: true, + }); + } + + return keyExpression; + } + private async queryTableMetadata({ database, table, @@ -700,8 +808,33 @@ export class Metadata { ]; const where = chSql`WHERE ${concatChSql(' AND ', ...whereConditions)}`; - const sql = key - ? chSql` + const colMeta = key + ? await this.getColumn({ + databaseName, + tableName, + column, + connectionId, + }) + : undefined; + const jsonValueExpression = + key && convertCHDataTypeToJSType(colMeta?.type ?? '') === JSDataType.JSON + ? renderJsonStringSubcolumn(column, key) + : undefined; + + let sql: ChSql; + if (jsonValueExpression) { + sql = chSql` + SELECT DISTINCT ${{ + UNSAFE_RAW_SQL: jsonValueExpression, + }} as value + FROM ${tableExpr({ database: databaseName, table: tableName })} + ${where} + LIMIT ${{ + Int32: maxValues, + }} + `; + } else if (key) { + sql = chSql` SELECT DISTINCT ${{ Identifier: column, }}[${{ String: key }}] as value @@ -710,8 +843,9 @@ export class Metadata { LIMIT ${{ Int32: maxValues, }} - ` - : chSql` + `; + } else { + sql = chSql` SELECT DISTINCT ${{ Identifier: column, }} as value @@ -721,6 +855,7 @@ export class Metadata { Int32: maxValues, }} `; + } return this.cache.getOrFetch(cacheKey, async () => { const values = await this.clickhouseClient @@ -1312,6 +1447,12 @@ export class Metadata { return this.cache.getOrFetch( `${objectHash(cacheKeyConfig)}.${key}.valuesDistribution`, async () => { + const renderedKey = await this.renderMetadataKeyExpression({ + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + connectionId: chartConfig.connection, + keyExpression: key, + }); const config: BuilderChartConfigWithDateRange = { ...chartConfig, with: [ @@ -1334,7 +1475,7 @@ export class Metadata { condition: `cityHash64(${chartConfig.timestampValueExpression}, rand()) % (SELECT sample_factor FROM tableStats) = 0`, }, ], - select: `${key} AS __hdx_value, count() as __hdx_count, __hdx_count / (sum(__hdx_count) OVER ()) * 100 AS __hdx_percentage`, + select: `${renderedKey} AS __hdx_value, count() as __hdx_count, __hdx_count / (sum(__hdx_count) OVER ()) * 100 AS __hdx_percentage`, orderBy: '__hdx_percentage DESC', groupBy: `__hdx_value`, limit: { limit }, @@ -1668,12 +1809,23 @@ export class Metadata { async () => { if (keys.length === 0) return []; + const renderedKeys = await Promise.all( + keys.map(key => + this.renderMetadataKeyExpression({ + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + connectionId: chartConfig.connection, + keyExpression: key, + }), + ), + ); + // When disableRowLimit is true, query directly without CTE // Otherwise, use CTE with row limits for sampling const sqlConfig = disableRowLimit ? { ...chartConfig, - select: keys + select: renderedKeys .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`) .join(', '), } @@ -1683,7 +1835,8 @@ export class Metadata { // than selecting just the JSON paths corresponding to the given keys. // paramN aliases are used to avoid issues with special characters or complex expressions in keys. const selectExpr = - keys.map((k, i) => `${k} as param${i}`).join(', ') || '*'; + renderedKeys.map((k, i) => `${k} as param${i}`).join(', ') || + '*'; return { with: [ From 6e0880a75226dc1fd69f82156548a48967a5f6d3 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 29 Jun 2026 12:29:40 -0400 Subject: [PATCH 115/728] feat: Add Known Columns List setting for distributed tables (#2539) ## Summary This PR adds a new source configuration setting for log and trace tables, intended to improve support for sources based on distributed tables which reference local tables with non-uniform column lists. ### The Problem Distributed tables may point to "local" tables with non-matching sets of columns. When this is the case, `SELECT *` queries request the list of columns declared by the distributed table from each local table. This causes errors when a local table is missing one of the declared columns. ### The Solution The solution in this PR is a new setting at the source level which allows the user to specify a list of "Known Columns" which exist on every local table. When this setting is provided, it is used instead of `*` when opening the side panel. To aid the user in diagnosing the failure when this setting is not provided: - The side panel error state has been updated from a vague `Error loading row details` to the ClickHouse query error and the query that was sent to ClickHouse - When the error indicates a referenced column could not be found AND when the source references a Distributed or Merge table, then the UI will also display an Alert hinting to the user that they might be able to solve it by setting a Known Columns List in source settings. ### Screenshots or video https://github.com/user-attachments/assets/26b04cd9-86f1-46d9-880a-b1ac6f4f5a87 ### How to test on Vercel preview This is tough to test on the preview environment because it requires a distributed table with non-matching columns to trigger the exact error. You could set this up locally by updating the development environment docker config to spin up a second ClickHouse node in the hdx_cluster (Claude is good at this) and creating the following tables + source:
DDL ```sql -- Local Table (Node 1) CREATE TABLE default.otel_logs_local ( `Timestamp` DateTime64(9) CODEC (Delta (8), ZSTD (1)), `TraceId` String CODEC (ZSTD (1)), `SpanId` String CODEC (ZSTD (1)), `TraceFlags` UInt8, `SeverityText` LowCardinality(String) CODEC (ZSTD (1)), `SeverityNumber` UInt8, `ServiceName` LowCardinality(String) CODEC (ZSTD (1)), `Body` String CODEC (ZSTD (1)), `ResourceSchemaUrl` LowCardinality(String) CODEC (ZSTD (1)), `ResourceAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `ScopeSchemaUrl` LowCardinality(String) CODEC (ZSTD (1)), `ScopeName` String CODEC (ZSTD (1)), `ScopeVersion` LowCardinality(String) CODEC (ZSTD (1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `LogAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `EventName` String CODEC (ZSTD (1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC (ZSTD (1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC (ZSTD (1)), `ResourceAttributeItems` Array(String) ALIAS arrayMap( arr -> concat(arr .1, '=', arr .2), CAST( ResourceAttributes, 'Array(Tuple(String, String))' ) ), `ScopeAttributeItems` Array(String) ALIAS arrayMap( arr -> concat(arr .1, '=', arr .2), CAST(ScopeAttributes, 'Array(Tuple(String, String))') ), `LogAttributeItems` Array(String) ALIAS arrayMap( arr -> concat(arr .1, '=', arr .2), CAST(LogAttributes, 'Array(Tuple(String, String))') ), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 100000000 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY ( toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp ) TTL toDateTime(Timestamp) + toIntervalDay(1) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1; -- Local Table (Node 2, missing SeverityNumber column) CREATE TABLE default.otel_logs_local ( `Timestamp` DateTime64(9) CODEC (Delta (8), ZSTD (1)), `TraceId` String CODEC (ZSTD (1)), `SpanId` String CODEC (ZSTD (1)), `TraceFlags` UInt8, `SeverityText` LowCardinality(String) CODEC (ZSTD (1)), `ServiceName` LowCardinality(String) CODEC (ZSTD (1)), `Body` String CODEC (ZSTD (1)), `ResourceSchemaUrl` LowCardinality(String) CODEC (ZSTD (1)), `ResourceAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `ScopeSchemaUrl` LowCardinality(String) CODEC (ZSTD (1)), `ScopeName` String CODEC (ZSTD (1)), `ScopeVersion` LowCardinality(String) CODEC (ZSTD (1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `LogAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `EventName` String CODEC (ZSTD (1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC (ZSTD (1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC (ZSTD (1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC (ZSTD (1)), `ResourceAttributeItems` Array(String) ALIAS arrayMap( arr -> concat(arr .1, '=', arr .2), CAST( ResourceAttributes, 'Array(Tuple(String, String))' ) ), `ScopeAttributeItems` Array(String) ALIAS arrayMap( arr -> concat(arr .1, '=', arr .2), CAST(ScopeAttributes, 'Array(Tuple(String, String))') ), `LogAttributeItems` Array(String) ALIAS arrayMap( arr -> concat(arr .1, '=', arr .2), CAST(LogAttributes, 'Array(Tuple(String, String))') ), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 100000000 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY ( toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp ) TTL toDateTime(Timestamp) + toIntervalDay(1) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1; -- Distributed Table (Both Nodes) CREATE TABLE default.otel_logs_distributed_mismatch ( `Timestamp` DateTime64(9) CODEC (Delta (8), ZSTD (1)), `TraceId` String CODEC (ZSTD (1)), `SpanId` String CODEC (ZSTD (1)), `TraceFlags` UInt8, `SeverityText` LowCardinality(String) CODEC (ZSTD (1)), `ServiceName` LowCardinality(String) CODEC (ZSTD (1)), `SeverityNumber` UInt8, -- Includes the SeverityNumber column `Body` String CODEC (ZSTD (1)), `ResourceSchemaUrl` LowCardinality(String) CODEC (ZSTD (1)), `ResourceAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `ScopeSchemaUrl` LowCardinality(String) CODEC (ZSTD (1)), `ScopeName` String CODEC (ZSTD (1)), `ScopeVersion` LowCardinality(String) CODEC (ZSTD (1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `LogAttributes` Map(LowCardinality(String), String) CODEC (ZSTD (1)), `EventName` String CODEC (ZSTD (1)) ) ENGINE Distributed('hdx_cluster', 'default', 'otel_logs_local', rand()); -- Populate the local table with some data from the original otel_logs table INSERT INTO otel_logs_local SELECT * FROM otel_logs; ```
### References - Linear Issue: Closes HDX-4588 - Related PRs: --- .changeset/spotty-roses-cheer.md | 7 + packages/api/openapi.json | 12 ++ packages/api/src/models/source.ts | 2 + .../src/routers/external-api/v2/sources.ts | 10 ++ .../app/src/components/DBRowDataPanel.tsx | 11 +- .../app/src/components/DBRowSidePanel.tsx | 10 ++ .../components/DBRowSidePanelErrorState.tsx | 138 ++++++++++++++++++ .../app/src/components/Sources/SourceForm.tsx | 34 +++++ .../__tests__/DBRowDataPanel.test.ts | 61 ++++++++ packages/app/src/hooks/useMetadata.tsx | 10 +- .../src/__tests__/metadata.test.ts | 52 +++++++ .../src/clickhouse/__tests__/index.test.ts | 39 +++++ packages/common-utils/src/clickhouse/index.ts | 13 ++ packages/common-utils/src/core/metadata.ts | 13 ++ packages/common-utils/src/types.ts | 2 + 15 files changed, 411 insertions(+), 3 deletions(-) create mode 100644 .changeset/spotty-roses-cheer.md create mode 100644 packages/app/src/components/DBRowSidePanelErrorState.tsx diff --git a/.changeset/spotty-roses-cheer.md b/.changeset/spotty-roses-cheer.md new file mode 100644 index 0000000000..da054a313c --- /dev/null +++ b/.changeset/spotty-roses-cheer.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add Known Columns List setting for distributed tables diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 60b2e9d7de..5e1511be80 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -3480,6 +3480,12 @@ "nullable": true, "example": "Body" }, + "knownColumnsListExpression": { + "type": "string", + "description": "For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns.", + "nullable": true, + "example": "Timestamp, Body, ServiceName" + }, "useTextIndexForImplicitColumn": { "type": "string", "enum": [ @@ -3706,6 +3712,12 @@ "nullable": true, "example": "SpanName" }, + "knownColumnsListExpression": { + "type": "string", + "description": "For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns.", + "nullable": true, + "example": "Timestamp, Body, ServiceName" + }, "useTextIndexForImplicitColumn": { "type": "string", "enum": [ diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 7623af007b..b1c63e0dc3 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -144,6 +144,7 @@ export const LogSource = Source.discriminator( traceIdExpression: String, spanIdExpression: String, implicitColumnExpression: String, + knownColumnsListExpression: String, useTextIndexForImplicitColumn: { type: String, enum: Object.values(UseTextIndex), @@ -197,6 +198,7 @@ export const TraceSource = Source.discriminator( eventAttributesExpression: String, spanEventsValueExpression: String, implicitColumnExpression: String, + knownColumnsListExpression: String, useTextIndexForImplicitColumn: { type: String, enum: Object.values(UseTextIndex), diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 106c9a4c76..4d68e6f7ee 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -360,6 +360,11 @@ function formatExternalSource(source: SourceDocument) { * description: Column used for full text search if no property is specified in a Lucene-based search. Typically the message body of a log. * nullable: true * example: Body + * knownColumnsListExpression: + * type: string + * description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns. + * nullable: true + * example: Timestamp, Body, ServiceName * useTextIndexForImplicitColumn: * type: string * enum: [auto, enabled, disabled] @@ -538,6 +543,11 @@ function formatExternalSource(source: SourceDocument) { * description: Column used for full text search if no property is specified in a Lucene-based search. Typically the message body of a log. * nullable: true * example: SpanName + * knownColumnsListExpression: + * type: string + * description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns. + * nullable: true + * example: Timestamp, Body, ServiceName * useTextIndexForImplicitColumn: * type: string * enum: [auto, enabled, disabled] diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index 77cc39c054..e91abd5e31 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -63,12 +63,21 @@ export function useRowData({ ) : []; + // `SELECT *` can fail against a Distributed/Merge table whose underlying + // target tables declare different column sets. When the source declares a + // "known columns" list (columns known to exist across all target tables) we + // select that instead of `*` when fetching full row data. + const knownColumns = + isLogSource(source) || isTraceSource(source) + ? source.knownColumnsListExpression?.trim() + : undefined; + const queryResult = useQueriedChartConfig( { connection: source.connection, select: [ { - valueExpression: '*', + valueExpression: knownColumns || '*', }, { valueExpression: getDisplayedTimestampValueExpression(source), diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index 1b1929dc28..ec5cb4a314 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -42,6 +42,7 @@ import ContextSubpanel from './ContextSidePanel'; import DBInfraPanel from './DBInfraPanel'; import { RowDataPanel, rowHasK8sContext, useRowData } from './DBRowDataPanel'; import { RowOverviewPanel } from './DBRowOverviewPanel'; +import { DBRowSidePanelErrorState } from './DBRowSidePanelErrorState'; import { DBSessionPanel, useSessionId } from './DBSessionPanel'; import DBTracePanel from './DBTracePanel'; import { INITIAL_DRAWER_WIDTH_PERCENT } from './DrawerUtils'; @@ -120,6 +121,8 @@ const DBRowSidePanel = ({ data: rowData, isLoading: isRowLoading, isSuccess: isRowSuccess, + isError: isRowError, + error: rowError, } = useRowData({ source, rowId, @@ -308,6 +311,13 @@ const DBRowSidePanel = ({ } if (!isRowSuccess) { + if (isRowError && rowError) { + return ( + + + + ); + } return
Error loading row data
; } diff --git a/packages/app/src/components/DBRowSidePanelErrorState.tsx b/packages/app/src/components/DBRowSidePanelErrorState.tsx new file mode 100644 index 0000000000..bb491eb802 --- /dev/null +++ b/packages/app/src/components/DBRowSidePanelErrorState.tsx @@ -0,0 +1,138 @@ +import Link from 'next/link'; +import { + ClickHouseQueryError, + isMissingColumnError, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { + isLogSource, + isTraceSource, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Alert, Anchor, Button, Code, Modal, Stack, Text } from '@mantine/core'; +import { useDisclosure } from '@mantine/hooks'; +import { IconAlertTriangle } from '@tabler/icons-react'; + +import { IS_LOCAL_MODE } from '@/config'; +import { useTableMetadata } from '@/hooks/useMetadata'; + +import { TableSourceForm } from './Sources/SourceForm'; +import { SQLPreview } from './ChartSQLPreview'; + +/** A hint to the user that setting the Known Columns List may resolve SELECT * failures on Distributed or Merge tables */ +function KnownColumnsListHint({ + onEditClick, + source, +}: { + onEditClick?: () => void; + source: TSource; +}) { + const hasKnownColumnsList = + (isLogSource(source) || isTraceSource(source)) && + !!source.knownColumnsListExpression; + + const message = hasKnownColumnsList ? ( + <> + This query may have failed due to an invalid Known Columns List{' '} + configuration. Check the Known Columns List for this source and + ensure that it references valid columns that exist in all target tables of + the Distributed or Merge table. + + ) : ( + <> + This query may have failed due to a{' '} + + SELECT * + {' '} + query on a Distributed table that declares columns missing in one or more + of its target tables. If this is the case, the{' '} + + SELECT * + {' '} + can be overridden by setting a Known Columns List for this source. + + ); + + return ( + } + title="SELECT * failure on Distributed or Merge table" + > + + {message} + {IS_LOCAL_MODE ? ( + + ) : ( + + Edit source settings + + )} + + + ); +} + +export function DBRowSidePanelErrorState({ + error, + source, +}: { + error: Error | ClickHouseQueryError; + source: TSource; +}) { + const [editOpened, editModal] = useDisclosure(false); + const { data: tableMetadata } = useTableMetadata(tcFromSource(source)); + + const showHint = + isMissingColumnError(error) && !!tableMetadata?.isPointerTable; + + return ( + + Error loading row data + + {showHint && ( + + )} + + + + Error Message: + + + {error.message} + + {error instanceof ClickHouseQueryError && ( + <> + + Sent Query: + + + + )} + + {IS_LOCAL_MODE && ( + + + + )} + + ); +} diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 18a64cdb3a..9ecc3858a0 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -179,6 +179,8 @@ function setCorrelationFieldValue( } const DEFAULT_DATABASE = 'default'; +const KNOWN_COLUMNS_EXPRESSION_HELP_TEXT = + 'For Distributed table sources whose target tables have non-matching column sets. Provide a list of columns supported across all target tables; it is used instead of SELECT * when fetching full row data (e.g. the row side panel). Leave blank to select all columns. This should be a comma-separated list of column names - do not include non-column expressions or aliases.'; // Placeholder written into from.databaseName / from.tableName when the // selected connection is Prometheus-only. @@ -1458,6 +1460,22 @@ function LogTableModelForm(props: TableModelProps) { sourceKind={SourceKind.Log} tableConnection={tableConnection} /> + + + + + + ({ + useQueriedChartConfig: jest.fn(), +})); + +const mockUseQueriedChartConfig = useQueriedChartConfig as jest.Mock; describe('DBRowDataPanel', () => { + const source: TLogSource = { + id: 'source-id', + kind: SourceKind.Log, + name: 'logs', + connection: 'conn-id', + from: { databaseName: 'default', tableName: 'logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, Body', + bodyExpression: 'Body', + }; + + beforeEach(() => { + jest.clearAllMocks(); + mockUseQueriedChartConfig.mockReturnValue({ + data: { + data: [], + meta: [], + rows: 0, + isComplete: true, + }, + isLoading: false, + isError: false, + isSuccess: true, + }); + }); + describe('getJSONColumnNames', () => { it('should return JSON column names', () => { const meta = [ @@ -16,6 +53,30 @@ describe('DBRowDataPanel', () => { }); }); + it('selects `*` when the source has no Known Columns List', () => { + renderHook(() => useRowData({ source, rowId: "id='abc123'" })); + + const [config] = mockUseQueriedChartConfig.mock.calls[0]; + expect(config.select[0]).toEqual({ valueExpression: '*' }); + }); + + it('selects the Known Columns List instead of `*` when set', () => { + const sourceWithKnownColumns: TLogSource = { + ...source, + knownColumnsListExpression: 'Timestamp, Body, ServiceName', + }; + + renderHook(() => + useRowData({ source: sourceWithKnownColumns, rowId: "id='abc123'" }), + ); + + const [config] = mockUseQueriedChartConfig.mock.calls[0]; + expect(config.select[0]).toEqual({ + valueExpression: 'Timestamp, Body, ServiceName', + }); + expect(config.select).not.toContainEqual({ valueExpression: '*' }); + }); + // Regression test for the OSS #2357 conflict-resolution merge. The // composed result wraps `Event Attributes` in a length check from // origin/main AND passes `mapColumns={mapColumns}` through to the diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index f7513f7cd0..93b9afde91 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -89,7 +89,10 @@ export function useColumns( ) { const metadata = useMetadataWithSettings(); return useQuery({ - queryKey: ['useMetadata.useColumns', { databaseName, tableName }], + queryKey: [ + 'useMetadata.useColumns', + { connectionId, databaseName, tableName }, + ], queryFn: async () => { return metadata.getColumns({ databaseName, @@ -297,7 +300,10 @@ export function useTableMetadata( ) { const metadata = useMetadataWithSettings(); return useQuery({ - queryKey: ['useMetadata.useTableMetadata', { databaseName, tableName }], + queryKey: [ + 'useMetadata.useTableMetadata', + { connectionId, databaseName, tableName }, + ], queryFn: async () => { return await metadata.getTableMetadata({ databaseName, diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 601e132602..166d66ab1f 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -213,6 +213,58 @@ describe('Metadata', () => { expect(result!.partition_key).toEqual('column1'); }); + it('does not set isPointerTable for tables that hold their own data', async () => { + const mockTableMetadata = { + database: 'test_db', + name: 'test_table', + engine: 'MergeTree', + engine_full: 'MergeTree() ORDER BY id', + partition_key: 'column1', + sorting_key: 'column2', + primary_key: 'column3', + }; + + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + data: [mockTableMetadata], + }), + }); + + const result = await metadata.getTableMetadata({ + databaseName: 'test_db', + tableName: 'test_table', + connectionId: 'test_connection', + }); + + expect(result!.isPointerTable).toBeFalsy(); + }); + + it('sets isPointerTable for a Merge table', async () => { + const mockTableMetadata = { + database: 'test_db', + name: 'merge_table', + engine: 'Merge', + engine_full: "Merge('test_db', '^events_')", + partition_key: '', + sorting_key: '', + primary_key: '', + }; + + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + data: [mockTableMetadata], + }), + }); + + const result = await metadata.getTableMetadata({ + databaseName: 'test_db', + tableName: 'merge_table', + connectionId: 'test_connection', + }); + + expect(result!.isPointerTable).toBe(true); + }); + it('should query via cluster() for Distributed table underlying metadata', async () => { const distributedMetadata = { database: 'test_db', diff --git a/packages/common-utils/src/clickhouse/__tests__/index.test.ts b/packages/common-utils/src/clickhouse/__tests__/index.test.ts index 80b85bbb22..a8bab44109 100644 --- a/packages/common-utils/src/clickhouse/__tests__/index.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/index.test.ts @@ -1,9 +1,48 @@ import { + ClickHouseQueryError, convertCHDataTypeToJSType, extractColumnReferencesFromKey, + isMissingColumnError, JSDataType, } from '@/clickhouse'; +describe('isMissingColumnError', () => { + it.each([ + 'Code: 47. DB::Exception: Unknown identifier: foo', + 'DB::Exception: Unknown expression identifier `bar`', + 'UNKNOWN_IDENTIFIER', + 'Missing columns: baz while processing query', + 'Code: 16. NO_SUCH_COLUMN_IN_TABLE', + 'There is no column with name qux', + 'There is no column with name qux', + "Identifier '__table1.skip_indices' cannot be resolved from table with name __table1.", + ])('returns true for missing-column error: %s', msg => { + expect(isMissingColumnError(new Error(msg))).toBe(true); + }); + + it('detects the error via a ClickHouseQueryError instance', () => { + expect( + isMissingColumnError( + new ClickHouseQueryError('Missing columns: foo', 'SELECT * FROM t'), + ), + ).toBe(true); + }); + + it.each([ + 'Code: 60. DB::Exception: Table default.foo does not exist', + 'Syntax error: failed at position 10', + 'Timeout exceeded', + '', + ])('returns false for unrelated error: %s', msg => { + expect(isMissingColumnError(new Error(msg))).toBe(false); + }); + + it('handles non-Error inputs', () => { + expect(isMissingColumnError(undefined)).toBe(false); + expect(isMissingColumnError('Unknown identifier: x')).toBe(true); + }); +}); + describe('extractColumnReferencesFromKey', () => { // Suppress expected console.error from parse failures in edge-case tests beforeAll(() => { diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 2dee2384a4..3123baec9a 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -299,6 +299,19 @@ export class ClickHouseQueryError extends Error { } } +/** + * Heuristically detects whether a query error is a ClickHouse "missing/unknown + * column" error. Useful for surfacing actionable hints (e.g. when a `SELECT *` + * against a Distributed/Merge table references a column absent from some target + * tables). + */ +export function isMissingColumnError(error: unknown): boolean { + const msg = error instanceof Error ? error.message : String(error ?? ''); + return /Unknown (expression|identifier)|UNKNOWN_IDENTIFIER|Missing columns|NO_SUCH_COLUMN_IN_TABLE|There is no column|cannot be resolved/i.test( + msg, + ); +} + /** * Returns columns referenced in given expression, where the expression is a comma-separated list of SQL expressions * E.g. "id, toStartOfInterval(timestamp, toIntervalDay(3)), user_id, json.a.b". diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 5fc5588dad..454c38ce52 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -154,6 +154,12 @@ export type TableMetadata = { create_table_query: string; /** DDL for the local (non-distributed) table, when the table is Distributed */ create_local_table_query?: string; + /** + * True when the queried table routes to other tables rather than holding its + * own data — i.e. a Distributed or Merge table (whose underlying target + * tables may declare differing column sets). + **/ + isPointerTable?: boolean; /** Note: This will contain the engine_full of the local table, when the table is Distributed */ engine_full: string; as_select: string; @@ -979,6 +985,7 @@ export class Metadata { // For Distributed tables, fetch metadata of the underlying local table to get correct partition key, sorting key, etc. if (tableMetadata?.engine === 'Distributed') { + tableMetadata.isPointerTable = true; try { const { cluster, database, table } = getDistributedTableArgs(tableMetadata) ?? {}; @@ -1028,6 +1035,12 @@ export class Metadata { } } + // Merge tables (including a Distributed table whose local table is a Merge + // table) also route to other tables rather than holding their own data. + if (tableMetadata?.engine === 'Merge') { + tableMetadata.isPointerTable = true; + } + // partition_key which includes parenthesis, unlike other keys such as 'primary_key' or 'sorting_key' if ( tableMetadata?.partition_key.startsWith('(') && diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index f7ea805eda..e1a2e79974 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1745,6 +1745,7 @@ export const LogSourceSchema = BaseSourceSchema.extend({ traceIdExpression: z.string().optional(), spanIdExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), + knownColumnsListExpression: z.string().optional(), /** * @deprecated Application-side SQL predicate AND'd into every query against * the source. Not a security boundary; bypassable by direct table SELECT. @@ -1795,6 +1796,7 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ eventAttributesExpression: z.string().optional(), spanEventsValueExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), + knownColumnsListExpression: z.string().optional(), displayedTimestampValueExpression: z.string().optional(), highlightedTraceAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(), From 9db2bbcd9c702e6bca713fff428a744d150509e6 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Jun 2026 13:01:05 -0400 Subject: [PATCH 116/728] ci: alert Slack on external-contributor issues & PRs (#2542) --- .../workflows/external-contributor-alerts.yml | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 .github/workflows/external-contributor-alerts.yml diff --git a/.github/workflows/external-contributor-alerts.yml b/.github/workflows/external-contributor-alerts.yml new file mode 100644 index 0000000000..dc31813d79 --- /dev/null +++ b/.github/workflows/external-contributor-alerts.yml @@ -0,0 +1,101 @@ +name: Alert on external contributions + +# Posts to Slack when an EXTERNAL contributor (author is not OWNER/MEMBER/ +# COLLABORATOR) opens an issue or PR, and adds an `external` label for triage. +# +# Why post directly instead of via the GitHub->Slack app's label subscription: +# that app only notifies on open/close/reopen and does NOT fire when a label is +# added after open — so a label-then-subscribe approach never alerts. Posting +# from the workflow itself is reliable and real-time. +# +# Drop-in per-repo workflow. Needs a SLACK_WEBHOOK_OSS_PRS secret — an org-level +# secret scoped to your OSS repos covers all of them at once. Uses +# pull_request_target so the token/secret are available for fork PRs; safe +# because it never checks out the PR's code (reads event metadata only). + +on: + issues: + types: [opened, reopened] + pull_request_target: + types: [opened, reopened, ready_for_review] + +permissions: + issues: write + pull-requests: write + +concurrency: + group: + external-contrib-${{ github.event.issue.number || + github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + alert: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v9 + env: + SLACK_WEBHOOK_OSS_PRS: ${{ secrets.SLACK_WEBHOOK_OSS_PRS }} + with: + script: | + const LABEL = 'external'; + const item = context.payload.issue ?? context.payload.pull_request; + if (!item) return; + + // Skip draft PRs — they re-trigger via ready_for_review. (Issues have no draft.) + if (item.draft) { core.info(`Skipping draft PR #${item.number}`); return; } + // Skip bots (Dependabot, etc.). + if (item.user?.type === 'Bot') { core.info(`Skipping bot ${item.user?.login}`); return; } + // External == author has no write association with the repo. + const INTERNAL = ['OWNER', 'MEMBER', 'COLLABORATOR']; + if (INTERNAL.includes(item.author_association)) { + core.info(`#${item.number} by ${item.user?.login} is internal (${item.author_association}); skipping.`); + return; + } + // The `external` label doubles as our "already alerted" marker, so a + // reopen (label persists) won't post to Slack twice. + if (item.labels?.some(l => l.name === LABEL)) { core.info(`#${item.number} already handled.`); return; } + + const { owner, repo } = context.repo; + + // Deliver the alert FIRST. We only mark the item handled (label it) + // after a confirmed send, so an unset/failing webhook leaves it + // unlabeled and a later event (reopen / ready_for_review) retries, + // rather than silently swallowing the required alert. + const webhook = process.env.SLACK_WEBHOOK_OSS_PRS; + if (!webhook) { + core.warning('SLACK_WEBHOOK_OSS_PRS not set — no alert sent and not labeled (will retry on next event).'); + return; + } + const kind = context.payload.pull_request ? 'PR' : 'issue'; + const text = `:sparkles: *New external ${kind}* in \`${owner}/${repo}\`\n` + + `<${item.html_url}|#${item.number}: ${item.title}>\n` + + `by \`${item.user.login}\` (${item.author_association})`; + // JSON.stringify safely escapes the attacker-controlled title. + let delivered = false; + try { + const res = await fetch(webhook, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ blocks: [{ type: 'section', text: { type: 'mrkdwn', text } }] }), + }); + delivered = res.ok; + if (!delivered) core.setFailed(`Slack post failed (${res.status}); leaving #${item.number} unlabeled to retry.`); + } catch (e) { + core.setFailed(`Slack post errored (${e.message}); leaving #${item.number} unlabeled to retry.`); + } + if (!delivered) return; + + // Delivered — mark handled by labeling (race-safe). The label also + // dedupes reopens via the guard above. A label failure here at worst + // risks one duplicate alert on a future reopen, never a lost alert. + try { + await github.rest.issues.createLabel({ + owner, repo, name: LABEL, color: 'D4C5F9', + description: 'Opened by an external contributor', + }); + core.info(`Created '${LABEL}' label.`); + } catch (e) { + if (e.status !== 422) throw e; // 422 = already exists; anything else is real + } + await github.rest.issues.addLabels({ owner, repo, issue_number: item.number, labels: [LABEL] }); + core.info(`Alerted Slack and labeled #${item.number} (${item.author_association}).`); From 9bbf680797faec8f7f1c7524d1702b37ad1e0cd5 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:28:33 -0400 Subject: [PATCH 117/728] fix: bug preventing deletion of metadataMVs from source (#2540) ## Summary Fixes a bug that prevented deleting some fields from a Source. Adds a regression test for the main interesting one, metadataMVs. ### References - Linear Issue: Closes HDX-4645 --- .changeset/wise-jobs-relax.md | 6 ++ packages/api/src/controllers/sources.ts | 4 +- .../src/routers/api/__tests__/sources.test.ts | 82 ++++++++++++++++++ .../app/tests/e2e/features/sources.spec.ts | 83 +++++++++++++++++++ .../app/tests/e2e/page-objects/SearchPage.ts | 4 + 5 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 .changeset/wise-jobs-relax.md diff --git a/.changeset/wise-jobs-relax.md b/.changeset/wise-jobs-relax.md new file mode 100644 index 0000000000..f860cc1e17 --- /dev/null +++ b/.changeset/wise-jobs-relax.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +fix: bug preventing deletion of nested subdocuments like metadataMVs diff --git a/packages/api/src/controllers/sources.ts b/packages/api/src/controllers/sources.ts index a34f931dff..26dae44717 100644 --- a/packages/api/src/controllers/sources.ts +++ b/packages/api/src/controllers/sources.ts @@ -61,8 +61,8 @@ export async function updateSource( // Same kind: simple update through the discriminator model if (existing.kind === source.kind) { - // @ts-expect-error The findOneAndUpdate method has incompatible type signatures but is actually safe - return getModelForKind(source.kind)?.findOneAndUpdate( + // @ts-expect-error The findOneAndReplace method has incompatible type signatures but is actually safe + return getModelForKind(source.kind)?.findOneAndReplace( { _id: sourceId, team }, source, { new: true }, diff --git a/packages/api/src/routers/api/__tests__/sources.test.ts b/packages/api/src/routers/api/__tests__/sources.test.ts index 1aeeeb7fbc..94acc1f905 100644 --- a/packages/api/src/routers/api/__tests__/sources.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.test.ts @@ -690,6 +690,88 @@ describe('sources router', () => { }); }); + describe('metadataMaterializedViews field', () => { + // Regression test for a bug where clearing the Metadata Materialized + // Views in the source form did not persist. The UI sets the field to + // undefined, JSON serialization drops it from the PUT payload, and the + // previous findOneAndUpdate-based controller silently preserved the + // old value because partial updates don't unset absent fields. + it('PUT /:id - removes metadataMaterializedViews when omitted from the payload', async () => { + const { agent, team } = await getLoggedInAgent(server); + + const source = await Source.create({ + ...MOCK_SOURCE, + team: team._id, + metadataMaterializedViews: { + keyRollupTable: 'test_table_key_rollup_15m', + kvRollupTable: 'test_table_kv_rollup_15m', + granularity: '15 minute', + }, + }); + + const created = await Source.findById(source._id).lean(); + if (created?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${created?.kind}`); + } + expect(created.metadataMaterializedViews).toMatchObject({ + keyRollupTable: 'test_table_key_rollup_15m', + kvRollupTable: 'test_table_kv_rollup_15m', + granularity: '15 minute', + }); + + await agent + .put(`/sources/${source._id}`) + .send({ + ...MOCK_SOURCE, + id: source._id.toString(), + }) + .expect(200); + + const updated = await Source.findById(source._id).lean(); + if (updated?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${updated?.kind}`); + } + expect(updated.metadataMaterializedViews).toBeUndefined(); + }); + + it('PUT /:id - updates metadataMaterializedViews when included in the payload', async () => { + const { agent, team } = await getLoggedInAgent(server); + + const source = await Source.create({ + ...MOCK_SOURCE, + team: team._id, + metadataMaterializedViews: { + keyRollupTable: 'old_key_rollup', + kvRollupTable: 'old_kv_rollup', + granularity: '15 minute', + }, + }); + + await agent + .put(`/sources/${source._id}`) + .send({ + ...MOCK_SOURCE, + id: source._id.toString(), + metadataMaterializedViews: { + keyRollupTable: 'new_key_rollup', + kvRollupTable: 'new_kv_rollup', + granularity: '1 hour', + }, + }) + .expect(200); + + const updated = await Source.findById(source._id).lean(); + if (updated?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${updated?.kind}`); + } + expect(updated.metadataMaterializedViews).toMatchObject({ + keyRollupTable: 'new_key_rollup', + kvRollupTable: 'new_kv_rollup', + granularity: '1 hour', + }); + }); + }); + describe('useTextIndexForImplicitColumn field', () => { it('POST / - persists useTextIndexForImplicitColumn on a Log source and returns it', async () => { const { agent } = await getLoggedInAgent(server); diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 3490a016d3..b37850d551 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -6,6 +6,7 @@ import { DEFAULT_METRICS_SOURCE_NAME, DEFAULT_SESSIONS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, + METADATA_MV_LOGS_SOURCE_NAME, } from '../utils/constants'; const COMMON_FIELDS = [ @@ -232,4 +233,86 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { } }, ); + + test( + 'source form sends the complete source on update (no field omission)', + { tag: ['@full-stack'] }, + async ({ page }) => { + // Pins the contract that updateSource relies on: saving from the + // source form must not drop any populated field. The controller + // uses findOneAndReplace, so any field omitted from the PUT body + // is silently deleted from MongoDB. If the frontend ever moves + // to a partial/PATCH-style payload, this test will fail because + // the before/after diff will show fields disappearing. + // + // METADATA_MV_LOGS has the broadest field coverage in fixtures, + // including metadataMaterializedViews — the field whose deletion + // bug motivated the controller change. + const logSources = await getSources(page, 'log'); + const sourceBefore = logSources.find( + (s: any) => s.name === METADATA_MV_LOGS_SOURCE_NAME, + ); + expect(sourceBefore).toBeDefined(); + expect(sourceBefore.metadataMaterializedViews).toBeDefined(); + const sourceId = sourceBefore._id; + + await searchPage.selectSource(METADATA_MV_LOGS_SOURCE_NAME); + await searchPage.openEditSourceModal(); + + // Gate on form hydration to avoid racing Save against + // react-hook-form's `values` reset. In full-stack mode "Edit + // source" navigates to /team and expands the source's + // TableSourceForm inline (no modal), so we scope by input name. + await expect(page.locator('input[name="name"]')).toHaveValue( + METADATA_MV_LOGS_SOURCE_NAME, + ); + + const putResponsePromise = page.waitForResponse( + res => + res.url().includes(`/sources/${sourceId}`) && + res.request().method() === 'PUT', + ); + await searchPage.saveSourceForm(); + + // The seeded source sets implicitColumnExpression without + // bodyExpression, which triggers the pairing-warnings dialog. + // The PUT only fires after the user confirms via "Save anyway". + await page.getByRole('button', { name: 'Save anyway' }).click(); + + const putResponse = await putResponsePromise; + expect(putResponse.ok()).toBeTruthy(); + + const sourcesAfter = await getSources(page, 'log'); + const sourceAfter = sourcesAfter.find((s: any) => s._id === sourceId); + expect(sourceAfter).toBeDefined(); + + // Specific regression: metadataMaterializedViews survived the + // form roundtrip with its user-meaningful fields intact. The + // embedded sub-document gets a fresh Mongoose-minted _id on + // each findOneAndReplace, which is fine — we only care that + // the rollup config the user configured is preserved. + expect(sourceAfter.metadataMaterializedViews).toMatchObject({ + keyRollupTable: sourceBefore.metadataMaterializedViews.keyRollupTable, + kvRollupTable: sourceBefore.metadataMaterializedViews.kvRollupTable, + granularity: sourceBefore.metadataMaterializedViews.granularity, + }); + + // Broader contract: every populated field present before the save + // is still present after the save. Server-managed bookkeeping + // fields are expected to differ (timestamps, version) or stay + // pinned (_id, team) on their own schedule. + const serverManagedKeys = new Set([ + '_id', + '__v', + 'team', + 'createdAt', + 'updatedAt', + ]); + for (const key of Object.keys(sourceBefore)) { + if (serverManagedKeys.has(key)) continue; + if (sourceBefore[key] == null) continue; + expect(sourceAfter).toHaveProperty(key); + } + }, + ); }); diff --git a/packages/app/tests/e2e/page-objects/SearchPage.ts b/packages/app/tests/e2e/page-objects/SearchPage.ts index 5b8dd5d36b..b34180224b 100644 --- a/packages/app/tests/e2e/page-objects/SearchPage.ts +++ b/packages/app/tests/e2e/page-objects/SearchPage.ts @@ -123,6 +123,10 @@ export class SearchPage { } } + async saveSourceForm() { + await this.page.getByRole('button', { name: 'Save Source' }).click(); + } + /** * Perform a search with the given query */ From dae3da543f556e45d7e7e1d42d8c61a3f8d1745d Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Jun 2026 16:56:46 -0400 Subject: [PATCH 118/728] ci: don't flag private org members as external contributors (#2546) Co-authored-by: Tom Alexander Co-authored-by: Claude Opus 4.8 (1M context) --- .../workflows/external-contributor-alerts.yml | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/external-contributor-alerts.yml b/.github/workflows/external-contributor-alerts.yml index dc31813d79..6350e081f1 100644 --- a/.github/workflows/external-contributor-alerts.yml +++ b/.github/workflows/external-contributor-alerts.yml @@ -46,18 +46,34 @@ jobs: if (item.draft) { core.info(`Skipping draft PR #${item.number}`); return; } // Skip bots (Dependabot, etc.). if (item.user?.type === 'Bot') { core.info(`Skipping bot ${item.user?.login}`); return; } - // External == author has no write association with the repo. - const INTERNAL = ['OWNER', 'MEMBER', 'COLLABORATOR']; - if (INTERNAL.includes(item.author_association)) { - core.info(`#${item.number} by ${item.user?.login} is internal (${item.author_association}); skipping.`); + // External == author has no write access to the repo. + const { owner, repo } = context.repo; + // Fast path (free): public members & direct collaborators. + let internal = ['OWNER', 'MEMBER', 'COLLABORATOR'].includes(item.author_association); + // Slow path: catches PRIVATE org members via their granted repo access. + // author_association only reports MEMBER for *public* members, so a + // member with concealed membership would otherwise be misflagged as + // external. Public repos grant everyone implicit 'read', so require + // triage+ (not permission != 'none'). + if (!internal) { + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner, repo, username: item.user.login, + }); + const p = data.user?.permissions ?? {}; + internal = !!(p.admin || p.maintain || p.push || p.triage); + } catch (e) { + if (e.status !== 404) throw e; // 404 = not a collaborator => external + } + } + if (internal) { + core.info(`#${item.number} by ${item.user?.login} is internal; skipping.`); return; } // The `external` label doubles as our "already alerted" marker, so a // reopen (label persists) won't post to Slack twice. if (item.labels?.some(l => l.name === LABEL)) { core.info(`#${item.number} already handled.`); return; } - const { owner, repo } = context.repo; - // Deliver the alert FIRST. We only mark the item handled (label it) // after a confirmed send, so an unset/failing webhook leaves it // unlabeled and a later event (reopen / ready_for_review) retries, From 63469fe0e911dafd97eb8aad6bc6c86fb597aabe Mon Sep 17 00:00:00 2001 From: Dan Hable <418679+dhable@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:12:26 -0500 Subject: [PATCH 119/728] feat(mcp): first-class metric source support (#2437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary **Why:** Metric sources were MVP-only in MCP — `describe_source` early-returned with just `metricTables`, builder tiles on metric sources saved but failed at render time, the query tools couldn't carry `metricName` / `metricType`, and the prompt told agents to fall back to raw SQL. Closes that gap. **What changed:** - **New tools** - `clickstack_list_metrics` — paginated metric-name catalog with `kind`, `namePattern` (ILIKE), time-window filters, and opaque cursor pagination. - `clickstack_describe_metric` — per-metric unit, description, attribute keys, sampled values. `kind` is required (paired with `metricName` by every upstream discovery tool, so requiring it eliminates the multi-kind probe path). - **Extended tools** - `clickstack_describe_source` now picks a representative metric table (gauge → sum → histogram) and runs the existing column / map-key / value-sampling pipeline against it, plus a per-kind metric-name sample. - `clickstack_timeseries` / `clickstack_table` accept `metricType`, `metricName`, `isDelta` on each select item, plus `aggFn:"increase"` for Sum counters. `valueExpression` defaults to `"Value"`. Emits a hint when the renderer's 20-group cap on `increase + groupBy` may apply. Reject up-front when `metricType` is paired with a non-metric source (or omitted on a metric source) instead of falling through to a cryptic ClickHouse error. - **Prompt** — replaces the "use raw SQL for metric tiles" workaround with positive discovery-workflow guidance and one worked example per supported kind. - **Out of scope**: `summary` and `"exponential histogram"` kinds (no renderer support). ## Live MCP testing findings & fixes Live testing against a populated metric source surfaced 5 issues. All are fixed in this PR. | # | Symptom | Root cause | Fix | |---|---|---|---| | 1 | `Cast to ObjectId failed for value "..." (type string) at path "_id" for model "Source"` leaked through `clickstack_describe_source` / `_list_metrics` / `_describe_metric` on a bad `sourceId` | `getSource()` didn't guard against malformed input | Pre-check `ObjectId.isValid` + try/catch around `findOne`; all tools surface their existing "Source not found" branch (`19bc4205`) | | 2 | `clickstack_timeseries` / `clickstack_table` accepted `metricType`+`metricName` on non-metric sources and failed at SQL render with `Unknown expression or function identifier "Value"` | No source-kind ↔ select-item shape guardrail in `runConfigTile` | New `assertSourceKindMatchesSelect()` helper called between source-not-found and the connection lookup — both directions of the mismatch return agent-actionable errors (`7001a915`) | | 3 | `metricTables` map in `clickstack_list_sources` / `_describe_source` responses included a stray `_id` ObjectId alongside the valid kind keys | Inline `type: { … }` Mongoose subdoc auto-adds `_id` | Declare a child `Schema(..., { _id: false })` for `metricTables`; defense-in-depth `sanitizeMetricTables()` at the MCP response boundary (`7ef9d9ca`) | | 4 | `clickstack_describe_metric` consistently timed out at 10 s on multi-kind metrics (e.g. `container.cpu.usage` lives in both gauge and sum) | Auto-detect path probed every populated kind serially, plus `fetchAttributeKeys` ran an unbounded `WHERE MetricName = ?` scan with no time filter | Require `kind` (every upstream tool already pairs it with name); bound `fetchAttributeKeys` with a CTE sample LIMIT + server-side `max_execution_time` (`30203f5e`) | | 5 | `clickstack_describe_source` on a populated metric source timed out on the cold call, then succeeded on retry | Stages 2–5 fell through to the no-rollup main-table-scan path without `timestampValueExpression`, so the helpers' time filter was silently dropped | Thread `source.timestampValueExpression` through to `getMapKeys`, both `getAllKeyValues` calls, and `sampleMetricNamesForKind` (`31bc21b5`) | ### Screenshots or video N/A — backend-only. ### How to test on Vercel preview N/A — non-UI change. ### References - Linear Issue: Closes [HDX-4347](https://linear.app/clickhouse/issue/HDX-4347/mcp-tools-improved-metrics-support) - Parent: HDX-4079 Co-authored-by: Brandon Pereira <7552738+brandon-pereira@users.noreply.github.com> --- .changeset/mcp-metric-source-support.md | 11 + MCP.md | 48 +- .../src/controllers/__tests__/sources.test.ts | 39 + packages/api/src/controllers/sources.ts | 18 +- .../mcp/__tests__/dashboards/prompts.test.ts | 78 +- .../mcp/__tests__/listMetricsCursor.test.ts | 104 +++ .../api/src/mcp/__tests__/metricKinds.test.ts | 86 ++ packages/api/src/mcp/__tests__/query.test.ts | 487 +++++++++++ .../api/src/mcp/__tests__/queryTool.test.ts | 242 +++++- .../api/src/mcp/__tests__/sources.test.ts | 545 +++++++++++- .../api/src/mcp/prompts/dashboards/content.ts | 28 +- .../api/src/mcp/tools/dashboards/schemas.ts | 75 +- packages/api/src/mcp/tools/query/helpers.ts | 216 ++++- packages/api/src/mcp/tools/query/schemas.ts | 247 +++++- packages/api/src/mcp/tools/query/table.ts | 55 +- .../api/src/mcp/tools/query/timeseries.ts | 101 ++- .../src/mcp/tools/sources/describeMetric.ts | 801 ++++++++++++++++++ .../src/mcp/tools/sources/describeSource.ts | 322 ++++++- packages/api/src/mcp/tools/sources/index.ts | 4 + .../api/src/mcp/tools/sources/listMetrics.ts | 504 +++++++++++ .../api/src/mcp/tools/sources/listSources.ts | 10 +- .../api/src/mcp/tools/sources/metricKinds.ts | 75 ++ packages/api/src/models/source.ts | 23 +- 23 files changed, 4012 insertions(+), 107 deletions(-) create mode 100644 .changeset/mcp-metric-source-support.md create mode 100644 packages/api/src/controllers/__tests__/sources.test.ts create mode 100644 packages/api/src/mcp/__tests__/listMetricsCursor.test.ts create mode 100644 packages/api/src/mcp/__tests__/metricKinds.test.ts create mode 100644 packages/api/src/mcp/tools/sources/describeMetric.ts create mode 100644 packages/api/src/mcp/tools/sources/listMetrics.ts create mode 100644 packages/api/src/mcp/tools/sources/metricKinds.ts diff --git a/.changeset/mcp-metric-source-support.md b/.changeset/mcp-metric-source-support.md new file mode 100644 index 0000000000..4dfd135051 --- /dev/null +++ b/.changeset/mcp-metric-source-support.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/api': minor +--- + +feat(mcp): first-class metric source support + +- Two new tools: `clickstack_list_metrics` paginates the metric-name catalog with optional kind / namePattern (ILIKE) / time-window filters and opaque cursor pagination; `clickstack_describe_metric` returns per-metric kind(s), unit, description, attribute keys, and sampled values (with kind auto-detection). +- `clickstack_describe_source` is metric-aware: picks a representative metric table (gauge → sum → histogram), runs column / map-key / value-sampling against it, and adds a per-kind metric-name sample. +- `clickstack_timeseries` and `clickstack_table` accept `metricType` (gauge / sum / histogram), `metricName`, and `isDelta` on each select item, plus `aggFn:"increase"` for Sum counters. `valueExpression` defaults to `"Value"` for metric sources. Surfaces the renderer's 20-group top-N cap on `increase + groupBy` as a neutral hint. +- Dashboard prompt's "use raw SQL for metric tiles" workaround is replaced with positive discovery-workflow guidance and one worked example per supported kind. +- `summary` and `"exponential histogram"` kinds remain out of scope (no query renderer support yet). diff --git a/MCP.md b/MCP.md index 3616e5c9a6..312276cb57 100644 --- a/MCP.md +++ b/MCP.md @@ -109,9 +109,11 @@ with: | Tool | Description | | ----------------------------- | -------------------------------------------------------------------------------------------- | | `clickstack_list_sources` | List all data sources and connections as a lightweight catalog (IDs, names, kinds) | -| `clickstack_describe_source` | Full column schema, attribute keys, and sampled low-cardinality values for a single source | -| `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart | -| `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart | +| `clickstack_describe_source` | Full column schema, attribute keys, and sampled low-cardinality values for a single source; for metric sources also returns a per-kind metric-name sample | +| `clickstack_list_metrics` | Paginated catalog of metric names on a metric source with optional kind / namePattern / time-window filters | +| `clickstack_describe_metric` | Per-metric drill-down: kind(s), unit, description, attribute keys per map column, and sampled values | +| `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart (works on log, trace, and metric sources) | +| `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart (works on log, trace, and metric sources) | | `clickstack_search` | Browse individual log, event, or trace rows | | `clickstack_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | | `clickstack_event_deltas` | Compare two row groups and rank properties by how their value distributions differ | @@ -127,3 +129,43 @@ with: | `clickstack_trace_waterfall` | Fetch all spans in a single trace as a parent/child waterfall tree with optional correlated logs | | `clickstack_trace_top_time_consuming_operations` | Aggregate breakdown of child operations by cumulative time across matching parent traces | | `clickstack_get_webhook` | List available webhook destinations for use as alert notification channels | + +### Metric Sources + +`clickstack_timeseries`, `clickstack_table`, and the dashboard builder tile +tools accept metric sources transparently. Each `select` item on a metric +query must set `metricType` (`"gauge"`, `"sum"`, or `"histogram"`) and +`metricName` (the OTel metric name, e.g. `system.cpu.utilization`). +`valueExpression` defaults to `"Value"` when omitted, so a typical metric +series looks like: + +```jsonc +{ "aggFn": "avg", "metricType": "gauge", "metricName": "system.cpu.utilization" } +``` + +Per-kind aggregation guidance: + +- **Gauge**: `avg`, `last_value`, `min`, or `max`. Set `"isDelta": true` for + Prometheus-style delta over each bucket. +- **Sum (counter)**: `"increase"` returns the per-bucket counter increase + (reset-aware), or `sum` / `avg` on the computed rate. `increase` combined + with `groupBy` is capped at the top 20 groups by the renderer; the tool + emits a neutral hint when the cap may apply. +- **Histogram**: `"quantile"` with `level` ∈ {0.5, 0.9, 0.95, 0.99} for + percentiles, or `"count"` for the total bucket count. + +`summary` and `"exponential histogram"` metric kinds are not yet supported +by the query renderer. + +Discovery workflow for metrics: + +1. `clickstack_list_sources` to find the metric source ID and its + `metricTables` map. +2. `clickstack_describe_source` to see columns, attribute keys, and a + per-kind metric-name sample. +3. `clickstack_list_metrics` for paginated catalog access with optional + kind / namePattern / time-window filters; pass `nextCursor` unchanged + for the next page. +4. `clickstack_describe_metric` to drill into a specific metric and + discover its attribute keys + sampled values before authoring queries. +5. `clickstack_timeseries` / `clickstack_table` to chart the metric. diff --git a/packages/api/src/controllers/__tests__/sources.test.ts b/packages/api/src/controllers/__tests__/sources.test.ts new file mode 100644 index 0000000000..71a7d73466 --- /dev/null +++ b/packages/api/src/controllers/__tests__/sources.test.ts @@ -0,0 +1,39 @@ +import mongoose from 'mongoose'; + +import { getSource } from '@/controllers/sources'; +import { clearDBCollections, closeDB, connectDB } from '@/fixtures'; + +describe('sources controller', () => { + beforeAll(async () => { + await connectDB(); + }); + + afterEach(async () => { + await clearDBCollections(); + }); + + afterAll(async () => { + await closeDB(); + }); + + describe('getSource', () => { + it('returns null when sourceId is not a valid ObjectId', async () => { + // Non-ObjectId strings used to bubble a Mongoose CastError up + // through MCP tools as "Cast to ObjectId failed for value ...". + // The wrapper now short-circuits before hitting MongoDB so the + // caller's not-found branch fires cleanly. + const team = new mongoose.Types.ObjectId().toString(); + + expect(await getSource(team, 'not-an-objectid')).toBeNull(); + expect(await getSource(team, '')).toBeNull(); + expect(await getSource(team, ' ')).toBeNull(); + }); + + it('returns null for a well-formed but missing ObjectId', async () => { + const team = new mongoose.Types.ObjectId().toString(); + const missingSourceId = new mongoose.Types.ObjectId().toString(); + + expect(await getSource(team, missingSourceId)).toBeNull(); + }); + }); +}); diff --git a/packages/api/src/controllers/sources.ts b/packages/api/src/controllers/sources.ts index 26dae44717..8d98094a94 100644 --- a/packages/api/src/controllers/sources.ts +++ b/packages/api/src/controllers/sources.ts @@ -1,4 +1,5 @@ import { SourceKind, SourceSchema } from '@hyperdx/common-utils/dist/types'; +import mongoose from 'mongoose'; import { ISourceInput, @@ -35,8 +36,21 @@ export function getSources(team: string) { return Source.find({ team }); } -export function getSource(team: string, sourceId: string) { - return Source.findOne({ _id: sourceId, team }); +export async function getSource(team: string, sourceId: string) { + // Pre-check the sourceId shape so a non-ObjectId input returns null + // (the caller's "not found" branch) instead of bubbling a Mongoose + // CastError. + if (!mongoose.Types.ObjectId.isValid(sourceId)) { + return null; + } + try { + return await Source.findOne({ _id: sourceId, team }); + } catch { + // Defense-in-depth: if Mongoose still throws (e.g. a future cast + // path), treat it as "not found" so the caller can surface a clean + // error. + return null; + } } type DistributiveOmit = T extends T diff --git a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts index 20a442e126..c8368712f0 100644 --- a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts @@ -50,25 +50,39 @@ describe('MCP Dashboard Prompts', () => { expect(numberFormat.toLowerCase()).toContain('per-series'); }); - it('declares the MCP metric-authoring gap rather than referencing fields that do not exist', () => { - // The MCP select-item schema does not carry metricName / metricType, - // so any prompt that tells the model to author a metric tile via - // the builder path is teaching it to ship JSON that gets silently - // stripped. Guard with an explicit assertion so a future diff that - // restores metric-tile guidance fails this test loudly. + it('documents the metric-source builder support with the discovery workflow', () => { + // Builder tiles on a metric source now work via the metricType + + // metricName + isDelta fields on each select item, with metricTables + // threaded through runConfigTile's builder branch. The prompt has to + // teach the model the discovery workflow (list_sources -> describe + // _source -> list_metrics -> describe_metric -> timeseries|table) + // and the per-kind aggFn rules so it doesn't fall back to raw SQL. const prompt = buildQueryGuidePrompt(); - expect(prompt).not.toMatch(/metricName \+ metricType/); - expect(prompt).not.toMatch(/exactly 1 select item with metricName/); - const constraintsIdx = prompt.indexOf('== PER-TILE TYPE CONSTRAINTS =='); - const constraintsBody = prompt.slice(constraintsIdx); - // The constraints section closes with an explicit note that builder - // tiles on a metric source are not reliable today, with a fallback - // recipe to raw SQL. Anchor on the phrase so a future diff that - // drops the gap-acknowledgement fails loudly. - expect(constraintsBody).toMatch( + const metricsIdx = prompt.indexOf('== METRIC SOURCES =='); + expect(metricsIdx).toBeGreaterThan(-1); + const metricsBody = prompt.slice( + metricsIdx, + prompt.indexOf('\n== ', metricsIdx + 1), + ); + // The four supported metric select fields are named. + expect(metricsBody).toContain('metricType'); + expect(metricsBody).toContain('metricName'); + expect(metricsBody).toContain('isDelta'); + // Per-kind aggregation guidance is present. + expect(metricsBody).toMatch(/gauge\s+Use aggFn:"last_value"/); + expect(metricsBody).toMatch(/sum\s+Use aggFn:"increase"/); + expect(metricsBody).toMatch(/histogram\s+Use aggFn:"quantile"/); + // The 20-group cap on increase + groupBy is documented. + expect(metricsBody).toMatch(/top 20 groups/); + // The four-tool discovery chain is documented in order. + expect(metricsBody).toContain('clickstack_describe_source'); + expect(metricsBody).toContain('clickstack_list_metrics'); + expect(metricsBody).toContain('clickstack_describe_metric'); + // The old "use raw SQL for metric tiles" workaround language is gone. + expect(prompt).not.toMatch( /Authoring builder tiles on a metric source is not reliable/, ); - expect(constraintsBody).toMatch(/MCP select-item shape does not carry/); + expect(prompt).not.toMatch(/Both table name and UUID are empty/); }); it('documents table-tile onClick linking features', () => { @@ -229,16 +243,30 @@ describe('MCP Dashboard Prompts', () => { ); }); - it('flags the metric source builder gap', () => { - // Builder tiles on metric sources currently save but render with - // "Both table name and UUID are empty". Claude went 100% raw SQL - // across 21 metric tiles for this reason; the prompt has to make - // the workaround obvious so the model doesn't try the builder - // first and fail silently. + it('walks the metric discovery workflow end-to-end with worked examples', () => { + // Metric source builder tiles now work — the prompt teaches the + // model how to find, characterise, and chart a metric without + // falling through to raw SQL. The examples cover one tile per + // supported metric kind so the pattern is unambiguous. const prompt = buildQueryGuidePrompt(); - expect(prompt).toMatch(/Both table name and UUID are empty/); - expect(prompt).toMatch(/use a raw SQL tile/); - expect(prompt).toMatch(/otel_metrics_gauge/); + const metricsIdx = prompt.indexOf('== METRIC SOURCES =='); + const metricsBody = prompt.slice( + metricsIdx, + prompt.indexOf('\n== ', metricsIdx + 1), + ); + // The five-step discovery workflow is enumerated. + expect(metricsBody).toMatch(/clickstack_list_sources/); + expect(metricsBody).toMatch(/clickstack_describe_source/); + expect(metricsBody).toMatch(/clickstack_list_metrics/); + expect(metricsBody).toMatch(/clickstack_describe_metric/); + expect(metricsBody).toMatch(/clickstack_timeseries/); + // One worked example per supported kind, each using a real OTel + // metric name so the agent has a concrete template. + expect(metricsBody).toContain('system.cpu.utilization'); + expect(metricsBody).toContain('http.server.request.count'); + expect(metricsBody).toContain('http.server.request.duration'); + // valueExpression default is documented. + expect(metricsBody).toMatch(/valueExpression defaults to "Value"/); }); it('contains no em-dashes or en-dashes used as em-dashes', () => { diff --git a/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts b/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts new file mode 100644 index 0000000000..bc4abb20e3 --- /dev/null +++ b/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts @@ -0,0 +1,104 @@ +// Mock heavy dependencies that break in unit-test context (no ClickHouse/Mongo) +jest.mock('@/models/source', () => ({})); +jest.mock('@/controllers/sources', () => ({})); +jest.mock('@/controllers/connection', () => ({})); +jest.mock('@/utils/trimToolResponse', () => ({ + trimToolResponse: (data: unknown) => ({ data, isTrimmed: false }), +})); + +import { decodeCursor, encodeCursor } from '@/mcp/tools/sources/listMetrics'; + +describe('listMetrics cursor', () => { + describe('encodeCursor / decodeCursor round-trip', () => { + it('round-trips a gauge cursor', () => { + const payload = { kind: 'gauge' as const, lastName: 'system.cpu.idle' }; + const encoded = encodeCursor(payload); + expect(encoded).toMatch(/^[A-Za-z0-9+/]+=*$/); // base64 + expect(decodeCursor(encoded)).toEqual(payload); + }); + + it('round-trips a sum cursor', () => { + const payload = { + kind: 'sum' as const, + lastName: 'http.server.request.count', + }; + expect(decodeCursor(encodeCursor(payload))).toEqual(payload); + }); + + it('round-trips a histogram cursor', () => { + const payload = { + kind: 'histogram' as const, + lastName: 'http.server.request.duration', + }; + expect(decodeCursor(encodeCursor(payload))).toEqual(payload); + }); + + it('round-trips metric names with dots, dashes, and unicode', () => { + const payload = { + kind: 'gauge' as const, + lastName: 'system.cpu.utilization-µ.naïve', + }; + expect(decodeCursor(encodeCursor(payload))).toEqual(payload); + }); + }); + + describe('decodeCursor rejection cases', () => { + it('returns null for non-base64 input', () => { + expect(decodeCursor('not-base64!')).toBeNull(); + }); + + it('returns null for base64 of invalid JSON', () => { + const garbage = Buffer.from('not json').toString('base64'); + expect(decodeCursor(garbage)).toBeNull(); + }); + + it('returns null when kind is missing', () => { + const malformed = Buffer.from(JSON.stringify({ lastName: 'x' })).toString( + 'base64', + ); + expect(decodeCursor(malformed)).toBeNull(); + }); + + it('returns null when lastName is missing', () => { + const malformed = Buffer.from(JSON.stringify({ kind: 'gauge' })).toString( + 'base64', + ); + expect(decodeCursor(malformed)).toBeNull(); + }); + + it('returns null when kind is not a queryable metric kind', () => { + const summaryCursor = Buffer.from( + JSON.stringify({ kind: 'summary', lastName: 'x' }), + ).toString('base64'); + expect(decodeCursor(summaryCursor)).toBeNull(); + + const expHistCursor = Buffer.from( + JSON.stringify({ kind: 'exponential histogram', lastName: 'x' }), + ).toString('base64'); + expect(decodeCursor(expHistCursor)).toBeNull(); + + const bogusCursor = Buffer.from( + JSON.stringify({ kind: 'bogus', lastName: 'x' }), + ).toString('base64'); + expect(decodeCursor(bogusCursor)).toBeNull(); + }); + + it('returns null when kind has the wrong type', () => { + const malformed = Buffer.from( + JSON.stringify({ kind: 1, lastName: 'x' }), + ).toString('base64'); + expect(decodeCursor(malformed)).toBeNull(); + }); + + it('returns null when lastName has the wrong type', () => { + const malformed = Buffer.from( + JSON.stringify({ kind: 'gauge', lastName: 42 }), + ).toString('base64'); + expect(decodeCursor(malformed)).toBeNull(); + }); + + it('returns null for empty string', () => { + expect(decodeCursor('')).toBeNull(); + }); + }); +}); diff --git a/packages/api/src/mcp/__tests__/metricKinds.test.ts b/packages/api/src/mcp/__tests__/metricKinds.test.ts new file mode 100644 index 0000000000..87bc664109 --- /dev/null +++ b/packages/api/src/mcp/__tests__/metricKinds.test.ts @@ -0,0 +1,86 @@ +import { sanitizeMetricTables } from '@/mcp/tools/sources/metricKinds'; + +describe('sanitizeMetricTables', () => { + it('returns undefined for null / undefined input', () => { + expect(sanitizeMetricTables(undefined)).toBeUndefined(); + expect(sanitizeMetricTables(null)).toBeUndefined(); + }); + + it('preserves valid queryable kind entries', () => { + expect( + sanitizeMetricTables({ + gauge: 'otel_metrics_gauge', + sum: 'otel_metrics_sum', + histogram: 'otel_metrics_histogram', + }), + ).toEqual({ + gauge: 'otel_metrics_gauge', + sum: 'otel_metrics_sum', + histogram: 'otel_metrics_histogram', + }); + }); + + it('preserves non-queryable kinds the schema still declares', () => { + // summary and "exponential histogram" are valid MetricsDataType + // members even though the query renderer cannot translate them. + expect( + sanitizeMetricTables({ + gauge: 'otel_metrics_gauge', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }), + ).toEqual({ + gauge: 'otel_metrics_gauge', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }); + }); + + it('strips Mongoose _id leaking from the metricTables subdoc', () => { + // Existing documents persisted before the schema fix may still + // carry a stray ObjectId in the embedded metricTables subdoc. + // The sanitizer keeps only valid kind keys. + const result = sanitizeMetricTables({ + gauge: 'otel_metrics_gauge', + sum: 'otel_metrics_sum', + _id: '6a2ad3b4da94be764e2deed8', + }); + expect(result).toEqual({ + gauge: 'otel_metrics_gauge', + sum: 'otel_metrics_sum', + }); + expect(result).not.toHaveProperty('_id'); + }); + + it('drops unknown keys', () => { + expect( + sanitizeMetricTables({ + gauge: 'otel_metrics_gauge', + not_a_real_kind: 'something', + }), + ).toEqual({ + gauge: 'otel_metrics_gauge', + }); + }); + + it('drops non-string values for valid keys', () => { + expect( + sanitizeMetricTables({ + gauge: 'otel_metrics_gauge', + sum: 12345, + histogram: null, + } as Record), + ).toEqual({ + gauge: 'otel_metrics_gauge', + }); + }); + + it('returns undefined when no valid entries remain', () => { + expect( + sanitizeMetricTables({ + _id: 'whatever', + not_a_real_kind: 'x', + }), + ).toBeUndefined(); + }); +}); diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index fffcdcbe38..bd3f00fe0b 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -7,10 +7,18 @@ jest.mock('@/utils/trimToolResponse', () => ({ })); import { + annotateIncreaseTopNHint, + assertSourceKindMatchesSelect, errorHint, + INCREASE_TOP_N_CAP, mergeWhereIntoSelectItems, parseTimeRange, } from '@/mcp/tools/query/helpers'; +import { + applyMetricSelectDefaults, + getMetricSelectIssues, + validateMetricSelectItems, +} from '@/mcp/tools/query/schemas'; import { resolveOrderBy } from '@/mcp/tools/query/table'; describe('parseTimeRange', () => { @@ -412,4 +420,483 @@ describe('resolveOrderBy', () => { ]), ).toBe('quantile'); }); + + it('should NOT synthesize for "increase" aggFn (metric-only marker)', () => { + // increase compiles to a multi-CTE sum(Rate) pipeline in the renderer, + // not a standalone SQL function. resolveOrderBy must leave bare + // "increase" alone so the renderer-assigned alias can take over. + expect( + resolveOrderBy('increase', [ + { aggFn: 'increase', valueExpression: 'Value' }, + ]), + ).toBe('increase'); + }); +}); + +// ─── getMetricSelectIssues ─────────────────────────────────────────────────── + +describe('getMetricSelectIssues', () => { + it('returns no issues for a plain non-metric count', () => { + expect(getMetricSelectIssues({ aggFn: 'count' })).toEqual([]); + }); + + it('returns no issues for a plain non-metric avg with valueExpression', () => { + expect( + getMetricSelectIssues({ aggFn: 'avg', valueExpression: 'Duration' }), + ).toEqual([]); + }); + + it('requires valueExpression for non-count non-metric aggregations', () => { + const issues = getMetricSelectIssues({ aggFn: 'avg' }); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['valueExpression']); + expect(issues[0].message).toContain('required for non-count'); + }); + + it('does NOT require valueExpression when metricType is set', () => { + // valueExpression defaults to "Value" for metric sources + expect( + getMetricSelectIssues({ + aggFn: 'avg', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + }), + ).toEqual([]); + }); + + it('rejects valueExpression on aggFn:"count"', () => { + const issues = getMetricSelectIssues({ + aggFn: 'count', + valueExpression: 'Duration', + }); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['valueExpression']); + }); + + it('rejects metricType without metricName', () => { + const issues = getMetricSelectIssues({ + aggFn: 'avg', + metricType: 'gauge', + }); + expect(issues.find(i => i.path[0] === 'metricName')).toBeDefined(); + }); + + it('rejects metricName without metricType', () => { + const issues = getMetricSelectIssues({ + aggFn: 'avg', + metricName: 'system.cpu.utilization', + }); + expect(issues.find(i => i.path[0] === 'metricType')).toBeDefined(); + }); + + it('rejects aggFn:"increase" on a gauge metric', () => { + const issues = getMetricSelectIssues({ + aggFn: 'increase', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + }); + expect(issues.find(i => i.path[0] === 'aggFn')).toBeDefined(); + }); + + it('accepts aggFn:"increase" on a sum metric', () => { + expect( + getMetricSelectIssues({ + aggFn: 'increase', + metricType: 'sum', + metricName: 'http.server.request.count', + }), + ).toEqual([]); + }); + + it('rejects aggFn:"avg" on a histogram metric', () => { + const issues = getMetricSelectIssues({ + aggFn: 'avg', + metricType: 'histogram', + metricName: 'http.server.request.duration', + }); + expect(issues.find(i => i.path[0] === 'aggFn')).toBeDefined(); + }); + + it('accepts aggFn:"count" on a histogram metric (no level required)', () => { + expect( + getMetricSelectIssues({ + aggFn: 'count', + metricType: 'histogram', + metricName: 'http.server.request.duration', + }), + ).toEqual([]); + }); + + it('requires level for aggFn:"quantile" on a histogram metric', () => { + const issues = getMetricSelectIssues({ + aggFn: 'quantile', + metricType: 'histogram', + metricName: 'http.server.request.duration', + }); + expect(issues.find(i => i.path[0] === 'level')).toBeDefined(); + }); + + it('accepts aggFn:"quantile" with level on a histogram metric', () => { + expect( + getMetricSelectIssues({ + aggFn: 'quantile', + level: 0.95, + metricType: 'histogram', + metricName: 'http.server.request.duration', + }), + ).toEqual([]); + }); + + it('rejects isDelta on a non-gauge metric', () => { + const issues = getMetricSelectIssues({ + aggFn: 'sum', + metricType: 'sum', + metricName: 'http.request.count', + isDelta: true, + }); + expect(issues.find(i => i.path[0] === 'isDelta')).toBeDefined(); + }); + + it('accepts isDelta on a gauge metric', () => { + expect( + getMetricSelectIssues({ + aggFn: 'avg', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + isDelta: true, + }), + ).toEqual([]); + }); + + it('rejects level when aggFn is not quantile', () => { + const issues = getMetricSelectIssues({ + aggFn: 'avg', + valueExpression: 'Duration', + level: 0.95, + }); + expect(issues.find(i => i.path[0] === 'level')).toBeDefined(); + }); +}); + +// ─── validateMetricSelectItems ─────────────────────────────────────────────── + +describe('validateMetricSelectItems', () => { + it('returns null for a valid items array', () => { + expect( + validateMetricSelectItems([ + { aggFn: 'count' }, + { aggFn: 'avg', valueExpression: 'Duration' }, + ]), + ).toBeNull(); + }); + + it('returns an error envelope and labels each item by index', () => { + const result = validateMetricSelectItems([ + { aggFn: 'avg' }, // missing valueExpression + { aggFn: 'increase', metricType: 'gauge', metricName: 'x' }, // increase requires sum + ]); + expect(result).not.toBeNull(); + if (!result) return; + expect(result.isError).toBe(true); + expect(result.content[0].type).toBe('text'); + const text = result.content[0].text; + expect(text).toContain('select[0].valueExpression'); + expect(text).toContain('select[1].aggFn'); + }); + + it('returns an error envelope for a single bad item', () => { + const result = validateMetricSelectItems([ + { + aggFn: 'quantile', + metricType: 'histogram', + metricName: 'http.server.request.duration', + }, // missing level + ]); + expect(result).not.toBeNull(); + if (!result) return; + expect(result.content[0].text).toContain('select[0].level'); + }); +}); + +// ─── applyMetricSelectDefaults ─────────────────────────────────────────────── + +// Typed as McpSelectItem so the inferred generic preserves the +// optional valueExpression field. Otherwise structural inference +// narrows away `valueExpression` and the assertions below stop +// type-checking. +type SelectItem = { + aggFn: string; + metricType?: 'gauge' | 'sum' | 'histogram'; + metricName?: string; + valueExpression?: string; +}; + +describe('applyMetricSelectDefaults', () => { + it('defaults valueExpression to "Value" when metricType is set', () => { + const input: SelectItem[] = [ + { + aggFn: 'avg', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + }, + ]; + const out = applyMetricSelectDefaults(input); + expect(out[0].valueExpression).toBe('Value'); + }); + + it('preserves an explicit valueExpression on metric items', () => { + const input: SelectItem[] = [ + { + aggFn: 'avg', + metricType: 'gauge', + metricName: 'x', + valueExpression: 'Value * 100', + }, + ]; + const out = applyMetricSelectDefaults(input); + expect(out[0].valueExpression).toBe('Value * 100'); + }); + + it('leaves non-metric items untouched', () => { + const input: SelectItem[] = [{ aggFn: 'count' }]; + const out = applyMetricSelectDefaults(input); + expect(out[0]).toEqual({ aggFn: 'count' }); + expect(out[0].valueExpression).toBeUndefined(); + }); + + it('returns new objects only for the items it mutates', () => { + const input: SelectItem[] = [ + { aggFn: 'count' }, + { aggFn: 'avg', metricType: 'gauge', metricName: 'x' }, + ]; + const out = applyMetricSelectDefaults(input); + expect(out[0]).toBe(input[0]); // unchanged item is the same reference + expect(out[1]).not.toBe(input[1]); // mutated item is a new object + }); +}); + +// ─── annotateIncreaseTopNHint ──────────────────────────────────────────────── + +function buildResult(data: unknown[]) { + return { + content: [ + { + type: 'text', + text: JSON.stringify({ result: { data } }), + }, + ], + isError: false, + }; +} + +/** Rows with `count` distinct ServiceName group values. */ +function buildGroupedRows(count: number) { + return Array.from({ length: count }, (_, i) => ({ + ServiceName: `svc-${i}`, + Series: i, + })); +} + +describe('annotateIncreaseTopNHint', () => { + it('exposes the renderer cap as a constant', () => { + expect(INCREASE_TOP_N_CAP).toBe(20); + }); + + it('appends a hint when the distinct group count reaches the cap', () => { + const result = buildResult(buildGroupedRows(INCREASE_TOP_N_CAP)); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toHaveLength(1); + expect(parsed.hints[0]).toContain('top 20'); + expect(parsed.hints[0]).toContain('aggFn:"increase"'); + }); + + it('does NOT annotate when the distinct group count is below the cap', () => { + const result = buildResult(buildGroupedRows(INCREASE_TOP_N_CAP - 1)); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('counts distinct groups across repeated time buckets, not raw rows', () => { + // 3 groups × 10 buckets = 30 rows but only 3 distinct groups. + const rows = Array.from({ length: 30 }, (_, i) => ({ + ServiceName: `svc-${i % 3}`, + bucket: i, + })); + const result = buildResult(rows); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('does NOT annotate when the groupBy column cannot be resolved from rows', () => { + const result = buildResult([{ x: 1 }, { x: 2 }]); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('supports multi-column groupBy by counting distinct combinations', () => { + // 4 ServiceName values × 5 SpanName values = 20 distinct combos. + const rows = Array.from({ length: 20 }, (_, i) => ({ + ServiceName: `svc-${i % 4}`, + SpanName: `op-${Math.floor(i / 4)}`, + })); + const result = buildResult(rows); + annotateIncreaseTopNHint( + result, + [{ aggFn: 'increase' }], + 'ServiceName, SpanName', + ); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toHaveLength(1); + }); + + it('appends alongside an existing hint instead of overwriting', () => { + const result = { + content: [ + { + type: 'text', + text: JSON.stringify({ + result: { data: buildGroupedRows(INCREASE_TOP_N_CAP) }, + hints: ['previous hint from another writer'], + }), + }, + ], + isError: false, + }; + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toHaveLength(2); + expect(parsed.hints[0]).toBe('previous hint from another writer'); + expect(parsed.hints[1]).toContain('top 20'); + }); + + it('does NOT annotate when increase is used WITHOUT a groupBy', () => { + const result = buildResult(buildGroupedRows(INCREASE_TOP_N_CAP)); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], undefined); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('does NOT annotate when groupBy is an empty string', () => { + const result = buildResult(buildGroupedRows(INCREASE_TOP_N_CAP)); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], ' '); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('does NOT annotate when no select item uses increase', () => { + const result = buildResult(buildGroupedRows(INCREASE_TOP_N_CAP)); + annotateIncreaseTopNHint( + result, + [{ aggFn: 'sum' }, { aggFn: 'avg' }], + 'ServiceName', + ); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('does NOT annotate empty results (already labelled by formatQueryResult)', () => { + const result = buildResult([]); + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.hints).toBeUndefined(); + }); + + it('does NOT annotate error results', () => { + const result = { + content: [{ type: 'text', text: 'an error message' }], + isError: true, + }; + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + expect(result.content[0].text).toBe('an error message'); + }); + + it('leaves unparseable text content unchanged', () => { + const result = { + content: [{ type: 'text', text: 'not json' }], + isError: false, + }; + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'); + expect(result.content[0].text).toBe('not json'); + }); + + it('is a no-op when content is missing', () => { + const result = {} as Parameters[0]; + expect(() => + annotateIncreaseTopNHint(result, [{ aggFn: 'increase' }], 'ServiceName'), + ).not.toThrow(); + }); +}); + +describe('assertSourceKindMatchesSelect', () => { + it('returns null when metric select runs against a metric source', () => { + const source = { kind: 'metric' }; + const select = [ + { + aggFn: 'avg', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + }, + ]; + expect(assertSourceKindMatchesSelect(source, select)).toBeNull(); + }); + + it('returns null when non-metric select runs against a trace source', () => { + const source = { kind: 'trace' }; + const select = [{ aggFn: 'count' }]; + expect(assertSourceKindMatchesSelect(source, select)).toBeNull(); + }); + + it('errors when metric params target a non-metric source', () => { + const source = { kind: 'trace' }; + const select = [ + { + aggFn: 'avg', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + }, + ]; + const result = assertSourceKindMatchesSelect(source, select); + expect(result?.isError).toBe(true); + expect(result?.content[0].text).toMatch(/not metric/); + expect(result?.content[0].text).toMatch(/clickstack_list_sources/); + }); + + it('errors when a metric source receives select items without metricType', () => { + const source = { kind: 'metric' }; + const select = [{ aggFn: 'avg', valueExpression: 'Value' }]; + const result = assertSourceKindMatchesSelect(source, select); + expect(result?.isError).toBe(true); + expect(result?.content[0].text).toMatch(/metricType \+ metricName/); + expect(result?.content[0].text).toMatch(/clickstack_describe_source/); + }); + + it('counts only items whose metricType is a non-empty string', () => { + const source = { kind: 'trace' }; + // Items with empty/missing metricType should not trip the check + expect( + assertSourceKindMatchesSelect(source, [ + { aggFn: 'count' }, + { aggFn: 'avg', valueExpression: 'Duration', metricType: '' }, + ]), + ).toBeNull(); + }); + + it('returns null for a raw string select', () => { + // Raw-string selects bypass the metric-annotation check; the + // renderer handles them directly. + expect( + assertSourceKindMatchesSelect({ kind: 'metric' }, 'count()'), + ).toBeNull(); + }); + + it('returns null when select is not an array', () => { + expect( + assertSourceKindMatchesSelect({ kind: 'trace' }, undefined), + ).toBeNull(); + expect(assertSourceKindMatchesSelect({ kind: 'trace' }, null)).toBeNull(); + }); }); diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index 5a82f1e917..87e83fbf9c 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -1,10 +1,13 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import * as config from '@/config'; import { bulkInsertLogs, + bulkInsertMetricsGauge, + bulkInsertMetricsHistogram, + bulkInsertMetricsSum, DEFAULT_DATABASE, DEFAULT_LOGS_TABLE, DEFAULT_TRACES_TABLE, @@ -802,4 +805,241 @@ describe('MCP Query Tools', () => { }); }); }); + + // ── Metric source query coverage ───────────────────────────────────────── + + describe('Metric sources (timeseries + table)', () => { + const createMetricSource = () => + Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', + [MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum', + [MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram', + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'Metrics', + }); + + const seedGauge = async (now: Date) => { + await bulkInsertMetricsGauge([ + { + MetricName: 'system.cpu.utilization', + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: now, + Value: 0.42, + }, + { + MetricName: 'system.cpu.utilization', + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: new Date(now.getTime() + 1000), + Value: 0.61, + }, + ]); + }; + + const seedSum = async (now: Date) => { + await bulkInsertMetricsSum([ + { + MetricName: 'http.server.request.count', + AggregationTemporality: 1, // DELTA + IsMonotonic: true, + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: now, + Value: 100, + }, + { + MetricName: 'http.server.request.count', + AggregationTemporality: 1, + IsMonotonic: true, + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: new Date(now.getTime() + 1000), + Value: 25, + }, + ]); + }; + + const seedHistogram = async (now: Date) => { + await bulkInsertMetricsHistogram([ + { + MetricName: 'http.server.request.duration', + ResourceAttributes: { 'service.name': 'svc-a' }, + TimeUnix: now, + BucketCounts: [1, 2, 3, 4], + ExplicitBounds: [10, 100, 1000], + AggregationTemporality: 1, + }, + ]); + }; + + it('runs clickstack_timeseries against a gauge metric source', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await seedGauge(now); + + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'avg', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + alias: 'cpu', + }, + ], + startTime: new Date(now.getTime() - 60_000).toISOString(), + endTime: new Date(now.getTime() + 60_000).toISOString(), + granularity: '1 minute', + }); + + expect(result.isError).toBeFalsy(); + // No "Both table name and UUID are empty" error from the renderer. + expect(getFirstText(result)).not.toMatch(/UUID are empty/); + + // Granularity must survive into the renderer: the result is bucketed + // over time (the __hdx_time_bucket column is present) rather than + // collapsed into a single per-group aggregate. Regression guard for + // the granularity-stripping bug. + const output = JSON.parse(getFirstText(result)); + const metaNames = (output.result?.meta ?? []).map( + (m: { name: string }) => m.name, + ); + expect(metaNames).toContain('__hdx_time_bucket'); + // Since the result IS bucketed, the "not bucketed over time" hint + // must never fire (it would be wrong advice). + const hints: string[] = output.hints ?? []; + expect(hints.some(h => /not bucketed over time/.test(h))).toBe(false); + }); + + it('runs clickstack_table aggFn:"increase" against a sum metric source', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await seedSum(now); + + const result = await callTool(client, 'clickstack_table', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'increase', + metricType: 'sum', + metricName: 'http.server.request.count', + alias: 'requests', + }, + ], + startTime: new Date(now.getTime() - 60_000).toISOString(), + endTime: new Date(now.getTime() + 60_000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(getFirstText(result)).not.toMatch(/UUID are empty/); + }); + + it('runs clickstack_timeseries quantile against a histogram metric source', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await seedHistogram(now); + + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'quantile', + level: 0.95, + metricType: 'histogram', + metricName: 'http.server.request.duration', + alias: 'p95', + }, + ], + startTime: new Date(now.getTime() - 60_000).toISOString(), + endTime: new Date(now.getTime() + 60_000).toISOString(), + granularity: '1 minute', + }); + + expect(result.isError).toBeFalsy(); + }); + + it('rejects aggFn:"increase" on a gauge select item with a friendly schema error', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'increase', + metricType: 'gauge', + metricName: 'system.cpu.utilization', + }, + ], + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/"increase" is only valid for sum/); + }); + + it('rejects aggFn:"quantile" on a histogram without level', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'quantile', + metricType: 'histogram', + metricName: 'http.server.request.duration', + }, + ], + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/level is required/); + }); + + it('rejects aggFn:"avg" on a histogram select item', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'avg', + metricType: 'histogram', + metricName: 'http.server.request.duration', + }, + ], + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch( + /Histogram metrics only support aggFn/, + ); + }); + + it('rejects metricType without metricName', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [{ aggFn: 'avg', metricType: 'gauge' }], + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/metricName is required/); + }); + + it('rejects isDelta on a non-gauge metric', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_timeseries', { + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'increase', + metricType: 'sum', + metricName: 'http.server.request.count', + isDelta: true, + }, + ], + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/isDelta is only valid for gauge/); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/sources.test.ts b/packages/api/src/mcp/__tests__/sources.test.ts index a1a714d0f9..8da78bc7b7 100644 --- a/packages/api/src/mcp/__tests__/sources.test.ts +++ b/packages/api/src/mcp/__tests__/sources.test.ts @@ -3,6 +3,9 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import * as config from '@/config'; import { + bulkInsertMetricsGauge, + bulkInsertMetricsHistogram, + bulkInsertMetricsSum, DEFAULT_DATABASE, DEFAULT_LOGS_TABLE, DEFAULT_TRACES_TABLE, @@ -249,7 +252,7 @@ describe('MCP Source Tools', () => { expect(output.source.keyColumns).toHaveProperty('spanId'); }); - it('should return metricTables for a metric source (no column schema)', async () => { + it('should return metricTables AND column schema for a metric source', async () => { const metricSource = await Source.create({ kind: SourceKind.Metric, team: team._id, @@ -280,13 +283,34 @@ describe('MCP Source Tools', () => { kind: SourceKind.Metric, }); - // Metric sources have metricTables but no column schema or value samples + // Metric sources surface metricTables AND run column / map-key + // discovery against the representative metric table (gauge picked + // first by pickRepresentativeMetricTable). expect(output.source.metricTables).toBeDefined(); expect(output.source.metricTables).toHaveProperty('gauge'); - expect(output.source.columns).toBeUndefined(); - expect(output.source.lowCardinalityValues).toBeUndefined(); - expect(output.source.mapAttributeKeys).toBeUndefined(); - expect(output.source.mapAttributeValues).toBeUndefined(); + // metricTables should only contain valid kind keys — not a stray + // Mongoose `_id` from the embedded subdoc. + expect(output.source.metricTables).not.toHaveProperty('_id'); + expect(Object.keys(output.source.metricTables).sort()).toEqual( + ['gauge', 'histogram', 'sum'].sort(), + ); + expect(output.source.discoveryMetricKind).toBe('gauge'); + expect(output.source.columns).toBeDefined(); + expect(Array.isArray(output.source.columns)).toBe(true); + // The OTel Collector gauge schema includes MetricName + Value + // as native columns. + const columnNames = output.source.columns.map( + (c: { name: string }) => c.name, + ); + expect(columnNames).toContain('MetricName'); + expect(columnNames).toContain('Value'); + + // nextSteps points at the new metric discovery tools. + expect(output.nextSteps.query).toContain('metricType'); + expect(output.nextSteps.discovery).toContain('clickstack_list_metrics'); + expect(output.nextSteps.discovery).toContain( + 'clickstack_describe_metric', + ); }); it('should include usage guidance and nextSteps', async () => { @@ -338,4 +362,513 @@ describe('MCP Source Tools', () => { expect(getFirstText(result)).toContain('not found'); }); }); + + // ── clickstack_list_metrics ────────────────────────────────────────────── + + describe('clickstack_list_metrics', () => { + const createMetricSource = () => + Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', + [MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum', + [MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram', + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'Metrics', + }); + + const seedMetricNames = async () => { + const now = new Date(); + await bulkInsertMetricsGauge([ + { + MetricName: 'system.cpu.utilization', + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: now, + Value: 0.42, + }, + { + MetricName: 'system.memory.usage', + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: now, + Value: 12345, + }, + ]); + await bulkInsertMetricsSum([ + { + MetricName: 'http.server.request.count', + AggregationTemporality: 1, + IsMonotonic: true, + ResourceAttributes: { 'service.name': 'svc-a' }, + ServiceName: 'svc-a', + TimeUnix: now, + Value: 100, + }, + ]); + await bulkInsertMetricsHistogram([ + { + MetricName: 'http.server.request.duration', + ResourceAttributes: { 'service.name': 'svc-a' }, + TimeUnix: now, + BucketCounts: [1, 2, 3], + ExplicitBounds: [10, 100, 1000], + AggregationTemporality: 1, + }, + ]); + }; + + it('rejects non-metric sources with a friendly error', async () => { + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: traceSource._id.toString(), + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not a metric source'); + }); + + it('returns 404 for unknown source IDs', async () => { + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: '000000000000000000000000', + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('returns metrics across all populated kinds when kind is omitted', async () => { + const metricSource = await createMetricSource(); + await seedMetricNames(); + + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + const namesByKind: Record = {}; + for (const entry of output.metrics ?? []) { + if (!namesByKind[entry.kind]) namesByKind[entry.kind] = []; + namesByKind[entry.kind].push(entry.name); + } + expect(namesByKind.gauge ?? []).toEqual( + expect.arrayContaining([ + 'system.cpu.utilization', + 'system.memory.usage', + ]), + ); + expect(namesByKind.sum ?? []).toEqual( + expect.arrayContaining(['http.server.request.count']), + ); + expect(namesByKind.histogram ?? []).toEqual( + expect.arrayContaining(['http.server.request.duration']), + ); + }); + + it('restricts results to a single kind when kind is set', async () => { + const metricSource = await createMetricSource(); + await seedMetricNames(); + + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + kind: 'gauge', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + for (const entry of output.metrics ?? []) { + expect(entry.kind).toBe('gauge'); + } + }); + + it('applies namePattern as a server-side ILIKE filter', async () => { + const metricSource = await createMetricSource(); + await seedMetricNames(); + + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + namePattern: 'system.cpu.%', + }); + const output = JSON.parse(getFirstText(result)); + for (const entry of output.metrics ?? []) { + expect(entry.name).toMatch(/^system\.cpu\./); + } + }); + + it('paginates via opaque nextCursor and resumes cleanly', async () => { + const metricSource = await createMetricSource(); + await seedMetricNames(); + + // First page: only ask for one entry so the cap is forced even with + // a small seed; sanity-check the cursor round-trip. + const first = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + kind: 'gauge', + limit: 1, + }); + const firstOutput = JSON.parse(getFirstText(first)); + expect(firstOutput.metrics.length).toBe(1); + expect(typeof firstOutput.nextCursor).toBe('string'); + + const second = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + kind: 'gauge', + limit: 5, + cursor: firstOutput.nextCursor, + }); + expect(second.isError).toBeFalsy(); + const secondOutput = JSON.parse(getFirstText(second)); + // The second page must not repeat the first page's last name. + const firstNames = new Set( + (firstOutput.metrics as { name: string }[]).map(m => m.name), + ); + for (const entry of secondOutput.metrics as { name: string }[]) { + expect(firstNames.has(entry.name)).toBe(false); + } + }); + + it('rejects a malformed cursor with an actionable error', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + cursor: 'not-base64!', + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/Invalid cursor/); + }); + + it('returns an empty-result hint when nothing matches', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + namePattern: 'this.name.does.not.exist.%', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.metrics).toEqual([]); + expect(output.hint).toMatch(/widening|removing|omitting/); + }); + + it('surfaces partialFailure instead of the empty hint when a kind fetch fails', async () => { + // Point the gauge kind at the logs table: it exists (so source + // resolution succeeds) but has no MetricName/TimeUnix columns, so + // the per-kind listing query throws. + const brokenSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: DEFAULT_LOGS_TABLE, + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'BrokenMetrics', + }); + const result = await callTool(client, 'clickstack_list_metrics', { + sourceId: brokenSource._id.toString(), + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.metrics).toEqual([]); + expect(output.partialFailure).toHaveLength(1); + expect(output.partialFailure[0].kind).toBe('gauge'); + expect(output.partialFailure[0].error).toBeTruthy(); + // The misleading "No metrics matched … widen the window" hint must + // NOT appear — the agent should retry, not widen. + expect(output.hint).not.toMatch(/No metrics matched/); + }); + }); + + // ── clickstack_describe_metric ─────────────────────────────────────────── + + describe('clickstack_describe_metric', () => { + const createMetricSource = () => + Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', + [MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum', + [MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram', + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'Metrics', + }); + + it('rejects non-metric sources', async () => { + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: traceSource._id.toString(), + metricName: 'system.cpu.utilization', + kind: 'gauge', + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not a metric source'); + }); + + it('rejects calls that omit kind', async () => { + // `kind` is required. Calling without it returns an MCP error + // surfaced from the schema parser rather than reaching ClickHouse. + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'system.cpu.utilization', + }); + expect(result.isError).toBe(true); + }); + + it('returns an actionable hint when the metric has no data in the kind table', async () => { + const metricSource = await createMetricSource(); + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'definitely.not.a.metric', + kind: 'gauge', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + // We always return a single kindDetail for the requested kind — + // attribute keys are empty when the metric has no data. + expect(output.kinds).toHaveLength(1); + expect(output.kinds[0].kind).toBe('gauge'); + expect(output.kinds[0].attributeKeys).toEqual({}); + expect(output.hint).toMatch(/No data found/); + expect(output.partialFailure).toBeUndefined(); + }); + + it('surfaces partialFailure instead of the no-data hint when discovery fails', async () => { + // Point the gauge kind at the logs table: it exists (so getColumns + // succeeds) but lacks MetricName/TimeUnix columns, so the + // attribute-keys discovery query throws. + const brokenSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: DEFAULT_LOGS_TABLE, + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'BrokenMetrics', + }); + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: brokenSource._id.toString(), + metricName: 'whatever', + kind: 'gauge', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.partialFailure).toBeDefined(); + expect( + output.partialFailure.map((f: { stage: string }) => f.stage), + ).toContain('attributeKeys'); + // The misleading "No data found … widen startTime/endTime" hint + // must NOT appear — the fetch failed; widening would not help. + expect(output.hint).not.toMatch(/No data found/); + }); + + it('returns attribute keys for a gauge metric when kind is specified', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await bulkInsertMetricsGauge([ + { + MetricName: 'system.cpu.utilization', + ResourceAttributes: { 'service.name': 'auto-detect-svc' }, + ServiceName: 'auto-detect-svc', + TimeUnix: now, + Value: 0.5, + }, + ]); + + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'system.cpu.utilization', + kind: 'gauge', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.metricName).toBe('system.cpu.utilization'); + expect(output.kinds.length).toBe(1); + const detail = output.kinds[0]; + expect(detail.kind).toBe('gauge'); + expect(detail.usage).toContain('aggFn'); + expect(detail.attributeKeys).toBeDefined(); + // ResourceAttributes['service.name'] should land in the keys map + // under ResourceAttributes. + expect(detail.attributeKeys.ResourceAttributes ?? []).toEqual( + expect.arrayContaining(['service.name']), + ); + // Attribute values should be sampled by default. + expect(detail.attributeValues).toBeDefined(); + // nextSteps.query carries a worked example matching the requested kind. + expect(output.nextSteps.query).toContain('metricType: "gauge"'); + }); + + it('reports sampledKeys and truncatedKeys so unsampled keys are distinguishable', async () => { + // 15 attribute keys > MAX_ATTR_KEYS_TO_SAMPLE (12): the overflow + // must land in truncatedKeys so the agent knows those keys were + // never queried (vs. sampled-but-empty). + const metricSource = await createMetricSource(); + const manyAttrs: Record = {}; + for (let i = 0; i < 15; i++) { + manyAttrs[`attr.key.${String(i).padStart(2, '0')}`] = `value-${i}`; + } + await bulkInsertMetricsGauge([ + { + MetricName: 'many.attrs.metric', + ResourceAttributes: manyAttrs, + ServiceName: 'many-attrs-svc', + TimeUnix: new Date(), + Value: 1, + }, + ]); + + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'many.attrs.metric', + kind: 'gauge', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + const detail = output.kinds[0]; + const meta = detail.attributeValuesMeta; + expect(meta).toBeDefined(); + expect(meta.sampledKeys).toHaveLength(12); + expect(meta.truncatedKeys).toHaveLength(3); + // Sampled and truncated sets are disjoint and cover all 15 keys. + expect(new Set([...meta.sampledKeys, ...meta.truncatedKeys]).size).toBe( + 15, + ); + // Every key with values must have been sampled. + for (const key of Object.keys(detail.attributeValues ?? {})) { + expect(meta.sampledKeys).toContain(key); + } + }); + + it('reports empty truncatedKeys when all attribute keys fit the cap', async () => { + const metricSource = await createMetricSource(); + await bulkInsertMetricsGauge([ + { + MetricName: 'few.attrs.metric', + ResourceAttributes: { 'service.name': 'few-attrs-svc' }, + ServiceName: 'few-attrs-svc', + TimeUnix: new Date(), + Value: 1, + }, + ]); + + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'few.attrs.metric', + kind: 'gauge', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + const meta = output.kinds[0].attributeValuesMeta; + expect(meta).toBeDefined(); + expect(meta.truncatedKeys).toEqual([]); + expect(meta.sampledKeys).toContain("ResourceAttributes['service.name']"); + }); + + it('skips value sampling when sampleValues is false', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await bulkInsertMetricsGauge([ + { + MetricName: 'system.cpu.utilization', + ResourceAttributes: { 'service.name': 'no-sample-svc' }, + ServiceName: 'no-sample-svc', + TimeUnix: now, + Value: 0.5, + }, + ]); + + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'system.cpu.utilization', + kind: 'gauge', + sampleValues: false, + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.kinds[0].attributeValues).toBeUndefined(); + }); + + it('returns the correct usage example for a sum metric', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await bulkInsertMetricsSum([ + { + MetricName: 'http.server.request.count', + AggregationTemporality: 1, + IsMonotonic: true, + ResourceAttributes: { 'service.name': 'sum-svc' }, + ServiceName: 'sum-svc', + TimeUnix: now, + Value: 42, + }, + ]); + + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'http.server.request.count', + kind: 'sum', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.kinds[0].kind).toBe('sum'); + expect(output.kinds[0].usage).toContain('increase'); + expect(output.nextSteps.query).toContain('"increase"'); + }); + + it('returns the quantile + level example for a histogram metric', async () => { + const metricSource = await createMetricSource(); + const now = new Date(); + await bulkInsertMetricsHistogram([ + { + MetricName: 'http.server.request.duration', + ResourceAttributes: { 'service.name': 'hist-svc' }, + TimeUnix: now, + BucketCounts: [1, 2, 3], + ExplicitBounds: [10, 100, 1000], + AggregationTemporality: 1, + }, + ]); + + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: metricSource._id.toString(), + metricName: 'http.server.request.duration', + kind: 'histogram', + }); + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.kinds[0].kind).toBe('histogram'); + expect(output.kinds[0].usage).toContain('quantile'); + expect(output.nextSteps.query).toContain('"quantile"'); + expect(output.nextSteps.query).toContain('level: 0.95'); + }); + + it('rejects explicit kind when the source has no table for that kind', async () => { + // Source with only gauge populated. + const gaugeOnly = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'GaugeOnly', + }); + const result = await callTool(client, 'clickstack_describe_metric', { + sourceId: gaugeOnly._id.toString(), + metricName: 'whatever', + kind: 'sum', + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/no "sum" metric table/); + }); + }); }); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 4c040f6bc2..e62837872e 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -124,7 +124,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t ] Use tabs when one container needs to show different views of the same data (Throughput / Latency / Errors over time, for instance); the tab bar appears when a container has two or more tabs declared. -12. VALIDATE EVERY TILE AFTER SAVE. After clickstack_save_dashboard, call clickstack_query_tile on EVERY tile (not just one). Save validates input shape; it does NOT validate query semantics. Some queries pass save and fail at render time (known gaps: Lucene comparison/wildcard on map attributes, metric tiles with multiple metricTables, malformed having clauses). If query_tile returns an error, fix the tile and re-save before declaring the dashboard ready. +12. VALIDATE EVERY TILE AFTER SAVE. After clickstack_save_dashboard, call clickstack_query_tile on EVERY tile (not just one). Save validates input shape; it does NOT validate query semantics. Some queries pass save and fail at render time (known gaps: Lucene comparison/wildcard on map attributes, malformed having clauses). If query_tile returns an error, fix the tile and re-save before declaring the dashboard ready. 13. NO TITLE-RECAP MARKDOWN TILE. The dashboard's name shows in the title bar. Adding a markdown tile with the dashboard name (or a "About this dashboard" header) doubles the title and eats a row of vertical space because markdown heading styles render at title-bar scale. Skip the markdown tile entirely on starter dashboards. @@ -958,7 +958,29 @@ For configType: "sql" tiles, write ClickHouse SQL with template macros: search No select items (select is a column list string). where is the filter. markdown No select items. Set markdown field with content. -NOTE: Authoring builder tiles on a metric source is not reliable today. The MCP select-item shape does not carry the metricName / metricType fields the metric query path needs, and a save with a metric sourceId may render in the UI as "Both table name and UUID are empty" even though the save itself succeeded. For metrics, use a raw SQL tile (configType: "sql") with explicit table reference. The standard tables that back a metric source are otel_metrics_gauge, otel_metrics_sum, and otel_metrics_histogram; clickstack_list_sources returns the metric source's metricTables map so you know which table holds which metric kind. Discovery: metric source schemas today do NOT publish mapAttributeKeys for ResourceAttributes / Attributes the way log and trace sources do, so attribute keys must be discovered by sampling (SELECT DISTINCT mapKeys(Attributes) FROM ...). +== METRIC SOURCES == + +Builder tiles work on metric sources. Each select item on a metric tile MUST set metricType ("gauge" | "sum" | "histogram") and metricName (the OTel metric name, e.g. "system.cpu.utilization"). valueExpression defaults to "Value" when omitted, so a typical metric series is { aggFn: "", metricType: "", metricName: "" }. summary and "exponential histogram" kinds are not yet supported by the renderer. + +Per-kind aggregation guidance: + gauge Use aggFn:"last_value" | "avg" | "min" | "max". Set isDelta:true for Prometheus-style delta over each bucket. + sum Use aggFn:"increase" for the per-bucket counter increase (reset-aware), or aggFn:"sum" | "avg" on the computed rate. increase + groupBy is capped at the top 20 groups by the renderer; pre-filter via where or pick a coarser groupBy when you need broader coverage. + histogram Use aggFn:"quantile" with level ∈ {0.5, 0.9, 0.95, 0.99} for percentiles, or aggFn:"count" for the total bucket count. quantile without level is rejected. + +Discovery workflow for metrics: + 1. clickstack_list_sources: find the metric source ID and its metricTables map (which kinds are populated). + 2. clickstack_describe_source(sourceId): returns columns, attribute keys, low-cardinality values, AND a per-kind metric-name sample (up to 20 names per kind). + 3. clickstack_list_metrics(sourceId, ...): paginate the full metric catalog with optional kind + namePattern (ILIKE) + time-window filters. Pass nextCursor unchanged for the next page. + 4. clickstack_describe_metric(sourceId, metricName): drill into a single metric: kind(s), unit, description, attribute keys per map column, and sampled values per attribute. Attribute keys vary per metric, not per source, so always call this before authoring tiles for a metric you've never queried. + 5. clickstack_timeseries | clickstack_table: author the chart. Set metricType + metricName on each select item; pass the discovered attribute keys via groupBy / where. + +Examples: + Gauge p95-like spread by service (use last_value or avg): + { aggFn: "avg", metricType: "gauge", metricName: "system.cpu.utilization" }, groupBy: "ServiceName" + Sum counter increase: + { aggFn: "increase", metricType: "sum", metricName: "http.server.request.count", alias: "Requests" }, groupBy: "ServiceName" + Histogram p95 latency: + { aggFn: "quantile", level: 0.95, metricType: "histogram", metricName: "http.server.request.duration", alias: "P95 Latency" }, groupBy: "ServiceName" == NUMBER FORMAT == @@ -1279,7 +1301,7 @@ Example: find top patterns for production services over the last 4 hours: The dashboard filter applies globally; tiles do not need the literal. On mixed-source dashboards where only some tiles carry the column, add appliesToSourceIds: ["", ...] to scope the filter to just those sources instead of breaking the unrelated tiles. 8. Forgetting to validate tiles after saving - Always call clickstack_query_tile on EVERY tile after clickstack_save_dashboard, not just one. Save validates input shape but not query semantics. Several known gaps (Lucene comparison/wildcard on map attributes, builder tiles on metric sources, malformed having) pass save and fail at render time. A dashboard with one bad tile renders the whole page in a degraded state; the user sees "Error loading chart" with no way to know which tile broke unless you validated. If query_tile returns an error, fix the where / SQL and re-save before declaring the dashboard ready. + Always call clickstack_query_tile on EVERY tile after clickstack_save_dashboard, not just one. Save validates input shape but not query semantics. Several known gaps (Lucene comparison/wildcard on map attributes, malformed having) pass save and fail at render time. A dashboard with one bad tile renders the whole page in a degraded state; the user sees "Error loading chart" with no way to know which tile broke unless you validated. If query_tile returns an error, fix the where / SQL and re-save before declaring the dashboard ready. 9. Using connectionId with builder tiles, or omitting connectionId or sourceId on a single-table SQL tile Builder tiles (line, table, etc.) use sourceId (no connectionId). diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 8f0bf68049..4d415c0173 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -15,8 +15,18 @@ import { } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; +import { getMetricSelectIssues } from '@/mcp/tools/query/schemas'; +import { QUERYABLE_METRIC_KINDS } from '@/mcp/tools/sources/metricKinds'; import { externalQuantileLevelSchema, objectIdSchema } from '@/utils/zod'; +/** + * Metric type values exposed on dashboard tile select items. Restricted to + * the three kinds the query renderer can translate today; summary and + * exponential histogram are intentionally excluded. Imports the shared + * `QUERYABLE_METRIC_KINDS` source-of-truth tuple from `../sources/metricKinds`. + */ +const mcpTileMetricTypeSchema = z.enum(QUERYABLE_METRIC_KINDS); + // ─── Shared tile schemas for MCP dashboard tools ───────────────────────────── const seriesLevelNumberFormatDescription = @@ -109,7 +119,10 @@ const mcpNumberFormatSchema = z.object({ const mcpTileSelectItemSchema = z .object({ aggFn: AggregateFunctionSchema.describe( - 'Aggregation function. "count" requires no valueExpression; all others do.', + 'Aggregation function. "count" requires no valueExpression; all others do. ' + + 'METRIC SOURCES: "increase" computes the per-bucket counter increase for Sum metrics ' + + '(reset-aware). For Gauges use last_value/avg/min/max. For Histograms use "quantile" ' + + 'with level or "count".', ), valueExpression: z .string() @@ -118,7 +131,9 @@ const mcpTileSelectItemSchema = z 'Column or expression to aggregate. Required for all aggFn except "count". ' + 'Use PascalCase for top-level columns (e.g. "Duration", "StatusCode"). ' + "For span attributes use: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + - "For resource attributes use: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", + "For resource attributes use: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).\n\n" + + 'METRIC SOURCES: optional — defaults to "Value" (the metric value column) when ' + + 'metricType/metricName are set.', ), where: z .string() @@ -138,29 +153,53 @@ const mcpTileSelectItemSchema = z ), level: externalQuantileLevelSchema .optional() - .describe('Percentile level for aggFn="quantile"'), + .describe( + 'Percentile level for aggFn="quantile". REQUIRED for histogram metrics with aggFn:"quantile".', + ), numberFormat: mcpNumberFormatSchema .optional() .describe(seriesLevelNumberFormatDescription), + metricType: mcpTileMetricTypeSchema + .optional() + .describe( + 'METRIC SOURCES ONLY. OTel metric kind: gauge, sum, or histogram. ' + + 'Required (with metricName) when the tile sourceId is a metric source. ' + + 'summary and exponential histogram are not supported by the renderer yet.', + ), + metricName: z + .string() + .optional() + .describe( + 'METRIC SOURCES ONLY. OTel metric name (e.g. "system.cpu.utilization"). ' + + 'Required when metricType is set.', + ), + isDelta: z + .boolean() + .optional() + .describe( + 'METRIC SOURCES ONLY (gauge metrics). When true, computes the Prometheus-style ' + + 'delta over each bucket. Default false.', + ), }) .superRefine((data, ctx) => { - if (data.level && data.aggFn !== 'quantile') { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: 'Level can only be used with quantile aggregation function', - }); - } - if (data.valueExpression && data.aggFn === 'count') { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: - 'Value expression cannot be used with count aggregation function', - }); - } else if (!data.valueExpression && data.aggFn !== 'count') { + const narrow = { + aggFn: typeof data.aggFn === 'string' ? data.aggFn : undefined, + metricType: + typeof data.metricType === 'string' ? data.metricType : undefined, + metricName: + typeof data.metricName === 'string' ? data.metricName : undefined, + isDelta: typeof data.isDelta === 'boolean' ? data.isDelta : undefined, + level: typeof data.level === 'number' ? data.level : undefined, + valueExpression: + typeof data.valueExpression === 'string' + ? data.valueExpression + : undefined, + }; + for (const issue of getMetricSelectIssues(narrow)) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: - 'Value expression is required for non-count aggregation functions', + path: issue.path, + message: issue.message, }); } }); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 87f59c362c..377750e28a 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -78,6 +78,90 @@ const MCP_CLICKHOUSE_SETTINGS = { */ const MCP_REQUEST_TIMEOUT = 32_000; // 30s query limit + 2s buffer +// ─── Increase top-N cap hint ──────────────────────────────────────────────── + +/** + * Group limit applied by the metric renderer when `aggFn:"increase"` is + * combined with `groupBy`. Mirrors `INCREASE_MAX_NUM_GROUPS` in + * `packages/common-utils/src/core/renderChartConfig.ts`. Surfaced to MCP + * callers as a hint so they can reason about why high-cardinality groupBys + * may be truncated. + */ +export const INCREASE_TOP_N_CAP = 20; + +/** + * Append a hint message onto a parsed tool response body. All hint + * writers share `hints: string[]` so multiple advisories can coexist + * (e.g. the increase top-N cap and the single-bucket collapse hint can + * legitimately both apply to the same result). + */ +export function appendHint(parsed: Record, hint: string) { + const existing = Array.isArray(parsed.hints) ? parsed.hints : []; + parsed.hints = [...existing, hint]; +} + +/** + * Count the distinct groupBy combinations present in a result set. + * Resolves each comma-separated groupBy segment against the row keys + * (the renderer aliases group columns by their literal expression). + * Returns null when no segment matches any row key — the caller should + * treat that as "count unknown". + */ +function countDistinctGroups( + data: Array>, + groupBy: string, +): number | null { + const segments = splitAndTrimWithBracket(groupBy); + const sample = data[0]; + if (!sample || typeof sample !== 'object') return null; + const resolvable = segments.filter(seg => seg in sample); + if (resolvable.length === 0) return null; + const seen = new Set(); + for (const row of data) { + seen.add(JSON.stringify(resolvable.map(seg => row[seg]))); + } + return seen.size; +} + +/** + * Mutates a successful tool result envelope in place to add a neutral + * informational hint when the renderer's increase+groupBy top-N cap + * likely applied. Safe to call unconditionally — does nothing for empty + * results, error results, queries that don't combine `aggFn:"increase"` + * with a non-empty `groupBy`, or results whose distinct group count is + * below the cap (no truncation possible). + */ +export function annotateIncreaseTopNHint( + result: { content?: { type: string; text?: string }[]; isError?: boolean }, + selectItems: ReadonlyArray<{ aggFn?: string }>, + groupBy: string | undefined, +): void { + if (result.isError) return; + const first = result.content?.[0]; + if (first?.type !== 'text' || typeof first.text !== 'string') return; + if (!groupBy || groupBy.trim() === '') return; + if (!selectItems.some(s => s.aggFn === 'increase')) return; + try { + const parsed = JSON.parse(first.text); + const data = parsed?.result?.data; + if (!Array.isArray(data) || data.length === 0) return; + // Only warn when the result actually carries enough distinct groups + // to have hit the renderer cap. When the group column cannot be + // resolved from the rows, skip the hint rather than crying wolf. + const distinctGroups = countDistinctGroups(data, groupBy); + if (distinctGroups === null || distinctGroups < INCREASE_TOP_N_CAP) return; + appendHint( + parsed, + `aggFn:"increase" combined with groupBy is capped at the top ${INCREASE_TOP_N_CAP} ` + + `groups by max bucket sum at the renderer layer. ` + + `Results may not include every group present in the underlying data.`, + ); + first.text = JSON.stringify(parsed, null, 2); + } catch { + // leave result unmodified on parse failure + } +} + // ─── Where merging ─────────────────────────────────────────────────────────── export interface MergeWhereResult { @@ -210,7 +294,9 @@ function formatQueryResult(result: unknown) { : {}), ...(empty ? { - hint: 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.', + hints: [ + 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.', + ], } : {}), }, @@ -222,6 +308,80 @@ function formatQueryResult(result: unknown) { }; } +// ─── Source-kind / select-shape guardrail ──────────────────────────────────── + +type SelectItemForKindCheck = { metricType?: unknown }; + +/** + * Reject builder-config queries where the select items' metric annotations + * don't match the source kind: + * - Non-metric source with any select item carrying `metricType` would + * fall through to SQL generation that references the metric `Value` + * column and fail with a cryptic ClickHouse error. + * - Metric source with no select item carrying `metricType` would also + * reach the renderer in a broken state. + * + * Catching both up-front gives the agent a clear next action that mirrors + * the wording on `clickstack_list_metrics` / `clickstack_describe_metric`. + * + * Returns an error envelope on mismatch, or `null` when the select shape + * is consistent with the source kind (or the select is a raw string that + * the caller already parsed by hand). + */ +export function assertSourceKindMatchesSelect( + source: { kind: string }, + select: unknown, +): { isError: true; content: [{ type: 'text'; text: string }] } | null { + // Raw-string select (rare on the builder path) — the renderer handles + // it; no metric annotations to inspect. + if (typeof select === 'string') return null; + if (!Array.isArray(select)) return null; + + const metricItemCount = (select as SelectItemForKindCheck[]).filter( + item => + typeof item === 'object' && + item !== null && + typeof item.metricType === 'string' && + item.metricType.length > 0, + ).length; + + const isMetricSource = source.kind === SourceKind.Metric; + + if (isMetricSource && metricItemCount === 0) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'Source kind is "metric", but no select item specifies metricType + metricName. ' + + 'Each select item on a metric source must set metricType ("gauge" | "sum" | "histogram") ' + + 'and metricName (e.g. metricName:"system.cpu.utilization"). Call ' + + 'clickstack_describe_source or clickstack_list_metrics to discover available metric names.', + }, + ], + }; + } + + if (!isMetricSource && metricItemCount > 0) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + `Source kind is "${source.kind}", not metric — but ${metricItemCount} select item(s) ` + + 'set metricType. metricType + metricName only work on metric sources. ' + + 'Drop the metric fields to query this source, or call clickstack_list_sources to find a ' + + 'source whose kind is "metric".', + }, + ], + }; + } + + return null; +} + // ─── Tile execution ────────────────────────────────────────────────────────── export async function runConfigTile( @@ -229,7 +389,7 @@ export async function runConfigTile( tile: ExternalDashboardTileWithId, startDate: Date, endDate: Date, - options?: { maxResults?: number }, + options?: { maxResults?: number; granularity?: string }, ) { if (!isConfigTile(tile)) { return { @@ -273,6 +433,14 @@ export async function runConfigTile( }; } + // Reject metric-style select against a non-metric source (and vice + // versa) before the renderer composes SQL against the wrong table. + const kindMismatch = assertSourceKindMatchesSelect( + source, + builderConfig.select, + ); + if (kindMismatch) return kindMismatch; + const connection = await getConnectionById( teamId, source.connection.toString(), @@ -298,6 +466,7 @@ export async function runConfigTile( }); const isSearch = builderConfig.displayType === DisplayType.Search; + const isMetricSource = source.kind === SourceKind.Metric; const defaultTableSelect = 'defaultTableSelectExpression' in source ? source.defaultTableSelectExpression @@ -327,13 +496,54 @@ export async function runConfigTile( } : {}; + // Metric sources need three adjustments before the renderer can + // translate the query: + // 1. `from.tableName` is blank — the renderer picks the correct + // per-kind ClickHouse table from `metricTables` at render time. + // 2. `metricTables` must be threaded onto the chart config (mirrors + // packages/api/src/routers/external-api/v2/charts.ts:261-267). + // 3. Each select item's `valueExpression` defaults to "Value" (the + // metric value column) when missing — agents normally omit it. + const metricSelectOverrides = + isMetricSource && Array.isArray(builderConfig.select) + ? { + select: builderConfig.select.map(item => + typeof item === 'string' + ? item + : { + ...item, + valueExpression: item.valueExpression ?? 'Value', + }, + ), + } + : {}; + + // Re-inject granularity for time charts. The MCP tool input carries + // it, but buildTile parses through externalDashboardTileSchemaWithId, + // whose line/stacked_bar schemas don't declare `granularity`, so Zod + // strips it. Without this the renderer's `granularity != null` guard + // fails and no __hdx_time_bucket is emitted — every timeseries call + // collapses to one row per group. Default to "auto" so the renderer + // picks a bucket, mirroring the REST charts path + // (packages/api/src/routers/external-api/v2/charts.ts:289). + // Search tiles intentionally have no granularity (handled above). + const granularityOverride = + !isSearch && + (builderConfig.displayType === DisplayType.Line || + builderConfig.displayType === DisplayType.StackedBar) + ? { granularity: options?.granularity ?? 'auto' } + : {}; + const chartConfig = { ...builderConfig, ...searchOverrides, + ...metricSelectOverrides, + ...granularityOverride, from: { databaseName: source.from.databaseName, - tableName: source.from.tableName, + tableName: isMetricSource ? '' : source.from.tableName, }, + ...(isMetricSource && { metricTables: source.metricTables }), connection: source.connection.toString(), timestampValueExpression: source.timestampValueExpression, implicitColumnExpression: implicitColumn, diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 8777e25ee8..e000b0626a 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; +import { QUERYABLE_METRIC_KINDS } from '@/mcp/tools/sources/metricKinds'; + // ─── Shared description fragments ──────────────────────────────────────────── const WHERE_DESCRIPTION = @@ -48,6 +50,10 @@ export const MCP_AGG_FN_OPTIONS = [ 'quantile', 'sum', 'none', + // 'increase' is only valid for Sum (counter) metrics. The renderer + // computes the per-bucket counter increase with reset-handling; the + // bare aggFn string maps to that behavior, not a SQL function. + 'increase', ] as const; const mcpAggFnSchema = z @@ -59,9 +65,146 @@ const mcpAggFnSchema = z ' count_distinct – unique value count (valueExpression required)\n' + ' quantile – percentile; also set level (valueExpression required)\n' + ' last_value – most recent value of a column\n' + - ' none – pass a raw expression through unchanged', + ' none – pass a raw expression through unchanged\n' + + ' increase – METRIC-ONLY (Sum/counter): per-bucket counter increase, ' + + 'reset-aware. Requires metricType:"sum" and metricName.', ); +/** + * Metric type values exposed to MCP tool callers. Restricted to the three + * kinds the renderer can translate today; summary and exponential histogram + * are intentionally excluded. See `../sources/metricKinds` for the shared + * source-of-truth constant used by every metric-aware tool. + */ +const mcpMetricTypeSchema = z + .enum(QUERYABLE_METRIC_KINDS) + .describe( + 'METRIC SOURCES ONLY. OTel metric kind. Required (along with metricName) ' + + 'when querying a metric source — discover via clickstack_describe_source ' + + 'or clickstack_describe_metric.\n' + + ' gauge – instantaneous values (CPU, memory, queue depth). Use last_value/avg/min/max; set isDelta:true for Prometheus-style delta over the bucket.\n' + + ' sum – cumulative or delta counters (request counts, bytes processed). Use aggFn:"increase" for counter increase, or sum/avg on the computed rate.\n' + + ' histogram – bucketed distributions (request duration). Use aggFn:"quantile" with level for percentiles, or aggFn:"count" for total bucket count.\n' + + 'NOTE: summary and exponential histogram are not supported by the query renderer yet.', + ); + +/** + * Shared cross-field validation issues for MCP select items. Returns the + * list of Zod issues a `.superRefine` callback should emit. Kept as a pure + * function so both `mcpSelectItemSchema` (query tools) and + * `mcpTileSelectItemSchema` (dashboard tile tools) can call it inline + * without widening Zod's output type inference. + * + * Constraints enforced: + * - metricType + metricName must be set together + * - aggFn:"increase" is Sum-only + * - histogram metrics only support quantile (with level) or count + * - isDelta is Gauge-only + * - level still requires aggFn:"quantile" + * - valueExpression is required for non-count aggFns UNLESS metricType is + * set (defaults to "Value" for metric sources) + * + * Note: summary and exponential histogram metric kinds are not in the input + * enum so they cannot reach this function — they are rejected by the field + * schema itself. + */ +export function getMetricSelectIssues(data: { + aggFn?: string; + metricType?: string; + metricName?: string; + isDelta?: boolean; + level?: number; + valueExpression?: string; +}): { path: (string | number)[]; message: string }[] { + const issues: { path: (string | number)[]; message: string }[] = []; + + // metricType ↔ metricName must be set together + if (data.metricType && !data.metricName) { + issues.push({ + path: ['metricName'], + message: + 'metricName is required when metricType is set. Discover metric names ' + + 'via clickstack_list_metrics or clickstack_describe_source.', + }); + } + if (data.metricName && !data.metricType) { + issues.push({ + path: ['metricType'], + message: + 'metricType is required when metricName is set. Use one of: gauge, sum, histogram.', + }); + } + + // increase is Sum-only + if (data.aggFn === 'increase' && data.metricType !== 'sum') { + issues.push({ + path: ['aggFn'], + message: + 'aggFn "increase" is only valid for sum (counter) metrics. ' + + 'Set metricType:"sum" and metricName, or pick a different aggFn.', + }); + } + + // Histogram supports only quantile (+ level) or count today + if (data.metricType === 'histogram') { + if (data.aggFn !== 'quantile' && data.aggFn !== 'count') { + issues.push({ + path: ['aggFn'], + message: + 'Histogram metrics only support aggFn "quantile" (with level) or "count" today.', + }); + } + if (data.aggFn === 'quantile' && data.level == null) { + issues.push({ + path: ['level'], + message: + 'level is required when aggFn is "quantile" on a histogram metric. ' + + 'Use 0.5, 0.9, 0.95, or 0.99.', + }); + } + } + + // isDelta is Gauge-only + if (data.isDelta && data.metricType !== 'gauge') { + issues.push({ + path: ['isDelta'], + message: 'isDelta is only valid for gauge metrics (metricType:"gauge").', + }); + } + + // level requires aggFn:"quantile" + if (data.level != null && data.aggFn !== 'quantile') { + issues.push({ + path: ['level'], + message: 'level is only valid with aggFn:"quantile".', + }); + } + + // valueExpression rules: + // - "count" never takes a valueExpression (existing rule) + // - non-count aggFns require valueExpression UNLESS metricType is set + // (metric sources default valueExpression to "Value" in the helper) + if (data.valueExpression && data.aggFn === 'count') { + issues.push({ + path: ['valueExpression'], + message: 'valueExpression cannot be used with aggFn:"count".', + }); + } else if ( + !data.valueExpression && + data.aggFn !== 'count' && + !data.metricType + ) { + issues.push({ + path: ['valueExpression'], + message: + 'valueExpression is required for non-count aggregation functions ' + + '(or set metricType to query a metric source, which defaults valueExpression to "Value").', + }); + } + + return issues; +} + export const mcpSelectItemSchema = z.object({ aggFn: mcpAggFnSchema, valueExpression: z @@ -74,7 +217,10 @@ export const mcpSelectItemSchema = z.object({ 'Any ClickHouse expression is allowed — common useful forms: ' + '"Duration / 1e6" (ns→ms), ' + '"toFloat64OrZero(SpanAttributes[\'response.size_bytes\'])" (cast attribute), ' + - '"if(StatusCode = \'STATUS_CODE_ERROR\', 1, 0)" (boolean→numeric for ratios).', + '"if(StatusCode = \'STATUS_CODE_ERROR\', 1, 0)" (boolean→numeric for ratios).\n\n' + + 'METRIC SOURCES: optional — defaults to "Value" (the metric value column) when ' + + 'metricType/metricName are set. Set explicitly only if you want to transform the ' + + 'metric value (e.g. "Value / 1e6").', ), where: z .string() @@ -113,10 +259,105 @@ export const mcpSelectItemSchema = z.object({ .union([z.literal(0.5), z.literal(0.9), z.literal(0.95), z.literal(0.99)]) .optional() .describe( - 'Percentile level. Only applicable when aggFn is "quantile". ' + + 'Percentile level. Required when aggFn is "quantile" on a histogram metric, ' + + 'optional otherwise. ' + 'Allowed values: 0.5, 0.9, 0.95, 0.99', ), + metricType: mcpMetricTypeSchema + .optional() + .describe( + 'METRIC SOURCES ONLY. OTel metric kind: gauge, sum, or histogram. ' + + 'Required (with metricName) when sourceId is a metric source. ' + + 'Discover via clickstack_describe_source (sample) or clickstack_describe_metric.', + ), + metricName: z + .string() + .optional() + .describe( + 'METRIC SOURCES ONLY. OTel metric name (e.g. "system.cpu.utilization", ' + + '"http.server.request.duration"). Required when metricType is set. ' + + 'Discover via clickstack_list_metrics or clickstack_describe_source.', + ), + isDelta: z + .boolean() + .optional() + .describe( + 'METRIC SOURCES ONLY (gauge metrics). When true, computes the Prometheus-style ' + + 'delta over each bucket: (argMax(Value) - argMin(Value)) * bucketSecs / timeDiff. ' + + 'Use for cumulative gauges where you want to chart growth per bucket. Default false.', + ), }); +// NOTE: cross-field validation (metric rules + level + valueExpression) is +// applied imperatively in the timeseries / table tool handlers via +// `validateMetricSelectItems`. We intentionally skip `.superRefine` here +// because Zod 3.x widens optional-field types post-refine, which breaks +// strict-typed downstream consumers like `mergeWhereIntoSelectItems` and +// `resolveOrderBy`. The dashboard tile schema, whose consumers don't trip +// on the widening, keeps its own `.superRefine`. + +/** + * Concrete shape of a parsed MCP select item. Mirrors the runtime values + * produced by `mcpSelectItemSchema` — needed as an explicit type because + * Zod 3.x's structural inference of `z.object({...})` callbacks (e.g. the + * MCP SDK's tool-handler input) can widen optional-field types into + * `unknown`. Cast `input.select` to `McpSelectItem[]` at tool boundaries. + */ +export type McpSelectItem = { + aggFn: string; + valueExpression?: string; + where?: string; + whereLanguage?: 'lucene' | 'sql'; + alias?: string; + level?: number; + metricType?: 'gauge' | 'sum' | 'histogram'; + metricName?: string; + isDelta?: boolean; +}; + +/** + * Default `valueExpression` to `"Value"` for every metric-tagged select + * item that omits it. Must be called BEFORE `buildTile`, because the + * external dashboard tile schema's `superRefine` rejects non-count + * aggregations with an empty `valueExpression`. The runtime renderer + * looks for `Value` on metric tables, so defaulting it here matches what + * `external-api/v2/charts.ts:240-250` does on the REST path. + */ +export function applyMetricSelectDefaults( + items: ReadonlyArray, +): T[] { + return items.map(item => + item.metricType && !item.valueExpression + ? { ...item, valueExpression: 'Value' } + : item, + ); +} + +/** + * Apply `getMetricSelectIssues` to every select item in a tool input. + * Returns an error-shaped tool response when any issue is detected, or + * `null` when all items pass. Call this from a tool handler before + * passing items to `runConfigTile`. + */ +export function validateMetricSelectItems( + items: ReadonlyArray, +): { isError: true; content: [{ type: 'text'; text: string }] } | null { + const errors: string[] = []; + items.forEach((item, idx) => { + for (const issue of getMetricSelectIssues(item)) { + errors.push(`select[${idx}].${issue.path.join('.')}: ${issue.message}`); + } + }); + if (errors.length === 0) return null; + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: errors.join('\n'), + }, + ], + }; +} export const startTimeSchema = z .string() diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index eb8eb46374..34fcdf6512 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -5,19 +5,23 @@ import type { McpContext } from '@/mcp/tools/types'; import { withToolTracing } from '@/mcp/utils/tracing'; import { + annotateIncreaseTopNHint, buildTile, mergeWhereIntoSelectItems, parseTimeRange, runConfigTile, } from './helpers'; import { + applyMetricSelectDefaults, endTimeSchema, groupBySchema, MCP_AGG_FN_OPTIONS, + McpSelectItem, mcpSelectItemSchema, orderBySchema, sourceIdSchema, startTimeSchema, + validateMetricSelectItems, whereLanguageSchema, whereSchema, } from './schemas'; @@ -62,9 +66,13 @@ const tableSchema = z.object({ // ─── orderBy resolution ────────────────────────────────────────────────────── /** Aggregation function names that ClickHouse cannot resolve as bare identifiers in ORDER BY. - * 'none' is excluded — it passes a raw expression through unchanged and has no synthesizable form. */ + * Excluded: + * - 'none' passes a raw expression through unchanged and has no synthesizable form. + * - 'increase' is a metric-only renderer marker that compiles to a multi-CTE + * sum(Rate) pipeline — there is no standalone SQL function to synthesize. + * The renderer auto-aliases the resulting column; agents should orderBy by alias. */ const AGG_FN_NAMES: ReadonlySet = new Set( - MCP_AGG_FN_OPTIONS.filter(fn => fn !== 'none'), + MCP_AGG_FN_OPTIONS.filter(fn => fn !== 'none' && fn !== 'increase'), ); /** A bare ClickHouse identifier: a letter or underscore followed by word chars. @@ -209,7 +217,18 @@ export function registerTable(server: McpServer, context: McpContext) { 'Map attributes work in groupBy and valueExpression, including ' + "toFloat64OrZero(SpanAttributes['key']).\n\n" + 'Shape auto-upgrade: if shape is "number" or "pie" but select has >1 item, ' + - 'it is transparently upgraded to "table".', + 'it is transparently upgraded to "table".\n\n' + + '── METRIC SOURCES ──\n' + + 'When sourceId is a metric source, each select item MUST set ' + + 'metricType ("gauge"|"sum"|"histogram") and metricName (the OTel metric name). ' + + 'valueExpression defaults to "Value" — set it explicitly only to transform the value.\n' + + 'Discovery: clickstack_describe_source returns a per-kind metric-name sample; ' + + 'clickstack_list_metrics paginates the full catalog; clickstack_describe_metric ' + + 'returns attribute keys + sampled values for a single metric.\n' + + 'Per kind: gauge uses last_value/avg/min/max; sum uses aggFn:"increase" for counter increase ' + + '(top-N capped at 20 groups when combined with groupBy), or sum/avg on the rate; ' + + 'histogram uses aggFn:"quantile" + level for percentiles, or aggFn:"count" for total bucket count.\n' + + 'summary and exponential histogram kinds are not supported by the query renderer yet.', inputSchema: tableSchema, }, withToolTracing('clickstack_table', context, async input => { @@ -222,12 +241,30 @@ export function registerTable(server: McpServer, context: McpContext) { } const { startDate, endDate } = timeRange; + // Cast to the concrete `McpSelectItem[]` because Zod 3.x widens + // optional-field inference at the MCP-SDK tool boundary; the + // runtime parser still produces the correct shape. + const rawSelect = input.select as McpSelectItem[]; + + // Validate cross-field constraints (metric rules, level/quantile, + // valueExpression presence) and surface friendly errors before we + // touch ClickHouse. + const validation = validateMetricSelectItems(rawSelect); + if (validation) return validation; + + // Default valueExpression="Value" for metric items BEFORE we call + // buildTile, because the external dashboard tile schema's + // superRefine rejects non-count aggregations with empty + // valueExpression and agents normally omit the field on metric + // queries. + const select = applyMetricSelectDefaults(rawSelect); + // Auto-upgrade shape when select has multiple items but shape is // single-value (number/pie). This is the #1 Zod error class from agents. let displayType: 'table' | 'number' | 'pie' = input.shape; if ( (displayType === 'number' || displayType === 'pie') && - input.select.length > 1 + select.length > 1 ) { displayType = 'table'; } @@ -236,11 +273,7 @@ export function registerTable(server: McpServer, context: McpContext) { // of the aggCondition for every metric. Table/line/number/pie display // types don't have a chart-level where — filtering is per-select-item. const { items: selectItems, warnings: mergeWarnings } = - mergeWhereIntoSelectItems( - input.select, - input.where, - input.whereLanguage, - ); + mergeWhereIntoSelectItems(select, input.where, input.whereLanguage); const tile = buildTile('MCP Table', 12, 4, { displayType, @@ -269,6 +302,10 @@ export function registerTable(server: McpServer, context: McpContext) { } } + // Surface the increase+groupBy top-N cap so the agent knows results + // may be truncated to 20 groups. + annotateIncreaseTopNHint(result, select, input.groupBy); + return result; }), ); diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index 0c5d5fbdd1..52a36be8c3 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -1,3 +1,4 @@ +import { FIXED_TIME_BUCKET_EXPR_ALIAS } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; @@ -5,18 +6,23 @@ import type { McpContext } from '@/mcp/tools/types'; import { withToolTracing } from '@/mcp/utils/tracing'; import { + annotateIncreaseTopNHint, + appendHint, buildTile, mergeWhereIntoSelectItems, parseTimeRange, runConfigTile, } from './helpers'; import { + applyMetricSelectDefaults, endTimeSchema, groupBySchema, + McpSelectItem, mcpSelectItemSchema, orderBySchema, sourceIdSchema, startTimeSchema, + validateMetricSelectItems, whereLanguageSchema, whereSchema, } from './schemas'; @@ -80,7 +86,20 @@ export function registerTimeseries(server: McpServer, context: McpContext) { 'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first. ' + 'Each select item defines one plotted series.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + - "Map attributes use bracket syntax: SpanAttributes['http.method'].", + "Map attributes use bracket syntax: SpanAttributes['http.method'].\n\n" + + '── METRIC SOURCES ──\n' + + 'When sourceId is a metric source, each select item MUST set ' + + 'metricType ("gauge"|"sum"|"histogram") and metricName (the OTel metric name). ' + + 'valueExpression defaults to "Value" — set it explicitly only to transform the value.\n' + + 'Discovery: clickstack_describe_source returns a per-kind metric-name sample; ' + + 'clickstack_list_metrics paginates the full catalog; clickstack_describe_metric ' + + 'returns attribute keys + sampled values for a single metric.\n' + + 'Per kind: gauge uses last_value/avg/min/max (or aggFn:any + isDelta:true for Prometheus-style delta); ' + + 'sum uses aggFn:"increase" for the counter increase, or sum/avg on the computed rate; ' + + 'histogram uses aggFn:"quantile" + level for percentiles, or aggFn:"count" for total bucket count.\n' + + 'TOP-N CAP: aggFn:"increase" + groupBy is capped at 20 groups by the renderer ' + + '(top by max bucket sum). Narrow with where/groupBy to see other groups.\n' + + 'summary and exponential histogram kinds are not supported by the query renderer yet.', inputSchema: timeseriesSchema, }, withToolTracing('clickstack_timeseries', context, async input => { @@ -93,14 +112,28 @@ export function registerTimeseries(server: McpServer, context: McpContext) { } const { startDate, endDate } = timeRange; + // Cast to the concrete `McpSelectItem[]` because Zod 3.x widens + // optional-field inference at the MCP-SDK tool boundary; the + // runtime parser still produces the correct shape. + const rawSelect = input.select as McpSelectItem[]; + + // Validate cross-field constraints (metric rules, level/quantile, + // valueExpression presence) and surface friendly errors before we + // touch ClickHouse. + const validation = validateMetricSelectItems(rawSelect); + if (validation) return validation; + + // Default valueExpression="Value" for metric items BEFORE we call + // buildTile, because the external dashboard tile schema's + // superRefine rejects non-count aggregations with empty + // valueExpression and agents normally omit the field on metric + // queries. + const select = applyMetricSelectDefaults(rawSelect); + // Inject top-level where into each select item (timeseries uses // per-select-item aggCondition, not chart-level where) const { items: selectItems, warnings: mergeWarnings } = - mergeWhereIntoSelectItems( - input.select, - input.where, - input.whereLanguage, - ); + mergeWhereIntoSelectItems(select, input.where, input.whereLanguage); const tile = buildTile('MCP Timeseries', 12, 4, { displayType: input.shape, @@ -116,6 +149,7 @@ export function registerTimeseries(server: McpServer, context: McpContext) { tile, startDate, endDate, + { granularity: input.granularity }, ); // Surface language-mismatch warnings so the agent knows the top-level @@ -130,9 +164,16 @@ export function registerTimeseries(server: McpServer, context: McpContext) { } } - // Detect single-bucket collapse: when a timeseries query returns only - // 1 row, the data likely collapsed into a single time bucket. Add a - // hint so the agent knows to adjust the time range. + // Surface the increase+groupBy top-N cap so the agent knows results + // may be truncated to 20 groups. + annotateIncreaseTopNHint(result, select, input.groupBy); + + // Time-dimension hints, keyed on the renderer's time-bucket column + // (FIXED_TIME_BUCKET_EXPR_ALIAS) in result.meta rather than on a raw + // row count. Counting rows conflated three cases: genuine collapse, + // a single group under groupBy (working as intended), and the + // no-time-bucket case — where "use a coarser granularity" is + // actively wrong advice. if ( result.content?.[0]?.type === 'text' && !('isError' in result && result.isError) @@ -140,14 +181,40 @@ export function registerTimeseries(server: McpServer, context: McpContext) { try { const parsed = JSON.parse(result.content[0].text); const data = parsed?.result?.data; - if (Array.isArray(data) && data.length === 1 && input.granularity) { - parsed.hint = - `Timeseries returned only 1 time bucket. ` + - `All data may have collapsed into a single "${input.granularity}" bucket. ` + - `The queried range was ${startDate.toISOString()} to ${endDate.toISOString()}. ` + - `If this looks wrong, adjust startTime/endTime to match the actual data range, ` + - `or try a coarser granularity.`; - result.content[0].text = JSON.stringify(parsed, null, 2); + const meta = parsed?.result?.meta; + if (Array.isArray(data) && data.length > 0) { + const hasTimeBucket = + Array.isArray(meta) && + meta.some( + (m: { name?: string }) => + m?.name === FIXED_TIME_BUCKET_EXPR_ALIAS, + ); + if (!hasTimeBucket) { + // No time bucket column — the result is not bucketed over + // time at all (a single aggregate per group). Adjusting + // granularity cannot help here. + appendHint( + parsed, + `Result is not bucketed over time (no ${FIXED_TIME_BUCKET_EXPR_ALIAS} column). ` + + `Each row is a single aggregate over the whole range ` + + `(${startDate.toISOString()} to ${endDate.toISOString()}). ` + + `Pass a granularity (e.g. "1 minute", "1 hour") to get a per-bucket time series.`, + ); + result.content[0].text = JSON.stringify(parsed, null, 2); + } else if (data.length === 1) { + // Genuine single-bucket collapse: the time bucket is present + // but everything landed in one bucket. Coarser/adjust-range + // advice is appropriate here. + appendHint( + parsed, + `Timeseries returned only 1 time bucket. ` + + `All data may have collapsed into a single bucket over ` + + `${startDate.toISOString()} to ${endDate.toISOString()}. ` + + `If this looks wrong, widen startTime/endTime to match the actual data range, ` + + `or try a finer granularity.`, + ); + result.content[0].text = JSON.stringify(parsed, null, 2); + } } } catch { // If parsing fails, return the original result unmodified diff --git a/packages/api/src/mcp/tools/sources/describeMetric.ts b/packages/api/src/mcp/tools/sources/describeMetric.ts new file mode 100644 index 0000000000..d823975cfd --- /dev/null +++ b/packages/api/src/mcp/tools/sources/describeMetric.ts @@ -0,0 +1,801 @@ +import { + chSql, + concatChSql, + convertCHDataTypeToJSType, + filterColumnMetaByType, + JSDataType, + tableExpr, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; +import type { McpContext } from '@/mcp/tools/types'; +import { withToolTracing } from '@/mcp/utils/tracing'; +import logger from '@/utils/logger'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +import { + QUERYABLE_METRIC_KINDS, + type QueryableMetricKind, +} from './metricKinds'; + +const DEFAULT_LOOKBACK_MS = 24 * 60 * 60 * 1000; +const DESCRIBE_TIMEOUT_MS = 10_000; + +// Server-side safety nets for the attribute-keys discovery query. +// Sample at most N rows that match (MetricName, time range), then +// aggregate from that sample. ClickHouse can stop scanning once the +// CTE has N matching rows so a hot metric does not starve the +// wall-clock budget. 100k rows is plenty to surface every unique map +// key on a healthy OTel metric. +const METRIC_ATTR_KEYS_SAMPLE_SIZE = 100_000; +const METRIC_ATTR_KEYS_MAX_EXEC_SECONDS = 8; + +// Max sampled values per attribute key (when sampleValues is true). +const MAX_ATTR_VALUES = 10; +// Cap on the number of attribute keys we sample values for per kind to +// avoid runaway fan-out on high-cardinality metrics. +const MAX_ATTR_KEYS_TO_SAMPLE = 12; + +// Per-kind aggregation guidance baked into the response so the agent can +// build a valid clickstack_timeseries / clickstack_table call without +// re-reading the schemas. +const KIND_USAGE: Record = { + gauge: + 'Gauge: use aggFn:"last_value"|"avg"|"min"|"max" on Value. Set isDelta:true for Prometheus-style delta over each bucket.', + sum: 'Sum (counter): use aggFn:"increase" for the per-bucket counter increase (reset-aware), or aggFn:"sum"/"avg" on the computed rate. increase+groupBy is capped at the top 20 groups.', + histogram: + 'Histogram: use aggFn:"quantile" with level ∈ {0.5, 0.9, 0.95, 0.99} for percentiles, or aggFn:"count" for the total bucket count.', +}; + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const describeMetricSchema = z.object({ + sourceId: z + .string() + .describe( + 'Source ID. Must reference a metric source — get IDs from clickstack_list_sources.', + ), + metricName: z + .string() + .min(1) + .describe( + 'OTel metric name to describe (e.g. "system.cpu.utilization", "http.server.request.duration"). ' + + 'Discover via clickstack_list_metrics or clickstack_describe_source.', + ), + kind: z + .enum(QUERYABLE_METRIC_KINDS) + .describe( + 'Metric kind: "gauge" | "sum" | "histogram". Required. ' + + 'Discover via clickstack_list_metrics (which returns name + kind per entry) ' + + 'or clickstack_describe_source (which groups metric-name samples by kind). ' + + 'A metric name can legitimately live in more than one kind (e.g. ' + + '"container.cpu.usage" appears in both gauge and sum) — pick the kind ' + + 'matching the value you want to query.', + ), + startTime: z + .string() + .optional() + .describe( + 'Restrict discovery to data points after this ISO 8601 timestamp. Default: 24 hours before endTime (or now).', + ), + endTime: z + .string() + .optional() + .describe('End of the discovery window as ISO 8601. Default: now.'), + sampleValues: z + .boolean() + .optional() + .default(true) + .describe( + 'When true (default), sample up to ' + + `${MAX_ATTR_VALUES} distinct values per attribute key for the top ${MAX_ATTR_KEYS_TO_SAMPLE} attributes. ` + + 'Set false to skip value sampling for faster responses on high-cardinality metrics.', + ), +}); + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +type AttributeValuesMeta = { + /** Display names (`MapColumn['key']`) queried for value samples. */ + sampledKeys: string[]; + /** Display names skipped because the MAX_ATTR_KEYS_TO_SAMPLE cap fired. */ + truncatedKeys: string[]; +}; + +type KindDetail = { + kind: QueryableMetricKind; + tableName: string; + unit?: string; + description?: string; + attributeKeys: Record; + attributeValues?: Record; + attributeValuesMeta?: AttributeValuesMeta; + usage: string; +}; + +/** + * Discriminated result for the discovery sub-queries so the caller can + * distinguish "fetch failed" from "genuinely empty" — the two cases + * need different agent guidance (retry/report vs. widen the window). + */ +type FetchResult = { ok: true; data: T } | { ok: false; error: string }; + +/** + * Compact an error for inclusion in a tool response: single line, + * capped length, no stack frames. + */ +function sanitizeFetchError(e: unknown): string { + const message = e instanceof Error ? e.message : String(e); + return message.replace(/\s+/g, ' ').trim().slice(0, 200); +} + +function parseTimeRange( + startTime?: string, + endTime?: string, +): { error: string } | { startDate: Date; endDate: Date } { + const endDate = endTime ? new Date(endTime) : new Date(); + const startDate = startTime + ? new Date(startTime) + : new Date(endDate.getTime() - DEFAULT_LOOKBACK_MS); + if (isNaN(endDate.getTime()) || isNaN(startDate.getTime())) { + return { + error: 'Invalid startTime or endTime: must be valid ISO 8601 strings', + }; + } + if (startDate >= endDate) { + return { error: 'endTime must be greater than startTime' }; + } + return { startDate, endDate }; +} + +/** + * Fetch unit and description for a metric name on a single kind table. + * Returns undefined values when the columns are absent from the schema. + */ +async function fetchUnitAndDescription({ + clickhouseClient, + databaseName, + tableName, + connectionId, + metricName, + startDate, + endDate, + hasUnit, + hasDescription, + signal, +}: { + clickhouseClient: ClickhouseClient; + databaseName: string; + tableName: string; + connectionId: string; + metricName: string; + startDate: Date; + endDate: Date; + hasUnit: boolean; + hasDescription: boolean; + signal: AbortSignal; +}): Promise<{ unit?: string; description?: string }> { + if (!hasUnit && !hasDescription) return {}; + + const innerProjections = [ + ...(hasUnit ? [chSql`${{ Identifier: 'MetricUnit' }}`] : []), + ...(hasDescription ? [chSql`${{ Identifier: 'MetricDescription' }}`] : []), + ]; + const outerProjections = [ + ...(hasUnit + ? [chSql`anyLast(${{ Identifier: 'MetricUnit' }}) AS MetricUnit`] + : []), + ...(hasDescription + ? [ + chSql`anyLast(${{ Identifier: 'MetricDescription' }}) AS MetricDescription`, + ] + : []), + ]; + + // Wrap in a bounded subquery so ClickHouse stops scanning once it + // has enough rows — unit and description are typically constant per + // metric, so a small sample is sufficient. Mirrors the bounding + // pattern used by fetchAttributeKeys / sampleAttributeValues. + const sql = chSql` + SELECT ${concatChSql(', ', outerProjections)} + FROM ( + SELECT ${concatChSql(', ', innerProjections)} + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE MetricName = ${{ String: metricName }} + AND TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: startDate.getTime() }}) + AND TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: endDate.getTime() }}) + LIMIT ${{ Int32: METRIC_ATTR_KEYS_SAMPLE_SIZE }} + ) + `; + try { + const response = await clickhouseClient.query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + connectionId, + clickhouse_settings: { + max_execution_time: METRIC_ATTR_KEYS_MAX_EXEC_SECONDS, + timeout_overflow_mode: 'break', + }, + abort_signal: signal, + }); + const result = (await response.json()) as { + data: Array<{ MetricUnit?: string; MetricDescription?: string }>; + }; + const row = result.data[0]; + if (!row) return {}; + return { + ...(row.MetricUnit ? { unit: row.MetricUnit } : {}), + ...(row.MetricDescription ? { description: row.MetricDescription } : {}), + }; + } catch (e) { + logger.warn( + { tableName, error: e instanceof Error ? e.message : String(e) }, + 'fetchUnitAndDescription failed', + ); + return {}; + } +} + +/** + * Discover the distinct attribute keys present for a single metric + * name on a single kind's table, grouped by the Map column they live on + * (typically ResourceAttributes / Attributes / ScopeAttributes on the + * OTel Collector default schema). Issues one query per Map column with + * `mapKeys(col) AS keys` and aggregates the distinct keys server-side. + * + * Bounds the scan two ways so it stays fast on production-shaped metric + * tables (where a single MetricName can match millions of rows): + * - WHERE TimeUnix BETWEEN startDate AND endDate scopes to the + * discovery window the caller already passed. + * - max_rows_to_read caps the per-query scan server-side so a hot + * metric cannot starve the wall-clock budget on its own. + * + * The earlier inline SQL did neither — an unbounded + * `WHERE MetricName = ?` scan consistently exceeded the wall-clock + * timeout on production tables. + */ +async function fetchAttributeKeys({ + clickhouseClient, + databaseName, + tableName, + connectionId, + metricName, + columns, + startDate, + endDate, + signal, +}: { + clickhouseClient: ClickhouseClient; + databaseName: string; + tableName: string; + connectionId: string; + metricName: string; + columns: { name: string; type: string }[]; + startDate: Date; + endDate: Date; + signal: AbortSignal; +}): Promise>> { + const mapColumns = + filterColumnMetaByType(columns, [JSDataType.Map])?.filter( + c => convertCHDataTypeToJSType(c.type) === JSDataType.Map, + ) ?? []; + if (mapColumns.length === 0) return { ok: true, data: {} }; + + const sampleProjections = mapColumns.map( + col => chSql`${{ Identifier: col.name }}`, + ); + + const aggProjections = mapColumns.map( + col => + chSql`arrayDistinct(arrayFlatten(groupArray(mapKeys(${{ Identifier: col.name }})))) AS ${{ Identifier: col.name }}`, + ); + + // Aggregate from a bounded sample of matching rows. The inner LIMIT + // lets ClickHouse stop scanning once it has SAMPLE_SIZE rows that + // match (MetricName, time range), which keeps the query fast on hot + // metric tables. ResourceAttributes / Attributes / ScopeAttributes + // are rarely keyed independently per row, so 100k rows surfaces + // every realistic key set. + const sql = chSql` + SELECT ${concatChSql(', ', aggProjections)} + FROM ( + SELECT ${concatChSql(', ', sampleProjections)} + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE MetricName = ${{ String: metricName }} + AND TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: startDate.getTime() }}) + AND TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: endDate.getTime() }}) + LIMIT ${{ Int32: METRIC_ATTR_KEYS_SAMPLE_SIZE }} + ) + `; + + try { + const response = await clickhouseClient.query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + connectionId, + clickhouse_settings: { + max_execution_time: METRIC_ATTR_KEYS_MAX_EXEC_SECONDS, + timeout_overflow_mode: 'break', + }, + abort_signal: signal, + }); + const result = (await response.json()) as { + data: Array>; + }; + const row = result.data[0]; + if (!row) return { ok: true, data: {} }; + const out: Record = {}; + for (const col of mapColumns) { + const keys = row[col.name]; + if (Array.isArray(keys) && keys.length > 0) { + out[col.name] = keys.filter(k => typeof k === 'string' && k.length > 0); + } + } + return { ok: true, data: out }; + } catch (e) { + logger.warn( + { tableName, error: e instanceof Error ? e.message : String(e) }, + 'fetchAttributeKeys failed', + ); + return { ok: false, error: sanitizeFetchError(e) }; + } +} + +type SampleAttributeValuesData = { + /** Keys with at least one non-empty sampled value. */ + values: Record; + meta: AttributeValuesMeta; +}; + +/** + * Sample distinct values per attribute key. Uses one composed + * groupArray-per-key query so all keys are fetched in a single round + * trip. + * + * The returned meta records which keys were actually queried + * (`sampledKeys`) vs. skipped by the MAX_ATTR_KEYS_TO_SAMPLE cap + * (`truncatedKeys`) so callers can distinguish "not sampled" from + * "sampled but empty in the window". + */ +async function sampleAttributeValues({ + clickhouseClient, + databaseName, + tableName, + connectionId, + metricName, + attributeKeys, + startDate, + endDate, + signal, +}: { + clickhouseClient: ClickhouseClient; + databaseName: string; + tableName: string; + connectionId: string; + metricName: string; + // attributeKeys is shape { mapColumn: keyName[] } e.g. { Attributes: ['http.method', 'http.route'] } + attributeKeys: Record; + startDate: Date; + endDate: Date; + signal: AbortSignal; +}): Promise> { + const flatKeyExprs: { display: string; mapColumn: string; key: string }[] = + []; + const truncatedKeys: string[] = []; + for (const [mapColumn, keys] of Object.entries(attributeKeys)) { + for (const key of keys) { + const display = `${mapColumn}['${key}']`; + if (flatKeyExprs.length >= MAX_ATTR_KEYS_TO_SAMPLE) { + truncatedKeys.push(display); + continue; + } + flatKeyExprs.push({ display, mapColumn, key }); + } + } + const meta: AttributeValuesMeta = { + sampledKeys: flatKeyExprs.map(e => e.display), + truncatedKeys, + }; + if (flatKeyExprs.length === 0) { + return { ok: true, data: { values: {}, meta } }; + } + + const projections = flatKeyExprs.map( + ({ mapColumn, key }, idx) => chSql` + groupUniqArray(${{ Int32: MAX_ATTR_VALUES }})( + ${{ Identifier: mapColumn }}[${{ String: key }}] + ) AS param${{ UNSAFE_RAW_SQL: String(idx) }} + `, + ); + + // Distinct set of map columns we need from each sampled row, so the + // inner subquery only projects what's required for the outer + // groupUniqArray aggregates. + const sampledMapColumns = Array.from( + new Set(flatKeyExprs.map(e => e.mapColumn)), + ); + const sampleProjections = sampledMapColumns.map( + col => chSql`${{ Identifier: col }}`, + ); + + // Aggregate from a bounded sample of matching rows. Mirrors + // fetchAttributeKeys: the inner LIMIT lets ClickHouse stop scanning + // once it has SAMPLE_SIZE rows that match (MetricName, time range), + // and the matching clickhouse_settings cap server-side execution so a + // hot metric cannot starve the wall-clock budget. + const sql = chSql` + SELECT ${concatChSql(', ', projections)} + FROM ( + SELECT ${concatChSql(', ', sampleProjections)} + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE MetricName = ${{ String: metricName }} + AND TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: startDate.getTime() }}) + AND TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: endDate.getTime() }}) + LIMIT ${{ Int32: METRIC_ATTR_KEYS_SAMPLE_SIZE }} + ) + `; + + try { + const response = await clickhouseClient.query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + connectionId, + clickhouse_settings: { + max_execution_time: METRIC_ATTR_KEYS_MAX_EXEC_SECONDS, + timeout_overflow_mode: 'break', + }, + abort_signal: signal, + }); + const result = (await response.json()) as { + data: Array>; + }; + const row = result.data[0]; + if (!row) return { ok: true, data: { values: {}, meta } }; + const values: Record = {}; + flatKeyExprs.forEach((keyExpr, idx) => { + const sample = (row[`param${idx}`] ?? []).filter(v => v !== ''); + if (sample.length > 0) { + values[keyExpr.display] = sample; + } + }); + return { ok: true, data: { values, meta } }; + } catch (e) { + logger.warn( + { tableName, error: e instanceof Error ? e.message : String(e) }, + 'sampleAttributeValues failed', + ); + return { ok: false, error: sanitizeFetchError(e) }; + } +} + +async function describeMetricImpl( + teamId: string, + input: z.infer, + signal: AbortSignal, +) { + const source = await getSource(teamId, input.sourceId); + if (!source) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Source "${input.sourceId}" not found. Call clickstack_list_sources to see available source IDs.`, + }, + ], + }; + } + if (source.kind !== SourceKind.Metric) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Source "${input.sourceId}" is a "${source.kind}" source, not a metric source. clickstack_describe_metric only works on metric sources.`, + }, + ], + }; + } + + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Connection not found for source "${input.sourceId}".`, + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + const databaseName = source.from.databaseName; + const connectionId = source.connection.toString(); + + // Validate the requested kind has a populated table on this source. + // The schema requires `kind`, so we always have an exact target. + const kind = input.kind; + const tableName = source.metricTables[kind]; + if (!tableName) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Source "${input.sourceId}" has no "${kind}" metric table populated. Populated kinds: ${Object.keys(source.metricTables).join(', ')}.`, + }, + ], + }; + } + + // Defensive column-presence check before referencing MetricUnit / + // MetricDescription on this kind's table. + let columns: { name: string; type: string }[]; + try { + columns = await metadata.getColumns({ + databaseName, + tableName, + connectionId, + }); + } catch (e) { + logger.warn( + { kind, error: e instanceof Error ? e.message : String(e) }, + 'describeMetric: getColumns failed', + ); + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Failed to load columns for "${tableName}". The metric table may be missing or unreachable.`, + }, + ], + }; + } + const columnNames = new Set(columns.map(c => c.name)); + const hasUnit = columnNames.has('MetricUnit'); + const hasDescription = columnNames.has('MetricDescription'); + + const [meta, attributeKeysResult] = await Promise.all([ + fetchUnitAndDescription({ + clickhouseClient, + databaseName, + tableName, + connectionId, + metricName: input.metricName, + startDate, + endDate, + hasUnit, + hasDescription, + signal, + }), + fetchAttributeKeys({ + clickhouseClient, + databaseName, + tableName, + connectionId, + metricName: input.metricName, + columns, + startDate, + endDate, + signal, + }), + ]); + + // Track discovery sub-queries that failed so the agent can distinguish + // "fetch failed" (retry / report) from "genuinely empty" (widen the + // window). Without this, transient ClickHouse errors surfaced as the + // misleading "No data found" hint. + const partialFailure: { stage: string; error: string }[] = []; + const attributeKeys = attributeKeysResult.ok ? attributeKeysResult.data : {}; + if (!attributeKeysResult.ok) { + partialFailure.push({ + stage: 'attributeKeys', + error: attributeKeysResult.error, + }); + } + + const kindDetail: KindDetail = { + kind, + tableName, + ...(meta.unit ? { unit: meta.unit } : {}), + ...(meta.description ? { description: meta.description } : {}), + attributeKeys, + usage: KIND_USAGE[kind], + }; + + if (input.sampleValues && Object.keys(attributeKeys).length > 0) { + const valuesResult = await sampleAttributeValues({ + clickhouseClient, + databaseName, + tableName, + connectionId, + metricName: input.metricName, + attributeKeys, + startDate, + endDate, + signal, + }); + if (valuesResult.ok) { + kindDetail.attributeValues = valuesResult.data.values; + // sampledKeys / truncatedKeys let the agent tell "key absent + // because we never queried it" (truncated) apart from "queried + // but empty in the window" (in sampledKeys, missing from values). + kindDetail.attributeValuesMeta = valuesResult.data.meta; + } else { + partialFailure.push({ + stage: 'attributeValues', + error: valuesResult.error, + }); + } + } + + const kindDetails: KindDetail[] = [kindDetail]; + + // Heuristic "no data in this kind" hint: when neither attribute keys, + // unit, nor description came back, the (metric, kind) pair likely has + // no data in the requested window. The agent's recourse is to widen + // startTime/endTime or double-check the kind via clickstack_list_metrics. + // Suppressed when any discovery stage failed — an empty attributeKeys + // can't be trusted as "no data" then. + const noSignal = + partialFailure.length === 0 && + Object.keys(attributeKeys).length === 0 && + !meta.unit && + !meta.description; + + const queryExample = `clickstack_timeseries({ sourceId: "${input.sourceId}", select: [{ aggFn: ${ + kind === 'sum' + ? '"increase"' + : kind === 'histogram' + ? '"quantile", level: 0.95' + : '"avg"' + }, metricType: "${kind}", metricName: "${input.metricName}" }] })`; + + const responseObj: Record = { + metricName: input.metricName, + kinds: kindDetails, + ...(partialFailure.length > 0 && { + partialFailure, + hint: + 'Some discovery queries failed — the fields above may be incomplete. ' + + 'Retry the call; if the failure persists, narrow startTime/endTime or set sampleValues:false.', + }), + ...(noSignal && { + hint: + `No data found for MetricName "${input.metricName}" with kind "${kind}" ` + + `between ${startDate.toISOString()} and ${endDate.toISOString()}. ` + + 'Try widening startTime/endTime, or call clickstack_list_metrics to ' + + 'confirm the metric name + kind combination exists.', + }), + nextSteps: { + query: `Example: ${queryExample}`, + }, + }; + + const { data: trimmed, isTrimmed } = trimToolResponse(responseObj); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + isTrimmed + ? { + ...trimmed, + note: 'Result was trimmed for context size. Set sampleValues:false or narrow startTime/endTime.', + } + : trimmed, + null, + 2, + ), + }, + ], + }; +} + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerDescribeMetric( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'clickstack_describe_metric', + { + title: 'Describe Metric', + description: + 'DRILL-DOWN: Use after clickstack_list_metrics (or after a clickstack_describe_source ' + + 'sample) to get attribute keys, sampled values, unit, and description for a ' + + 'specific (metricName, kind) pair. Attribute keys vary per metric — not per source — ' + + "so always call this before clickstack_timeseries / clickstack_table for any metric you've never queried.\n\n" + + 'REQUIRES `kind` — pass the gauge/sum/histogram value emitted alongside the metric name by ' + + 'clickstack_list_metrics or clickstack_describe_source. A metric name can legitimately ' + + 'live in more than one kind (e.g. "container.cpu.usage" appears in both gauge and sum); ' + + 'call this tool once per kind you care about.\n\n' + + 'attributeValuesMeta on each kind reports sampledKeys (queried for values) and ' + + 'truncatedKeys (skipped by the per-call sampling cap) — a key in truncatedKeys was ' + + 'never queried, so query it directly if you need its values.\n\n' + + 'Workflow: clickstack_list_sources → clickstack_list_metrics → ' + + 'clickstack_describe_metric → clickstack_timeseries|clickstack_table.', + inputSchema: describeMetricSchema, + }, + withToolTracing('clickstack_describe_metric', context, async rawInput => { + // Re-parse explicitly: the MCP SDK callback signature widens + // optional-field types into `unknown`, but the parser produces + // the typed shape we need for downstream calls. + const input = describeMetricSchema.parse(rawInput); + const controller = new AbortController(); + // Hoist the timer handle so the finally block can cancel it on the + // success path — otherwise a stale controller.abort() fires + // DESCRIBE_TIMEOUT_MS after every successful call and the + // setTimeout closure stays pinned for the same duration. + let timeoutId: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + controller.abort(); + reject(new Error('DESCRIBE_METRIC_TIMEOUT')); + }, DESCRIBE_TIMEOUT_MS); + }); + try { + return await Promise.race([ + describeMetricImpl(teamId.toString(), input, controller.signal), + timeoutPromise, + ]); + } catch (e) { + if (e instanceof Error && e.message === 'DESCRIBE_METRIC_TIMEOUT') { + logger.warn( + { teamId, sourceId: input.sourceId, metricName: input.metricName }, + 'clickstack_describe_metric timed out', + ); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'Discovery timed out. The metric table may be under load or the ' + + 'attribute set may be very high-cardinality. Try narrowing ' + + 'startTime/endTime or setting sampleValues:false to skip the ' + + 'value-sampling stage.', + }, + ], + }; + } + throw e; + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } + }), + ); +} diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index f2437b4b1f..5d9b54a624 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -1,11 +1,14 @@ import { + chSql, + concatChSql, convertCHDataTypeToJSType, filterColumnMetaByType, JSDataType, + tableExpr, } from '@hyperdx/common-utils/dist/clickhouse'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; @@ -16,6 +19,12 @@ import { withToolTracing } from '@/mcp/utils/tracing'; import logger from '@/utils/logger'; import { trimToolResponse } from '@/utils/trimToolResponse'; +import { + QUERYABLE_METRIC_KINDS, + type QueryableMetricKind, + sanitizeMetricTables, +} from './metricKinds'; + // How far back to look when querying the rollup tables for value samples. const VALUE_SAMPLE_LOOKBACK_MS = 24 * 60 * 60 * 1000; // 24 hours @@ -27,6 +36,200 @@ const MAX_LC_VALUES = 20; const MAX_MAP_KEY_VALUES = 5; const MAX_MAP_KEYS_TO_SAMPLE = 10; +// Max MetricName values returned per metric kind by the starter sample. +// clickstack_list_metrics provides paginated discovery beyond this cap. +const MAX_METRIC_NAMES_PER_KIND = 20; + +/** + * Pick the representative metric table to use as the starting point for + * schema/attribute discovery on a metric source. Prefers gauge → sum → + * histogram from the source's populated metricTables map. Returns the + * ClickHouse table name, or undefined when no queryable metric table is + * populated. + */ +function pickRepresentativeMetricTable( + metricTables: MetricTable, +): { kind: QueryableMetricKind; tableName: string } | undefined { + for (const kind of QUERYABLE_METRIC_KINDS) { + const tableName = metricTables[kind]; + if (tableName) { + return { kind, tableName }; + } + } + return undefined; +} + +type MetricNameSample = { + name: string; + unit?: string; + description?: string; +}; + +/** + * Sample distinct MetricName values for a single metric kind. Optionally + * enriches each name with MetricUnit / MetricDescription when those + * columns are present on the table (the OTel Collector default schema + * includes them; custom schemas may not). + */ +async function sampleMetricNamesForKind({ + metadata, + clickhouseClient, + databaseName, + tableName, + connectionId, + dateRange, + timestampValueExpression, + signal, + cachedColumns, +}: { + metadata: ReturnType; + clickhouseClient: ClickhouseClient; + databaseName: string; + tableName: string; + connectionId: string; + dateRange: [Date, Date]; + timestampValueExpression: string; + signal: AbortSignal; + cachedColumns?: { name: string }[]; +}): Promise { + // Defensive column presence check for MetricUnit / MetricDescription. + const kindColumns = + cachedColumns ?? + (await metadata.getColumns({ databaseName, tableName, connectionId })); + const columnNames = new Set(kindColumns.map(c => c.name)); + const hasUnit = columnNames.has('MetricUnit'); + const hasDescription = columnNames.has('MetricDescription'); + + // First fetch the distinct metric names; this is the only step that + // strictly needs to succeed for the kind to appear in the response. + // Pass timestampValueExpression so the no-rollup fallback path scopes + // its scan to dateRange instead of going unbounded against the raw + // metric table on cold cache. + const nameResults = await metadata.getAllKeyValues({ + databaseName, + tableName, + keyExpressions: ['MetricName'], + maxValuesPerKey: MAX_METRIC_NAMES_PER_KIND, + connectionId, + dateRange, + timestampValueExpression, + signal, + }); + const names = nameResults[0]?.value ?? []; + if (names.length === 0) return []; + + // Best-effort enrichment with unit + description. One small query + // returns one row per metric name with the most-recent unit / desc. + let enrichments: Record = {}; + if ((hasUnit || hasDescription) && !signal.aborted) { + try { + enrichments = await fetchMetricNameEnrichments({ + clickhouseClient, + databaseName, + tableName, + connectionId, + names, + dateRange, + hasUnit, + hasDescription, + signal, + }); + } catch (e) { + logger.warn( + { databaseName, tableName, error: e }, + 'Failed to enrich metric names with unit/description', + ); + } + } + + return names.map(name => { + const enrichment = enrichments[name] ?? {}; + const sample: MetricNameSample = { name }; + if (enrichment.unit) sample.unit = enrichment.unit; + if (enrichment.description) sample.description = enrichment.description; + return sample; + }); +} + +/** + * Fetch MetricUnit and MetricDescription for a batch of metric names. + * Uses `anyLast` so the most-recent value wins when a metric has changed + * unit/description over time. + */ +async function fetchMetricNameEnrichments({ + clickhouseClient, + databaseName, + tableName, + connectionId, + names, + dateRange, + hasUnit, + hasDescription, + signal, +}: { + clickhouseClient: ClickhouseClient; + databaseName: string; + tableName: string; + connectionId: string; + names: string[]; + dateRange: [Date, Date]; + hasUnit: boolean; + hasDescription: boolean; + signal: AbortSignal; +}): Promise> { + // Build the projection fragments via the parameterised chSql DSL so + // identifiers are quoted and the unit/description columns only appear + // when present on the source table. + const projections = [ + chSql`MetricName`, + ...(hasUnit + ? [chSql`anyLast(${{ Identifier: 'MetricUnit' }}) AS MetricUnit`] + : []), + ...(hasDescription + ? [ + chSql`anyLast(${{ Identifier: 'MetricDescription' }}) AS MetricDescription`, + ] + : []), + ]; + const namePlaceholders = concatChSql( + ',', + names.map(name => chSql`${{ String: name }}`), + ); + const sql = chSql` + SELECT ${concatChSql(', ', projections)} + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE MetricName IN (${namePlaceholders}) + AND TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }}) + AND TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }}) + GROUP BY MetricName + `; + + type EnrichmentRow = { + MetricName: string; + MetricUnit?: string; + MetricDescription?: string; + }; + + const response = await clickhouseClient.query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + connectionId, + abort_signal: signal, + }); + const result = (await response.json()) as { data: EnrichmentRow[] }; + + const enrichments: Record = + {}; + for (const row of result.data) { + enrichments[row.MetricName] = { + ...(row.MetricUnit ? { unit: row.MetricUnit } : {}), + ...(row.MetricDescription ? { description: row.MetricDescription } : {}), + }; + } + return enrichments; +} + /** * Core schema-discovery logic. Extracted so the caller can wrap it in * Promise.race for wall-clock timeout enforcement. @@ -71,6 +274,9 @@ async function describeSourceSchema( } // Key columns by source kind + let representativeMetric: + | { kind: QueryableMetricKind; tableName: string } + | undefined; if (source.kind === SourceKind.Trace) { meta.keyColumns = { spanName: source.spanNameExpression, @@ -89,11 +295,29 @@ async function describeSourceSchema( traceId: source.traceIdExpression, }; } else if (source.kind === SourceKind.Metric) { - meta.metricTables = source.metricTables; + // Filter out implementation-detail keys (e.g. a stray Mongoose `_id` + // on the metricTables subdoc) so the agent only sees valid metric + // kinds. + const tables = sanitizeMetricTables( + source.metricTables as Record | undefined, + ); + if (tables) meta.metricTables = tables; + representativeMetric = pickRepresentativeMetricTable(source.metricTables); + if (representativeMetric) { + meta.discoveryMetricKind = representativeMetric.kind; + } } - // For sources without a table (e.g. metric sources), return early - if (!source.from.tableName) { + // Resolve the table name we'll use for column / map-key / value + // discovery. For non-metric sources this is just source.from.tableName. + // For metric sources we use the representative metric table picked + // above (gauge → sum → histogram). + const discoveryTableName = + source.from.tableName || representativeMetric?.tableName || ''; + + // Only early-return when there is truly no table to discover schema + // against (e.g. a metric source with no populated metric tables). + if (!discoveryTableName) { return { content: [ { @@ -102,7 +326,7 @@ async function describeSourceSchema( { source: meta, nextSteps: { - query: `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}" and the metric tables above.`, + query: `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}".`, }, }, null, @@ -136,7 +360,8 @@ async function describeSourceSchema( password: connection.password, }); const metadata = getMetadata(clickhouseClient); - const { databaseName, tableName } = source.from; + const databaseName = source.from.databaseName; + const tableName = discoveryTableName; const connectionId = source.connection.toString(); // Track which sampling stages were skipped due to timeout @@ -169,6 +394,12 @@ async function describeSourceSchema( })); // ── 2. Map attribute keys ───────────────────────────────────────────── + // timestampValueExpression is threaded into getMapKeys / getAllKeyValues / + // sampleMetricNamesForKind below so the no-rollup fallback path (i.e. + // metric sources, which don't have metadataMaterializedViews configured) + // can scope its scan to dateRange instead of going unbounded against + // the raw metric table on cold cache. + const timestampValueExpression = source.timestampValueExpression; const mapColumns = filterColumnMetaByType(columns, [JSDataType.Map]); const mapKeysResults: Record = {}; @@ -184,6 +415,7 @@ async function describeSourceSchema( connectionId, metadataMVs, dateRange, + timestampValueExpression, signal, }); mapKeysResults[col.name] = keys; @@ -225,6 +457,7 @@ async function describeSourceSchema( connectionId, metadataMVs, dateRange, + timestampValueExpression, signal, }); for (const { key, value } of results) { @@ -264,6 +497,7 @@ async function describeSourceSchema( connectionId, metadataMVs, dateRange, + timestampValueExpression, signal, }); for (const { key, value } of results) { @@ -286,6 +520,55 @@ async function describeSourceSchema( skippedStages.push('mapAttributeValues'); } + // ── 5. Metric name + unit + description sampling ────────────────────── + // For metric sources, sample distinct MetricName values per queryable + // kind so the agent has a starter list without needing a follow-up call + // to clickstack_list_metrics for the common case (<= 20 metrics/kind). + // Defensively check for MetricUnit / MetricDescription columns: they + // exist on the standard OTel Collector schema but a custom metric table + // may not declare them. + if (source.kind === SourceKind.Metric && !signal.aborted) { + const metricNames: Record = {}; + await Promise.all( + QUERYABLE_METRIC_KINDS.map(async kind => { + const kindTableName = source.metricTables[kind]; + if (!kindTableName) return; + try { + const samples = await sampleMetricNamesForKind({ + metadata, + clickhouseClient, + databaseName, + tableName: kindTableName, + connectionId, + dateRange, + timestampValueExpression, + signal, + // Reuse representative columns when the kind matches the + // representative table to avoid a second getColumns round-trip. + cachedColumns: + representativeMetric?.tableName === kindTableName + ? columns + : undefined, + }); + if (samples.length > 0) { + metricNames[kind] = samples; + } + } catch (e) { + logger.warn( + { sourceId, kind, error: e }, + 'Failed to sample metric names for kind', + ); + } + }), + ); + if (signal.aborted && Object.keys(metricNames).length === 0) { + skippedStages.push('metricNames'); + } + if (Object.keys(metricNames).length > 0) { + meta.metricNames = metricNames; + } + } + // Flag partial results so the LLM knows value samples may be incomplete if (skippedStages.length > 0) { meta.partial = true; @@ -299,6 +582,14 @@ async function describeSourceSchema( : 'These are the REAL values in your data — use them in filters instead of guessing. ' + 'Example: where: "SeverityText:error" (if \'error\' appears in the sampled values above).'; + const isMetricSource = source.kind === SourceKind.Metric; + const queryNextStep = isMetricSource + ? `Use clickstack_timeseries or clickstack_table with sourceId "${sourceId}" and metricType/metricName from above.` + : `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}" and the columns/attributes above.`; + const discoveryNextStep = isMetricSource + ? `For more metric names than the sample above, call clickstack_list_metrics with sourceId "${sourceId}". For per-metric attribute keys + sampled values, call clickstack_describe_metric with sourceId and metricName.` + : undefined; + const { data: output, isTrimmed } = trimToolResponse({ source: meta, usage: { @@ -307,11 +598,17 @@ async function describeSourceSchema( mapAttributes: "Use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']", lowCardinalityValues: lcValuesHint, + ...(isMetricSource && { + metricNames: + 'Each entry maps a metric kind (gauge/sum/histogram) to a sample of metric names ' + + 'available on that table. Pass metricType + metricName on each select item.', + }), }, nextSteps: { - query: `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}" and the columns/attributes above.`, + query: queryNextStep, mapAttributeAccess: "Use bracket syntax for map columns: ResourceAttributes['service.name'], SpanAttributes['http.method']", + ...(discoveryNextStep && { discovery: discoveryNextStep }), }, }); @@ -371,9 +668,14 @@ export function registerDescribeSource( const controller = new AbortController(); // Promise.race enforces wall-clock timeout regardless of whether - // internal ClickHouse calls honour the AbortSignal. + // internal ClickHouse calls honour the AbortSignal. Hoist the + // timer handle so the finally block can cancel it on the success + // path — otherwise a stale controller.abort() fires + // DESCRIBE_TIMEOUT_MS after every successful call and the + // setTimeout closure stays pinned for the same duration. + let timeoutId: ReturnType | undefined; const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => { + timeoutId = setTimeout(() => { controller.abort(); reject(new Error('DESCRIBE_TIMEOUT')); }, DESCRIBE_TIMEOUT_MS); @@ -411,6 +713,8 @@ export function registerDescribeSource( 'Failed to describe source schema', ); throw e; + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); } }, ), diff --git a/packages/api/src/mcp/tools/sources/index.ts b/packages/api/src/mcp/tools/sources/index.ts index ae4255dd52..0efefcd786 100644 --- a/packages/api/src/mcp/tools/sources/index.ts +++ b/packages/api/src/mcp/tools/sources/index.ts @@ -2,7 +2,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { McpContext, ToolDefinition } from '@/mcp/tools/types'; +import { registerDescribeMetric } from './describeMetric'; import { registerDescribeSource } from './describeSource'; +import { registerListMetrics } from './listMetrics'; import { registerListSources } from './listSources'; const sourcesTools: ToolDefinition = ( @@ -11,6 +13,8 @@ const sourcesTools: ToolDefinition = ( ) => { registerListSources(server, context); registerDescribeSource(server, context); + registerListMetrics(server, context); + registerDescribeMetric(server, context); }; export default sourcesTools; diff --git a/packages/api/src/mcp/tools/sources/listMetrics.ts b/packages/api/src/mcp/tools/sources/listMetrics.ts new file mode 100644 index 0000000000..bd59133e85 --- /dev/null +++ b/packages/api/src/mcp/tools/sources/listMetrics.ts @@ -0,0 +1,504 @@ +import { + chSql, + concatChSql, + tableExpr, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; +import type { McpContext } from '@/mcp/tools/types'; +import { withToolTracing } from '@/mcp/utils/tracing'; +import logger from '@/utils/logger'; + +import { + QUERYABLE_METRIC_KINDS, + type QueryableMetricKind, +} from './metricKinds'; + +const DEFAULT_LIMIT = 50; +const MAX_LIMIT = 500; +const DEFAULT_LOOKBACK_MS = 24 * 60 * 60 * 1000; + +// Hard timeout for the entire list operation (ms). +const LIST_TIMEOUT_MS = 10_000; + +// Server-side ClickHouse execution cap per query. Matches the bounds +// used by describeMetric's fetchAttributeKeys / sampleAttributeValues. +const MAX_EXEC_SECONDS = 8; + +// ─── Cursor ────────────────────────────────────────────────────────────────── + +export type ListMetricsCursorPayload = { + kind: QueryableMetricKind; + lastName: string; +}; + +/** @internal Exported for testing. */ +export function encodeCursor(payload: ListMetricsCursorPayload): string { + return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64'); +} + +/** @internal Exported for testing. */ +export function decodeCursor(raw: string): ListMetricsCursorPayload | null { + try { + const decoded = Buffer.from(raw, 'base64').toString('utf8'); + const parsed = JSON.parse(decoded); + if ( + typeof parsed === 'object' && + parsed !== null && + typeof parsed.kind === 'string' && + typeof parsed.lastName === 'string' && + (QUERYABLE_METRIC_KINDS as readonly string[]).includes(parsed.kind) + ) { + return { + kind: parsed.kind as QueryableMetricKind, + lastName: parsed.lastName, + }; + } + } catch { + // fall through + } + return null; +} + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const listMetricsSchema = z.object({ + sourceId: z + .string() + .describe( + 'Source ID. Must reference a metric source — get IDs from clickstack_list_sources.', + ), + kind: z + .enum(QUERYABLE_METRIC_KINDS) + .optional() + .describe( + 'Optional metric kind filter. Omit to scan every populated kind on the source ' + + '(gauge → sum → histogram). Set to narrow results to one kind.', + ), + namePattern: z + .string() + .optional() + .describe( + 'Optional ClickHouse ILIKE pattern applied to MetricName server-side. ' + + 'Use % as the wildcard. Examples: "system.cpu.%", "%duration%", "http.server.%".', + ), + startTime: z + .string() + .optional() + .describe( + 'Restrict to metrics with data points after this ISO 8601 timestamp. ' + + 'Default: 24 hours before endTime (or now).', + ), + endTime: z + .string() + .optional() + .describe('End of the time window as ISO 8601. Default: now.'), + limit: z + .number() + .int() + .min(1) + .max(MAX_LIMIT) + .optional() + .default(DEFAULT_LIMIT) + .describe( + `Max metrics returned per page. Default ${DEFAULT_LIMIT}, max ${MAX_LIMIT}.`, + ), + cursor: z + .string() + .optional() + .describe( + 'Opaque pagination cursor returned by a previous call as `nextCursor`. ' + + 'Pass it back unchanged to get the next page.', + ), +}); + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +type MetricEntry = { + name: string; + kind: QueryableMetricKind; + unit?: string; + description?: string; +}; + +function parseTimeRange( + startTime?: string, + endTime?: string, +): { error: string } | { startDate: Date; endDate: Date } { + const endDate = endTime ? new Date(endTime) : new Date(); + const startDate = startTime + ? new Date(startTime) + : new Date(endDate.getTime() - DEFAULT_LOOKBACK_MS); + if (isNaN(endDate.getTime()) || isNaN(startDate.getTime())) { + return { + error: 'Invalid startTime or endTime: must be valid ISO 8601 strings', + }; + } + if (startDate >= endDate) { + return { error: 'endTime must be greater than startTime' }; + } + return { startDate, endDate }; +} + +async function fetchMetricsForKind({ + clickhouseClient, + metadata, + kind, + databaseName, + tableName, + connectionId, + startDate, + endDate, + namePattern, + afterName, + limit, + signal, +}: { + clickhouseClient: ClickhouseClient; + metadata: ReturnType; + kind: QueryableMetricKind; + databaseName: string; + tableName: string; + connectionId: string; + startDate: Date; + endDate: Date; + namePattern: string | undefined; + afterName: string | undefined; + limit: number; + signal: AbortSignal; +}): Promise { + // Defensive column-presence check so we don't reference MetricUnit / + // MetricDescription on non-OTel-default schemas. + const columns = await metadata.getColumns({ + databaseName, + tableName, + connectionId, + }); + const columnNames = new Set(columns.map(c => c.name)); + const hasUnit = columnNames.has('MetricUnit'); + const hasDescription = columnNames.has('MetricDescription'); + + const projections = [ + chSql`MetricName`, + ...(hasUnit + ? [chSql`anyLast(${{ Identifier: 'MetricUnit' }}) AS MetricUnit`] + : []), + ...(hasDescription + ? [ + chSql`anyLast(${{ Identifier: 'MetricDescription' }}) AS MetricDescription`, + ] + : []), + ]; + + const whereParts = [ + chSql`TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: startDate.getTime() }})`, + chSql`TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: endDate.getTime() }})`, + ...(afterName !== undefined + ? [chSql`MetricName > ${{ String: afterName }}`] + : []), + ...(namePattern + ? [chSql`MetricName ILIKE ${{ String: namePattern }}`] + : []), + ]; + + const sql = chSql` + SELECT ${concatChSql(', ', projections)} + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE ${concatChSql(' AND ', whereParts)} + GROUP BY MetricName + ORDER BY MetricName ASC + LIMIT ${{ Int32: limit }} + `; + + type Row = { + MetricName: string; + MetricUnit?: string; + MetricDescription?: string; + }; + + const response = await clickhouseClient.query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + connectionId, + clickhouse_settings: { + max_execution_time: MAX_EXEC_SECONDS, + timeout_overflow_mode: 'break', + }, + abort_signal: signal, + }); + const result = (await response.json()) as { data: Row[] }; + return result.data.map(row => { + const entry: MetricEntry = { name: row.MetricName, kind }; + if (row.MetricUnit) entry.unit = row.MetricUnit; + if (row.MetricDescription) entry.description = row.MetricDescription; + return entry; + }); +} + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerListMetrics( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'clickstack_list_metrics', + { + title: 'List Metric Names', + description: + 'DISCOVERY: Use this after clickstack_describe_source when you need more metric ' + + 'names than the per-kind sample shows, or when you want to narrow by ' + + 'kind / name pattern / time window. ' + + 'Returns paginated metric names per kind (gauge/sum/histogram) ' + + 'with optional unit and description (when the OTel-default columns are present). ' + + 'Pass the returned `nextCursor` back unchanged to fetch the next page.\n\n' + + 'Workflow: clickstack_list_sources → clickstack_describe_source → ' + + 'clickstack_list_metrics → clickstack_describe_metric → ' + + 'clickstack_timeseries|clickstack_table.', + inputSchema: listMetricsSchema, + }, + withToolTracing('clickstack_list_metrics', context, async rawInput => { + // Re-parse explicitly: the MCP SDK callback signature widens + // optional-field types into `unknown`, but the parser produces + // the typed shape we need for downstream calls. + const input: z.infer = + listMetricsSchema.parse(rawInput); + + const controller = new AbortController(); + let timeoutId: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + controller.abort(); + reject(new Error('LIST_METRICS_TIMEOUT')); + }, LIST_TIMEOUT_MS); + }); + + try { + return await Promise.race([ + listMetricsImpl(teamId.toString(), input, controller.signal), + timeoutPromise, + ]); + } catch (e) { + if (e instanceof Error && e.message === 'LIST_METRICS_TIMEOUT') { + logger.warn( + { teamId, sourceId: input.sourceId }, + 'clickstack_list_metrics timed out', + ); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'clickstack_list_metrics timed out. Try narrowing the time window ' + + '(startTime/endTime), pinning a single `kind`, or adding a namePattern filter.', + }, + ], + }; + } + throw e; + } finally { + clearTimeout(timeoutId); + } + }), + ); +} + +async function listMetricsImpl( + teamId: string, + input: z.infer, + signal: AbortSignal, +) { + const source = await getSource(teamId, input.sourceId); + if (!source) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Source "${input.sourceId}" not found. Call clickstack_list_sources to see available source IDs.`, + }, + ], + }; + } + if (source.kind !== SourceKind.Metric) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Source "${input.sourceId}" is a "${source.kind}" source, not a metric source. clickstack_list_metrics only works on metric sources — call clickstack_list_sources to find one whose kind is "metric".`, + }, + ], + }; + } + + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + // Decode cursor; reject silently and start over if malformed so a + // truncated or tampered cursor does not surface internals. + const cursor = input.cursor ? decodeCursor(input.cursor) : null; + if (input.cursor && !cursor) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: 'Invalid cursor. Omit cursor to start over, or pass the exact `nextCursor` value returned by a previous call.', + }, + ], + }; + } + + // Resolve which kinds to scan, in order. When a cursor is set, + // skip kinds before the cursor's kind (already returned) and start + // the cursor's kind at the lastName-exclusive position. + const requestedKinds: QueryableMetricKind[] = input.kind + ? [input.kind] + : QUERYABLE_METRIC_KINDS.filter(k => Boolean(source.metricTables[k])); + const startKindIdx = cursor ? requestedKinds.indexOf(cursor.kind) : 0; + if (startKindIdx < 0) { + // Cursor points at a kind that's not in scope for this call — + // safer to error than silently skip. + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Cursor references kind "${cursor!.kind}" but that kind is not in scope for this call. Drop the kind filter or pass a matching cursor.`, + }, + ], + }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Connection not found for source "${input.sourceId}".`, + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + + const limit = input.limit ?? DEFAULT_LIMIT; + const databaseName = source.from.databaseName; + + const metrics: MetricEntry[] = []; + // Per-kind fetch failures, surfaced on the response so the agent + // can distinguish "kind genuinely has no metrics" from "the fetch + // for that kind failed" — the two need different recovery steps. + const partialFailure: { kind: string; error: string }[] = []; + let nextCursor: string | undefined; + for (let i = startKindIdx; i < requestedKinds.length; i++) { + const kind = requestedKinds[i]; + const tableName = source.metricTables[kind]; + if (!tableName) continue; + const afterName = + cursor && cursor.kind === kind && i === startKindIdx + ? cursor.lastName + : undefined; + const remaining = limit - metrics.length; + if (remaining <= 0) break; + // Fetch one extra row so we can detect more-data-available. + let kindMetrics: MetricEntry[]; + try { + kindMetrics = await fetchMetricsForKind({ + clickhouseClient, + metadata, + kind, + databaseName, + tableName, + connectionId: source.connection.toString(), + startDate, + endDate, + namePattern: input.namePattern, + afterName, + limit: remaining + 1, + signal, + }); + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + logger.warn( + { sourceId: input.sourceId, kind, error: message }, + 'Failed to list metrics for kind', + ); + partialFailure.push({ + kind, + error: message.replace(/\s+/g, ' ').trim().slice(0, 200), + }); + continue; + } + if (kindMetrics.length > remaining) { + // We hit the cap for this kind; emit cursor pointing at the + // last returned name and stop iterating further kinds. + const truncated = kindMetrics.slice(0, remaining); + metrics.push(...truncated); + nextCursor = encodeCursor({ + kind, + lastName: truncated[truncated.length - 1].name, + }); + break; + } + metrics.push(...kindMetrics); + } + + const responseObj: Record = { + metrics, + ...(nextCursor && { nextCursor }), + ...(partialFailure.length > 0 && { + partialFailure, + hint: + 'Listing failed for some metric kinds — results may be incomplete. ' + + 'Retry the call; if the failure persists, narrow startTime/endTime or pin a single `kind`.', + }), + ...(metrics.length === 0 && + partialFailure.length === 0 && { + hint: + 'No metrics matched. Try widening the time window (startTime/endTime), ' + + 'removing the namePattern filter, or omitting `kind` to scan every populated metric table.', + }), + usage: + 'Pass `metricType` + `metricName` from each entry to clickstack_timeseries / clickstack_table. ' + + 'For per-metric attribute keys and sampled values, call clickstack_describe_metric.', + }; + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify(responseObj, null, 2), + }, + ], + }; +} diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index ffa4cd4730..f394c28c76 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -7,6 +7,8 @@ import { getSources } from '@/controllers/sources'; import type { McpContext } from '@/mcp/tools/types'; import { withToolTracing } from '@/mcp/utils/tracing'; +import { sanitizeMetricTables } from './metricKinds'; + export function registerListSources( server: McpServer, context: McpContext, @@ -71,7 +73,13 @@ export function registerListSources( traceId: s.traceIdExpression, }; } else if (s.kind === SourceKind.Metric) { - meta.metricTables = s.metricTables; + // Filter out implementation-detail keys (e.g. a stray Mongoose + // `_id` on the metricTables subdoc) so the agent only sees + // valid metric kinds. + const tables = sanitizeMetricTables( + s.metricTables as Record | undefined, + ); + if (tables) meta.metricTables = tables; } return meta; diff --git a/packages/api/src/mcp/tools/sources/metricKinds.ts b/packages/api/src/mcp/tools/sources/metricKinds.ts new file mode 100644 index 0000000000..bdaac920ba --- /dev/null +++ b/packages/api/src/mcp/tools/sources/metricKinds.ts @@ -0,0 +1,75 @@ +import { MetricsDataType } from '@hyperdx/common-utils/dist/types'; + +/** + * Metric kinds the query renderer can translate today. Mirrors the three + * MetricsDataType members that `translateMetricChartConfig` in + * common-utils branches over; `summary` and `"exponential histogram"` + * are intentionally excluded because the renderer throws on them. + * + * Declared as plain string literals (not MetricsDataType enum members) + * so `z.enum(...)` narrows correctly at the MCP SDK callback boundary — + * referencing the enum here pessimises Zod's inference to `unknown` on + * every optional field of the surrounding tool input schema, which then + * cascades into "not assignable to type string" errors at every handler + * call site. The compile-time assertion below guarantees the literals + * stay in sync with the enum so adding a new queryable kind cannot be + * done in only one place. + * + * Imported by every metric-aware MCP tool (clickstack_describe_source, + * clickstack_list_metrics, clickstack_describe_metric) and by the + * clickstack_timeseries / clickstack_table select-item schema. + */ +export const QUERYABLE_METRIC_KINDS = ['gauge', 'sum', 'histogram'] as const; + +export type QueryableMetricKind = (typeof QUERYABLE_METRIC_KINDS)[number]; + +// Compile-time assertion that the string literals above still match the +// MetricsDataType enum members. If a future MetricsDataType rename +// breaks this list, this line fails to type-check. +const _assertKindsMatchEnum: readonly QueryableMetricKind[] = [ + MetricsDataType.Gauge, + MetricsDataType.Sum, + MetricsDataType.Histogram, +]; +void _assertKindsMatchEnum; + +/** + * Allowed kind keys on the `metricTables` map when serialising a metric + * source into an MCP response. Includes the non-queryable kinds + * (`summary`, `exponential histogram`) because the model schema declares + * tables for them even though the query renderer cannot yet translate + * them. + */ +const ALLOWED_METRIC_TABLE_KINDS: readonly string[] = + Object.values(MetricsDataType); + +/** + * Filter source.metricTables to the known kind keys before emitting it + * in a clickstack_list_sources / clickstack_describe_source response. + * + * Belt-and-braces defense: even after the model schema declares + * `_id: false` on the metricTables subdoc, existing documents persisted + * before the schema fix may still carry a stray `_id` value that + * Mongoose serialises alongside the legitimate kind keys. Filtering at + * the response boundary keeps the agent-facing payload free of + * implementation-detail keys. + * + * Accepts both plain objects and Mongoose subdocuments — we look up + * each allowed kind via property access rather than enumerating keys, + * because Mongoose subdoc instances expose field values through getters + * (not as own enumerable properties). + */ +export function sanitizeMetricTables( + metricTables: Record | undefined | null, +): Record | undefined { + if (!metricTables) return undefined; + const source = metricTables as Record; + const out: Record = {}; + for (const kind of ALLOWED_METRIC_TABLE_KINDS) { + const value = source[kind]; + if (typeof value === 'string' && value.length > 0) { + out[kind] = value; + } + } + return Object.keys(out).length > 0 ? out : undefined; +} diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index b1c63e0dc3..5840505ee1 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -240,18 +240,27 @@ export const SessionSource = Source.discriminator( // -------------------------- // Metric discriminator // -------------------------- +// metricTables is declared as a nested Schema with `_id: false` so the +// embedded subdoc does not auto-generate an ObjectId. Without this opt-out +// Mongoose adds an `_id` field that leaks into MCP responses alongside +// the queryable kind keys (gauge/sum/histogram/...). +const MetricTablesSchema = new Schema( + { + [MetricsDataType.Gauge]: String, + [MetricsDataType.Histogram]: String, + [MetricsDataType.Sum]: String, + [MetricsDataType.Summary]: String, + [MetricsDataType.ExponentialHistogram]: String, + }, + { _id: false }, +); + type IMetricSource = Extract; export const MetricSource = Source.discriminator( SourceKind.Metric, new Schema>({ metricTables: { - type: { - [MetricsDataType.Gauge]: String, - [MetricsDataType.Histogram]: String, - [MetricsDataType.Sum]: String, - [MetricsDataType.Summary]: String, - [MetricsDataType.ExponentialHistogram]: String, - }, + type: MetricTablesSchema, default: undefined, }, resourceAttributesExpression: String, From 5e19a2b42704901ee6836ff6e5ef740a63dbaecb Mon Sep 17 00:00:00 2001 From: Vineet Ahirkar Date: Tue, 30 Jun 2026 15:27:12 +0530 Subject: [PATCH 120/728] Show elapsed-time and generated-SQL for search timeline view (#2384) Co-authored-by: Karl Power Co-authored-by: Karl Power <85935352+karl-power@users.noreply.github.com> --- .changeset/early-banks-retire.md | 5 + packages/app/src/DBSearchPage.tsx | 181 +++++- .../src/__tests__/useSearchTelemetry.test.tsx | 558 ++++++++++++++++++ .../app/src/components/ChartSQLPreview.tsx | 16 +- packages/app/src/hooks/useChartConfig.tsx | 2 +- packages/app/src/hooks/useExplainQuery.tsx | 6 +- 6 files changed, 751 insertions(+), 17 deletions(-) create mode 100644 .changeset/early-banks-retire.md create mode 100644 packages/app/src/__tests__/useSearchTelemetry.test.tsx diff --git a/.changeset/early-banks-retire.md b/.changeset/early-banks-retire.md new file mode 100644 index 0000000000..735f033102 --- /dev/null +++ b/.changeset/early-banks-retire.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +Show elapsed time and Generated SQL for search timeline view diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 3aea34d706..6b1c0b1542 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -25,6 +25,7 @@ import { import { useForm, useWatch } from 'react-hook-form'; import { z } from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; +import HyperDX from '@hyperdx/browser'; import { ClickHouseQueryError, ColumnMeta, @@ -72,13 +73,14 @@ import { notifications } from '@mantine/notifications'; import { IconArrowBarToRight, IconBolt, + IconCode, IconPlayerPlay, IconPlus, IconStack2, IconTags, IconX, } from '@tabler/icons-react'; -import { useIsFetching } from '@tanstack/react-query'; +import { keepPreviousData, useIsFetching } from '@tanstack/react-query'; import { SortingState } from '@tanstack/react-table'; import CodeMirror from '@uiw/react-codemirror'; @@ -120,9 +122,14 @@ import { parseTimeQuery, useNewTimeQuery, } from '@/timeQuery'; -import { QUERY_LOCAL_STORAGE, useLocalStorage, usePrevious } from '@/utils'; +import { + formatDurationMs, + QUERY_LOCAL_STORAGE, + useLocalStorage, + usePrevious, +} from '@/utils'; -import { SQLPreview } from './components/ChartSQLPreview'; +import ChartSQLPreview, { SQLPreview } from './components/ChartSQLPreview'; import DBSqlRowTableWithSideBar from './components/DBSqlRowTableWithSidebar'; import PatternTable from './components/PatternTable'; import { DBSearchHeatmapChart } from './components/Search/DBSearchHeatmapChart'; @@ -330,30 +337,93 @@ function SearchResultsCountGroup({ ); } -function SearchNumRows({ +export function SearchNumRows({ config, + sqlConfig, enabled, + searchElapsedMs, + isSearching, + isLiveTail = false, }: { config: ChartConfigWithDateRange; + sqlConfig?: ChartConfigWithDateRange; enabled: boolean; + searchElapsedMs: number | null; + isSearching: boolean; + isLiveTail?: boolean; }) { + const [statsOpened, { open: openStats, close: closeStats }] = + useDisclosure(false); const { data, isLoading, error } = useExplainQuery(config, { enabled, + // Keep the previous row count on screen while a new EXPLAIN runs so the + // "Scanned Rows" value doesn't flash a loading state on every live-tail + // poll (each poll changes the dateRange, and thus the query key). + placeholderData: keepPreviousData, }); if (!enabled) { return null; } - const numRows = data?.[0]?.rows; + const explainRow = data?.[0]; + const numRows = explainRow?.rows; + const hasData = !isLoading && !error && numRows != null; + + // During live tail we keep showing the last measured elapsed time and never + // flash the "..." loading state, so the value doesn't flicker between polls. + const showElapsedLoading = isSearching && !isLiveTail; + const showElapsed = showElapsedLoading || searchElapsedMs != null; + return ( - - {isLoading - ? 'Scanned Rows ...' - : error || !numRows - ? '' - : `Scanned Rows: ${Number.parseInt(numRows)?.toLocaleString()}`} - + <> + + + + + + {isLoading + ? 'Scanned Rows ...' + : error || numRows == null + ? '' + : `Scanned Rows: ${Number(numRows).toLocaleString()}`} + + {showElapsed && ( + <> + {(hasData || isLoading) && ( + + | + + )} + + {showElapsedLoading + ? 'Elapsed Time: ...' + : `Elapsed Time: ${formatDurationMs(searchElapsedMs!)}`} + + + )} + {/* The generated-SQL preview is derived purely from config, not the + explain query, so it renders unconditionally. Gating it on explain + loading/data would make it flicker on every live-tail poll, since + each poll changes the dateRange (and thus the explain queryKey). */} + + + + + + + ); } @@ -810,6 +880,79 @@ const queryStateMap = { orderBy: parseAsStringEncoded, }; +export function useSearchTelemetry({ + isAnyQueryFetching, + isLive, + sourceId, +}: { + isAnyQueryFetching: boolean; + /** When true the hook suppresses recording and emission so live-tail + * background refetches do not flood the metric. */ + isLive: boolean; + sourceId: string | null; +}) { + const searchStartTimeRef = useRef(null); + const wasFetchingRef = useRef(false); + // Whether the in-flight cycle began as a live-tail refresh, captured on the + // rising edge so a mid-cycle isLive flip can't change how it's treated. + const cycleIsLiveRef = useRef(false); + + // Snapshot latency_ms and source_id together so a later sourceId change does + // not cause the emission effect to re-fire with stale latency data. `emit` + // records whether this cycle should be reported to telemetry (user-initiated + // searches only); latency is still surfaced for display in every case. + const [completedSearch, setCompletedSearch] = useState<{ + latency_ms: number; + source_id: string; + emit: boolean; + } | null>(null); + + useEffect(() => { + if (isAnyQueryFetching) { + // Start the timer once per fetch cycle (for live tail too — we display + // its elapsed time, we just don't emit telemetry for it). + if (!wasFetchingRef.current) { + searchStartTimeRef.current = performance.now(); + cycleIsLiveRef.current = isLive; + // Only blank the displayed timer for user-initiated searches. During + // live tail we keep the previous value so it doesn't flicker between + // background refreshes. + if (!isLive) { + setCompletedSearch(null); + } + } + wasFetchingRef.current = true; + } else { + if (searchStartTimeRef.current != null) { + setCompletedSearch({ + latency_ms: Math.round( + performance.now() - searchStartTimeRef.current, + ), + source_id: sourceId ?? '', + emit: !cycleIsLiveRef.current, + }); + searchStartTimeRef.current = null; + } + wasFetchingRef.current = false; + } + }, [isAnyQueryFetching, isLive, sourceId]); + + // completedSearch is the only dep here — sourceId was snapshotted at + // completion time so changing source after a finished search does not + // re-emit the previous run's latency against the new source. Live-tail + // cycles are recorded for display but flagged emit=false so they never flood + // telemetry. + useEffect(() => { + if (completedSearch == null || !completedSearch.emit) return; + HyperDX.addAction('search executed', { + latency_ms: completedSearch.latency_ms, + source_id: completedSearch.source_id, + }); + }, [completedSearch]); + + return { searchElapsedMs: completedSearch?.latency_ms ?? null }; +} + export function DBSearchPage() { const brandName = useBrandDisplayName(); // Next router is laggy behind window.location, which causes race @@ -1339,6 +1482,12 @@ export function DBSearchPage() { queryKey: [QUERY_KEY_PREFIX], }) > 0; + const { searchElapsedMs } = useSearchTelemetry({ + isAnyQueryFetching, + isLive: isLive ?? false, + sourceId: chartConfig?.source ?? null, + }); + const isTabVisible = useDocumentVisibility(); // State for collapsing all expanded rows when resuming live tail @@ -2193,7 +2342,11 @@ export function DBSearchPage() { ...chartConfig, dateRange: searchedTimeRange, }} + sqlConfig={histogramTimeChartConfig ?? undefined} enabled={isReady} + searchElapsedMs={searchElapsedMs} + isSearching={isAnyQueryFetching} + isLiveTail={isLive ?? false} /> @@ -2287,7 +2440,11 @@ export function DBSearchPage() { ...chartConfig, dateRange: searchedTimeRange, }} + sqlConfig={histogramTimeChartConfig ?? undefined} enabled={isReady} + searchElapsedMs={searchElapsedMs} + isSearching={isAnyQueryFetching} + isLiveTail={isLive ?? false} /> diff --git a/packages/app/src/__tests__/useSearchTelemetry.test.tsx b/packages/app/src/__tests__/useSearchTelemetry.test.tsx new file mode 100644 index 0000000000..15892b5b67 --- /dev/null +++ b/packages/app/src/__tests__/useSearchTelemetry.test.tsx @@ -0,0 +1,558 @@ +import React from 'react'; +import { act, fireEvent, renderHook, screen } from '@testing-library/react'; + +import { SearchNumRows, useSearchTelemetry } from '@/DBSearchPage'; + +jest.mock('@/layout', () => ({ + withAppNav: (component: unknown) => component, +})); + +const mockAddAction = jest.fn(); +jest.mock('@hyperdx/browser', () => ({ + __esModule: true, + default: { addAction: (...args: unknown[]) => mockAddAction(...args) }, +})); + +let mockExplainData: unknown[] | undefined = undefined; +let mockExplainIsLoading = false; +let mockExplainError: Error | null = null; + +jest.mock('@/hooks/useExplainQuery', () => ({ + useExplainQuery: () => ({ + data: mockExplainData, + isLoading: mockExplainIsLoading, + error: mockExplainError, + }), +})); + +// Capture the last rendered config so we can assert on it without fighting portals +let lastSQLPreviewConfig: unknown = undefined; +jest.mock('../components/ChartSQLPreview', () => ({ + __esModule: true, + default: (props: { config: unknown }) => { + lastSQLPreviewConfig = props.config; + return
; + }, + SQLPreview: () =>
, +})); + +// Render Mantine Modal content inline (no portal / no transition) so jsdom can see it +jest.mock('@mantine/core', () => { + const actual = jest.requireActual('@mantine/core'); + return { + ...actual, + Modal: ({ opened, onClose, title, children }: any) => + opened ? ( +
+ {title} + + {children} +
+ ) : null, + }; +}); + +// --------------------------------------------------------------------------- +// useSearchTelemetry +// --------------------------------------------------------------------------- + +describe('useSearchTelemetry', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('emits "search executed" action with latency_ms and source_id when a search completes', async () => { + const { result, rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: false, + sourceId: 'my-source', + }, + }, + ); + + await act(async () => { + rerender({ + isAnyQueryFetching: false, + isLive: false, + sourceId: 'my-source', + }); + }); + + expect(result.current.searchElapsedMs).toBeGreaterThanOrEqual(0); + expect(mockAddAction).toHaveBeenCalledTimes(1); + expect(mockAddAction).toHaveBeenCalledWith('search executed', { + latency_ms: expect.any(Number), + source_id: 'my-source', + }); + }); + + it('does NOT re-emit when sourceId changes after a completed search (P0 fix)', async () => { + const { rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: false, + sourceId: 'src-a', + }, + }, + ); + + // Complete the search + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: false, sourceId: 'src-a' }); + }); + + expect(mockAddAction).toHaveBeenCalledTimes(1); + expect(mockAddAction).toHaveBeenCalledWith('search executed', { + latency_ms: expect.any(Number), + source_id: 'src-a', + }); + + // Change sourceId after search is done — must NOT fire again + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: false, sourceId: 'src-b' }); + }); + + expect(mockAddAction).toHaveBeenCalledTimes(1); // still just 1 + }); + + it('does NOT emit during live-tail ticks (P0 fix)', async () => { + const { rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: true, + sourceId: 'src', + }, + }, + ); + + // Live-tail cycle completes + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: true, sourceId: 'src' }); + }); + + expect(mockAddAction).not.toHaveBeenCalled(); + }); + + it('measures elapsed time during live tail for display but does NOT emit', async () => { + const { result, rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: true, + sourceId: 'src', + }, + }, + ); + + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: true, sourceId: 'src' }); + }); + + // Latency is surfaced for display ... + expect(result.current.searchElapsedMs).toBeGreaterThanOrEqual(0); + // ... but live-tail cycles are never reported to telemetry. + expect(mockAddAction).not.toHaveBeenCalled(); + }); + + it('keeps the previous elapsed value when a live-tail poll starts (no blink)', async () => { + const { result, rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: true, + sourceId: 'src', + }, + }, + ); + + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: true, sourceId: 'src' }); + }); + const firstElapsed = result.current.searchElapsedMs; + expect(firstElapsed).toBeGreaterThanOrEqual(0); + + // Next poll begins — the displayed value must NOT reset to null, otherwise + // the timer would flicker between live-tail refreshes. + await act(async () => { + rerender({ isAnyQueryFetching: true, isLive: true, sourceId: 'src' }); + }); + expect(result.current.searchElapsedMs).toBe(firstElapsed); + }); + + it('does NOT re-stamp the start clock if isAnyQueryFetching is already true (P2 fix)', async () => { + const { result, rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: false, + sourceId: 'src', + }, + }, + ); + + // Queries temporarily dip to 0 then back up before true completion + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: false, sourceId: 'src' }); + }); + const firstElapsed = result.current.searchElapsedMs; + + await act(async () => { + rerender({ isAnyQueryFetching: true, isLive: false, sourceId: 'src' }); + }); + // completedSearch is reset to null while re-fetching + expect(result.current.searchElapsedMs).toBeNull(); + + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: false, sourceId: 'src' }); + }); + + // Two emits (one per false transition) — the clock was re-anchored + // only on the first genuine false→true, so elapsed on both are valid numbers + expect(mockAddAction).toHaveBeenCalledTimes(2); + expect(firstElapsed).toBeGreaterThanOrEqual(0); + expect(result.current.searchElapsedMs).toBeGreaterThanOrEqual(0); + }); + + it('resets elapsed time and does not emit when a new search starts', async () => { + const { result, rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: false, + isLive: false, + sourceId: 'src', + }, + }, + ); + + await act(async () => { + rerender({ isAnyQueryFetching: true, isLive: false, sourceId: 'src' }); + }); + + expect(result.current.searchElapsedMs).toBeNull(); + expect(mockAddAction).not.toHaveBeenCalled(); + }); + + it('falls back to empty string source_id when sourceId is null', async () => { + const { rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: true, + isLive: false, + sourceId: null as string | null, + }, + }, + ); + + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: false, sourceId: null }); + }); + + expect(mockAddAction).toHaveBeenCalledWith('search executed', { + latency_ms: expect.any(Number), + source_id: '', + }); + }); + + it('does not emit when fetching stops without a prior start', async () => { + const { rerender } = renderHook( + ({ isAnyQueryFetching, isLive, sourceId }) => + useSearchTelemetry({ isAnyQueryFetching, isLive, sourceId }), + { + initialProps: { + isAnyQueryFetching: false, + isLive: false, + sourceId: 'src', + }, + }, + ); + + await act(async () => { + rerender({ isAnyQueryFetching: false, isLive: false, sourceId: 'src' }); + }); + + expect(mockAddAction).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// SearchNumRows +// --------------------------------------------------------------------------- + +const baseConfig = { + source: 'test-source', + dateRange: { from: new Date('2024-01-01'), to: new Date('2024-01-02') }, +} as any; + +describe('SearchNumRows', () => { + beforeEach(() => { + mockExplainData = undefined; + mockExplainIsLoading = false; + mockExplainError = null; + }); + + it('renders nothing when enabled=false', () => { + renderWithMantine( + , + ); + expect(screen.queryByText(/Scanned Rows/)).not.toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /show generated sql/i }), + ).not.toBeInTheDocument(); + }); + + it('shows loading state while explain query is in flight', () => { + mockExplainIsLoading = true; + renderWithMantine( + , + ); + expect(screen.getByText('Scanned Rows ...')).toBeInTheDocument(); + }); + + it('keeps the SQL icon visible while explain is loading (no flicker on poll)', () => { + mockExplainIsLoading = true; + renderWithMantine( + , + ); + expect( + screen.getByRole('button', { name: /show generated sql/i }), + ).toBeInTheDocument(); + }); + + it('renders empty text on error', () => { + mockExplainError = new Error('fail'); + renderWithMantine( + , + ); + // No row count text shown on error + expect(screen.queryByText(/Scanned Rows:/)).not.toBeInTheDocument(); + }); + + it('shows formatted scanned row count when data is present', () => { + mockExplainData = [{ rows: 1482447 }]; + renderWithMantine( + , + ); + expect(screen.getByText('Scanned Rows: 1,482,447')).toBeInTheDocument(); + }); + + it('shows "Elapsed Time: ..." while searching', () => { + mockExplainData = [{ rows: 100 }]; + renderWithMantine( + , + ); + expect(screen.getByText('Elapsed Time: ...')).toBeInTheDocument(); + }); + + it('does not flash the "..." elapsed loading state during live tail', () => { + mockExplainData = [{ rows: 100 }]; + renderWithMantine( + , + ); + // While live and fetching, keep showing the last value, not the "..." + expect(screen.queryByText('Elapsed Time: ...')).not.toBeInTheDocument(); + expect(screen.getByText(/Elapsed Time:/)).toBeInTheDocument(); + }); + + it('hides elapsed during live tail before the first measurement', () => { + mockExplainData = [{ rows: 100 }]; + renderWithMantine( + , + ); + expect(screen.queryByText(/Elapsed Time:/)).not.toBeInTheDocument(); + }); + + it('shows formatted elapsed time after search completes', () => { + mockExplainData = [{ rows: 100 }]; + renderWithMantine( + , + ); + expect(screen.getByText(/Elapsed Time:/)).toBeInTheDocument(); + expect(screen.queryByText('Elapsed Time: ...')).not.toBeInTheDocument(); + }); + + it('hides elapsed time section when there is no elapsed value and not searching', () => { + mockExplainData = [{ rows: 100 }]; + renderWithMantine( + , + ); + expect(screen.queryByText(/Elapsed Time:/)).not.toBeInTheDocument(); + }); + + it('still shows elapsed time and SQL icon when the explain query fails', () => { + mockExplainError = new Error('explain timed out'); + renderWithMantine( + , + ); + // Elapsed time is independent of the explain query and must remain visible + expect(screen.getByText(/Elapsed Time:/)).toBeInTheDocument(); + // SQL preview is also independent of explain data + expect( + screen.getByRole('button', { name: /show generated sql/i }), + ).toBeInTheDocument(); + }); + + it('shows elapsed time while searching even when explain has no data yet', () => { + mockExplainData = undefined; + renderWithMantine( + , + ); + expect(screen.getByText('Elapsed Time: ...')).toBeInTheDocument(); + }); + + it('opens the modal when the SQL button is clicked', async () => { + mockExplainData = [{ rows: 50 }]; + renderWithMantine( + , + ); + + await act(async () => { + fireEvent.click( + screen.getByRole('button', { name: /show generated sql/i }), + ); + }); + + expect(screen.getByTestId('modal')).toBeInTheDocument(); + expect(screen.getByTestId('chart-sql-preview')).toBeInTheDocument(); + }); + + it('uses sqlConfig in the modal when provided — title says "Timeline" and content uses sqlConfig', async () => { + mockExplainData = [{ rows: 50 }]; + const sqlConfig = { ...baseConfig, source: 'timeline-source' } as any; + + renderWithMantine( + , + ); + + await act(async () => { + fireEvent.click( + screen.getByRole('button', { name: /show generated sql/i }), + ); + }); + + expect(screen.getByTestId('modal-title')).toHaveTextContent( + 'Generated SQL (Timeline)', + ); + expect(lastSQLPreviewConfig).toEqual(sqlConfig); + }); + + it('uses plain "Generated SQL" title and rows config when no sqlConfig is passed', async () => { + mockExplainData = [{ rows: 50 }]; + const rowsConfig = { ...baseConfig, source: 'rows-source' } as any; + + renderWithMantine( + , + ); + + await act(async () => { + fireEvent.click( + screen.getByRole('button', { name: /show generated sql/i }), + ); + }); + + expect(screen.getByTestId('modal-title')).toHaveTextContent( + 'Generated SQL', + ); + expect( + screen.queryByText('Generated SQL (Timeline)'), + ).not.toBeInTheDocument(); + expect(lastSQLPreviewConfig).toEqual(rowsConfig); + }); +}); diff --git a/packages/app/src/components/ChartSQLPreview.tsx b/packages/app/src/components/ChartSQLPreview.tsx index a07333cf8f..86663577c9 100644 --- a/packages/app/src/components/ChartSQLPreview.tsx +++ b/packages/app/src/components/ChartSQLPreview.tsx @@ -4,6 +4,7 @@ import { format } from '@hyperdx/common-utils/dist/sqlFormatter'; import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; import { Button, Paper, Text, useMantineColorScheme } from '@mantine/core'; import { IconCheck, IconCopy } from '@tabler/icons-react'; +import { keepPreviousData } from '@tanstack/react-query'; import CodeMirror, { EditorView } from '@uiw/react-codemirror'; import { useRenderedSqlChartConfig } from '@/hooks/useChartConfig'; @@ -97,7 +98,12 @@ export default function ChartSQLPreview({ config: ChartConfigWithOptDateRange; enableCopy?: boolean; }) { - const { data, error, isLoading } = useRenderedSqlChartConfig(config); + const { data, error, isLoading } = useRenderedSqlChartConfig(config, { + // Keep the previously rendered SQL visible while a new one is generated so + // the preview doesn't flicker when the config changes (e.g. live tail + // refreshes the dateRange each poll). + placeholderData: keepPreviousData, + }); return ( - {isLoading ? ( + {data ? ( + // Prefer showing the (possibly placeholder) SQL over the loading state + // so the preview doesn't flicker when the query re-runs — e.g. live tail + // refreshes the dateRange each poll, and builder configs briefly report + // isLoading via the MV-optimization lookup. + + ) : isLoading ? ( Loading query preview... diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index ac2a78c82c..e4393dc780 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -460,7 +460,7 @@ export function useQueriedChartConfig( export function useRenderedSqlChartConfig( config: ChartConfigWithOptDateRange, - options?: UseQueryOptions, + options?: Partial>, ) { const { enabled = true } = options ?? {}; diff --git a/packages/app/src/hooks/useExplainQuery.tsx b/packages/app/src/hooks/useExplainQuery.tsx index 0ead671e6f..8e1e5d078d 100644 --- a/packages/app/src/hooks/useExplainQuery.tsx +++ b/packages/app/src/hooks/useExplainQuery.tsx @@ -11,6 +11,8 @@ export function useExplainQuery( _config: ChartConfigWithOptDateRange, options?: Omit, 'queryKey' | 'queryFn'>, ) { + const { enabled, ...restOptions } = options ?? {}; + const config = { ..._config, with: undefined, @@ -42,7 +44,7 @@ export function useExplainQuery( }, retry: false, staleTime: 1000 * 60, - enabled: !isSourceLoading, - ...options, + enabled: enabled && !isSourceLoading, + ...restOptions, }); } From f9fab8ed6f1ddb5815d7e92a05f041ed70e4b3bc Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 30 Jun 2026 11:20:13 -0400 Subject: [PATCH 121/728] fix(table): keep sticky table header above body cells when scrolling (#2553) ## Summary Fixes a bug causing text content from the rightmost column in the HDXMultiSeriesTableChart overlapping the table header text. The `.lastCell` class [associated with table onClick action tooltips](https://github.com/hyperdxio/hyperdx/pull/2380/changes#diff-134df20f2798566e6a607358f3d2c12f7b5560c331004d4aec24a02cf502477dR30) applies relative positioning, which creates a new stacking context for the rightmost cell, above the header. The `z-index: 2` now applied to the header ensures that the header is positioned above the cell content. This mirrors the solution used for the existing logs table. ### Screenshots or video #### Before https://github.com/user-attachments/assets/855477fe-dc15-4a26-8f7b-4d2cfc803dab #### After https://github.com/user-attachments/assets/5ae4fdbc-6679-4913-ac06-3a9c0407e9e3 ### How to test on Vercel preview - Create a table with a few columns and enough rows to trigger scrolling - Confirm that none of the content overlaps the headers ### References - Linear Issue: HDX-4657 - Related PRs: --- .changeset/fix-table-header-overlap.md | 5 +++++ packages/app/src/HDXMultiSeriesTableChart.tsx | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 .changeset/fix-table-header-overlap.md diff --git a/.changeset/fix-table-header-overlap.md b/.changeset/fix-table-header-overlap.md new file mode 100644 index 0000000000..7e80e58b80 --- /dev/null +++ b/.changeset/fix-table-header-overlap.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: Prevent table content from overlapping table headers diff --git a/packages/app/src/HDXMultiSeriesTableChart.tsx b/packages/app/src/HDXMultiSeriesTableChart.tsx index bd85ced210..9c32d17eea 100644 --- a/packages/app/src/HDXMultiSeriesTableChart.tsx +++ b/packages/app/src/HDXMultiSeriesTableChart.tsx @@ -349,6 +349,12 @@ export const Table = ({ style={{ position: 'sticky', top: 0, + // Body cells include positioned elements (e.g. the + // position: relative `.lastCell`), which otherwise paint on + // top of the sticky header as rows scroll underneath it. + // A stacking context above those cells keeps the header on + // top. Mirrors the log table's sticky head (LogTable.module.scss). + zIndex: 2, background: variant === 'muted' ? 'var(--color-bg-muted)' From e2103f78e82724a2b03e3d069c78c9d1e672e1e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:01:59 -0700 Subject: [PATCH 122/728] Release HyperDX (#2378) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .changeset/add-accesskey-index.md | 5 - .changeset/add-hdx-eval-framework.md | 15 - .changeset/alias-map-projection-fallback.md | 14 - .changeset/angry-boats-lie.md | 5 - .changeset/breezy-foxes-look.md | 5 - .changeset/breezy-students-clap.md | 7 - .changeset/browser-rum-dashboard-template.md | 12 - .changeset/bump-http-proxy-middleware.md | 6 - .changeset/bump-otel-collector-0-154-0.md | 18 - .changeset/bump-otel-collector-0-155-0.md | 19 - .changeset/chilled-drinks-pay.md | 6 - .changeset/clickhouse-client-1-23-head.md | 18 - .../clickhouse-client-forward-compat.md | 11 - .changeset/connect-your-ai-assistant.md | 11 - .../connection-prometheus-endpoint-ui.md | 9 - .changeset/curvy-foxes-give.md | 5 - .../dashboard-number-tile-font-resize.md | 10 - .changeset/datetime-filter-exclude.md | 6 - .../duplicate-series-in-chart-builder.md | 12 - .changeset/early-banks-retire.md | 5 - .changeset/editable-filter-pills.md | 7 - .changeset/eight-phones-protect.md | 5 - .changeset/event-patterns-column-selector.md | 5 - .changeset/external-api-connections.md | 5 - .changeset/filter-key-special-characters.md | 6 - .changeset/fix-bool-filter.md | 5 - .changeset/fix-describe-source-timeout.md | 5 - .../fix-exclude-filter-pill-contrast.md | 7 - .changeset/fix-invite-password.md | 5 - .../fix-log-detail-timestamp-timezone.md | 7 - .changeset/fix-markdown-tile-save.md | 5 - ...x-parametric-aggregate-cast-distributed.md | 7 - .../fix-series-limit-chunk-consistency.md | 6 - .changeset/fix-table-header-overlap.md | 5 - .changeset/fluffy-cows-shave.md | 8 - .changeset/fuzzy-knives-bathe.md | 5 - .changeset/giant-waves-argue.md | 5 - .changeset/good-parents-check.md | 5 - .changeset/good-swans-hammer.md | 5 - .../hdx-4369-mergepath-map-subscript.md | 23 -- .changeset/hdx-4405-trailing-action-hint.md | 5 - .changeset/hot-impalas-flash.md | 5 - .changeset/lazy-load-dashboard-tiles.md | 7 - .changeset/many-comics-rule.md | 5 - .changeset/mcp-alias-descriptions.md | 5 - .changeset/mcp-dashboard-improvements.md | 21 -- .changeset/mcp-error-hints-readonly.md | 16 - .changeset/mcp-metric-source-support.md | 11 - .changeset/mcp-number-tile-color.md | 5 - .changeset/mcp-objectid-validation.md | 10 - .changeset/mcp-orderby-alias-quoting.md | 17 - .changeset/mcp-search-denoise.md | 20 - .changeset/mcp-sql-tile-macro-hints.md | 6 - .changeset/metric-table-autofill.md | 11 - .changeset/nasty-crews-shop.md | 5 - .changeset/nodenext-module-resolution.md | 5 - ...mber-tile-background-chart-external-api.md | 5 - .changeset/number-tile-background-chart.md | 15 - .changeset/number-tile-color-external-api.md | 5 - .../number-tile-format-survives-apply.md | 12 - .changeset/number-tile-sparkline-groupby.md | 7 - .../observability-instrumentation-rollout.md | 21 -- .../observability-instrumentation-uplift.md | 11 - .changeset/onclick-external-link-variant.md | 7 - .../page-header-z-index-below-drawers.md | 20 - .changeset/page-migrate-dashboard.md | 7 - .changeset/pattern-table-alias-with.md | 13 - .changeset/polite-moons-give.md | 5 - .changeset/ratio-number-tiles.md | 5 - .changeset/raw-sql-chart-shortcut.md | 5 - .changeset/remove-mcp-max-result-rows.md | 14 - .changeset/rename-chart-palette-tokens.md | 85 ----- .changeset/rename-mcp-tools-to-clickstack.md | 10 - .changeset/selfish-oranges-cheer.md | 9 - .changeset/serious-gifts-rest.md | 6 - .../service-map-percentiles-filtering.md | 12 - .changeset/silly-toes-agree.md | 5 - .changeset/smooth-turkeys-dream.md | 5 - .changeset/solid-pigs-rhyme.md | 6 - .changeset/source-chip-kebab-menu.md | 5 - .changeset/source-form-dropdown-clipping.md | 5 - .changeset/source-section-autocomplete.md | 5 - .changeset/source-section-field.md | 12 - .../source-selector-section-grouping.md | 5 - .changeset/sources-list-section.md | 5 - .changeset/spotty-roses-cheer.md | 7 - .changeset/standalone-promql-conditional.md | 22 -- .changeset/tile-alert-attributes.md | 5 - .changeset/time-chart-series-limit.md | 7 - .../tooltip-highlight-nearest-series.md | 5 - .changeset/tough-ads-pay.md | 5 - .../type-json-metadata-filter-attributes.md | 5 - .changeset/variadic-cityhash-aligns.md | 31 -- .changeset/wet-ties-kneel.md | 5 - .changeset/wise-jobs-relax.md | 6 - .env | 4 +- packages/api/CHANGELOG.md | 295 +++++++++++++++ packages/api/package.json | 4 +- packages/app/CHANGELOG.md | 344 ++++++++++++++++++ packages/app/package.json | 4 +- packages/cli/CHANGELOG.md | 26 ++ packages/cli/package.json | 4 +- packages/common-utils/CHANGELOG.md | 204 +++++++++++ packages/common-utils/package.json | 2 +- packages/hdx-eval/CHANGELOG.md | 22 ++ packages/hdx-eval/package.json | 2 +- packages/otel-collector/CHANGELOG.md | 57 +++ packages/otel-collector/package.json | 2 +- yarn.lock | 8 +- 109 files changed, 963 insertions(+), 924 deletions(-) delete mode 100644 .changeset/add-accesskey-index.md delete mode 100644 .changeset/add-hdx-eval-framework.md delete mode 100644 .changeset/alias-map-projection-fallback.md delete mode 100644 .changeset/angry-boats-lie.md delete mode 100644 .changeset/breezy-foxes-look.md delete mode 100644 .changeset/breezy-students-clap.md delete mode 100644 .changeset/browser-rum-dashboard-template.md delete mode 100644 .changeset/bump-http-proxy-middleware.md delete mode 100644 .changeset/bump-otel-collector-0-154-0.md delete mode 100644 .changeset/bump-otel-collector-0-155-0.md delete mode 100644 .changeset/chilled-drinks-pay.md delete mode 100644 .changeset/clickhouse-client-1-23-head.md delete mode 100644 .changeset/clickhouse-client-forward-compat.md delete mode 100644 .changeset/connect-your-ai-assistant.md delete mode 100644 .changeset/connection-prometheus-endpoint-ui.md delete mode 100644 .changeset/curvy-foxes-give.md delete mode 100644 .changeset/dashboard-number-tile-font-resize.md delete mode 100644 .changeset/datetime-filter-exclude.md delete mode 100644 .changeset/duplicate-series-in-chart-builder.md delete mode 100644 .changeset/early-banks-retire.md delete mode 100644 .changeset/editable-filter-pills.md delete mode 100644 .changeset/eight-phones-protect.md delete mode 100644 .changeset/event-patterns-column-selector.md delete mode 100644 .changeset/external-api-connections.md delete mode 100644 .changeset/filter-key-special-characters.md delete mode 100644 .changeset/fix-bool-filter.md delete mode 100644 .changeset/fix-describe-source-timeout.md delete mode 100644 .changeset/fix-exclude-filter-pill-contrast.md delete mode 100644 .changeset/fix-invite-password.md delete mode 100644 .changeset/fix-log-detail-timestamp-timezone.md delete mode 100644 .changeset/fix-markdown-tile-save.md delete mode 100644 .changeset/fix-parametric-aggregate-cast-distributed.md delete mode 100644 .changeset/fix-series-limit-chunk-consistency.md delete mode 100644 .changeset/fix-table-header-overlap.md delete mode 100644 .changeset/fluffy-cows-shave.md delete mode 100644 .changeset/fuzzy-knives-bathe.md delete mode 100644 .changeset/giant-waves-argue.md delete mode 100644 .changeset/good-parents-check.md delete mode 100644 .changeset/good-swans-hammer.md delete mode 100644 .changeset/hdx-4369-mergepath-map-subscript.md delete mode 100644 .changeset/hdx-4405-trailing-action-hint.md delete mode 100644 .changeset/hot-impalas-flash.md delete mode 100644 .changeset/lazy-load-dashboard-tiles.md delete mode 100644 .changeset/many-comics-rule.md delete mode 100644 .changeset/mcp-alias-descriptions.md delete mode 100644 .changeset/mcp-dashboard-improvements.md delete mode 100644 .changeset/mcp-error-hints-readonly.md delete mode 100644 .changeset/mcp-metric-source-support.md delete mode 100644 .changeset/mcp-number-tile-color.md delete mode 100644 .changeset/mcp-objectid-validation.md delete mode 100644 .changeset/mcp-orderby-alias-quoting.md delete mode 100644 .changeset/mcp-search-denoise.md delete mode 100644 .changeset/mcp-sql-tile-macro-hints.md delete mode 100644 .changeset/metric-table-autofill.md delete mode 100644 .changeset/nasty-crews-shop.md delete mode 100644 .changeset/nodenext-module-resolution.md delete mode 100644 .changeset/number-tile-background-chart-external-api.md delete mode 100644 .changeset/number-tile-background-chart.md delete mode 100644 .changeset/number-tile-color-external-api.md delete mode 100644 .changeset/number-tile-format-survives-apply.md delete mode 100644 .changeset/number-tile-sparkline-groupby.md delete mode 100644 .changeset/observability-instrumentation-rollout.md delete mode 100644 .changeset/observability-instrumentation-uplift.md delete mode 100644 .changeset/onclick-external-link-variant.md delete mode 100644 .changeset/page-header-z-index-below-drawers.md delete mode 100644 .changeset/page-migrate-dashboard.md delete mode 100644 .changeset/pattern-table-alias-with.md delete mode 100644 .changeset/polite-moons-give.md delete mode 100644 .changeset/ratio-number-tiles.md delete mode 100644 .changeset/raw-sql-chart-shortcut.md delete mode 100644 .changeset/remove-mcp-max-result-rows.md delete mode 100644 .changeset/rename-chart-palette-tokens.md delete mode 100644 .changeset/rename-mcp-tools-to-clickstack.md delete mode 100644 .changeset/selfish-oranges-cheer.md delete mode 100644 .changeset/serious-gifts-rest.md delete mode 100644 .changeset/service-map-percentiles-filtering.md delete mode 100644 .changeset/silly-toes-agree.md delete mode 100644 .changeset/smooth-turkeys-dream.md delete mode 100644 .changeset/solid-pigs-rhyme.md delete mode 100644 .changeset/source-chip-kebab-menu.md delete mode 100644 .changeset/source-form-dropdown-clipping.md delete mode 100644 .changeset/source-section-autocomplete.md delete mode 100644 .changeset/source-section-field.md delete mode 100644 .changeset/source-selector-section-grouping.md delete mode 100644 .changeset/sources-list-section.md delete mode 100644 .changeset/spotty-roses-cheer.md delete mode 100644 .changeset/standalone-promql-conditional.md delete mode 100644 .changeset/tile-alert-attributes.md delete mode 100644 .changeset/time-chart-series-limit.md delete mode 100644 .changeset/tooltip-highlight-nearest-series.md delete mode 100644 .changeset/tough-ads-pay.md delete mode 100644 .changeset/type-json-metadata-filter-attributes.md delete mode 100644 .changeset/variadic-cityhash-aligns.md delete mode 100644 .changeset/wet-ties-kneel.md delete mode 100644 .changeset/wise-jobs-relax.md create mode 100644 packages/hdx-eval/CHANGELOG.md diff --git a/.changeset/add-accesskey-index.md b/.changeset/add-accesskey-index.md deleted file mode 100644 index 4a9b44d73c..0000000000 --- a/.changeset/add-accesskey-index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": minor ---- - -Add unique MongoDB index on accessKey field in User model to eliminate full collection scans during API key authentication. This could cause startup failures if any existing users share duplicate accessKey values. diff --git a/.changeset/add-hdx-eval-framework.md b/.changeset/add-hdx-eval-framework.md deleted file mode 100644 index 9d7f75966b..0000000000 --- a/.changeset/add-hdx-eval-framework.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -'@hyperdx/hdx-eval': minor ---- - -feat: add AI eval framework for benchmarking MCP servers - -New `@hyperdx/hdx-eval` package for benchmarking AI agents against -observability MCP servers. Generates deterministic synthetic telemetry -with planted anomalies, spawns Claude Code as an SRE agent, records full -trajectories, and grades answers using programmatic checks and an -LLM-as-judge. - -Includes 5 scenarios (error-root-cause, latency-spike, noisy-signals, -segmented-regression, service-health-check), MCP-agnostic N-way -comparison, blinded judging, and a web viewer for browsing results. diff --git a/.changeset/alias-map-projection-fallback.md b/.changeset/alias-map-projection-fallback.md deleted file mode 100644 index f7b48c3d46..0000000000 --- a/.changeset/alias-map-projection-fallback.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -'@hyperdx/common-utils': patch ---- - -fix: recover the SELECT-alias map when a query has ClickHouse-specific SQL the parser rejects - -`chSqlToAliasMap` returned an empty map whenever the rendered query contained -SQL that node-sql-parser's Postgresql dialect cannot parse, for example a -sampling CTE with `greatest(CAST(total / N AS UInt32), 1)`. An empty alias map -drops the `WITH` clauses that define the source's select aliases, so filters on -aliased columns (Event Patterns, histogram, alerts) failed with `Unknown -identifier`. It now falls back to parsing only the outer SELECT projection, -which is all the alias map needs, so the aliases are recovered even when the -rest of the statement is unparseable. diff --git a/.changeset/angry-boats-lie.md b/.changeset/angry-boats-lie.md deleted file mode 100644 index a5e9d5fc74..0000000000 --- a/.changeset/angry-boats-lie.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix(search-filters): prevent nested filter dropdowns from disappearing on reopen diff --git a/.changeset/breezy-foxes-look.md b/.changeset/breezy-foxes-look.md deleted file mode 100644 index 267f9b44d2..0000000000 --- a/.changeset/breezy-foxes-look.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": minor ---- - -Feat: Added a V2 endpoint for team management. diff --git a/.changeset/breezy-students-clap.md b/.changeset/breezy-students-clap.md deleted file mode 100644 index 2e08cb5b03..0000000000 --- a/.changeset/breezy-students-clap.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -feat: Add option to fit time chart y-axis lower bound diff --git a/.changeset/browser-rum-dashboard-template.md b/.changeset/browser-rum-dashboard-template.md deleted file mode 100644 index fd16b607c6..0000000000 --- a/.changeset/browser-rum-dashboard-template.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -'@hyperdx/app': minor ---- - -feat: add Browser RUM dashboard template - -- New "Browser RUM" template in the dashboards gallery for browser sessions instrumented with the HyperDX Browser SDK (or any OTel browser instrumentation emitting a `rum.sessionId` resource attribute) -- Performance Overview section: page-view/session/error KPIs, Core Web Vitals (LCP/INP/CLS) p75, median/p75/p90 page-load percentiles, and long-task health -- Page Views Breakdown section: traffic grouped by URL, browser (parsed from the `http.user_agent` the document-load instrumentation emits), country, and device size (derived from `screen.xy`) -- Errors section with tabs for an overview, JS exceptions (by message and by page), and failing API calls -- Five dashboard-level filters: Service, Environment, Service Version, Page URL, and Country -- Top Countries tile and the Country filter populate when the OTel collector's `geoip` processor is enabled (geo can't be derived in the browser) diff --git a/.changeset/bump-http-proxy-middleware.md b/.changeset/bump-http-proxy-middleware.md deleted file mode 100644 index 06a2586660..0000000000 --- a/.changeset/bump-http-proxy-middleware.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -Bump http-proxy-middleware to v4, replacing http-proxy with httpxy diff --git a/.changeset/bump-otel-collector-0-154-0.md b/.changeset/bump-otel-collector-0-154-0.md deleted file mode 100644 index 920f7c1e03..0000000000 --- a/.changeset/bump-otel-collector-0-154-0.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -'@hyperdx/otel-collector': minor ---- - -chore(otel-collector): bump base collector to v0.154.0 - -Upgrade the custom OTel Collector base from contrib v0.149.0 (core 1.55.0) to -v0.154.0 (core 1.60.0). Updates `OTEL_COLLECTOR_VERSION` / -`OTEL_COLLECTOR_CORE_VERSION` in `.env`, both Dockerfile ARG defaults, and the -smoke-test compose fallbacks. - -Compatibility: no config changes required. Reviewed contrib and core breaking -changes across v0.150–v0.154 against every component HyperDX uses. All affected -upstream changes are either backward-compatible deprecation aliases -(`prometheusremotewrite`, `resourcedetection`), explicit-config no-ops for -HyperDX (clickhouse exporter already sets `json:` directly; transform/routing -connectors set `error_mode: ignore` explicitly), or internal core feature-gate -stabilizations. diff --git a/.changeset/bump-otel-collector-0-155-0.md b/.changeset/bump-otel-collector-0-155-0.md deleted file mode 100644 index 6a6cbf822e..0000000000 --- a/.changeset/bump-otel-collector-0-155-0.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -'@hyperdx/otel-collector': minor ---- - -chore(otel-collector): bump base collector to v0.155.0 - -Upgrade the custom OTel Collector base from contrib v0.154.0 (core 1.60.0) to -v0.155.0 (core 1.61.0). Updates `OTEL_COLLECTOR_VERSION` / -`OTEL_COLLECTOR_CORE_VERSION` in `.env`, both Dockerfile ARG defaults, and the -smoke-test compose fallbacks. - -Compatibility: no config changes required. Reviewed contrib and core breaking -changes for v0.155.0 against every component HyperDX uses. The removed -`telemetry.UseLocalHostAsDefaultMetricsAddress` core gate has no impact because -the telemetry metrics endpoint is set explicitly (`host: 0.0.0.0`, `port: -8888`), and the `memory_limiter` metric rename does not affect the smoke tests -(which assert on the startup log line and the `batch/lowlatency` metric label, -not memory_limiter metrics). All other breaking changes are in unused components -or internal feature-gate removals. diff --git a/.changeset/chilled-drinks-pay.md b/.changeset/chilled-drinks-pay.md deleted file mode 100644 index d5a8b7d1b9..0000000000 --- a/.changeset/chilled-drinks-pay.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch ---- - -fix: Add sourceId to MCP Raw SQL Tile schema diff --git a/.changeset/clickhouse-client-1-23-head.md b/.changeset/clickhouse-client-1-23-head.md deleted file mode 100644 index 448c950d9e..0000000000 --- a/.changeset/clickhouse-client-1-23-head.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/cli": patch ---- - -Bump `@clickhouse/client*` to `1.23.0-head.fae5998.1` and fix the type -incompatibility it introduces. - -In `@clickhouse/client*` 1.23 each platform package (`@clickhouse/client`, -`@clickhouse/client-web`) bundles its own copy of the shared types, so their -`ClickHouseSettings` types — which reference the nominally-compared `SettingsMap` -class — are no longer the same type as `@clickhouse/client-common`'s. The shared -`processClickhouseSettings()` helper produces the `client-common` flavor, so -assigning it into the per-platform clients' `query()` now requires an explicit -bridge. Guard the existing `as ClickHouseSettings` assertions at those -boundaries (`node.ts`, `browser.ts`, `cli`) with a scoped -`@typescript-eslint/no-unsafe-type-assertion` disable, matching the existing -"client library type mismatch" pattern. No runtime behavior changes. diff --git a/.changeset/clickhouse-client-forward-compat.md b/.changeset/clickhouse-client-forward-compat.md deleted file mode 100644 index 1e5972925b..0000000000 --- a/.changeset/clickhouse-client-forward-compat.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/cli": patch ---- - -Import ClickHouse client types from the platform packages -(`@clickhouse/client` / `@clickhouse/client-web`) instead of the deprecated -`@clickhouse/client-common`. This makes the packages forward-compatible with -`@clickhouse/client*` 1.23 (where `client-common` is deprecated and each -platform package bundles and re-exports its own copy of the shared types) -without bumping the pinned version. No runtime behavior changes. diff --git a/.changeset/connect-your-ai-assistant.md b/.changeset/connect-your-ai-assistant.md deleted file mode 100644 index e39775e851..0000000000 --- a/.changeset/connect-your-ai-assistant.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -feat: add a "Connect your AI assistant" section to Team Settings - -A new section on the Team Settings page (Integrations tab, above the API Keys -card) lets a user install the HyperDX MCP server in Claude Code, Cursor, -VS Code + Copilot, Codex CLI, or any MCP-compatible host without hand-rolling -JSON. Per-host snippets carry the user's personal access key so the install -works against the existing `/api/mcp` route without extra setup. diff --git a/.changeset/connection-prometheus-endpoint-ui.md b/.changeset/connection-prometheus-endpoint-ui.md deleted file mode 100644 index 14bf3b6a48..0000000000 --- a/.changeset/connection-prometheus-endpoint-ui.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -'@hyperdx/app': minor -'@hyperdx/api': minor -'@hyperdx/common-utils': minor ---- - -Add UI support for configuring an external Prometheus-compatible endpoint on a -connection. Modify Connections model to now have a boolean -`isPrometheusEndpoint` field and use host for storing the host. diff --git a/.changeset/curvy-foxes-give.md b/.changeset/curvy-foxes-give.md deleted file mode 100644 index 84f55a7ee6..0000000000 --- a/.changeset/curvy-foxes-give.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -fix: Add missing numberFormats, compareToPreviousPeriod fields to MCP Schemas diff --git a/.changeset/dashboard-number-tile-font-resize.md b/.changeset/dashboard-number-tile-font-resize.md deleted file mode 100644 index 1bdcaa5297..0000000000 --- a/.changeset/dashboard-number-tile-font-resize.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat(dashboard): auto-resize font in number tiles to fit container - -Number tiles now automatically scale their font size to fit the available -width, preventing text overflow on narrow tiles and making better use of -space on wide ones. Includes an error boundary so a single broken tile -does not crash the entire dashboard. diff --git a/.changeset/datetime-filter-exclude.md b/.changeset/datetime-filter-exclude.md deleted file mode 100644 index e65ddb1b09..0000000000 --- a/.changeset/datetime-filter-exclude.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch ---- - -fix(search): wrap date column values in a type-matching parse/convert expression when building IN/NOT IN filters, so including/excluding a timestamp value no longer fails with "Cannot convert string ... to type DateTime64" or "Type mismatch in IN ... Expected: DateTime. Got: Decimal64". Date column types are now resolved from the query result set, so aliased (`TimestampTime AS time`) and computed (`toDate(TimestampTime)`) DateTime/Date columns are also wrapped correctly when added to filters. diff --git a/.changeset/duplicate-series-in-chart-builder.md b/.changeset/duplicate-series-in-chart-builder.md deleted file mode 100644 index 30b38ae6ba..0000000000 --- a/.changeset/duplicate-series-in-chart-builder.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat(chart-explorer): duplicate a series in the chart builder - -Add a Duplicate button to each series row in the chart builder that inserts a -copy of that series directly below it, so building a near-identical variant -(for example avg and p95 of the same column) no longer requires re-entering -every field by hand. "Add Series" still creates a blank series. The copy -starts with an empty alias so it does not collide with the original's alias in -the generated SQL. diff --git a/.changeset/early-banks-retire.md b/.changeset/early-banks-retire.md deleted file mode 100644 index 735f033102..0000000000 --- a/.changeset/early-banks-retire.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Show elapsed time and Generated SQL for search timeline view diff --git a/.changeset/editable-filter-pills.md b/.changeset/editable-filter-pills.md deleted file mode 100644 index f61e38c47b..0000000000 --- a/.changeset/editable-filter-pills.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -feat(search): make active filter pills editable in place - -Clicking an active filter pill under the search bar now opens a small menu to copy the value, flip the filter polarity (include vs exclude), or switch to a different value of the same field, without removing and re-adding the filter. The polarity is preserved when changing the value, and the one-click remove on each pill is unchanged. Range and not-applied pills keep their remove-only behavior. diff --git a/.changeset/eight-phones-protect.md b/.changeset/eight-phones-protect.md deleted file mode 100644 index 25334b05f4..0000000000 --- a/.changeset/eight-phones-protect.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -feat: Use optimistic updates for favorites diff --git a/.changeset/event-patterns-column-selector.md b/.changeset/event-patterns-column-selector.md deleted file mode 100644 index a6b2f1a4fc..0000000000 --- a/.changeset/event-patterns-column-selector.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat: Allow selecting the column or SQL expression used for event pattern grouping (with shareable URL state) diff --git a/.changeset/external-api-connections.md b/.changeset/external-api-connections.md deleted file mode 100644 index b430d6e8ca..0000000000 --- a/.changeset/external-api-connections.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": minor ---- - -Add connection management endpoints to the external API (`/api/v2/connections`), supporting list, get, create, update, and delete with Bearer token authentication. Passwords are write-only and never returned by the API. \ No newline at end of file diff --git a/.changeset/filter-key-special-characters.md b/.changeset/filter-key-special-characters.md deleted file mode 100644 index 49d766ae7a..0000000000 --- a/.changeset/filter-key-special-characters.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@hyperdx/common-utils': patch -'@hyperdx/app': patch ---- - -fix: Correct filter handling for filter keys with special characters diff --git a/.changeset/fix-bool-filter.md b/.changeset/fix-bool-filter.md deleted file mode 100644 index 759cb05485..0000000000 --- a/.changeset/fix-bool-filter.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -fix: handle boolean values in JSON viewer filter actions diff --git a/.changeset/fix-describe-source-timeout.md b/.changeset/fix-describe-source-timeout.md deleted file mode 100644 index 716b355bc0..0000000000 --- a/.changeset/fix-describe-source-timeout.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -fix(mcp): reduce describe_source timeouts by using rollup tables for map key discovery diff --git a/.changeset/fix-exclude-filter-pill-contrast.md b/.changeset/fix-exclude-filter-pill-contrast.md deleted file mode 100644 index 39ddb9dbec..0000000000 --- a/.changeset/fix-exclude-filter-pill-contrast.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: improve contrast of excluded search filter pills - -Excluded ("!=") filter pills above the search results used a saturated red background with red text and a red remove button, which made them hard to read in the light theme. They now use a soft red tint with a readable accent, legible in both light and dark themes. diff --git a/.changeset/fix-invite-password.md b/.changeset/fix-invite-password.md deleted file mode 100644 index 360fb33569..0000000000 --- a/.changeset/fix-invite-password.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -fix: enforce password complexity on team invite acceptance diff --git a/.changeset/fix-log-detail-timestamp-timezone.md b/.changeset/fix-log-detail-timestamp-timezone.md deleted file mode 100644 index ec21d9a613..0000000000 --- a/.changeset/fix-log-detail-timestamp-timezone.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix(app): format log detail Timestamp in local timezone - -The log detail JSON viewer rendered Timestamp and TimestampTime as raw UTC ISO strings while the results table used the shared FormatTime helper. diff --git a/.changeset/fix-markdown-tile-save.md b/.changeset/fix-markdown-tile-save.md deleted file mode 100644 index 32a3475c6f..0000000000 --- a/.changeset/fix-markdown-tile-save.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: allow saving edits to markdown dashboard tiles that have a minimal config shape (no resolved source) diff --git a/.changeset/fix-parametric-aggregate-cast-distributed.md b/.changeset/fix-parametric-aggregate-cast-distributed.md deleted file mode 100644 index dbf2167c7e..0000000000 --- a/.changeset/fix-parametric-aggregate-cast-distributed.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@hyperdx/common-utils': patch -'@hyperdx/api': patch -'@hyperdx/app': patch ---- - -fix: inline parametric aggregate function arguments instead of passing as query parameters diff --git a/.changeset/fix-series-limit-chunk-consistency.md b/.changeset/fix-series-limit-chunk-consistency.md deleted file mode 100644 index 255666fc5a..0000000000 --- a/.changeset/fix-series-limit-chunk-consistency.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@hyperdx/common-utils': patch -'@hyperdx/app': patch ---- - -feat(charts): the time-chart series limit is now configured per chart in the Display Settings drawer instead of as a workspace-wide team setting (the team "Time Chart Series Limit" setting is removed). It is disabled by default — charts fetch every series and no `__hdx_series_limit` CTE is emitted — and is cleared back to disabled by emptying the field. The control only appears for builder line/bar charts; the limit and its Generated SQL preview now come from the chart's own config. When a limit is set, chunked time-chart queries keep a consistent top-N series set: previously each time-window chunk ranked its own top-N, so charts could render more series than the limit and adjacent windows disagreed; the ranking is now pinned to the newest chunk window for every chunk so the union across chunks equals the limit. diff --git a/.changeset/fix-table-header-overlap.md b/.changeset/fix-table-header-overlap.md deleted file mode 100644 index 7e80e58b80..0000000000 --- a/.changeset/fix-table-header-overlap.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: Prevent table content from overlapping table headers diff --git a/.changeset/fluffy-cows-shave.md b/.changeset/fluffy-cows-shave.md deleted file mode 100644 index eb9dfde37b..0000000000 --- a/.changeset/fluffy-cows-shave.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -"@hyperdx/otel-collector": patch -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -fix: polish promql experience across the app diff --git a/.changeset/fuzzy-knives-bathe.md b/.changeset/fuzzy-knives-bathe.md deleted file mode 100644 index ab7e1b03e8..0000000000 --- a/.changeset/fuzzy-knives-bathe.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix: Skip rendering empty aggConditions diff --git a/.changeset/giant-waves-argue.md b/.changeset/giant-waves-argue.md deleted file mode 100644 index 50974a7279..0000000000 --- a/.changeset/giant-waves-argue.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: Navigate to the dashboard listing page after deleting a dashboard diff --git a/.changeset/good-parents-check.md b/.changeset/good-parents-check.md deleted file mode 100644 index e21e03acf8..0000000000 --- a/.changeset/good-parents-check.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/hdx-eval': patch ---- - -Support multi-model comparison in eval batches via comma-separated --model flag diff --git a/.changeset/good-swans-hammer.md b/.changeset/good-swans-hammer.md deleted file mode 100644 index a5ba3e0de5..0000000000 --- a/.changeset/good-swans-hammer.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -Adding filters to dashboard exports. Implemented validation on dashboard imports to catch potential issues with generated JSON or manually tweaked JSON. diff --git a/.changeset/hdx-4369-mergepath-map-subscript.md b/.changeset/hdx-4369-mergepath-map-subscript.md deleted file mode 100644 index 00c07cd9ba..0000000000 --- a/.changeset/hdx-4369-mergepath-map-subscript.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix(row-panel): mergePath now emits string-key subscripts for Map columns, -preventing a crash when expanding rows with numeric-looking attribute keys - -`mergePath` converted numeric path segments to 1-based array subscripts -(`[N+1]`) regardless of whether the parent column was a Map or an Array. -On a `Map(String, String)` column this produced SQL like `LogAttributes[2]`, -which ClickHouse rejects with `Illegal types of arguments: -Map(String, String), UInt8 for function arrayElement`. The grid row -"expand" view failed for any row whose attribute path included a -numeric-looking key under a Map column. - -`mergePath` now accepts a `mapColumns` argument alongside `jsonColumns`. -For Map-typed parents, sub-keys always render as string subscripts -(`Map['1']`) regardless of whether the key looks numeric. The three -callers (`useAutoCompleteOptions`, `DBRowJsonViewer` via the row panels, -`DBSearchPageFilters`) now thread Map-column names from the source -schema. A new `useMapColumns` hook mirrors `useJsonColumns`. - -Fixes HDX-4369. diff --git a/.changeset/hdx-4405-trailing-action-hint.md b/.changeset/hdx-4405-trailing-action-hint.md deleted file mode 100644 index 2dba609e2b..0000000000 --- a/.changeset/hdx-4405-trailing-action-hint.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -Dashboard table tiles configured with a row-click action now show a trailing arrow-up-right icon at the right edge of each row, revealed on hover, with a small tooltip that names the destination. Actionable rows get a stronger background highlight on hover to reinforce interactivity before the user sees the arrow fade in. The icon click navigates to the same URL as a row click, with all the standard native browser behaviors (cmd-click new tab, middle-click new tab, right-click context menu). diff --git a/.changeset/hot-impalas-flash.md b/.changeset/hot-impalas-flash.md deleted file mode 100644 index 1d74d040dd..0000000000 --- a/.changeset/hot-impalas-flash.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -Fix: Resolved an issue with markdown tiles breaking dashboard imports. diff --git a/.changeset/lazy-load-dashboard-tiles.md b/.changeset/lazy-load-dashboard-tiles.md deleted file mode 100644 index c1153e9058..0000000000 --- a/.changeset/lazy-load-dashboard-tiles.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@hyperdx/app': minor ---- - -feat: lazy-load dashboard tiles based on viewport visibility - -Dashboard tiles now only run their ClickHouse queries once they scroll into the browser viewport, instead of every tile querying on page load. A tile loads the first time it becomes visible and keeps its data afterward. This significantly reduces the number of queries fired when opening dashboards with many tiles. diff --git a/.changeset/many-comics-rule.md b/.changeset/many-comics-rule.md deleted file mode 100644 index 26eae6d49e..0000000000 --- a/.changeset/many-comics-rule.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -feat: add direct_read optimization for filters diff --git a/.changeset/mcp-alias-descriptions.md b/.changeset/mcp-alias-descriptions.md deleted file mode 100644 index 61cc48e4b8..0000000000 --- a/.changeset/mcp-alias-descriptions.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -fix(mcp): improve alias field descriptions and examples for readable chart legends diff --git a/.changeset/mcp-dashboard-improvements.md b/.changeset/mcp-dashboard-improvements.md deleted file mode 100644 index 375224e649..0000000000 --- a/.changeset/mcp-dashboard-improvements.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -feat(mcp): add patch_dashboard, get_dashboard_tile, search_dashboards tools - -Add three new MCP dashboard tools for granular operations: - -- `hyperdx_get_dashboard_tile` — retrieve a single tile by tileId -- `hyperdx_patch_dashboard` — update name/tags and/or replace one tile - without resubmitting the full dashboard -- `hyperdx_search_dashboards` — search by name and/or tags - -Fix empty parameter schema on patch/search tools caused by Zod -`.refine()` wrapping. Document Lucene substring matching limitations -prominently in tool descriptions and query guide prompt. - -**Breaking (minor):** Tile `name` on `hyperdx_save_dashboard` now requires -at least 1 character (`.min(1)`). Previously empty string `""` was accepted -and silently persisted as a blank title. Callers sending `name: ""` will -now receive a validation error. diff --git a/.changeset/mcp-error-hints-readonly.md b/.changeset/mcp-error-hints-readonly.md deleted file mode 100644 index 095316d290..0000000000 --- a/.changeset/mcp-error-hints-readonly.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -fix(mcp): improve error hints and fix readonly mode for query safety settings - -Switch MCP ClickHouse safety settings from readonly=1 to readonly=2 so -max_execution_time and max_result_rows are actually applied (readonly=1 -silently rejects all setting changes). - -Improve DateTime64 cast error hint to recommend parseDateTime64BestEffort() -which works on both DateTime and DateTime64 columns, replacing -toDateTime64() which only works on DateTime64. - -Add error hint for unknown column/identifier errors directing agents to -call describe_source before retrying. diff --git a/.changeset/mcp-metric-source-support.md b/.changeset/mcp-metric-source-support.md deleted file mode 100644 index 4dfd135051..0000000000 --- a/.changeset/mcp-metric-source-support.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -'@hyperdx/api': minor ---- - -feat(mcp): first-class metric source support - -- Two new tools: `clickstack_list_metrics` paginates the metric-name catalog with optional kind / namePattern (ILIKE) / time-window filters and opaque cursor pagination; `clickstack_describe_metric` returns per-metric kind(s), unit, description, attribute keys, and sampled values (with kind auto-detection). -- `clickstack_describe_source` is metric-aware: picks a representative metric table (gauge → sum → histogram), runs column / map-key / value-sampling against it, and adds a per-kind metric-name sample. -- `clickstack_timeseries` and `clickstack_table` accept `metricType` (gauge / sum / histogram), `metricName`, and `isDelta` on each select item, plus `aggFn:"increase"` for Sum counters. `valueExpression` defaults to `"Value"` for metric sources. Surfaces the renderer's 20-group top-N cap on `increase + groupBy` as a neutral hint. -- Dashboard prompt's "use raw SQL for metric tiles" workaround is replaced with positive discovery-workflow guidance and one worked example per supported kind. -- `summary` and `"exponential histogram"` kinds remain out of scope (no query renderer support yet). diff --git a/.changeset/mcp-number-tile-color.md b/.changeset/mcp-number-tile-color.md deleted file mode 100644 index c83f70fe00..0000000000 --- a/.changeset/mcp-number-tile-color.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -Support number-tile color in the MCP dashboard tools. `save_dashboard` and `patch_dashboard` now accept a static `color` and conditional `colorRules` on builder number tiles, and a static `color` on raw SQL number tiles, matching the external REST dashboards API. diff --git a/.changeset/mcp-objectid-validation.md b/.changeset/mcp-objectid-validation.md deleted file mode 100644 index 6087e50143..0000000000 --- a/.changeset/mcp-objectid-validation.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -refactor(mcp): simplify ObjectId validation with shared helpers and schema-level checks - -Add `mcpError()` and `validateObjectId()` utilities to reduce boilerplate -across MCP tool handlers. Move ObjectId validation into Zod input schemas -for always-required ID fields, eliminating inline checks entirely. Remaining -conditional checks use the new one-liner helper. diff --git a/.changeset/mcp-orderby-alias-quoting.md b/.changeset/mcp-orderby-alias-quoting.md deleted file mode 100644 index 8d8ea67b4c..0000000000 --- a/.changeset/mcp-orderby-alias-quoting.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -fix(mcp): quote multi-word aliases in orderBy and steer event-pattern usage - -Quote resolved aliases that are not bare identifiers (e.g. `"P95 Latency"`) -in `resolveOrderBy` output, in both the direct alias-match and aggFn-match -paths. Previously an unquoted multi-word alias produced SQL-invalid -`ORDER BY` output. Incoming orderBy values are stripped of surrounding -double-quote/backtick quoting before matching, so agents that already quote -the alias resolve correctly without being double-quoted. - -Also document the alias-quoting requirement in the `orderBy` schema -descriptions, and update the `clickstack_event_patterns` tool description to -steer agents toward it (over `clickstack_search` / `clickstack_table`) when -exploring what messages, errors, or events exist. diff --git a/.changeset/mcp-search-denoise.md b/.changeset/mcp-search-denoise.md deleted file mode 100644 index 53b5495980..0000000000 --- a/.changeset/mcp-search-denoise.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -'@hyperdx/api': patch -'@hyperdx/app': patch -'@hyperdx/common-utils': patch ---- - -feat(mcp): add denoise option to clickstack_search tool - -Add a `denoise` boolean parameter to the MCP `clickstack_search` tool that -automatically filters out high-frequency repetitive event patterns from -search results, mirroring the web app's "Denoise Results" feature. - -When enabled, the tool samples 10k random events, mines patterns using -the Drain algorithm, identifies noisy patterns (>10% of sample), and -filters them out of result rows. Returns filtered rows plus metadata -listing removed patterns with estimated counts. - -Extracts shared denoise constants (`DENOISE_SAMPLE_SIZE`, -`DENOISE_NOISE_THRESHOLD`) into `@hyperdx/common-utils` so the web app -and MCP server use the same values. diff --git a/.changeset/mcp-sql-tile-macro-hints.md b/.changeset/mcp-sql-tile-macro-hints.md deleted file mode 100644 index 1bf70fbf95..0000000000 --- a/.changeset/mcp-sql-tile-macro-hints.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/api": patch -"@hyperdx/common-utils": patch ---- - -fix: Nudge agents towards macros in raw SQL tiles diff --git a/.changeset/metric-table-autofill.md b/.changeset/metric-table-autofill.md deleted file mode 100644 index d8a37040d2..0000000000 --- a/.changeset/metric-table-autofill.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat: auto-fill metric table dropdowns when creating a Metrics source - -The 5 metric-table dropdowns (Gauge, Histogram, Sum, Summary, Exponential -Histogram) now auto-populate by matching table names in the selected database -to their metric type via suffix patterns. Prefers `otel_metrics_` prefixed -names, never overwrites user selections, and shows a green notification on -successful autofill. diff --git a/.changeset/nasty-crews-shop.md b/.changeset/nasty-crews-shop.md deleted file mode 100644 index 0b53ead24f..0000000000 --- a/.changeset/nasty-crews-shop.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -fix(mcp): add ClickHouse safety settings (max_execution_time, max_result_rows, readonly) for MCP query execution diff --git a/.changeset/nodenext-module-resolution.md b/.changeset/nodenext-module-resolution.md deleted file mode 100644 index f50ed4c1b0..0000000000 --- a/.changeset/nodenext-module-resolution.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -chore: upgrade moduleResolution to NodeNext and simplify clickhouseProxy static import diff --git a/.changeset/number-tile-background-chart-external-api.md b/.changeset/number-tile-background-chart-external-api.md deleted file mode 100644 index 653ec8f4f8..0000000000 --- a/.changeset/number-tile-background-chart-external-api.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -Add `backgroundChart` support to number tiles in the external dashboards API (`/api/v2/dashboards`). Builder number tiles can now carry an optional background trend sparkline (`type` line or area, with an optional palette-token `color`) over the v2 REST API, matching the dashboard editor. Raw SQL number tiles do not support it. diff --git a/.changeset/number-tile-background-chart.md b/.changeset/number-tile-background-chart.md deleted file mode 100644 index 07eb9f2929..0000000000 --- a/.changeset/number-tile-background-chart.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -'@hyperdx/app': minor -'@hyperdx/common-utils': minor ---- - -feat(dashboards): add a background trend sparkline to number tiles - -Number tiles can now render a faint line or area sparkline behind the value, -derived from a time-bucketed version of the same query, so the value's trend -over the selected range is visible at a glance. This is handy for SLO / -error-budget tiles where the burn over time matters as much as the current -number. The sparkline inherits the tile's color by default and can be -overridden to any palette token. Configure it under Display Settings > -Background chart on a number tile. Available on builder number tiles (raw SQL -number tiles return a single value with no time dimension to bucket). diff --git a/.changeset/number-tile-color-external-api.md b/.changeset/number-tile-color-external-api.md deleted file mode 100644 index 3739b66c4f..0000000000 --- a/.changeset/number-tile-color-external-api.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": minor ---- - -Support number-tile color authoring through the external dashboards API. The v2 REST API and OpenAPI spec now accept `color` (a palette token) and `colorRules` (ordered conditional color rules, last match wins) on builder number tiles, and `color` on raw SQL number tiles, matching what the in-product number-tile editor persists. Color rules accept the numeric and equality operators the editor offers (`gt`, `gte`, `lt`, `lte`, `between`, `eq`, `neq`). Existing dashboards keep working: tiles saved before the palette was renamed to hue names are normalized to the current token names on read. diff --git a/.changeset/number-tile-format-survives-apply.md b/.changeset/number-tile-format-survives-apply.md deleted file mode 100644 index 709f98becd..0000000000 --- a/.changeset/number-tile-format-survives-apply.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix(dashboards): keep the auto-detected number format when applying display settings - -Opening Display Settings on a number tile that auto-detects its format from the -datasource (for example p95 of a trace Duration column) and clicking Apply no -longer rewrites the format to Number. The drawer now reflects the -datasource-derived format, and Apply persists `numberFormat` only when the user -explicitly changes it; otherwise it stays unset so render-time auto-detection -keeps driving the format. diff --git a/.changeset/number-tile-sparkline-groupby.md b/.changeset/number-tile-sparkline-groupby.md deleted file mode 100644 index 375d52826c..0000000000 --- a/.changeset/number-tile-sparkline-groupby.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix(dashboards): match the number-tile background sparkline to the displayed value - -The big number on a number tile is a single aggregate (its query drops `groupBy`), but the background sparkline kept any `groupBy` the tile carried over from a prior Line display type. It then plotted only the first group's trend behind a value that aggregates every group. The sparkline now drops `groupBy` as well, so its trend reflects the same single series as the value it sits behind. diff --git a/.changeset/observability-instrumentation-rollout.md b/.changeset/observability-instrumentation-rollout.md deleted file mode 100644 index f5d2b660be..0000000000 --- a/.changeset/observability-instrumentation-rollout.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -'@hyperdx/api': minor ---- - -Extend observability instrumentation to the remaining API surfaces using the -shared helpers. Add custom metrics and tracing to previously log-only paths: -OpAMP message handling (message outcomes, agent status reports, remote configs -sent), the Prometheus proxy router (query duration + swallowed-error counters -labeled by endpoint and backend), alert webhook/notification delivery (delivery -attempts and duration labeled by service and outcome), and MongoDB connection -lifecycle events. - -Add a reusable SLO primitive (`withOperationMetrics` / `recordOperationOutcome`) -that emits standard availability + latency SLIs (`hyperdx.operation.requests` -and `hyperdx.operation.duration_ms`, labeled by `operation` and `outcome`) so -SLOs can be defined per piece of application functionality. Apply it to the AI -assistant generation call, the ClickHouse proxy (query passthrough + -connection test), and alert processing — both the end-to-end alert evaluation -(`alerts.evaluate`, excluding scheduling skips) and its ClickHouse data fetch -(`alerts.query`) — paths whose failures previously surfaced only as logs or -failures-only counters with no latency or denominator. diff --git a/.changeset/observability-instrumentation-uplift.md b/.changeset/observability-instrumentation-uplift.md deleted file mode 100644 index a20e705a7c..0000000000 --- a/.changeset/observability-instrumentation-uplift.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -'@hyperdx/api': minor ---- - -Improve API observability instrumentation. Add a centralized tracing + metrics -helper library (`withSpan`, `setBusinessContext`, `getStaticFeatureFlags`, -memoized `getCounter`/`getHistogram`, `recordDuration`), attach consistent -team/user/feature-flag context to traces across all auth paths (session, -access-key, local mode), and add custom metrics for previously log-only hot -paths: API errors, alert evaluation outcomes/query/process failures, and -external API search/charts query duration and errors. diff --git a/.changeset/onclick-external-link-variant.md b/.changeset/onclick-external-link-variant.md deleted file mode 100644 index ee51e5b194..0000000000 --- a/.changeset/onclick-external-link-variant.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/common-utils": minor -"@hyperdx/app": minor -"@hyperdx/api": minor ---- - -feat: Add an "external link" row-click action for dashboard table tiles diff --git a/.changeset/page-header-z-index-below-drawers.md b/.changeset/page-header-z-index-below-drawers.md deleted file mode 100644 index 31d4dd178e..0000000000 --- a/.changeset/page-header-z-index-below-drawers.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -fix(z-index): keep sticky header below drawers and drawers above the fullscreen tile modal - -Two related z-index regressions: - -- `PageHeader` was pinned at `z-index: 100`, but app drawers opt into a - much lower stack via `ZIndexContext` (`contextZIndex + 10`, so a - top-level drawer renders at `z-index: 10`). The sticky header therefore - floated above the drawer overlay. The header now sits at `z-index: 2` so - drawer overlays reliably cover the page chrome while the header still - wins against normal scrolling content. -- `FullscreenPanelModal` used Mantine's default modal z-index (`200`) and - didn't propagate it through `ZIndexContext`. Clicking a row in a - fullscreen search tile opened a `DBRowSidePanel` drawer at `z-index: 10` - that was hidden behind the modal. The modal now follows the existing - `contextZIndex + 10` pattern and wraps its children in a - `ZIndexContext.Provider`, so child drawers stack on top of it. diff --git a/.changeset/page-migrate-dashboard.md b/.changeset/page-migrate-dashboard.md deleted file mode 100644 index dc70c2ec0a..0000000000 --- a/.changeset/page-migrate-dashboard.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -chore: migrate the custom Dashboard page to shared `PageLayout` / `PageHeader`. Breadcrumbs, the editable dashboard name, dashboard actions (Favorite, Tags, Menu), and the "Created by … Updated …" meta now live in a single page header, while the query toolbar (SQL/Lucene WHERE, time range, granularity, Live, refresh, edit filters, Run) is pinned to the top of the scroll container as a dedicated sticky row — the chrome above scrolls away and only the toolbar follows the user. The "Updated …" meta moves to the right side of the breadcrumbs row instead of sitting as a separate body line. - -`PageHeader` gains a `stickyRow` slot that any page can use to declare a single row that should be the only pinned element, with the rest of the header treated as scrolling chrome. Other pages are unaffected — a `PageHeader` without `stickyRow` keeps the existing fully-sticky behavior. diff --git a/.changeset/pattern-table-alias-with.md b/.changeset/pattern-table-alias-with.md deleted file mode 100644 index 7d37aa7fef..0000000000 --- a/.changeset/pattern-table-alias-with.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -fix(search): keep select-alias filters working in Event Patterns - -Filtering on a column the source exposes only under an alias (for example a -default select of `ServiceName as service`) failed in the Event Patterns view -with `Unknown expression or table expression identifier 'service'`. The -results table works because its own SELECT defines the alias, but Event -Patterns rebuilds the SELECT and did not carry the alias definitions. The -pattern query now receives the same alias `WITH` clauses already threaded into -the results, histogram, and heatmap queries, so the filter resolves. diff --git a/.changeset/polite-moons-give.md b/.changeset/polite-moons-give.md deleted file mode 100644 index 8ee1597d14..0000000000 --- a/.changeset/polite-moons-give.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -feat: add source field suggestions diff --git a/.changeset/ratio-number-tiles.md b/.changeset/ratio-number-tiles.md deleted file mode 100644 index c1cbb65897..0000000000 --- a/.changeset/ratio-number-tiles.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": minor ---- - -Number chart tiles now support a second series with the "As Ratio" toggle (`series[0] / series[1]`), matching line and bar charts. Combined with a `percent` number format, this renders a percentage (e.g. success/error rate) as a single big number with the trend sparkline behind it. diff --git a/.changeset/raw-sql-chart-shortcut.md b/.changeset/raw-sql-chart-shortcut.md deleted file mode 100644 index 4fb19b88d7..0000000000 --- a/.changeset/raw-sql-chart-shortcut.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Add Cmd/Ctrl+Enter support for running raw SQL chart queries from the SQL editor. diff --git a/.changeset/remove-mcp-max-result-rows.md b/.changeset/remove-mcp-max-result-rows.md deleted file mode 100644 index 4c5d65505a..0000000000 --- a/.changeset/remove-mcp-max-result-rows.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -fix(mcp): remove max_result_rows from MCP safety settings - -Remove the hardcoded max_result_rows=100000 setting from MCP query -execution. Some ClickHouse connections impose profile constraints that -cap max_result_rows below our default, causing SETTING_CONSTRAINT_VIOLATION -errors. The remaining safety settings (max_execution_time=30, readonly=2) -and trimToolResponse provide sufficient protection. - -Add a SETTING_CONSTRAINT_VIOLATION error hint so constrained settings -surface actionable guidance instead of raw ClickHouse errors. diff --git a/.changeset/rename-chart-palette-tokens.md b/.changeset/rename-chart-palette-tokens.md deleted file mode 100644 index cea4e38ac2..0000000000 --- a/.changeset/rename-chart-palette-tokens.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -'@hyperdx/common-utils': minor -'@hyperdx/app': minor -'@hyperdx/api': patch ---- - -refactor(theme): rename chart palette tokens from chart-1..10 to hue-named -(chart-blue, chart-orange, ...) and unify the categorical palette across HyperDX -and ClickStack - -Stored configs from the initial color picker (#2265) keep working. -`ChartPaletteTokenSchema` stays strict (a plain `z.enum`, so its `z.input` -matches `z.output` — wrapping it in `z.preprocess` would poison -`validateRequest`'s `req.body` inference all the way up to -`Dashboard.tiles[i].config.color`). Migration of legacy `chart-1` .. `chart-10` -happens at five complementary points so no entry or wire-format path can slip -through, all composing over a single shared walker -(`walkRawDashboardTileColors` in `common-utils`) so the per-tile traversal -stays in lockstep: - -- **Fetch-time / write-time (React)**: `normalizeDashboardTileColors` in - `packages/app/src/dashboard.ts` heals dashboards on read - (`useDashboards` / `fetchLocalDashboards` / `fetchDashboards`) and on write - (`useUpdateDashboard` / `useCreateDashboard`). Unresolvable color strings - (stale hexes, hand-edited values, forward-rolled future tokens) are - preserved so the user's chosen value survives a render pass — the strict - server-side schema surfaces a clear error on next save instead of the - normalizer quietly dropping the field. -- **JSON import**: `DBDashboardImportPage` runs - `normalizeRawDashboardTileColors` on the parsed JSON *before* the strict - `DashboardTemplateSchema.safeParse`, so templates exported from a - pre-rename deploy import cleanly. -- **Server-side GET response healing**: `getDashboards` / `getDashboard` in - `packages/api/src/controllers/dashboard.ts` rewrite legacy tile colors on - the way out. Pre-rename Mongo docs are served on the wire as - hue-named tokens so non-React HTTP clients (CI scripts, stale bundle - tabs during a rolling deploy, the external API) can round-trip - GET → PATCH without ever resurrecting `chart-N` through the strict - schema. -- **Server-side write shim**: the dashboards POST / PATCH routes mount - a request-body preprocessor that rewrites legacy tile colors before - `validateRequest` runs `ChartPaletteTokenSchema`. Catches non-React - HTTP callers (stale-bundle tabs during a rolling deploy, CI scripts, - MCP, the upcoming external-API parity work) for a one-release - deprecation window without weakening the schema's input/output equality. - The dashboard provisioner task applies the same shim before parsing - on-disk template files. -- **Render-time (belt-and-suspenders)**: `DBNumberChart` and - `ColorSwatchInput` also call `resolveChartPaletteToken` for tiles - constructed in memory between fetch and save (`ChartEditor` form - state, unit-test fixtures, hand-rolled `Tile` literals). - -The migration preserves the HyperDX slot ordering from #2265 (slot 1 = brand -green, slot 2 = blue, etc.). - -**ClickStack legacy color caveat:** Pre-rename ClickStack used a different slot -ordering than HyperDX (`--color-chart-1` was brand blue `#437eef`, not brand -green). The migration map uses HyperDX slot ordering, so any ClickStack -dashboard saved via #2265 with `color: 'chart-1'` will flip from blue to -Observable green after migration. We chose this trade-off deliberately over -branching the legacy map by active theme: `LEGACY_CHART_PALETTE_TOKEN_MAP` lives -in `common-utils` (shared with the API), and migration is one-shot persisted on -next save — theme-branching would couple common-utils to browser DOM state and -still produce wrong results for users whose active theme changed since the -original pick. Affected users can manually re-pick the desired hue via the (now -hue-labeled) color picker. - -The categorical palette is based on Observable 10, with `chart-blue` swapped to -`#437eef` to match the brand link color -(`--click-global-color-text-link-default`); all other hues are straight from -Observable 10. The palette resolves identically on both themes — picking -`chart-blue` always renders the brand blue. Brand identity for charts moves -entirely into the semantic layer: `--color-chart-success` and `--color-chart-info` -resolve to categorical `chart-green` (`#3ca951`) and `chart-blue` (`#437eef`) on -both HyperDX and ClickStack, so success fills, info-level logs, and the -matching multi-series slots all read consistently across brands. - -Internally, JS (`CATEGORICAL_HEX_BY_TOKEN` in `packages/app/src/utils.ts`) is -the source of truth for categorical hues — `getColorFromCSSVariable` and -`getColorFromCSSToken` skip `getComputedStyle` for categorical tokens since the -palette is unified across themes. The matching `--color-chart-{hue}` CSS vars in -`_tokens.scss` remain as a stylesheet-author affordance (inline `var()` use, -devtools inspection) and a hook for any future per-brand override. Semantic -tokens still resolve through `getComputedStyle` because they genuinely vary per -theme. diff --git a/.changeset/rename-mcp-tools-to-clickstack.md b/.changeset/rename-mcp-tools-to-clickstack.md deleted file mode 100644 index 70347fa33c..0000000000 --- a/.changeset/rename-mcp-tools-to-clickstack.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -"@hyperdx/api": patch -"@hyperdx/common-utils": patch ---- - -refactor(mcp): rename all MCP tool prefixes from `hyperdx_` to `clickstack_` - -Rename the MCP server name from `hyperdx` to `clickstack` and update all 19 -tool names (e.g. `hyperdx_search` → `clickstack_search`), along with -descriptions, prompts, error messages, and test references. diff --git a/.changeset/selfish-oranges-cheer.md b/.changeset/selfish-oranges-cheer.md deleted file mode 100644 index a29fc26031..0000000000 --- a/.changeset/selfish-oranges-cheer.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/hdx-eval": patch -"@hyperdx/api": patch -"@hyperdx/app": patch -"@hyperdx/cli": patch ---- - -Removing relative imports and using path aliases diff --git a/.changeset/serious-gifts-rest.md b/.changeset/serious-gifts-rest.md deleted file mode 100644 index 2d434ae3aa..0000000000 --- a/.changeset/serious-gifts-rest.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch ---- - -fix: fallback to body or implicit column expression when other empty diff --git a/.changeset/service-map-percentiles-filtering.md b/.changeset/service-map-percentiles-filtering.md deleted file mode 100644 index 2e80fab9b0..0000000000 --- a/.changeset/service-map-percentiles-filtering.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat(service-map): server-side filtering, latency percentiles, throughput & focus - -The Service Map gains server-side filtering (Lucene/SQL `where` plus a -service-name multi-select with inbound/outbound neighbor expansion), latency -percentiles (p50/p95/p99) and request throughput (req/s) in node and edge -tooltips, a "Focus" action to scope the map to a service and its immediate -dependencies, and node sizing by total throughput (incoming + outgoing). -Percentiles are computed server-side via a single GROUPING SETS query. diff --git a/.changeset/silly-toes-agree.md b/.changeset/silly-toes-agree.md deleted file mode 100644 index de1482b26c..0000000000 --- a/.changeset/silly-toes-agree.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: next-runtime-env runtime env var injection fixed for images diff --git a/.changeset/smooth-turkeys-dream.md b/.changeset/smooth-turkeys-dream.md deleted file mode 100644 index ddfa44ee5a..0000000000 --- a/.changeset/smooth-turkeys-dream.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: Fix height of source select RHS menu diff --git a/.changeset/solid-pigs-rhyme.md b/.changeset/solid-pigs-rhyme.md deleted file mode 100644 index 344bcb74b8..0000000000 --- a/.changeset/solid-pigs-rhyme.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Transition the local development server from Webpack to Turbopack to -significantly improve build performance and hot-reloading speed. diff --git a/.changeset/source-chip-kebab-menu.md b/.changeset/source-chip-kebab-menu.md deleted file mode 100644 index 9ec99239ab..0000000000 --- a/.changeset/source-chip-kebab-menu.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat(source-picker): chip + kebab menu UX diff --git a/.changeset/source-form-dropdown-clipping.md b/.changeset/source-form-dropdown-clipping.md deleted file mode 100644 index 08f54d1c83..0000000000 --- a/.changeset/source-form-dropdown-clipping.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Fix the database, table, and connection dropdowns being clipped inside the source setup modal. The dropdowns now render in a portal, so the full list is visible and scrollable when configuring or editing a source. diff --git a/.changeset/source-section-autocomplete.md b/.changeset/source-section-autocomplete.md deleted file mode 100644 index 83c0e93cd8..0000000000 --- a/.changeset/source-section-autocomplete.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Suggest existing section names in the source form's **Section** field. The field is now an autocomplete fed by the sections already in use, so a new source can reuse an existing section instead of retyping it (which is how a section ends up split into near-duplicates like "Billing" and "billing"). The field stays free-text, so any new section name is still accepted. diff --git a/.changeset/source-section-field.md b/.changeset/source-section-field.md deleted file mode 100644 index e37cc5c64d..0000000000 --- a/.changeset/source-section-field.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -feat: add an optional Section field to data sources - -Sources can now carry an optional free-text Section label, set from the source -settings form. The value is persisted and returned by GET /api/v2/sources, so -external API consumers can read it. This lays the groundwork for grouping and -searching sources by section in the source selector. diff --git a/.changeset/source-selector-section-grouping.md b/.changeset/source-selector-section-grouping.md deleted file mode 100644 index 54616cb920..0000000000 --- a/.changeset/source-selector-section-grouping.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Group the data source selector by section and add tag-style search. When sources have a Section assigned, the selector lists them under section headers; search matches on both the source name and its section, so a section name acts as a tag (typing "billing" returns every source in the Billing section, including ones whose name does not contain "billing"). The selector stays flat until at least one source has a section, so deployments that have not adopted sections see no change. The grouped dropdown is also widened and pinned to the picker's left edge so section headers and source names are not cramped. diff --git a/.changeset/sources-list-section.md b/.changeset/sources-list-section.md deleted file mode 100644 index bf1ec1a4c2..0000000000 --- a/.changeset/sources-list-section.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Show each source's **Section** on the Manage Sources list. A sectioned source now displays its section, with a folder icon, in the dimmed metadata row alongside its connection and table, so the list mirrors the grouped selector. Sources without a section are unchanged. diff --git a/.changeset/spotty-roses-cheer.md b/.changeset/spotty-roses-cheer.md deleted file mode 100644 index da054a313c..0000000000 --- a/.changeset/spotty-roses-cheer.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -feat: Add Known Columns List setting for distributed tables diff --git a/.changeset/standalone-promql-conditional.md b/.changeset/standalone-promql-conditional.md deleted file mode 100644 index 58d904538f..0000000000 --- a/.changeset/standalone-promql-conditional.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -'@hyperdx/otel-collector': patch ---- - -fix(otel-collector): only enable the prometheus remote-write exporter in -standalone mode when `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` is set - -The standalone collector config used to unconditionally declare a -`prometheusremotewrite` exporter and a `metrics/promql` pipeline. When -`CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` was unset the exporter rendered -with an empty endpoint and every metrics batch failed to export. - -The exporter and pipeline have been moved to -`docker/otel-collector/config.standalone.promql.yaml`, which is now only -loaded by `entrypoint.sh` when `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` is -non-empty. This mirrors the OpAMP-managed gating in -`packages/api/src/opamp/controllers/opampController.ts` (which already -only adds the exporter when `IS_PROMQL_ENABLED` is true). - -No action required if `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` is set; the -behavior is unchanged. If it was unset, the collector now stops emitting -the failing prometheus-remote-write attempts. diff --git a/.changeset/tile-alert-attributes.md b/.changeset/tile-alert-attributes.md deleted file mode 100644 index 14f6c8a7cd..0000000000 --- a/.changeset/tile-alert-attributes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/api": patch ---- - -fix(alerts): populate `{{attributes.*}}` template variables for tile/chart alerts from group-by fields diff --git a/.changeset/time-chart-series-limit.md b/.changeset/time-chart-series-limit.md deleted file mode 100644 index 27aae22282..0000000000 --- a/.changeset/time-chart-series-limit.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch -"@hyperdx/api": patch ---- - -feat(charts): cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys. The cap defaults to 100 (the number of series rendered) and is configurable per team via a new "Time Chart Series Limit" setting; series beyond the cap remain available in the series selector. diff --git a/.changeset/tooltip-highlight-nearest-series.md b/.changeset/tooltip-highlight-nearest-series.md deleted file mode 100644 index c10b29ffcf..0000000000 --- a/.changeset/tooltip-highlight-nearest-series.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -feat: emphasize the series nearest the cursor in multi-series time charts. The nearest line is thickened and the others fade back, and its tooltip row is bolded while the rest dim, so a value is easy to trace back to its line. diff --git a/.changeset/tough-ads-pay.md b/.changeset/tough-ads-pay.md deleted file mode 100644 index 792a3b8c3d..0000000000 --- a/.changeset/tough-ads-pay.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": minor ---- - -feat: trace panel inline split detail diff --git a/.changeset/type-json-metadata-filter-attributes.md b/.changeset/type-json-metadata-filter-attributes.md deleted file mode 100644 index 0b4174904c..0000000000 --- a/.changeset/type-json-metadata-filter-attributes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/common-utils': patch ---- - -Type JSON metadata filter attribute paths before value sampling. diff --git a/.changeset/variadic-cityhash-aligns.md b/.changeset/variadic-cityhash-aligns.md deleted file mode 100644 index 94e6ccff6f..0000000000 --- a/.changeset/variadic-cityhash-aligns.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -'@hyperdx/common-utils': patch -'@hyperdx/api': patch ---- - -refactor(metrics): unify AttributesHash to variadic cityHash64 across Map and -JSON metric schemas - -Sum / Gauge / Histogram metric queries now compute AttributesHash as -`cityHash64(ScopeAttributes, ResourceAttributes, Attributes)` for both -Map(LowCardinality(String), String) and JSON attribute columns. Previously -the Map-schema path wrapped the three maps in `mapConcat()` before hashing, -and the JSON-schema path used the variadic form; the schema-detection -ClickHouse round-trip and the `attrHashExpr` helper / `isJsonSchema` -plumbing are gone. - -Compatibility: -- Per-row AttributesHash values change for every Map-schema metric row, - but the hash is recomputed inside CTEs on every query — no materialized - view, projection, ALIAS column, or cache persists it, so no downstream - consumer is affected (audit: OSS only). -- Cross-scope same-key behaviour shifts: two rows that carry the same - logical key in different attribute scopes (e.g. `host` in - `ResourceAttributes` for one emission and `host` in `Attributes` for the - next) now hash distinctly and land in separate series. Previously the - mapConcat path collapsed them into one series. This only matters when an - OTel collector processor promotes attributes across scopes mid-stream; - most SDKs emit attributes in stable scopes. The new behaviour is captured - by an integration test in `packages/api/src/clickhouse/__tests__`. - -HDX-4466. diff --git a/.changeset/wet-ties-kneel.md b/.changeset/wet-ties-kneel.md deleted file mode 100644 index 11aa39b0a1..0000000000 --- a/.changeset/wet-ties-kneel.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -chore: Make error states consistent across chart types diff --git a/.changeset/wise-jobs-relax.md b/.changeset/wise-jobs-relax.md deleted file mode 100644 index f860cc1e17..0000000000 --- a/.changeset/wise-jobs-relax.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -fix: bug preventing deletion of nested subdocuments like metadataMVs diff --git a/.env b/.env index 00d04c800b..72f542411b 100644 --- a/.env +++ b/.env @@ -8,8 +8,8 @@ NEXT_ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=clickhouse/clickstack-all-in-one ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=clickhouse/clickstack-otel-collector OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector -CODE_VERSION=2.28.0 -IMAGE_VERSION_SUB_TAG=.28.0 +CODE_VERSION=2.29.0 +IMAGE_VERSION_SUB_TAG=.29.0 IMAGE_VERSION=2 IMAGE_NIGHTLY_TAG=2-nightly IMAGE_LATEST_TAG=latest diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 5f4e8b8e17..0991b232fa 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,300 @@ # @hyperdx/api +## 2.29.0 + +### Minor Changes + +- 9119de5f: Add unique MongoDB index on accessKey field in User model to eliminate full collection scans during API key authentication. This could cause startup failures if any existing users share duplicate accessKey values. +- 9f23b7e58: Feat: Added a V2 endpoint for team management. +- 5cd709020: Add UI support for configuring an external Prometheus-compatible endpoint on a + connection. Modify Connections model to now have a boolean + `isPrometheusEndpoint` field and use host for storing the host. +- b798f91f: Add connection management endpoints to the external API (`/api/v2/connections`), supporting list, get, create, update, and delete with Bearer token authentication. Passwords are write-only and never returned by the API. +- 63469fe0e: feat(mcp): first-class metric source support + + - Two new tools: `clickstack_list_metrics` paginates the metric-name catalog with optional kind / namePattern (ILIKE) / time-window filters and opaque cursor pagination; `clickstack_describe_metric` returns per-metric kind(s), unit, description, attribute keys, and sampled values (with kind auto-detection). + - `clickstack_describe_source` is metric-aware: picks a representative metric table (gauge → sum → histogram), runs column / map-key / value-sampling against it, and adds a per-kind metric-name sample. + - `clickstack_timeseries` and `clickstack_table` accept `metricType` (gauge / sum / histogram), `metricName`, and `isDelta` on each select item, plus `aggFn:"increase"` for Sum counters. `valueExpression` defaults to `"Value"` for metric sources. Surfaces the renderer's 20-group top-N cap on `increase + groupBy` as a neutral hint. + - Dashboard prompt's "use raw SQL for metric tiles" workaround is replaced with positive discovery-workflow guidance and one worked example per supported kind. + - `summary` and `"exponential histogram"` kinds remain out of scope (no query renderer support yet). + +- f126d5b1: Support number-tile color authoring through the external dashboards API. The v2 REST API and OpenAPI spec now accept `color` (a palette token) and `colorRules` (ordered conditional color rules, last match wins) on builder number tiles, and `color` on raw SQL number tiles, matching what the in-product number-tile editor persists. Color rules accept the numeric and equality operators the editor offers (`gt`, `gte`, `lt`, `lte`, `between`, `eq`, `neq`). Existing dashboards keep working: tiles saved before the palette was renamed to hue names are normalized to the current token names on read. +- ebfc2e80a: Extend observability instrumentation to the remaining API surfaces using the + shared helpers. Add custom metrics and tracing to previously log-only paths: + OpAMP message handling (message outcomes, agent status reports, remote configs + sent), the Prometheus proxy router (query duration + swallowed-error counters + labeled by endpoint and backend), alert webhook/notification delivery (delivery + attempts and duration labeled by service and outcome), and MongoDB connection + lifecycle events. + + Add a reusable SLO primitive (`withOperationMetrics` / `recordOperationOutcome`) + that emits standard availability + latency SLIs (`hyperdx.operation.requests` + and `hyperdx.operation.duration_ms`, labeled by `operation` and `outcome`) so + SLOs can be defined per piece of application functionality. Apply it to the AI + assistant generation call, the ClickHouse proxy (query passthrough + + connection test), and alert processing — both the end-to-end alert evaluation + (`alerts.evaluate`, excluding scheduling skips) and its ClickHouse data fetch + (`alerts.query`) — paths whose failures previously surfaced only as logs or + failures-only counters with no latency or denominator. + +- bbc29859d: Improve API observability instrumentation. Add a centralized tracing + metrics + helper library (`withSpan`, `setBusinessContext`, `getStaticFeatureFlags`, + memoized `getCounter`/`getHistogram`, `recordDuration`), attach consistent + team/user/feature-flag context to traces across all auth paths (session, + access-key, local mode), and add custom metrics for previously log-only hot + paths: API errors, alert evaluation outcomes/query/process failures, and + external API search/charts query duration and errors. +- 17e1eb19d: feat: Add an "external link" row-click action for dashboard table tiles + +### Patch Changes + +- 998ea5d0: feat: Add option to fit time chart y-axis lower bound +- 0497ca5dd: Bump http-proxy-middleware to v4, replacing http-proxy with httpxy +- ee907386: fix: Add sourceId to MCP Raw SQL Tile schema +- 9a7e392a: fix: Add missing numberFormats, compareToPreviousPeriod fields to MCP Schemas +- cdd7ca07: fix(mcp): reduce describe_source timeouts by using rollup tables for map key discovery +- d11991b0c: fix: enforce password complexity on team invite acceptance +- 8261b461: fix: inline parametric aggregate function arguments instead of passing as query parameters +- 973d1201b: fix: polish promql experience across the app +- 8164492f: fix(mcp): improve alias field descriptions and examples for readable chart legends +- a19ba549: feat(mcp): add patch_dashboard, get_dashboard_tile, search_dashboards tools + + Add three new MCP dashboard tools for granular operations: + + - `hyperdx_get_dashboard_tile` — retrieve a single tile by tileId + - `hyperdx_patch_dashboard` — update name/tags and/or replace one tile + without resubmitting the full dashboard + - `hyperdx_search_dashboards` — search by name and/or tags + + Fix empty parameter schema on patch/search tools caused by Zod + `.refine()` wrapping. Document Lucene substring matching limitations + prominently in tool descriptions and query guide prompt. + + **Breaking (minor):** Tile `name` on `hyperdx_save_dashboard` now requires + at least 1 character (`.min(1)`). Previously empty string `""` was accepted + and silently persisted as a blank title. Callers sending `name: ""` will + now receive a validation error. + +- 7e7159a5: fix(mcp): improve error hints and fix readonly mode for query safety settings + + Switch MCP ClickHouse safety settings from readonly=1 to readonly=2 so + max_execution_time and max_result_rows are actually applied (readonly=1 + silently rejects all setting changes). + + Improve DateTime64 cast error hint to recommend parseDateTime64BestEffort() + which works on both DateTime and DateTime64 columns, replacing + toDateTime64() which only works on DateTime64. + + Add error hint for unknown column/identifier errors directing agents to + call describe_source before retrying. + +- f34a31fdc: Support number-tile color in the MCP dashboard tools. `save_dashboard` and `patch_dashboard` now accept a static `color` and conditional `colorRules` on builder number tiles, and a static `color` on raw SQL number tiles, matching the external REST dashboards API. +- f6bda8c5: refactor(mcp): simplify ObjectId validation with shared helpers and schema-level checks + + Add `mcpError()` and `validateObjectId()` utilities to reduce boilerplate + across MCP tool handlers. Move ObjectId validation into Zod input schemas + for always-required ID fields, eliminating inline checks entirely. Remaining + conditional checks use the new one-liner helper. + +- f326ccf8: fix(mcp): quote multi-word aliases in orderBy and steer event-pattern usage + + Quote resolved aliases that are not bare identifiers (e.g. `"P95 Latency"`) + in `resolveOrderBy` output, in both the direct alias-match and aggFn-match + paths. Previously an unquoted multi-word alias produced SQL-invalid + `ORDER BY` output. Incoming orderBy values are stripped of surrounding + double-quote/backtick quoting before matching, so agents that already quote + the alias resolve correctly without being double-quoted. + + Also document the alias-quoting requirement in the `orderBy` schema + descriptions, and update the `clickstack_event_patterns` tool description to + steer agents toward it (over `clickstack_search` / `clickstack_table`) when + exploring what messages, errors, or events exist. + +- 750b8afe: feat(mcp): add denoise option to clickstack_search tool + + Add a `denoise` boolean parameter to the MCP `clickstack_search` tool that + automatically filters out high-frequency repetitive event patterns from + search results, mirroring the web app's "Denoise Results" feature. + + When enabled, the tool samples 10k random events, mines patterns using + the Drain algorithm, identifies noisy patterns (>10% of sample), and + filters them out of result rows. Returns filtered rows plus metadata + listing removed patterns with estimated counts. + + Extracts shared denoise constants (`DENOISE_SAMPLE_SIZE`, + `DENOISE_NOISE_THRESHOLD`) into `@hyperdx/common-utils` so the web app + and MCP server use the same values. + +- caba7c255: fix: Nudge agents towards macros in raw SQL tiles +- f113ea36: fix(mcp): add ClickHouse safety settings (max_execution_time, max_result_rows, readonly) for MCP query execution +- 634101c33: chore: upgrade moduleResolution to NodeNext and simplify clickhouseProxy static import +- ba626ef96: Add `backgroundChart` support to number tiles in the external dashboards API (`/api/v2/dashboards`). Builder number tiles can now carry an optional background trend sparkline (`type` line or area, with an optional palette-token `color`) over the v2 REST API, matching the dashboard editor. Raw SQL number tiles do not support it. +- 60a91e43: fix(mcp): remove max_result_rows from MCP safety settings + + Remove the hardcoded max_result_rows=100000 setting from MCP query + execution. Some ClickHouse connections impose profile constraints that + cap max_result_rows below our default, causing SETTING_CONSTRAINT_VIOLATION + errors. The remaining safety settings (max_execution_time=30, readonly=2) + and trimToolResponse provide sufficient protection. + + Add a SETTING_CONSTRAINT_VIOLATION error hint so constrained settings + surface actionable guidance instead of raw ClickHouse errors. + +- e03971b0: refactor(theme): rename chart palette tokens from chart-1..10 to hue-named + (chart-blue, chart-orange, ...) and unify the categorical palette across HyperDX + and ClickStack + + Stored configs from the initial color picker (#2265) keep working. + `ChartPaletteTokenSchema` stays strict (a plain `z.enum`, so its `z.input` + matches `z.output` — wrapping it in `z.preprocess` would poison + `validateRequest`'s `req.body` inference all the way up to + `Dashboard.tiles[i].config.color`). Migration of legacy `chart-1` .. `chart-10` + happens at five complementary points so no entry or wire-format path can slip + through, all composing over a single shared walker + (`walkRawDashboardTileColors` in `common-utils`) so the per-tile traversal + stays in lockstep: + + - **Fetch-time / write-time (React)**: `normalizeDashboardTileColors` in + `packages/app/src/dashboard.ts` heals dashboards on read + (`useDashboards` / `fetchLocalDashboards` / `fetchDashboards`) and on write + (`useUpdateDashboard` / `useCreateDashboard`). Unresolvable color strings + (stale hexes, hand-edited values, forward-rolled future tokens) are + preserved so the user's chosen value survives a render pass — the strict + server-side schema surfaces a clear error on next save instead of the + normalizer quietly dropping the field. + - **JSON import**: `DBDashboardImportPage` runs + `normalizeRawDashboardTileColors` on the parsed JSON _before_ the strict + `DashboardTemplateSchema.safeParse`, so templates exported from a + pre-rename deploy import cleanly. + - **Server-side GET response healing**: `getDashboards` / `getDashboard` in + `packages/api/src/controllers/dashboard.ts` rewrite legacy tile colors on + the way out. Pre-rename Mongo docs are served on the wire as + hue-named tokens so non-React HTTP clients (CI scripts, stale bundle + tabs during a rolling deploy, the external API) can round-trip + GET → PATCH without ever resurrecting `chart-N` through the strict + schema. + - **Server-side write shim**: the dashboards POST / PATCH routes mount + a request-body preprocessor that rewrites legacy tile colors before + `validateRequest` runs `ChartPaletteTokenSchema`. Catches non-React + HTTP callers (stale-bundle tabs during a rolling deploy, CI scripts, + MCP, the upcoming external-API parity work) for a one-release + deprecation window without weakening the schema's input/output equality. + The dashboard provisioner task applies the same shim before parsing + on-disk template files. + - **Render-time (belt-and-suspenders)**: `DBNumberChart` and + `ColorSwatchInput` also call `resolveChartPaletteToken` for tiles + constructed in memory between fetch and save (`ChartEditor` form + state, unit-test fixtures, hand-rolled `Tile` literals). + + The migration preserves the HyperDX slot ordering from #2265 (slot 1 = brand + green, slot 2 = blue, etc.). + + **ClickStack legacy color caveat:** Pre-rename ClickStack used a different slot + ordering than HyperDX (`--color-chart-1` was brand blue `#437eef`, not brand + green). The migration map uses HyperDX slot ordering, so any ClickStack + dashboard saved via #2265 with `color: 'chart-1'` will flip from blue to + Observable green after migration. We chose this trade-off deliberately over + branching the legacy map by active theme: `LEGACY_CHART_PALETTE_TOKEN_MAP` lives + in `common-utils` (shared with the API), and migration is one-shot persisted on + next save — theme-branching would couple common-utils to browser DOM state and + still produce wrong results for users whose active theme changed since the + original pick. Affected users can manually re-pick the desired hue via the (now + hue-labeled) color picker. + + The categorical palette is based on Observable 10, with `chart-blue` swapped to + `#437eef` to match the brand link color + (`--click-global-color-text-link-default`); all other hues are straight from + Observable 10. The palette resolves identically on both themes — picking + `chart-blue` always renders the brand blue. Brand identity for charts moves + entirely into the semantic layer: `--color-chart-success` and `--color-chart-info` + resolve to categorical `chart-green` (`#3ca951`) and `chart-blue` (`#437eef`) on + both HyperDX and ClickStack, so success fills, info-level logs, and the + matching multi-series slots all read consistently across brands. + + Internally, JS (`CATEGORICAL_HEX_BY_TOKEN` in `packages/app/src/utils.ts`) is + the source of truth for categorical hues — `getColorFromCSSVariable` and + `getColorFromCSSToken` skip `getComputedStyle` for categorical tokens since the + palette is unified across themes. The matching `--color-chart-{hue}` CSS vars in + `_tokens.scss` remain as a stylesheet-author affordance (inline `var()` use, + devtools inspection) and a hook for any future per-brand override. Semantic + tokens still resolve through `getComputedStyle` because they genuinely vary per + theme. + +- adac913d: refactor(mcp): rename all MCP tool prefixes from `hyperdx_` to `clickstack_` + + Rename the MCP server name from `hyperdx` to `clickstack` and update all 19 + tool names (e.g. `hyperdx_search` → `clickstack_search`), along with + descriptions, prompts, error messages, and test references. + +- 1a64796c1: Removing relative imports and using path aliases +- 03f9dd70: feat: add an optional Section field to data sources + + Sources can now carry an optional free-text Section label, set from the source + settings form. The value is persisted and returned by GET /api/v2/sources, so + external API consumers can read it. This lays the groundwork for grouping and + searching sources by section in the source selector. + +- 6e0880a75: feat: Add Known Columns List setting for distributed tables +- fc3ef2dc: fix(alerts): populate `{{attributes.*}}` template variables for tile/chart alerts from group-by fields +- 81e524c2: feat(charts): cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys. The cap defaults to 100 (the number of series rendered) and is configurable per team via a new "Time Chart Series Limit" setting; series beyond the cap remain available in the series selector. +- 55a255a0a: refactor(metrics): unify AttributesHash to variadic cityHash64 across Map and + JSON metric schemas + + Sum / Gauge / Histogram metric queries now compute AttributesHash as + `cityHash64(ScopeAttributes, ResourceAttributes, Attributes)` for both + Map(LowCardinality(String), String) and JSON attribute columns. Previously + the Map-schema path wrapped the three maps in `mapConcat()` before hashing, + and the JSON-schema path used the variadic form; the schema-detection + ClickHouse round-trip and the `attrHashExpr` helper / `isJsonSchema` + plumbing are gone. + + Compatibility: + + - Per-row AttributesHash values change for every Map-schema metric row, + but the hash is recomputed inside CTEs on every query — no materialized + view, projection, ALIAS column, or cache persists it, so no downstream + consumer is affected (audit: OSS only). + - Cross-scope same-key behaviour shifts: two rows that carry the same + logical key in different attribute scopes (e.g. `host` in + `ResourceAttributes` for one emission and `host` in `Attributes` for the + next) now hash distinctly and land in separate series. Previously the + mapConcat path collapsed them into one series. This only matters when an + OTel collector processor promotes attributes across scopes mid-stream; + most SDKs emit attributes in stable scopes. The new behaviour is captured + by an integration test in `packages/api/src/clickhouse/__tests__`. + + HDX-4466. + +- 9bbf68079: fix: bug preventing deletion of nested subdocuments like metadataMVs +- Updated dependencies [1d44098e5] +- Updated dependencies [998ea5d0] +- Updated dependencies [ee907386] +- Updated dependencies [5c46215f8] +- Updated dependencies [45954c318] +- Updated dependencies [5cd709020] +- Updated dependencies [5a1dde4d3] +- Updated dependencies [ae39bc436] +- Updated dependencies [8261b461] +- Updated dependencies [bf6e1f29] +- Updated dependencies [973d1201b] +- Updated dependencies [677e3f71] +- Updated dependencies [89949b1b] +- Updated dependencies [747352f3] +- Updated dependencies [750b8afe] +- Updated dependencies [caba7c255] +- Updated dependencies [f40cf686b] +- Updated dependencies [17e1eb19d] +- Updated dependencies [e03971b0] +- Updated dependencies [adac913d] +- Updated dependencies [1a64796c1] +- Updated dependencies [c74744a5] +- Updated dependencies [03f9dd70] +- Updated dependencies [6e0880a75] +- Updated dependencies [81e524c2] +- Updated dependencies [da3caab43] +- Updated dependencies [55a255a0a] + - @hyperdx/common-utils@0.21.0 + ## 2.28.0 ### Minor Changes diff --git a/packages/api/package.json b/packages/api/package.json index 31cea0a484..328e210910 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/api", - "version": "2.28.0", + "version": "2.29.0", "license": "MIT", "private": true, "engines": { @@ -11,7 +11,7 @@ "@ai-sdk/openai": "^3.0.47", "@braintree/sanitize-url": "^7.1.1", "@esm2cjs/p-queue": "^7.3.0", - "@hyperdx/common-utils": "^0.20.0", + "@hyperdx/common-utils": "^0.21.0", "@hyperdx/node-opentelemetry": "^0.9.0", "@hyperdx/passport-local-mongoose": "^9.0.1", "@modelcontextprotocol/sdk": "^1.27.1", diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index f8c17791c2..dc34c4667d 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,349 @@ # @hyperdx/app +## 2.29.0 + +### Minor Changes + +- 9af8cbab: feat: add Browser RUM dashboard template + + - New "Browser RUM" template in the dashboards gallery for browser sessions instrumented with the HyperDX Browser SDK (or any OTel browser instrumentation emitting a `rum.sessionId` resource attribute) + - Performance Overview section: page-view/session/error KPIs, Core Web Vitals (LCP/INP/CLS) p75, median/p75/p90 page-load percentiles, and long-task health + - Page Views Breakdown section: traffic grouped by URL, browser (parsed from the `http.user_agent` the document-load instrumentation emits), country, and device size (derived from `screen.xy`) + - Errors section with tabs for an overview, JS exceptions (by message and by page), and failing API calls + - Five dashboard-level filters: Service, Environment, Service Version, Page URL, and Country + - Top Countries tile and the Country filter populate when the OTel collector's `geoip` processor is enabled (geo can't be derived in the browser) + +- 5cd709020: Add UI support for configuring an external Prometheus-compatible endpoint on a + connection. Modify Connections model to now have a boolean + `isPrometheusEndpoint` field and use host for storing the host. +- b6a4b3b3: feat: lazy-load dashboard tiles based on viewport visibility + + Dashboard tiles now only run their ClickHouse queries once they scroll into the browser viewport, instead of every tile querying on page load. A tile loads the first time it becomes visible and keeps its data afterward. This significantly reduces the number of queries fired when opening dashboards with many tiles. + +- f40cf686b: feat(dashboards): add a background trend sparkline to number tiles + + Number tiles can now render a faint line or area sparkline behind the value, + derived from a time-bucketed version of the same query, so the value's trend + over the selected range is visible at a glance. This is handy for SLO / + error-budget tiles where the burn over time matters as much as the current + number. The sparkline inherits the tile's color by default and can be + overridden to any palette token. Configure it under Display Settings > + Background chart on a number tile. Available on builder number tiles (raw SQL + number tiles return a single value with no time dimension to bucket). + +- 17e1eb19d: feat: Add an "external link" row-click action for dashboard table tiles +- c1403a7a7: Number chart tiles now support a second series with the "As Ratio" toggle (`series[0] / series[1]`), matching line and bar charts. Combined with a `percent` number format, this renders a percentage (e.g. success/error rate) as a single big number with the trend sparkline behind it. +- e03971b0: refactor(theme): rename chart palette tokens from chart-1..10 to hue-named + (chart-blue, chart-orange, ...) and unify the categorical palette across HyperDX + and ClickStack + + Stored configs from the initial color picker (#2265) keep working. + `ChartPaletteTokenSchema` stays strict (a plain `z.enum`, so its `z.input` + matches `z.output` — wrapping it in `z.preprocess` would poison + `validateRequest`'s `req.body` inference all the way up to + `Dashboard.tiles[i].config.color`). Migration of legacy `chart-1` .. `chart-10` + happens at five complementary points so no entry or wire-format path can slip + through, all composing over a single shared walker + (`walkRawDashboardTileColors` in `common-utils`) so the per-tile traversal + stays in lockstep: + + - **Fetch-time / write-time (React)**: `normalizeDashboardTileColors` in + `packages/app/src/dashboard.ts` heals dashboards on read + (`useDashboards` / `fetchLocalDashboards` / `fetchDashboards`) and on write + (`useUpdateDashboard` / `useCreateDashboard`). Unresolvable color strings + (stale hexes, hand-edited values, forward-rolled future tokens) are + preserved so the user's chosen value survives a render pass — the strict + server-side schema surfaces a clear error on next save instead of the + normalizer quietly dropping the field. + - **JSON import**: `DBDashboardImportPage` runs + `normalizeRawDashboardTileColors` on the parsed JSON _before_ the strict + `DashboardTemplateSchema.safeParse`, so templates exported from a + pre-rename deploy import cleanly. + - **Server-side GET response healing**: `getDashboards` / `getDashboard` in + `packages/api/src/controllers/dashboard.ts` rewrite legacy tile colors on + the way out. Pre-rename Mongo docs are served on the wire as + hue-named tokens so non-React HTTP clients (CI scripts, stale bundle + tabs during a rolling deploy, the external API) can round-trip + GET → PATCH without ever resurrecting `chart-N` through the strict + schema. + - **Server-side write shim**: the dashboards POST / PATCH routes mount + a request-body preprocessor that rewrites legacy tile colors before + `validateRequest` runs `ChartPaletteTokenSchema`. Catches non-React + HTTP callers (stale-bundle tabs during a rolling deploy, CI scripts, + MCP, the upcoming external-API parity work) for a one-release + deprecation window without weakening the schema's input/output equality. + The dashboard provisioner task applies the same shim before parsing + on-disk template files. + - **Render-time (belt-and-suspenders)**: `DBNumberChart` and + `ColorSwatchInput` also call `resolveChartPaletteToken` for tiles + constructed in memory between fetch and save (`ChartEditor` form + state, unit-test fixtures, hand-rolled `Tile` literals). + + The migration preserves the HyperDX slot ordering from #2265 (slot 1 = brand + green, slot 2 = blue, etc.). + + **ClickStack legacy color caveat:** Pre-rename ClickStack used a different slot + ordering than HyperDX (`--color-chart-1` was brand blue `#437eef`, not brand + green). The migration map uses HyperDX slot ordering, so any ClickStack + dashboard saved via #2265 with `color: 'chart-1'` will flip from blue to + Observable green after migration. We chose this trade-off deliberately over + branching the legacy map by active theme: `LEGACY_CHART_PALETTE_TOKEN_MAP` lives + in `common-utils` (shared with the API), and migration is one-shot persisted on + next save — theme-branching would couple common-utils to browser DOM state and + still produce wrong results for users whose active theme changed since the + original pick. Affected users can manually re-pick the desired hue via the (now + hue-labeled) color picker. + + The categorical palette is based on Observable 10, with `chart-blue` swapped to + `#437eef` to match the brand link color + (`--click-global-color-text-link-default`); all other hues are straight from + Observable 10. The palette resolves identically on both themes — picking + `chart-blue` always renders the brand blue. Brand identity for charts moves + entirely into the semantic layer: `--color-chart-success` and `--color-chart-info` + resolve to categorical `chart-green` (`#3ca951`) and `chart-blue` (`#437eef`) on + both HyperDX and ClickStack, so success fills, info-level logs, and the + matching multi-series slots all read consistently across brands. + + Internally, JS (`CATEGORICAL_HEX_BY_TOKEN` in `packages/app/src/utils.ts`) is + the source of truth for categorical hues — `getColorFromCSSVariable` and + `getColorFromCSSToken` skip `getComputedStyle` for categorical tokens since the + palette is unified across themes. The matching `--color-chart-{hue}` CSS vars in + `_tokens.scss` remain as a stylesheet-author affordance (inline `var()` use, + devtools inspection) and a hook for any future per-brand override. Semantic + tokens still resolve through `getComputedStyle` because they genuinely vary per + theme. + +- 418567ff: feat: trace panel inline split detail + +### Patch Changes + +- 56c58663: fix(search-filters): prevent nested filter dropdowns from disappearing on reopen +- 998ea5d0: feat: Add option to fit time chart y-axis lower bound +- 0497ca5dd: Bump http-proxy-middleware to v4, replacing http-proxy with httpxy +- 20fabc65: feat: add a "Connect your AI assistant" section to Team Settings + + A new section on the Team Settings page (Integrations tab, above the API Keys + card) lets a user install the HyperDX MCP server in Claude Code, Cursor, + VS Code + Copilot, Codex CLI, or any MCP-compatible host without hand-rolling + JSON. Per-host snippets carry the user's personal access key so the install + works against the existing `/api/mcp` route without extra setup. + +- 8e52cef4: feat(dashboard): auto-resize font in number tiles to fit container + + Number tiles now automatically scale their font size to fit the available + width, preventing text overflow on narrow tiles and making better use of + space on wide ones. Includes an error boundary so a single broken tile + does not crash the entire dashboard. + +- 5a1dde4d3: fix(search): wrap date column values in a type-matching parse/convert expression when building IN/NOT IN filters, so including/excluding a timestamp value no longer fails with "Cannot convert string ... to type DateTime64" or "Type mismatch in IN ... Expected: DateTime. Got: Decimal64". Date column types are now resolved from the query result set, so aliased (`TimestampTime AS time`) and computed (`toDate(TimestampTime)`) DateTime/Date columns are also wrapped correctly when added to filters. +- 31b87816: feat(chart-explorer): duplicate a series in the chart builder + + Add a Duplicate button to each series row in the chart builder that inserts a + copy of that series directly below it, so building a near-identical variant + (for example avg and p95 of the same column) no longer requires re-entering + every field by hand. "Add Series" still creates a blank series. The copy + starts with an empty alias so it does not collide with the original's alias in + the generated SQL. + +- 5e19a2b42: Show elapsed time and Generated SQL for search timeline view +- 65931e37: feat(search): make active filter pills editable in place + + Clicking an active filter pill under the search bar now opens a small menu to copy the value, flip the filter polarity (include vs exclude), or switch to a different value of the same field, without removing and re-adding the filter. The polarity is preserved when changing the value, and the one-click remove on each pill is unchanged. Range and not-applied pills keep their remove-only behavior. + +- 7152d2b6: feat: Use optimistic updates for favorites +- 497d50b4: feat: Allow selecting the column or SQL expression used for event pattern grouping (with shareable URL state) +- ae39bc436: fix: Correct filter handling for filter keys with special characters +- bd31ea982: fix: handle boolean values in JSON viewer filter actions +- 052315b1: fix: improve contrast of excluded search filter pills + + Excluded ("!=") filter pills above the search results used a saturated red background with red text and a red remove button, which made them hard to read in the light theme. They now use a soft red tint with a readable accent, legible in both light and dark themes. + +- 7b6db8d91: fix(app): format log detail Timestamp in local timezone + + The log detail JSON viewer rendered Timestamp and TimestampTime as raw UTC ISO strings while the results table used the shared FormatTime helper. + +- bcec17635: fix: allow saving edits to markdown dashboard tiles that have a minimal config shape (no resolved source) +- 8261b461: fix: inline parametric aggregate function arguments instead of passing as query parameters +- bf6e1f29: feat(charts): the time-chart series limit is now configured per chart in the Display Settings drawer instead of as a workspace-wide team setting (the team "Time Chart Series Limit" setting is removed). It is disabled by default — charts fetch every series and no `__hdx_series_limit` CTE is emitted — and is cleared back to disabled by emptying the field. The control only appears for builder line/bar charts; the limit and its Generated SQL preview now come from the chart's own config. When a limit is set, chunked time-chart queries keep a consistent top-N series set: previously each time-window chunk ranked its own top-N, so charts could render more series than the limit and adjacent windows disagreed; the ranking is now pinned to the newest chunk window for every chunk so the union across chunks equals the limit. +- f9fab8ed6: fix: Prevent table content from overlapping table headers +- 973d1201b: fix: polish promql experience across the app +- 712ba11c: fix: Navigate to the dashboard listing page after deleting a dashboard +- 21307756: fix(row-panel): mergePath now emits string-key subscripts for Map columns, + preventing a crash when expanding rows with numeric-looking attribute keys + + `mergePath` converted numeric path segments to 1-based array subscripts + (`[N+1]`) regardless of whether the parent column was a Map or an Array. + On a `Map(String, String)` column this produced SQL like `LogAttributes[2]`, + which ClickHouse rejects with `Illegal types of arguments: +Map(String, String), UInt8 for function arrayElement`. The grid row + "expand" view failed for any row whose attribute path included a + numeric-looking key under a Map column. + + `mergePath` now accepts a `mapColumns` argument alongside `jsonColumns`. + For Map-typed parents, sub-keys always render as string subscripts + (`Map['1']`) regardless of whether the key looks numeric. The three + callers (`useAutoCompleteOptions`, `DBRowJsonViewer` via the row panels, + `DBSearchPageFilters`) now thread Map-column names from the source + schema. A new `useMapColumns` hook mirrors `useJsonColumns`. + + Fixes HDX-4369. + +- 2cecc9f4: Dashboard table tiles configured with a row-click action now show a trailing arrow-up-right icon at the right edge of each row, revealed on hover, with a small tooltip that names the destination. Actionable rows get a stronger background highlight on hover to reinforce interactivity before the user sees the arrow fade in. The icon click navigates to the same URL as a row click, with all the standard native browser behaviors (cmd-click new tab, middle-click new tab, right-click context menu). +- d985895fa: Fix: Resolved an issue with markdown tiles breaking dashboard imports. +- 750b8afe: feat(mcp): add denoise option to clickstack_search tool + + Add a `denoise` boolean parameter to the MCP `clickstack_search` tool that + automatically filters out high-frequency repetitive event patterns from + search results, mirroring the web app's "Denoise Results" feature. + + When enabled, the tool samples 10k random events, mines patterns using + the Drain algorithm, identifies noisy patterns (>10% of sample), and + filters them out of result rows. Returns filtered rows plus metadata + listing removed patterns with estimated counts. + + Extracts shared denoise constants (`DENOISE_SAMPLE_SIZE`, + `DENOISE_NOISE_THRESHOLD`) into `@hyperdx/common-utils` so the web app + and MCP server use the same values. + +- cd6a17daf: feat: auto-fill metric table dropdowns when creating a Metrics source + + The 5 metric-table dropdowns (Gauge, Histogram, Sum, Summary, Exponential + Histogram) now auto-populate by matching table names in the selected database + to their metric type via suffix patterns. Prefers `otel_metrics_` prefixed + names, never overwrites user selections, and shows a green notification on + successful autofill. + +- 6747cf963: fix(dashboards): keep the auto-detected number format when applying display settings + + Opening Display Settings on a number tile that auto-detects its format from the + datasource (for example p95 of a trace Duration column) and clicking Apply no + longer rewrites the format to Number. The drawer now reflects the + datasource-derived format, and Apply persists `numberFormat` only when the user + explicitly changes it; otherwise it stays unset so render-time auto-detection + keeps driving the format. + +- a258fcfe8: fix(dashboards): match the number-tile background sparkline to the displayed value + + The big number on a number tile is a single aggregate (its query drops `groupBy`), but the background sparkline kept any `groupBy` the tile carried over from a prior Line display type. It then plotted only the first group's trend behind a value that aggregates every group. The sparkline now drops `groupBy` as well, so its trend reflects the same single series as the value it sits behind. + +- 9d713999: fix(z-index): keep sticky header below drawers and drawers above the fullscreen tile modal + + Two related z-index regressions: + + - `PageHeader` was pinned at `z-index: 100`, but app drawers opt into a + much lower stack via `ZIndexContext` (`contextZIndex + 10`, so a + top-level drawer renders at `z-index: 10`). The sticky header therefore + floated above the drawer overlay. The header now sits at `z-index: 2` so + drawer overlays reliably cover the page chrome while the header still + wins against normal scrolling content. + - `FullscreenPanelModal` used Mantine's default modal z-index (`200`) and + didn't propagate it through `ZIndexContext`. Clicking a row in a + fullscreen search tile opened a `DBRowSidePanel` drawer at `z-index: 10` + that was hidden behind the modal. The modal now follows the existing + `contextZIndex + 10` pattern and wraps its children in a + `ZIndexContext.Provider`, so child drawers stack on top of it. + +- 538a1c4e: chore: migrate the custom Dashboard page to shared `PageLayout` / `PageHeader`. Breadcrumbs, the editable dashboard name, dashboard actions (Favorite, Tags, Menu), and the "Created by … Updated …" meta now live in a single page header, while the query toolbar (SQL/Lucene WHERE, time range, granularity, Live, refresh, edit filters, Run) is pinned to the top of the scroll container as a dedicated sticky row — the chrome above scrolls away and only the toolbar follows the user. The "Updated …" meta moves to the right side of the breadcrumbs row instead of sitting as a separate body line. + + `PageHeader` gains a `stickyRow` slot that any page can use to declare a single row that should be the only pinned element, with the rest of the header treated as scrolling chrome. Other pages are unaffected — a `PageHeader` without `stickyRow` keeps the existing fully-sticky behavior. + +- 5e3e541bb: fix(search): keep select-alias filters working in Event Patterns + + Filtering on a column the source exposes only under an alias (for example a + default select of `ServiceName as service`) failed in the Event Patterns view + with `Unknown expression or table expression identifier 'service'`. The + results table works because its own SELECT defines the alias, but Event + Patterns rebuilds the SELECT and did not carry the alias definitions. The + pattern query now receives the same alias `WITH` clauses already threaded into + the results, histogram, and heatmap queries, so the filter resolves. + +- e4922804: feat: add source field suggestions +- defbe1f9: Add Cmd/Ctrl+Enter support for running raw SQL chart queries from the SQL editor. +- 1a64796c1: Removing relative imports and using path aliases +- c74744a5: fix: fallback to body or implicit column expression when other empty +- d1d91d74: feat(service-map): server-side filtering, latency percentiles, throughput & focus + + The Service Map gains server-side filtering (Lucene/SQL `where` plus a + service-name multi-select with inbound/outbound neighbor expansion), latency + percentiles (p50/p95/p99) and request throughput (req/s) in node and edge + tooltips, a "Focus" action to scope the map to a service and its immediate + dependencies, and node sizing by total throughput (incoming + outgoing). + Percentiles are computed server-side via a single GROUPING SETS query. + +- b763ba64: fix: next-runtime-env runtime env var injection fixed for images +- 53e8bd17: fix: Fix height of source select RHS menu +- 5e8af09be: Transition the local development server from Webpack to Turbopack to + significantly improve build performance and hot-reloading speed. +- 2a681456: feat(source-picker): chip + kebab menu UX +- f95687b0: Fix the database, table, and connection dropdowns being clipped inside the source setup modal. The dropdowns now render in a portal, so the full list is visible and scrollable when configuring or editing a source. +- 48e19e8b: Suggest existing section names in the source form's **Section** field. The field is now an autocomplete fed by the sections already in use, so a new source can reuse an existing section instead of retyping it (which is how a section ends up split into near-duplicates like "Billing" and "billing"). The field stays free-text, so any new section name is still accepted. +- 03f9dd70: feat: add an optional Section field to data sources + + Sources can now carry an optional free-text Section label, set from the source + settings form. The value is persisted and returned by GET /api/v2/sources, so + external API consumers can read it. This lays the groundwork for grouping and + searching sources by section in the source selector. + +- fdb18f26: Group the data source selector by section and add tag-style search. When sources have a Section assigned, the selector lists them under section headers; search matches on both the source name and its section, so a section name acts as a tag (typing "billing" returns every source in the Billing section, including ones whose name does not contain "billing"). The selector stays flat until at least one source has a section, so deployments that have not adopted sections see no change. The grouped dropdown is also widened and pinned to the picker's left edge so section headers and source names are not cramped. +- 34aa906f0: Show each source's **Section** on the Manage Sources list. A sectioned source now displays its section, with a folder icon, in the dimmed metadata row alongside its connection and table, so the list mirrors the grouped selector. Sources without a section are unchanged. +- 6e0880a75: feat: Add Known Columns List setting for distributed tables +- 81e524c2: feat(charts): cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys. The cap defaults to 100 (the number of series rendered) and is configurable per team via a new "Time Chart Series Limit" setting; series beyond the cap remain available in the series selector. +- bc5cd0021: feat: emphasize the series nearest the cursor in multi-series time charts. The nearest line is thickened and the others fade back, and its tooltip row is bolded while the rest dim, so a value is easy to trace back to its line. +- a6e7dcde: chore: Make error states consistent across chart types +- 9bbf68079: fix: bug preventing deletion of nested subdocuments like metadataMVs +- Updated dependencies [9119de5f] +- Updated dependencies [1d44098e5] +- Updated dependencies [9f23b7e58] +- Updated dependencies [998ea5d0] +- Updated dependencies [0497ca5dd] +- Updated dependencies [ee907386] +- Updated dependencies [5c46215f8] +- Updated dependencies [45954c318] +- Updated dependencies [5cd709020] +- Updated dependencies [9a7e392a] +- Updated dependencies [5a1dde4d3] +- Updated dependencies [b798f91f] +- Updated dependencies [ae39bc436] +- Updated dependencies [cdd7ca07] +- Updated dependencies [d11991b0c] +- Updated dependencies [8261b461] +- Updated dependencies [bf6e1f29] +- Updated dependencies [973d1201b] +- Updated dependencies [677e3f71] +- Updated dependencies [89949b1b] +- Updated dependencies [747352f3] +- Updated dependencies [8164492f] +- Updated dependencies [a19ba549] +- Updated dependencies [7e7159a5] +- Updated dependencies [63469fe0e] +- Updated dependencies [f34a31fdc] +- Updated dependencies [f6bda8c5] +- Updated dependencies [f326ccf8] +- Updated dependencies [750b8afe] +- Updated dependencies [caba7c255] +- Updated dependencies [f113ea36] +- Updated dependencies [634101c33] +- Updated dependencies [ba626ef96] +- Updated dependencies [f40cf686b] +- Updated dependencies [f126d5b1] +- Updated dependencies [ebfc2e80a] +- Updated dependencies [bbc29859d] +- Updated dependencies [17e1eb19d] +- Updated dependencies [60a91e43] +- Updated dependencies [e03971b0] +- Updated dependencies [adac913d] +- Updated dependencies [1a64796c1] +- Updated dependencies [c74744a5] +- Updated dependencies [03f9dd70] +- Updated dependencies [6e0880a75] +- Updated dependencies [fc3ef2dc] +- Updated dependencies [81e524c2] +- Updated dependencies [da3caab43] +- Updated dependencies [55a255a0a] +- Updated dependencies [9bbf68079] + - @hyperdx/api@2.29.0 + - @hyperdx/common-utils@0.21.0 + ## 2.28.0 ### Minor Changes diff --git a/packages/app/package.json b/packages/app/package.json index 92d9138243..df48d0cd8f 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/app", - "version": "2.28.0", + "version": "2.29.0", "private": true, "license": "MIT", "engines": { @@ -39,7 +39,7 @@ "@hookform/resolvers": "^3.9.0", "@hyperdx/api": "workspace:*", "@hyperdx/browser": "^0.22.1", - "@hyperdx/common-utils": "^0.20.0", + "@hyperdx/common-utils": "^0.21.0", "@hyperdx/node-opentelemetry": "^0.9.0", "@mantine/core": "^9.0.0", "@mantine/dates": "^9.0.0", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 4f7eececf6..7c51e72196 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,31 @@ # @hyperdx/cli +## 0.5.1 + +### Patch Changes + +- 5c46215f8: Bump `@clickhouse/client*` to `1.23.0-head.fae5998.1` and fix the type + incompatibility it introduces. + + In `@clickhouse/client*` 1.23 each platform package (`@clickhouse/client`, + `@clickhouse/client-web`) bundles its own copy of the shared types, so their + `ClickHouseSettings` types — which reference the nominally-compared `SettingsMap` + class — are no longer the same type as `@clickhouse/client-common`'s. The shared + `processClickhouseSettings()` helper produces the `client-common` flavor, so + assigning it into the per-platform clients' `query()` now requires an explicit + bridge. Guard the existing `as ClickHouseSettings` assertions at those + boundaries (`node.ts`, `browser.ts`, `cli`) with a scoped + `@typescript-eslint/no-unsafe-type-assertion` disable, matching the existing + "client library type mismatch" pattern. No runtime behavior changes. + +- 45954c318: Import ClickHouse client types from the platform packages + (`@clickhouse/client` / `@clickhouse/client-web`) instead of the deprecated + `@clickhouse/client-common`. This makes the packages forward-compatible with + `@clickhouse/client*` 1.23 (where `client-common` is deprecated and each + platform package bundles and re-exports its own copy of the shared types) + without bumping the pinned version. No runtime behavior changes. +- 1a64796c1: Removing relative imports and using path aliases + ## 0.5.0 ### Minor Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index ac29a95407..79569ef309 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/cli", - "version": "0.5.0", + "version": "0.5.1", "license": "MIT", "type": "module", "repository": { @@ -40,7 +40,7 @@ }, "devDependencies": { "@clickhouse/client": "1.23.0-head.fae5998.1", - "@hyperdx/common-utils": "^0.20.0", + "@hyperdx/common-utils": "^0.21.0", "@jest/globals": "^30.2.0", "@types/crypto-js": "^4.2.2", "@types/jest": "^29.5.14", diff --git a/packages/common-utils/CHANGELOG.md b/packages/common-utils/CHANGELOG.md index 450dcf1b0e..eb96fafcec 100644 --- a/packages/common-utils/CHANGELOG.md +++ b/packages/common-utils/CHANGELOG.md @@ -1,5 +1,209 @@ # @hyperdx/common-utils +## 0.21.0 + +### Minor Changes + +- 5cd709020: Add UI support for configuring an external Prometheus-compatible endpoint on a + connection. Modify Connections model to now have a boolean + `isPrometheusEndpoint` field and use host for storing the host. +- f40cf686b: feat(dashboards): add a background trend sparkline to number tiles + + Number tiles can now render a faint line or area sparkline behind the value, + derived from a time-bucketed version of the same query, so the value's trend + over the selected range is visible at a glance. This is handy for SLO / + error-budget tiles where the burn over time matters as much as the current + number. The sparkline inherits the tile's color by default and can be + overridden to any palette token. Configure it under Display Settings > + Background chart on a number tile. Available on builder number tiles (raw SQL + number tiles return a single value with no time dimension to bucket). + +- 17e1eb19d: feat: Add an "external link" row-click action for dashboard table tiles +- e03971b0: refactor(theme): rename chart palette tokens from chart-1..10 to hue-named + (chart-blue, chart-orange, ...) and unify the categorical palette across HyperDX + and ClickStack + + Stored configs from the initial color picker (#2265) keep working. + `ChartPaletteTokenSchema` stays strict (a plain `z.enum`, so its `z.input` + matches `z.output` — wrapping it in `z.preprocess` would poison + `validateRequest`'s `req.body` inference all the way up to + `Dashboard.tiles[i].config.color`). Migration of legacy `chart-1` .. `chart-10` + happens at five complementary points so no entry or wire-format path can slip + through, all composing over a single shared walker + (`walkRawDashboardTileColors` in `common-utils`) so the per-tile traversal + stays in lockstep: + + - **Fetch-time / write-time (React)**: `normalizeDashboardTileColors` in + `packages/app/src/dashboard.ts` heals dashboards on read + (`useDashboards` / `fetchLocalDashboards` / `fetchDashboards`) and on write + (`useUpdateDashboard` / `useCreateDashboard`). Unresolvable color strings + (stale hexes, hand-edited values, forward-rolled future tokens) are + preserved so the user's chosen value survives a render pass — the strict + server-side schema surfaces a clear error on next save instead of the + normalizer quietly dropping the field. + - **JSON import**: `DBDashboardImportPage` runs + `normalizeRawDashboardTileColors` on the parsed JSON _before_ the strict + `DashboardTemplateSchema.safeParse`, so templates exported from a + pre-rename deploy import cleanly. + - **Server-side GET response healing**: `getDashboards` / `getDashboard` in + `packages/api/src/controllers/dashboard.ts` rewrite legacy tile colors on + the way out. Pre-rename Mongo docs are served on the wire as + hue-named tokens so non-React HTTP clients (CI scripts, stale bundle + tabs during a rolling deploy, the external API) can round-trip + GET → PATCH without ever resurrecting `chart-N` through the strict + schema. + - **Server-side write shim**: the dashboards POST / PATCH routes mount + a request-body preprocessor that rewrites legacy tile colors before + `validateRequest` runs `ChartPaletteTokenSchema`. Catches non-React + HTTP callers (stale-bundle tabs during a rolling deploy, CI scripts, + MCP, the upcoming external-API parity work) for a one-release + deprecation window without weakening the schema's input/output equality. + The dashboard provisioner task applies the same shim before parsing + on-disk template files. + - **Render-time (belt-and-suspenders)**: `DBNumberChart` and + `ColorSwatchInput` also call `resolveChartPaletteToken` for tiles + constructed in memory between fetch and save (`ChartEditor` form + state, unit-test fixtures, hand-rolled `Tile` literals). + + The migration preserves the HyperDX slot ordering from #2265 (slot 1 = brand + green, slot 2 = blue, etc.). + + **ClickStack legacy color caveat:** Pre-rename ClickStack used a different slot + ordering than HyperDX (`--color-chart-1` was brand blue `#437eef`, not brand + green). The migration map uses HyperDX slot ordering, so any ClickStack + dashboard saved via #2265 with `color: 'chart-1'` will flip from blue to + Observable green after migration. We chose this trade-off deliberately over + branching the legacy map by active theme: `LEGACY_CHART_PALETTE_TOKEN_MAP` lives + in `common-utils` (shared with the API), and migration is one-shot persisted on + next save — theme-branching would couple common-utils to browser DOM state and + still produce wrong results for users whose active theme changed since the + original pick. Affected users can manually re-pick the desired hue via the (now + hue-labeled) color picker. + + The categorical palette is based on Observable 10, with `chart-blue` swapped to + `#437eef` to match the brand link color + (`--click-global-color-text-link-default`); all other hues are straight from + Observable 10. The palette resolves identically on both themes — picking + `chart-blue` always renders the brand blue. Brand identity for charts moves + entirely into the semantic layer: `--color-chart-success` and `--color-chart-info` + resolve to categorical `chart-green` (`#3ca951`) and `chart-blue` (`#437eef`) on + both HyperDX and ClickStack, so success fills, info-level logs, and the + matching multi-series slots all read consistently across brands. + + Internally, JS (`CATEGORICAL_HEX_BY_TOKEN` in `packages/app/src/utils.ts`) is + the source of truth for categorical hues — `getColorFromCSSVariable` and + `getColorFromCSSToken` skip `getComputedStyle` for categorical tokens since the + palette is unified across themes. The matching `--color-chart-{hue}` CSS vars in + `_tokens.scss` remain as a stylesheet-author affordance (inline `var()` use, + devtools inspection) and a hook for any future per-brand override. Semantic + tokens still resolve through `getComputedStyle` because they genuinely vary per + theme. + +### Patch Changes + +- 1d44098e5: fix: recover the SELECT-alias map when a query has ClickHouse-specific SQL the parser rejects + + `chSqlToAliasMap` returned an empty map whenever the rendered query contained + SQL that node-sql-parser's Postgresql dialect cannot parse, for example a + sampling CTE with `greatest(CAST(total / N AS UInt32), 1)`. An empty alias map + drops the `WITH` clauses that define the source's select aliases, so filters on + aliased columns (Event Patterns, histogram, alerts) failed with `Unknown +identifier`. It now falls back to parsing only the outer SELECT projection, + which is all the alias map needs, so the aliases are recovered even when the + rest of the statement is unparseable. + +- 998ea5d0: feat: Add option to fit time chart y-axis lower bound +- ee907386: fix: Add sourceId to MCP Raw SQL Tile schema +- 5c46215f8: Bump `@clickhouse/client*` to `1.23.0-head.fae5998.1` and fix the type + incompatibility it introduces. + + In `@clickhouse/client*` 1.23 each platform package (`@clickhouse/client`, + `@clickhouse/client-web`) bundles its own copy of the shared types, so their + `ClickHouseSettings` types — which reference the nominally-compared `SettingsMap` + class — are no longer the same type as `@clickhouse/client-common`'s. The shared + `processClickhouseSettings()` helper produces the `client-common` flavor, so + assigning it into the per-platform clients' `query()` now requires an explicit + bridge. Guard the existing `as ClickHouseSettings` assertions at those + boundaries (`node.ts`, `browser.ts`, `cli`) with a scoped + `@typescript-eslint/no-unsafe-type-assertion` disable, matching the existing + "client library type mismatch" pattern. No runtime behavior changes. + +- 45954c318: Import ClickHouse client types from the platform packages + (`@clickhouse/client` / `@clickhouse/client-web`) instead of the deprecated + `@clickhouse/client-common`. This makes the packages forward-compatible with + `@clickhouse/client*` 1.23 (where `client-common` is deprecated and each + platform package bundles and re-exports its own copy of the shared types) + without bumping the pinned version. No runtime behavior changes. +- 5a1dde4d3: fix(search): wrap date column values in a type-matching parse/convert expression when building IN/NOT IN filters, so including/excluding a timestamp value no longer fails with "Cannot convert string ... to type DateTime64" or "Type mismatch in IN ... Expected: DateTime. Got: Decimal64". Date column types are now resolved from the query result set, so aliased (`TimestampTime AS time`) and computed (`toDate(TimestampTime)`) DateTime/Date columns are also wrapped correctly when added to filters. +- ae39bc436: fix: Correct filter handling for filter keys with special characters +- 8261b461: fix: inline parametric aggregate function arguments instead of passing as query parameters +- bf6e1f29: feat(charts): the time-chart series limit is now configured per chart in the Display Settings drawer instead of as a workspace-wide team setting (the team "Time Chart Series Limit" setting is removed). It is disabled by default — charts fetch every series and no `__hdx_series_limit` CTE is emitted — and is cleared back to disabled by emptying the field. The control only appears for builder line/bar charts; the limit and its Generated SQL preview now come from the chart's own config. When a limit is set, chunked time-chart queries keep a consistent top-N series set: previously each time-window chunk ranked its own top-N, so charts could render more series than the limit and adjacent windows disagreed; the ranking is now pinned to the newest chunk window for every chunk so the union across chunks equals the limit. +- 973d1201b: fix: polish promql experience across the app +- 677e3f71: fix: Skip rendering empty aggConditions +- 89949b1b: Adding filters to dashboard exports. Implemented validation on dashboard imports to catch potential issues with generated JSON or manually tweaked JSON. +- 747352f3: feat: add direct_read optimization for filters +- 750b8afe: feat(mcp): add denoise option to clickstack_search tool + + Add a `denoise` boolean parameter to the MCP `clickstack_search` tool that + automatically filters out high-frequency repetitive event patterns from + search results, mirroring the web app's "Denoise Results" feature. + + When enabled, the tool samples 10k random events, mines patterns using + the Drain algorithm, identifies noisy patterns (>10% of sample), and + filters them out of result rows. Returns filtered rows plus metadata + listing removed patterns with estimated counts. + + Extracts shared denoise constants (`DENOISE_SAMPLE_SIZE`, + `DENOISE_NOISE_THRESHOLD`) into `@hyperdx/common-utils` so the web app + and MCP server use the same values. + +- caba7c255: fix: Nudge agents towards macros in raw SQL tiles +- adac913d: refactor(mcp): rename all MCP tool prefixes from `hyperdx_` to `clickstack_` + + Rename the MCP server name from `hyperdx` to `clickstack` and update all 19 + tool names (e.g. `hyperdx_search` → `clickstack_search`), along with + descriptions, prompts, error messages, and test references. + +- 1a64796c1: Removing relative imports and using path aliases +- c74744a5: fix: fallback to body or implicit column expression when other empty +- 03f9dd70: feat: add an optional Section field to data sources + + Sources can now carry an optional free-text Section label, set from the source + settings form. The value is persisted and returned by GET /api/v2/sources, so + external API consumers can read it. This lays the groundwork for grouping and + searching sources by section in the source selector. + +- 6e0880a75: feat: Add Known Columns List setting for distributed tables +- 81e524c2: feat(charts): cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys. The cap defaults to 100 (the number of series rendered) and is configurable per team via a new "Time Chart Series Limit" setting; series beyond the cap remain available in the series selector. +- da3caab43: Type JSON metadata filter attribute paths before value sampling. +- 55a255a0a: refactor(metrics): unify AttributesHash to variadic cityHash64 across Map and + JSON metric schemas + + Sum / Gauge / Histogram metric queries now compute AttributesHash as + `cityHash64(ScopeAttributes, ResourceAttributes, Attributes)` for both + Map(LowCardinality(String), String) and JSON attribute columns. Previously + the Map-schema path wrapped the three maps in `mapConcat()` before hashing, + and the JSON-schema path used the variadic form; the schema-detection + ClickHouse round-trip and the `attrHashExpr` helper / `isJsonSchema` + plumbing are gone. + + Compatibility: + + - Per-row AttributesHash values change for every Map-schema metric row, + but the hash is recomputed inside CTEs on every query — no materialized + view, projection, ALIAS column, or cache persists it, so no downstream + consumer is affected (audit: OSS only). + - Cross-scope same-key behaviour shifts: two rows that carry the same + logical key in different attribute scopes (e.g. `host` in + `ResourceAttributes` for one emission and `host` in `Attributes` for the + next) now hash distinctly and land in separate series. Previously the + mapConcat path collapsed them into one series. This only matters when an + OTel collector processor promotes attributes across scopes mid-stream; + most SDKs emit attributes in stable scopes. The new behaviour is captured + by an integration test in `packages/api/src/clickhouse/__tests__`. + + HDX-4466. + ## 0.20.0 ### Minor Changes diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 0345b775f6..4888d51074 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -1,7 +1,7 @@ { "name": "@hyperdx/common-utils", "description": "Common utilities for HyperDX application", - "version": "0.20.0", + "version": "0.21.0", "license": "MIT", "private": true, "files": [ diff --git a/packages/hdx-eval/CHANGELOG.md b/packages/hdx-eval/CHANGELOG.md new file mode 100644 index 0000000000..43014d2da6 --- /dev/null +++ b/packages/hdx-eval/CHANGELOG.md @@ -0,0 +1,22 @@ +# @hyperdx/hdx-eval + +## 0.2.0 + +### Minor Changes + +- 5bd1c681: feat: add AI eval framework for benchmarking MCP servers + + New `@hyperdx/hdx-eval` package for benchmarking AI agents against + observability MCP servers. Generates deterministic synthetic telemetry + with planted anomalies, spawns Claude Code as an SRE agent, records full + trajectories, and grades answers using programmatic checks and an + LLM-as-judge. + + Includes 5 scenarios (error-root-cause, latency-spike, noisy-signals, + segmented-regression, service-health-check), MCP-agnostic N-way + comparison, blinded judging, and a web viewer for browsing results. + +### Patch Changes + +- 6a800318: Support multi-model comparison in eval batches via comma-separated --model flag +- 1a64796c1: Removing relative imports and using path aliases diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json index 1982842236..bfc542bb33 100644 --- a/packages/hdx-eval/package.json +++ b/packages/hdx-eval/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/hdx-eval", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "private": true, "engines": { diff --git a/packages/otel-collector/CHANGELOG.md b/packages/otel-collector/CHANGELOG.md index 82e33e8c74..6893333277 100644 --- a/packages/otel-collector/CHANGELOG.md +++ b/packages/otel-collector/CHANGELOG.md @@ -1,5 +1,62 @@ # @hyperdx/otel-collector +## 2.29.0 + +### Minor Changes + +- 34a855969: chore(otel-collector): bump base collector to v0.154.0 + + Upgrade the custom OTel Collector base from contrib v0.149.0 (core 1.55.0) to + v0.154.0 (core 1.60.0). Updates `OTEL_COLLECTOR_VERSION` / + `OTEL_COLLECTOR_CORE_VERSION` in `.env`, both Dockerfile ARG defaults, and the + smoke-test compose fallbacks. + + Compatibility: no config changes required. Reviewed contrib and core breaking + changes across v0.150–v0.154 against every component HyperDX uses. All affected + upstream changes are either backward-compatible deprecation aliases + (`prometheusremotewrite`, `resourcedetection`), explicit-config no-ops for + HyperDX (clickhouse exporter already sets `json:` directly; transform/routing + connectors set `error_mode: ignore` explicitly), or internal core feature-gate + stabilizations. + +- 6b6c340fc: chore(otel-collector): bump base collector to v0.155.0 + + Upgrade the custom OTel Collector base from contrib v0.154.0 (core 1.60.0) to + v0.155.0 (core 1.61.0). Updates `OTEL_COLLECTOR_VERSION` / + `OTEL_COLLECTOR_CORE_VERSION` in `.env`, both Dockerfile ARG defaults, and the + smoke-test compose fallbacks. + + Compatibility: no config changes required. Reviewed contrib and core breaking + changes for v0.155.0 against every component HyperDX uses. The removed + `telemetry.UseLocalHostAsDefaultMetricsAddress` core gate has no impact because + the telemetry metrics endpoint is set explicitly (`host: 0.0.0.0`, `port: +8888`), and the `memory_limiter` metric rename does not affect the smoke tests + (which assert on the startup log line and the `batch/lowlatency` metric label, + not memory_limiter metrics). All other breaking changes are in unused components + or internal feature-gate removals. + +### Patch Changes + +- 973d1201b: fix: polish promql experience across the app +- a5dfce4b: fix(otel-collector): only enable the prometheus remote-write exporter in + standalone mode when `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` is set + + The standalone collector config used to unconditionally declare a + `prometheusremotewrite` exporter and a `metrics/promql` pipeline. When + `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` was unset the exporter rendered + with an empty endpoint and every metrics batch failed to export. + + The exporter and pipeline have been moved to + `docker/otel-collector/config.standalone.promql.yaml`, which is now only + loaded by `entrypoint.sh` when `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` is + non-empty. This mirrors the OpAMP-managed gating in + `packages/api/src/opamp/controllers/opampController.ts` (which already + only adds the exporter when `IS_PROMQL_ENABLED` is true). + + No action required if `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` is set; the + behavior is unchanged. If it was unset, the collector now stops emitting + the failing prometheus-remote-write attempts. + ## 2.28.0 ### Minor Changes diff --git a/packages/otel-collector/package.json b/packages/otel-collector/package.json index 61b68c11c9..79c9b67bee 100644 --- a/packages/otel-collector/package.json +++ b/packages/otel-collector/package.json @@ -1,7 +1,7 @@ { "name": "@hyperdx/otel-collector", "description": "HyperDX OpenTelemetry Collector configuration and Docker image", - "version": "2.28.0", + "version": "2.29.0", "license": "MIT", "private": true } diff --git a/yarn.lock b/yarn.lock index e0ab5a0eb1..de48626159 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4424,7 +4424,7 @@ __metadata: "@ai-sdk/openai": "npm:^3.0.47" "@braintree/sanitize-url": "npm:^7.1.1" "@esm2cjs/p-queue": "npm:^7.3.0" - "@hyperdx/common-utils": "npm:^0.20.0" + "@hyperdx/common-utils": "npm:^0.21.0" "@hyperdx/node-opentelemetry": "npm:^0.9.0" "@hyperdx/passport-local-mongoose": "npm:^9.0.1" "@modelcontextprotocol/sdk": "npm:^1.27.1" @@ -4513,7 +4513,7 @@ __metadata: "@hookform/resolvers": "npm:^3.9.0" "@hyperdx/api": "workspace:*" "@hyperdx/browser": "npm:^0.22.1" - "@hyperdx/common-utils": "npm:^0.20.0" + "@hyperdx/common-utils": "npm:^0.21.0" "@hyperdx/node-opentelemetry": "npm:^0.9.0" "@jedmao/location": "npm:^3.0.0" "@mantine/core": "npm:^9.0.0" @@ -4640,7 +4640,7 @@ __metadata: resolution: "@hyperdx/cli@workspace:packages/cli" dependencies: "@clickhouse/client": "npm:1.23.0-head.fae5998.1" - "@hyperdx/common-utils": "npm:^0.20.0" + "@hyperdx/common-utils": "npm:^0.21.0" "@jest/globals": "npm:^30.2.0" "@types/crypto-js": "npm:^4.2.2" "@types/jest": "npm:^29.5.14" @@ -4666,7 +4666,7 @@ __metadata: languageName: unknown linkType: soft -"@hyperdx/common-utils@npm:^0.20.0, @hyperdx/common-utils@workspace:packages/common-utils": +"@hyperdx/common-utils@npm:^0.21.0, @hyperdx/common-utils@workspace:packages/common-utils": version: 0.0.0-use.local resolution: "@hyperdx/common-utils@workspace:packages/common-utils" dependencies: From 89f34db833f4a5bf076672c29324bb984581441f Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:30:39 -0700 Subject: [PATCH 123/728] [HDX] Fix duplicate git tag push in release workflow (#2377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The `Release` workflow's `Check Changesets` job failed on the `Create Release Pull Request or Publish to npm` step, which gates **every** downstream job (Docker app/local/all-in-one/otel-collector builds, CLI binary release, downstream notifications). Failed run: https://github.com/hyperdxio/hyperdx/actions/runs/26666123898/job/78599749256 The npm publish of `@hyperdx/cli@0.5.0` actually **succeeded**, but the job still exited non-zero on the subsequent git tag push: ``` [command]/usr/bin/git push origin @hyperdx/cli@0.5.0 [command]/usr/bin/git push origin @hyperdx/cli@0.5.0 ← pushed twice * [new tag] @hyperdx/cli@0.5.0 -> @hyperdx/cli@0.5.0 ! [remote rejected] @hyperdx/cli@0.5.0 -> @hyperdx/cli@0.5.0 (cannot lock ref 'refs/tags/@hyperdx/cli@0.5.0': reference already exists) error: failed to push some refs to 'https://github.com/hyperdxio/hyperdx' ##[error]Error: The process '/usr/bin/git' failed with exit code 1 ``` ## Root cause The `release` script was: ```jsonc "release": "npx changeset tag && npx changeset publish" ``` - `changeset tag` creates git tags for **all** packages (including the 4 private ones that are never published to npm). - `changeset publish` then publishes to npm **and also creates a git tag for each package it publishes**. When a publishable package (`@hyperdx/cli`) is published for the **first time** in a run, both commands create its tag. The changesets GitHub Action then pushes each queued tag individually, so `@hyperdx/cli@0.5.0` got pushed twice. The first push succeeds; the second is rejected with `reference already exists`, and git exits 1 — failing the whole job. (The 4 private packages only get tagged once, by `changeset tag`, so they never collide. Only the freshly-published public package double-pushes.) ## Fix ```jsonc "release": "npx changeset publish --no-git-tag && npx changeset tag" ``` `changeset publish --no-git-tag` handles npm only and no longer creates git tags. `changeset tag` becomes the single source of truth for git tags across **all** packages (public and private), so each tag is created and pushed exactly once. This preserves the existing behavior of tagging every package while eliminating the duplicate push. `--no-git-tag` is supported by the pinned `@changesets/cli@2.26.2`. ## Note on the affected release The blocked `2.28.0` / `cli@0.5.0` release was unblocked out-of-band by deleting the stale git tags and the leftover changeset-generated GitHub Releases and re-running the workflow; that release has since completed. This PR only prevents the failure from recurring on future first-time package publishes. ## Testing - `make ci-lint` (formatting/lint) - Verified `changeset publish --no-git-tag` is a valid flag in `@changesets/cli@2.26.2`. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e2133c7e01..4e1cfa3c04 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "lint": "npx nx run-many -t ci:lint", "lint:fix": "npx nx run-many -t lint:fix", "version": "make version", - "release": "npx changeset tag && npx changeset publish" + "release": "npx changeset publish --no-git-tag && npx changeset tag" }, "lint-staged": { "packages/api/src/routers/external-api/**/*.ts": [ From d1802e1c8b7de302d11992974826c28adfda0ef6 Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:06:31 +0200 Subject: [PATCH 124/728] feat: add trace timeline minimap (#2552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - New `TimelineMinimap`: time axis + one bar per span over the full trace range. Brush to zoom, drag the viewport rect to pan, drag its edges to resize, double-click or reset button to zoom out. Dimmed overlay + framed rect show the current viewport. - New `TimelineViewportController` in `TimelineChart` — imperative bridge (`getState`, rAF-throttled `subscribe`, `zoomToRange`, `panToOffset`, `reset`) exposed via `onReady`, so the minimap reads/drives the chart's scroll-based zoom/pan without re-rendering virtualized rows. Wheel-zoom refactored through a shared `commitViewport` so wheel, native scroll, and minimap stay in sync. - Extracted `getMaxEventValue` so chart and minimap derive an identical range. - Wired into `DBTraceWaterfallChart` (shown once rows load with visible spans). ### Screenshots or video https://github.com/user-attachments/assets/4496e134-877e-42f4-adde-1ee7677376a4 ### How to test on Vercel preview **Preview routes:** `/search` **Steps:** 1. Open the trace viewer and explore a timeline with the minimap. ### References - Linear Issue: Closes HDX-3951 - Related PRs: Prototype PR: https://github.com/hyperdxio/hyperdx/pull/1970 --- .changeset/trace-minimap-brush-zoom-pan.md | 5 + .../src/components/DBTraceWaterfallChart.tsx | 21 +- .../TimelineChart/TimelineChart.tsx | 178 ++++++++- .../TimelineChart/TimelineMinimap.module.scss | 91 +++++ .../TimelineChart/TimelineMinimap.tsx | 370 ++++++++++++++++++ .../__tests__/TimelineMinimap.test.tsx | 205 ++++++++++ .../TimelineChart/__tests__/utils.test.ts | 30 +- .../app/src/components/TimelineChart/index.ts | 2 + .../app/src/components/TimelineChart/utils.ts | 15 + .../__tests__/DBTraceWaterfallChart.test.tsx | 9 +- 10 files changed, 909 insertions(+), 17 deletions(-) create mode 100644 .changeset/trace-minimap-brush-zoom-pan.md create mode 100644 packages/app/src/components/TimelineChart/TimelineMinimap.module.scss create mode 100644 packages/app/src/components/TimelineChart/TimelineMinimap.tsx create mode 100644 packages/app/src/components/TimelineChart/__tests__/TimelineMinimap.test.tsx diff --git a/.changeset/trace-minimap-brush-zoom-pan.md b/.changeset/trace-minimap-brush-zoom-pan.md new file mode 100644 index 0000000000..f76050780d --- /dev/null +++ b/.changeset/trace-minimap-brush-zoom-pan.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat(trace): add a trace minimap above the waterfall diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index be1c75a2fa..cc1e786820 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -40,7 +40,11 @@ import { import { ContactSupportText } from '@/components/ContactSupportText'; import SearchInputV2 from '@/components/SearchInput/SearchInputV2'; -import { TimelineChart } from '@/components/TimelineChart'; +import { + TimelineChart, + TimelineMinimap, + type TimelineViewportController, +} from '@/components/TimelineChart'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; import useRowWhere, { WithClause } from '@/hooks/useRowWhere'; import useWaterfallSearchState from '@/hooks/useWaterfallSearchState'; @@ -1004,8 +1008,22 @@ export function DBTraceWaterfallChartContainer({ const { ref: timelineWrapperRef, height: timelineWrapperHeight } = useElementSize(); + const [viewportController, setViewportController] = + useState(null); + + const showMinimap = + !isFetching && !error && rows != null && visibleNodes.length > 0; + return ( <> + {showMinimap && ( + + + + )} {isFilterExpanded && (
)}
diff --git a/packages/app/src/components/TimelineChart/TimelineChart.tsx b/packages/app/src/components/TimelineChart/TimelineChart.tsx index 6f340f1f78..732b61f2d7 100644 --- a/packages/app/src/components/TimelineChart/TimelineChart.tsx +++ b/packages/app/src/components/TimelineChart/TimelineChart.tsx @@ -18,6 +18,7 @@ import { TimelineChartRowEvents, type TTimelineEvent, } from './TimelineChartRowEvents'; +import { getMaxEventValue } from './utils'; import styles from './TimelineChart.module.scss'; import resizeStyles from '@styles/ResizablePanel.module.scss'; @@ -30,6 +31,37 @@ type Row = { type?: string; }; +/** + * Imperative bridge between the scroll-based zoom/pan model owned by + * TimelineChart and the TimelineMinimap (which renders above the controls row, + * outside this component). The minimap reads the current viewport via + * getState(), reflects live scroll/zoom changes via subscribe(), and drives + * zoom/pan through zoomToRange/panToOffset/reset. + * + * All fractions are in [0, 1] over the events timeline (the label column is + * excluded from the events area, matching how spans are positioned). + */ +export type TimelineViewportState = { + // Current zoom scale (>= 1; 1 = fully zoomed out). + scale: number; + // Fraction of the timeline scrolled past the left edge of the events area. + offsetFrac: number; + // Fraction of the timeline currently visible in the events area. + viewportWidthFrac: number; +}; + +export type TimelineViewportController = { + getState: () => TimelineViewportState; + // Subscribe to scroll/zoom changes (rAF-throttled). Returns an unsubscribe. + subscribe: (cb: () => void) => () => void; + // Zoom so the events area shows exactly [startFrac, endFrac]. + zoomToRange: (startFrac: number, endFrac: number) => void; + // Pan (at the current scale) so the events area's left edge is at offsetFrac. + panToOffset: (offsetFrac: number) => void; + // Return to the fully zoomed-out view. + reset: () => void; +}; + type TimelineChartProps = { initialScrollRowIndex: number; labelWidth: number; @@ -37,8 +69,13 @@ type TimelineChartProps = { rowHeight: number; rows: Row[]; onEventClick: (e: Row) => void; + onReady?: (controller: TimelineViewportController) => void; }; +// Smallest selectable viewport width as a timeline fraction. Guards against +// divide-by-zero / runaway scale when brushing or resizing to a tiny range. +const MIN_VIEWPORT_FRAC = 0.02; + const axisHeight = 24; const rowsMarginTop = 32; const maxScale = 100; @@ -51,6 +88,7 @@ export const TimelineChart = memo(function (props: TimelineChartProps) { rowHeight, rows, onEventClick, + onReady, } = props; const initialWidthPercent = (initialLabelWidth / window.innerWidth) * 100; @@ -127,17 +165,133 @@ export const TimelineChart = memo(function (props: TimelineChartProps) { } }, [initialScrollRowIndex, rowVirtualizer]); - const maxVal = useMemo(() => { - let max = 0; + const maxVal = useMemo(() => getMaxEventValue(rows), [rows]); - for (const row of rows) { - for (const event of row.events) { - max = Math.max(max, event.end); - } + // Subscribers (the minimap) notified on scroll/zoom so they can re-read the + // viewport. rAF-throttled so native scroll bursts collapse to one read. + const subscribersRef = useRef void>>(new Set()); + const notifyRafRef = useRef(null); + + const notifyViewport = useStableCallback(() => { + if (notifyRafRef.current != null) return; + notifyRafRef.current = requestAnimationFrame(() => { + notifyRafRef.current = null; + subscribersRef.current.forEach(cb => cb()); + }); + }); + + // Low-level commit shared by wheel-zoom and the minimap: set the scale, the + // scroller width, and the scroll position, then re-layout the axis/cursor and + // notify viewport subscribers. The browser clamps scrollLeft to a valid range. + const commitViewport = useStableCallback( + (newScale: number, newScrollLeft: number) => { + const container = timelineRef.current; + const scroller = timelineScrollerRef.current; + if (!container || !scroller) return; + + const clamped = Math.min(Math.max(newScale, 1), maxScale); + scaleRef.current = clamped; + scroller.style.width = `${100 * clamped}%`; + container.scrollLeft = newScrollLeft; + + xAxisHandleRef.current?.recompute(); + mouseCursorHandleRef.current?.recompute(); + notifyViewport(); + }, + ); + + const getViewportState = useStableCallback((): TimelineViewportState => { + const container = timelineRef.current; + const scale = scaleRef.current; + const clientW = container?.clientWidth ?? 1; + const labelW = labelWidthRef.current; + // Events area width at the current scale; matches the geometry used by + // flushWheel (scroller width is 100*scale% of the container). + const eventsW = Math.max(1, clientW * scale - labelW); + const offsetFrac = container ? container.scrollLeft / eventsW : 0; + const viewportWidthFrac = Math.min(1, (clientW - labelW) / eventsW); + return { scale, offsetFrac, viewportWidthFrac }; + }); + + const zoomToRange = useStableCallback( + (startFrac: number, endFrac: number) => { + const container = timelineRef.current; + if (!container) return; + const clientW = container.clientWidth; + const labelW = labelWidthRef.current; + const widthFrac = Math.max(endFrac - startFrac, MIN_VIEWPORT_FRAC); + // Invert viewportWidthFrac = (W - labelW) / (W*scale - labelW). + const newScale = (labelW + (clientW - labelW) / widthFrac) / clientW; + const clamped = Math.min(Math.max(newScale, 1), maxScale); + const eventsWNew = Math.max(1, clientW * clamped - labelW); + commitViewport(clamped, startFrac * eventsWNew); + }, + ); + + const panToOffset = useStableCallback((offsetFrac: number) => { + const container = timelineRef.current; + if (!container) return; + const clientW = container.clientWidth; + const labelW = labelWidthRef.current; + const eventsW = Math.max(1, clientW * scaleRef.current - labelW); + commitViewport(scaleRef.current, offsetFrac * eventsW); + }); + + const resetViewport = useStableCallback(() => { + commitViewport(1, 0); + }); + + const subscribeViewport = useStableCallback((cb: () => void) => { + subscribersRef.current.add(cb); + return () => { + subscribersRef.current.delete(cb); + }; + }); + + // Stable controller object created once; methods are stable (useStableCallback). + const controllerRef = useRef(null); + if (controllerRef.current == null) { + controllerRef.current = { + getState: getViewportState, + subscribe: subscribeViewport, + zoomToRange, + panToOffset, + reset: resetViewport, + }; + } + + useEffect(() => { + if (controllerRef.current != null) { + onReady?.(controllerRef.current); } + }, [onReady]); - return max * 1.1; - }, [rows]); + // Native horizontal scroll (trackpad / scrollbar) pans the events area; keep + // the minimap viewport rectangle in sync. + useEffect(() => { + const element = timelineRef.current; + if (element == null) return; + const onScroll = () => notifyViewport(); + element.addEventListener('scroll', onScroll, { passive: true }); + return () => element.removeEventListener('scroll', onScroll); + }, [notifyViewport]); + + // Re-notify subscribers when the label column is resized, because the + // events-area width fraction changes even though scale and scrollLeft don't. + // Without this the minimap viewport rect keeps stale offset/width fractions + // until the next scroll or wheel-zoom. + useEffect(() => { + notifyViewport(); + }, [labelWidth, notifyViewport]); + + useEffect(() => { + return () => { + if (notifyRafRef.current != null) { + cancelAnimationFrame(notifyRafRef.current); + notifyRafRef.current = null; + } + }; + }, []); // Wheel deltas accumulate into a pending state and the zoom commit runs // once per frame via rAF, so a fast scroll in one frame collapses to a @@ -191,12 +345,8 @@ export const TimelineChart = memo(function (props: TimelineChartProps) { const fraction = (container.scrollLeft + cursorPx) / eventsWOld; - scaleRef.current = newScale; - scroller.style.width = `${100 * newScale}%`; - container.scrollLeft = fraction * eventsWNew - cursorPx; - - xAxisHandleRef.current?.recompute(); - mouseCursorHandleRef.current?.recompute(); + // Keep the cursor anchored to the same timeline fraction across the zoom. + commitViewport(newScale, fraction * eventsWNew - cursorPx); }); const onWheel = useStableCallback((e: WheelEvent) => { diff --git a/packages/app/src/components/TimelineChart/TimelineMinimap.module.scss b/packages/app/src/components/TimelineChart/TimelineMinimap.module.scss new file mode 100644 index 0000000000..3144eebd00 --- /dev/null +++ b/packages/app/src/components/TimelineChart/TimelineMinimap.module.scss @@ -0,0 +1,91 @@ +.container { + position: relative; + border: 1px solid var(--color-border); + background-color: var(--color-bg-surface); + overflow: hidden; + user-select: none; +} + +.tick { + position: absolute; + top: 0; + border-left: 1px solid var(--color-border-muted); + pointer-events: none; +} + +.tickLabel { + font-size: 10px; + color: var(--color-text-muted); + margin-left: 3px; + white-space: nowrap; + user-select: none; +} + +.bar { + position: absolute; + border-radius: 1px; + pointer-events: none; +} + +.dimmedOverlay { + position: absolute; + top: 0; + height: 100%; + background-color: var(--color-viz-range-overlay); + pointer-events: none; +} + +.viewportFrame { + position: absolute; + top: 0; + height: 100%; + box-sizing: border-box; + border: 1px solid var(--color-viz-range-border); + pointer-events: none; +} + +.handleGrip { + position: absolute; + top: 50%; + transform: translateY(-50%); + width: 6px; + height: 20px; + background-color: var(--color-viz-range-handle); + border-radius: 2px; + display: flex; + align-items: center; + justify-content: center; + gap: 1px; + + &Left { + left: -3px; + } + + &Right { + right: -3px; + } +} + +.handleGripLine { + width: 0; + height: 8px; + border-left: 1px solid var(--color-viz-range-handle-grip); +} + +.brushOverlay { + position: absolute; + top: 0; + height: 100%; + background-color: var(--color-viz-range-brush); + border: 1px solid var(--color-viz-range-brush-border); + box-sizing: border-box; + pointer-events: none; +} + +.resetButton { + position: absolute; + top: 3px; + right: 3px; + z-index: 4; + pointer-events: auto; +} diff --git a/packages/app/src/components/TimelineChart/TimelineMinimap.tsx b/packages/app/src/components/TimelineChart/TimelineMinimap.tsx new file mode 100644 index 0000000000..ba93fffe6e --- /dev/null +++ b/packages/app/src/components/TimelineChart/TimelineMinimap.tsx @@ -0,0 +1,370 @@ +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { ActionIcon, Tooltip } from '@mantine/core'; +import { IconZoomReset } from '@tabler/icons-react'; + +import type { TimelineViewportController } from './TimelineChart'; +import type { TTimelineEvent } from './TimelineChartRowEvents'; +import { getMaxEventValue, renderMs } from './utils'; + +import styles from './TimelineMinimap.module.scss'; + +type MinimapRow = { + events: TTimelineEvent[]; +}; + +type DragMode = 'pan' | 'resize-left' | 'resize-right' | 'brush' | null; + +type TimelineMinimapProps = { + rows: MinimapRow[]; + controller: TimelineViewportController | null; +}; + +const TICK_HEIGHT = 18; +const BAR_AREA_HEIGHT = 34; +const MINIMAP_HEIGHT = TICK_HEIGHT + BAR_AREA_HEIGHT; +// Pixel width of the grab zone on each side of the viewport for resize handles. +const HANDLE_HIT_AREA = 8; +const BAR_HEIGHT = 2; +// Below this brushed width (timeline fraction) a brush is treated as a click, +// not a zoom — avoids accidental extreme zooms on a stray drag. +const MIN_BRUSH_FRAC = 0.01; +// Smallest range a resize handle can produce; mirrors MIN_VIEWPORT_FRAC in +// TimelineChart so the two never disagree about the minimum window. +const MIN_RANGE_FRAC = 0.02; +// scale must exceed this to count as "zoomed" (guards float noise near 1). +const ZOOMED_EPSILON = 1.01; + +function getXFraction(clientX: number, containerEl: HTMLDivElement): number { + const rect = containerEl.getBoundingClientRect(); + return Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); +} + +const HandleGrip = ({ side }: { side: 'left' | 'right' }) => ( +
+
+
+
+); + +export const TimelineMinimap = memo(function ({ + rows, + controller, +}: TimelineMinimapProps) { + const containerRef = useRef(null); + const dragModeRef = useRef(null); + const dragStartXRef = useRef(0); + // Viewport at drag start, captured from the controller so pan/resize math is + // relative to a stable origin even as the controller mutates mid-drag. + const dragStartOffsetFracRef = useRef(0); + const dragStartWidthFracRef = useRef(1); + const brushStartFracRef = useRef(0); + const brushRangeRef = useRef<{ start: number; end: number } | null>(null); + + const [brushRange, setBrushRange] = useState<{ + start: number; + end: number; + } | null>(null); + + // Local mirror of the chart's viewport, refreshed via the controller + // subscription. Keeps the rectangle in sync with native scroll + wheel zoom + // without re-rendering the (virtualized) waterfall rows. + const [viewport, setViewport] = useState({ + offsetFrac: 0, + widthFrac: 1, + scale: 1, + }); + + useEffect(() => { + if (!controller) return; + const update = () => { + const s = controller.getState(); + setViewport({ + offsetFrac: s.offsetFrac, + widthFrac: s.viewportWidthFrac, + scale: s.scale, + }); + }; + update(); + return controller.subscribe(update); + }, [controller]); + + const maxVal = useMemo(() => getMaxEventValue(rows), [rows]); + + const viewportStartFrac = viewport.offsetFrac; + const viewportWidthFrac = viewport.widthFrac; + const viewportEndFrac = Math.min(viewportStartFrac + viewportWidthFrac, 1); + const isZoomed = viewport.scale > ZOOMED_EPSILON; + + const ticks = useMemo(() => { + if (maxVal <= 0) return []; + const TARGET_TICKS = 6; + const rawInterval = maxVal / TARGET_TICKS; + const magnitude = Math.pow(10, Math.floor(Math.log10(rawInterval))); + let interval = magnitude; + if (rawInterval >= 2 * magnitude) interval = 2 * magnitude; + if (rawInterval >= 5 * magnitude) interval = 5 * magnitude; + + const result = []; + for (let i = 0; ; i++) { + const val = i * interval; + const frac = val / maxVal; + if (frac > 1) break; + result.push({ frac, label: renderMs(val) }); + } + return result; + }, [maxVal]); + + const handlePointerDown = useCallback( + (e: React.PointerEvent) => { + const el = containerRef.current; + if (!el || !controller) return; + + e.preventDefault(); + e.stopPropagation(); + el.setPointerCapture(e.pointerId); + + const x = getXFraction(e.clientX, el); + const handleFrac = HANDLE_HIT_AREA / el.getBoundingClientRect().width; + + const s = controller.getState(); + const startEdge = s.offsetFrac; + const endEdge = Math.min(s.offsetFrac + s.viewportWidthFrac, 1); + const zoomed = s.scale > ZOOMED_EPSILON; + + dragStartXRef.current = e.clientX; + dragStartOffsetFracRef.current = s.offsetFrac; + dragStartWidthFracRef.current = s.viewportWidthFrac; + + if (zoomed && Math.abs(x - startEdge) < handleFrac) { + dragModeRef.current = 'resize-left'; + } else if (zoomed && Math.abs(x - endEdge) < handleFrac) { + dragModeRef.current = 'resize-right'; + } else if (zoomed && x >= startEdge && x <= endEdge) { + dragModeRef.current = 'pan'; + } else { + dragModeRef.current = 'brush'; + brushStartFracRef.current = x; + const range = { start: x, end: x }; + brushRangeRef.current = range; + setBrushRange(range); + } + }, + [controller], + ); + + const handlePointerMove = useCallback( + (e: React.PointerEvent) => { + if (!dragModeRef.current || !controller) return; + const el = containerRef.current; + if (!el) return; + + const rect = el.getBoundingClientRect(); + + if (dragModeRef.current === 'brush') { + const x = getXFraction(e.clientX, el); + const start = Math.min(brushStartFracRef.current, x); + const end = Math.max(brushStartFracRef.current, x); + const range = { start, end }; + brushRangeRef.current = range; + setBrushRange(range); + return; + } + + const deltaFrac = (e.clientX - dragStartXRef.current) / rect.width; + + if (dragModeRef.current === 'pan') { + const maxOffset = Math.max(0, 1 - dragStartWidthFracRef.current); + const newOffset = Math.min( + Math.max(dragStartOffsetFracRef.current + deltaFrac, 0), + maxOffset, + ); + controller.panToOffset(newOffset); + } else if (dragModeRef.current === 'resize-left') { + const currentStart = dragStartOffsetFracRef.current; + const currentEnd = Math.min( + currentStart + dragStartWidthFracRef.current, + 1, + ); + const newStart = Math.max( + 0, + Math.min(currentStart + deltaFrac, currentEnd - MIN_RANGE_FRAC), + ); + controller.zoomToRange(newStart, currentEnd); + } else if (dragModeRef.current === 'resize-right') { + const currentStart = dragStartOffsetFracRef.current; + const currentEnd = Math.min( + currentStart + dragStartWidthFracRef.current, + 1, + ); + const newEnd = Math.min( + 1, + Math.max(currentEnd + deltaFrac, currentStart + MIN_RANGE_FRAC), + ); + controller.zoomToRange(currentStart, newEnd); + } + }, + [controller], + ); + + const handlePointerUp = useCallback(() => { + if (dragModeRef.current === 'brush' && controller) { + const range = brushRangeRef.current; + if (range && range.end - range.start > MIN_BRUSH_FRAC) { + controller.zoomToRange(range.start, range.end); + } + } + dragModeRef.current = null; + brushRangeRef.current = null; + setBrushRange(null); + }, [controller]); + + const handleDoubleClick = useCallback(() => { + if (controller && isZoomed) { + controller.reset(); + } + }, [controller, isZoomed]); + + const handleReset = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + controller?.reset(); + }, + [controller], + ); + + const getCursor = useCallback( + (e: React.PointerEvent) => { + if (!isZoomed) return 'crosshair'; + const el = containerRef.current; + if (!el) return 'crosshair'; + const x = getXFraction(e.clientX, el); + const handleFrac = HANDLE_HIT_AREA / el.getBoundingClientRect().width; + + if (Math.abs(x - viewportStartFrac) < handleFrac) return 'col-resize'; + if (Math.abs(x - viewportEndFrac) < handleFrac) return 'col-resize'; + if (x >= viewportStartFrac && x <= viewportEndFrac) return 'grab'; + return 'crosshair'; + }, + [viewportStartFrac, viewportEndFrac, isZoomed], + ); + + return ( +
{ + handlePointerMove(e); + if (!dragModeRef.current) { + e.currentTarget.style.cursor = getCursor(e); + } else if (dragModeRef.current === 'pan') { + e.currentTarget.style.cursor = 'grabbing'; + } else if (dragModeRef.current === 'brush') { + e.currentTarget.style.cursor = 'crosshair'; + } else { + e.currentTarget.style.cursor = 'col-resize'; + } + }} + onPointerUp={handlePointerUp} + onPointerCancel={handlePointerUp} + > + {ticks.map(({ frac, label }) => ( +
+ {label} +
+ ))} + + {maxVal > 0 && + rows.map((row, rowIdx) => + row.events.map(event => { + const left = (event.start / maxVal) * 100; + const width = Math.max( + ((event.end - event.start) / maxVal) * 100, + 0.15, + ); + return ( +
+ ); + }), + )} + + {isZoomed && ( + <> +
+
+
+ + +
+ + )} + + {brushRange && ( +
+ )} + + {isZoomed && ( + + e.stopPropagation()} + onDoubleClick={e => e.stopPropagation()} + > + + + + )} +
+ ); +}); + +TimelineMinimap.displayName = 'TimelineMinimap'; diff --git a/packages/app/src/components/TimelineChart/__tests__/TimelineMinimap.test.tsx b/packages/app/src/components/TimelineChart/__tests__/TimelineMinimap.test.tsx new file mode 100644 index 0000000000..687b476ed7 --- /dev/null +++ b/packages/app/src/components/TimelineChart/__tests__/TimelineMinimap.test.tsx @@ -0,0 +1,205 @@ +import { fireEvent, screen } from '@testing-library/react'; + +import type { TimelineViewportController } from '@/components/TimelineChart'; +import { TimelineMinimap } from '@/components/TimelineChart/TimelineMinimap'; + +// jsdom has no layout engine, so getBoundingClientRect returns zeros and +// setPointerCapture is unimplemented. Pin the container to a known 1000px-wide +// box and stub pointer capture so getXFraction math is deterministic +// (clientX / 1000 = fraction). +const CONTAINER_WIDTH = 1000; + +beforeAll(() => { + // jsdom doesn't implement PointerEvent, so fireEvent.pointerXxx would drop + // clientX. Back it with MouseEvent (which honors clientX from init). + if (typeof window.PointerEvent === 'undefined') { + // @ts-expect-error minimal polyfill for tests + window.PointerEvent = class extends MouseEvent { + constructor(type: string, params: MouseEventInit = {}) { + super(type, params); + } + }; + } + Element.prototype.setPointerCapture = jest.fn(); + Element.prototype.releasePointerCapture = jest.fn(); + jest.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ + left: 0, + top: 0, + right: CONTAINER_WIDTH, + bottom: 52, + width: CONTAINER_WIDTH, + height: 52, + x: 0, + y: 0, + toJSON: () => ({}), + } as DOMRect); +}); + +afterAll(() => { + jest.restoreAllMocks(); +}); + +const rows = [ + { + events: [ + { + id: 'a', + start: 0, + end: 100, + tooltip: '', + color: '#fff', + backgroundColor: 'rgb(10, 20, 30)', + body: null, + }, + { + id: 'b', + start: 100, + end: 300, + tooltip: '', + color: '#fff', + backgroundColor: 'rgb(40, 50, 60)', + body: null, + }, + ], + }, +]; + +function makeController(state: { + scale: number; + offsetFrac: number; + viewportWidthFrac: number; +}): TimelineViewportController { + return { + getState: jest.fn(() => state), + subscribe: jest.fn(() => () => {}), + zoomToRange: jest.fn(), + panToOffset: jest.fn(), + reset: jest.fn(), + }; +} + +describe('TimelineMinimap', () => { + it('renders service-colored bars from event.backgroundColor', () => { + const controller = makeController({ + scale: 1, + offsetFrac: 0, + viewportWidthFrac: 1, + }); + renderWithMantine(); + const minimap = screen.getByTestId('timeline-minimap'); + const bars = Array.from(minimap.querySelectorAll('div')).filter(el => + /rgb\(/.test(el.style.backgroundColor), + ); + expect(bars.length).toBe(2); + expect(bars.map(b => b.style.backgroundColor)).toEqual( + expect.arrayContaining(['rgb(10, 20, 30)', 'rgb(40, 50, 60)']), + ); + }); + + it('hides viewport chrome (reset button) when fully zoomed out', () => { + const controller = makeController({ + scale: 1, + offsetFrac: 0, + viewportWidthFrac: 1, + }); + renderWithMantine(); + expect( + screen.queryByRole('button', { name: /reset minimap zoom/i }), + ).not.toBeInTheDocument(); + }); + + it('shows the reset button when zoomed and resets on click', () => { + const controller = makeController({ + scale: 4, + offsetFrac: 0.25, + viewportWidthFrac: 0.25, + }); + renderWithMantine(); + const resetBtn = screen.getByRole('button', { + name: /reset minimap zoom/i, + }); + expect(resetBtn).toBeInTheDocument(); + fireEvent.click(resetBtn); + expect(controller.reset).toHaveBeenCalledTimes(1); + }); + + it('brush-to-zoom: dragging a range calls zoomToRange with that range', () => { + const controller = makeController({ + scale: 1, + offsetFrac: 0, + viewportWidthFrac: 1, + }); + renderWithMantine(); + const minimap = screen.getByTestId('timeline-minimap'); + + fireEvent.pointerDown(minimap, { clientX: 200, pointerId: 1 }); + fireEvent.pointerMove(minimap, { clientX: 600, pointerId: 1 }); + fireEvent.pointerUp(minimap, { clientX: 600, pointerId: 1 }); + + expect(controller.zoomToRange).toHaveBeenCalledTimes(1); + const [start, end] = (controller.zoomToRange as jest.Mock).mock.calls[0]; + expect(start).toBeCloseTo(0.2, 5); + expect(end).toBeCloseTo(0.6, 5); + }); + + it('drag-to-pan: dragging inside the viewport when zoomed calls panToOffset', () => { + const controller = makeController({ + scale: 4, + offsetFrac: 0.25, + viewportWidthFrac: 0.25, + }); + renderWithMantine(); + const minimap = screen.getByTestId('timeline-minimap'); + + // viewport spans frac [0.25, 0.5]; start the drag at its middle (0.375). + fireEvent.pointerDown(minimap, { clientX: 375, pointerId: 1 }); + fireEvent.pointerMove(minimap, { clientX: 475, pointerId: 1 }); + + expect(controller.panToOffset).toHaveBeenCalled(); + const lastCall = (controller.panToOffset as jest.Mock).mock.calls.at(-1); + // moved +0.1 frac → offset 0.25 -> ~0.35 + expect(lastCall[0]).toBeCloseTo(0.35, 5); + expect(controller.zoomToRange).not.toHaveBeenCalled(); + }); + + it('resize handle: dragging the right edge re-zooms to the new range', () => { + const controller = makeController({ + scale: 4, + offsetFrac: 0.25, + viewportWidthFrac: 0.25, + }); + renderWithMantine(); + const minimap = screen.getByTestId('timeline-minimap'); + + // right edge at frac 0.5 (px 500); grab it and drag right by 0.2 (200px). + fireEvent.pointerDown(minimap, { clientX: 500, pointerId: 1 }); + fireEvent.pointerMove(minimap, { clientX: 700, pointerId: 1 }); + + expect(controller.zoomToRange).toHaveBeenCalled(); + const lastCall = (controller.zoomToRange as jest.Mock).mock.calls.at(-1); + expect(lastCall[0]).toBeCloseTo(0.25, 5); // start unchanged + expect(lastCall[1]).toBeCloseTo(0.7, 5); // end pushed to 0.7 + }); + + it('double-click resets when zoomed', () => { + const controller = makeController({ + scale: 4, + offsetFrac: 0.25, + viewportWidthFrac: 0.25, + }); + renderWithMantine(); + fireEvent.doubleClick(screen.getByTestId('timeline-minimap')); + expect(controller.reset).toHaveBeenCalledTimes(1); + }); + + it('double-click does nothing when not zoomed', () => { + const controller = makeController({ + scale: 1, + offsetFrac: 0, + viewportWidthFrac: 1, + }); + renderWithMantine(); + fireEvent.doubleClick(screen.getByTestId('timeline-minimap')); + expect(controller.reset).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app/src/components/TimelineChart/__tests__/utils.test.ts b/packages/app/src/components/TimelineChart/__tests__/utils.test.ts index 2975775189..8b05166549 100644 --- a/packages/app/src/components/TimelineChart/__tests__/utils.test.ts +++ b/packages/app/src/components/TimelineChart/__tests__/utils.test.ts @@ -1,4 +1,8 @@ -import { calculateInterval, renderMs } from '@/components/TimelineChart/utils'; +import { + calculateInterval, + getMaxEventValue, + renderMs, +} from '@/components/TimelineChart/utils'; describe('renderMs', () => { it('formats sub-second values as ms', () => { @@ -106,3 +110,27 @@ describe('calculateInterval', () => { expect(calculateInterval(-5)).toBe(1); }); }); + +describe('getMaxEventValue', () => { + it('returns the max event end plus 10% padding', () => { + const rows = [ + { events: [{ end: 100 }, { end: 250 }] }, + { events: [{ end: 300 }] }, + ]; + expect(getMaxEventValue(rows)).toBeCloseTo(330); + }); + + it('ignores rows with no events and finds the global max', () => { + const rows = [ + { events: [] }, + { events: [{ end: 50 }] }, + { events: [{ end: 10 }, { end: 500 }] }, + ]; + expect(getMaxEventValue(rows)).toBeCloseTo(550); + }); + + it('returns 0 for empty input', () => { + expect(getMaxEventValue([])).toBe(0); + expect(getMaxEventValue([{ events: [] }])).toBe(0); + }); +}); diff --git a/packages/app/src/components/TimelineChart/index.ts b/packages/app/src/components/TimelineChart/index.ts index c059823eb9..37eac98d67 100644 --- a/packages/app/src/components/TimelineChart/index.ts +++ b/packages/app/src/components/TimelineChart/index.ts @@ -1 +1,3 @@ +export type { TimelineViewportController } from './TimelineChart'; export { TimelineChart } from './TimelineChart'; +export { TimelineMinimap } from './TimelineMinimap'; diff --git a/packages/app/src/components/TimelineChart/utils.ts b/packages/app/src/components/TimelineChart/utils.ts index 998827e997..0bf8cd3dfc 100644 --- a/packages/app/src/components/TimelineChart/utils.ts +++ b/packages/app/src/components/TimelineChart/utils.ts @@ -1,3 +1,18 @@ +// Shared maxVal used to position events across the waterfall and the minimap. +// Adds 10% right padding so the last span never touches the edge. Kept here so +// TimelineChart and TimelineMinimap derive an identical timeline range. +export function getMaxEventValue( + rows: { events: { end: number }[] }[], +): number { + let max = 0; + for (const row of rows) { + for (const event of row.events) { + max = Math.max(max, event.end); + } + } + return max * 1.1; +} + export function renderMs(ms: number) { if (ms < 1) { const µsRounded = Math.round(ms * 1000); diff --git a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx index 60b591b706..b7df1ae082 100644 --- a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx @@ -55,7 +55,14 @@ jest.mock('@/components/TimelineChart', () => { ); }; mockComponent.latestProps = {}; - return { TimelineChart: mockComponent }; + + const MockTimelineMinimap = function MockTimelineMinimap(props: any) { + MockTimelineMinimap.latestProps = props; + return
TimelineMinimap
; + }; + MockTimelineMinimap.latestProps = {}; + + return { TimelineChart: mockComponent, TimelineMinimap: MockTimelineMinimap }; }); jest.mock('@/hooks/useOffsetPaginatedQuery'); From c7785f3f57625a7ddfb18c697c84e024f23dc050 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:11:03 -0700 Subject: [PATCH 125/728] [HDX-4666] Add OSS release reminder workflow (#2557) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary OSS releases are automated end-to-end by [changesets](https://github.com/changesets/changesets) + `.github/workflows/release.yml`: a push to `main` opens/updates an auto-generated **"Release HyperDX"** version-bump PR, and merging that PR is what actually cuts the release (publishes the npm packages, CLI binaries, and multi-arch Docker images, then fires the downstream syncs). The catch: **merging that Release PR is a manual step**, and it's easy to forget. That's how the last OSS release slipped ~a month — the community noticed in #observability that releases had stalled while changesets piled up (HDX-4666). This PR adds a safety net. It deliberately does **not** rebuild the publish pipeline (already exists) and does **not** auto-merge the Release PR (we keep the human review gate on the version bump). **Change:** - **`.github/workflows/release-reminder.yml`** — a scheduled workflow (Friday 20:00 UTC = 12pm PST / 1pm PDT) plus `workflow_dispatch`. It queries open PRs (read-only, no checkout) and, if an unmerged **"Release HyperDX"** PR is open, posts to **#hyperdx-pull-requests** with the PR link, how long it's been open, and a merge nudge — escalating the wording to "overdue" once it's been pending more than 7 days, and `@`-mentioning the release owner (via the `RELEASE_SLACK_MEMBER_ID` repo/org variable) so they get notified. If no Release PR is open, it stays silent. Reuses the existing `SLACK_WEBHOOK_OSS_PRS` secret and mirrors the conventions in `external-contributor-alerts.yml` (fail-soft, JSON-escaped payloads). No changeset is included: this is a CI workflow change with no published-package behavior change (exempt per `AGENTS.md`). ### Setup required Set the `RELEASE_SLACK_MEMBER_ID` repo variable to the release owner's Slack member ID so the reminder `@`-mentions them (leave unset to skip the mention). Already configured on this repo. ### How to test on Vercel preview N/A — non-UI change (CI workflow only). ### References - Linear Issue: [HDX-4666](https://linear.app/clickhouse/issue/HDX-4666/automate-the-oss-release-process-or-notify-hyperdx-pull-requests-so) - Related PRs: ### Testing notes - Workflow YAML validated (parses cleanly) and is Prettier-compliant. - Live Slack delivery can be verified after merge via the workflow's `workflow_dispatch` trigger. --- .github/workflows/release-reminder.yml | 76 ++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/release-reminder.yml diff --git a/.github/workflows/release-reminder.yml b/.github/workflows/release-reminder.yml new file mode 100644 index 0000000000..c0f3f762f3 --- /dev/null +++ b/.github/workflows/release-reminder.yml @@ -0,0 +1,76 @@ +name: Release Reminder + +on: + schedule: + - cron: '0 20 * * 5' + workflow_dispatch: + +permissions: + pull-requests: read + +concurrency: + group: release-reminder + cancel-in-progress: false + +jobs: + remind: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v9 + env: + SLACK_WEBHOOK_OSS_PRS: ${{ secrets.SLACK_WEBHOOK_OSS_PRS }} + with: + script: | + const MENTION_ID = '${{ vars.RELEASE_SLACK_MEMBER_ID }}'; + const RELEASE_PR_TITLE = 'Release HyperDX'; + const OVERDUE_DAYS = 7; + + const { owner, repo } = context.repo; + + const prs = await github.paginate(github.rest.pulls.list, { + owner, repo, state: 'open', per_page: 100, + }); + const releasePr = prs.find(pr => + pr.title === RELEASE_PR_TITLE && + pr.user?.type === 'Bot' && + (pr.user?.login === 'github-actions[bot]' || pr.user?.login === 'github-actions') + ); + + if (!releasePr) { + core.info('No open "Release HyperDX" PR found — nothing pending. Staying silent.'); + return; + } + + const openedAt = new Date(releasePr.created_at); + const daysOpen = Math.floor((Date.now() - openedAt.getTime()) / 86_400_000); + const overdue = daysOpen >= OVERDUE_DAYS; + + const mention = MENTION_ID ? `<@${MENTION_ID}> ` : ''; + const header = overdue + ? `:rotating_light: *OSS release is overdue* — the Release PR has been open ${daysOpen} day(s).` + : `:package: *OSS release is pending* — a Release PR has been open ${daysOpen} day(s).`; + const text = `${mention}${header}\n` + + `Merge it to cut the release (publishes Docker/npm/CLI artifacts):\n` + + `<${releasePr.html_url}|#${releasePr.number}: ${releasePr.title}>`; + + const webhook = process.env.SLACK_WEBHOOK_OSS_PRS; + if (!webhook) { + core.warning('SLACK_WEBHOOK_OSS_PRS not set — no reminder sent.'); + return; + } + + try { + const res = await fetch(webhook, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ blocks: [{ type: 'section', text: { type: 'mrkdwn', text } }] }), + }); + if (!res.ok) { + core.setFailed(`Slack post failed (${res.status}).`); + return; + } + } catch (e) { + core.setFailed(`Slack post errored (${e.message}).`); + return; + } + core.info(`Reminder sent for Release PR #${releasePr.number} (open ${daysOpen} day(s)).`); From abf5b537381155d815babe687389985a4ec55153 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 2 Jul 2026 00:34:54 +1000 Subject: [PATCH 126/728] feat(api): add POST /api/v2/dashboards/validate (#2562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds a `POST /api/v2/dashboards/validate` endpoint to the external v2 API. It validates a dashboard body **without persisting anything** and returns a structured result, so external clients can check a dashboard before writing it. ## Why The ClickStack Terraform provider (`clickstack_dashboard`) manages dashboards as a JSON body it sends verbatim to `POST/PUT /api/v2/dashboards`. Without a validate endpoint, the only way to surface a bad dashboard is at apply time (mid-`terraform apply`), after other resources may have changed. This endpoint lets the provider validate at **plan** time and show precise errors before any mutation. It is generally useful to any API consumer that wants a dry-run check. ## Contract `POST /api/v2/dashboards/validate` (Bearer auth, team-scoped, rate-limited like the other v2 routes). Always returns **HTTP 200** for both valid and invalid bodies: ```json { "valid": true|false, "errors": [{ "path": "tiles.0.config", "message": "..." }], "normalized": { ...defaults-applied body... } | null } ``` - 200 always (not 400 on invalid) so callers can distinguish "validation ran and found problems" from "the request itself failed" (auth/5xx). - errors aggregates all zod issues; a validateDashboardTiles failure is returned as a single {path:"", message}. - normalized echoes the zod-parsed body with defaults applied (no persistence, so no server-assigned tile IDs) for consumers that want a canonical form. Implementation - Reuses the exact createDashboardBodySchema and validateDashboardTiles the create path uses, so validation cannot drift from what create enforces (FK existence for sources/connections, container/tab refs, PromQL rejection). - No DB writes / no side effects; safe to call repeatedly (e.g. on every plan). - OpenAPI annotation added; /validate added to the auth-coverage test list. Testing packages/api/src/routers/external-api/__tests__/dashboards.test.ts: valid body → valid:true; malformed body → valid:false with errors; no-persist assertion (dashboard count unchanged); normalized-body-on-valid. Requires MongoDB + ClickHouse (run in CI / docker-compose.ci.yml); tsc --noEmit is clean. Notes PromQL tiles are rejected here (as on create) — not yet supported in the external API. Paired with the provider PR: . contributes to HDX-4671 --- .changeset/gold-lions-marry.md | 5 + packages/api/openapi.json | 108 ++++++++++++++++ .../external-api/__tests__/dashboards.test.ts | 40 ++++++ .../src/routers/external-api/v2/dashboards.ts | 121 ++++++++++++++++++ 4 files changed, 274 insertions(+) create mode 100644 .changeset/gold-lions-marry.md diff --git a/.changeset/gold-lions-marry.md b/.changeset/gold-lions-marry.md new file mode 100644 index 0000000000..fe090d8833 --- /dev/null +++ b/.changeset/gold-lions-marry.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +Adds a POST /api/v2/dashboards/validate endpoint to the external v2 API. It diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 5e1511be80..086996f928 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5779,6 +5779,114 @@ } } }, + "/api/v2/dashboards/validate": { + "post": { + "summary": "Validate Dashboard", + "description": "Validates a dashboard body against the same schema and tile rules used\nby POST /api/v2/dashboards. The dashboard is **never persisted**. Use\nthis endpoint at plan time (e.g. from a Terraform provider) to check\nthat a dashboard configuration is valid before applying it.\n", + "operationId": "validateDashboard", + "tags": [ + "Dashboards" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateDashboardRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Validation result. HTTP 200 is always returned for valid **and**\ninvalid bodies — a non-200 response means the request itself\nfailed (auth, server error, etc.).\n", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "valid", + "errors", + "normalized" + ], + "properties": { + "valid": { + "type": "boolean", + "description": "True when the body passes all validation rules." + }, + "errors": { + "type": "array", + "description": "Validation errors. Empty when valid is true.", + "items": { + "type": "object", + "required": [ + "path", + "message" + ], + "properties": { + "path": { + "type": "string", + "description": "Dot-separated field path, or empty string for top-level errors." + }, + "message": { + "type": "string", + "description": "Human-readable error description." + } + } + } + }, + "normalized": { + "type": "object", + "nullable": true, + "description": "The parsed dashboard body with defaults applied (no\npersistence, so no server-assigned tile IDs). Populated\nwhen valid is true, null when valid is false.\n" + } + } + }, + "examples": { + "valid": { + "summary": "Valid dashboard body", + "value": { + "valid": true, + "errors": [], + "normalized": { + "name": "My Dashboard", + "tiles": [] + } + } + }, + "invalid": { + "summary": "Invalid dashboard body", + "value": { + "valid": false, + "errors": [ + { + "path": "name", + "message": "Required" + } + ], + "normalized": null + } + } + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Unauthorized access. API key is missing or invalid." + } + } + } + } + } + } + }, "/api/v2/search": { "post": { "summary": "Search Raw Logs and Traces", diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts index 9222a7f191..079a1273aa 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts @@ -5893,6 +5893,7 @@ describe('External API v2 Dashboards - new format', () => { { method: 'post', path: BASE_URL }, { method: 'put', path: `${BASE_URL}/${testId}` }, { method: 'delete', path: `${BASE_URL}/${testId}` }, + { method: 'post', path: `${BASE_URL}/validate` }, ]; for (const { method, path } of routes) { @@ -5900,4 +5901,43 @@ describe('External API v2 Dashboards - new format', () => { } }); }); + + describe('POST /api/v2/dashboards/validate', () => { + it('returns valid:true for a well-formed builder dashboard', async () => { + const res = await authRequest('post', `${BASE_URL}/validate`).send({ + name: 'D', + tiles: [], + }); + expect(res.status).toBe(200); + expect(res.body.valid).toBe(true); + expect(res.body.errors).toEqual([]); + }); + + it('returns valid:false with errors for a malformed body', async () => { + const res = await authRequest('post', `${BASE_URL}/validate`).send({ + tiles: 'nope', + }); // name missing, tiles wrong type + expect(res.status).toBe(200); + expect(res.body.valid).toBe(false); + expect(res.body.errors.length).toBeGreaterThan(0); + }); + + it('returns a normalized body with zod defaults applied when valid', async () => { + const res = await authRequest('post', `${BASE_URL}/validate`).send({ + name: 'D', + tiles: [], + }); + expect(res.body.valid).toBe(true); + expect(res.body.normalized).toMatchObject({ name: 'D', tiles: [] }); + }); + + it('does not persist (dashboard count unchanged)', async () => { + const before = await Dashboard.countDocuments({}); + await authRequest('post', `${BASE_URL}/validate`).send({ + name: 'D', + tiles: [], + }); + expect(await Dashboard.countDocuments({})).toBe(before); + }); + }); }); diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 39a4b6a5b7..e812a916d9 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -1929,6 +1929,127 @@ router.get( }, ); +/** + * @openapi + * /api/v2/dashboards/validate: + * post: + * summary: Validate Dashboard + * description: | + * Validates a dashboard body against the same schema and tile rules used + * by POST /api/v2/dashboards. The dashboard is **never persisted**. Use + * this endpoint at plan time (e.g. from a Terraform provider) to check + * that a dashboard configuration is valid before applying it. + * operationId: validateDashboard + * tags: [Dashboards] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/CreateDashboardRequest' + * responses: + * '200': + * description: | + * Validation result. HTTP 200 is always returned for valid **and** + * invalid bodies — a non-200 response means the request itself + * failed (auth, server error, etc.). + * content: + * application/json: + * schema: + * type: object + * required: [valid, errors, normalized] + * properties: + * valid: + * type: boolean + * description: True when the body passes all validation rules. + * errors: + * type: array + * description: Validation errors. Empty when valid is true. + * items: + * type: object + * required: [path, message] + * properties: + * path: + * type: string + * description: Dot-separated field path, or empty string for top-level errors. + * message: + * type: string + * description: Human-readable error description. + * normalized: + * type: object + * nullable: true + * description: | + * The parsed dashboard body with defaults applied (no + * persistence, so no server-assigned tile IDs). Populated + * when valid is true, null when valid is false. + * examples: + * valid: + * summary: Valid dashboard body + * value: + * valid: true + * errors: [] + * normalized: + * name: "My Dashboard" + * tiles: [] + * invalid: + * summary: Invalid dashboard body + * value: + * valid: false + * errors: + * - path: "name" + * message: "Required" + * normalized: null + * '401': + * description: Unauthorized + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Unauthorized access. API key is missing or invalid." + */ +router.post('/validate', async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const parsed = createDashboardBodySchema.safeParse(req.body); + if (!parsed.success) { + return res.json({ + valid: false, + errors: parsed.error.issues.map(i => ({ + path: i.path.join('.'), + message: i.message, + })), + normalized: null, + }); + } + + const { tiles, filters, containers } = parsed.data; + // Cast: createDashboardBodySchema tiles have optional `id`; the validator + // only reads sourceId/config — the cast is safe for a no-persist call. + const validationError = await validateDashboardTiles({ + teamId: teamId.toString(), + tiles: tiles as ExternalDashboardTileWithId[], + filters, + containers: containers ?? [], + }); + if (validationError) { + return res.json({ + valid: false, + errors: [{ path: '', message: validationError }], + normalized: null, + }); + } + + return res.json({ valid: true, errors: [], normalized: parsed.data }); + } catch (e) { + next(e); + } +}); + /** * @openapi * /api/v2/dashboards: From 64d0bbe567bb3c5a106b73c673f1f901ffd4100c Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 1 Jul 2026 09:30:02 -0600 Subject: [PATCH 127/728] feat(hdx-eval): add scenario hooks for custom prompts, tools, grading, and inspection (#2547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds optional hooks to the `Scenario` type so new eval scenario kinds can customize harness and grading behavior without modifying framework files. All existing scenarios are unaffected — hooks are optional with defaults preserved. ## Hooks | Hook | Purpose | |---|---| | `buildSystemPrompt(ctx)` | Replace the default SRE-investigation system prompt | | `allowedToolPatterns` | Selectively unblock denied MCP tools (e.g. dashboard tools) | | `judgeSystemPreamble` | Replace the default LLM judge preamble | | `postRunInspection(ctx)` | Inspect artifacts post-run, collect evidence for the judge, clean up | ## Why Adding the dashboard-build eval (separate PR) required changes to 10+ framework files because `ScenarioKind` was threaded through the entire call stack. With hooks, adding a new scenario kind requires only the scenario files + one import line in `index.ts`. ## Other changes - `runCell` takes a `Scenario` object instead of a scenario name string (needed for hook dispatch) - `GradeRecord` has generic `inspectionSummary` and `inspectionEvidence` fields - Re-grade safety: cached inspection summary + evidence are reused on `--rerun-judge` instead of re-firing the hook (artifacts may have been cleaned up) - `SystemPromptContext` includes `variant` so custom hooks can adapt to hypothesis-mode runs - `buildInspectionConfig()` helper extracted (used by both `run` and `grade` commands) - CLI: `--email`/`--password` options on `run` and `grade` commands (extracted to `DEFAULT_EVAL_EMAIL`/`DEFAULT_EVAL_PASSWORD` constants) - `.gitignore`: ignore `eval.config.*.json` backup configs ## Test plan - 149 unit tests pass (16 suites) - TypeScript compiles clean - All existing scenarios produce identical results (hooks are optional, defaults preserved) - New test: `buildSystemPrompt` hook dispatch verified with a fake scenario --- .changeset/eval-scenario-hooks.md | 5 + packages/hdx-eval/.gitignore | 1 + .../src/__tests__/systemPrompt.test.ts | 33 +++++-- packages/hdx-eval/src/cli.ts | 80 +++++++++++++++- packages/hdx-eval/src/grading/grade.ts | 81 +++++++++++++++- packages/hdx-eval/src/grading/judge.ts | 11 ++- packages/hdx-eval/src/grading/judgePrompt.ts | 21 ++++- packages/hdx-eval/src/grading/types.ts | 19 +++- packages/hdx-eval/src/harness/claudeSpawn.ts | 11 ++- packages/hdx-eval/src/harness/runRun.ts | 12 ++- packages/hdx-eval/src/harness/settingsFile.ts | 23 ++++- packages/hdx-eval/src/harness/systemPrompt.ts | 38 +++++++- packages/hdx-eval/src/scenarios/types.ts | 94 +++++++++++++++++++ 13 files changed, 399 insertions(+), 30 deletions(-) create mode 100644 .changeset/eval-scenario-hooks.md diff --git a/.changeset/eval-scenario-hooks.md b/.changeset/eval-scenario-hooks.md new file mode 100644 index 0000000000..8a82dbb10d --- /dev/null +++ b/.changeset/eval-scenario-hooks.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/hdx-eval": patch +--- + +Add optional scenario hooks for custom system prompts, tool permissions, judge preambles, and post-run artifact inspection diff --git a/packages/hdx-eval/.gitignore b/packages/hdx-eval/.gitignore index 423f3d8fde..9c3fe04566 100644 --- a/packages/hdx-eval/.gitignore +++ b/packages/hdx-eval/.gitignore @@ -1,2 +1,3 @@ /runs/ eval.config.json +eval.config.*.json diff --git a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts index 24b6325bc3..08e41f5150 100644 --- a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts +++ b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts @@ -1,15 +1,17 @@ import { buildSystemPrompt } from '@/harness/systemPrompt'; +import { getScenario } from '@/scenarios'; +import type { Scenario, SystemPromptContext } from '@/scenarios/types'; describe('buildSystemPrompt', () => { it('includes scenario-specific table names', () => { - const p = buildSystemPrompt('error-root-cause'); + const p = buildSystemPrompt(getScenario('error-root-cause')); expect(p).toContain('default.eval_error_root_cause_otel_traces'); expect(p).toContain('default.eval_error_root_cause_otel_logs'); }); it('includes generic tool-environment guidance', () => { - const p1 = buildSystemPrompt('latency-spike'); - const p2 = buildSystemPrompt('noisy-signals'); + const p1 = buildSystemPrompt(getScenario('latency-spike')); + const p2 = buildSystemPrompt(getScenario('noisy-signals')); // Both prompts should explain tool environment and Read availability for (const p of [p1, p2]) { expect(p).toContain('TOOL ENVIRONMENT'); @@ -19,7 +21,7 @@ describe('buildSystemPrompt', () => { }); it('asks for service, operation, root cause, and ruled-out section', () => { - const p = buildSystemPrompt('error-root-cause'); + const p = buildSystemPrompt(getScenario('error-root-cause')); expect(p.toLowerCase()).toContain('service'); expect(p.toLowerCase()).toContain('operation'); expect(p.toLowerCase()).toContain('root cause'); @@ -27,14 +29,33 @@ describe('buildSystemPrompt', () => { }); it('includes anchor time when provided', () => { - const p = buildSystemPrompt('error-root-cause', '2026-01-15T10:00:00.000Z'); + const p = buildSystemPrompt( + getScenario('error-root-cause'), + '2026-01-15T10:00:00.000Z', + ); expect(p).toContain('FIXED CURRENT TIME'); expect(p).toContain('2026-01-15T10:00:00.000Z'); }); it('includes hypothesis playbook when variant is hypothesis', () => { - const p = buildSystemPrompt('error-root-cause', undefined, 'hypothesis'); + const p = buildSystemPrompt( + getScenario('error-root-cause'), + undefined, + 'hypothesis', + ); expect(p).toContain('INVESTIGATION GUIDANCE'); expect(p).toContain('hypotheses'); }); + + it('uses custom buildSystemPrompt hook when provided', () => { + const customPrompt = 'You are a custom scenario agent.'; + const fakeScenario: Scenario = { + ...getScenario('error-root-cause'), + buildSystemPrompt: (_ctx: SystemPromptContext) => customPrompt, + }; + const p = buildSystemPrompt(fakeScenario); + expect(p).toBe(customPrompt); + // Should NOT contain investigation-specific content + expect(p).not.toContain("What's not the cause"); + }); }); diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts index 76d3825aa3..117c1d483f 100644 --- a/packages/hdx-eval/src/cli.ts +++ b/packages/hdx-eval/src/cli.ts @@ -96,6 +96,29 @@ function defaultApiUrl(): string { return 'http://localhost:8000'; } +// Default eval account credentials — used by setup-hyperdx, run, and grade. +const DEFAULT_EVAL_EMAIL = 'eval@local.test'; +const DEFAULT_EVAL_PASSWORD = 'EvalPass123!#'; + +/** Build the inspection config from an eval config + CLI credentials. */ +function buildInspectionConfig( + config: import('./hyperdx/config').EvalConfig, + creds: { email: string; password: string }, + anchorTimeIso?: string, +): + | NonNullable + | undefined { + if (!config.hyperdxApi) return undefined; + return { + apiUrl: config.hyperdxApi.apiUrl, + accessKey: config.hyperdxApi.accessKey, + email: creds.email, + password: creds.password, + anchorTimeIso, + cleanup: true, + }; +} + const program = new Command(); program @@ -215,8 +238,8 @@ program 'One-time setup: register/login + create Connection + per-scenario Sources', ) .option('--api-url ', 'HyperDX API URL', defaultApiUrl()) - .option('--email ', 'Account email', 'eval@local.test') - .option('--password ', 'Account password', 'EvalPass123!#') + .option('--email ', 'Account email', DEFAULT_EVAL_EMAIL) + .option('--password ', 'Account password', DEFAULT_EVAL_PASSWORD) .option( '--creds-file ', 'JSON file with {"email":..., "password":...} (overrides --email/--password)', @@ -378,6 +401,16 @@ program '--no-judge', 'Run programmatic checks only during auto-grading (skip LLM judge)', ) + .option( + '--email ', + `HyperDX account email for post-run inspection (default: ${DEFAULT_EVAL_EMAIL})`, + DEFAULT_EVAL_EMAIL, + ) + .option( + '--password ', + `HyperDX account password for post-run inspection (default: ${DEFAULT_EVAL_PASSWORD})`, + DEFAULT_EVAL_PASSWORD, + ) .action( async ( scenarioName: string, @@ -398,6 +431,8 @@ program report: boolean; judgeModel: string; judge: boolean; + email: string; + password: string; }, ) => { const opts = program.opts(); @@ -582,7 +617,7 @@ program const startedMs = Date.now(); const record = await runCell({ config, - scenario: scenario.name, + scenario, agentPrompt: scenario.agentPrompt, mcp, model, @@ -660,10 +695,16 @@ program def: getMcpDefinition(config, k), })), ); + // Build inspection config when the scenario has a postRunInspection + // hook and the HyperDX API config is available. + const inspectionConfig = scenario.postRunInspection + ? buildInspectionConfig(config, cmdOpts, anchorTimeIso) + : undefined; const gradeOpts: GradeBatchOptions = { judgeModel: cmdOpts.judgeModel, skipJudge: cmdOpts.judge === false, blindingEntries, + inspectionConfig, }; const gradeResult = await gradeBatch(batchDir, gradeOpts); console.log( @@ -793,10 +834,26 @@ program .option('--judge-model ', 'Judge model ID', 'claude-opus-4-7') .option('--rerun-judge', 'Re-call judge even if a grade JSON already exists') .option('--no-judge', 'Run programmatic checks only (cheap regrade)') + .option( + '--email ', + 'HyperDX account email for post-run inspection', + DEFAULT_EVAL_EMAIL, + ) + .option( + '--password ', + 'HyperDX account password for post-run inspection', + DEFAULT_EVAL_PASSWORD, + ) .action( async ( batch: string, - cmdOpts: { judgeModel: string; rerunJudge?: boolean; judge: boolean }, + cmdOpts: { + judgeModel: string; + rerunJudge?: boolean; + judge: boolean; + email: string; + password: string; + }, ) => { const dir = resolveBatchDir(batch); console.log(`Grading batch at ${dir}`); @@ -815,11 +872,26 @@ program // Config may be stale — grade without blinding. } } + // Build inspection config from eval config if available. + let inspectionConfig; + if (configExists()) { + try { + const cfg = readConfig(); + inspectionConfig = buildInspectionConfig( + cfg, + cmdOpts, + cfg.anchorTime, + ); + } catch { + // Config may be stale — grade without inspection. + } + } const summary = await gradeBatch(dir, { judgeModel: cmdOpts.judgeModel, rerunJudge: cmdOpts.rerunJudge ?? false, skipJudge: cmdOpts.judge === false, blindingEntries, + inspectionConfig, }); console.log( `\nGraded ${summary.graded.length} run${summary.graded.length === 1 ? '' : 's'}; ${summary.errors.length} error${summary.errors.length === 1 ? '' : 's'}.`, diff --git a/packages/hdx-eval/src/grading/grade.ts b/packages/hdx-eval/src/grading/grade.ts index bf969a4acd..b5449936c5 100644 --- a/packages/hdx-eval/src/grading/grade.ts +++ b/packages/hdx-eval/src/grading/grade.ts @@ -6,6 +6,7 @@ import type { RunRecord } from '@/harness/types'; import { isModelSubdir, isRunJson, runsRoot, safeReaddir } from '@/runs/path'; import { readRun } from '@/runs/store'; import { getScenario, SCENARIO_NAMES } from '@/scenarios'; +import type { PostRunInspectionResult } from '@/scenarios/types'; import type { BlindingEntry } from './blind'; import { judgeTrajectory } from './judge'; @@ -76,6 +77,19 @@ export type GradeBatchOptions = { apiKey?: string; /** Blinding entries for anonymizing MCP identity during judging. */ blindingEntries?: BlindingEntry[]; + /** + * API config for post-run inspection hooks. When provided, scenarios + * with a `postRunInspection` hook will receive this config so they + * can call the HyperDX API to inspect artifacts. + */ + inspectionConfig?: { + apiUrl: string; + accessKey: string; + email: string; + password: string; + anchorTimeIso?: string; + cleanup?: boolean; + }; }; export type GradeBatchSummary = { @@ -142,12 +156,16 @@ export async function gradeBatch( ? ` (-${(grade.toolErrors.penalty * 100).toFixed(0)}pp)` : '') : ''; + // Show inspection summary if present. + const inspBit = grade.inspectionSummary + ? formatInspectionLogBit(grade.inspectionSummary) + : ''; console.log( ` ${grade.scenario}/${grade.mcp}/${grade.runId.split('-').slice(-1)[0]} prog=${( grade.programmatic.score * 100 ).toFixed( 0, - )}% ${judgeBit} combined=${(grade.combinedScore * 100).toFixed(0)}%${errBit}`, + )}% ${judgeBit} combined=${(grade.combinedScore * 100).toFixed(0)}%${errBit}${inspBit}`, ); } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -159,6 +177,23 @@ export async function gradeBatch( return { batchDir: resolved, graded, errors }; } +function formatInspectionLogBit(summary: Record): string { + const parts: string[] = []; + if ( + typeof summary.totalTiles === 'number' && + typeof summary.tilesWithData === 'number' + ) { + parts.push(`tiles=${summary.tilesWithData}/${summary.totalTiles}`); + } + if (typeof summary.createCalls === 'number') { + parts.push(`creates=${summary.createCalls}`); + } + if (typeof summary.patchCalls === 'number') { + parts.push(`patches=${summary.patchCalls}`); + } + return parts.length > 0 ? ` ${parts.join(' ')}` : ''; +} + async function gradeOne(args: { record: RunRecord; client?: Anthropic; @@ -174,6 +209,42 @@ async function gradeOne(args: { rubric.programmatic, ); + const toolErrors = computeToolErrorStats(record); + + // ── Post-run inspection (scenario hook) ────────────────────────── + // Runs BEFORE the judge so evidence can be passed to the judge prompt. + // On re-grade, reuse the cached inspectionSummary from the existing + // grade record — the artifacts were likely cleaned up on the first pass, + // so re-running the hook would fail or produce empty evidence. + let inspectionResult: PostRunInspectionResult | undefined; + + if (existing?.inspectionSummary) { + // Reuse cached inspection from previous grading pass. The artifacts + // were likely cleaned up, so re-running the hook would fail. The + // persisted evidence string lets --rerun-judge work without re-inspection. + inspectionResult = { + evidence: existing.inspectionEvidence ?? '', + summary: existing.inspectionSummary, + }; + } else if (scenario.postRunInspection && opts.inspectionConfig) { + try { + inspectionResult = await scenario.postRunInspection({ + toolCalls: record.toolCalls, + apiUrl: opts.inspectionConfig.apiUrl, + accessKey: opts.inspectionConfig.accessKey, + email: opts.inspectionConfig.email, + password: opts.inspectionConfig.password, + anchorTimeIso: opts.inspectionConfig.anchorTimeIso, + cleanup: opts.inspectionConfig.cleanup ?? true, + }); + } catch (err) { + console.warn( + ` post-run inspection failed: ${err instanceof Error ? err.message : err}`, + ); + } + } + + // ── LLM Judge ──────────────────────────────────────────────────── let judge: JudgeResult | null = existing?.judge ?? null; if (!opts.skipJudge && client && (!judge || opts.rerunJudge)) { judge = await judgeTrajectory({ @@ -185,9 +256,12 @@ async function gradeOne(args: { judgeModel: opts.judgeModel, client, blindingEntries: opts.blindingEntries, + judgeSystemPreamble: scenario.judgeSystemPreamble, + inspectionEvidence: inspectionResult?.evidence, }); } + // ── Combined score ─────────────────────────────────────────────── const judgeError = judge && 'error' in judge && judge.error; const judgeScore = judge?.weightedScore ?? 0; const rawCombined = @@ -195,8 +269,7 @@ async function gradeOne(args: { ? COMBINED_SCORE_PROGRAMMATIC_WEIGHT * programmatic.score + COMBINED_SCORE_JUDGE_WEIGHT * judgeScore : programmatic.score; - const toolErrors = computeToolErrorStats(record); - // Apply the tool-error penalty AFTER scoring the answer. Clamp to [0,1]. + const combinedScore = Math.max( 0, Math.min(1, rawCombined - toolErrors.penalty), @@ -210,6 +283,8 @@ async function gradeOne(args: { programmatic, judge, toolErrors, + inspectionSummary: inspectionResult?.summary, + inspectionEvidence: inspectionResult?.evidence || undefined, combinedScore, gradedAt: new Date().toISOString(), judgeModel: judge?.model ?? opts.judgeModel ?? 'skipped', diff --git a/packages/hdx-eval/src/grading/judge.ts b/packages/hdx-eval/src/grading/judge.ts index 7bcb076417..f3d449481a 100644 --- a/packages/hdx-eval/src/grading/judge.ts +++ b/packages/hdx-eval/src/grading/judge.ts @@ -21,6 +21,10 @@ export type JudgeOptions = { client?: Anthropic; /** Blinding entries for anonymizing MCP identity in the answer. */ blindingEntries?: BlindingEntry[]; + /** Custom judge system preamble from a scenario hook. */ + judgeSystemPreamble?: string; + /** Post-run inspection evidence to append to the judge prompt. */ + inspectionEvidence?: string; }; export async function judgeTrajectory( @@ -30,11 +34,16 @@ export async function judgeTrajectory( const client = opts.client ?? new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); - const systemPrompt = buildJudgeSystem(opts.scenarioName, opts.rubric); + const systemPrompt = buildJudgeSystem( + opts.scenarioName, + opts.rubric, + opts.judgeSystemPreamble, + ); const userPrompt = buildJudgeUser({ scenarioPrompt: opts.scenarioPrompt, groundTruthFacts: formatGroundTruthFacts(opts.groundTruth), candidateAnswer: blindAnswer(opts.finalAnswer, opts.blindingEntries), + dashboardEvidence: opts.inspectionEvidence, }); const startedMs = Date.now(); diff --git a/packages/hdx-eval/src/grading/judgePrompt.ts b/packages/hdx-eval/src/grading/judgePrompt.ts index 082ca8639c..9497335789 100644 --- a/packages/hdx-eval/src/grading/judgePrompt.ts +++ b/packages/hdx-eval/src/grading/judgePrompt.ts @@ -12,12 +12,17 @@ Return STRICT JSON of shape: { "scores": { "": { "score": N, "rationale": "..." } } } No prose outside the JSON. Include every criterion id from the rubric.`; -export function buildJudgeSystem(scenarioName: string, rubric: Rubric): string { +export function buildJudgeSystem( + scenarioName: string, + rubric: Rubric, + /** Custom system preamble from a scenario hook. */ + customPreamble?: string, +): string { const criteriaSection = rubric.judge.criteria .map(c => `- "${c.id}" (weight ${c.weight}): ${c.description}`) .join('\n'); return [ - SYSTEM_PREAMBLE, + customPreamble ?? SYSTEM_PREAMBLE, '', `SCENARIO: ${scenarioName}`, '', @@ -30,8 +35,10 @@ export function buildJudgeUser(args: { scenarioPrompt: string; groundTruthFacts: string; candidateAnswer: string; + /** Dashboard artifact evidence — appended for dashboard scenarios. */ + dashboardEvidence?: string; }): string { - return [ + const sections = [ 'SCENARIO QUESTION:', args.scenarioPrompt, '', @@ -40,7 +47,13 @@ export function buildJudgeUser(args: { '', 'CANDIDATE FINAL ANSWER (anonymized):', args.candidateAnswer, - ].join('\n'); + ]; + + if (args.dashboardEvidence) { + sections.push('', args.dashboardEvidence); + } + + return sections.join('\n'); } /** diff --git a/packages/hdx-eval/src/grading/types.ts b/packages/hdx-eval/src/grading/types.ts index 6252f8226b..440bf3ea24 100644 --- a/packages/hdx-eval/src/grading/types.ts +++ b/packages/hdx-eval/src/grading/types.ts @@ -80,14 +80,27 @@ export type GradeRecord = { programmatic: ProgrammaticResult; judge: JudgeResult | null; toolErrors: ToolErrorStats; + /** + * Scenario-specific inspection summary. Only present when the scenario + * provides a `postRunInspection` hook. The shape depends on the scenario + * (e.g., dashboard scenarios include tile details, alert scenarios might + * include alert evaluation results). Persisted as opaque JSON. + */ + inspectionSummary?: Record; + /** + * Human-readable inspection evidence that was passed to the LLM judge. + * Persisted so re-grades (e.g. --rerun-judge) can reuse the evidence + * without re-running the inspection hook (artifacts may be cleaned up). + */ + inspectionEvidence?: string; /** * combinedScore = clamp01( * PROGRAMMATIC_WEIGHT * programmatic + JUDGE_WEIGHT * judge * - toolErrors.penalty * ) - * The penalty is at most MAX_ERROR_PENALTY (0.2) — a run can't be reduced - * to zero from a high error rate alone, but spamming bad tool calls makes - * an otherwise-good answer measurably worse. + * When a postRunInspection hook provides evidence, the judge receives + * it alongside the ground truth — the judge score already incorporates + * artifact quality, so no separate mechanical blend is needed. */ combinedScore: number; gradedAt: string; diff --git a/packages/hdx-eval/src/harness/claudeSpawn.ts b/packages/hdx-eval/src/harness/claudeSpawn.ts index ea48d33d2c..83caf8cc86 100644 --- a/packages/hdx-eval/src/harness/claudeSpawn.ts +++ b/packages/hdx-eval/src/harness/claudeSpawn.ts @@ -30,6 +30,9 @@ export type SpawnOptions = { * `hypothesis` variant allows the Task tool so the agent can spawn * subagents. Default 'baseline'. */ promptVariant?: PromptVariant; + /** Tool name substrings to remove from the denied-tools list. + * Comes from `Scenario.allowedToolPatterns`. */ + allowedToolPatterns?: string[]; }; export type SpawnResult = { @@ -55,7 +58,11 @@ export async function runClaude(opts: SpawnOptions): Promise { const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; writeFileSync( settingsPath, - JSON.stringify(buildSettings(mcpDef, promptVariant, tempdir), null, 2), + JSON.stringify( + buildSettings(mcpDef, promptVariant, tempdir, opts.allowedToolPatterns), + null, + 2, + ), ); const argv = [ @@ -65,7 +72,7 @@ export async function runClaude(opts: SpawnOptions): Promise { '--allowedTools', `${allowedToolsPattern(mcpDef)},Read(${tempdir}/*)`, '--disallowedTools', - deniedToolsFor(promptVariant, mcpDef).join(','), + deniedToolsFor(promptVariant, mcpDef, opts.allowedToolPatterns).join(','), '--dangerously-skip-permissions', '--setting-sources', 'local', diff --git a/packages/hdx-eval/src/harness/runRun.ts b/packages/hdx-eval/src/harness/runRun.ts index 05def8039a..9e7762a7d8 100644 --- a/packages/hdx-eval/src/harness/runRun.ts +++ b/packages/hdx-eval/src/harness/runRun.ts @@ -1,4 +1,5 @@ import { type EvalConfig, getMcpDefinition } from '@/hyperdx/config'; +import type { Scenario } from '@/scenarios/types'; import { runClaude } from './claudeSpawn'; import type { ParsedEvent } from './streamParser'; @@ -13,7 +14,8 @@ import type { export type RunCellOptions = { config: EvalConfig; - scenario: string; + /** The full scenario object (provides hooks for custom behavior). */ + scenario: Scenario; agentPrompt: string; mcp: McpKind; model: string; @@ -35,6 +37,9 @@ export async function runCell(opts: RunCellOptions): Promise { const startedAtMs = Date.now(); const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; const mcpDef = getMcpDefinition(opts.config, opts.mcp); + + // Use scenario's custom system prompt builder if provided, + // otherwise fall back to the default investigation prompt. const systemPromptAppend = buildSystemPrompt( opts.scenario, opts.anchorTimeIso, @@ -44,7 +49,7 @@ export async function runCell(opts: RunCellOptions): Promise { const result = await runClaude({ config: opts.config, - scenario: opts.scenario, + scenario: opts.scenario.name, mcp: opts.mcp, mcpDef, model: opts.model, @@ -54,6 +59,7 @@ export async function runCell(opts: RunCellOptions): Promise { systemPromptAppend, apiKey: opts.apiKey, promptVariant, + allowedToolPatterns: opts.scenario.allowedToolPatterns, }); const endedAtIso = new Date().toISOString(); @@ -64,7 +70,7 @@ export async function runCell(opts: RunCellOptions): Promise { stderr: result.stderr, exitCode: result.exitCode, timedOut: result.timedOut, - scenario: opts.scenario, + scenario: opts.scenario.name, agentPrompt: opts.agentPrompt, systemPromptAppend, mcp: opts.mcp, diff --git a/packages/hdx-eval/src/harness/settingsFile.ts b/packages/hdx-eval/src/harness/settingsFile.ts index 5e3a510f58..9a8011b0e0 100644 --- a/packages/hdx-eval/src/harness/settingsFile.ts +++ b/packages/hdx-eval/src/harness/settingsFile.ts @@ -39,17 +39,35 @@ export const DENIED_BUILT_IN_TOOLS = [ /** * Build the list of denied tools for a given MCP and prompt variant. * Per-MCP denied tools come from `McpDefinition.deniedTools`. + * + * When `allowedToolPatterns` is provided (from a scenario hook), matching + * tools are removed from the deny list via substring matching. */ export function deniedToolsFor( variant: PromptVariant, def?: McpDefinition, + allowedToolPatterns?: string[], ): readonly string[] { const builtIn = variant === 'hypothesis' ? DENIED_BUILT_IN_TOOLS_BASE : DENIED_BUILT_IN_TOOLS; - const mcpDenied = def?.deniedTools ?? []; + let mcpDenied = def?.deniedTools ?? []; + + // Scenarios can selectively unblock tools by providing substring patterns. + // E.g., dashboard scenarios pass ['delete_dashboard', 'save_dashboard', ...] + // to remove those from the deny list. + if ( + allowedToolPatterns && + allowedToolPatterns.length > 0 && + mcpDenied.length > 0 + ) { + mcpDenied = mcpDenied.filter( + t => !allowedToolPatterns.some(p => t.includes(p)), + ); + } + if (mcpDenied.length > 0) { return [...builtIn, ...mcpDenied]; } @@ -60,6 +78,7 @@ export function buildSettings( def: McpDefinition, variant: PromptVariant = 'baseline', tempdir?: string, + allowedToolPatterns?: string[], ): Record { const allow: string[] = [allowedToolsPattern(def)]; @@ -73,7 +92,7 @@ export function buildSettings( return { permissions: { allow, - deny: [...deniedToolsFor(variant, def)], + deny: [...deniedToolsFor(variant, def, allowedToolPatterns)], }, }; } diff --git a/packages/hdx-eval/src/harness/systemPrompt.ts b/packages/hdx-eval/src/harness/systemPrompt.ts index 49a0f09220..dec059e686 100644 --- a/packages/hdx-eval/src/harness/systemPrompt.ts +++ b/packages/hdx-eval/src/harness/systemPrompt.ts @@ -1,14 +1,48 @@ import { scenarioTables } from '@/clickhouse/schema'; +import type { Scenario } from '@/scenarios/types'; import type { PromptVariant } from './types'; +/** + * Build the system prompt for a scenario run. + * + * If the scenario provides a `buildSystemPrompt` hook, it is called instead + * of the default investigation prompt. This lets new scenario kinds (dashboard, + * alert, etc.) define their own instructions without modifying this file. + */ export function buildSystemPrompt( - scenario: string, + scenario: Scenario, anchorTimeIso?: string, variant: PromptVariant = 'baseline', maxTurns?: number, ): string { - const { traces, logs } = scenarioTables(scenario); + // Scenario-provided custom system prompt — used by dashboard-build, etc. + if (scenario.buildSystemPrompt) { + const tables = scenarioTables(scenario.name); + return scenario.buildSystemPrompt({ + tables, + anchorTimeIso, + maxTurns, + variant, + }); + } + + // Default: SRE investigation prompt. + return buildInvestigationSystemPrompt( + scenario.name, + anchorTimeIso, + variant, + maxTurns, + ); +} + +function buildInvestigationSystemPrompt( + scenarioName: string, + anchorTimeIso?: string, + variant: PromptVariant = 'baseline', + maxTurns?: number, +): string { + const { traces, logs } = scenarioTables(scenarioName); const sharedSchema = ''; // `These follow the standard OpenTelemetry ClickHouse schema: // - traces have Timestamp DateTime64(9), TraceId, SpanId, ParentSpanId, diff --git a/packages/hdx-eval/src/scenarios/types.ts b/packages/hdx-eval/src/scenarios/types.ts index cd7bb28f50..98ec1748e7 100644 --- a/packages/hdx-eval/src/scenarios/types.ts +++ b/packages/hdx-eval/src/scenarios/types.ts @@ -1,4 +1,5 @@ import type { LogRow, TraceRow } from '@/generators/types'; +import type { PromptVariant, ToolCallRecord } from '@/harness/types'; import type { SeededRng } from '@/rng/seeded'; export type GenerateContext = { @@ -22,6 +23,55 @@ export type ScenarioBatch = { logs: LogRow[]; }; +// ─── Scenario hooks ────────────────────────────────────────────────────────── +// These optional hooks let scenarios customize harness and grading behavior +// without requiring changes to framework files. A new scenario kind (e.g., +// alert-build, saved-search-build) only needs to implement the relevant hooks +// in its generate.ts — no changes to cli.ts, grade.ts, settingsFile.ts, etc. + +/** + * Context passed to the system prompt builder hook. + */ +export type SystemPromptContext = { + /** Scenario-specific ClickHouse table names. */ + tables: { traces: string; logs: string }; + /** Fixed anchor time ISO string (if set). */ + anchorTimeIso?: string; + /** Max tool turns for the run. */ + maxTurns?: number; + /** Prompt variant — 'hypothesis' enables multi-hypothesis guidance. */ + variant?: PromptVariant; +}; + +/** + * Result of a post-run inspection hook. The `evidence` string is appended + * to the judge prompt so the LLM judge can evaluate the actual artifact + * (dashboard, alert, saved search, etc.) — not just the agent's text answer. + */ +export type PostRunInspectionResult = { + /** Human-readable evidence string appended to the judge prompt. */ + evidence: string; + /** Structured summary persisted in the grade record. */ + summary: Record; + /** Dashboard (or other artifact) IDs that were cleaned up by the hook. + * Informational only — the hook handles cleanup itself via the `cleanup` + * flag in PostRunInspectionContext. */ + cleanupIds?: string[]; +}; + +/** + * Context passed to the post-run inspection hook. + */ +export type PostRunInspectionContext = { + toolCalls: ToolCallRecord[]; + apiUrl: string; + accessKey: string; + email: string; + password: string; + anchorTimeIso?: string; + cleanup?: boolean; +}; + export type Scenario = { name: string; agentPrompt: string; @@ -33,6 +83,50 @@ export type Scenario = { */ generate(ctx: GenerateContext): Iterable; groundTruth: Record; + + // ─── Optional hooks ────────────────────────────────────────────── + // Scenarios that don't provide these hooks get the default investigation + // behavior — no framework changes needed. + + /** + * Custom system prompt builder. When provided, replaces the default + * SRE-investigation system prompt entirely. This is how non-investigation + * scenarios (dashboard-build, alert-build, etc.) inject their own + * instructions without modifying systemPrompt.ts. + */ + buildSystemPrompt?: (ctx: SystemPromptContext) => string; + + /** + * Tool name substrings to remove from the denied-tools list. Lets + * scenarios selectively unblock tools (e.g., dashboard tools) without + * modifying settingsFile.ts. Matched via substring against each + * denied tool name. + */ + allowedToolPatterns?: string[]; + + /** + * LLM judge system preamble override. When provided, replaces the + * default "evaluating an SRE investigation" preamble. The scenario + * can instruct the judge to evaluate artifacts (dashboards, alerts) + * rather than text answers. + */ + judgeSystemPreamble?: string; + + /** + * Post-run inspection hook. Runs after the agent completes but before + * grading. Can inspect the created artifacts (dashboards, alerts, etc.) + * via the API, collect evidence for the judge, and clean up. + * + * When this hook is provided AND returns evidence, the grading pipeline: + * 1. Passes the evidence string to the LLM judge prompt + * 2. Persists the summary in the grade record + * + * Cleanup is the hook's responsibility — use the `cleanup` flag in + * PostRunInspectionContext to decide whether to delete artifacts. + */ + postRunInspection?: ( + ctx: PostRunInspectionContext, + ) => Promise; }; /** Helper for tests: drain an iterable into one combined batch. */ From 555d88a99ce3c229bd5aa9813efb16ee8b86fab7 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:25:31 -0700 Subject: [PATCH 128/728] fix: keep parsed-JSON "Add to Filters" keys as valid SQL (#2561) "Add to Filters" on a value inside parsed JSON from a String column (for example `Body`) builds a query ClickHouse rejects. Expand a log whose `Body` is a JSON string, then filter on a nested value such as `app.user.currency = USD`, and the generated condition is: ``` JSONExtractString(Body, 'app['user.currency')'] IN ('USD') ``` which fails to parse. This fixes the filter-key serialization so the generated SQL is valid. Resolves HDX-4427. ## Summary The JSON viewer builds a correct ClickHouse expression for the filter key, `JSONExtractString(Body, 'app.user.currency')`. That key then flows through `escapeFilterStateKeys` into `toClickHouseKeyExpression`, which only short-circuited bracket access, backtick paths, and `toString(...)`. A `JSONExtract*(...)` expression fell through to `parseMapFieldName`, whose dot pattern split on the dot inside the quoted JSON path argument, and `mergePath` then rewrote it into the invalid `JSONExtractString(Body, 'app['user.currency')']`. The pass-through guard now treats any raw SQL function-call expression (`toString(...)`, `JSONExtractString/Float/Bool(...)`) as an already-valid ClickHouse expression and returns it unchanged, so it reaches `filtersToQuery` intact and serializes to `JSONExtractString(Body, 'app.user.currency') IN ('USD')`. Dot-form Map sub-keys like `LogAttributes.host.name` contain no parentheses, so they still rewrite to bracket form as before. ## Changes - `packages/app/src/components/DBSearchPageFilters/utils.ts`: generalize the `toClickHouseKeyExpression` pass-through guard from `toString(` to any function-call expression. - Unit tests for the guard; an integration test that runs the real `escapeFilterStateKeys` -> `filtersToQuery` -> `parseQuery` pipeline and asserts every emitted condition is valid, round-trippable SQL across a range of JSON and Map key shapes; and a component test locking the "Add to Filters" contract for a parsed-JSON String column. - An end-to-end test that adds a filter on a nested JSON `Body` value from the row side panel and confirms the search runs without a ClickHouse error and returns the matching row. ## Test plan - [x] `nx run-many -t ci:lint` (eslint + tsc) - [x] `nx run-many -t ci:unit` - [x] Confirmed the unit and integration tests fail without the fix and pass with it - [x] Drove the real search UI against ClickHouse: "Add to Filters" on a nested JSON `Body` value builds `JSONExtractString(Body, 'app.user.currency') IN ('USD')`, the query runs, and the row is returned - [x] e2e: `packages/app/tests/e2e/features/search/json-add-to-filter.spec.ts` [ui-check: allow] No visual surface; the change is filter-key SQL serialization, verified via unit, integration, and end-to-end tests. [viewport: allow] Layout-neutral logic fix. [ui-states: allow] No empty/loading/error UI states in scope. --- .changeset/fix-json-add-to-filters-sql.md | 5 + .../src/components/DBRowJsonViewer.test.tsx | 15 ++ .../jsonAddToFilter.pipeline.test.ts | 167 ++++++++++++++++++ .../DBSearchPageFilters/utils.test.ts | 40 +++++ .../components/DBSearchPageFilters/utils.ts | 18 +- packages/app/src/components/HyperJson.tsx | 1 + .../e2e/components/SidePanelComponent.ts | 41 +++++ .../search/json-add-to-filter.spec.ts | 61 +++++++ packages/app/tests/e2e/seed-clickhouse.ts | 44 +++++ 9 files changed, 389 insertions(+), 3 deletions(-) create mode 100644 .changeset/fix-json-add-to-filters-sql.md create mode 100644 packages/app/src/components/DBSearchPageFilters/jsonAddToFilter.pipeline.test.ts create mode 100644 packages/app/tests/e2e/features/search/json-add-to-filter.spec.ts diff --git a/.changeset/fix-json-add-to-filters-sql.md b/.changeset/fix-json-add-to-filters-sql.md new file mode 100644 index 0000000000..ccfc4c2b51 --- /dev/null +++ b/.changeset/fix-json-add-to-filters-sql.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +Fix "Add to Filters" on a value inside parsed JSON from a String column (for example `Body`) building invalid SQL. The `JSONExtractString(...)` expression the JSON viewer produces is now passed through unchanged instead of being mis-parsed as a dot-form Map sub-key and mangled into a query ClickHouse rejects. diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index dfa5ee2b17..75a97f4d97 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -164,6 +164,21 @@ describe('DBRowJsonViewer', () => { ); }); + // HDX-4427: "Add to Filters" on a value inside parsed JSON from a String + // column must hand searchFilters the JSONExtract* expression, which is what + // gets serialized into the WHERE clause. Body here is a String column holding + // a JSON string with a dotted key, mirroring the play-clickstack repro. + it('adds a JSONExtractString filter for a value inside a parsed-JSON string column', () => { + renderComponent({ Body: JSON.stringify({ 'app.user.currency': 'USD' }) }); + + expandAndClickButton('Body', 'app.user.currency', 'Add to Filters'); + + expect(mockOnPropertyAddClick).toHaveBeenCalledWith( + "JSONExtractString(Body, 'app.user.currency')", + 'USD', + ); + }); + it('toggles columns with correct path formatting', () => { renderComponent(logData); clickLineButton('field1', 'Column'); diff --git a/packages/app/src/components/DBSearchPageFilters/jsonAddToFilter.pipeline.test.ts b/packages/app/src/components/DBSearchPageFilters/jsonAddToFilter.pipeline.test.ts new file mode 100644 index 0000000000..f298bb04bf --- /dev/null +++ b/packages/app/src/components/DBSearchPageFilters/jsonAddToFilter.pipeline.test.ts @@ -0,0 +1,167 @@ +import { + type FilterState, + filtersToQuery, + isValidFilterCondition, +} from '@hyperdx/common-utils/dist/filters'; + +import { buildJSONExtractQuery } from '@/components/DBRowJsonViewer'; +import { escapeFilterStateKeys, parseQuery } from '@/searchFilters'; + +import { cleanClickHouseExpression } from './utils'; + +// End-to-end coverage for the "Add to Filters" -> WHERE-clause path, driven +// through the real producer (buildJSONExtractQuery) and the real serialization +// (escapeFilterStateKeys -> filtersToQuery). This is the regression net for +// HDX-4427: a filter key that is a raw ClickHouse expression must serialize to +// valid SQL and survive a parseQuery round-trip. Asserting validity (rather +// than an exact string) catches this bug class for any JSON/Map key shape, not +// just the one the user happened to hit. + +// Real top-level columns on a typical OTel logs table. +const knownColumns = new Set(['Body', 'LogAttributes', 'ServiceName']); + +// buildJSONExtractQuery returns string | null; in every case here a nested path +// exists, so null is a test-setup error rather than an expected branch. +const jsonKey = ( + keyPath: string[], + parsedJsonRootPath: string[], + jsonColumns: string[] = [], + jsonExtractFn: + | 'JSONExtractString' + | 'JSONExtractFloat' + | 'JSONExtractBool' = 'JSONExtractString', + mapColumns: string[] = [], +): string => { + const out = buildJSONExtractQuery( + keyPath, + parsedJsonRootPath, + jsonColumns, + jsonExtractFn, + mapColumns, + ); + if (out == null) { + throw new Error('buildJSONExtractQuery returned null for a nested path'); + } + return out; +}; + +const runAddToFilter = ( + key: string, + values: { included?: string[]; excluded?: string[] }, +) => { + const state: FilterState = { + [key]: { + included: new Set(values.included ?? []), + excluded: new Set(values.excluded ?? []), + }, + }; + // Mirror updateFilterQuery: escape keys, then serialize to the query. + return filtersToQuery(escapeFilterStateKeys(state, knownColumns)); +}; + +describe('parsed-JSON "Add to Filters" -> valid SQL pipeline (HDX-4427)', () => { + const cases: { + desc: string; + key: string; + values: { included?: string[]; excluded?: string[] }; + }[] = [ + { + desc: 'flat dotted key in a String column (the reported play-clickstack case)', + key: jsonKey(['Body', 'app.user.currency'], ['Body']), + values: { included: ['USD'] }, + }, + { + desc: 'nested path in a String column', + key: jsonKey(['Body', 'app', 'user', 'id'], ['Body']), + values: { included: ['u-1'] }, + }, + { + desc: 'simple top-level key in a String column', + key: jsonKey(['Body', 'level'], ['Body']), + values: { included: ['error'], excluded: ['debug'] }, + }, + { + desc: 'numeric metric via JSONExtractFloat with a dotted path', + key: jsonKey( + ['Body', 'metrics.latency'], + ['Body'], + [], + 'JSONExtractFloat', + ), + values: { included: ['200'] }, + }, + { + desc: 'boolean flag via JSONExtractBool with a dotted path', + key: jsonKey(['Body', 'flags.enabled'], ['Body'], [], 'JSONExtractBool'), + values: { included: ['true'] }, + }, + { + desc: 'Map sub-value holding JSON (bracketed base column)', + key: jsonKey( + ['LogAttributes', 'config', 'db.host'], + ['LogAttributes', 'config'], + [], + 'JSONExtractString', + ['LogAttributes'], + ), + values: { included: ['localhost'] }, + }, + { + desc: 'value containing a single quote (SQL escaping)', + key: jsonKey(['Body', 'user.name'], ['Body']), + values: { included: ["O'Brien"] }, + }, + ]; + + it.each(cases)( + 'emits only valid ClickHouse SQL for $desc', + ({ key, values }) => { + const query = runAddToFilter(key, values); + + expect(query.length).toBeGreaterThan(0); + for (const filter of query) { + expect(filter.type).toBe('sql'); + if (filter.type === 'sql') { + expect(isValidFilterCondition(filter.condition, 'sql')).toBe(true); + } + } + }, + ); + + it('emits the corrected, valid condition for the reported case', () => { + const key = jsonKey(['Body', 'app.user.currency'], ['Body']); + expect(runAddToFilter(key, { included: ['USD'] })).toEqual([ + { + type: 'sql', + condition: "JSONExtractString(Body, 'app.user.currency') IN ('USD')", + }, + ]); + }); + + it('round-trips included and excluded values through parseQuery', () => { + const key = jsonKey(['Body', 'app.user.currency'], ['Body']); + const query = runAddToFilter(key, { + included: ['USD', 'EUR'], + excluded: ['JPY'], + }); + + const back = parseQuery(query).filters; + // parseQuery keys are the canonical (escaped) form; clean them the same way + // unescapeFilterStateKeys does before comparing to the in-memory key. + const restoredKeys = Object.keys(back).map(cleanClickHouseExpression); + expect(restoredKeys).toEqual([key]); + + const entry = back[Object.keys(back)[0]]; + expect(entry.included).toEqual(new Set(['USD', 'EUR'])); + expect(entry.excluded).toEqual(new Set(['JPY'])); + }); + + it('round-trips a value containing a single quote', () => { + const key = jsonKey(['Body', 'user.name'], ['Body']); + const back = parseQuery( + runAddToFilter(key, { included: ["O'Brien"] }), + ).filters; + const entry = back[Object.keys(back)[0]]; + expect(entry.included).toEqual(new Set(["O'Brien"])); + }); +}); diff --git a/packages/app/src/components/DBSearchPageFilters/utils.test.ts b/packages/app/src/components/DBSearchPageFilters/utils.test.ts index 986a86f0bd..10fa53c521 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.test.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.test.ts @@ -180,6 +180,34 @@ describe('toClickHouseKeyExpression', () => { "LogAttributes['42.foo']", ); }); + + // HDX-4427: "Add to Filters" on a value inside parsed JSON from a String + // column builds a JSONExtract* function call as the filter key. These are + // already valid ClickHouse expressions and must pass through untouched. + // Previously the dot inside the quoted JSON path argument made + // parseMapFieldName treat the whole expression as a dot-form Map sub-key, and + // mergePath mangled it into invalid SQL like + // `JSONExtractString(Body, 'app['user.currency')']`. + describe('raw SQL function-call expression keys (parsed-JSON "Add to Filters")', () => { + it.each([ + "JSONExtractString(Body, 'app.user.currency')", + "JSONExtractString(Body, 'app', 'user.currency')", + "JSONExtractString(Body, 'level')", + "JSONExtractFloat(Body, 'metrics.latency')", + "JSONExtractBool(Body, 'flags.enabled')", + "JSONExtractString(LogAttributes['weird.key.payload'], 'abc.def.jqk/abcd')", + ])('leaves the JSON-extract expression %s unchanged', key => { + expect(toClickHouseKeyExpression(key)).toBe(key); + }); + + // The guard generalizes the previous `startsWith('toString(')` special case, + // so a toString() wrapper with no bracket access still passes through. + it('leaves a toString() wrapper without bracket access unchanged', () => { + expect(toClickHouseKeyExpression('toString(Body)')).toBe( + 'toString(Body)', + ); + }); + }); }); describe('toQuotedClickHouseKeyExpression', () => { @@ -241,6 +269,18 @@ describe('toQuotedClickHouseKeyExpression', () => { ).toBe('`service-name`'); }); + // HDX-4427: the JSONExtract* key from a parsed-JSON "Add to Filters" reaches + // toQuotedClickHouseKeyExpression via escapeFilterStateKeys. It is not a known + // column and is already valid SQL, so it must pass through unquoted/unmangled. + it('leaves a JSON-extract function-call key unchanged', () => { + expect( + toQuotedClickHouseKeyExpression( + "JSONExtractString(Body, 'app.user.currency')", + new Set(['Body']), + ), + ).toBe("JSONExtractString(Body, 'app.user.currency')"); + }); + describe('with knownColumns (schema-aware)', () => { it('quotes a flat column whose name contains dots as a single identifier', () => { const cols = new Set(['__hdx_materialized_k8s.cluster.name']); diff --git a/packages/app/src/components/DBSearchPageFilters/utils.ts b/packages/app/src/components/DBSearchPageFilters/utils.ts index b2297a6cc4..86f0465426 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.ts @@ -142,12 +142,19 @@ export function getFilterStateEntry( ); } +// A key that begins with `identifier(` is a raw SQL function call (e.g. +// `toString(...)`, `JSONExtractString(...)`), not a column name or a dot-form +// Map sub-key, so it is already a valid ClickHouse expression. +const isSqlFunctionCallExpression = (key: string): boolean => + /^[A-Za-z_]\w*\(/.test(key); + // Coerce a filterState key into a ClickHouse expression suitable for raw SQL. // A dot-form Map sub-key like `LogAttributes.host.name` is rewritten to bracket // form `LogAttributes['host.name']` via `mergePath` so the conversion stays // consistent with the keys produced by the facet-discovery path. Bracket form, -// backtick-quoted JSON paths, `toString(...)` wrappers, and plain column names -// are returned unchanged. Use this when handing a filterState key off to a SQL +// backtick-quoted JSON paths, raw SQL function-call expressions +// (`toString(...)`, `JSONExtractString(...)`), and plain column names are +// returned unchanged. Use this when handing a filterState key off to a SQL // caller (e.g. "Load more" via metadata.getKeyValues), since `setFilterValue` // normalizes Map sub-keys to dot form which ClickHouse cannot resolve as map // access. @@ -162,7 +169,12 @@ export function toClickHouseKeyExpression(key: string): string { key.includes("['") || key.includes('["') || key.includes('`') || - key.startsWith('toString(') + // "Add to Filters" on a value inside parsed JSON from a String column builds + // a function-call key (e.g. JSONExtractString(Body, 'app.user.currency')); + // it must pass through untouched. Without this, parseMapFieldName splits on + // the dot inside the quoted argument and mergePath mangles it into the + // invalid `JSONExtractString(Body, 'app['user.currency')']`. HDX-4427. + isSqlFunctionCallExpression(key) ) { return key; } diff --git a/packages/app/src/components/HyperJson.tsx b/packages/app/src/components/HyperJson.tsx index c047983097..8a6a51aaf7 100644 --- a/packages/app/src/components/HyperJson.tsx +++ b/packages/app/src/components/HyperJson.tsx @@ -272,6 +272,7 @@ const Line = React.memo( <>
0, diff --git a/packages/app/tests/e2e/components/SidePanelComponent.ts b/packages/app/tests/e2e/components/SidePanelComponent.ts index 50559ce0b2..7153dc58b4 100644 --- a/packages/app/tests/e2e/components/SidePanelComponent.ts +++ b/packages/app/tests/e2e/components/SidePanelComponent.ts @@ -62,6 +62,47 @@ export class SidePanelComponent { } } + /** + * In the parsed JSON view, expand a field whose value is a JSON string, then + * click the "Add to Filters" line action on a nested key. Each row in the JSON + * viewer carries `data-testid="json-viewer-line"`, and the action is a + * ` + + ) : null} + { + const w = width ?? 1; + sizeRef.current = [w, height ?? 1]; + setContainerWidth(prev => (prev === w ? prev : w)); }} - onMouseUp={e => { - const MIN_DRAG_DISTANCE = 20; // Minimum horizontal drag distance in pixels - let dragDistance = 0; - - if (mouseDownPosRef.current != null && e?.chartX != null) { - dragDistance = Math.abs(e.chartX - mouseDownPosRef.current); - } + className={isLoading ? 'effect-pulse' : ''} + > + setIsHovered(true)} + onMouseLeave={() => { + setIsHovered(false); + setNearestSeriesKey(undefined); - if (e?.activeLabel != null && highlightStart === e.activeLabel) { - // If it's just a click, don't zoom setHighlightStart(undefined); setHighlightEnd(undefined); mouseDownPosRef.current = null; - } else if ( - highlightStart != null && - highlightEnd != null && - dragDistance >= MIN_DRAG_DISTANCE - ) { - try { - onTimeRangeSelect?.( - new Date( - Number.parseInt( - highlightStart <= highlightEnd - ? highlightStart - : highlightEnd, - ) * 1000, - ), - new Date( - Number.parseInt( - highlightEnd >= highlightStart - ? highlightEnd - : highlightStart, - ) * 1000, - ), - ); - } catch (e) { - console.error('failed to highlight range', e); + }} + onMouseDown={e => { + if (e != null && e.chartX != null && e.chartY != null) { + setHighlightStart(e.activeLabel); + mouseDownPosRef.current = e.chartX; } - setHighlightStart(undefined); - setHighlightEnd(undefined); - mouseDownPosRef.current = null; - } else { - // Drag was too short, clear the highlight - setHighlightStart(undefined); - setHighlightEnd(undefined); - mouseDownPosRef.current = null; - } - }} - onClick={(state, e) => { - if ( - state != null && - state.chartX != null && - state.chartY != null && - state.activeLabel != null && - // If we didn't drag and highlight yet - highlightStart == null - ) { - setIsClickActive({ - x: state.chartX, - y: state.chartY, - activeLabel: state.activeLabel, - xPerc: state.chartX / sizeRef.current[0], - yPerc: state.chartY / sizeRef.current[1], - activePayload: state.activePayload, - }); - // The click-frozen tooltip hides the live tooltip, so drop any - // line emphasis to match. - setNearestSeriesKey(undefined); - } else { - // We clicked on the chart but outside of a line - setIsClickActive(undefined); - } + }} + onMouseMove={e => { + setIsHovered(true); + + // Track which series' line is nearest the cursor so the lines can + // emphasize it. The active dots captured their pixel Y on the prior + // frame; comparing the pointer's chartY picks the nearest line. Skip + // while a click-frozen tooltip is shown, matching the tooltip, and + // only set state when the key changes to keep re-renders rare. + const activePointYByKey = activePointYByKeyRef.current; + const nextNearest = + isClickActive == null && + activePointYByKey.size > 1 && + e?.chartY != null + ? findNearestSeriesKey( + activePointYByKey, + Array.from(activePointYByKey.keys()), + e.chartY, + NEAREST_SERIES_MAX_DISTANCE_PX, + ) + : undefined; + setNearestSeriesKey(prev => + prev === nextNearest ? prev : nextNearest, + ); - // TODO: Properly detect clicks outside of the fake tooltip - e.stopPropagation(); - }} - > - - {/* Gradient defs cover every hex that any fill may reference. + if (highlightStart != null) { + setHighlightEnd(e.activeLabel); + setIsClickActive(undefined); // Clear out any click state as we're highlighting + } + }} + onMouseUp={e => { + const MIN_DRAG_DISTANCE = 20; // Minimum horizontal drag distance in pixels + let dragDistance = 0; + + if (mouseDownPosRef.current != null && e?.chartX != null) { + dragDistance = Math.abs(e.chartX - mouseDownPosRef.current); + } + + if (e?.activeLabel != null && highlightStart === e.activeLabel) { + // If it's just a click, don't zoom + setHighlightStart(undefined); + setHighlightEnd(undefined); + mouseDownPosRef.current = null; + } else if ( + highlightStart != null && + highlightEnd != null && + dragDistance >= MIN_DRAG_DISTANCE + ) { + try { + // Remember the range we're zooming away from so "Reset zoom" can + // restore it. Keep the earliest origin across consecutive zooms. + const originStart = dateRange[0]; + const originEnd = dateRange[1]; + setZoomOrigin(prev => prev ?? [originStart, originEnd]); + justZoomedRef.current = true; + onTimeRangeSelect?.( + new Date( + Number.parseInt( + highlightStart <= highlightEnd + ? highlightStart + : highlightEnd, + ) * 1000, + ), + new Date( + Number.parseInt( + highlightEnd >= highlightStart + ? highlightEnd + : highlightStart, + ) * 1000, + ), + ); + } catch (e) { + console.error('failed to highlight range', e); + justZoomedRef.current = false; + setZoomOrigin(null); + } + setHighlightStart(undefined); + setHighlightEnd(undefined); + mouseDownPosRef.current = null; + } else { + // Drag was too short, clear the highlight + setHighlightStart(undefined); + setHighlightEnd(undefined); + mouseDownPosRef.current = null; + } + }} + onClick={(state, e) => { + if ( + state != null && + state.chartX != null && + state.chartY != null && + state.activeLabel != null && + // If we didn't drag and highlight yet + highlightStart == null + ) { + setIsClickActive({ + x: state.chartX, + y: state.chartY, + activeLabel: state.activeLabel, + xPerc: state.chartX / sizeRef.current[0], + yPerc: state.chartY / sizeRef.current[1], + activePayload: state.activePayload, + }); + // The click-frozen tooltip hides the live tooltip, so drop any + // line emphasis to match. + setNearestSeriesKey(undefined); + } else { + // We clicked on the chart but outside of a line + setIsClickActive(undefined); + } + + // TODO: Properly detect clicks outside of the fake tooltip + e.stopPropagation(); + }} + > + + {/* Gradient defs cover every hex that any fill may reference. `COLORS` (the unified categorical palette) is included up-front as a baseline; semantic colors returned by the `getChartColor{Info,Success,Warning,Error}` helpers can also @@ -892,96 +968,97 @@ export const MemoChart = memo(function MemoChart({ resolve to `--color-chart-info`, chart blue `#437eef`, on both brands, which matches categorical slot 0). Union them here so the referenced `url(#time-chart-lin-grad-…)` always exists. */} - {collectMemoChartGradientHexes(lineData).map(c => { - return ( - - - - - ); - })} - - {isHovered && ( - - )} - - - {lines} - {isClickActive == null && ( - - } - wrapperStyle={{ - zIndex: 1, - }} - /> - )} - {referenceLines} - {highlightStart && highlightEnd ? ( - - ) : null} - {showLegend && ( - - } - offset={-100} + {collectMemoChartGradientHexes(lineData).map(c => { + return ( + + + + + ); + })} + + {isHovered && ( + + )} + - )} - {/** Needs to be at the bottom to prevent re-rendering */} - {isClickActive != null ? ( - - ) : null} - {logReferenceTimestamp != null ? ( - - ) : null} - - + {lines} + {isClickActive == null && ( + + } + wrapperStyle={{ + zIndex: 1, + }} + /> + )} + {referenceLines} + {highlightStart && highlightEnd ? ( + + ) : null} + {showLegend && ( + + } + offset={-100} + /> + )} + {/** Needs to be at the bottom to prevent re-rendering */} + {isClickActive != null ? ( + + ) : null} + {logReferenceTimestamp != null ? ( + + ) : null} + + +
); }); From 0c72543604a835c9b785baef72e8ba59ca7ce4d0 Mon Sep 17 00:00:00 2001 From: matt Date: Mon, 6 Jul 2026 05:34:12 -0400 Subject: [PATCH 148/728] feat: consecutive-window alerts (#2472) Co-authored-by: Karl Power <85935352+karl-power@users.noreply.github.com> Co-authored-by: Karl Power --- .changeset/rare-spoons-change.md | 9 + packages/api/src/controllers/alertHistory.ts | 16 +- packages/api/src/controllers/alerts.ts | 3 + packages/api/src/models/alert.ts | 9 + packages/api/src/models/alertHistory.ts | 5 + packages/api/src/routers/api/alerts.ts | 1 + .../checkAlerts/__tests__/checkAlerts.test.ts | 468 +++++++++++++++++- packages/api/src/tasks/checkAlerts/index.ts | 238 +++++++-- .../tasks/checkAlerts/providers/default.ts | 32 +- .../src/tasks/checkAlerts/providers/index.ts | 3 + packages/api/src/utils/zod.ts | 1 + packages/app/src/AlertsPage.tsx | 19 + packages/app/src/DBDashboardPage.tsx | 3 + packages/app/src/DBSearchPageAlertModal.tsx | 7 + .../src/components/AlertScheduleFields.tsx | 50 +- .../app/src/components/AlertStatusIcon.tsx | 13 +- .../DBEditTimeChartForm/TileAlertEditor.tsx | 8 +- .../components/alerts/AlertHistoryCards.tsx | 25 +- packages/app/styles/AlertsPage.module.scss | 4 + packages/common-utils/src/types.ts | 3 + 20 files changed, 861 insertions(+), 56 deletions(-) create mode 100644 .changeset/rare-spoons-change.md diff --git a/.changeset/rare-spoons-change.md b/.changeset/rare-spoons-change.md new file mode 100644 index 0000000000..e68e316b5a --- /dev/null +++ b/.changeset/rare-spoons-change.md @@ -0,0 +1,9 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +Adding consecutive-window configuration to alerts, so that you can specify a condition like "only fire this alert after some condition is met for N consecutive windows." This helps prevent flaky alerts (and pages), and cut down on alert noise in many cases. + +Also adds a `PENDING` alert state for alarms that _will_ fire if current trends continue. diff --git a/packages/api/src/controllers/alertHistory.ts b/packages/api/src/controllers/alertHistory.ts index 93ed9c1468..86f5701d5b 100644 --- a/packages/api/src/controllers/alertHistory.ts +++ b/packages/api/src/controllers/alertHistory.ts @@ -18,14 +18,24 @@ type GroupedAlertHistory = { lastValues: IAlertHistory['lastValues'][]; }; +function groupStateToOverallState(states: string[]): AlertState { + if (states.includes(AlertState.ALERT)) { + return AlertState.ALERT; + } + + if (states.includes(AlertState.PENDING)) { + return AlertState.PENDING; + } + + return AlertState.OK; +} + function mapGroupedHistories( groupedHistories: GroupedAlertHistory[], ): Omit[] { return groupedHistories.map(group => ({ createdAt: group._id, - state: group.states.includes(AlertState.ALERT) - ? AlertState.ALERT - : AlertState.OK, + state: groupStateToOverallState(group.states), counts: group.counts, lastValues: group.lastValues .flat() diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 095d317d3b..238b069946 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -155,6 +155,9 @@ const makeAlert = (alert: AlertInput, userId?: ObjectId): Partial => { // Chart alerts dashboard: alert.dashboardId as unknown as ObjectId, tileId: alert.tileId, + + // Multi-window alerting + numConsecutiveWindows: alert.numConsecutiveWindows ?? null, }; }; diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index bea7653d7b..e607629f78 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -14,6 +14,7 @@ export enum AlertState { DISABLED = 'DISABLED', INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', OK = 'OK', + PENDING = 'PENDING', } export interface IAlertError { @@ -84,6 +85,9 @@ export interface IAlert { until: Date; }; + // Multi-window alerting: fire only after N violations in M consecutive windows + numConsecutiveWindows?: number | null; + // Errors recorded during the most recent execution executionErrors?: IAlertError[]; createdAt: Date; @@ -190,6 +194,11 @@ const AlertSchema = new Schema( type: String, required: false, }, + numConsecutiveWindows: { + type: Number, + required: false, + min: 1, + }, silenced: { required: false, type: { diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 93cd4ca254..5cf8630f81 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -12,6 +12,7 @@ export interface IAlertHistory { state: AlertState; lastValues: { startTime: Date; count: number }[]; group?: string; // For group-by alerts, stores the group identifier + fired?: boolean; } const AlertHistorySchema = new Schema({ @@ -45,6 +46,10 @@ const AlertHistorySchema = new Schema({ type: String, required: false, }, + fired: { + type: Boolean, + required: false, + }, }); AlertHistorySchema.index( diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index e3fdde4e42..9da8c63b31 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -84,6 +84,7 @@ const formatAlertResponse = ( 'createdAt', 'updatedAt', 'executionErrors', + 'numConsecutiveWindows', ]), }; }; diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts index 152cfcf36a..5ebfb95f96 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts @@ -25,7 +25,7 @@ import { RAW_SQL_ALERT_TEMPLATE, RAW_SQL_NUMBER_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { AlertSource } from '@/models/alert'; +import Alert, { AlertSource, IAlert } from '@/models/alert'; import AlertHistory from '@/models/alertHistory'; import Connection, { IConnection } from '@/models/connection'; import Dashboard, { IDashboard } from '@/models/dashboard'; @@ -37,6 +37,7 @@ import * as checkAlert from '@/tasks/checkAlerts'; import { alertHasGroupBy, doesExceedThreshold, + getConsecutiveWindowHistories, getPreviousAlertHistories, getScheduledWindowStart, parseAlertData, @@ -2114,15 +2115,16 @@ describe('checkAlerts', () => { alertProvider: AlertProvider, teamWebhooksById: Map, ) => { - const previousMap = await getPreviousAlertHistories( - [details.alert.id], - now, - ); + const [previousMap, recentHistoryMap] = await Promise.all([ + getPreviousAlertHistories([details.alert.id], now), + getConsecutiveWindowHistories([details.alert], now), + ]); await processAlert( now, { ...details, previousMap, + recentHistoryMap, }, clickhouseClient, connection.id, @@ -8537,6 +8539,351 @@ describe('checkAlerts', () => { ); expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); }); + + describe('multi-window alerting (numConsecutiveWindows)', () => { + it('fires on the first violation when numConsecutiveWindows=1', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 0, + savedSearchId: savedSearch.id, + numConsecutiveWindows: 1, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date('2024-01-01T00:05:00Z'), + SeverityText: 'error', + Body: 'err', + }, + ]); + + await processAlertAtTime( + new Date('2024-01-01T00:12:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const histories = await AlertHistory.find({ alert: details.alert.id }); + expect(histories).toHaveLength(1); + expect(histories[0].state).toBe('ALERT'); + expect(histories[0].fired).toBe(true); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + }); + + it('suppresses notification until all M windows have violated', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue(null as any); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 0, + savedSearchId: savedSearch.id, + numConsecutiveWindows: 3, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date('2024-01-01T00:05:00Z'), + SeverityText: 'error', + Body: 'err', + }, + { + ServiceName: 'api', + Timestamp: new Date('2024-01-01T00:10:00Z'), + SeverityText: 'error', + Body: 'err', + }, + { + ServiceName: 'api', + Timestamp: new Date('2024-01-01T00:15:00Z'), + SeverityText: 'error', + Body: 'err', + }, + ]); + + // don't fire on windows 1 or 2 since we need 3 consecutive violations to fire + await processAlertAtTime( + new Date('2024-01-01T00:12:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); + let histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories).toHaveLength(1); + expect(histories[0].state).toBe('PENDING'); + expect(histories[0].fired).toBeFalsy(); // shouldn't fire yet + + await processAlertAtTime( + new Date('2024-01-01T00:17:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); + histories = await AlertHistory.find({ alert: details.alert.id }).sort({ + createdAt: 1, + }); + expect(histories).toHaveLength(2); + expect(histories[1].state).toBe('PENDING'); + expect(histories[1].fired).toBeFalsy(); // shouldn't fire yet + + await processAlertAtTime( + new Date('2024-01-01T00:22:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + histories = await AlertHistory.find({ alert: details.alert.id }).sort({ + createdAt: 1, + }); + expect(histories).toHaveLength(3); + expect(histories[2].state).toBe('ALERT'); + expect(histories[2].fired).toBe(true); // fires here b/c it's the third consecutive violation + }); + + it('does not send a resolution notification when no alert had previously fired', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue(null as any); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, // threshold=1 so zero-fill (no data) produces OK, not ALERT + savedSearchId: savedSearch.id, + numConsecutiveWindows: 3, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date('2024-01-01T00:05:00Z'), + SeverityText: 'error', + Body: 'err', + }, + { + ServiceName: 'api', + Timestamp: new Date('2024-01-01T00:10:00Z'), + SeverityText: 'error', + Body: 'err', + }, + ]); + + await processAlertAtTime( + new Date('2024-01-01T00:12:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); + + await processAlertAtTime( + new Date('2024-01-01T00:17:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); + + // window 3 is ok (no violation) + await processAlertAtTime( + new Date('2024-01-01T00:22:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); // webhook should never fire in this case + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories).toHaveLength(3); + expect(histories[0].state).toBe('PENDING'); + expect(histories[0].fired).toBeFalsy(); + expect(histories[1].state).toBe('PENDING'); + expect(histories[1].fired).toBeFalsy(); + expect(histories[2].state).toBe('OK'); + }); + + it('tracks consecutive windows independently per group', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue(null as any); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 0, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + numConsecutiveWindows: 2, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Window 1: only service-a violates. + // Window 2: both service-a (2nd consecutive) and service-b (1st) violate. + await bulkInsertLogs([ + { + ServiceName: 'service-a', + Timestamp: new Date('2024-01-01T00:05:00Z'), + SeverityText: 'error', + Body: 'err', + }, + { + ServiceName: 'service-a', + Timestamp: new Date('2024-01-01T00:10:00Z'), + SeverityText: 'error', + Body: 'err', + }, + { + ServiceName: 'service-b', + Timestamp: new Date('2024-01-01T00:10:00Z'), + SeverityText: 'error', + Body: 'err', + }, + ]); + + // Window 1: service-a has its first violation -> PENDING, no notification. + await processAlertAtTime( + new Date('2024-01-01T00:12:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); + + // Window 2: service-a hits 2 consecutive violations -> ALERT (fires); + // service-b is on its first violation -> PENDING (does not fire). + await processAlertAtTime( + new Date('2024-01-01T00:17:00Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + // Exactly one notification: service-a's transition to ALERT. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + const serviceAHistories = histories + .filter(h => h.group === 'ServiceName:service-a') + .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()); + const serviceBHistories = histories.filter( + h => h.group === 'ServiceName:service-b', + ); + + // service-a: first window PENDING, second window ALERT (fired). + expect(serviceAHistories).toHaveLength(2); + expect(serviceAHistories[0].state).toBe('PENDING'); + expect(serviceAHistories[0].fired).toBeFalsy(); + expect(serviceAHistories[1].state).toBe('ALERT'); + expect(serviceAHistories[1].fired).toBe(true); + + // service-b: only one violation so far, so it must still be PENDING even + // though service-a fired in the same run. + expect(serviceBHistories).toHaveLength(1); + expect(serviceBHistories[0].state).toBe('PENDING'); + expect(serviceBHistories[0].fired).toBeFalsy(); + }); + }); }); describe('processAlert with materialized views', () => { @@ -9094,4 +9441,115 @@ describe('checkAlerts', () => { ); }); }); + + describe('getConsecutiveWindowHistories', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + jest.clearAllMocks(); + }); + + afterAll(async () => { + await server.stop(); + }); + + // getConsecutiveWindowHistories only reads scheduling/config fields off the + // alert (id, interval, numConsecutiveWindows, schedule*) and never loads the + // alert from the DB, so a lightweight stub is sufficient. + const makeAlert = ( + id: mongoose.Types.ObjectId, + numConsecutiveWindows?: number, + interval = '5m', + ): IAlert => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + ({ + id: id.toString(), + interval, + numConsecutiveWindows, + }) as unknown as IAlert; + + const saveHistory = ( + alertId: mongoose.Types.ObjectId, + createdAt: Date, + opts: { group?: string; state?: AlertState } = {}, + ) => + new AlertHistory({ + alert: alertId, + createdAt, + state: opts.state ?? AlertState.ALERT, + group: opts.group, + }).save(); + + it('skips alerts with numConsecutiveWindows <= 1 (no query, empty map)', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveHistory(alertId, new Date('2025-01-01T00:10:00Z')); + + const aggregateSpy = jest.spyOn(AlertHistory, 'aggregate'); + + const result = await getConsecutiveWindowHistories( + [makeAlert(alertId, 1), makeAlert(new mongoose.Types.ObjectId())], + new Date('2025-01-01T00:17:00Z'), + ); + + expect(aggregateSpy).not.toHaveBeenCalled(); + expect(result.size).toBe(0); + }); + + it('buckets recent histories per group, newest-first, within the lookback window', async () => { + const alertId = new mongoose.Types.ObjectId(); + // now=00:17 -> windowStart=00:15; lookback = (3-1)*5m -> [00:05, 00:15) + await saveHistory(alertId, new Date('2025-01-01T00:00:00Z'), { + group: 'ServiceName:a', + }); // excluded: before earliestAllowedTime + await saveHistory(alertId, new Date('2025-01-01T00:05:00Z'), { + group: 'ServiceName:a', + }); + await saveHistory(alertId, new Date('2025-01-01T00:10:00Z'), { + group: 'ServiceName:a', + }); + await saveHistory(alertId, new Date('2025-01-01T00:15:00Z'), { + group: 'ServiceName:a', + }); // excluded: == windowStart (the current window being evaluated) + await saveHistory(alertId, new Date('2025-01-01T00:10:00Z'), { + group: 'ServiceName:b', + }); + + const result = await getConsecutiveWindowHistories( + [makeAlert(alertId, 3)], + new Date('2025-01-01T00:17:00Z'), + ); + + const aKey = `${alertId.toString()}||ServiceName:a`; + const bKey = `${alertId.toString()}||ServiceName:b`; + + expect(result.get(aKey)!.map(h => h.createdAt)).toEqual([ + new Date('2025-01-01T00:10:00Z'), + new Date('2025-01-01T00:05:00Z'), + ]); + expect(result.get(bKey)!.map(h => h.createdAt)).toEqual([ + new Date('2025-01-01T00:10:00Z'), + ]); + }); + + it('keys ungrouped histories by the bare alert id', async () => { + const alertId = new mongoose.Types.ObjectId(); + // now=00:17 -> windowStart=00:15; lookback = (2-1)*5m -> [00:10, 00:15) + await saveHistory(alertId, new Date('2025-01-01T00:10:00Z')); + + const result = await getConsecutiveWindowHistories( + [makeAlert(alertId, 2)], + new Date('2025-01-01T00:17:00Z'), + ); + + expect(result.size).toBe(1); + expect(result.get(alertId.toString())!.map(h => h.state)).toEqual([ + AlertState.ALERT, + ]); + }); + }); }); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 3c0199f796..82f15525df 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -324,6 +324,31 @@ export const getScheduledWindowStart = ( return fns.addMinutes(roundedShiftedNow, scheduleOffsetMinutes); }; +/** + * Compute the scheduled window start ("now rounded down to the window") for an + * alert at the given time. This mirrors the computation inside processAlert so + * that history fetched up-front (see getConsecutiveWindowHistories) lines up + * exactly with the window processAlert evaluates. + */ +export const getAlertWindowStart = (alert: IAlert, now: Date): Date => { + const windowSizeInMins = ms(alert.interval) / 60000; + const scheduleStartAt = normalizeScheduleStartAt({ + alertId: alert.id, + scheduleStartAt: alert.scheduleStartAt, + }); + const scheduleOffsetMinutes = normalizeScheduleOffsetMinutes({ + alertId: alert.id, + scheduleOffsetMinutes: alert.scheduleOffsetMinutes, + windowSizeInMins, + }); + return getScheduledWindowStart( + now, + windowSizeInMins, + scheduleOffsetMinutes, + scheduleStartAt, + ); +}; + const fireChannelEvent = async ({ alert, alertProvider, @@ -738,7 +763,7 @@ export const processAlert = async ( alertProvider: AlertProvider, teamWebhooksById: Map, ) => { - const { alert, previousMap } = details; + const { alert, previousMap, recentHistoryMap } = details; const source = 'source' in details ? details.source : undefined; // Errors collected during this execution. Webhook errors accumulate here; query // and validation errors are recorded via recordAlertErrors before returning. @@ -1057,13 +1082,38 @@ export const processAlert = async ( } }; + const numWindowsToLookBack = alert.numConsecutiveWindows ?? 1; + + const shouldFireBasedOnConsecutiveWindows = ( + groupKey?: string, + ): boolean => { + if (numWindowsToLookBack <= 1) { + return true; + } + + // recentHistoryMap entries are pre-filtered to the lookback window and + // sorted newest-first, so take the most recent M-1 for this group. + const key = computeHistoryMapKey(alert.id, groupKey || ''); + const groupHistories = recentHistoryMap?.get(key) ?? []; + const relevant = groupHistories.slice(0, numWindowsToLookBack - 1); + + return ( + relevant.length === numWindowsToLookBack - 1 && + relevant.every( + h => h.state === AlertState.ALERT || h.state === AlertState.PENDING, + ) + ); + }; + const sendNotificationIfResolved = async ( previousHistory: AggregatedAlertHistory | undefined, currentHistory: IAlertHistory, groupKey: string, ) => { if ( - previousHistory?.state === AlertState.ALERT && + (previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING) && + previousHistory?.fired !== false && currentHistory.state === AlertState.OK ) { const lastValue = @@ -1099,19 +1149,26 @@ export const processAlert = async ( : 0; history.lastValues.push({ count: value, startTime: alertTimestamp }); + const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); if (doesExceedThreshold(alert, value)) { - history.state = AlertState.ALERT; history.counts += 1; - await trySendNotification({ - state: AlertState.ALERT, - group: '', - totalCount: value, - startTime: alertTimestamp, - }); + if (shouldFireBasedOnConsecutiveWindows()) { + history.state = AlertState.ALERT; + history.fired = true; + await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: value, + startTime: alertTimestamp, + }); + } else { + history.state = AlertState.PENDING; + // Carry forward fired=true if a notification was previously sent and not yet resolved. + history.fired = previous?.fired === true; + } } // Auto-resolve - const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); await sendNotificationIfResolved(previous, history, ''); const historyRecords = Array.from(histories.values()); @@ -1161,19 +1218,32 @@ export const processAlert = async ( const hasAlertsInPreviousMap = previousMap .values() - .some(history => history.state === AlertState.ALERT); + .some( + history => + history.state === AlertState.ALERT || + history.state === AlertState.PENDING, + ); if (zeroValueIsAlert) { const history = getOrCreateHistory(''); history.lastValues.push({ count: 0, startTime: bucketStart }); - history.state = AlertState.ALERT; history.counts += 1; - await trySendNotification({ - state: AlertState.ALERT, - group: '', - totalCount: 0, - startTime: bucketStart, - }); + if (shouldFireBasedOnConsecutiveWindows()) { + history.state = AlertState.ALERT; + history.fired = true; + await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: 0, + startTime: bucketStart, + }); + } else { + history.state = AlertState.PENDING; + // Carry forward fired=true if a notification was previously sent and not yet resolved. + history.fired = + previousMap.get(computeHistoryMapKey(alert.id, ''))?.fired === + true; + } } else if (!hasGroupBy || !hasAlertsInPreviousMap) { // For grouped alerts, if there are alerts in the previous map, // we will handle creating a history as part of auto-resolve later @@ -1201,16 +1271,24 @@ export const processAlert = async ( const history = getOrCreateHistory(groupKey); if (doesExceedThreshold(alert, value)) { - history.state = AlertState.ALERT; - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: value, - startTime: bucketStart, - attributes, - }); - history.counts += 1; + if (shouldFireBasedOnConsecutiveWindows(groupKey)) { + history.state = AlertState.ALERT; + history.fired = true; + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: value, + startTime: bucketStart, + attributes, + }); + } else { + history.state = AlertState.PENDING; + // Carry forward fired=true if a notification was previously sent and not yet resolved. + history.fired = + previousMap.get(computeHistoryMapKey(alert.id, groupKey)) + ?.fired === true; + } } else { // TODO: if the alert was previously alerting (different bucket), should we set state to OK (plus auto-resolve)? } @@ -1218,16 +1296,17 @@ export const processAlert = async ( } } - // Handle missing groups: If current check found no data, check if any previously alerting groups need to be resolved - // For group-by alerts, check if any previously alerting groups are missing from current data + // Handle missing groups: If current check found no data, check if any previously alerting/pending groups need to be resolved + // For group-by alerts, check if any previously alerting or pending groups are missing from current data if (hasGroupBy && previousMap && previousMap.size > 0) { for (const [previousKey, previousHistory] of previousMap.entries()) { const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); - // If this group was previously ALERT but is missing from current data and would be resolved by a 0 value, + // If this group was previously ALERT or PENDING but is missing from current data and would be resolved by a 0 value, // create an OK history for the group if ( - previousHistory.state === AlertState.ALERT && + (previousHistory.state === AlertState.ALERT || + previousHistory.state === AlertState.PENDING) && !histories.has(groupKey) && !doesExceedThreshold(alert, 0) ) { @@ -1236,7 +1315,7 @@ export const processAlert = async ( alertId: alert.id, group: groupKey, }, - `Group "${groupKey}" is missing from current data but was previously alerting - creating OK history`, + `Group "${groupKey}" is missing from current data but was previously ${previousHistory.state} - creating OK history`, ); const history = getOrCreateHistory(groupKey); history.lastValues.push({ count: 0, startTime: expectedBuckets[0] }); @@ -1317,6 +1396,7 @@ export interface AggregatedAlertHistory { createdAt: Date; state: AlertState; group?: string; + fired?: boolean; } /** @@ -1337,12 +1417,14 @@ export interface AggregatedAlertHistory { export const getPreviousAlertHistories = async ( alertIds: string[], now: Date, + sharedQueue?: PQueue, ) => { const lookbackDate = new Date(now.getTime() - ms('7d')); - // Use a concurrency-limited queue to avoid overwhelming the connection pool - // when there are many alerts (e.g., 200+ alert IDs). - const queue = new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); + // Concurrency-limited per-alert queries to avoid overwhelming the connection + // pool when there are many alerts (e.g., 200+ alert IDs). + const queue = + sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); const results = await Promise.all( alertIds.map(alertId => @@ -1371,6 +1453,7 @@ export const getPreviousAlertHistories = async ( }, createdAt: { $first: '$createdAt' }, state: { $first: '$state' }, + fired: { $first: '$fired' }, }, }, { @@ -1379,6 +1462,7 @@ export const getPreviousAlertHistories = async ( createdAt: 1, state: 1, group: '$_id.group', + fired: 1, }, }, ]); @@ -1401,6 +1485,90 @@ export const getPreviousAlertHistories = async ( ); }; +/** + * For alerts that use multi-window lookback (numConsecutiveWindows > 1), + * batch-fetch the per-group history needed to decide whether the alert condition + * has been met in M consecutive windows. + * + * Alerts with numConsecutiveWindows <= 1 are skipped entirely (no query is run). + * + * For each multi-window alert we fetch the AlertHistory records whose createdAt + * falls in [windowStart - (M-1)*window, windowStart), sorted newest-first, then + * bucket them by group. processAlert takes the most recent M-1 per group and + * requires every one of them to be ALERT/PENDING to fire. The window start is + * computed with getAlertWindowStart so it matches the window processAlert + * evaluates for the same `now`. + */ +export const getConsecutiveWindowHistories = async ( + alerts: IAlert[], + now: Date, + sharedQueue?: PQueue, +): Promise> => { + const map = new Map(); + + const multiWindowAlerts = alerts.filter( + alert => (alert.numConsecutiveWindows ?? 1) > 1, + ); + if (multiWindowAlerts.length === 0) { + return map; + } + + // Concurrency-limited per-alert queries (same approach as getPreviousAlertHistories) + const queue = + sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); + + const results = await Promise.all( + multiWindowAlerts.map(alert => + queue.add(async () => { + const numWindowsToLookBack = alert.numConsecutiveWindows ?? 1; + const windowSizeInMins = ms(alert.interval) / 60000; + const windowStart = getAlertWindowStart(alert, now); + const earliestAllowedTime = new Date( + windowStart.getTime() - + (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, + ); + const id = new mongoose.Types.ObjectId(alert.id); + const histories = await AlertHistory.aggregate([ + { + $match: { + alert: id, + createdAt: { $gte: earliestAllowedTime, $lt: windowStart }, + }, + }, + { $sort: { alert: 1, group: 1, createdAt: -1 } }, + { + $project: { + _id: '$alert', + createdAt: 1, + state: 1, + group: 1, + fired: 1, + }, + }, + ]); + return { alertId: alert.id, histories }; + }), + ), + ); + + for (const result of results) { + if (!result) { + continue; + } + for (const history of result.histories) { + const key = computeHistoryMapKey(result.alertId, history.group || ''); + const bucket = map.get(key); + if (bucket) { + bucket.push(history); + } else { + map.set(key, [history]); + } + } + } + + return map; +}; + export default class CheckAlertTask implements HdxTask { private provider!: AlertProvider; private task_queue: PQueue; diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 7a255bff76..079cdcc89d 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -1,3 +1,4 @@ +import PQueue from '@esm2cjs/p-queue'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { displayTypeSupportsRawSqlAlerts } from '@hyperdx/common-utils/dist/core/utils'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; @@ -7,6 +8,7 @@ import ms from 'ms'; import { URLSearchParams } from 'url'; import * as config from '@/config'; +import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; import { LOCAL_APP_TEAM } from '@/controllers/team'; import { connectDB, mongooseConnection, ObjectId } from '@/models'; import Alert, { @@ -23,6 +25,7 @@ import { type ISource, Source } from '@/models/source'; import Webhook, { IWebhook } from '@/models/webhook'; import { AggregatedAlertHistory, + getConsecutiveWindowHistories, getPreviousAlertHistories, } from '@/tasks/checkAlerts'; import { @@ -216,6 +219,7 @@ async function loadAlert( alert: IAlert, groupedTasks: Map, previousAlerts: Map, + recentHistoryMap: Map, now: Date, ) { if (!alert.source) { @@ -258,7 +262,11 @@ async function loadAlert( if (!v) { throw new Error(`provider did not set key ${conn.id} before appending`); } - v.alerts.push({ ...details, previousMap: previousAlerts }); + v.alerts.push({ + ...details, + previousMap: previousAlerts, + recentHistoryMap, + }); } export default class DefaultAlertProvider implements AlertProvider { @@ -276,11 +284,25 @@ export default class DefaultAlertProvider implements AlertProvider { const now = new Date(); const alertIds = alerts.map(({ id }) => id); - const previousAlerts = await getPreviousAlertHistories(alertIds, now); + // Share a single queue across both history fetches so their combined + // in-flight per-alert queries stay within one global cap. + const historyQueryQueue = new PQueue({ + concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, + }); + const [previousAlerts, recentHistoryMap] = await Promise.all([ + getPreviousAlertHistories(alertIds, now, historyQueryQueue), + getConsecutiveWindowHistories(alerts, now, historyQueryQueue), + ]); for (const alert of alerts) { try { - await loadAlert(alert, groupedTasks, previousAlerts, now); + await loadAlert( + alert, + groupedTasks, + previousAlerts, + recentHistoryMap, + now, + ); } catch (e) { logger.error({ message: `failed to load alert: ${e}`, @@ -382,7 +404,9 @@ export default class DefaultAlertProvider implements AlertProvider { const finalState = historiesToCheck.some(h => h.state === AlertState.ALERT) ? AlertState.ALERT - : AlertState.OK; + : historiesToCheck.some(h => h.state === AlertState.PENDING) + ? AlertState.PENDING + : AlertState.OK; // Update alert state + errors based on this execution await Alert.updateOne( diff --git a/packages/api/src/tasks/checkAlerts/providers/index.ts b/packages/api/src/tasks/checkAlerts/providers/index.ts index 116be8c588..aeed3e5407 100644 --- a/packages/api/src/tasks/checkAlerts/providers/index.ts +++ b/packages/api/src/tasks/checkAlerts/providers/index.ts @@ -32,6 +32,9 @@ export type PopulatedAlertChannel = { type: 'webhook' } & { channel: IWebhook }; export type AlertDetails = { alert: IAlert; previousMap: Map; // Map of alertId||group -> history for group-by alerts + // For multi-window alerts (numConsecutiveWindows > 1): the recent per-group + // history (alertId||group -> histories, newest-first). + recentHistoryMap?: Map; } & ( | { taskType: AlertTaskType.SAVED_SEARCH; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 3675f0dac1..5dd1798299 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -636,6 +636,7 @@ export const alertSchema = z name: z.string().min(1).max(512).nullish(), message: z.string().min(1).max(4096).nullish(), note: alertNoteSchema, + numConsecutiveWindows: z.number().int().min(1).nullish(), }) .and(zSavedSearchAlert.or(zTileAlert)) .superRefine(validateAlertScheduleOffsetMinutes) diff --git a/packages/app/src/AlertsPage.tsx b/packages/app/src/AlertsPage.tsx index 5ddf11828e..1d1345c0f3 100644 --- a/packages/app/src/AlertsPage.tsx +++ b/packages/app/src/AlertsPage.tsx @@ -30,6 +30,7 @@ import { IconChevronDown, IconChevronRight, IconHelpCircle, + IconHourglass, IconInfoCircleFilled, IconNote, IconSearch, @@ -213,6 +214,11 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) { Alert )} + {alert.state === AlertState.PENDING && ( + + Pending + + )} {alert.state === AlertState.OK && Ok} {alert.state === AlertState.DISABLED && ( @@ -269,6 +275,9 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) { function AlertCardList({ alerts }: { alerts: AlertsPageItem[] }) { const alarmAlerts = alerts.filter(alert => alert.state === AlertState.ALERT); + const pendingAlerts = alerts.filter( + alert => alert.state === AlertState.PENDING, + ); const okData = alerts.filter(alert => alert.state === AlertState.OK); return ( @@ -283,6 +292,16 @@ function AlertCardList({ alerts }: { alerts: AlertsPageItem[] }) { ))}
)} + {pendingAlerts.length > 0 && ( +
+ + Pending + + {pendingAlerts.map(alert => ( + + ))} +
+ )}
OK diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index a35ab57c7c..75d10d1e7f 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -593,6 +593,9 @@ const Tile = forwardRef( if (alert.silenced?.at) { return 'yellow'; } + if (alert.state === AlertState.PENDING) { + return 'orange'; + } return 'red'; }, [alert]); diff --git a/packages/app/src/DBSearchPageAlertModal.tsx b/packages/app/src/DBSearchPageAlertModal.tsx index 9f2cd43098..3af7e81fe3 100644 --- a/packages/app/src/DBSearchPageAlertModal.tsx +++ b/packages/app/src/DBSearchPageAlertModal.tsx @@ -77,6 +77,7 @@ const SavedSearchAlertFormSchema = z scheduleStartAt: scheduleStartAtSchema, thresholdType: z.nativeEnum(AlertThresholdType), channel: zAlertChannel, + numConsecutiveWindows: z.number().int().min(1).optional(), }) .passthrough() .superRefine(validateAlertScheduleOffsetMinutes) @@ -150,6 +151,10 @@ const AlertForm = ({ const groupByValue = useWatch({ control, name: 'groupBy' }); const threshold = useWatch({ control, name: 'threshold' }); const thresholdMax = useWatch({ control, name: 'thresholdMax' }); + const numConsecutiveWindows = useWatch({ + control, + name: 'numConsecutiveWindows', + }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, 0, @@ -264,6 +269,8 @@ const AlertForm = ({ scheduleOffsetMinutes={scheduleOffsetMinutes} maxScheduleOffsetMinutes={maxScheduleOffsetMinutes} offsetWindowLabel={`from each ${intervalLabel} window`} + numConsecutiveWindowsName="numConsecutiveWindows" + numConsecutiveWindows={numConsecutiveWindows ?? undefined} /> grouped by diff --git a/packages/app/src/components/AlertScheduleFields.tsx b/packages/app/src/components/AlertScheduleFields.tsx index ccdba37367..774baca58a 100644 --- a/packages/app/src/components/AlertScheduleFields.tsx +++ b/packages/app/src/components/AlertScheduleFields.tsx @@ -36,6 +36,8 @@ type AlertScheduleFieldsProps = { scheduleOffsetMinutes: number | null | undefined; maxScheduleOffsetMinutes: number; offsetWindowLabel: string; + numConsecutiveWindowsName?: FieldPath; + numConsecutiveWindows?: number; }; export function AlertScheduleFields({ @@ -46,6 +48,8 @@ export function AlertScheduleFields({ scheduleOffsetMinutes, maxScheduleOffsetMinutes, offsetWindowLabel, + numConsecutiveWindowsName, + numConsecutiveWindows, }: AlertScheduleFieldsProps) { const showScheduleOffsetInput = maxScheduleOffsetMinutes > 0; const scheduleStartAtValue = useWatch({ @@ -54,7 +58,9 @@ export function AlertScheduleFields({ }) as string | null | undefined; const hasScheduleStartAtAnchor = scheduleStartAtValue != null; const hasAdvancedScheduleValues = - (scheduleOffsetMinutes ?? 0) > 0 || hasScheduleStartAtAnchor; + (scheduleOffsetMinutes ?? 0) > 0 || + hasScheduleStartAtAnchor || + (numConsecutiveWindows ?? 1) > 1; const [opened, setOpened] = useState(hasAdvancedScheduleValues); useEffect(() => { @@ -101,6 +107,48 @@ export function AlertScheduleFields({ Optional schedule controls for aligning alert windows. + {numConsecutiveWindowsName && ( + + + + Consecutive windows + + + + + + + + ( + { + const num = typeof v === 'number' ? v : 1; + field.onChange(num > 1 ? num : undefined); + }} + min={1} + size="xs" + w={70} + /> + )} + /> + + {(numConsecutiveWindows ?? 1) === 1 + ? 'window' + : 'consecutive windows'} + + + )} {showScheduleOffsetInput && ( <> diff --git a/packages/app/src/components/AlertStatusIcon.tsx b/packages/app/src/components/AlertStatusIcon.tsx index 310311155a..74b3bd5f31 100644 --- a/packages/app/src/components/AlertStatusIcon.tsx +++ b/packages/app/src/components/AlertStatusIcon.tsx @@ -9,12 +9,17 @@ export function AlertStatusIcon({ }) { if (!Array.isArray(alerts) || alerts.length === 0) return null; const alertingCount = alerts.filter(a => a.state === AlertState.ALERT).length; + const pendingCount = alerts.filter( + a => a.state === AlertState.PENDING, + ).length; return ( 0 ? `${alertingCount} alert${alertingCount > 1 ? 's' : ''} triggered` - : 'Alerts configured' + : pendingCount > 0 + ? `${pendingCount} alert${pendingCount > 1 ? 's' : ''} pending` + : 'Alerts configured' } > {alertingCount > 0 ? ( @@ -23,6 +28,12 @@ export function AlertStatusIcon({ color="var(--mantine-color-red-filled)" data-testid="alert-status-icon-triggered" /> + ) : pendingCount > 0 ? ( + ) : ( )} diff --git a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx index 187aaaaa69..8bfb0180fa 100644 --- a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx @@ -72,6 +72,10 @@ export function TileAlertEditor({ control, name: 'alert.scheduleOffsetMinutes', }); + const alertnumConsecutiveWindows = useWatch({ + control, + name: 'alert.numConsecutiveWindows', + }); const maxAlertScheduleOffsetMinutes = alert?.interval ? Math.max(intervalToMinutes(alert.interval) - 1, 0) : 0; @@ -206,7 +210,7 @@ export function TileAlertEditor({ )} /> - window via + via Send to diff --git a/packages/app/src/components/alerts/AlertHistoryCards.tsx b/packages/app/src/components/alerts/AlertHistoryCards.tsx index 22aad47cfa..a20537308f 100644 --- a/packages/app/src/components/alerts/AlertHistoryCards.tsx +++ b/packages/app/src/components/alerts/AlertHistoryCards.tsx @@ -26,6 +26,17 @@ import styles from '@styles/AlertsPage.module.scss'; const HISTORY_ITEMS = 18; +function stateToBgColorClass(state: AlertState) { + switch (state) { + case AlertState.OK: + return styles.ok; + case AlertState.PENDING: + return styles.pending; + default: + return styles.alarm; + } +} + function AlertHistoryCard({ history, alertUrl, @@ -58,18 +69,20 @@ function AlertHistoryCard({
); + const count = history.counts ?? 0; + const pending = history.state === AlertState.PENDING ? 'pending' : ''; + const alert = `alert${count === 0 || count > 1 ? 's' : ''}`; + const time = formatRelative(start, today); + const label = `${count} ${pending} ${alert} ${time}`; + return ( - + {href ? ( {content} diff --git a/packages/app/styles/AlertsPage.module.scss b/packages/app/styles/AlertsPage.module.scss index cf95722e94..6261985499 100644 --- a/packages/app/styles/AlertsPage.module.scss +++ b/packages/app/styles/AlertsPage.module.scss @@ -39,6 +39,10 @@ background-color: var(--color-bg-danger); } + &.pending { + background-color: var(--color-bg-warning); + } + &.clickable { cursor: pointer; } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index e1a2e79974..e8c597fb03 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -431,6 +431,7 @@ export enum AlertState { DISABLED = 'DISABLED', INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', OK = 'OK', + PENDING = 'PENDING', } export enum AlertErrorType { @@ -604,6 +605,7 @@ export const AlertBaseObjectSchema = z.object({ until: z.string(), }) .optional(), + numConsecutiveWindows: z.number().int().min(1).nullish(), }); // Keep AlertBaseSchema as a ZodObject for backwards compatibility with @@ -2077,6 +2079,7 @@ export const AlertsPageItemSchema = z.object({ }) .optional(), executionErrors: z.array(AlertErrorSchema).optional(), + numConsecutiveWindows: z.number().int().min(1).nullish(), }); export type AlertsPageItem = z.infer; From 5081c8cbb202273a2bda2d401ba3ab0004602efb Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:00:05 -0700 Subject: [PATCH 149/728] feat(mcp): include source section in list/describe source tools (#2485) The data source `section` label (added in #2432, surfaced in the selector in #2476) is read wherever a user looks at sources, but the MCP source tools never returned it. This adds it to `clickstack_list_sources` and `clickstack_describe_source` so an agent sees the same grouping a user does. HDX-4584. ## Summary - `clickstack_list_sources`: each source entry now includes `section` when the source has one. - `clickstack_describe_source`: the returned `source` metadata now includes `section` when set. - The key is omitted when a source has no section, matching the read-only `GET /api/v2/sources` behavior shipped in #2432. ## Why `section` lives on the shared source schema, so the tools surface a field they already receive: no controller or schema change. Adding it conditionally (only when present) mirrors the existing optional fields in the same `meta` builder (`eventAttributesColumn`, `resourceAttributesColumn`). ## Test plan - [x] `npx nx run @hyperdx/api:ci:lint` (eslint + tsc) clean - [x] Integration: `@hyperdx/api:dev:int mcp/__tests__/sources.test` green (13/13), including new cases asserting `section` is returned when set and omitted when unset for both tools - [x] No external API / OpenAPI change (read-only GET parity already shipped in #2432); `openapi.json` regenerates with no diff [ui-check: allow] MCP tool output only, no UI. [viewport: allow] Not applicable. [ui-states: allow] Not applicable. --- .changeset/mcp-source-section.md | 10 ++++++ .../api/src/mcp/__tests__/sources.test.ts | 32 +++++++++++++++++++ .../src/mcp/tools/sources/describeSource.ts | 4 +++ .../api/src/mcp/tools/sources/listSources.ts | 4 +++ 4 files changed, 50 insertions(+) create mode 100644 .changeset/mcp-source-section.md diff --git a/.changeset/mcp-source-section.md b/.changeset/mcp-source-section.md new file mode 100644 index 0000000000..d27abc7a7c --- /dev/null +++ b/.changeset/mcp-source-section.md @@ -0,0 +1,10 @@ +--- +"@hyperdx/api": patch +--- + +feat: include the source Section in MCP source tools + +The `clickstack_list_sources` and `clickstack_describe_source` MCP tools now +return the optional Section label on each source, so agents see the same source +grouping that the source selector shows. Sources without a section are +unchanged. diff --git a/packages/api/src/mcp/__tests__/sources.test.ts b/packages/api/src/mcp/__tests__/sources.test.ts index 8da78bc7b7..50541a52c6 100644 --- a/packages/api/src/mcp/__tests__/sources.test.ts +++ b/packages/api/src/mcp/__tests__/sources.test.ts @@ -81,6 +81,7 @@ describe('MCP Source Tools', () => { traceIdExpression: 'TraceId', connection: connection._id, name: 'Logs', + section: 'Billing', }); const context: McpContext = { @@ -152,6 +153,19 @@ describe('MCP Source Tools', () => { expect(log.keyColumns).toHaveProperty('body'); }); + it('includes a source section when set and omits it when unset', async () => { + const result = await callTool(client, 'clickstack_list_sources'); + const output = JSON.parse(getFirstText(result)); + + const log = output.sources.find((s: any) => s.kind === SourceKind.Log); + expect(log.section).toBe('Billing'); + + const trace = output.sources.find( + (s: any) => s.kind === SourceKind.Trace, + ); + expect(trace.section).toBeUndefined(); + }); + it('should return empty sources for a team with no sources', async () => { await client.close(); await server.clearDBs(); @@ -225,6 +239,24 @@ describe('MCP Source Tools', () => { }); }); + it('includes the source section when set and omits it when unset', async () => { + const withSection = await callTool(client, 'clickstack_describe_source', { + sourceId: logSource._id.toString(), + }); + expect(JSON.parse(getFirstText(withSection)).source.section).toBe( + 'Billing', + ); + + const withoutSection = await callTool( + client, + 'clickstack_describe_source', + { sourceId: traceSource._id.toString() }, + ); + expect( + JSON.parse(getFirstText(withoutSection)).source.section, + ).toBeUndefined(); + }); + it('should include map attribute keys', async () => { const result = await callTool(client, 'clickstack_describe_source', { sourceId: traceSource._id.toString(), diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 69033303f2..453d2b9efd 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -253,6 +253,10 @@ async function describeSourceSchema( timestampColumn: source.timestampValueExpression, }; + if (source.section) { + meta.section = source.section; + } + if ( 'eventAttributesExpression' in source && source.eventAttributesExpression diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index 2984df7010..b209387aed 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -43,6 +43,10 @@ export function registerListSources({ timestampColumn: s.timestampValueExpression, }; + if (s.section) { + meta.section = s.section; + } + if ('eventAttributesExpression' in s && s.eventAttributesExpression) { meta.eventAttributesColumn = s.eventAttributesExpression; } From d16db2557e2ab7607adc42212f129cefa5b9ff0a Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:21:49 -0700 Subject: [PATCH 150/728] feat(api): support number-tile backgroundChart in the MCP dashboard tools (#2584) Add `backgroundChart` to number tiles in the MCP dashboard tools, so an agent can author the background trend sparkline through `clickstack_save_dashboard` and `clickstack_patch_dashboard`. Mirrors the number-tile color MCP work in #2480 and the v2 REST parity in #2509. Part of #1360. Replaces #2510 (closed; that branch began as a stacked PR and the commit history got noisy once the base merged). Same code, single clean commit against `main`. ## Summary Add `backgroundChart` (a required `type` of `line` or `area`, plus an optional palette-token `color`) to the builder number-tile config in the MCP save and patch tools, reusing `BackgroundChartSchema` from `common-utils`. The MCP tools already share the external-api conversion, so this is schema, prompt, and tests only, no conversion edits. Raw SQL number tiles do not expose it (no time dimension to bucket), matching the editor and the REST API. ## What - Add `backgroundChart` to `mcpNumberTileSchema.config` with a tool description; the patch tool inherits it via `mcpPatchTileSchema`. - Add a "NUMBER TILE BACKGROUND CHART" section to the dashboard authoring prompt with a worked builder example. - Tests: round-trip through save/get and through patch/get; reject a `type` outside `line`/`area`; drop the field on a raw SQL number tile. ## Test plan - [x] `yarn workspace @hyperdx/api lint` - [x] `yarn workspace @hyperdx/api tsc --noEmit` - [x] integration: `mcp/__tests__/dashboards/{saveDashboard,patchDashboard}.test.ts` (75 passed) CI re-runs lint, typecheck, and the test suite against the current `main` on this branch. ### What's NOT in this PR (follow-up) - Customer docs for the sparkline option (tracked separately). --- .../mcp-number-tile-background-chart.md | 5 ++ .../dashboards/patchDashboard.test.ts | 54 ++++++++++++ .../dashboards/saveDashboard.test.ts | 85 +++++++++++++++++++ .../api/src/mcp/prompts/dashboards/content.ts | 17 ++++ .../api/src/mcp/tools/dashboards/schemas.ts | 13 +++ 5 files changed, 174 insertions(+) create mode 100644 .changeset/mcp-number-tile-background-chart.md diff --git a/.changeset/mcp-number-tile-background-chart.md b/.changeset/mcp-number-tile-background-chart.md new file mode 100644 index 0000000000..e14f2f71aa --- /dev/null +++ b/.changeset/mcp-number-tile-background-chart.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +Add `backgroundChart` support to number tiles in the MCP dashboard tools (`clickstack_save_dashboard` and `clickstack_patch_dashboard`). Builder number tiles can now carry an optional background trend sparkline (`type` line or area, with an optional palette-token `color`), matching the dashboard editor and the v2 REST API. Raw SQL number tiles do not support it. diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts index fe6cb93c68..f318568c07 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.test.ts @@ -651,6 +651,60 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { ]); }); + it('should round-trip backgroundChart: patch then get_dashboard_tile', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Background Chart Patch Test', + tiles: [ + { + name: 'Original', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }, + ); + const created = JSON.parse(getFirstText(createResult)); + const tileId = created.tiles[0].id; + + await callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId: created.id, + tileId, + tile: { + name: 'Patched', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'avg', valueExpression: 'Duration' }], + backgroundChart: { type: 'line', color: 'chart-blue' }, + }, + }, + }); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { + dashboardId: created.id, + tileId, + }, + ); + + expect(getResult.isError).toBeFalsy(); + const tile = JSON.parse(getFirstText(getResult)); + expect(tile.config.displayType).toBe('number'); + expect(tile.config.backgroundChart).toEqual({ + type: 'line', + color: 'chart-blue', + }); + }); + describe('raw SQL macro warnings', () => { // Patching a tile to a macro-less raw SQL config succeeds (non-blocking) // but surfaces an advisory `warnings` array on the response. diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts index 0b51e7e2cf..8929ef1889 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.test.ts @@ -833,6 +833,91 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { expect(saved.tiles[0].config.colorRules).toBeUndefined(); }); + it('should round-trip number-tile backgroundChart through save and get', async () => { + const sourceId = ctx.traceSource._id.toString(); + const numberConfig = { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + color: 'chart-green', + backgroundChart: { type: 'area', color: 'chart-blue' }, + }; + + const saveResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Number Background Chart', + tiles: [{ name: 'Number', config: numberConfig }], + }, + ); + expect(saveResult.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(saveResult)); + expect(saved.tiles[0].config).toMatchObject(numberConfig); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard', + { + id: saved.id, + }, + ); + expect(getResult.isError).toBeFalsy(); + const fetched = JSON.parse(getFirstText(getResult)); + expect(fetched.tiles[0].config).toMatchObject(numberConfig); + }); + + it('should reject a number-tile backgroundChart type outside line/area', async () => { + const sourceId = ctx.traceSource._id.toString(); + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Bad BackgroundChart', + tiles: [ + { + name: 'Number', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + backgroundChart: { type: 'bar' }, + }, + }, + ], + }); + + expect(result.isError).toBe(true); + }); + + it('should drop backgroundChart on a raw SQL number tile', async () => { + // The raw SQL number tile schema has no backgroundChart (builder-only; + // a raw SQL tile returns a single value with no time dimension to + // bucket), so a sent value is stripped on save. + const connectionId = ctx.connection._id.toString(); + const saveResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'SQL Number Background Chart', + tiles: [ + { + name: 'SLO', + config: { + configType: 'sql', + displayType: 'number', + connectionId, + sqlTemplate: 'SELECT 0.99 AS value LIMIT 1', + color: 'chart-success', + backgroundChart: { type: 'line' }, + }, + }, + ], + }, + ); + expect(saveResult.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(saveResult)); + expect(saved.tiles[0].config.color).toBe('chart-success'); + expect(saved.tiles[0].config.backgroundChart).toBeUndefined(); + }); + it('should preserve display fields on builder tiles through save, get, update, and re-get', async () => { const sourceId = ctx.traceSource._id.toString(); const numberFormat = { diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 9e72381d9b..1ae7d62323 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -1028,6 +1028,23 @@ Colors are palette tokens, not hex. Order rules from least to most severe so the colorRules is for number builder tiles only. Raw SQL number tiles (configType: "sql") support color but not colorRules. +== NUMBER TILE BACKGROUND CHART == + +number tiles can show a faint background trend sparkline behind the value, derived from a time-bucketed version of the same query. Use it for SLO / error-budget tiles where the trend over the window matters as much as the current value. One field on the tile config: + + backgroundChart { type, color? }. type is "line" or "area". color is an optional palette token override; when unset the sparkline inherits the tile color. + +Example: an availability tile that shows the current value over a faint area trend: + config: { + displayType: "number", + sourceId: "...", + select: [{ aggFn: "avg", valueExpression: "Success", numberFormat: { output: "percent" } }], + color: "chart-green", + backgroundChart: { type: "area" } + } + +backgroundChart is for number builder tiles only. Raw SQL number tiles (configType: "sql") return a single value with no time dimension to bucket, so they do not support it. + == asRatio == Set asRatio: true on line / stacked_bar / table tiles with exactly 2 select items diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index d90e759af4..54ecd6d867 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -4,6 +4,7 @@ // readability. Reformatting all separators is out of scope here. import { AggregateFunctionSchema, + BackgroundChartSchema, ChartPaletteTokenSchema, DASHBOARD_CONTAINER_ID_MAX, DASHBOARD_MAX_CONTAINERS, @@ -60,6 +61,15 @@ const rawSqlNumberTileColorDescription = '"number", ignored otherwise. Raw SQL number tiles do not support ' + 'conditional colorRules.'; +const numberTileBackgroundChartDescription = + 'Optional background trend sparkline drawn behind the number, derived ' + + 'from a time-bucketed version of the same query (useful for SLO / ' + + 'error-budget tiles where the trend over the window matters). ' + + '{ type, color? }: type is "line" or "area"; color is an optional ' + + 'palette token override (the sparkline inherits the tile color when ' + + 'unset). Builder number tiles only; raw SQL number tiles have no time ' + + 'dimension to bucket. Example: { type: "area", color: "chart-blue" }.'; + const mcpNumberFormatSchema = z.object({ output: z .enum([ @@ -597,6 +607,9 @@ const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ .max(10) .optional() .describe(numberTileColorRulesDescription), + backgroundChart: BackgroundChartSchema.optional().describe( + numberTileBackgroundChartDescription, + ), }), }); From b53c03723b7134c6787513a994d56eadc50bf753 Mon Sep 17 00:00:00 2001 From: Elizabet Oliveira Date: Mon, 6 Jul 2026 14:32:25 +0100 Subject: [PATCH 151/728] Polish dashboard tile styling and header layout (#2592) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary A visual-polish pass on dashboards to make tiles read as distinct cards and give groups a cleaner, more consistent layout. ### Tiles - **Card styling**: tiles use a `bg-body` (white in light mode) background with a border, sitting on a recessed "sunken" page background so they read as elevated cards. - **Compact, consistent header**: every tile has a slim header strip with a full-bleed separator (titled or not), reducing empty space and preventing actions from overlapping content. - **Right-aligned kebab menu**: tile actions (duplicate, view fullscreen, edit, delete, move-to-group) are consolidated into a `⋮` menu rendered as a toolbar *suffix*, so it sits at the far right after each chart's own controls (display switcher, granularity). The alert affordance stays alongside it and is always visible. - **Alert affordance**: uses `IconBellPlus` when no alert is configured (reads clearly on the light header) and `IconBell` + a colored status indicator when an alert exists. - **Non-muted table tiles**: dashboard table tiles use the default (white) table variant so they blend into the card instead of showing a muted panel. - **Modern resize handle**: replaced the default corner bracket with a subtle, hover-revealed dotted triangular grip. ### Groups - **Always-visible controls**: the drag handle, add-tile, and `⋮` overflow controls are always visible in the group header (no longer hover-only). - **Header / body split**: group header uses `bg-body`; the tile area below uses the sunken background with padding around the nested tiles. - **Removed redundant tab underline**: the group's Mantine tab list no longer draws its default `::before` underline, since the group border + header/body color split already separate the tab row from the content (the active-tab indicator is preserved). ### Theming - Renamed `--color-bg-muted-subtle` → `--color-bg-sunken` (and `.bg-muted-subtle` → `.bg-sunken`) with explanatory comments, since the token is a recessed backdrop that sits *below* `bg-body` cards rather than a "subtler muted". ### Screenshots Before Browser-RUM-–-HyperDX-07-06-2026_11_22_AM (1) Screenshot 2026-07-06 at 12 35 16 After Browser-RUM-–-HyperDX-07-06-2026_11_22_AM Screenshot 2026-07-06 at 12 34 26 ### Tests - Updated dashboard E2E specs/page objects so tile edit/duplicate/delete are exercised through the kebab menu. ## Test plan - [ ] Tiles render as bordered `bg-body` cards on a sunken page background (light + dark). - [ ] Tile header is compact and consistent for titled and untitled tiles; separator reaches both card edges. - [ ] Kebab menu sits at the far right after chart controls; alert bell + all actions (edit/duplicate/delete/fullscreen/move) work. - [ ] Alert icon shows `bell +` with no alert and a status dot when an alert exists; both visible on light/dark headers. - [ ] Table tiles no longer show a muted background. - [ ] Group controls (drag/add/kebab) are always visible; header is `bg-body`, tile area is sunken with padding; no underline under the group tabs (active-tab indicator still shows). - [ ] Resize handle grip appears on hover and resizing still works. - [ ] E2E: `make dev-e2e FILE=dashboard` --- .changeset/dashboard-tile-visual-polish.md | 9 + packages/app/src/DBDashboardPage.tsx | 312 ++++++++++-------- .../app/src/components/DashboardContainer.tsx | 18 +- packages/app/src/components/GroupTabBar.tsx | 2 +- .../src/components/charts/ChartContainer.tsx | 139 +++++--- .../src/theme/themes/clickstack/_tokens.scss | 18 + .../app/src/theme/themes/hyperdx/_tokens.scss | 10 + packages/app/styles/_utilities.scss | 7 + packages/app/styles/app.scss | 53 ++- .../dashboard-external-api-config.spec.ts | 11 +- .../dashboard-external-api-series.spec.ts | 11 +- .../app/tests/e2e/features/dashboard.spec.ts | 37 ++- .../tests/e2e/page-objects/DashboardPage.ts | 19 +- 13 files changed, 417 insertions(+), 229 deletions(-) create mode 100644 .changeset/dashboard-tile-visual-polish.md diff --git a/.changeset/dashboard-tile-visual-polish.md b/.changeset/dashboard-tile-visual-polish.md new file mode 100644 index 0000000000..5e9cba5fd8 --- /dev/null +++ b/.changeset/dashboard-tile-visual-polish.md @@ -0,0 +1,9 @@ +--- +"@hyperdx/app": patch +--- + +Polish dashboard tiles: white (surface) card backgrounds with a border, a subtle +muted page background, a modern dotted resize handle, and a compact, consistent +tile header. Tile actions are consolidated into a right-aligned kebab menu (with +the alert bell) that sits after each chart's own controls, and a full-bleed +separator gives every tile a consistent header strip. diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 75d10d1e7f..1ed9cf72ad 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -76,6 +76,7 @@ import { IconAlertTriangle, IconArrowsMaximize, IconBell, + IconBellPlus, IconChartBar, IconChevronsDown, IconChevronsUp, @@ -133,7 +134,10 @@ import useDashboardContainers, { } from '@/hooks/useDashboardContainers'; import { calculateNextTilePosition, makeId } from '@/utils/tilePositioning'; -import ChartContainer from './components/charts/ChartContainer'; +import ChartContainer, { + ChartContainerCardHeaderProvider, + DASHBOARD_TILE_PADDING_INLINE, +} from './components/charts/ChartContainer'; import DBHeatmapChart, { toHeatmapChartConfig, } from './components/DBHeatmapChart'; @@ -188,7 +192,8 @@ function HeatmapTile({ keyPrefix, chartId, title, - toolbar, + toolbarPrefix, + toolbarSuffix, queriedConfig, source, dateRange, @@ -197,7 +202,8 @@ function HeatmapTile({ keyPrefix: string; chartId: string; title: React.ReactNode; - toolbar: React.ReactNode[]; + toolbarPrefix: React.ReactNode[]; + toolbarSuffix: React.ReactNode[]; queriedConfig: BuilderChartConfigWithDateRange; source: TSource | undefined; dateRange: [Date, Date]; @@ -243,7 +249,8 @@ function HeatmapTile({ 0; return ( e.stopPropagation()} key="hover-toolbar" - my={4} // Margin to ensure that the Alert Indicator doesn't clip on non-Line/Bar display types - style={{ visibility: hovered ? 'visible' : 'hidden' }} + my={2} // Margin to ensure that the Alert Indicator doesn't clip on non-Line/Bar display types > - {displayTypeSupportsAlerts && ( - +} - mr={4} - > + {displayTypeSupportsAlerts && + (alert ? ( + // Existing alert: bell with a colored status dot indicator. + + + + + + + + ) : ( + // No alert yet: a dedicated "bell +" icon reads clearly on any + // background, unlike an overlaid indicator badge. - + - - )} + ))} - - - - { - e.stopPropagation(); - openFullscreen(); - }} - title="View Fullscreen (f)" - > - - - - - - {onMoveToGroup && moveTargets && moveTargets.length > 0 && ( - - - - - - - - - - Move to Group - {chart.containerId && ( - onMoveToGroup(undefined)}> - (Ungrouped) - - )} - {moveTargets - .filter( - t => - !( - t.containerId === chart.containerId && - t.tabId === chart.tabId - ), - ) - .map(t => ( + + + + + + + + + e.stopPropagation()}> + } + onClick={onDuplicateClick} + > + Duplicate + + } + onClick={() => openFullscreen()} + > + View fullscreen + + } + onClick={onEditClick} + > + Edit + + {canMoveToGroup && ( + <> + + Move to Group + {chart.containerId && ( onMoveToGroup(t.containerId, t.tabId)} + leftSection={} + onClick={() => onMoveToGroup(undefined)} > - {t.allTabs ? ( - - {t.allTabs.map((tab, i) => ( - - {i > 0 && ( + (Ungrouped) + + )} + {moveTargets + .filter( + t => + !( + t.containerId === chart.containerId && + t.tabId === chart.tabId + ), + ) + .map(t => ( + } + onClick={() => onMoveToGroup(t.containerId, t.tabId)} + > + {t.allTabs ? ( + + {t.allTabs.map((tab, i) => ( + + {i > 0 && ( + + {' | '} + + )} - {' | '} + {tab.title} - )} - - {tab.title} - - ))} - - ) : ( - t.label - )} - - ))} - - - )} - - - + ))} + + ) : ( + t.label + )} + + ))} + + )} + + } + onClick={onDeleteClick} + > + Delete + + + ); }, [ @@ -809,7 +831,6 @@ const Tile = forwardRef( chart.id, chart.containerId, chart.tabId, - hovered, onDeleteClick, onDuplicateClick, onEditClick, @@ -818,20 +839,24 @@ const Tile = forwardRef( ]); const title = useMemo( - () => ( - - {chart.config.name} - - ), + () => + chart.config.name ? ( + {chart.config.name} + ) : undefined, [chart.config.name], ); // Render chart content (used in both tile and fullscreen views) const renderChartContent = useCallback( (hideToolbar: boolean = false, isFullscreenView: boolean = false) => { - const toolbar = hideToolbar - ? [filterWarning] - : [hoverToolbar, filterWarning]; + // Tile-level actions (alert bell + kebab) render as a suffix so they + // sit to the right of each chart's own controls (display switcher, + // granularity, etc.), keeping the kebab at the far right edge. + const toolbarPrefixItems = [filterWarning]; + const toolbarSuffixItems = hideToolbar ? [] : [hoverToolbar]; + // Combined + ordered for containers that only accept `toolbarItems` + // (they have no chart-specific controls to interleave). + const toolbar = [...toolbarPrefixItems, ...toolbarSuffixItems]; const keyPrefix = isFullscreenView ? 'fullscreen' : 'tile'; // The fullscreen view is always visible, so it should always load. @@ -893,7 +918,8 @@ const Tile = forwardRef( )} {effectiveQueriedConfig?.displayType === DisplayType.Table && ( - + @@ -941,7 +968,8 @@ const Tile = forwardRef( @@ -950,7 +978,8 @@ const Tile = forwardRef( @@ -961,7 +990,8 @@ const Tile = forwardRef( keyPrefix={keyPrefix} chartId={chart.id} title={title} - toolbar={toolbar} + toolbarPrefix={toolbarPrefixItems} + toolbarSuffix={toolbarSuffixItems} enabled={chartEnabled} queriedConfig={effectiveQueriedConfig} source={source} @@ -1013,7 +1043,7 @@ const Tile = forwardRef( }} isLive={false} queryKeyPrefix={'search'} - variant="muted" + variant="default" errorVariant="collapsible" /> @@ -1045,7 +1075,9 @@ const Tile = forwardRef( <>
e.stopPropagation()} > - {renderChartContent()} + + {renderChartContent()} +
{children}
@@ -2827,6 +2862,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { } padded + contentClassName="bg-sunken" content={dashboardBody} /> ); diff --git a/packages/app/src/components/DashboardContainer.tsx b/packages/app/src/components/DashboardContainer.tsx index 380afd6a65..73f39544ae 100644 --- a/packages/app/src/components/DashboardContainer.tsx +++ b/packages/app/src/components/DashboardContainer.tsx @@ -138,8 +138,6 @@ export default function DashboardContainer({ @@ -154,8 +152,6 @@ export default function DashboardContainer({ @@ -211,8 +207,6 @@ export default function DashboardContainer({ style={{ cursor: 'grab', flexShrink: 0, - opacity: showControls ? 1 : 0, - transition: 'opacity 150ms', }} data-testid={`group-drag-handle-${container.id}`} > @@ -247,6 +241,7 @@ export default function DashboardContainer({ onMouseLeave={() => setHovered(false)} mt={8} style={{ + backgroundColor: 'var(--color-bg-body)', border: bordered ? '1px solid var(--color-border)' : undefined, borderRadius: bordered ? 4 : undefined, }} @@ -364,7 +359,16 @@ export default function DashboardContainer({ )} {!isCollapsed && ( - {children(hasTabs ? resolvedActiveTabId : undefined)} + + {children(hasTabs ? resolvedActiveTabId : undefined)} + )} - + {tabs.map(tab => ( + {children} + + ); +} + +// Horizontal padding a dashboard tile applies to its content. The card header +// bleeds its separator to the tile edge by cancelling exactly this inset, so +// the tile must consume the same value (see DASHBOARD_TILE_PADDING_INLINE usage +// in DBDashboardPage) instead of a matching `.px-2` utility that could drift. +export const DASHBOARD_TILE_PADDING_INLINE = + 'calc(var(--mantine-spacing-md) * 0.5)'; +const HEADER_SPACING = DASHBOARD_TILE_PADDING_INLINE; +const HEADER_SPACING_SLIM = 'calc(var(--mantine-spacing-md) * 0.25)'; + function ChartContainer({ title, toolbarItems, children, disableReactiveContainer, }: ChartContainerProps) { + const cardHeader = use(ChartContainerCardHeaderContext); + const hasToolbar = !!toolbarItems?.length; + const showHeader = !!title || hasToolbar; + return ( - - {(!!title || !!toolbarItems?.length) && ( - - + + {showHeader && ( + spacing-md * 0.5) so the separator reaches the + // card edges, then re-inset the content. + marginInline: `calc(${HEADER_SPACING} * -1)`, + paddingInline: HEADER_SPACING, + // Keep the header strip compact: slim vertical padding for + // both titled and title-less tiles, plus a full-bleed + // separator so every tile has a consistent header. + paddingTop: HEADER_SPACING_SLIM, + paddingBottom: HEADER_SPACING_SLIM, + borderBottom: '1px solid var(--color-border)', + } + : undefined + } > - {title} - - {toolbarItems && ( - - {toolbarItems} - - )} - - )} - {disableReactiveContainer ? ( - children - ) : ( -
+ + {title} + + {toolbarItems && ( + + {toolbarItems} + + )} + + )} + {disableReactiveContainer ? ( + children + ) : (
- {children} +
+ {children} +
-
- )} - + )} + + ); } diff --git a/packages/app/src/theme/themes/clickstack/_tokens.scss b/packages/app/src/theme/themes/clickstack/_tokens.scss index 72b36cca71..915aae14f1 100644 --- a/packages/app/src/theme/themes/clickstack/_tokens.scss +++ b/packages/app/src/theme/themes/clickstack/_tokens.scss @@ -110,6 +110,15 @@ --color-bg-surface: var(--click-global-color-background-default); --color-bg-inverted: var(--mantine-color-dark-1); --color-bg-muted: var(--click-global-color-background-muted); + + /* "Sunken" = a recessed backdrop that sits *below* raised content. Use it for + the area behind `--color-bg-body` cards (e.g. dashboard tiles) so the cards + visually pop. It is intentionally darker/greyer than the body itself. */ + --color-bg-sunken: color-mix( + in srgb, + var(--click-global-color-background-default), + #000 20% + ); --color-bg-highlighted: rgb(80 80 80 / 70%); --color-bg-sidenav: var(--click-global-color-background-default); --color-bg-sidenav-link-active: var(--palette-neutral-712); @@ -302,6 +311,15 @@ --color-bg-surface: var(--click-global-color-background-default); --color-bg-inverted: var(--mantine-color-dark-3); --color-bg-muted: var(--click-global-color-background-muted); + + /* "Sunken" = a recessed backdrop that sits *below* raised content. Use it for + the area behind `--color-bg-body` cards (e.g. dashboard tiles) so the cards + visually pop. It is intentionally slightly darker than the body itself. */ + --color-bg-sunken: color-mix( + in srgb, + var(--click-global-color-background-muted) 55%, + #fff + ); --color-bg-highlighted: rgb(241 243 245 / 80%); --color-bg-sidenav: var(--click-global-color-background-default); --color-bg-sidenav-link-active: var(--click-global-color-stroke-default); diff --git a/packages/app/src/theme/themes/hyperdx/_tokens.scss b/packages/app/src/theme/themes/hyperdx/_tokens.scss index 3ec5280fe0..595181f6e5 100644 --- a/packages/app/src/theme/themes/hyperdx/_tokens.scss +++ b/packages/app/src/theme/themes/hyperdx/_tokens.scss @@ -29,6 +29,11 @@ --color-bg-surface: var(--mantine-color-dark-5); --color-bg-inverted: var(--mantine-color-dark-1); --color-bg-muted: var(--mantine-color-dark-7); + + /* "Sunken" = a recessed backdrop that sits *below* raised content. Use it for + the area behind `--color-bg-body` cards (e.g. dashboard tiles) so the cards + visually pop. It is intentionally darker/greyer than the body itself. */ + --color-bg-sunken: color-mix(in srgb, var(--mantine-color-dark-9), #000 50%); --color-bg-highlighted: rgb(55 58 64 / 70%); --color-bg-sidenav: var(--mantine-color-dark-9); --color-bg-sidenav-link-active: var(--mantine-color-dark-7); @@ -131,6 +136,11 @@ --color-bg-surface: var(--mantine-color-white); --color-bg-inverted: var(--mantine-color-dark-3); --color-bg-muted: #f6f6fa; + + /* "Sunken" = a recessed backdrop that sits *below* raised content. Use it for + the area behind `--color-bg-body` cards (e.g. dashboard tiles) so the cards + visually pop. It is intentionally slightly darker than the body itself. */ + --color-bg-sunken: #fafafc; --color-bg-highlighted: rgb(230 232 237 / 70%); --color-bg-sidenav: var(--mantine-color-white); --color-bg-sidenav-link-active: #ebebef; diff --git a/packages/app/styles/_utilities.scss b/packages/app/styles/_utilities.scss index df0d4d5f99..2b7a743724 100644 --- a/packages/app/styles/_utilities.scss +++ b/packages/app/styles/_utilities.scss @@ -15,6 +15,13 @@ background-color: var(--color-bg-muted); } +// "Sunken" = a recessed backdrop that sits *below* raised content. Use it for +// the area behind `.bg-body` cards (e.g. dashboard tiles / group bodies) so the +// cards visually pop. +.bg-sunken { + background-color: var(--color-bg-sunken, var(--color-bg-muted)); +} + .bg-muted-hover { &:hover { background-color: var(--color-bg-muted); diff --git a/packages/app/styles/app.scss b/packages/app/styles/app.scss index 73c494f03e..6a1c5f89f3 100644 --- a/packages/app/styles/app.scss +++ b/packages/app/styles/app.scss @@ -209,14 +209,51 @@ button:focus-visible { opacity: 0.4 !important; } -.react-grid-item > .react-resizable-handle::after { - border-color: var(--color-text) !important; - right: -4px; - bottom: -4px; - width: 4px; - height: 4px; - border-right: 2px solid var(--color-text) !important; - border-bottom: 2px solid var(--color-text) !important; +.react-grid-item > .react-resizable-handle { + padding: 0; + // Drop react-resizable's default corner glyph (a low-opacity SVG angle it + // sets via background-image); we render our own grip below instead. + background-image: none; + + // Modern dotted grip: a triangular cluster of dots tucked into the corner. + &::after { + content: ''; + position: absolute; + right: 3px; + bottom: 3px; + width: 12px; + height: 12px; + border: none !important; + background-image: radial-gradient( + circle, + var(--color-text-muted) 40%, + transparent 45% + ); + background-size: 4px 4px; + background-position: bottom right; + + // Clip the dot grid into a lower-right triangle for a classic grip look. + clip-path: polygon(100% 0, 100% 100%, 0 100%); + opacity: 0; + transition: opacity 0.15s ease; + } +} + +.react-grid-item:hover > .react-resizable-handle::after { + opacity: 0.65; +} + +.react-grid-item > .react-resizable-handle:hover::after { + opacity: 1; +} + +// Mantine's default Tabs variant draws a full-width underline beneath the tab +// list via a ::before pseudo-element. Dashboard groups already have their own +// border plus a white-header / muted-body split, so this line is redundant. +// Scoped to the list's ::before so the active-tab indicator (a separate +// element that reuses --tab-border-color) is left intact. +.dashboard-group-tabs-list::before { + display: none !important; } // custom overrides for jsontree diff --git a/packages/app/tests/e2e/features/dashboard-external-api-config.spec.ts b/packages/app/tests/e2e/features/dashboard-external-api-config.spec.ts index 4b8ac8fd28..9cd14beb15 100644 --- a/packages/app/tests/e2e/features/dashboard-external-api-config.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-external-api-config.spec.ts @@ -181,15 +181,8 @@ test.describe( for (let i = 0; i < dashboardPayload.tiles.length; i++) { const tileData = dashboardPayload.tiles[i]; - // Hover over tile to reveal edit button - await tiles.nth(i).hover(); - - // Click edit button for this tile - const editButton = page - .locator(`[data-testid^="tile-edit-button-"]`) - .nth(i); - await expect(editButton).toBeVisible(); - await editButton.click(); + // Open the tile's actions menu and click Edit + await dashboardPage.editTile(i); // Wait for chart editor modal to open const chartNameInput = page.getByTestId('chart-name-input'); diff --git a/packages/app/tests/e2e/features/dashboard-external-api-series.spec.ts b/packages/app/tests/e2e/features/dashboard-external-api-series.spec.ts index 0a49dbce6a..f1412d7410 100644 --- a/packages/app/tests/e2e/features/dashboard-external-api-series.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-external-api-series.spec.ts @@ -188,15 +188,8 @@ test.describe( const tileData = tilesData[i]; const series = tileData.series; - // Hover over tile to reveal edit button - await tiles.nth(i).hover(); - - // Click edit button for this tile - const editButton = page - .locator(`[data-testid^="tile-edit-button-"]`) - .nth(i); - await expect(editButton).toBeVisible(); - await editButton.click(); + // Open the tile's actions menu and click Edit + await dashboardPage.editTile(i); // Wait for chart editor modal to open const chartNameInput = page.getByTestId('chart-name-input'); diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 34eb579ca8..1dfb96ea28 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -159,20 +159,23 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { const dashboardTiles = dashboardPage.getTiles(); await expect(dashboardTiles).toHaveCount(2, { timeout: 10000 }); - // Hover over first tile to reveal action buttons + // The alert affordance lives directly in the always-visible tile header. await dashboardPage.hoverOverTile(0); + await expect(dashboardPage.getTileButton('alerts')).toBeVisible(); - // Verify all action buttons are visible - const buttons: Array<'edit' | 'duplicate' | 'delete' | 'alerts'> = [ + // Edit / duplicate / delete now live inside the tile actions (kebab) menu. + await dashboardPage.openTileActionsMenu(0); + const menuButtons: Array<'edit' | 'duplicate' | 'delete'> = [ 'edit', 'duplicate', 'delete', - 'alerts', ]; - for (const button of buttons) { - const buttonLocator = dashboardPage.getTileButton(button); - await expect(buttonLocator).toBeVisible(); + for (const button of menuButtons) { + await expect(dashboardPage.getTileButton(button)).toBeVisible(); } + + // Close the menu so it doesn't intercept subsequent interactions. + await dashboardPage.page.keyboard.press('Escape'); }); await test.step('Test duplicate tile', async () => { @@ -338,20 +341,23 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { const dashboardTiles = dashboardPage.getTiles(); await expect(dashboardTiles).toHaveCount(1, { timeout: 10000 }); - // Hover over first tile to reveal action buttons + // The alert affordance lives directly in the always-visible tile header. await dashboardPage.hoverOverTile(0); + await expect(dashboardPage.getTileButton('alerts')).toBeVisible(); - // Verify all action buttons are visible - const buttons: Array<'edit' | 'duplicate' | 'delete' | 'alerts'> = [ + // Edit / duplicate / delete now live inside the tile actions (kebab) menu. + await dashboardPage.openTileActionsMenu(0); + const menuButtons: Array<'edit' | 'duplicate' | 'delete'> = [ 'edit', 'duplicate', 'delete', - 'alerts', ]; - for (const button of buttons) { - const buttonLocator = dashboardPage.getTileButton(button); - await expect(buttonLocator).toBeVisible(); + for (const button of menuButtons) { + await expect(dashboardPage.getTileButton(button)).toBeVisible(); } + + // Close the menu so it doesn't intercept subsequent interactions. + await dashboardPage.page.keyboard.press('Escape'); }); let dashboardUrl: string; @@ -991,7 +997,8 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { }); await test.step('Verify tile can be edited when source is missing', async () => { - await dashboardPage.hoverOverTile(0); + // Edit now lives inside the tile actions (kebab) menu. + await dashboardPage.openTileActionsMenu(0); const editButton = dashboardPage.getTileButton('edit'); await expect(editButton).toBeVisible(); diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index 95c8296efb..192fa644dc 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -588,11 +588,22 @@ export class DashboardPage { return this.page.locator(`[data-testid^="tile-${action}-button-"]`).first(); } + /** + * Open a tile's actions (kebab) menu, revealing the Duplicate / View + * fullscreen / Edit / Delete items (which now live inside the menu). + */ + async openTileActionsMenu(tileIndex: number) { + await this.page + .locator('[data-testid^="tile-actions-button-"]') + .nth(tileIndex) + .click(); + } + /** * Edit a tile */ async editTile(tileIndex: number) { - await this.hoverOverTile(tileIndex); + await this.openTileActionsMenu(tileIndex); await this.getTileButton('edit').click(); } @@ -600,7 +611,7 @@ export class DashboardPage { * Duplicate a tile */ async duplicateTile(tileIndex: number) { - await this.hoverOverTile(tileIndex); + await this.openTileActionsMenu(tileIndex); await this.getTileButton('duplicate').click(); const confirmButton = this.page.locator( @@ -613,7 +624,7 @@ export class DashboardPage { * Delete a tile */ async deleteTile(tileIndex: number) { - await this.hoverOverTile(tileIndex); + await this.openTileActionsMenu(tileIndex); await this.getTileButton('delete').click(); const confirmButton = this.page.locator( @@ -1267,7 +1278,7 @@ export class DashboardPage { * Waits for the fullscreen modal's TimePicker to appear before returning. */ async openFullscreenForTile(index: number) { - await this.hoverOverTile(index); + await this.openTileActionsMenu(index); const fullscreenBtn = this.page .locator('[data-testid^="tile-fullscreen-button-"]') .first(); From 707e646668a90930d6336846d22979b5fd4bacb0 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 6 Jul 2026 09:43:12 -0400 Subject: [PATCH 152/728] fix(table-chart): wrap mode now breaks long URLs/IDs instead of overflowing into adjacent columns (#2234) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Re-add `.text-break`, `.align-top`, `.align-middle`, `.align-bottom` utility classes to `_bootstrap-utilities.scss`. These were referenced by `HDXMultiSeriesTableChart` and `DBRowTable` but dropped during the Bootstrap removal refactor (#1347), leaving wrap mode silently broken. ## Why With `.text-break` undefined, toggling wrap mode on a Table chart cell containing a long URL/identifier without whitespace leaves the cell with only `overflow: hidden` and default `word-break: normal`. The browser can't break the string, partial fragments leak across cell boundaries during fallback rendering, and the visible result is text appearing to overlap into adjacent columns. Reproducible with many narrow columns + URLs like `login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=…`. ## Test plan - [ ] Build a Table chart over data with long unbreakable strings (URLs, trace IDs, hashes) - [ ] Toggle the wrap-lines icon — long values should break onto multiple lines within their column rather than fragmenting across columns - [ ] Toggle wrap-lines off — truncate-with-ellipsis behavior unchanged - [ ] Existing `DBTableChart.test.tsx` still passes - [ ] Before: image After: image Ref: HDX-4180 --- .changeset/cool-bikes-suffer.md | 5 +++++ packages/app/styles/_bootstrap-utilities.scss | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 .changeset/cool-bikes-suffer.md diff --git a/.changeset/cool-bikes-suffer.md b/.changeset/cool-bikes-suffer.md new file mode 100644 index 0000000000..07cf1f4359 --- /dev/null +++ b/.changeset/cool-bikes-suffer.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix(table-chart): wrap mode now breaks long URLs/IDs instead of overflowing into adjacent columns diff --git a/packages/app/styles/_bootstrap-utilities.scss b/packages/app/styles/_bootstrap-utilities.scss index fa97c1583c..6fa6fee8cb 100644 --- a/packages/app/styles/_bootstrap-utilities.scss +++ b/packages/app/styles/_bootstrap-utilities.scss @@ -968,6 +968,26 @@ white-space: nowrap !important; } +// Word break — needed for cells in wrap mode so long URLs/identifiers +// without whitespace can break instead of overflowing into the next column. +.text-break { + word-wrap: break-word !important; + word-break: break-word !important; +} + +// Vertical alignment — applies to table cells (and inline/inline-block). +.align-top { + vertical-align: top !important; +} + +.align-middle { + vertical-align: middle !important; +} + +.align-bottom { + vertical-align: bottom !important; +} + // Font size utilities .fs-1 { font-size: 2.5rem !important; // 40px From cf61a8eaedbe9f6dc1ae9bfb108d038757f77e4f Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 6 Jul 2026 10:46:05 -0400 Subject: [PATCH 153/728] feat: Support plugin as a comparison axis in evals (#2579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR adds support for `plugin` configs in the MCP evals framework, so that claude plugin versions can be A/B tested (including the no-plugin case). This follows the pattern established by model configuration / comparison. ### Screenshots or video Screenshot 2026-07-02 at 9 53 38 AM Screenshot 2026-07-02 at 10 01 56 AM ### How to test on Vercel preview Testing: 1. Clone the [plugin](https://github.com/ClickHouse/clickstack-claude-plugin) 2. Add the plugin to your evals config file ```json "plugins": { "clickstack": { "label": "ClickStack", "dir": "/Users/pulpdrew/Documents/repos/clickstack-claude-plugin/plugin" } }, ``` 4. Run the evals ```sh yarn workspace @hyperdx/hdx-eval dev run error-root-cause --mcp hyperdx-main,hyperdx-mcp-integration --plugin none,clickstack ``` ### References - Linear Issue: Closes HDX-4692 - Related PRs: --- packages/hdx-eval/README.md | 54 +++++- packages/hdx-eval/eval.config.example.json | 10 ++ .../hdx-eval/scripts/viewer/public/app.js | 27 ++- packages/hdx-eval/scripts/viewer/server.js | 33 +++- .../hdx-eval/src/__tests__/aggregate.test.ts | 146 +++++++++++++++- .../hdx-eval/src/__tests__/config.test.ts | 133 +++++++++++++++ packages/hdx-eval/src/__tests__/grade.test.ts | 66 ++++++++ .../hdx-eval/src/__tests__/markdown.test.ts | 48 +++++- .../hdx-eval/src/__tests__/mcpConfig.test.ts | 20 +++ .../src/__tests__/reportsStore.test.ts | 158 ++++++++++++++++++ .../hdx-eval/src/__tests__/runPath.test.ts | 113 +++++++++++++ .../hdx-eval/src/__tests__/runRun.test.ts | 16 ++ .../hdx-eval/src/__tests__/runsStore.test.ts | 63 +++++++ packages/hdx-eval/src/cli.ts | 116 ++++++++++--- packages/hdx-eval/src/grading/grade.ts | 61 +++---- packages/hdx-eval/src/harness/claudeSpawn.ts | 27 ++- packages/hdx-eval/src/harness/runRun.ts | 33 +++- packages/hdx-eval/src/harness/types.ts | 22 +++ packages/hdx-eval/src/hyperdx/config.ts | 68 +++++++- packages/hdx-eval/src/reports/aggregate.ts | 92 +++++++--- packages/hdx-eval/src/reports/markdown.ts | 8 +- packages/hdx-eval/src/reports/store.ts | 84 +++++----- packages/hdx-eval/src/runs/instrument.ts | 36 +--- packages/hdx-eval/src/runs/path.ts | 94 +++++++++-- packages/hdx-eval/src/runs/store.ts | 40 +---- 25 files changed, 1352 insertions(+), 216 deletions(-) create mode 100644 packages/hdx-eval/src/__tests__/config.test.ts create mode 100644 packages/hdx-eval/src/__tests__/reportsStore.test.ts create mode 100644 packages/hdx-eval/src/__tests__/runPath.test.ts create mode 100644 packages/hdx-eval/src/__tests__/runsStore.test.ts diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index 8edd9b5d59..35a4291552 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -161,11 +161,58 @@ anonymize the MCP identity for fair LLM judging. | Section | Required | Purpose | |---|---|---| | `mcps` | yes | MCP registry (see above) | +| `plugins` | no | Claude Code plugin registry (see [Plugins](#plugins)) | | `scenarios` | no | Per-scenario HyperDX Source IDs (populated by `setup-hyperdx`) | | `hyperdxApi` | no | HyperDX API URL + access key (used only by `setup-hyperdx`) | | `clickhouse` | no | ClickHouse connection for `seed`, `drop`, `runs-instrument` commands | | `anchorTime` | no | Fixed "now" anchor (ISO 8601). Auto-generated and saved on first `run`. See [Anchor Time](#anchor-time). | +### Plugins + +You can test a [Claude Code plugin](https://code.claude.com/docs/en/plugins) as +a comparison variable — for example a plugin that installs a hook. Define +plugins under the top-level `plugins` key, then select them with `--plugin`: + +```json +{ + "plugins": { + "myplugin": { + "label": "My Plugin", + "url": "https://example.com/my-plugin.zip" + }, + "dev-plugin": { + "label": "Dev Plugin", + "dir": "/absolute/path/to/my-plugin" + } + } +} +``` + +| Field | Required | Notes | +| ------- | ------------------ | -------------------------------------------------- | +| `label` | yes | Human-readable label for reports/CLI output | +| `url` | one of `url`/`dir` | URL to a plugin `.zip` (loaded via `--plugin-url`) | +| `dir` | one of `url`/`dir` | Local plugin directory (loaded via `--plugin-dir`) | + +Each entry must set **exactly one** of `url` or `dir`. + +`--plugin` works like `--mcp` and `--model`: the variants are the names you pass +(comma-separated), and the literal `none` is the no-plugin variant. When +`--plugin` is omitted the default is the single `none` variant, so behavior is +unchanged. + +- `--plugin myplugin` — run **only** the plugin variant. The plugin doesn't + vary, so the column key is just ``. +- `--plugin none,myplugin` — run **both** the no-plugin baseline and the plugin + variant (column keys `/none` and `/myplugin`), so the report shows + the plugin variant as a challenger with per-metric deltas vs. the no-plugin + baseline. The baseline defaults to the first variant (`none` here); override + with `--baseline`. +- `--plugin none,a,b` — compare several plugins against no-plugin at once. + +Each named plugin is loaded into the isolated agent session via `--plugin-url` +(for `url`) or `--plugin-dir` (for `dir`). + ### Example: HTTP MCP (HyperDX) ```json @@ -260,6 +307,7 @@ dev run --mcp hyperdx # single MCP dev run --mcp hyperdx-main,hyperdx-feature # compare two dev run --mcp all # all enabled MCPs (default) dev run --baseline hyperdx-main # set baseline for delta reports +dev run --plugin none,myplugin # compare no-plugin baseline vs plugin variant dev run --runs 5 # 5 runs per cell dev run --model claude-opus-4-6 dev run --max-turns 20 # default: 15 @@ -322,9 +370,9 @@ Combined score = `0.4 * programmatic + 0.6 * judge - toolErrorPenalty` ### Baseline + Challengers -Reports use a **baseline + challengers** model. One MCP is designated as the -baseline (default: first MCP alphabetically, or set explicitly with -`--baseline`). All other MCPs show deltas relative to it: +Reports use a **baseline + challengers** model. One column is designated as the +baseline (default: the first mcp/model/plugin set, or set explicitly with +`--baseline `). All other columns show deltas relative to it: ``` | Metric | hyperdx-main | hyperdx-feature | Δ (hyperdx-feature) | diff --git a/packages/hdx-eval/eval.config.example.json b/packages/hdx-eval/eval.config.example.json index b71d7a3448..1093f5e29d 100644 --- a/packages/hdx-eval/eval.config.example.json +++ b/packages/hdx-eval/eval.config.example.json @@ -47,6 +47,16 @@ "brandTerms": ["ClickHouse MCP", "clickhouse"] } }, + "plugins": { + "myplugin": { + "label": "My Plugin", + "url": "https://example.com/my-plugin.zip" + }, + "dev-plugin": { + "label": "Dev Plugin (local)", + "dir": "/absolute/path/to/my-plugin" + } + }, "scenarios": { "error-root-cause": { "tracesSourceId": "mongo-objectid", diff --git a/packages/hdx-eval/scripts/viewer/public/app.js b/packages/hdx-eval/scripts/viewer/public/app.js index c57f71ef7e..ee80f2bc01 100644 --- a/packages/hdx-eval/scripts/viewer/public/app.js +++ b/packages/hdx-eval/scripts/viewer/public/app.js @@ -221,16 +221,37 @@ // ----- rendering: sidebar nav ----- + /** Base model name of a cell (plugin-arm cells store `model/plugin` in `model`). */ + function cellModel(c) { + if (!c.model) return null; + return c.plugin ? c.model.split('/')[0] : c.model; + } + /** True when the current batch has more than one distinct model across cells. */ function isMultiModel() { const models = new Set(); - for (const c of state.cells) if (c.model) models.add(c.model); + for (const c of state.cells) { + const m = cellModel(c); + if (m) models.add(m); + } return models.size > 1; } - /** Heading for a cell: matches the columnKeyFor logic in aggregate.ts. */ + /** True when the current batch has more than one distinct plugin arm. */ + function isMultiPlugin() { + const plugins = new Set(); + for (const c of state.cells) plugins.add(c.plugin || 'none'); + return plugins.size > 1; + } + + /** Heading for a cell: matches the columnKeyFor logic in aggregate.ts — + * the MCP name, plus `/` and the varying dimensions joined with `+`. */ function cellHeading(c) { - return c.model && isMultiModel() ? `${c.mcp}/${c.model}` : c.mcp; + const varying = []; + const model = cellModel(c); + if (model && isMultiModel()) varying.push(model); + if (isMultiPlugin()) varying.push(c.plugin || 'none'); + return varying.length > 0 ? `${c.mcp}/${varying.join('+')}` : c.mcp; } function renderNav() { diff --git a/packages/hdx-eval/scripts/viewer/server.js b/packages/hdx-eval/scripts/viewer/server.js index d2e138d925..012a2a7c69 100644 --- a/packages/hdx-eval/scripts/viewer/server.js +++ b/packages/hdx-eval/scripts/viewer/server.js @@ -117,12 +117,38 @@ function listCells(batch) { if (runs.length > 0) out.push({ scenario, mcp, model: null, runs }); } - // New layout: ///.json + // Layouts: + // - ////.json (current; the + // no-plugin arm lives under /none/) + // - ///.json (legacy, pre-plugin-level) for (const entry of entries) { if (!entry.isDirectory()) continue; const modelDir = path.join(mcpDir, entry.name); + // Legacy no-plugin arm: runs directly in the model dir. const runs = collectRuns(modelDir); if (runs.length > 0) out.push({ scenario, mcp, model: entry.name, runs }); + // Plugin arms (incl. `none`): one level deeper. The cell's `model` + // carries the `/` relative path so the run-detail + // route resolves it. + let modelEntries = []; + try { + modelEntries = fs.readdirSync(modelDir, { withFileTypes: true }); + } catch { + modelEntries = []; + } + for (const pe of modelEntries) { + if (!pe.isDirectory()) continue; + const pluginRuns = collectRuns(path.join(modelDir, pe.name)); + if (pluginRuns.length > 0) { + out.push({ + scenario, + mcp, + model: `${entry.name}/${pe.name}`, + plugin: pe.name, + runs: pluginRuns, + }); + } + } } } } @@ -154,9 +180,10 @@ const ROUTES = [ sendJson(res, 200, { batch, summary, cells }); }, ], - // New layout: /runs/:scenario/:mcp/:model/:idx + // New layout: /runs/:scenario/:mcp/:model/:idx — `model` may be a nested + // `/` path for plugin arms, so it captures across slashes. [ - /^\/api\/batches\/([^/]+)\/runs\/([^/]+)\/([^/]+)\/([^/]+)\/(\d+)$/, + /^\/api\/batches\/([^/]+)\/runs\/([^/]+)\/([^/]+)\/(.+)\/(\d+)$/, (m, _q, res) => { const [, batch, scenario, mcp, model, idx] = m.map((x, i) => i === 0 ? x : decodeURIComponent(x), diff --git a/packages/hdx-eval/src/__tests__/aggregate.test.ts b/packages/hdx-eval/src/__tests__/aggregate.test.ts index aedead5a8b..cca6801106 100644 --- a/packages/hdx-eval/src/__tests__/aggregate.test.ts +++ b/packages/hdx-eval/src/__tests__/aggregate.test.ts @@ -1,6 +1,10 @@ import type { GradeRecord } from '@/grading/types'; import type { RunRecord } from '@/harness/types'; -import { buildAggregate, type GradedRunPair } from '@/reports/aggregate'; +import { + buildAggregate, + columnKeyFor, + type GradedRunPair, +} from '@/reports/aggregate'; function pair(args: { scenario: string; @@ -11,13 +15,16 @@ function pair(args: { toolCalls: number; outputTokens: number; durationMs: number; + model?: string; + plugin?: string; }): GradedRunPair { const run: RunRecord = { schemaVersion: 1, runId: `${args.scenario}-${args.mcp}-${args.i}`, scenario: args.scenario, mcp: args.mcp, - model: 'claude-sonnet-4-6', + model: args.model ?? 'claude-sonnet-4-6', + plugin: args.plugin ?? 'none', runIndex: args.i, seed: 42, startedAt: '2026-05-09T07:50:00.000Z', @@ -94,6 +101,66 @@ function pair(args: { return { run, grade }; } +describe('columnKeyFor', () => { + it('is just the mcp name when neither models nor plugins vary', () => { + expect( + columnKeyFor('hyperdx', 'opus', 'none', { + multiModel: false, + multiPlugin: false, + }), + ).toBe('hyperdx'); + // A single shared plugin is also omitted. + expect( + columnKeyFor('hyperdx', 'opus', 'myplugin', { + multiModel: false, + multiPlugin: false, + }), + ).toBe('hyperdx'); + }); + + it('appends the model after "/" when models vary', () => { + expect( + columnKeyFor('hyperdx', 'opus', 'none', { + multiModel: true, + multiPlugin: false, + }), + ).toBe('hyperdx/opus'); + }); + + it('appends the plugin after "/" when plugins vary, rendering the no-plugin arm as "none"', () => { + expect( + columnKeyFor('hyperdx', 'opus', 'myplugin', { + multiModel: false, + multiPlugin: true, + }), + ).toBe('hyperdx/myplugin'); + expect( + columnKeyFor('hyperdx', 'opus', 'none', { + multiModel: false, + multiPlugin: true, + }), + ).toBe('hyperdx/none'); + }); + + it('joins model and plugin with "+" when both vary', () => { + expect( + columnKeyFor('hyperdx', 'opus', 'myplugin', { + multiModel: true, + multiPlugin: true, + }), + ).toBe('hyperdx/opus+myplugin'); + }); + + it('sanitizes model and plugin segments so keys match directory names', () => { + expect( + columnKeyFor('hyperdx', 'claude-sonnet-4.5', 'my.plugin', { + multiModel: true, + multiPlugin: true, + }), + ).toBe('hyperdx/claude-sonnet-4-5+my-plugin'); + }); +}); + describe('buildAggregate', () => { it('groups by scenario and computes per-cell means with baseline delta', () => { const pairs: GradedRunPair[] = [ @@ -186,6 +253,15 @@ describe('buildAggregate', () => { // Delta should be clickhouse (challenger) - hyperdx (baseline) expect(sc.deltas['clickhouse']).toBeDefined(); expect(sc.deltas['hyperdx']).toBeUndefined(); + + // A baseline that matches no column falls back to the first column + // instead of producing all-null deltas. + const fallback = buildAggregate({ + batchDir: '/tmp/x', + pairs, + baseline: 'not-a-column', + }); + expect(fallback.baseline).toBe('clickhouse'); }); it('produces an ordered, multi-scenario summary', () => { @@ -306,4 +382,70 @@ describe('buildAggregate', () => { // Deltas: challenger vs baseline (first alphabetically) expect(sc.deltas['hyperdx/claude-sonnet-4-6']?.combinedScore).toBeDefined(); }); + + it('appends the plugin after "/" when multiple plugin arms are present', () => { + const base = { + scenario: 'error-root-cause', + mcp: 'hyperdx', + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 10, + outputTokens: 3000, + durationMs: 40_000, + }; + const pairs: GradedRunPair[] = [ + pair({ ...base, i: 0, plugin: 'none' }), + pair({ ...base, i: 1, plugin: 'myplugin' }), + ]; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.multiModel).toBe(false); + expect(summary.multiPlugin).toBe(true); + // The no-plugin arm renders as the literal "none". + expect(summary.columnOrder).toEqual(['hyperdx/myplugin', 'hyperdx/none']); + const sc = summary.scenarios[0]; + expect(sc.cells['hyperdx/none']).toBeDefined(); + expect(sc.cells['hyperdx/myplugin']).toBeDefined(); + }); + + it('joins varying model and plugin with "+" when both vary', () => { + const base = { + scenario: 'error-root-cause', + mcp: 'hyperdx', + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 10, + outputTokens: 3000, + durationMs: 40_000, + }; + const pairs: GradedRunPair[] = [ + pair({ ...base, i: 0, model: 'claude-sonnet-4-6', plugin: 'none' }), + pair({ ...base, i: 1, model: 'claude-haiku-4-5', plugin: 'myplugin' }), + ]; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.multiModel).toBe(true); + expect(summary.multiPlugin).toBe(true); + expect(summary.columnOrder).toEqual([ + 'hyperdx/claude-haiku-4-5+myplugin', + 'hyperdx/claude-sonnet-4-6+none', + ]); + }); + + it('omits a single shared plugin from column keys', () => { + const base = { + scenario: 'error-root-cause', + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 10, + outputTokens: 3000, + durationMs: 40_000, + plugin: 'myplugin', + }; + const pairs: GradedRunPair[] = [ + pair({ ...base, mcp: 'hyperdx', i: 0 }), + pair({ ...base, mcp: 'clickhouse', i: 0 }), + ]; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.multiPlugin).toBe(false); + expect(summary.columnOrder).toEqual(['clickhouse', 'hyperdx']); + }); }); diff --git a/packages/hdx-eval/src/__tests__/config.test.ts b/packages/hdx-eval/src/__tests__/config.test.ts new file mode 100644 index 0000000000..c97124195f --- /dev/null +++ b/packages/hdx-eval/src/__tests__/config.test.ts @@ -0,0 +1,133 @@ +import { mkdirSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { + configPluginNames, + type EvalConfig, + getPluginDefinition, + readConfig, +} from '@/hyperdx/config'; + +const VALID_MCP = { + type: 'http', + url: 'http://localhost:30196/mcp', + toolPattern: 'mcp__hyperdx__*', + label: 'HyperDX', +}; + +const baseConfig: EvalConfig = { + mcps: { hyperdx: VALID_MCP as EvalConfig['mcps'][string] }, + plugins: { + urlplugin: { label: 'Url Plugin', url: 'https://example.com/p.zip' }, + dirplugin: { label: 'Dir Plugin', dir: '/abs/path/to/plugin' }, + }, +}; + +describe('readConfig plugin validation', () => { + const tmpRoot = join('/tmp', `hdx-eval-config-test-${Date.now()}`); + + beforeAll(() => { + mkdirSync(tmpRoot, { recursive: true }); + }); + + afterAll(() => { + rmSync(tmpRoot, { recursive: true, force: true }); + }); + + function writeTmpConfig(name: string, config: unknown): string { + const p = join(tmpRoot, `${name}.json`); + writeFileSync(p, JSON.stringify(config, null, 2)); + return p; + } + + it('accepts a config with valid url and dir plugins', () => { + const p = writeTmpConfig('valid', baseConfig); + const config = readConfig(p); + expect(configPluginNames(config).sort()).toEqual([ + 'dirplugin', + 'urlplugin', + ]); + }); + + it('accepts a config with no plugins section', () => { + const p = writeTmpConfig('no-plugins', { mcps: baseConfig.mcps }); + expect(configPluginNames(readConfig(p))).toEqual([]); + }); + + it('rejects a plugin without a label', () => { + const p = writeTmpConfig('no-label', { + mcps: baseConfig.mcps, + plugins: { bad: { url: 'https://example.com/p.zip' } }, + }); + expect(() => readConfig(p)).toThrow(/plugins\.bad\.label/); + }); + + it('rejects a plugin that sets both url and dir', () => { + const p = writeTmpConfig('both', { + mcps: baseConfig.mcps, + plugins: { + bad: { label: 'Bad', url: 'https://example.com/p.zip', dir: '/x' }, + }, + }); + expect(() => readConfig(p)).toThrow(/exactly one of 'url' or 'dir'/); + }); + + it('rejects a plugin that sets neither url nor dir', () => { + const p = writeTmpConfig('neither', { + mcps: baseConfig.mcps, + plugins: { bad: { label: 'Bad' } }, + }); + expect(() => readConfig(p)).toThrow(/exactly one of 'url' or 'dir'/); + }); + + it('rejects a plugin using the reserved name "none"', () => { + const p = writeTmpConfig('reserved-none', { + mcps: baseConfig.mcps, + plugins: { none: { label: 'None', url: 'https://example.com/p.zip' } }, + }); + expect(() => readConfig(p)).toThrow(/reserved name "none"/); + }); + + it('rejects a non-object plugins section', () => { + const p = writeTmpConfig('not-object', { + mcps: baseConfig.mcps, + plugins: 'nope', + }); + expect(() => readConfig(p)).toThrow(/'plugins' must be an object/); + }); +}); + +describe('getPluginDefinition', () => { + it('returns the definition for a known plugin', () => { + expect(getPluginDefinition(baseConfig, 'urlplugin')).toEqual({ + label: 'Url Plugin', + url: 'https://example.com/p.zip', + }); + expect(getPluginDefinition(baseConfig, 'dirplugin')).toEqual({ + label: 'Dir Plugin', + dir: '/abs/path/to/plugin', + }); + }); + + it('throws for an unknown plugin, listing available names', () => { + expect(() => getPluginDefinition(baseConfig, 'missing')).toThrow( + /Plugin "missing" not found.*urlplugin, dirplugin/, + ); + }); + + it('throws when no plugins are defined at all', () => { + expect(() => + getPluginDefinition({ mcps: baseConfig.mcps }, 'missing'), + ).toThrow(/\(none defined\)/); + }); + + it('throws for a definition with both url and dir set', () => { + const config: EvalConfig = { + mcps: baseConfig.mcps, + plugins: { bad: { label: 'Bad', url: 'https://x/p.zip', dir: '/x' } }, + }; + expect(() => getPluginDefinition(config, 'bad')).toThrow( + /exactly one of 'url' or 'dir'/, + ); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/grade.test.ts b/packages/hdx-eval/src/__tests__/grade.test.ts index 54a8198cf7..34a256b1a7 100644 --- a/packages/hdx-eval/src/__tests__/grade.test.ts +++ b/packages/hdx-eval/src/__tests__/grade.test.ts @@ -24,6 +24,7 @@ function buildRun(args: { scenario: args.scenario, mcp: 'hyperdx', model: 'claude-sonnet-4-6', + plugin: 'none', runIndex: 0, seed: 42, startedAt: '2026-05-10T00:00:00.000Z', @@ -166,4 +167,69 @@ describe('gradeBatch tool-error penalty', () => { expect(grade.programmatic.score).toBeCloseTo(1, 5); expect(grade.combinedScore).toBeCloseTo(0.8, 5); }); + + it('grades runs stored in the current /// layout', async () => { + const batchDir = join(tmpRoot, 'plugin-layout'); + const run = buildRun({ + scenario: 'error-root-cause', + toolCalls: [{ name: 'mcp__hyperdx__hyperdx_query', isError: false }], + finalAnswer: ANSWER, + }); + run.plugin = 'myplugin'; + const dir = join( + batchDir, + 'error-root-cause', + 'hyperdx', + 'claude-sonnet-4-6', + 'myplugin', + ); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, '0.json'), JSON.stringify(run, null, 2)); + + const result = await gradeBatch(batchDir, { skipJudge: true }); + expect(result.errors).toHaveLength(0); + expect(result.graded).toHaveLength(1); + expect(result.graded[0].programmatic.score).toBeCloseTo(1, 5); + // The grade sidecar lands next to the run file, inside the plugin dir. + const sidecar = JSON.parse( + readFileSync(join(dir, '0.grade.json'), 'utf8'), + ) as { runId: string }; + expect(sidecar.runId).toBe(run.runId); + }); + + it('labels log lines with the plugin column key when plugin arms vary', async () => { + const batchDir = join(tmpRoot, 'plugin-arm-labels'); + for (const plugin of ['none', 'myplugin']) { + const run = buildRun({ + scenario: 'error-root-cause', + toolCalls: [{ name: 'mcp__hyperdx__hyperdx_query', isError: false }], + finalAnswer: ANSWER, + }); + run.plugin = plugin; + const dir = join( + batchDir, + 'error-root-cause', + 'hyperdx', + 'claude-sonnet-4-6', + plugin, + ); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, '0.json'), JSON.stringify(run, null, 2)); + } + + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + let lines: string[] = []; + try { + await gradeBatch(batchDir, { skipJudge: true }); + lines = logSpy.mock.calls.map(c => String(c[0])); + } finally { + logSpy.mockRestore(); + } + expect(lines.some(l => l.includes('error-root-cause/hyperdx/none/0'))).toBe( + true, + ); + expect( + lines.some(l => l.includes('error-root-cause/hyperdx/myplugin/0')), + ).toBe(true); + }); }); diff --git a/packages/hdx-eval/src/__tests__/markdown.test.ts b/packages/hdx-eval/src/__tests__/markdown.test.ts index 610de55c27..a2cc7e17ef 100644 --- a/packages/hdx-eval/src/__tests__/markdown.test.ts +++ b/packages/hdx-eval/src/__tests__/markdown.test.ts @@ -3,13 +3,19 @@ import type { RunRecord } from '@/harness/types'; import { buildAggregate, type GradedRunPair } from '@/reports/aggregate'; import { renderMarkdownReport } from '@/reports/markdown'; -function pair(scenario: string, mcp: string, i: number): GradedRunPair { +function pair( + scenario: string, + mcp: string, + i: number, + plugin = 'none', +): GradedRunPair { const run: RunRecord = { schemaVersion: 1, - runId: `${scenario}-${mcp}-${i}`, + runId: `${scenario}-${mcp}-${plugin}-${i}`, scenario, mcp, model: 'claude-sonnet-4-6', + plugin, runIndex: i, seed: 42, startedAt: '2026-05-09T07:50:00.000Z', @@ -114,3 +120,41 @@ describe('renderMarkdownReport', () => { expect(md).toContain('Baseline: clickhouse'); }); }); + +describe('renderMarkdownReport with plugin arms', () => { + const summary = buildAggregate({ + batchDir: '/tmp/2026-05-09T07-50-58-566Z', + pairs: [ + pair('error-root-cause', 'hyperdx', 0), + pair('error-root-cause', 'hyperdx', 0, 'myplugin'), + ], + }); + const md = renderMarkdownReport(summary); + + it('uses plugin column keys and the mcp/plugin format hint in the header', () => { + expect(md).toContain( + 'Columns: hyperdx/myplugin, hyperdx/none _(mcp/plugin)_', + ); + expect(md).toContain('Baseline: hyperdx/myplugin'); + }); + + it('renders a delta column for the challenger plugin arm', () => { + expect(md).toContain('Δ (hyperdx/none)'); + }); +}); + +describe('renderMarkdownReport with model and plugin arms', () => { + const modelPair = pair('error-root-cause', 'hyperdx', 0); + modelPair.run.model = 'claude-haiku-4-5'; + const summary = buildAggregate({ + batchDir: '/tmp/2026-05-09T07-50-58-566Z', + pairs: [modelPair, pair('error-root-cause', 'hyperdx', 0, 'myplugin')], + }); + const md = renderMarkdownReport(summary); + + it('uses the mcp/model+plugin format hint when both vary', () => { + expect(md).toContain('_(mcp/model+plugin)_'); + expect(md).toContain('hyperdx/claude-haiku-4-5+none'); + expect(md).toContain('hyperdx/claude-sonnet-4-6+myplugin'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/mcpConfig.test.ts b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts index f9c6db2c28..47ebb18714 100644 --- a/packages/hdx-eval/src/__tests__/mcpConfig.test.ts +++ b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts @@ -1,3 +1,4 @@ +import { pluginCliArgs } from '@/harness/claudeSpawn'; import { allowedToolsPattern, buildMcpConfig } from '@/harness/mcpConfig'; import { buildSettings, @@ -187,3 +188,22 @@ describe('enabledMcpNames', () => { expect(enabledMcpNames(cfg)).toContain('alpha'); }); }); + +describe('pluginCliArgs', () => { + it('returns --plugin-url args for a url plugin', () => { + expect( + pluginCliArgs({ label: 'P', url: 'https://example.com/p.zip' }), + ).toEqual(['--plugin-url', 'https://example.com/p.zip']); + }); + + it('returns --plugin-dir args for a dir plugin', () => { + expect(pluginCliArgs({ label: 'P', dir: '/abs/path/to/plugin' })).toEqual([ + '--plugin-dir', + '/abs/path/to/plugin', + ]); + }); + + it('returns no args for the no-plugin baseline', () => { + expect(pluginCliArgs(undefined)).toEqual([]); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/reportsStore.test.ts b/packages/hdx-eval/src/__tests__/reportsStore.test.ts new file mode 100644 index 0000000000..5e5b790b80 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/reportsStore.test.ts @@ -0,0 +1,158 @@ +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import type { GradeRecord } from '@/grading/types'; +import type { RunRecord } from '@/harness/types'; +import type { BatchSummary } from '@/reports/aggregate'; +import { writeBatchSummary } from '@/reports/store'; + +function buildRun(plugin: string): RunRecord { + return { + schemaVersion: 1, + runId: `run-${plugin}-0`, + scenario: 'error-root-cause', + mcp: 'hyperdx', + model: 'claude-sonnet-4-6', + plugin, + runIndex: 0, + seed: 42, + startedAt: '2026-07-02T00:00:00.000Z', + endedAt: '2026-07-02T00:01:00.000Z', + durationMs: 60_000, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: [], + messages: [], + finalAnswer: 'a', + tokens: { input: 100, output: 1000, cacheCreation: 0, cacheRead: 0 }, + totalCostUsd: 0.01, + stderr: '', + }; +} + +function buildGrade(run: RunRecord, combined: number): GradeRecord { + return { + schemaVersion: 2, + runId: run.runId, + scenario: run.scenario, + mcp: run.mcp, + programmatic: { + hits: [{ id: 'check-a', weight: 1, matched: true, satisfied: true }], + score: combined, + }, + judge: { + model: 'claude-opus-4-7', + scores: { correctness: { score: 5, rationale: '' } }, + weightedScore: combined, + rawResponse: '', + durationMs: 1000, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + }, + toolErrors: { total: 0, errors: 0, rate: 0, penalty: 0, samples: [] }, + combinedScore: combined, + gradedAt: '', + judgeModel: 'claude-opus-4-7', + }; +} + +describe('writeBatchSummary over a plugin-arm batch', () => { + const tmpRoot = join('/tmp', `hdx-eval-reports-store-test-${Date.now()}`); + const batchDir = join(tmpRoot, 'batch'); + + beforeAll(() => { + // Current layout: ////.json with a + // .grade.json sidecar next to each run. + for (const [plugin, combined] of [ + ['none', 0.8], + ['myplugin', 0.9], + ] as const) { + const dir = join( + batchDir, + 'error-root-cause', + 'hyperdx', + 'claude-sonnet-4-6', + plugin, + ); + mkdirSync(dir, { recursive: true }); + const run = buildRun(plugin); + writeFileSync(join(dir, '0.json'), JSON.stringify(run, null, 2)); + writeFileSync( + join(dir, '0.grade.json'), + JSON.stringify(buildGrade(run, combined), null, 2), + ); + } + }); + + afterAll(() => { + rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it('aggregates plugin arms into plugin-keyed columns', () => { + const outPath = join(batchDir, '_summary.md'); + const result = writeBatchSummary(batchDir, outPath); + expect(result.pairsCount).toBe(2); + + const summary = JSON.parse( + readFileSync(result.jsonPath, 'utf8'), + ) as BatchSummary; + expect(summary.multiModel).toBe(false); + expect(summary.multiPlugin).toBe(true); + expect(summary.columnOrder).toEqual(['hyperdx/myplugin', 'hyperdx/none']); + // Default baseline: first column. + expect(summary.baseline).toBe('hyperdx/myplugin'); + + const scenario = summary.scenarios[0]; + expect(scenario.cells['hyperdx/none'].plugin).toBe('none'); + expect(scenario.cells['hyperdx/myplugin'].plugin).toBe('myplugin'); + // Delta = challenger (none, 0.8) − baseline (myplugin, 0.9). + expect(scenario.deltas['hyperdx/none'].combinedScore).toBeCloseTo(-0.1, 5); + + const md = readFileSync(result.mdPath, 'utf8'); + expect(md).toContain('hyperdx/myplugin'); + expect(md).toContain('Δ (hyperdx/none)'); + }); + + it('honors an explicit column-key baseline', () => { + const outPath = join(batchDir, '_summary-explicit.md'); + const result = writeBatchSummary(batchDir, outPath, 'hyperdx/none'); + const summary = JSON.parse( + readFileSync(result.jsonPath, 'utf8'), + ) as BatchSummary; + expect(summary.baseline).toBe('hyperdx/none'); + expect( + summary.scenarios[0].deltas['hyperdx/myplugin'].combinedScore, + ).toBeCloseTo(0.1, 5); + }); + + it('reuses the persisted baseline when regenerating without --baseline', () => { + // Simulate the run command's auto-report: explicit baseline written to + // the canonical _summary.json location. + writeBatchSummary(batchDir, join(batchDir, '_summary.md'), 'hyperdx/none'); + + // Regenerate with no explicit baseline: reuses the persisted one instead + // of falling back to the alphabetically-first column. + const result = writeBatchSummary(batchDir, join(batchDir, '_regen.md')); + const summary = JSON.parse( + readFileSync(result.jsonPath, 'utf8'), + ) as BatchSummary; + expect(summary.baseline).toBe('hyperdx/none'); + expect( + summary.scenarios[0].deltas['hyperdx/myplugin'].combinedScore, + ).toBeCloseTo(0.1, 5); + }); + + it('ignores a stale persisted baseline that matches no column', () => { + writeFileSync( + join(batchDir, '_summary.json'), + JSON.stringify({ baseline: 'hyperdx+oldformat' }), + ); + const result = writeBatchSummary(batchDir, join(batchDir, '_stale.md')); + const summary = JSON.parse( + readFileSync(result.jsonPath, 'utf8'), + ) as BatchSummary; + expect(summary.baseline).toBe('hyperdx/myplugin'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/runPath.test.ts b/packages/hdx-eval/src/__tests__/runPath.test.ts new file mode 100644 index 0000000000..a33f9e47bd --- /dev/null +++ b/packages/hdx-eval/src/__tests__/runPath.test.ts @@ -0,0 +1,113 @@ +import { mkdirSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { getRunFilesInBatch, runFilePath, runsRoot } from '@/runs/path'; + +describe('runFilePath', () => { + it('always includes a plugin directory level, using "none" for the no-plugin arm', () => { + const p = runFilePath({ + batchDir: 'batch', + scenario: 'error-root-cause', + mcp: 'hyperdx', + model: 'claude-sonnet-4-6', + plugin: 'none', + runIndex: 0, + }); + expect(p).toBe( + join( + runsRoot(), + 'batch', + 'error-root-cause', + 'hyperdx', + 'claude-sonnet-4-6', + 'none', + '0.json', + ), + ); + }); + + it('defaults an omitted plugin to the "none" directory', () => { + const p = runFilePath({ + batchDir: 'batch', + scenario: 'error-root-cause', + mcp: 'hyperdx', + model: 'claude-sonnet-4-6', + runIndex: 1, + }); + expect(p).toContain(join('claude-sonnet-4-6', 'none', '1.json')); + }); + + it('sanitizes model and plugin segments the same way', () => { + const p = runFilePath({ + batchDir: 'batch', + scenario: 'error-root-cause', + mcp: 'hyperdx', + model: 'claude-sonnet-4.5', + plugin: 'my.plugin', + runIndex: 2, + }); + expect(p).toContain(join('claude-sonnet-4-5', 'my-plugin', '2.json')); + }); +}); + +describe('getRunFilesInBatch', () => { + const tmpRoot = join('/tmp', `hdx-eval-run-path-test-${Date.now()}`); + const batchDir = join(tmpRoot, 'batch'); + const mcpDir = join(batchDir, 'error-root-cause', 'hyperdx'); + const modelDir = join(mcpDir, 'claude-sonnet-4-6'); + + beforeAll(() => { + mkdirSync(join(modelDir, 'none'), { recursive: true }); + mkdirSync(join(modelDir, 'myplugin'), { recursive: true }); + // Current layout: per-plugin subdirs, no-plugin arm under none/. + writeFileSync(join(modelDir, 'none', '0.json'), '{}'); + writeFileSync(join(modelDir, 'myplugin', '1.json'), '{}'); + // Legacy layout: run directly in the model dir (pre-plugin-level). + writeFileSync(join(modelDir, '2.json'), '{}'); + // Legacy layout: run directly in the mcp dir (pre-multi-model). + writeFileSync(join(mcpDir, '3.json'), '{}'); + // Sidecars — excluded everywhere. + writeFileSync(join(modelDir, 'none', '0.grade.json'), '{}'); + writeFileSync(join(modelDir, '2.timing.json'), '{}'); + // A second scenario, for scenarioFilter. + const otherDir = join( + batchDir, + 'latency-spike', + 'hyperdx', + 'model', + 'none', + ); + mkdirSync(otherDir, { recursive: true }); + writeFileSync(join(otherDir, '0.json'), '{}'); + }); + + afterAll(() => { + rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it('collects runs from the current layout and both legacy layouts, skipping sidecars', () => { + const files = getRunFilesInBatch(batchDir); + expect(files).toEqual( + [ + join(modelDir, 'none', '0.json'), + join(modelDir, 'myplugin', '1.json'), + join(modelDir, '2.json'), + join(mcpDir, '3.json'), + join(batchDir, 'latency-spike', 'hyperdx', 'model', 'none', '0.json'), + ].sort(), + ); + }); + + it('restricts walking to scenarios matching scenarioFilter', () => { + const files = getRunFilesInBatch(batchDir, { + scenarioFilter: s => s === 'latency-spike', + }); + expect(files).toEqual([ + join(batchDir, 'latency-spike', 'hyperdx', 'model', 'none', '0.json'), + ]); + }); + + it('returns [] for a missing batch directory', () => { + expect(getRunFilesInBatch(join(tmpRoot, 'does-not-exist'))).toEqual([]); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/runRun.test.ts b/packages/hdx-eval/src/__tests__/runRun.test.ts index b58c73d5fe..960cd2f12c 100644 --- a/packages/hdx-eval/src/__tests__/runRun.test.ts +++ b/packages/hdx-eval/src/__tests__/runRun.test.ts @@ -19,6 +19,7 @@ const baseInput = { systemPromptAppend: 'sys', mcp: 'hyperdx' as const, model: 'claude-sonnet-4-6', + plugin: 'none', runIndex: 0, seed: 42, startedAtIso: '2026-05-09T10:00:00.000Z', @@ -97,6 +98,21 @@ describe('assembleRecord', () => { expect(record.tools.map(t => t.name)).toEqual([ 'mcp__hyperdx__hyperdx_query', ]); + expect(record.plugin).toBe('none'); + }); + + it('propagates the plugin arm into the record and the runId', () => { + const record = assembleRecord({ + ...baseInput, + plugin: 'myplugin', + events: [], + exitCode: 0, + timedOut: false, + }); + expect(record.plugin).toBe('myplugin'); + // The runId disambiguates arms, so same-index runs from different + // plugin arms never collide. + expect(record.runId).toContain('-myplugin-0'); }); it('marks termination as max_turns when result subtype indicates it', () => { diff --git a/packages/hdx-eval/src/__tests__/runsStore.test.ts b/packages/hdx-eval/src/__tests__/runsStore.test.ts new file mode 100644 index 0000000000..555bc029b1 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/runsStore.test.ts @@ -0,0 +1,63 @@ +import { existsSync, rmSync } from 'fs'; +import { join } from 'path'; + +import type { RunRecord } from '@/harness/types'; +import { runsRoot } from '@/runs/path'; +import { listRunsInBatch, writeRun } from '@/runs/store'; + +function buildRun(plugin: string, runIndex: number): RunRecord { + return { + schemaVersion: 1, + runId: `run-${plugin}-${runIndex}`, + scenario: 'error-root-cause', + mcp: 'hyperdx', + model: 'claude-sonnet-4-6', + plugin, + runIndex, + seed: 42, + startedAt: '2026-07-02T00:00:00.000Z', + endedAt: '2026-07-02T00:01:00.000Z', + durationMs: 60_000, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: [], + messages: [], + finalAnswer: 'a', + tokens: { input: 1, output: 1, cacheCreation: 0, cacheRead: 0 }, + totalCostUsd: 0, + stderr: '', + }; +} + +describe('writeRun / listRunsInBatch round-trip', () => { + const batch = `test-batch-runs-store-${Date.now()}`; + + afterAll(() => { + rmSync(join(runsRoot(), batch), { recursive: true, force: true }); + }); + + it('writes every arm under // and lists them back', () => { + const nonePath = writeRun({ record: buildRun('none', 0), batchDir: batch }); + const pluginPath = writeRun({ + record: buildRun('myplugin', 0), + batchDir: batch, + }); + + // Model and plugin each get a directory level; the no-plugin arm lives + // under the literal `none` directory. + expect(nonePath).toContain(join('claude-sonnet-4-6', 'none', '0.json')); + expect(pluginPath).toContain( + join('claude-sonnet-4-6', 'myplugin', '0.json'), + ); + expect(existsSync(nonePath)).toBe(true); + expect(existsSync(pluginPath)).toBe(true); + + // The batch walker finds exactly what the writer produced. + expect(listRunsInBatch(batch).sort()).toEqual( + [nonePath, pluginPath].sort(), + ); + }); +}); diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts index 117c1d483f..3f0916e0d1 100644 --- a/packages/hdx-eval/src/cli.ts +++ b/packages/hdx-eval/src/cli.ts @@ -16,14 +16,17 @@ import { resolveBatchDir, } from './grading/grade'; import { runCell } from './harness/runRun'; -import type { McpKind, PromptVariant } from './harness/types'; +import { type McpKind, PLUGIN_NONE, type PromptVariant } from './harness/types'; import { configExists, configMcpNames, configPath, + configPluginNames, enabledMcpNames, ensureAnchorTime, + type EvalConfig, getMcpDefinition, + getPluginDefinition, readConfig, } from './hyperdx/config'; import { runCheck, runSetup } from './hyperdx/setup'; @@ -338,9 +341,17 @@ program 'Comma-separated MCP names from config, or "all" (default: all)', 'all', ) + .option( + '--plugin ', + 'Comma-separated Claude Code plugin variants from config (like --mcp/--model). ' + + 'The literal "none" is the no-plugin variant. Default when omitted: "none". ' + + 'Pass "none," to compare a plugin against the no-plugin baseline.', + ) .option( '--baseline ', - 'MCP to use as baseline in reports (default: first MCP in list)', + 'Column key to use as baseline in reports. Keys are the MCP name, plus ' + + '"/", "/", or "/+" when models/plugins ' + + 'vary (default: the first listed mcp/model/plugin variants)', ) .option('--runs ', 'Number of runs per (scenario,MCP) cell', '3') .option( @@ -416,6 +427,7 @@ program scenarioName: string, cmdOpts: { mcp: string; + plugin?: string; baseline?: string; runs: string; model: string; @@ -457,15 +469,28 @@ program getMcpDefinition(config, mcp); } const models = parseModelFlag(cmdOpts.model); - const multiModel = models.length > 1; - // For baseline resolution: when multi-model, the column key is - // mcp/sanitizedModel; otherwise just mcp. - const firstColumnKey = columnKeyFor(mcpKinds[0], models[0], multiModel); + const plugins = parsePluginFlag(cmdOpts.plugin, config); + const keyOpts = { + multiModel: models.length > 1, + multiPlugin: plugins.length > 1, + }; + // Default baseline: the first listed variant of each dimension (first + // mcp, first model, first plugin — or their defaults when a flag is + // omitted). The auto-report persists it in _summary.json, and `report` + // regenerations reuse the persisted value, so delta signs stay stable. + const firstColumnKey = columnKeyFor( + mcpKinds[0], + models[0], + plugins[0], + keyOpts, + ); const baseline = cmdOpts.baseline ?? firstColumnKey; if (cmdOpts.baseline) { // Validate: baseline must match one of the column keys. const allColumnKeys = mcpKinds.flatMap(m => - models.map(mod => columnKeyFor(m, mod, multiModel)), + models.flatMap(mod => + plugins.map(pl => columnKeyFor(m, mod, pl, keyOpts)), + ), ); if (!allColumnKeys.includes(cmdOpts.baseline)) { throw new Error( @@ -570,6 +595,7 @@ program console.log(`Scenario: ${scenario.name}`); console.log(`MCPs: ${mcpKinds.join(', ')}`); console.log(`Models: ${models.join(', ')}`); + console.log(`Plugins: ${plugins.join(', ')}`); console.log( `Runs/cell: ${runs}, max-turns: ${maxTurns}, concurrency: ${concurrency}, prompt-variant: ${promptVariant}\n`, ); @@ -577,6 +603,7 @@ program type SummaryRow = { mcp: McpKind; model: string; + plugin: string; i: number; toolCalls: number; inputTokens: number; @@ -586,12 +613,20 @@ program path: string; }; - // Flatten the (mcp, model, runIndex) matrix into a single queue and - // pull from it with a worker pool of size `concurrency`. - const cells: Array<{ mcp: McpKind; model: string; i: number }> = []; + // Flatten the (mcp, model, plugin, runIndex) matrix into a single queue + // and pull from it with a worker pool of size `concurrency`. + const cells: Array<{ + mcp: McpKind; + model: string; + plugin: string; + i: number; + }> = []; for (const mcp of mcpKinds) { for (const model of models) { - for (let i = 0; i < runs; i++) cells.push({ mcp, model, i }); + for (const plugin of plugins) { + for (let i = 0; i < runs; i++) + cells.push({ mcp, model, plugin, i }); + } } } @@ -600,6 +635,7 @@ program const errors: Array<{ mcp: string; model: string; + plugin: string; i: number; error: string; }> = []; @@ -607,8 +643,8 @@ program while (true) { const idx = cursor++; if (idx >= cells.length) return; - const { mcp, model, i } = cells[idx]; - const cellLabel = columnKeyFor(mcp, model, multiModel); + const { mcp, model, plugin, i } = cells[idx]; + const cellLabel = columnKeyFor(mcp, model, plugin, keyOpts); const label = concurrency > 1 ? `[w${workerId}] ` : ' '; try { process.stdout.write( @@ -621,6 +657,7 @@ program agentPrompt: scenario.agentPrompt, mcp, model, + plugin, maxTurns, timeoutMs, runIndex: i, @@ -637,6 +674,7 @@ program summary.push({ mcp, model, + plugin, i, toolCalls: record.toolCalls.length, inputTokens: record.tokens.input, @@ -650,7 +688,7 @@ program console.error( `${label}${cellLabel} run ${i + 1}/${runs}: FAILED — ${msg}`, ); - errors.push({ mcp, model, i, error: msg }); + errors.push({ mcp, model, plugin, i, error: msg }); } } } @@ -664,24 +702,26 @@ program `\n${errors.length} run(s) failed:\n` + errors .map(e => { - const cl = columnKeyFor(e.mcp, e.model, multiModel); + const cl = columnKeyFor(e.mcp, e.model, e.plugin, keyOpts); return ` ${cl} #${e.i}: ${e.error}`; }) .join('\n'), ); } console.log('\nResults written under runs/' + batchDir); - // Sort by (mcp, model, i) for stable summary output. + // Sort by (mcp, model, plugin, i) for stable summary output. summary.sort( (a, b) => a.mcp.localeCompare(b.mcp) || a.model.localeCompare(b.model) || + a.plugin.localeCompare(b.plugin) || a.i - b.i, ); + const labelWidth = keyOpts.multiModel || keyOpts.multiPlugin ? 34 : 10; for (const row of summary) { - const cl = columnKeyFor(row.mcp, row.model, multiModel); + const cl = columnKeyFor(row.mcp, row.model, row.plugin, keyOpts); console.log( - ` ${cl.padEnd(multiModel ? 30 : 10)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, + ` ${cl.padEnd(labelWidth)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, ); } @@ -940,7 +980,8 @@ program .option('--out ', 'Output markdown path (default: /_summary.md)') .option( '--baseline ', - 'MCP to use as baseline for delta computation (default: first MCP found)', + 'Column key to use as baseline for delta computation (default: the ' + + "baseline recorded in the batch's _summary.json, else the first column)", ) .action( async (batch: string, cmdOpts: { out?: string; baseline?: string }) => { @@ -1014,6 +1055,43 @@ function parsePromptVariant(v: string): PromptVariant { ); } +/** + * Resolve the plugin variants for a run. Mirrors `--model`/`--mcp`: the plugins are + * exactly the names passed (deduped, order-preserving). When `--plugin` is + * omitted the default is the single no-plugin arm (`PLUGIN_NONE`). + */ +function parsePluginFlag(v: string | undefined, config: EvalConfig): string[] { + if (!v) return [PLUGIN_NONE]; + const names = [ + ...new Set( + v + .split(',') + .map(s => s.trim()) + .filter(Boolean), + ), + ]; + if (names.length === 0) return [PLUGIN_NONE]; + const available = configPluginNames(config); + const arms: string[] = []; + for (const name of names) { + if (name === PLUGIN_NONE) { + arms.push(PLUGIN_NONE); + continue; + } + if (!available.includes(name)) { + throw new Error( + `--plugin: "${name}" not found in config 'plugins'. Available: ${ + available.join(', ') || '(none defined)' + }`, + ); + } + // Validate the definition (exactly one of url/dir) up front. + getPluginDefinition(config, name); + arms.push(name); + } + return arms; +} + program.parseAsync(process.argv).catch(err => { console.error(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/packages/hdx-eval/src/grading/grade.ts b/packages/hdx-eval/src/grading/grade.ts index b5449936c5..ee099e9d03 100644 --- a/packages/hdx-eval/src/grading/grade.ts +++ b/packages/hdx-eval/src/grading/grade.ts @@ -2,8 +2,9 @@ import Anthropic from '@anthropic-ai/sdk'; import { existsSync, readFileSync, writeFileSync } from 'fs'; import { dirname, join } from 'path'; -import type { RunRecord } from '@/harness/types'; -import { isModelSubdir, isRunJson, runsRoot, safeReaddir } from '@/runs/path'; +import { PLUGIN_NONE, type RunRecord } from '@/harness/types'; +import { columnKeyFor } from '@/reports/aggregate'; +import { getRunFilesInBatch, runsRoot } from '@/runs/path'; import { readRun } from '@/runs/store'; import { getScenario, SCENARIO_NAMES } from '@/scenarios'; import type { PostRunInspectionResult } from '@/scenarios/types'; @@ -108,6 +109,24 @@ export async function gradeBatch( } const runFiles = listRunFiles(resolved); + // Detect whether this batch varies models/plugins so the per-run log labels + // use the same column keys as the run output (e.g. `hyperdx/none/0`). + const models = new Set(); + const plugins = new Set(); + for (const p of runFiles) { + try { + const r = readRun(p); + models.add(r.model); + plugins.add(r.plugin ?? PLUGIN_NONE); + } catch { + // Unreadable runs are reported by the grading loop below. + } + } + const keyOpts = { + multiModel: models.size > 1, + multiPlugin: plugins.size > 1, + }; + // Decide whether we'll actually need the judge: skipJudge wins, otherwise // we need it if any run is missing a cached judge OR rerun was requested. const needsJudge = @@ -160,8 +179,14 @@ export async function gradeBatch( const inspBit = grade.inspectionSummary ? formatInspectionLogBit(grade.inspectionSummary) : ''; + const cellLabel = columnKeyFor( + record.mcp, + record.model, + record.plugin ?? PLUGIN_NONE, + keyOpts, + ); console.log( - ` ${grade.scenario}/${grade.mcp}/${grade.runId.split('-').slice(-1)[0]} prog=${( + ` ${grade.scenario}/${cellLabel}/${grade.runId.split('-').slice(-1)[0]} prog=${( grade.programmatic.score * 100 ).toFixed( 0, @@ -291,33 +316,11 @@ async function gradeOne(args: { }; } -/** - * List run JSON files. Supports both the new - * `///.json` layout and the legacy - * `//.json` layout. - */ +/** List run JSON files. See `getRunFilesInBatch` for the supported layouts. */ function listRunFiles(batchDir: string): string[] { - const out: string[] = []; - for (const scenario of safeReaddir(batchDir)) { - if (!SCENARIO_NAMES.includes(scenario)) continue; - const sceneDir = join(batchDir, scenario); - for (const mcp of safeReaddir(sceneDir)) { - const mcpDir = join(sceneDir, mcp); - for (const entry of safeReaddir(mcpDir)) { - if (isRunJson(entry)) { - // Legacy layout: //.json - out.push(join(mcpDir, entry)); - } else if (isModelSubdir(mcpDir, entry)) { - // New layout: ///.json - const modelDir = join(mcpDir, entry); - for (const file of safeReaddir(modelDir)) { - if (isRunJson(file)) out.push(join(modelDir, file)); - } - } - } - } - } - return out.sort(); + return getRunFilesInBatch(batchDir, { + scenarioFilter: s => SCENARIO_NAMES.includes(s), + }); } function readExistingGrade(path: string): GradeRecord | null { diff --git a/packages/hdx-eval/src/harness/claudeSpawn.ts b/packages/hdx-eval/src/harness/claudeSpawn.ts index 83caf8cc86..8c6307f7d0 100644 --- a/packages/hdx-eval/src/harness/claudeSpawn.ts +++ b/packages/hdx-eval/src/harness/claudeSpawn.ts @@ -12,7 +12,12 @@ import { type ParsedEvent, parseStreamLine, } from './streamParser'; -import type { McpDefinition, McpKind, PromptVariant } from './types'; +import type { + McpDefinition, + McpKind, + PluginDefinition, + PromptVariant, +} from './types'; export type SpawnOptions = { config: EvalConfig; @@ -33,6 +38,9 @@ export type SpawnOptions = { /** Tool name substrings to remove from the denied-tools list. * Comes from `Scenario.allowedToolPatterns`. */ allowedToolPatterns?: string[]; + /** When set, load this Claude Code plugin into the isolated agent session + * via `--plugin-url`/`--plugin-dir`. Undefined for the no-plugin baseline. */ + pluginDef?: PluginDefinition; }; export type SpawnResult = { @@ -45,6 +53,19 @@ export type SpawnResult = { tempdir: string; }; +/** + * CLI args that load a plugin into the isolated agent session: `--plugin-url` + * for `url` definitions, `--plugin-dir` for `dir` definitions, and no args + * for the no-plugin baseline (`def` undefined). + */ +export function pluginCliArgs(def?: PluginDefinition): string[] { + return def?.url + ? ['--plugin-url', def.url] + : def?.dir + ? ['--plugin-dir', def.dir] + : []; +} + export async function runClaude(opts: SpawnOptions): Promise { const tempdir = mkdtempSync(join(tmpdir(), 'hdx-eval-')); const mcpConfigPath = join(tempdir, 'mcp-config.json'); @@ -65,6 +86,9 @@ export async function runClaude(opts: SpawnOptions): Promise { ), ); + // Load a plugin for this arm, if any + const pluginArgs = pluginCliArgs(opts.pluginDef); + const argv = [ '-p', '--mcp-config', @@ -73,6 +97,7 @@ export async function runClaude(opts: SpawnOptions): Promise { `${allowedToolsPattern(mcpDef)},Read(${tempdir}/*)`, '--disallowedTools', deniedToolsFor(promptVariant, mcpDef, opts.allowedToolPatterns).join(','), + ...pluginArgs, '--dangerously-skip-permissions', '--setting-sources', 'local', diff --git a/packages/hdx-eval/src/harness/runRun.ts b/packages/hdx-eval/src/harness/runRun.ts index 9e7762a7d8..0496ff24b0 100644 --- a/packages/hdx-eval/src/harness/runRun.ts +++ b/packages/hdx-eval/src/harness/runRun.ts @@ -1,15 +1,20 @@ -import { type EvalConfig, getMcpDefinition } from '@/hyperdx/config'; +import { + type EvalConfig, + getMcpDefinition, + getPluginDefinition, +} from '@/hyperdx/config'; import type { Scenario } from '@/scenarios/types'; import { runClaude } from './claudeSpawn'; import type { ParsedEvent } from './streamParser'; import { buildSystemPrompt } from './systemPrompt'; -import type { - McpKind, - PromptVariant, - RunRecord, - Termination, - ToolCallRecord, +import { + type McpKind, + PLUGIN_NONE, + type PromptVariant, + type RunRecord, + type Termination, + type ToolCallRecord, } from './types'; export type RunCellOptions = { @@ -30,13 +35,21 @@ export type RunCellOptions = { /** Prompt variant — 'baseline' (default) or 'hypothesis' (forces * hypothesis-enumeration + parallel Task subagents). */ promptVariant?: PromptVariant; + /** Plugin key — a key in the config `plugins` map, or `PLUGIN_NONE` + * (default) for the no-plugin baseline. */ + plugin?: string; }; export async function runCell(opts: RunCellOptions): Promise { const startedAtIso = new Date().toISOString(); const startedAtMs = Date.now(); const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; + const plugin = opts.plugin ?? PLUGIN_NONE; const mcpDef = getMcpDefinition(opts.config, opts.mcp); + const pluginDef = + plugin === PLUGIN_NONE + ? undefined + : getPluginDefinition(opts.config, plugin); // Use scenario's custom system prompt builder if provided, // otherwise fall back to the default investigation prompt. @@ -60,6 +73,7 @@ export async function runCell(opts: RunCellOptions): Promise { apiKey: opts.apiKey, promptVariant, allowedToolPatterns: opts.scenario.allowedToolPatterns, + pluginDef, }); const endedAtIso = new Date().toISOString(); @@ -75,6 +89,7 @@ export async function runCell(opts: RunCellOptions): Promise { systemPromptAppend, mcp: opts.mcp, model: opts.model, + plugin, runIndex: opts.runIndex, seed: opts.seed, startedAtIso, @@ -93,6 +108,7 @@ type AssembleInput = { systemPromptAppend: string; mcp: McpKind; model: string; + plugin: string; runIndex: number; seed: number; startedAtIso: string; @@ -183,10 +199,11 @@ export function assembleRecord(input: AssembleInput): RunRecord { return { schemaVersion: 1, - runId: `${input.startedAtIso}-${input.scenario}-${input.mcp}-${input.runIndex}`, + runId: `${input.startedAtIso}-${input.scenario}-${input.mcp}-${input.plugin}-${input.runIndex}`, scenario: input.scenario, mcp: input.mcp, model: input.model, + plugin: input.plugin, runIndex: input.runIndex, seed: input.seed, startedAt: input.startedAtIso, diff --git a/packages/hdx-eval/src/harness/types.ts b/packages/hdx-eval/src/harness/types.ts index 84b97de9e5..c86fc3fb5d 100644 --- a/packages/hdx-eval/src/harness/types.ts +++ b/packages/hdx-eval/src/harness/types.ts @@ -43,6 +43,27 @@ export type McpDefinition = (HttpMcpTransport | StdioMcpTransport) & { enabled?: boolean; }; +/** + * A Claude Code plugin definition in the eval config. A plugin is loaded into + * the isolated agent session (via `--plugin-url`/`--plugin-dir`) and treated + * as a evaluation variant. Exactly one of `url` (a `.zip` URL) or `dir` + * (a local plugin directory) should be set. + */ +export type PluginDefinition = { + /** Human-readable label for reports and CLI output. */ + label: string; + /** URL to a plugin `.zip` archive (loaded via `--plugin-url`). */ + url?: string; + /** Local plugin directory (loaded via `--plugin-dir`). */ + dir?: string; +}; + +/** + * Sentinel plugin key meaning "no plugin". This is the implicit default + * for a run with no `--plugin` flag. + */ +export const PLUGIN_NONE = 'none'; + /** * Prompt variants for the system-prompt A/B. * - `baseline`: the default investigative prompt, no subagent affordance. @@ -70,6 +91,7 @@ export type RunRecord = { scenario: string; mcp: McpKind; model: string; + plugin?: string; runIndex: number; seed: number; startedAt: string; diff --git a/packages/hdx-eval/src/hyperdx/config.ts b/packages/hdx-eval/src/hyperdx/config.ts index d3b385235a..5e6ac71062 100644 --- a/packages/hdx-eval/src/hyperdx/config.ts +++ b/packages/hdx-eval/src/hyperdx/config.ts @@ -1,7 +1,12 @@ import { existsSync, readFileSync, writeFileSync } from 'fs'; import { resolve } from 'path'; -import type { McpDefinition, McpKind } from '@/harness/types'; +import { + type McpDefinition, + type McpKind, + PLUGIN_NONE, + type PluginDefinition, +} from '@/harness/types'; type ScenarioSourceIds = { tracesSourceId: string; @@ -21,6 +26,8 @@ type HyperdxApiConfig = { export type EvalConfig = { /** Registry of MCP definitions keyed by a free-form name. */ mcps: Record; + /** Registry of Claude Code plugin definitions keyed by a free-form name. */ + plugins?: Record; /** Per-scenario HyperDX Source IDs (only needed when an MCP points at * HyperDX and the Sources must exist). */ scenarios?: Record; @@ -100,6 +107,32 @@ export function getMcpDefinition( return def; } +/** Return all plugin names defined in the config. */ +export function configPluginNames(config: EvalConfig): string[] { + return Object.keys(config.plugins ?? {}); +} + +/** Get a single plugin definition by name, or throw. Validates that exactly + * one of `url`/`dir` is set. */ +export function getPluginDefinition( + config: EvalConfig, + name: string, +): PluginDefinition { + const def = config.plugins?.[name]; + if (!def) { + const available = configPluginNames(config).join(', ') || '(none defined)'; + throw new Error( + `Plugin "${name}" not found in config. Available: ${available}`, + ); + } + const hasUrl = typeof def.url === 'string' && def.url.length > 0; + const hasDir = typeof def.dir === 'string' && def.dir.length > 0; + if (hasUrl === hasDir) { + throw new Error(`Plugin "${name}" must set exactly one of 'url' or 'dir'.`); + } + return def; +} + /** * Return the config's saved `anchorTime`, creating and persisting a default * (current wall-clock time) when none exists yet. This makes anchor-time @@ -181,5 +214,38 @@ function validateConfig(raw: unknown, path: string): EvalConfig { } } + // Validate the optional `plugins` section. + const plugins = obj.plugins as Record | undefined; + if (plugins !== undefined) { + if (typeof plugins !== 'object' || plugins === null) { + throw new Error(`Eval config at ${path} 'plugins' must be an object`); + } + for (const [name, def] of Object.entries(plugins)) { + if (name === PLUGIN_NONE) { + throw new Error( + `Eval config 'plugins' must not use the reserved name ` + + `"${PLUGIN_NONE}" — it always means the no-plugin baseline, so ` + + `a plugin with that name could never be selected`, + ); + } + if (!def || typeof def !== 'object') { + throw new Error(`Eval config 'plugins.${name}' must be an object`); + } + const d = def as Record; + if (typeof d.label !== 'string' || !d.label) { + throw new Error( + `Eval config 'plugins.${name}.label' must be a non-empty string`, + ); + } + const hasUrl = typeof d.url === 'string' && d.url.length > 0; + const hasDir = typeof d.dir === 'string' && d.dir.length > 0; + if (hasUrl === hasDir) { + throw new Error( + `Eval config 'plugins.${name}' must set exactly one of 'url' or 'dir'`, + ); + } + } + } + return raw as EvalConfig; } diff --git a/packages/hdx-eval/src/reports/aggregate.ts b/packages/hdx-eval/src/reports/aggregate.ts index 03ef8e6a0b..237ce2abf3 100644 --- a/packages/hdx-eval/src/reports/aggregate.ts +++ b/packages/hdx-eval/src/reports/aggregate.ts @@ -1,12 +1,12 @@ import type { GradeRecord } from '@/grading/types'; -import type { McpKind, RunRecord } from '@/harness/types'; -import { modelDirName } from '@/runs/path'; +import { type McpKind, PLUGIN_NONE, type RunRecord } from '@/harness/types'; +import { escapeDirSegment } from '@/runs/path'; /** - * A column key uniquely identifies a (mcp, model) combination in reports. - * When a batch has a single model, the column key equals the MCP name - * (backward-compatible). When multiple models are present, the column - * key is `mcp/model`. + * A column key uniquely identifies a (mcp, model, plugin) combination in + * reports. The MCP name is always the base; when the batch compares multiple + * models and/or plugins, the varying components are appended after a `/`, + * joined with `+`: `mcp`, `mcp/model`, `mcp/plugin`, or `mcp/model+plugin`. */ export type ColumnKey = string; @@ -14,6 +14,7 @@ export type CellSummary = { scenario: string; mcp: McpKind; model: string; + plugin: string; /** Display key used in report columns. */ columnKey: ColumnKey; n: number; @@ -47,7 +48,7 @@ export type DeltaSummary = { export type ScenarioSummary = { scenario: string; - /** Cells keyed by column key (mcp or mcp/model). */ + /** Cells keyed by column key (e.g. mcp, mcp/model, mcp/model+plugin). */ cells: Record; /** Which column key is the baseline. */ baseline?: ColumnKey; @@ -64,6 +65,8 @@ export type BatchSummary = { columnOrder: ColumnKey[]; /** Whether this batch compares multiple models. */ multiModel: boolean; + /** Whether this batch compares multiple plugin arms. */ + multiPlugin: boolean; scenarios: ScenarioSummary[]; /** * @deprecated Use `columnOrder`. Preserved for backward compatibility @@ -78,56 +81,96 @@ export type GradedRunPair = { }; /** - * Build a column key from a (mcp, model) pair. - * When `multiModel` is false the column key is just the mcp name. + * Build a column key from a (mcp, model, plugin) tuple. + * The MCP name is always the base. Model and plugin follow the same rule: + * a component is included only when the batch compares multiple values of + * that dimension. Varying components are appended after a `/`, joined with + * `+` (e.g. `hyperdx/opus`, `hyperdx/myplugin`, `hyperdx/opus+myplugin`). + * When plugins vary, the no-plugin variant renders as the literal `none`. * - * The model component is sanitized through `modelDirName` so that the - * column key matches the directory name on disk (which is what the - * viewer reads). Without this, model IDs containing `/`, `:`, or `.` - * would produce different keys in the summary JSON vs the viewer. + * The model/plugin components are sanitized through `escapeDirSegment` so the + * column key matches the directory name on disk (which is what the viewer + * reads). */ export function columnKeyFor( mcp: McpKind, model: string, - multiModel: boolean, + plugin: string, + opts: { multiModel: boolean; multiPlugin: boolean }, ): ColumnKey { - return multiModel ? `${mcp}/${modelDirName(model)}` : mcp; + const varying: string[] = []; + if (opts.multiModel) varying.push(escapeDirSegment(model)); + if (opts.multiPlugin) varying.push(escapeDirSegment(plugin || PLUGIN_NONE)); + return varying.length > 0 ? `${mcp}/${varying.join('+')}` : mcp; } export function buildAggregate(args: { batchDir: string; pairs: GradedRunPair[]; - /** Explicit baseline column key. If not set, the first column encountered - * is used. When using a single model, pass just the mcp name. */ + /** Explicit baseline column key. Ignored when it doesn't match any column + * in the batch; without a (valid) baseline the first column is used. + * Callers that know the CLI variant order (the `run` command) or a + * previously persisted baseline (`writeBatchSummary`) pass it here. */ baseline?: ColumnKey; }): BatchSummary { - // Detect whether this batch involves multiple models. + // Detect whether this batch compares multiple models / plugins. const modelSet = new Set(); - for (const p of args.pairs) modelSet.add(p.run.model); + const pluginSet = new Set(); + for (const p of args.pairs) { + modelSet.add(p.run.model); + pluginSet.add(p.run.plugin ?? PLUGIN_NONE); + } const multiModel = modelSet.size > 1; + const multiPlugin = pluginSet.size > 1; + const keyOpts = { multiModel, multiPlugin }; const byScenario = new Map(); const columnSet = new Set(); for (const p of args.pairs) { if (!byScenario.has(p.run.scenario)) byScenario.set(p.run.scenario, []); byScenario.get(p.run.scenario)!.push(p); - columnSet.add(columnKeyFor(p.run.mcp, p.run.model, multiModel)); + columnSet.add( + columnKeyFor( + p.run.mcp, + p.run.model, + p.run.plugin ?? PLUGIN_NONE, + keyOpts, + ), + ); } const columnOrder = [...columnSet].sort(); - const baseline = args.baseline ?? columnOrder[0]; + // Honor the requested baseline only if it actually matches a column (a + // persisted baseline can go stale, e.g. after a key-format change); + // otherwise fall back to the first column. + const baseline = + args.baseline && columnSet.has(args.baseline) + ? args.baseline + : columnOrder[0]; const scenarios: ScenarioSummary[] = []; for (const [name, ps] of byScenario.entries()) { const byCol = new Map(); for (const p of ps) { - const key = columnKeyFor(p.run.mcp, p.run.model, multiModel); + const key = columnKeyFor( + p.run.mcp, + p.run.model, + p.run.plugin ?? PLUGIN_NONE, + keyOpts, + ); if (!byCol.has(key)) byCol.set(key, []); byCol.get(key)!.push(p); } const cells: Record = {}; for (const [col, list] of byCol.entries()) { const first = list[0].run; - cells[col] = buildCellSummary(name, first.mcp, first.model, col, list); + cells[col] = buildCellSummary( + name, + first.mcp, + first.model, + first.plugin ?? PLUGIN_NONE, + col, + list, + ); } // Compute deltas: each non-baseline column vs the baseline. @@ -153,6 +196,7 @@ export function buildAggregate(args: { baseline, columnOrder, multiModel, + multiPlugin, mcpOrder: columnOrder, scenarios, }; @@ -162,6 +206,7 @@ function buildCellSummary( scenario: string, mcp: McpKind, model: string, + plugin: string, columnKey: ColumnKey, pairs: GradedRunPair[], ): CellSummary { @@ -227,6 +272,7 @@ function buildCellSummary( scenario, mcp, model, + plugin, columnKey, n, programmatic: { mean: programmaticScore, perCheck }, diff --git a/packages/hdx-eval/src/reports/markdown.ts b/packages/hdx-eval/src/reports/markdown.ts index 4e0c86373b..34753f4f76 100644 --- a/packages/hdx-eval/src/reports/markdown.ts +++ b/packages/hdx-eval/src/reports/markdown.ts @@ -15,8 +15,12 @@ export function renderMarkdownReport(summary: BatchSummary): string { if (summary.baseline) { lines.push(`Baseline: ${summary.baseline}`); } - if (summary.multiModel) { - lines.push(`Columns: ${columns.join(', ')} _(mcp/model)_`); + const varying = [ + ...(summary.multiModel ? ['model'] : []), + ...(summary.multiPlugin ? ['plugin'] : []), + ]; + if (varying.length > 0) { + lines.push(`Columns: ${columns.join(', ')} _(mcp/${varying.join('+')})_`); } else { lines.push(`MCPs: ${columns.join(', ')}`); } diff --git a/packages/hdx-eval/src/reports/store.ts b/packages/hdx-eval/src/reports/store.ts index b45ed15dcf..5337920c56 100644 --- a/packages/hdx-eval/src/reports/store.ts +++ b/packages/hdx-eval/src/reports/store.ts @@ -2,64 +2,68 @@ import { existsSync, readFileSync, writeFileSync } from 'fs'; import { join } from 'path'; import type { GradeRecord } from '@/grading/types'; -import type { McpKind, RunRecord } from '@/harness/types'; -import { isModelSubdir, isRunJson, safeReaddir } from '@/runs/path'; +import type { RunRecord } from '@/harness/types'; +import { getRunFilesInBatch } from '@/runs/path'; import { SCENARIO_NAMES } from '@/scenarios'; -import { buildAggregate, type GradedRunPair } from './aggregate'; +import { + buildAggregate, + type ColumnKey, + type GradedRunPair, +} from './aggregate'; import { renderMarkdownReport } from './markdown'; -function collectRunGradePairs(dir: string, pairs: GradedRunPair[]): void { - for (const file of safeReaddir(dir)) { - if (isRunJson(file)) { - const runPath = join(dir, file); - const gradePath = runPath.replace(/\.json$/, '.grade.json'); - if (!existsSync(gradePath)) continue; - try { - const run = JSON.parse(readFileSync(runPath, 'utf8')) as RunRecord; - const grade = JSON.parse( - readFileSync(gradePath, 'utf8'), - ) as GradeRecord; - pairs.push({ run, grade }); - } catch { - // skip malformed - } - } - } -} - /** - * Load graded pairs from a batch. Supports both the new - * `///.json` layout and the legacy - * `//.json` layout. + * Load graded pairs from a batch: every run JSON (see `getRunFilesInBatch` for + * the supported layouts) that has a `.grade.json` sidecar next to it. */ function loadGradedPairs(batchDir: string): GradedRunPair[] { const pairs: GradedRunPair[] = []; - for (const scenario of safeReaddir(batchDir)) { - if (!SCENARIO_NAMES.includes(scenario)) continue; - const sceneDir = join(batchDir, scenario); - for (const mcp of safeReaddir(sceneDir)) { - const mcpDir = join(sceneDir, mcp); - // Collect runs directly in mcpDir (legacy layout). - collectRunGradePairs(mcpDir, pairs); - // Also check subdirectories (new model layout). - for (const entry of safeReaddir(mcpDir)) { - if (isModelSubdir(mcpDir, entry)) { - collectRunGradePairs(join(mcpDir, entry), pairs); - } - } + const runPaths = getRunFilesInBatch(batchDir, { + scenarioFilter: s => SCENARIO_NAMES.includes(s), + }); + for (const runPath of runPaths) { + const gradePath = runPath.replace(/\.json$/, '.grade.json'); + if (!existsSync(gradePath)) continue; + try { + const run = JSON.parse(readFileSync(runPath, 'utf8')) as RunRecord; + const grade = JSON.parse(readFileSync(gradePath, 'utf8')) as GradeRecord; + pairs.push({ run, grade }); + } catch { + // skip malformed } } return pairs; } +/** + * Baseline recorded by a previous report at the batch's canonical + * `_summary.json`, if any. Lets `report` regenerations keep the baseline the + * `run` auto-report chose (CLI variant order), which `buildAggregate` cannot + * reconstruct from the run data alone. + */ +function persistedBaseline(batchDir: string): ColumnKey | undefined { + try { + const raw = JSON.parse( + readFileSync(join(batchDir, '_summary.json'), 'utf8'), + ) as { baseline?: unknown }; + return typeof raw.baseline === 'string' ? raw.baseline : undefined; + } catch { + return undefined; + } +} + export function writeBatchSummary( batchDir: string, outPath: string, - baseline?: McpKind, + baseline?: ColumnKey, ): { jsonPath: string; mdPath: string; pairsCount: number } { const pairs = loadGradedPairs(batchDir); - const summary = buildAggregate({ batchDir, pairs, baseline }); + const summary = buildAggregate({ + batchDir, + pairs, + baseline: baseline ?? persistedBaseline(batchDir), + }); const jsonPath = outPath.replace(/\.md$/, '.json'); writeFileSync(jsonPath, JSON.stringify(summary, null, 2) + '\n', 'utf8'); diff --git a/packages/hdx-eval/src/runs/instrument.ts b/packages/hdx-eval/src/runs/instrument.ts index 63f088729a..c618ccff8d 100644 --- a/packages/hdx-eval/src/runs/instrument.ts +++ b/packages/hdx-eval/src/runs/instrument.ts @@ -7,7 +7,7 @@ import { scenarioTables } from '@/clickhouse/schema'; import type { RunRecord } from '@/harness/types'; import { SCENARIO_NAMES } from '@/scenarios'; -import { isModelSubdir, isRunJson, runsRoot, safeReaddir } from './path'; +import { getRunFilesInBatch, runsRoot } from './path'; const QUERY_LOG_BUFFER_MS = 5_000; @@ -283,34 +283,12 @@ export async function instrumentBatch( }); try { const out: TimingRecord[] = []; - for (const scenario of safeReaddir(resolved)) { - if (!SCENARIO_NAMES.includes(scenario)) continue; - const sceneDir = join(resolved, scenario); - for (const mcp of safeReaddir(sceneDir)) { - const mcpDir = join(sceneDir, mcp); - for (const entry of safeReaddir(mcpDir)) { - if (isRunJson(entry)) { - // Legacy layout: //.json - const timing = await instrumentRun({ - runPath: join(mcpDir, entry), - client, - }); - out.push(timing); - } else if (isModelSubdir(mcpDir, entry)) { - // New layout: ///.json - const modelDir = join(mcpDir, entry); - for (const file of safeReaddir(modelDir)) { - if (isRunJson(file)) { - const timing = await instrumentRun({ - runPath: join(modelDir, file), - client, - }); - out.push(timing); - } - } - } - } - } + const runPaths = getRunFilesInBatch(resolved, { + scenarioFilter: s => SCENARIO_NAMES.includes(s), + }); + for (const runPath of runPaths) { + const timing = await instrumentRun({ runPath, client }); + out.push(timing); } return out; } finally { diff --git a/packages/hdx-eval/src/runs/path.ts b/packages/hdx-eval/src/runs/path.ts index 0ff707118b..e64354788b 100644 --- a/packages/hdx-eval/src/runs/path.ts +++ b/packages/hdx-eval/src/runs/path.ts @@ -1,7 +1,7 @@ import { readdirSync, statSync } from 'fs'; import { resolve } from 'path'; -import type { McpKind } from '@/harness/types'; +import { type McpKind, PLUGIN_NONE } from '@/harness/types'; export function runsRoot(): string { return resolve(__dirname, '..', '..', 'runs'); @@ -12,21 +12,28 @@ export function batchDirName(date: Date = new Date()): string { } /** - * Sanitize a model ID for use as a directory name. Replaces characters - * that are problematic in file paths (`:`, `/`, `\`, `.`) with dashes. + * Sanitize a model ID or plugin name for use as a directory name. Replaces + * characters that are problematic in file paths (`:`, `/`, `\`, `.`) with + * dashes. * * Dots are replaced to avoid confusion with file extensions when listing * directory entries alongside run files like `0.json`. */ -export function modelDirName(model: string): string { - return model.replace(/[:/\\.]/g, '-'); +export function escapeDirSegment(segment: string): string { + return segment.replace(/[:/\\.]/g, '-'); } +/** + * Path of a run JSON file: `/////.json`. + * Model and plugin are treated consistently — each always gets a directory + * level, with the no-plugin arm stored under the literal `none` directory. + */ export function runFilePath(args: { batchDir: string; scenario: string; mcp: McpKind; model: string; + plugin?: string; runIndex: number; }): string { return resolve( @@ -34,14 +41,16 @@ export function runFilePath(args: { args.batchDir, args.scenario, args.mcp, - modelDirName(args.model), + escapeDirSegment(args.model), + escapeDirSegment(args.plugin || PLUGIN_NONE), `${args.runIndex}.json`, ); } // --------------------------------------------------------------------------- -// Shared filesystem helpers — used by store, instrument, grade, reports, and -// the viewer. Centralised here to avoid duplication. +// Batch walking — `getRunFilesInBatch` is the single walker used by store, +// instrument, grade, and reports. (The viewer keeps its own plain-JS copy +// because it needs cell grouping rather than a flat file list.) // --------------------------------------------------------------------------- /** @@ -50,7 +59,7 @@ export function runFilePath(args: { * `.timing.json`) and any other non-run JSON by requiring the `^\d+\.json$` * pattern. */ -export function isRunJson(file: string): boolean { +function isRunJson(file: string): boolean { return /^\d+\.json$/.test(file); } @@ -58,7 +67,7 @@ export function isRunJson(file: string): boolean { * Like `readdirSync` but returns `[]` on any error (missing dir, permission * issues, etc.). */ -export function safeReaddir(dirPath: string): string[] { +function safeReaddir(dirPath: string): string[] { try { return readdirSync(dirPath); } catch { @@ -67,15 +76,70 @@ export function safeReaddir(dirPath: string): string[] { } /** - * Returns true when `entry` inside `parentDir` is a subdirectory (i.e. a - * model directory in the new layout), as opposed to a file. Uses `stat` - * instead of a heuristic so model names containing dots are handled - * correctly. + * Returns true when `entry` inside `parentDir` is a subdirectory, as opposed + * to a file. Uses `stat` instead of a heuristic so model names containing + * dots are handled correctly. */ -export function isModelSubdir(parentDir: string, entry: string): boolean { +function isModelSubdir(parentDir: string, entry: string): boolean { try { return statSync(resolve(parentDir, entry)).isDirectory(); } catch { return false; } } + +/** + * Absolute paths of the run JSON files under a model directory: those one + * level deeper in per-plugin subdirectories (`//.json`, + * with the no-plugin arm under `/none/`) plus those directly in + * `modelDir` (legacy batches created before the plugin level was always + * emitted). + */ +function getRunFilesInModelDir(modelDir: string): string[] { + const out: string[] = []; + for (const entry of safeReaddir(modelDir)) { + if (isRunJson(entry)) { + // Legacy no-plugin arm: /.json + out.push(resolve(modelDir, entry)); + } else if (isModelSubdir(modelDir, entry)) { + // Plugin arm (incl. `none`): //.json + const pluginDir = resolve(modelDir, entry); + for (const file of safeReaddir(pluginDir)) { + if (isRunJson(file)) out.push(resolve(pluginDir, file)); + } + } + } + return out; +} + +/** + * Absolute paths of all run JSON files in a batch directory, sorted. Handles + * the current `////.json` layout (the + * no-plugin arm lives under `/none/`) plus two legacy layouts: runs + * directly in the model dir (pre-plugin-level) and directly in the mcp dir + * (pre-multi-model). Sidecar files (`.grade.json`, `.timing.json`) are + * excluded. Pass `scenarioFilter` to restrict which top-level scenario + * directories are walked. + */ +export function getRunFilesInBatch( + batchRoot: string, + opts: { scenarioFilter?: (name: string) => boolean } = {}, +): string[] { + const out: string[] = []; + for (const scenario of safeReaddir(batchRoot)) { + if (opts.scenarioFilter && !opts.scenarioFilter(scenario)) continue; + const sceneDir = resolve(batchRoot, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = resolve(sceneDir, mcp); + for (const entry of safeReaddir(mcpDir)) { + if (isRunJson(entry)) { + // Legacy layout: //.json + out.push(resolve(mcpDir, entry)); + } else if (isModelSubdir(mcpDir, entry)) { + out.push(...getRunFilesInModelDir(resolve(mcpDir, entry))); + } + } + } + } + return out.sort(); +} diff --git a/packages/hdx-eval/src/runs/store.ts b/packages/hdx-eval/src/runs/store.ts index fe95a4bd79..0034762a6c 100644 --- a/packages/hdx-eval/src/runs/store.ts +++ b/packages/hdx-eval/src/runs/store.ts @@ -3,13 +3,7 @@ import { dirname, join } from 'path'; import type { RunRecord } from '@/harness/types'; -import { - isModelSubdir, - isRunJson, - runFilePath, - runsRoot, - safeReaddir, -} from './path'; +import { getRunFilesInBatch, runFilePath, runsRoot } from './path'; export function writeRun(args: { record: RunRecord; @@ -20,6 +14,7 @@ export function writeRun(args: { scenario: args.record.scenario, mcp: args.record.mcp, model: args.record.model, + plugin: args.record.plugin, runIndex: args.record.runIndex, }); mkdirSync(dirname(path), { recursive: true }); @@ -39,34 +34,7 @@ export function listBatches(): string[] { } } -/** - * List run JSON files in a batch. Supports both the new - * `///.json` layout and the legacy - * `//.json` layout (for batches created before - * the multi-model feature). - */ +/** List run JSON files in a batch. See `getRunFilesInBatch` for the layouts. */ export function listRunsInBatch(batchDir: string): string[] { - const root = join(runsRoot(), batchDir); - const out: string[] = []; - for (const scenario of safeReaddir(root)) { - const sceneDir = join(root, scenario); - for (const mcp of safeReaddir(sceneDir)) { - const mcpDir = join(sceneDir, mcp); - for (const entry of safeReaddir(mcpDir)) { - if (isRunJson(entry)) { - // Legacy layout: //.json - out.push(join(mcpDir, entry)); - } else if (isModelSubdir(mcpDir, entry)) { - // New layout: ///.json - const modelDir = join(mcpDir, entry); - for (const file of safeReaddir(modelDir)) { - if (isRunJson(file)) { - out.push(join(modelDir, file)); - } - } - } - } - } - } - return out.sort(); + return getRunFilesInBatch(join(runsRoot(), batchDir)); } From bfa977ec1e21b4998167d3b35cb41f8a49d179a5 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 6 Jul 2026 09:37:52 -0600 Subject: [PATCH 154/728] chore(worktrees): use install instead of symlink (#2585) turbopack does not support symlinks for node_modules, so any new worktrunk worktrees will fail until you manually fix the problem link: https://nextjs.org/docs/app/api-reference/turbopack#filesystem-root --- .config/wt.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.config/wt.toml b/.config/wt.toml index 1ea3d91cce..0b668a6b8e 100644 --- a/.config/wt.toml +++ b/.config/wt.toml @@ -5,11 +5,9 @@ # hooks that run when creating/switching/removing worktrees via `wt`. # --------------------------------------------------------------------------- -# pre-start: blocking — symlink node_modules from primary worktree for -# instant dependency access. If the branch needs different deps: -# rm node_modules && yarn install +# pre-start: install dependencies so app is ready to run # --------------------------------------------------------------------------- -pre-start = "ln -sf {{ primary_worktree_path }}/node_modules {{ worktree_path }}/node_modules" +pre-start = "yarn install" # --------------------------------------------------------------------------- # post-start: background tasks From 143010cddf53f1ac3d71a43941d4af014099b72f Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 6 Jul 2026 09:48:57 -0600 Subject: [PATCH 155/728] docs: add eval framework guide to agent_docs (#2581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds `agent_docs/evals.md` — a comprehensive guide for running MCP evals, focused on the dual-slot A/B comparison workflow. ## Why The eval framework has grown significantly (6 scenarios, dual-slot architecture, hook-based dashboard evals) but the only documentation lived in `packages/hdx-eval/README.md`, which covers the CLI reference and config schema. There was no high-level guide explaining the end-to-end workflow for comparing branches, common pitfalls, or alternative use cases. ## Changes - **`agent_docs/evals.md`** (new) — covers: - Architecture diagram (eval harness → two HyperDX instances) - Step-by-step dual-slot A/B setup - Recommended flags table - Alternative workflows (single MCP, model comparison, non-HyperDX MCP) - Reading and inspecting results - Common pitfalls (port binding, stale keys, timeouts, data loss) - Cleanup - **`AGENTS.md`** — adds cross-reference to the new doc from the hdx-eval package description - **`agent_docs/README.md`** — adds `evals.md` to the directory index - **`packages/hdx-eval/README.md`** — expands the `yarn dev` prereq description and aligns the example slot number with the new docs --- AGENTS.md | 4 +- agent_docs/README.md | 1 + agent_docs/evals.md | 372 ++++++++++++++++++++++++++++++++++++ packages/hdx-eval/README.md | 6 +- 4 files changed, 380 insertions(+), 3 deletions(-) create mode 100644 agent_docs/evals.md diff --git a/AGENTS.md b/AGENTS.md index 10bc1b1b7a..4222c86a6d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,9 @@ This is a **monorepo** with six packages: - `packages/hdx-eval` - AI eval framework for benchmarking MCP servers against observability scenarios. Generates deterministic synthetic telemetry, spawns agents, and grades with programmatic checks + LLM-as-judge. See its - [`README.md`](packages/hdx-eval/README.md) for setup and usage. + [`README.md`](packages/hdx-eval/README.md) for setup and usage, and + [`agent_docs/evals.md`](agent_docs/evals.md) for the dual-slot A/B + comparison workflow. **Data flow**: Apps → OpenTelemetry Collector → ClickHouse (telemetry data) / MongoDB (configuration/metadata) diff --git a/agent_docs/README.md b/agent_docs/README.md index fd273f61db..3a1f5444ea 100644 --- a/agent_docs/README.md +++ b/agent_docs/README.md @@ -19,6 +19,7 @@ Instead of stuffing all instructions into `AGENTS.md` (which goes into every con - **`page_layout.md`** - PageHeader, PageLayout, and consistent page chrome (titles, actions, tool pages) - **`data_viz_colors.md`** - Chart, heatmap, and semantic status colors. Read before adding or changing any color in a chart, sparkline, heatmap, legend, or status pill. - **`themes.md`** - How the brand theme system (HyperDX vs ClickStack) and color mode (light/dark/system) work. Read before changing anything in `packages/app/src/theme/`, adding semantic CSS variables, or touching brand-conditional UI. +- **`evals.md`** - MCP eval framework: dual-slot setup, running A/B comparisons between branches, interpreting results. Read before running evals or benchmarking MCP changes. ## Usage Pattern diff --git a/agent_docs/evals.md b/agent_docs/evals.md new file mode 100644 index 0000000000..ff918e3df4 --- /dev/null +++ b/agent_docs/evals.md @@ -0,0 +1,372 @@ +# MCP Eval Framework + +Read this before running evals, comparing MCP branches, or investigating eval +results. For the full CLI reference, config schema, scenario details, and +scoring formula, see [`packages/hdx-eval/README.md`](../packages/hdx-eval/README.md). + +## What It Does + +The eval framework spawns Claude Code as an SRE agent, gives it access to an +MCP server, and asks it to solve observability scenarios (find a root cause, +build a dashboard, etc.) against synthetic telemetry seeded into ClickHouse. +Answers are graded with programmatic regex checks (40%) + an LLM judge (60%), +minus a penalty for tool-call errors. The framework is MCP-agnostic — you can +compare any two (or more) MCP servers side by side. + +### Available scenarios + +| Scenario | Type | What It Tests | +|---|---|---| +| `error-root-cause` | Investigation | Find a cascading DB timeout causing checkout failures | +| `latency-spike` | Investigation | Identify a segmented p99 spike for enterprise tenants | +| `noisy-signals` | Investigation | Find load-bearing log patterns among noise | +| `segmented-regression` | Investigation | Detect a regression hidden by single-axis aggregation | +| `service-health-check` | Investigation | Generate a peace-time health report | +| `dashboard-build` | Dashboard creation | Build monitoring dashboards with diverse tile types | + +## Prerequisites + +| Requirement | Notes | +|---|---| +| `yarn dev` running | Each worktree needs `yarn dev` running with a fixed slot. This handles `yarn install`, `yarn build:common-utils`, Docker containers (ClickHouse, MongoDB), and the API server. See [Development Setup](development.md). | +| `ANTHROPIC_API_KEY` or `AI_API_KEY` | Auto-loaded from `.env.local` at the monorepo root. Check `.env.local` in other worktrees for existing keys if missing. | +| `claude` CLI | Installed globally (`which claude`). The harness spawns it in streaming JSON mode. | +| `uv` | Only needed for `stdio`-type MCPs (e.g. raw ClickHouse MCP). Install via `curl -LsSf https://astral.sh/uv/install.sh \| sh`. | +| Two git worktrees | One checked out to the branch under test, one to `main`. Use `git worktree list` to verify. | + +## Dual-Slot A/B Comparison (Primary Workflow) + +This is the standard workflow for comparing a feature branch against main. + +### Architecture + +Evals run from **one worktree** (the branch being tested). The eval harness +talks to **two separate HyperDX API instances** running on different slots: + +``` + ┌──────────────────────┐ + │ Eval Harness │ + │ (this worktree) │ + │ │ + │ eval.config.json │ + │ ┌────────┬────────┐ │ + │ │branch │ main │ │ + │ │MCP │ MCP │ │ + │ └───┬────┴───┬────┘ │ + └──────┼────────┼──────┘ + │ │ + ┌────────────────┘ └────────────────┐ + ▼ ▼ + ┌─────────────────────┐ ┌─────────────────────┐ + │ Slot 98 (main) │ │ Slot 99 (branch) │ + │ API :30198 │ │ API :30199 │ + │ CH :30598 │ │ CH :30599 │ + │ Mongo:30498 │ │ Mongo:30499 │ + │ MCP :30198/mcp │ │ MCP :30199/mcp │ + └─────────────────────┘ └─────────────────────┘ + main worktree branch worktree +``` + +Each slot runs its own ClickHouse instance. `setup-hyperdx` (step 2) creates +the HyperDX account, Connection, and Sources on each instance, but does **not** +seed telemetry data. The `run` command auto-seeds on first run, but only into +the single ClickHouse specified in the top-level `clickhouse` config — so for +dual-slot comparisons you must seed each instance explicitly (step 4b). + +### 1. Start both dev stacks + +`yarn dev` starts Docker infra (ClickHouse, MongoDB, OTel Collector) **and** +the API + App servers in one command. Run it from each worktree with a fixed +slot: + +```bash +# From the main worktree (slot 98) +HDX_DEV_SLOT=98 yarn dev + +# From the branch worktree (slot 99) +HDX_DEV_SLOT=99 yarn dev +``` + +Each command prints a banner with its port assignments. Wait for both to +finish starting, then verify: + +```bash +curl -s -o /dev/null -w "Slot 98 API: %{http_code}\n" http://localhost:30198/api/v1/me +curl -s -o /dev/null -w "Slot 99 API: %{http_code}\n" http://localhost:30199/api/v1/me +# Expected: 404 (no auth token — but the server is responding) + +curl -s -o /dev/null -w "Slot 98 CH: %{http_code}\n" http://localhost:30598/ +curl -s -o /dev/null -w "Slot 99 CH: %{http_code}\n" http://localhost:30599/ +# Expected: 200 +``` + +### 2. Run `setup-hyperdx` against each instance (if needed) + +> Run these commands from the worktree that has the eval framework code (the +> branch under test). + +This registers an eval account, creates a ClickHouse Connection, and creates +per-scenario Sources on each HyperDX instance. The command is **idempotent** — +it skips anything that already exists, so it's safe to run every time. + +**Skip this step** if `eval.config.branch.json` and `eval.config.main.json` +already exist and pass the check: + +```bash +# Quick check: do saved configs exist and are the MCPs reachable? +# NOTE: These cp commands temporarily overwrite eval.config.json with a +# single-instance config. If you already have a combined two-MCP config, +# you must re-run step 3 (jq merge) afterward. +cp packages/hdx-eval/eval.config.branch.json packages/hdx-eval/eval.config.json 2>/dev/null && \ + yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --check && \ + echo "Branch config OK" || echo "Branch config needs setup" + +cp packages/hdx-eval/eval.config.main.json packages/hdx-eval/eval.config.json 2>/dev/null && \ + yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --check && \ + echo "Main config OK" || echo "Main config needs setup" +``` + +If either needs setup (or this is the first time): + +```bash +# Setup main instance (slot 98) +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --api-url http://localhost:30198 +cp packages/hdx-eval/eval.config.json packages/hdx-eval/eval.config.main.json + +# Setup branch instance (slot 99) +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --api-url http://localhost:30199 +cp packages/hdx-eval/eval.config.json packages/hdx-eval/eval.config.branch.json +``` + +### 3. Build the combined eval config + +The eval runner needs a single `eval.config.json` with both MCPs so it can +compare them in one batch. Use `jq` to merge: + +```bash +cat packages/hdx-eval/eval.config.branch.json | jq \ + --slurpfile main packages/hdx-eval/eval.config.main.json ' + # Helper: rewrite mcp__hyperdx__ prefixes in toolPattern and deniedTools + def reprefix($new): + .toolPattern = ("mcp__" + $new + "__*") + | if .deniedTools then + .deniedTools = [.deniedTools[] | gsub("^mcp__hyperdx__"; "mcp__" + $new + "__")] + else . end; + + . as $branch | + .mcps = { + "hyperdx-branch": ($branch.mcps.hyperdx + {"label": "HyperDX Branch", "enabled": true} | reprefix("hyperdx-branch")), + "hyperdx-main": ($main[0].mcps.hyperdx + {"label": "HyperDX Main", "enabled": true} | reprefix("hyperdx-main")) + } | + .hyperdxApi = $branch.hyperdxApi +' > packages/hdx-eval/eval.config.json +``` + +> **Why the `reprefix` step?** The harness uses the config key (e.g. +> `hyperdx-branch`) as the MCP server name, so Claude Code registers tools as +> `mcp__hyperdx-branch__*`. The `toolPattern` and `deniedTools` from the +> original single-MCP config still say `mcp__hyperdx__*`, which won't match. +> The `reprefix` helper rewrites both to use the new key. + +Verify the config has both MCPs: + +```bash +jq '.mcps | keys' packages/hdx-eval/eval.config.json +# Expected: ["hyperdx-branch", "hyperdx-main"] +``` + +### 4. Verify MCP connectivity + +```bash +BRANCH_KEY=$(jq -r '.hyperdxApi.accessKey' packages/hdx-eval/eval.config.branch.json) +MAIN_KEY=$(jq -r '.mcps["hyperdx-main"].headers.Authorization' packages/hdx-eval/eval.config.json | sed 's/Bearer //') +curl -s -o /dev/null -w "Branch MCP: %{http_code}\n" \ + -H "Authorization: Bearer $BRANCH_KEY" http://localhost:30199/mcp +curl -s -o /dev/null -w "Main MCP: %{http_code}\n" \ + -H "Authorization: Bearer $MAIN_KEY" http://localhost:30198/mcp +# Expected: 406 (SSE endpoint, wrong HTTP method — but proves auth works) +``` + +### 5. Seed both ClickHouse instances + +`setup-hyperdx` creates the HyperDX account and Sources but does **not** seed +telemetry data. The `run` command's auto-seed only writes to the single +ClickHouse in the top-level `clickhouse` config, so you must seed each +instance explicitly: + +```bash +# Seed the main instance (slot 98, CH on port 30598) +HDX_DEV_SLOT=98 yarn workspace @hyperdx/hdx-eval dev seed + +# Seed the branch instance (slot 99, CH on port 30599) +HDX_DEV_SLOT=99 yarn workspace @hyperdx/hdx-eval dev seed +``` + +The seed command is idempotent — it skips if data already exists. Replace +`` with the scenario name (e.g. `error-root-cause`). You can seed +multiple scenarios by running the command once per scenario. + +### 6. Run the comparison + +```bash +yarn workspace @hyperdx/hdx-eval dev run \ + --mcp hyperdx-branch,hyperdx-main \ + --baseline hyperdx-main \ + --runs 3 \ + --max-turns 15 \ + --timeout 600000 \ + --judge-model claude-opus-4-7 +``` + +The eval harness automatically loads `ANTHROPIC_API_KEY` (or `AI_API_KEY`) +from `.env.local` at the monorepo root via `dotenvx`. If the key isn't there, +either add it to `.env.local` or export it in your shell. Check `.env.local` +files in other worktrees or `~/sites/hyperdx/.env.local` for existing keys. + +For single-MCP runs, the harness auto-seeds on first run if the scenario +tables are empty. For dual-slot A/B runs, seed both instances explicitly +(step 5 above). Subsequent runs reuse existing data and the saved anchor time. + +### Recommended flags + +| Flag | Recommended Value | Why | +|---|---|---| +| `--timeout` | `600000` (10 min) | The default 5 min causes false timeouts on complex scenarios like `dashboard-build`. | +| `--concurrency` | omit (default 1) | Concurrent runs cause resource contention and false timeouts. Run sequentially. | +| `--runs` | `3` minimum | Eval results have high variance. 3 runs is the minimum for meaningful comparison. | +| `--baseline` | `hyperdx-main` | Sets main as the reference for delta computation in reports. | +| `--judge-model` | `claude-opus-4-7` | Default. Use a strong model for judging. | +| `--no-judge` | use for fast iteration | Skips the LLM judge (saves cost/time). Programmatic scores only. | + +## Other Use Cases + +The dual-slot A/B comparison is the primary workflow, but the eval framework +supports other configurations with a single dev stack. + +### Quick score check (single MCP) + +Run one MCP against a scenario to get a baseline score without comparison. +Useful for validating that a scenario works or checking score after a change. + +```bash +# Single slot — just one `yarn dev` needed +export HDX_DEV_SLOT=98 +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx +yarn workspace @hyperdx/hdx-eval dev run error-root-cause --mcp hyperdx --runs 1 --no-judge +``` + +### Model comparison (same MCP, different models) + +Compare how different models perform on the same MCP and data. Useful for +evaluating model upgrades or cost/quality tradeoffs. + +```bash +yarn workspace @hyperdx/hdx-eval dev run error-root-cause \ + --mcp hyperdx --model claude-sonnet-4-20250514 --runs 3 +yarn workspace @hyperdx/hdx-eval dev run error-root-cause \ + --mcp hyperdx --model claude-opus-4-6 --runs 3 +``` + +### Non-HyperDX MCP (e.g. raw ClickHouse) + +The framework works with any MCP. The `clickhouse` stdio MCP is included by +default in `eval.config.json` after setup. Use it to compare HyperDX's MCP +against querying ClickHouse directly. + +```bash +yarn workspace @hyperdx/hdx-eval dev run error-root-cause \ + --mcp hyperdx,clickhouse --baseline clickhouse --runs 3 +``` + +## Reading Results + +Results are written to `packages/hdx-eval/runs//`. Each batch +contains: + +``` +_summary.md # Markdown comparison report +_summary.json # Structured data for programmatic use +/ + / + / + 0.json # Full run trajectory (tool calls, tokens, timing) + 0.grade.json # Grade record (programmatic + judge scores) +``` + +Useful inspection commands: + +```bash +# List all batches +yarn workspace @hyperdx/hdx-eval dev runs-list + +# Show a specific run's tool calls and queries +yarn workspace @hyperdx/hdx-eval dev runs-show --queries + +# Show the agent's final answer +yarn workspace @hyperdx/hdx-eval dev runs-show --final-answer + +# Re-grade with only programmatic checks (fast, no API key needed) +yarn workspace @hyperdx/hdx-eval dev grade --no-judge + +# Re-generate the comparison report with a different baseline +yarn workspace @hyperdx/hdx-eval dev report --baseline hyperdx-main + +# Browse results in the web viewer +yarn workspace @hyperdx/hdx-eval viewer +``` + +## Common Pitfalls + +### Containers bind to wrong ports + +If you run `docker compose up` without sourcing `scripts/dev-env.sh` first, +containers bind to default ports (8123, 27017) instead of slot-specific ones. +Always use the `bash -c 'export HDX_DEV_SLOT=... && source scripts/dev-env.sh +&& docker compose ...'` pattern (see [Cleanup](#cleanup) below). + +### Access keys go stale after container restarts + +When MongoDB is recreated, the eval account's access key changes. Re-run +`setup-hyperdx` against the affected instance and rebuild the combined config. + +### False timeouts + +Two common causes: +1. **Default 5-minute timeout is too short** for complex scenarios. Use + `--timeout 600000` (10 minutes). +2. **`--concurrency > 1`** causes resource contention between parallel Claude + processes. Run sequentially (omit `--concurrency`). + +### `eval.config.json` is gitignored + +The eval config files (`eval.config.json`, `eval.config.*.json`) are +gitignored because they contain instance-specific access keys and Source IDs. +They must be regenerated per environment using `setup-hyperdx`. + +### ClickHouse data missing after container recreation + +If Docker volumes are removed (e.g. `docker compose down -v`), seeded data is +lost. The eval harness auto-seeds on the next run if it detects empty tables, +but this adds time. Use `--reseed` explicitly to force a fresh seed. + +### Dev stack not starting + +If `yarn dev` fails: +- Port already in use — another stack on the same slot is running. Check with + `lsof -i :301XX` and kill stale processes. +- Docker not running — start Docker Desktop or OrbStack first. +- Missing dependencies — run `yarn setup` from the repo root. + +## Cleanup + +```bash +# If you started with `yarn dev`, Ctrl-C in each terminal stops everything. +# If you started Docker separately: +bash -c 'export HDX_DEV_SLOT=98 && source scripts/dev-env.sh 2>/dev/null && \ + docker compose -p hdx-dev-98 -f docker-compose.dev.yml down' +bash -c 'export HDX_DEV_SLOT=99 && source scripts/dev-env.sh 2>/dev/null && \ + docker compose -p hdx-dev-99 -f docker-compose.dev.yml down' + +# Drop eval data from ClickHouse (optional) +yarn workspace @hyperdx/hdx-eval dev drop +``` diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index 35a4291552..99ee9d0d70 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -11,7 +11,9 @@ selected at run time. ## Prerequisites -- **`yarn dev` running** on at least one worktree (ClickHouse, MongoDB, API) +- **`yarn dev` running** on at least one worktree — this handles + `yarn install`, `yarn build:common-utils`, Docker containers (ClickHouse, + MongoDB), and the API server. See `agent_docs/development.md` for setup. - **`claude` CLI installed** (the harness spawns it in streaming JSON mode) - **`ANTHROPIC_API_KEY`** or **`AI_API_KEY`** in `.env.local` at the monorepo root (for `run` and `grade` commands) @@ -28,7 +30,7 @@ stack is using — you can find this in the banner printed by `yarn dev`. ```bash # Set the slot for your session (must match the running dev stack) -export HDX_DEV_SLOT=80 +export HDX_DEV_SLOT=98 # 1. One-time setup: register eval account + create Sources + write config yarn workspace @hyperdx/hdx-eval dev setup-hyperdx From 73e6e876e6246b775a8af4d8d6c6262ef7f479eb Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:09:49 +0200 Subject: [PATCH 156/728] feat: expose window alerting on the external API (#2595) ## Summary Follow up to https://github.com/hyperdxio/hyperdx/pull/2472 to expose the `numConsecutiveWindows` field on the external API. ### How to test on Vercel preview **Preview routes:** N/A **Steps:** 1. Test via external API ### References - Related PRs: https://github.com/hyperdxio/hyperdx/pull/2472 --- .../external-api-consecutive-window-alerts.md | 5 + packages/api/openapi.json | 14 ++- .../external-api/__tests__/alerts.test.ts | 101 ++++++++++++++++++ .../api/src/routers/external-api/v2/alerts.ts | 10 +- .../src/utils/__tests__/externalApi.test.ts | 26 +++++ packages/api/src/utils/externalApi.ts | 2 + 6 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 .changeset/external-api-consecutive-window-alerts.md diff --git a/.changeset/external-api-consecutive-window-alerts.md b/.changeset/external-api-consecutive-window-alerts.md new file mode 100644 index 0000000000..b67210a79c --- /dev/null +++ b/.changeset/external-api-consecutive-window-alerts.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +Expose consecutive-window alerting on the external API. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 18734b646d..7d938b5c14 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -102,7 +102,8 @@ "ALERT", "OK", "INSUFFICIENT_DATA", - "DISABLED" + "DISABLED", + "PENDING" ], "description": "Current alert state." }, @@ -292,6 +293,13 @@ "minLength": 1, "maxLength": 4096, "example": "Threshold raised from 50 to 100 on 2026-01-15. See [runbook](https://wiki.example.com/runbook)." + }, + "numConsecutiveWindows": { + "type": "integer", + "minimum": 1, + "nullable": true, + "description": "Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state.", + "example": 3 } } }, @@ -4293,6 +4301,7 @@ "teamId": "65f5e4a3b9e77c001a345678", "tileId": "65f5e4a3b9e77c001a901234", "dashboardId": "65f5e4a3b9e77c001a567890", + "numConsecutiveWindows": 3, "createdAt": "2023-03-15T10:20:30.000Z", "updatedAt": "2023-03-15T14:25:10.000Z" } @@ -4558,7 +4567,8 @@ "webhookId": "65f5e4a3b9e77c001a789012" }, "name": "Error Spike Alert", - "message": "Error rate has exceeded 100 in the last hour" + "message": "Error rate has exceeded 100 in the last hour", + "numConsecutiveWindows": 3 } } } diff --git a/packages/api/src/routers/external-api/__tests__/alerts.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.test.ts index 3eb8856e8d..02669737b6 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.test.ts @@ -223,6 +223,7 @@ describe('External API Alerts', () => { name: 'Format Test Alert', message: 'This is a test alert for format verification', note: null, + numConsecutiveWindows: null, threshold: 123, interval: '15m', source: AlertSource.TILE, @@ -1232,4 +1233,104 @@ describe('External API Alerts', () => { expect(match.executionErrors[0].message).toBe('webhook delivery failed'); }); }); + + describe('Consecutive-window alerting (numConsecutiveWindows)', () => { + it('should create an alert with numConsecutiveWindows and return it', async () => { + const { alert } = await createTestAlert({ numConsecutiveWindows: 3 }); + + expect(alert.numConsecutiveWindows).toBe(3); + }); + + it('should default numConsecutiveWindows to null when not provided', async () => { + const { alert } = await createTestAlert(); + + expect(alert.numConsecutiveWindows).toBeNull(); + }); + + it('should reject numConsecutiveWindows below 1', async () => { + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 100, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.ABOVE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + numConsecutiveWindows: 0, + }) + .expect(400); + + consoleErrorSpy.mockRestore(); + }); + + it('should return numConsecutiveWindows on GET by id', async () => { + const { alert } = await createTestAlert({ numConsecutiveWindows: 5 }); + + const getResponse = await authRequest( + 'get', + `${ALERTS_BASE_URL}/${alert.id}`, + ).expect(200); + + expect(getResponse.body.data.numConsecutiveWindows).toBe(5); + }); + + it('should return numConsecutiveWindows on the list endpoint', async () => { + const { alert } = await createTestAlert({ numConsecutiveWindows: 4 }); + + const listResponse = await authRequest('get', ALERTS_BASE_URL).expect( + 200, + ); + + const match = listResponse.body.data.find((a: any) => a.id === alert.id); + expect(match).toBeDefined(); + expect(match.numConsecutiveWindows).toBe(4); + }); + + it('should update an alert to set numConsecutiveWindows', async () => { + const { alert, alertInput } = await createTestAlert(); + + expect(alert.numConsecutiveWindows).toBeNull(); + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send({ ...alertInput, numConsecutiveWindows: 2 }) + .expect(200); + + const getResponse = await authRequest( + 'get', + `${ALERTS_BASE_URL}/${alert.id}`, + ).expect(200); + + expect(getResponse.body.data.numConsecutiveWindows).toBe(2); + }); + + it('should update an alert to clear numConsecutiveWindows (null)', async () => { + const { alert, alertInput } = await createTestAlert({ + numConsecutiveWindows: 3, + }); + + expect(alert.numConsecutiveWindows).toBe(3); + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send({ ...alertInput, numConsecutiveWindows: null }) + .expect(200); + + const getResponse = await authRequest( + 'get', + `${ALERTS_BASE_URL}/${alert.id}`, + ).expect(200); + + expect(getResponse.body.data.numConsecutiveWindows).toBeNull(); + }); + }); }); diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index 68cf2e67c1..0aab0457ee 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -42,7 +42,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * description: Alert source type. * AlertState: * type: string - * enum: [ALERT, OK, INSUFFICIENT_DATA, DISABLED] + * enum: [ALERT, OK, INSUFFICIENT_DATA, DISABLED, PENDING] * description: Current alert state. * AlertChannelType: * type: string @@ -186,6 +186,12 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * minLength: 1 * maxLength: 4096 * example: "Threshold raised from 50 to 100 on 2026-01-15. See [runbook](https://wiki.example.com/runbook)." + * numConsecutiveWindows: + * type: integer + * minimum: 1 + * nullable: true + * description: Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state. + * example: 3 * * AlertResponse: * allOf: @@ -310,6 +316,7 @@ const router = express.Router(); * teamId: "65f5e4a3b9e77c001a345678" * tileId: "65f5e4a3b9e77c001a901234" * dashboardId: "65f5e4a3b9e77c001a567890" + * numConsecutiveWindows: 3 * createdAt: "2023-03-15T10:20:30.000Z" * updatedAt: "2023-03-15T14:25:10.000Z" * '401': @@ -443,6 +450,7 @@ router.get('/', async (req, res, next) => { * webhookId: "65f5e4a3b9e77c001a789012" * name: "Error Spike Alert" * message: "Error rate has exceeded 100 in the last hour" + * numConsecutiveWindows: 3 * responses: * '200': * description: Successfully created alert diff --git a/packages/api/src/utils/__tests__/externalApi.test.ts b/packages/api/src/utils/__tests__/externalApi.test.ts index d64cb40480..499ec69a52 100644 --- a/packages/api/src/utils/__tests__/externalApi.test.ts +++ b/packages/api/src/utils/__tests__/externalApi.test.ts @@ -71,4 +71,30 @@ describe('utils/externalApi', () => { expect(translated.note).toBe('threshold raised to 100'); }); }); + + describe('numConsecutiveWindows handling', () => { + it('returns numConsecutiveWindows as null when the value is null', () => { + const alert = createAlertDocument({ numConsecutiveWindows: null }); + + const translated = translateAlertDocumentToExternalAlert(alert); + + expect(translated.numConsecutiveWindows).toBeNull(); + }); + + it('returns numConsecutiveWindows as null when the value is undefined', () => { + const alert = createAlertDocument({ numConsecutiveWindows: undefined }); + + const translated = translateAlertDocumentToExternalAlert(alert); + + expect(translated.numConsecutiveWindows).toBeNull(); + }); + + it('returns numConsecutiveWindows when the value is a positive integer', () => { + const alert = createAlertDocument({ numConsecutiveWindows: 3 }); + + const translated = translateAlertDocumentToExternalAlert(alert); + + expect(translated.numConsecutiveWindows).toBe(3); + }); + }); }); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 1c6debccd4..c25e76cbcc 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -237,6 +237,7 @@ export type ExternalAlert = { interval: AlertInterval; scheduleOffsetMinutes?: number; scheduleStartAt?: string | null; + numConsecutiveWindows?: number | null; thresholdType: AlertThresholdType; source?: string; state: AlertState; @@ -338,6 +339,7 @@ export function translateAlertDocumentToExternalAlert( scheduleOffsetMinutes: alertObj.scheduleOffsetMinutes, }), scheduleStartAt: transformScheduleStartAt(alertObj.scheduleStartAt), + numConsecutiveWindows: alertObj.numConsecutiveWindows ?? null, thresholdType: alertObj.thresholdType, source: alertObj.source, state: alertObj.state, From 81e2b3022e20b661f3bf185c245c85ec739c7b5d Mon Sep 17 00:00:00 2001 From: Elizabet Oliveira Date: Tue, 7 Jul 2026 10:49:33 +0100 Subject: [PATCH 157/728] fix(theme): use Mantine's --button-hover-color for primary buttons (#2598) Co-authored-by: Cursor --- .changeset/fix-primary-button-hover-color.md | 8 ++++++++ packages/app/src/theme/themes/clickstack/mantineTheme.ts | 2 +- packages/app/src/theme/themes/hyperdx/mantineTheme.ts | 2 +- packages/app/styles/variants.module.scss | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 .changeset/fix-primary-button-hover-color.md diff --git a/.changeset/fix-primary-button-hover-color.md b/.changeset/fix-primary-button-hover-color.md new file mode 100644 index 0000000000..71b00c6c5e --- /dev/null +++ b/.changeset/fix-primary-button-hover-color.md @@ -0,0 +1,8 @@ +--- +'@hyperdx/app': patch +--- + +Fix primary button hover text color by using Mantine's `--button-hover-color` +variable (the theme previously set the non-existent `--button-color-hover`, so +the hover text color was never applied and could fall through to an inherited +page color). diff --git a/packages/app/src/theme/themes/clickstack/mantineTheme.ts b/packages/app/src/theme/themes/clickstack/mantineTheme.ts index f6b59ab841..a67d79b32a 100644 --- a/packages/app/src/theme/themes/clickstack/mantineTheme.ts +++ b/packages/app/src/theme/themes/clickstack/mantineTheme.ts @@ -251,7 +251,7 @@ const makeTheme = ({ baseVars['--button-bg'] = 'var(--color-primary-button-bg)'; baseVars['--button-hover'] = 'var(--color-primary-button-bg-hover)'; baseVars['--button-color'] = 'var(--color-primary-button-text)'; - baseVars['--button-color-hover'] = 'var(--color-primary-button-text)'; + baseVars['--button-hover-color'] = 'var(--color-primary-button-text)'; } if (props.variant === 'secondary') { diff --git a/packages/app/src/theme/themes/hyperdx/mantineTheme.ts b/packages/app/src/theme/themes/hyperdx/mantineTheme.ts index c9f77b8d00..7528244074 100644 --- a/packages/app/src/theme/themes/hyperdx/mantineTheme.ts +++ b/packages/app/src/theme/themes/hyperdx/mantineTheme.ts @@ -264,7 +264,7 @@ const makeTheme = ({ baseVars['--button-bg'] = 'var(--color-primary-button-bg)'; baseVars['--button-hover'] = 'var(--color-primary-button-bg-hover)'; baseVars['--button-color'] = 'var(--color-primary-button-text)'; - baseVars['--button-color-hover'] = 'var(--color-primary-button-text)'; + baseVars['--button-hover-color'] = 'var(--color-primary-button-text)'; } if (props.variant === 'secondary') { diff --git a/packages/app/styles/variants.module.scss b/packages/app/styles/variants.module.scss index 1ed9c0fc5e..3e86d97c42 100644 --- a/packages/app/styles/variants.module.scss +++ b/packages/app/styles/variants.module.scss @@ -1,6 +1,6 @@ // Hover: brighten text to full contrast. // Both Button and ActionIcon use SCSS for hover color because -// Mantine only provides --button-color-hover (no ActionIcon equivalent). +// Mantine only provides --button-hover-color (no ActionIcon equivalent). // Using SCSS for both keeps the approach consistent. .buttonLink:hover:not(:disabled, [data-disabled]) { --button-color: var(--color-text); From f6dbdd149102131b4ca24556d1c98f82fb5326de Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:23:58 +0200 Subject: [PATCH 158/728] feat: single drawer navigation (#2541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Replaces the old stacked/layered side-panel drawers with a **single right-hand drawer** where all event navigation happens in-place. Everything now renders inside one shell instead of opening a second drawer on top of another. - **Single-drawer navigation.** `DBRowSidePanel` is refactored into `DBRowSidePanelInner`, which owns two URL-persisted stacks: `sidePanelNavStack` for same-source drilldowns (e.g. Surrounding Context, replaced in-place) and `sidePanelSourceStack` for cross-source navigation (log → trace via the new **View Trace** action, or session → event). Each frame stores its source id, row id, label, and originating tab. State lives in the URL (`sidePanelTab`, `sidePanelSourceStack`, `sidePanelNavStack`, `sessionPanelEvent`), so the trail survives reload and shared links. - **Unified breadcrumbs (`SidePanelBreadcrumbs`).** New shared component used across log, trace, and session panels: source-kind icons (`IconLogs`/`IconConnection`/`IconDeviceLaptop`), label truncation with tooltips, and a clickable trail to jump back to any ancestor. - **Session panel alignment.** `SessionSidePanel` embeds `DBRowSidePanelInner` for the selected event instead of portaling a second `DBRowSidePanel`, so session → event → trace shares the same structure and breadcrumbs. `SessionSubpanel`/`DBSessionPanel` report selections via `onEventNavigate`. - **Header redesign.** Metadata row (timestamp, service, duration, status, span kind), **Copy Trace ID**, and **View Trace** for logs with trace context. View Trace / Trace ID are hidden when there's no trace context. Shared `SidePanelHeaderActions` for shortcuts/full-width/share/close; `DBRowSidePanelHeader` slimmed to the body section. - **Navigation fixes.** Session Close (X) closes the whole panel; back/Esc pops one level. Returning from a trace restores the prior tab; opening a different event resets the tab and clears stale stacks; pushes jump to the destination's default tab. - **Background scroll.** Removes the side panel overlay on search and sessions pages so that the background lists are scrollable and the user can click to select an alternative log/trace/session. ### Screenshots or video https://github.com/user-attachments/assets/bae94470-b697-43a4-b48b-d0c555a869ed ### How to test on Vercel preview **Preview routes:** `/search`, `/sessions` Navigate around the side panel drawer on both the search and sessions pages. ### References - **Linear Issue**: Closes HDX-3942, HDX-3946, HDX-3947, HDX-3948, HDX-3949 - **Prototype PR**: https://github.com/hyperdxio/hyperdx/pull/1970 --- .changeset/single-drawer-navigation.md | 5 + .../app/src/NamespaceDetailsSidePanel.tsx | 2 - packages/app/src/NodeDetailsSidePanel.tsx | 6 +- packages/app/src/PodDetailsSidePanel.tsx | 3 - packages/app/src/SessionSidePanel.tsx | 286 +++++-- packages/app/src/SessionSubpanel.tsx | 60 +- packages/app/src/SessionsPage.tsx | 12 +- .../src/__tests__/SessionSidePanel.test.tsx | 86 ++ .../app/src/components/ContextSidePanel.tsx | 113 +-- .../app/src/components/DBRowDataPanel.tsx | 24 +- .../app/src/components/DBRowOverviewPanel.tsx | 1 - .../app/src/components/DBRowSidePanel.tsx | 804 ++++++++++++++---- .../src/components/DBRowSidePanel.types.ts | 38 + .../src/components/DBRowSidePanelHeader.tsx | 211 +---- packages/app/src/components/DBRowTable.tsx | 7 +- .../app/src/components/DBSessionPanel.tsx | 7 +- .../components/DBSqlRowTableWithSidebar.tsx | 23 +- packages/app/src/components/DBTracePanel.tsx | 68 +- .../app/src/components/PatternSidePanel.tsx | 2 - .../Search/DirectTraceSidePanel.tsx | 3 + .../ServiceDashboardSlowestEventsTile.tsx | 2 - .../src/components/SidePanelBreadcrumbs.tsx | 138 +++ .../DBRowSidePanel.missingSource.test.tsx | 275 ++++++ .../DBRowSidePanel.staleStack.test.tsx | 255 ++++++ .../__tests__/DBRowSidePanelHeader.test.tsx | 10 +- .../__tests__/SidePanelBreadcrumbs.test.tsx | 72 ++ .../__tests__/useSidePanelStack.test.tsx | 309 +++++++ packages/app/src/hooks/useRowWhere.tsx | 5 +- packages/app/src/hooks/useSidePanelStack.ts | 237 ++++++ .../src/utils/__tests__/queryParsers.test.ts | 39 + packages/app/src/utils/queryParsers.ts | 29 +- packages/app/styles/LogSidePanel.module.scss | 1 - .../tests/e2e/features/search/search.spec.ts | 16 +- .../app/tests/e2e/features/sessions.spec.ts | 59 +- .../tests/e2e/page-objects/SessionsPage.ts | 19 - packages/common-utils/src/types.ts | 40 +- 36 files changed, 2552 insertions(+), 715 deletions(-) create mode 100644 .changeset/single-drawer-navigation.md create mode 100644 packages/app/src/components/DBRowSidePanel.types.ts create mode 100644 packages/app/src/components/SidePanelBreadcrumbs.tsx create mode 100644 packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx create mode 100644 packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx create mode 100644 packages/app/src/components/__tests__/SidePanelBreadcrumbs.test.tsx create mode 100644 packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx create mode 100644 packages/app/src/hooks/useSidePanelStack.ts diff --git a/.changeset/single-drawer-navigation.md b/.changeset/single-drawer-navigation.md new file mode 100644 index 0000000000..2a1c21cf62 --- /dev/null +++ b/.changeset/single-drawer-navigation.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": minor +--- + +Redesign the event side panel into a single right-hand drawer with breadcrumb-stack navigation. Logs, traces, and sessions now navigate in-place (surrounding-context drilldowns, log → trace via a new "View Trace" action, and session → event) instead of stacking layered drawers. diff --git a/packages/app/src/NamespaceDetailsSidePanel.tsx b/packages/app/src/NamespaceDetailsSidePanel.tsx index d174eac198..61c0734371 100644 --- a/packages/app/src/NamespaceDetailsSidePanel.tsx +++ b/packages/app/src/NamespaceDetailsSidePanel.tsx @@ -181,8 +181,6 @@ function NamespaceLogs({ - {/* + {/* Search - + */} @@ -194,8 +194,6 @@ function NodeLogs({ @@ -460,7 +458,6 @@ export default function PodDetailsSidePanel({ rowId={rowId} aliasWith={aliasWith} onClose={handleCloseRowSidePanel} - isNestedPanel={true} /> )}
diff --git a/packages/app/src/SessionSidePanel.tsx b/packages/app/src/SessionSidePanel.tsx index 2cee84a423..f59d499575 100644 --- a/packages/app/src/SessionSidePanel.tsx +++ b/packages/app/src/SessionSidePanel.tsx @@ -1,33 +1,74 @@ -import { useCallback, useMemo, useState } from 'react'; +import { useCallback, useMemo } from 'react'; +import { useQueryState } from 'nuqs'; +import { ErrorBoundary } from 'react-error-boundary'; import { useHotkeys } from 'react-hotkeys-hook'; +import { z } from 'zod'; import { DateRange, SearchCondition, SearchConditionLanguage, + SourceKind, TSessionSource, TTraceSource, + WithClauseSchema, } from '@hyperdx/common-utils/dist/types'; -import { ActionIcon, Box, Button, Drawer, Group } from '@mantine/core'; +import { + ActionIcon, + Box, + Button, + Drawer, + Flex, + Group, + Text, + Tooltip, +} from '@mantine/core'; import { notifications } from '@mantine/notifications'; import { IconLink, IconX } from '@tabler/icons-react'; +import { + DBRowSidePanelInner, + RowSidePanelContext, + SidePanelErrorFallback, +} from '@/components/DBRowSidePanel'; import { DrawerFullWidthToggle, INITIAL_DRAWER_WIDTH_PERCENT, } from '@/components/DrawerUtils'; +import SidePanelBreadcrumbs, { + BreadcrumbItem, +} from '@/components/SidePanelBreadcrumbs'; import useResizable from '@/hooks/useResizable'; +import { WithClause } from '@/hooks/useRowWhere'; import { CLIPBOARD_ERROR_MESSAGE, copyTextToClipboard, } from '@/utils/clipboard'; +import { parseAsJsonEncoded } from '@/utils/queryParsers'; +import { ZIndexContext } from '@/zIndex'; +import useSidePanelStack from './hooks/useSidePanelStack'; import { Session } from './sessions'; import SessionSubpanel from './SessionSubpanel'; import { formatDistanceToNowStrictShort } from './utils'; -import { ZIndexContext } from './zIndex'; import styles from '@/../styles/LogSidePanel.module.scss'; +type SelectedEvent = { + rowId: string; + aliasWith: WithClause[]; + sessionId: string; +}; + +const sessionEventSchema = z.object({ + rowId: z.string(), + aliasWith: z.array(WithClauseSchema), + sessionId: z.string(), +}); + +const sessionEventParser = parseAsJsonEncoded( + sessionEventSchema.parse, +); + export default function SessionSidePanel({ traceSource, sessionSource, @@ -50,8 +91,24 @@ export default function SessionSidePanel({ onClose: () => void; zIndex?: number; }) { - // Keep track of sub-drawers so we can disable closing this root drawer - const [subDrawerOpen, setSubDrawerOpen] = useState(false); + // A single in-place event view (session → event), persisted to the URL so it + // survives reload and shared links. Deeper navigation (View Trace, + // surrounding context) is handled by `DBRowSidePanelInner`, which persists + // its own state to the shared `sidePanel*` params below. + const [persistedEvent, setSelectedEvent] = useQueryState( + 'sessionPanelEvent', + sessionEventParser, + ); + + // Read-time ownership gate: a persisted event belongs to the session it was + // opened in. If the user clicks a different session card (which updates the + // session params without going through this drawer's handlers), the old event + // is simply not selected — it can never render inside the wrong session, + // regardless of remount timing. No evict effect required. + const selectedEvent = + persistedEvent != null && persistedEvent.sessionId === sessionId + ? persistedEvent + : null; const { size, setSize, startResize } = useResizable( INITIAL_DRAWER_WIDTH_PERCENT, @@ -61,14 +118,48 @@ export default function SessionSidePanel({ setSize(isFullWidth ? INITIAL_DRAWER_WIDTH_PERCENT : 100); }, [isFullWidth, setSize]); - useHotkeys( - ['esc'], - () => { - onClose(); - }, - { - enabled: subDrawerOpen === false, + const sessionLabel = session?.userEmail || `Anonymous Session ${sessionId}`; + + const sidePanelStack = useSidePanelStack({ + initialRowId: selectedEvent?.rowId, + }); + + const handleBackToSession = useCallback(() => { + setSelectedEvent(null); + sidePanelStack.clearTrail(); + }, [setSelectedEvent, sidePanelStack]); + + const handleEventNavigate = useCallback( + (rowId: string, aliasWith: WithClause[]) => { + sidePanelStack.clearTrail(); + setSelectedEvent({ rowId, aliasWith, sessionId }); }, + [setSelectedEvent, sidePanelStack, sessionId], + ); + + // X / Esc-at-root closes the whole panel and clears the session-panel params. + const handleClose = useCallback(() => { + setSelectedEvent(null); + sidePanelStack.clearTrail(); + onClose(); + }, [setSelectedEvent, sidePanelStack, onClose]); + + useHotkeys(['esc'], handleClose, { enabled: !selectedEvent }); + + const shareSession = useCallback(async () => { + const ok = await copyTextToClipboard(window.location.href); + notifications.show( + ok + ? { color: 'green', message: 'Copied link to clipboard' } + : { color: 'red', message: CLIPBOARD_ERROR_MESSAGE }, + ); + }, []); + + const breadcrumbs = useMemo( + (): BreadcrumbItem[] => [ + { label: sessionLabel, sourceKind: SourceKind.Session }, + ], + [sessionLabel], ); const timeAgo = useMemo(() => { @@ -81,93 +172,130 @@ export default function SessionSidePanel({ return ( { - if (!subDrawerOpen) { - onClose(); - } - }} + onClose={handleClose} position="right" size={`${size}vw`} withCloseButton={false} + closeOnEscape={false} + lockScroll={false} + withOverlay={false} + trapFocus={false} zIndex={zIndex} styles={{ + content: { + border: 'none', + boxShadow: 'var(--shadow-drawer)', + }, body: { padding: 0, - height: '100vh', + height: '100%', }, }} - className="border-start" > - +
-
-
-
- {session?.userEmail || `Anonymous Session ${sessionId}`} -
- Last active {timeAgo} ago - · - {Number.parseInt(session?.errorCount ?? '0') > 0 ? ( + {selectedEvent ? ( + + ( + + )} + > + + + + ) : ( + <> + + + + + + + + + + + + + + + Last active {timeAgo} ago + {Number.parseInt(session?.errorCount ?? '0') > 0 && ( <> - + {' · '} + {session?.errorCount} Errors - - · + - ) : null} - {session?.sessionCount} Events -
-
- - + +
+ - - - - - -
-
- {sessionId != null ? ( - - ) : null} +
+ + )}
- + ); } diff --git a/packages/app/src/SessionSubpanel.tsx b/packages/app/src/SessionSubpanel.tsx index abbca03ab5..c5ef31a75d 100644 --- a/packages/app/src/SessionSubpanel.tsx +++ b/packages/app/src/SessionSubpanel.tsx @@ -17,7 +17,6 @@ import { Button, Divider, Group, - Portal, SegmentedControl, Tooltip, } from '@mantine/core'; @@ -31,9 +30,7 @@ import { IconToggleRight, } from '@tabler/icons-react'; -import DBRowSidePanel from '@/components/DBRowSidePanel'; import { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; -import { useZIndex, ZIndexContext } from '@/zIndex'; import SearchWhereInput from './components/SearchInput/SearchWhereInput'; import useFieldExpressionGenerator from './hooks/useFieldExpressionGenerator'; @@ -149,12 +146,12 @@ function useSessionChartConfigs({ type: 'lucene' as const, condition: `${traceSource.resourceAttributesExpression}.rum.sessionId:"${rumSessionId}" AND ( - ${traceSource.eventAttributesExpression}.http.status_code:>299 - OR ${traceSource.eventAttributesExpression}.component:"error" - OR ${traceSource.spanNameExpression}:"routeChange" - OR ${traceSource.spanNameExpression}:"documentLoad" - OR ${traceSource.spanNameExpression}:"intercom.onShow" - OR ScopeName:"custom-action" + ${traceSource.eventAttributesExpression}.http.status_code:>299 + OR ${traceSource.eventAttributesExpression}.component:"error" + OR ${traceSource.spanNameExpression}:"routeChange" + OR ${traceSource.spanNameExpression}:"documentLoad" + OR ${traceSource.spanNameExpression}:"intercom.onShow" + OR ScopeName:"custom-action" )`, }), [traceSource, rumSessionId], @@ -166,13 +163,13 @@ function useSessionChartConfigs({ type: 'lucene' as const, condition: `${traceSource.resourceAttributesExpression}.rum.sessionId:"${rumSessionId}" AND ( - ${traceSource.eventAttributesExpression}.http.status_code:* - OR ${traceSource.eventAttributesExpression}.component:"console" - OR ${traceSource.eventAttributesExpression}.component:"error" - OR ${traceSource.spanNameExpression}:"routeChange" - OR ${traceSource.spanNameExpression}:"documentLoad" - OR ${traceSource.spanNameExpression}:"intercom.onShow" - OR ScopeName:"custom-action" + ${traceSource.eventAttributesExpression}.http.status_code:* + OR ${traceSource.eventAttributesExpression}.component:"console" + OR ${traceSource.eventAttributesExpression}.component:"error" + OR ${traceSource.spanNameExpression}:"routeChange" + OR ${traceSource.spanNameExpression}:"documentLoad" + OR ${traceSource.spanNameExpression}:"intercom.onShow" + OR ScopeName:"custom-action" )`, }; }, [traceSource, rumSessionId, getTraceSourceFieldExpression]); @@ -235,30 +232,25 @@ export default function SessionSubpanel({ traceSource, sessionSource, session, - setDrawerOpen, rumSessionId, start, end, initialTs, whereLanguage = 'lucene', onLanguageChange, + onEventNavigate, }: { traceSource: TTraceSource; sessionSource: TSessionSource; session: { serviceName: string }; - setDrawerOpen: (open: boolean) => void; rumSessionId: string; start: Date; end: Date; initialTs?: number; whereLanguage?: SearchConditionLanguage; onLanguageChange?: (lang: 'sql' | 'lucene') => void; + onEventNavigate?: (rowId: string, aliasWith: WithClause[]) => void; }) { - const contextZIndex = useZIndex(); - - const [rowId, setRowId] = useState(undefined); - const [aliasWith, setAliasWith] = useState([]); - const [tsQuery, setTsQuery] = useQueryState( 'ts', parseAsInteger.withOptions({ history: 'replace' }), @@ -430,11 +422,9 @@ export default function SessionSubpanel({ ); const onSessionEventClick = useCallback( (rowWhere: RowWhereResult) => { - setDrawerOpen(true); - setRowId(rowWhere.where); - setAliasWith(rowWhere.aliasWith); + onEventNavigate?.(rowWhere.where, rowWhere.aliasWith); }, - [setDrawerOpen, setRowId, setAliasWith], + [onEventNavigate], ); const onSessionEventTimeClick = useCallback( (ts: number) => { @@ -453,22 +443,6 @@ export default function SessionSubpanel({ return ( @@ -973,10 +1217,12 @@ export function DBTraceWaterfallChartContainer({ tooltip: `${displayText} ${tookMs >= 0 ? `took ${tookMs.toFixed(4)}ms` : ''} ${status ? `| Status: ${status}` : ''}${!isNaN(startOffset) ? ` | Started at ${formatTime(new Date(startOffset), { format: 'withMs' })}` : ''}`, color: 'var(--color-text-inverted)', backgroundColor: barColor({ - isError, - isWarn, isHighlighted, + isError, type, + serviceColor: serviceName + ? serviceColorMap.get(serviceName) + : undefined, }), body: {displayText}, minWidthPx: type === SourceKind.Log ? 10 : 2, @@ -992,13 +1238,11 @@ export function DBTraceWaterfallChartContainer({ visibleNodes, formatTime, highlightedRowWhere, - isFilterActive, minOffset, onClick, + serviceColorMap, showSpanEvents, toggleCollapse, - collapseTooltipShown, - setCollapseTooltipShown, ], ); const initialScrollRowIndex = visibleNodes.findIndex(v => { @@ -1026,50 +1270,134 @@ export function DBTraceWaterfallChartContainer({ )} {isFilterExpanded && ( - - Spans filter - - + + + + Spans filter + + + setValue('traceWhereLanguage', lang, { shouldDirty: true }) + } + lucenePlaceholder='Filter spans ex. StatusCode:"Error"' + sqlPlaceholder="Filter spans ex. StatusCode = 'Error'" + data-testid="trace-search-input" + // The waterfall lives inside an `overflow: hidden` column, which + // clips the SQL editor's autocomplete tooltip. Portal it to the + // document body so suggestions aren't cut off (Lucene mode + // already renders its dropdown in a portal). + parentRef={ + typeof document !== 'undefined' ? document.body : null + } + /> + {traceFilterError && ( + + + + )} + {logTableSource && ( - <> - Logs filter - + + Logs filter + + + setValue('logWhereLanguage', lang, { shouldDirty: true }) + } + lucenePlaceholder='Filter logs ex. SeverityText:"error"' + sqlPlaceholder="Filter logs ex. SeverityText = 'error'" data-testid="log-search-input" + parentRef={ + typeof document !== 'undefined' ? document.body : null + } /> - + {logError && ( + + + + )} + )} - + )} + {hasCollapsibleNodes && ( + + + + + + + + + + + + + + + + + + + + + + + )} {itemCountString},{' '} @@ -1123,26 +1451,29 @@ export function DBTraceWaterfallChartContainer({ - - setIsFilterExpanded(prev => !prev)} - size="xs" - > - {isFilterExpanded ? 'Hide Filters' : 'Show Filters'}{' '} - {isFilterActive && '(active)'} - - {isFilterActive && ( + + {controlsExtra} + setIsFilterExpanded(prev => !prev)} size="xs" - ms="xs" > - Clear Filters + {isFilterExpanded ? 'Hide Filters' : 'Show Filters'}{' '} + {isFilterActive && '(active)'} - )} - + {isFilterActive && ( + + Clear Filters + + )} + + {!isFetching && !error && highlightedAttributeValues?.length > 0 && ( diff --git a/packages/app/src/components/SearchInput/SearchWhereInput.tsx b/packages/app/src/components/SearchInput/SearchWhereInput.tsx index 296a23cb76..e4f60134b6 100644 --- a/packages/app/src/components/SearchInput/SearchWhereInput.tsx +++ b/packages/app/src/components/SearchInput/SearchWhereInput.tsx @@ -119,6 +119,7 @@ export type SearchWhereInputProps = { * Source id used in various queries */ sourceId?: string; + parentRef?: HTMLElement | null; } & TableConnectionChoice & UseControllerProps; @@ -166,6 +167,7 @@ export default function SearchWhereInput({ dateRange, languageName = `${name}Language`, sourceId, + parentRef, }: SearchWhereInputProps) { const [syntaxRefOpened, { open: openSyntaxRef, close: closeSyntaxRef }] = useDisclosure(false); @@ -240,6 +242,7 @@ export default function SearchWhereInput({ additionalSuggestions={additionalSuggestions} dateRange={dateRange} sourceId={sourceId} + parentRef={parentRef} /> ) : (
- {rowId != null && traceSource && ( - - - { - setDrawerOpen(false); - setRowId(undefined); - }} - /> - - - )}
(['sql', 'lucene']), }; -export default function SessionsPage() { +function SessionsPage() { const brandName = useBrandDisplayName(); const [appliedConfig, setAppliedConfig] = useQueryStates(appliedConfigMap); @@ -493,7 +494,14 @@ export default function SessionsPage() { ); } -SessionsPage.getLayout = withAppNav; +const SessionsPageDynamic = dynamic(async () => SessionsPage, { + ssr: false, +}); + +// @ts-expect-error for getLayout +SessionsPageDynamic.getLayout = withAppNav; + +export default SessionsPageDynamic; function SessionSetupInstructions() { const brandName = useBrandDisplayName(); diff --git a/packages/app/src/__tests__/SessionSidePanel.test.tsx b/packages/app/src/__tests__/SessionSidePanel.test.tsx index 7969b3dc6e..5bf4b5e467 100644 --- a/packages/app/src/__tests__/SessionSidePanel.test.tsx +++ b/packages/app/src/__tests__/SessionSidePanel.test.tsx @@ -13,6 +13,44 @@ jest.mock('../SessionSubpanel', () => ({ default: () =>
, })); +jest.mock('@/components/DBRowSidePanel', () => { + const ReactActual = jest.requireActual('react'); + return { + __esModule: true, + DBRowSidePanelInner: (props: { rowId: string }) => ( +
{props.rowId}
+ ), + RowSidePanelContext: ReactActual.createContext({}), + }; +}); + +// Controllable state for the `sessionPanelEvent` query param so tests can +// simulate a value left in the URL by a previously selected session. +const mockNuqs: { + sessionPanelEvent: unknown; + setSessionPanelEvent: jest.Mock; +} = { + sessionPanelEvent: null, + setSessionPanelEvent: jest.fn(), +}; + +jest.mock('nuqs', () => { + const actual = jest.requireActual('nuqs'); + // eslint-disable-next-line @typescript-eslint/no-empty-function + const noop = () => {}; + return { + ...actual, + // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => + key === 'sessionPanelEvent' + ? [mockNuqs.sessionPanelEvent, mockNuqs.setSessionPanelEvent] + : // Mirror real nuqs: a parser built with `.withDefault(...)` never + // yields null. Honor its defaultValue so hooks like + // useSidePanelStack read empty arrays, not null. + [parser?.defaultValue ?? null, noop], + }; +}); + jest.mock('../utils/clipboard', () => ({ __esModule: true, CLIPBOARD_ERROR_MESSAGE: @@ -72,6 +110,8 @@ describe('SessionSidePanel - Share Session', () => { beforeEach(() => { mockedCopy.mockReset(); mockedShow.mockReset(); + mockNuqs.sessionPanelEvent = null; + mockNuqs.setSessionPanelEvent.mockReset(); setLocationHref('/sessions?sessionSource=src&from=1&to=2'); }); @@ -116,3 +156,49 @@ describe('SessionSidePanel - Share Session', () => { ); }); }); + +describe('SessionSidePanel - persisted event session ownership', () => { + beforeEach(() => { + mockNuqs.sessionPanelEvent = null; + mockNuqs.setSessionPanelEvent.mockReset(); + setLocationHref('/sessions?sessionSource=src&from=1&to=2'); + }); + + it('renders a persisted event that belongs to the current session', () => { + mockNuqs.sessionPanelEvent = { + rowId: 'row-current', + aliasWith: [], + sessionId: 'sid-abc', + }; + + renderPanel(); + + // The event view is shown and the param is left untouched. + expect(screen.getByTestId('event-view-mock')).toHaveTextContent( + 'row-current', + ); + expect( + screen.queryByTestId('session-subpanel-mock'), + ).not.toBeInTheDocument(); + expect(mockNuqs.setSessionPanelEvent).not.toHaveBeenCalled(); + }); + + it('ignores a persisted event owned by a different session', () => { + // Simulates: opened an event in another session, then clicked this session + // card (which only updates sid/sfrom/sto and remounts the drawer). + mockNuqs.sessionPanelEvent = { + rowId: 'row-stale', + aliasWith: [], + sessionId: 'some-other-session', + }; + + renderPanel(); + + // The stale event must NOT render inside the newly selected session; the + // read-time ownership gate falls back to the session root. Correctness does + // not depend on any evict effect firing, so the param may harmlessly linger + // in the URL (it can never render for the wrong session). + expect(screen.getByTestId('session-subpanel-mock')).toBeInTheDocument(); + expect(screen.queryByTestId('event-view-mock')).not.toBeInTheDocument(); + }); +}); diff --git a/packages/app/src/components/ContextSidePanel.tsx b/packages/app/src/components/ContextSidePanel.tsx index e73f5a9532..a50a326515 100644 --- a/packages/app/src/components/ContextSidePanel.tsx +++ b/packages/app/src/components/ContextSidePanel.tsx @@ -1,12 +1,12 @@ import { useCallback, useContext, useMemo, useState } from 'react'; import ms from 'ms'; -import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { BuilderChartConfigWithDateRange, isLogSource, isTraceSource, + SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; import { Badge, Flex, Group, SegmentedControl } from '@mantine/core'; @@ -16,16 +16,10 @@ import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; -import { useSource } from '@/source'; import { formatAttributeClause } from '@/utils'; -import { parseAsStringEncoded } from '@/utils/queryParsers'; import { ROW_DATA_ALIASES } from './DBRowDataPanel'; -import DBRowSidePanel, { RowSidePanelContext } from './DBRowSidePanel'; -import { - BreadcrumbNavigationCallback, - BreadcrumbPath, -} from './DBRowSidePanelHeader'; +import { RowSidePanelContext } from './DBRowSidePanel'; import { DBSqlRowTable } from './DBRowTable'; enum ContextBy { @@ -42,35 +36,13 @@ interface ContextSubpanelProps { dbSqlRowTableConfig: BuilderChartConfigWithDateRange | undefined; rowData: Record; rowId: string | undefined; - breadcrumbPath?: BreadcrumbPath; - onBreadcrumbClick?: BreadcrumbNavigationCallback; -} - -// Custom hook to manage nested panel state -export function useNestedPanelState(isNested?: boolean) { - // Query state (URL-based) for root level - const queryState = { - contextRowId: useQueryState('contextRowId', parseAsStringEncoded), - // Source IDs are MongoDB ObjectIDs (hex strings) and contain no special - // characters, so no encoding is needed here. - contextRowSource: useQueryState('contextRowSource'), - }; - - // Local state for nested levels - const localState = { - contextRowId: useState(null), - contextRowSource: useState(null), - }; - - // Choose which state to use based on nesting level - const activeState = isNested ? localState : queryState; - - return { - contextRowId: activeState.contextRowId[0], - contextRowSource: activeState.contextRowSource[0], - setContextRowId: activeState.contextRowId[1], - setContextRowSource: activeState.contextRowSource[1], - }; + onNavigateToRow?: ( + rowId: string, + aliasWith: WithClause[], + label: string, + sourceKind?: SourceKind, + ) => void; + 'data-testid'?: string; } export default function ContextSubpanel({ @@ -78,8 +50,8 @@ export default function ContextSubpanel({ dbSqlRowTableConfig, rowData, rowId, - breadcrumbPath, - onBreadcrumbClick, + onNavigateToRow, + 'data-testid': dataTestId, }: ContextSubpanelProps) { const QUERY_KEY_PREFIX = 'context'; const origTimestamp = rowData[ROW_DATA_ALIASES.TIMESTAMP]; @@ -100,36 +72,21 @@ export default function ContextSubpanel({ const formWhere = useWatch({ control, name: 'where' }); const [debouncedWhere] = useDebouncedValue(formWhere, 1000); - // State management for nested panels - const isNested = !!breadcrumbPath?.length; - - const { - contextRowId, - contextRowSource, - setContextRowId, - setContextRowSource, - } = useNestedPanelState(isNested); - - const { data: contextRowSidePanelSource } = useSource({ - id: contextRowSource || '', - }); - - const [contextAliasWith, setContextAliasWith] = useState([]); - - const handleContextSidePanelClose = useCallback(() => { - setContextRowId(null); - setContextRowSource(null); - }, [setContextRowId, setContextRowSource]); - const { setChildModalOpen } = useContext(RowSidePanelContext); const handleRowExpandClick = useCallback( - (rowWhere: RowWhereResult) => { - setContextRowId(rowWhere.where); - setContextAliasWith(rowWhere.aliasWith); - setContextRowSource(source.id); + (rowWhere: RowWhereResult, row: Record) => { + const body = row?.[ROW_DATA_ALIASES.BODY]; + const fallback = isTraceSource(source) ? 'Span' : 'Log'; + const label = + typeof body === 'string' && body.length > 0 + ? body + : body != null + ? JSON.stringify(body) + : fallback; + onNavigateToRow?.(rowWhere.where, rowWhere.aliasWith, label, source.kind); }, - [source.id, setContextRowId, setContextRowSource], + [onNavigateToRow, source], ); const date = useMemo(() => new Date(origTimestamp), [origTimestamp]); @@ -144,7 +101,7 @@ export default function ContextSubpanel({ /* Functions to help generate WHERE clause based on which Context the user chooses (All, Host, Node, etc...). - Since we support lucene and sql, we need to format the condition + Since we support lucene and sql, we need to format the condition based on the language */ const { @@ -259,7 +216,12 @@ export default function ContextSubpanel({ return ( <> {config && ( - + )} - {contextRowId && contextRowSidePanelSource && ( - - )} ); } diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index e91abd5e31..1f631148ae 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -11,7 +11,11 @@ import { Box } from '@mantine/core'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { WithClause } from '@/hooks/useRowWhere'; -import { getDisplayedTimestampValueExpression, getEventBody } from '@/source'; +import { + getDisplayedTimestampValueExpression, + getDurationMsExpression, + getEventBody, +} from '@/source'; import { getSelectExpressionsForHighlightedAttributes } from '@/utils/highlightedAttributes'; import { DBRowJsonViewer } from './DBRowJsonViewer'; @@ -28,6 +32,8 @@ export enum ROW_DATA_ALIASES { EVENT_ATTRIBUTES = '__hdx_event_attributes', EVENTS_EXCEPTION_ATTRIBUTES = '__hdx_events_exception_attributes', SPAN_EVENTS = '__hdx_span_events', + DURATION_MS = '__hdx_duration_ms', + SPAN_KIND = '__hdx_span_kind', } export function useRowData({ @@ -154,6 +160,22 @@ export function useRowData({ }, ] : []), + ...(source.kind === SourceKind.Trace && source.durationExpression + ? [ + { + valueExpression: getDurationMsExpression(source), + alias: ROW_DATA_ALIASES.DURATION_MS, + }, + ] + : []), + ...(source.kind === SourceKind.Trace && source.spanKindExpression + ? [ + { + valueExpression: source.spanKindExpression, + alias: ROW_DATA_ALIASES.SPAN_KIND, + }, + ] + : []), ...selectHighlightedRowAttributes, ], where: rowId ?? '0=1', diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx index 2d8560fff9..1ea1e945dd 100644 --- a/packages/app/src/components/DBRowOverviewPanel.tsx +++ b/packages/app/src/components/DBRowOverviewPanel.tsx @@ -196,7 +196,6 @@ export function RowOverviewPanel({ ({}); -enum Tab { - Overview = 'overview', - Parsed = 'parsed', - Debug = 'debug', - Trace = 'trace', - ServiceMap = 'serviceMap', - Context = 'context', - Replay = 'replay', - Infrastructure = 'infrastructure', +function SidePanelHeaderActions({ + onClose, + isFullWidth, + onToggleFullWidth, +}: { + onClose: () => void; + isFullWidth?: boolean; + onToggleFullWidth?: () => void; +}) { + const [shortcutsOpen, setShortcutsOpen] = useState(false); + + return ( + <> + + {onToggleFullWidth && ( + + )} + + setShortcutsOpen(true)} + aria-label="Keyboard shortcuts" + > + + + + + {({ copied, copy }) => ( + + + + + + )} + + + + + + + + setShortcutsOpen(false)} + /> + + ); } +const SPAN_KIND_LABELS: Record = { + '1': 'Internal', + '2': 'Server', + '3': 'Client', + '4': 'Producer', + '5': 'Consumer', + Internal: 'Internal', + Server: 'Server', + Client: 'Client', + Producer: 'Producer', + Consumer: 'Consumer', + SPAN_KIND_INTERNAL: 'Internal', + SPAN_KIND_SERVER: 'Server', + SPAN_KIND_CLIENT: 'Client', + SPAN_KIND_PRODUCER: 'Producer', + SPAN_KIND_CONSUMER: 'Consumer', +}; + type DBRowSidePanelProps = { source: TSource; rowId: string | undefined; aliasWith?: WithClause[]; onClose: () => void; - isNestedPanel?: boolean; - breadcrumbPath?: BreadcrumbPath; - onBreadcrumbClick?: BreadcrumbNavigationCallback; }; -const DBRowSidePanel = ({ - rowId: rowId, - aliasWith, - source, - isNestedPanel = false, - setSubDrawerOpen, +type DBRowSidePanelInnerProps = DBRowSidePanelProps & { + isFullWidth?: boolean; + onToggleFullWidth?: () => void; + persistStacksInUrl?: boolean; + parentBreadcrumbs?: BreadcrumbItem[]; + onNavigateToParent?: () => void; + sidePanelStack: SidePanelStack; +}; + +export const DBRowSidePanelInner = ({ + rowId: initialRowId, + aliasWith: initialAliasWith, + source: rootSource, onClose, - breadcrumbPath, - onBreadcrumbClick, isFullWidth, onToggleFullWidth, -}: DBRowSidePanelProps & { - setSubDrawerOpen: Dispatch>; - isFullWidth?: boolean; - onToggleFullWidth?: () => void; -}) => { + parentBreadcrumbs, + onNavigateToParent, + sidePanelStack, +}: DBRowSidePanelInnerProps) => { + const { + sourceStack, + navStack, + tab: persistedTab, + pushSource, + pushNav, + popOne, + truncateTo, + setTab, + } = sidePanelStack; + + const activeSourceFrame = + sourceStack.length > 0 ? sourceStack[sourceStack.length - 1] : null; + + // Resolve the leaf source (cross-source navigation). Intermediate frames only + // need their stored label/kind for breadcrumbs. + const { + data: activeStackSource, + isLoading: isStackSourceLoading, + isSuccess: isStackSourceSettled, + } = useSource({ + id: activeSourceFrame?.sourceId ?? null, + }); + const isResolvingSource = activeSourceFrame != null && isStackSourceLoading; + + const isStackSourceMissing = + activeSourceFrame != null && + isStackSourceSettled && + activeStackSource == null; + const source = activeStackSource ?? rootSource; + + const baseRowId = activeSourceFrame?.rowId ?? initialRowId; + const baseAliasWith = activeSourceFrame?.aliasWith ?? initialAliasWith; + + const leafNav = navStack.length > 0 ? navStack[navStack.length - 1] : null; + const resolvedRowId = leafNav?.rowId ?? baseRowId; + const resolvedAliasWith = leafNav?.aliasWith ?? baseAliasWith; + + // Avoid querying the (transiently wrong or unavailable) root source with a + // leaf row id while the leaf source is still loading or can no longer be + // resolved. + const skipRowQuery = isResolvingSource || isStackSourceMissing; + const activeRowId = skipRowQuery ? undefined : resolvedRowId; + const activeAliasWith = skipRowQuery ? undefined : resolvedAliasWith; + const { data: rowData, isLoading: isRowLoading, @@ -125,39 +276,17 @@ const DBRowSidePanel = ({ error: rowError, } = useRowData({ source, - rowId, - aliasWith, + rowId: activeRowId, + aliasWith: activeAliasWith, }); - const { dbSqlRowTableConfig } = useContext(RowSidePanelContext); + const hasActiveStacks = activeSourceFrame != null || leafNav != null; - const handleBreadcrumbClick = useCallback( - (targetLevel: number) => { - // Current panel's level in the hierarchy - const currentLevel = breadcrumbPath?.length ?? 0; - - // The target panel level corresponds to the breadcrumb index: - // - targetLevel 0 = root panel (breadcrumbPath.length = 0) - // - targetLevel 1 = first nested panel (breadcrumbPath.length = 1) - // - etc. - - // If our current level is greater than the target panel level, close this panel - if (currentLevel > targetLevel) { - onClose(); - onBreadcrumbClick?.(targetLevel); - } - // If our current level equals the target panel level, we're the target - don't close - else if (currentLevel === targetLevel) { - // This is the panel the user wants to navigate to - do nothing (stay open) - return; - } - // If our current level is less than target, propagate up (this panel should stay open) - else { - onBreadcrumbClick?.(targetLevel); - } - }, - [breadcrumbPath?.length, onBreadcrumbClick, onClose], - ); + const parentContext = useContext(RowSidePanelContext); + // Nested rows shouldn't inherit the parent table's row config. + const dbSqlRowTableConfig = hasActiveStacks + ? undefined + : parentContext.dbSqlRowTableConfig; const hasOverviewPanel = useMemo(() => { if (isLogSource(source) || isTraceSource(source)) { @@ -178,30 +307,63 @@ const DBRowSidePanel = ({ return false; }, [source]); - const defaultTab = - source.kind === 'trace' - ? Tab.Trace - : hasOverviewPanel - ? Tab.Overview - : Tab.Parsed; + const sourceIsTrace = source.kind === SourceKind.Trace; + + const defaultTab = sourceIsTrace + ? Tab.Trace + : hasOverviewPanel + ? Tab.Overview + : Tab.Parsed; + + const handleNavigateToRow = useCallback( + ( + rowId: string, + aliasWith: WithClause[], + label: string, + sourceKind?: SourceKind, + ) => { + // Same-source drilldown (e.g. surrounding context) → jump to this + // source's default tab. + pushNav({ rowId, aliasWith, label, sourceKind }, defaultTab); + }, + [pushNav, defaultTab], + ); - const [queryTab, setQueryTab] = useQueryState( - 'sidePanelTab', - parseAsStringEnum(Object.values(Tab)).withDefault(defaultTab), + const handleSourceStackPush = useCallback( + (frame: SourceFrame) => { + // Cross-source push (e.g. "View Trace") → jump to the destination + // source's default tab. + const destinationTab = + frame.sourceKind === SourceKind.Trace ? Tab.Trace : Tab.Overview; + pushSource(frame, destinationTab); + }, + [pushSource], ); - const [stateTab, setStateTab] = useState(defaultTab); - // Nested panels can't share the query param or else they'll conflict, so we'll use local state for nested panels - // We'll need to handle this properly eventually... - const tab = isNestedPanel ? stateTab : queryTab; - const setTab = isNestedPanel ? setStateTab : setQueryTab; + const handlePanelBack = useCallback(() => { + // Pop one level (nav → source), restoring the tab active before that level + // was entered. When the trail is empty, leave the panel: hand off to an + // embedding parent (session) or close. + if (popOne() === 'none') { + if (onNavigateToParent) { + onNavigateToParent(); + } else { + onClose(); + } + } + }, [popOne, onNavigateToParent, onClose]); - const displayedTab = tab; + useHotkeys(['esc'], handlePanelBack); + + const handleBreadcrumbNavigation = useCallback( + (sourceLevel: number, navLevel: number) => + truncateTo(sourceLevel, navLevel), + [truncateTo], + ); const normalizedRow = rowData?.data?.[0]; const timestampValue = normalizedRow?.['__hdx_timestamp']; - // TODO: Improve parsing let timestampDate: Date; if (typeof timestampValue === 'number') { timestampDate = new Date(timestampValue * 1000); @@ -218,6 +380,18 @@ const DBRowSidePanel = ({ const severityText: string | undefined = normalizedRow?.['__hdx_severity_text']; + // Capture the root event body once for the root breadcrumb label. + const [initialMainContent, setInitialMainContent] = useState< + string | undefined + >(undefined); + + useEffect(() => { + if (mainContent != null && initialMainContent == null && !hasActiveStacks) { + // eslint-disable-next-line react-hooks/set-state-in-effect + setInitialMainContent(mainContent); + } + }, [mainContent, initialMainContent, hasActiveStacks]); + const highlightedAttributeValues = useMemo(() => { const attributeExpressions: NonNullable< (TLogSource | TTraceSource)['highlightedRowAttributeExpressions'] @@ -266,7 +440,10 @@ const DBRowSidePanel = ({ }, [timestampDate]); const focusDate = timestampDate; - const traceId: string | undefined = normalizedRow?.['__hdx_trace_id']; + // Coerce empty / falsy trace ids to undefined so "View Trace" / Trace ID + // is hidden for logs without trace context. + const traceId: string | undefined = + normalizedRow?.['__hdx_trace_id'] || undefined; const childSourceId = isLogSource(source) ? source.traceSourceId @@ -278,17 +455,61 @@ const DBRowSidePanel = ({ ? source.id : isLogSource(source) ? source.traceSourceId - : isSessionSource(source) + : source.kind === SourceKind.Session ? source.traceSourceId : undefined; const enableServiceMap = traceId && traceSourceId; + const { data: traceSourceData } = useSource({ id: traceSourceId }); + + const spanId = normalizedRow?.['__hdx_span_id']; + const traceIdExpression = + traceSourceData?.kind === SourceKind.Log || + traceSourceData?.kind === SourceKind.Trace + ? traceSourceData.traceIdExpression + : undefined; + const spanIdExpression = + traceSourceData?.kind === SourceKind.Log || + traceSourceData?.kind === SourceKind.Trace + ? traceSourceData.spanIdExpression + : undefined; + + const traceSpanRowId = useMemo(() => { + const clauses: string[] = []; + if (traceIdExpression && traceId) { + clauses.push( + SqlString.format('?=?', [SqlString.raw(traceIdExpression), traceId]), + ); + } + if (spanIdExpression && spanId) { + clauses.push( + SqlString.format('?=?', [SqlString.raw(spanIdExpression), spanId]), + ); + } + return clauses.length > 0 ? clauses.join(' AND ') : undefined; + }, [traceIdExpression, traceId, spanIdExpression, spanId]); + + const handleSessionEventNavigate = useCallback( + (rowId: string, aliasWith: WithClause[]) => { + if (traceSourceData) { + handleSourceStackPush({ + sourceId: traceSourceData.id, + rowId, + aliasWith, + label: mainContent || 'Session Replay', + sourceKind: traceSourceData.kind as SourceKind, + }); + } + }, + [traceSourceData, handleSourceStackPush, mainContent], + ); + const { rumSessionId, rumServiceName } = useSessionId({ sourceId: traceSourceId, traceId, dateRange: oneHourRange, - enabled: rowId != null, + enabled: activeRowId != null, }); const hasK8sContext = useMemo( @@ -306,10 +527,141 @@ const DBRowSidePanel = ({ } }, [normalizedRow]); - if (isRowLoading) { + const durationMs = normalizedRow?.[ROW_DATA_ALIASES.DURATION_MS]; + const spanKind = normalizedRow?.[ROW_DATA_ALIASES.SPAN_KIND]; + const serviceName = normalizedRow?.[ROW_DATA_ALIASES.SERVICE_NAME]; + const statusCode = normalizedRow?.[ROW_DATA_ALIASES.SEVERITY_TEXT]; + + const formattedDuration = useMemo(() => { + if (durationMs == null || isNaN(Number(durationMs))) return undefined; + return renderMs(Number(durationMs)); + }, [durationMs]); + + const spanKindLabel = useMemo(() => { + if (spanKind == null) return undefined; + return SPAN_KIND_LABELS[String(spanKind)] ?? String(spanKind); + }, [spanKind]); + + const allBreadcrumbs = useMemo((): BreadcrumbItem[] => { + const items: BreadcrumbItem[] = []; + + if (parentBreadcrumbs) { + items.push(...parentBreadcrumbs); + } + + const crumbSourceStack = sourceStack; + const crumbNavStack = navStack; + const hasStack = crumbSourceStack.length > 0 || crumbNavStack.length > 0; + const rootLabel = + initialMainContent || + (rootSource.kind === SourceKind.Trace ? 'Trace' : 'Log'); + + if (hasStack) { + items.push({ + label: rootLabel, + sourceKind: rootSource.kind as SourceKind, + onClick: () => handleBreadcrumbNavigation(0, 0), + }); + } + + crumbSourceStack.forEach((entry, i) => { + const isLeafSource = i === crumbSourceStack.length - 1; + const isCurrent = isLeafSource && crumbNavStack.length === 0; + items.push({ + label: entry.label, + sourceKind: entry.sourceKind, + onClick: isCurrent + ? undefined + : () => handleBreadcrumbNavigation(i + 1, 0), + }); + }); + + crumbNavStack.forEach((entry, i) => { + const isCurrent = i === crumbNavStack.length - 1; + items.push({ + label: entry.label, + sourceKind: entry.sourceKind, + onClick: isCurrent + ? undefined + : () => handleBreadcrumbNavigation(crumbSourceStack.length, i + 1), + }); + }); + + if (!hasStack) { + items.push({ + label: mainContent || (sourceIsTrace ? 'Trace' : 'Log'), + sourceKind: source.kind as SourceKind, + }); + } + + return items; + }, [ + sourceStack, + navStack, + rootSource.kind, + sourceIsTrace, + mainContent, + initialMainContent, + source.kind, + handleBreadcrumbNavigation, + parentBreadcrumbs, + ]); + + const availableTabs = useMemo(() => { + const tabs: Tab[] = []; + if (hasOverviewPanel && !sourceIsTrace) tabs.push(Tab.Overview); + if (!sourceIsTrace) tabs.push(Tab.Parsed); + if (sourceIsTrace) tabs.push(Tab.Trace); + if (enableServiceMap) tabs.push(Tab.ServiceMap); + tabs.push(Tab.Context); + if (rumSessionId != null) tabs.push(Tab.Replay); + if (hasK8sContext) tabs.push(Tab.Infrastructure); + return tabs; + }, [ + hasOverviewPanel, + sourceIsTrace, + enableServiceMap, + rumSessionId, + hasK8sContext, + ]); + + const displayedTab = reconcileTab(persistedTab, availableTabs, defaultTab); + + if (isRowLoading || isResolvingSource) { return
Loading...
; } + // The leaf cross-source frame points at a source that no longer resolves + // (deleted / renamed / another workspace via a shared link). Render an + // explicit message that keeps the breadcrumb trail, Back, and Close controls + // working instead of hanging on an indefinite "Loading...". + if (isStackSourceMissing) { + return ( + <> + + + + + + + + + This source is no longer available. It may have been deleted, + renamed, or belong to a different workspace. Use the back button or + the breadcrumbs above to return. + + + + ); + } + if (!isRowSuccess) { if (isRowError && rowError) { return ( @@ -321,34 +673,151 @@ const DBRowSidePanel = ({ return
Error loading row data
; } + const showLogTraceActions = !sourceIsTrace && traceId && traceSourceId; + return ( <> - + + + + + + + {!sourceIsTrace && severityText && } + {timestampDate && !isNaN(timestampDate.getTime()) && ( + + ·{' '} + {formatDistanceToNowStrictShort(timestampDate)} ago + + )} + {serviceName && ( + <> + + · + + + + Service + + + {serviceName} + + + + )} + {sourceIsTrace && formattedDuration && ( + <> + + · + + + + Duration + + + {formattedDuration} + + + + )} + {sourceIsTrace && statusCode && ( + <> + + · + + + + Status + + + {statusCode} + + + + )} + {sourceIsTrace && spanKindLabel && ( + + {spanKindLabel} + + )} + {(sourceIsTrace ? traceId : showLogTraceActions) && ( + <> + + · + + + {({ copied, copy }) => ( + + + + + Trace ID + + + + )} + + + )} + {showLogTraceActions && ( + + )} + - {/* */} )} - {displayedTab === Tab.Trace && ( + {displayedTab === Tab.Trace && sourceIsTrace && ( { console.error(err); @@ -474,8 +951,8 @@ const DBRowSidePanel = ({ )} @@ -495,9 +972,8 @@ const DBRowSidePanel = ({ source={source} dbSqlRowTableConfig={dbSqlRowTableConfig} rowData={normalizedRow} - rowId={rowId} - breadcrumbPath={breadcrumbPath} - onBreadcrumbClick={handleBreadcrumbClick} + rowId={activeRowId} + onNavigateToRow={handleNavigateToRow} /> )} @@ -519,7 +995,7 @@ const DBRowSidePanel = ({ void }) => ( + + + + +
+ An error occurred while rendering this event. +
+ +
+ {error?.message} +
+ +); + export default function DBRowSidePanelErrorBoundary({ onClose, rowId, aliasWith, source, - isNestedPanel, - breadcrumbPath, - onBreadcrumbClick, }: DBRowSidePanelProps) { const contextZIndex = useZIndex(); const drawerZIndex = contextZIndex + 10; @@ -572,42 +1072,35 @@ export default function DBRowSidePanelErrorBoundary({ setSize(isFullWidth ? INITIAL_DRAWER_WIDTH_PERCENT : 100); }, [isFullWidth, setSize]); - // Keep track of sub-drawers so we can disable closing this root drawer - const [subDrawerOpen, setSubDrawerOpen] = useState(false); - - const [_, setQueryTab] = useQueryState( - 'tab', - parseAsStringEnum(Object.values(Tab)), - ); - const { clear: clearTraceWaterfallSearchState } = useWaterfallSearchState({}); + const sidePanelStack = useSidePanelStack({ initialRowId: rowId }); + const _onClose = useCallback(() => { - // Reset tab to undefined when unmounting, so that when we open the drawer again, it doesn't open to the last tab - // (which might not be valid, ex session replay) - if (!isNestedPanel) { - setQueryTab(null); - } + // Reset the tab and navigation stacks so re-opening the drawer starts fresh. + sidePanelStack.clearTrail(); // Clear waterfall search state on close, so that filters don't // persist when reopening another trace. clearTraceWaterfallSearchState(); onClose(); - }, [setQueryTab, isNestedPanel, onClose, clearTraceWaterfallSearchState]); - - useHotkeys(['esc'], _onClose, { enabled: subDrawerOpen === false }); + }, [sidePanelStack, onClose, clearTraceWaterfallSearchState]); return ( { - if (!subDrawerOpen) { - _onClose(); - } - }} + closeOnEscape={false} + lockScroll={false} + withOverlay={false} + trapFocus={false} + onClose={_onClose} position="right" size={`${size}vw`} styles={{ + content: { + border: 'none', + boxShadow: 'var(--shadow-drawer)', + }, body: { padding: '0', height: '100%', @@ -615,37 +1108,26 @@ export default function DBRowSidePanelErrorBoundary({ }} zIndex={drawerZIndex} > - +
( - -
- An error occurred while rendering this event. -
- -
- {error?.error?.message} -
- + fallbackRender={fallbackProps => ( + )} > -
- + ); } diff --git a/packages/app/src/components/DBRowSidePanel.types.ts b/packages/app/src/components/DBRowSidePanel.types.ts new file mode 100644 index 0000000000..f36b22b5c1 --- /dev/null +++ b/packages/app/src/components/DBRowSidePanel.types.ts @@ -0,0 +1,38 @@ +import { z } from 'zod'; +import { SourceKind, WithClauseSchema } from '@hyperdx/common-utils/dist/types'; + +export enum Tab { + Overview = 'overview', + Parsed = 'parsed', + Debug = 'debug', + Trace = 'trace', + ServiceMap = 'serviceMap', + Context = 'context', + Replay = 'replay', + Infrastructure = 'infrastructure', +} + +const NavEntrySchema = z.object({ + rowId: z.string(), + aliasWith: z.array(WithClauseSchema), + label: z.string(), + sourceKind: z.nativeEnum(SourceKind).optional(), + originTab: z.nativeEnum(Tab).optional(), +}); + +export type NavEntry = z.infer; + +const SourceFrameSchema = z.object({ + sourceId: z.string(), + rowId: z.string(), + aliasWith: z.array(WithClauseSchema), + label: z.string(), + sourceKind: z.nativeEnum(SourceKind).optional(), + originTab: z.nativeEnum(Tab).optional(), +}); + +export type SourceFrame = z.infer; + +export const parseSourceStack = z.array(SourceFrameSchema).parse; + +export const parseNavStack = z.array(NavEntrySchema).parse; diff --git a/packages/app/src/components/DBRowSidePanelHeader.tsx b/packages/app/src/components/DBRowSidePanelHeader.tsx index b37a1f383a..a253d71fe9 100644 --- a/packages/app/src/components/DBRowSidePanelHeader.tsx +++ b/packages/app/src/components/DBRowSidePanelHeader.tsx @@ -1,135 +1,20 @@ -import React, { - useCallback, - useContext, - useEffect, - useMemo, - useState, -} from 'react'; -import { - ActionIcon, - Box, - Breadcrumbs, - Button, - Flex, - Group, - Paper, - Text, - Tooltip, - UnstyledButton, -} from '@mantine/core'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { Box, Button, Flex, Paper, Text } from '@mantine/core'; import { IconArrowsDiagonal, IconArrowsDiagonalMinimize2, - IconKeyboard, } from '@tabler/icons-react'; -import { KeyboardShortcutsModal } from '@/LogSidePanelElements'; -import { FormatTime } from '@/useFormatTime'; import { useUserPreferences } from '@/useUserPreferences'; -import { formatDistanceToNowStrictShort } from '@/utils'; import AISummarizeButton from './AISummarizeButton'; import { DBHighlightedAttributesList, HighlightedAttribute, } from './DBHighlightedAttributesList'; -import { RowSidePanelContext } from './DBRowSidePanel'; -import { DrawerFullWidthToggle } from './DrawerUtils'; -import LogLevel from './LogLevel'; - -const isValidDate = (date: Date) => 'getTime' in date && !isNaN(date.getTime()); const MAX_MAIN_CONTENT_LENGTH = 2000; -// Types for breadcrumb navigation -export type BreadcrumbEntry = { - label: string; - rowData?: Record; -}; - -export type BreadcrumbPath = BreadcrumbEntry[]; - -// Navigation callback type - called when user wants to navigate to a specific level -export type BreadcrumbNavigationCallback = (targetLevel: number) => void; - -function getBodyTextForBreadcrumb(rowData: Record): string { - const bodyText = (rowData.__hdx_body || '').trim(); - const BREADCRUMB_TOOLTIP_MAX_LENGTH = 200; - const BREADCRUMB_TOOLTIP_TRUNCATED_LENGTH = 197; - - return bodyText.length > BREADCRUMB_TOOLTIP_MAX_LENGTH - ? `${bodyText.substring(0, BREADCRUMB_TOOLTIP_TRUNCATED_LENGTH)}...` - : bodyText; -} - -function BreadcrumbNavigation({ - breadcrumbPath, - onNavigateToLevel, -}: { - breadcrumbPath: BreadcrumbPath; - onNavigateToLevel?: BreadcrumbNavigationCallback; -}) { - const handleBreadcrumbItemClick = useCallback( - (clickedIndex: number) => { - // Navigate to the clicked breadcrumb level - // This will close all panels above this level - onNavigateToLevel?.(clickedIndex); - }, - [onNavigateToLevel], - ); - - const breadcrumbItems = useMemo(() => { - if (breadcrumbPath.length === 0) return []; - - const items = []; - - // Add all previous levels from breadcrumbPath - breadcrumbPath.forEach((crumb, index) => { - const tooltipText = crumb.rowData - ? getBodyTextForBreadcrumb(crumb.rowData) - : ''; - - items.push( - - handleBreadcrumbItemClick(index)} - style={{ textDecoration: 'none' }} - > - - {index === 0 ? 'Original Event' : crumb.label} - - - , - ); - }); - - // Add current level - items.push( - - Selected Event - , - ); - - return items; - }, [breadcrumbPath, handleBreadcrumbItemClick]); - - if (breadcrumbPath.length === 0) return null; - - return ( - - - {breadcrumbItems} - - - ); -} - export default function DBRowSidePanelHeader({ attributes, mainContent = '', @@ -137,32 +22,19 @@ export default function DBRowSidePanelHeader({ // When `true`, the source has a body column configured. An empty value // for that column renders a soft empty-state paper. When `false` (the // source has neither body nor implicit column configured), the body - // paper is suppressed entirely; the header still shows timestamp, - // severity, and highlighted attributes. + // paper is suppressed entirely; the highlighted attributes still render. bodyConfigured = true, - date, severityText, rowData, - breadcrumbPath, - onBreadcrumbClick, - isFullWidth, - onToggleFullWidth, }: { - date: Date; mainContent?: string; mainContentHeader?: string; bodyConfigured?: boolean; attributes?: HighlightedAttribute[]; severityText?: string; rowData?: Record; - breadcrumbPath?: BreadcrumbPath; - onBreadcrumbClick?: BreadcrumbNavigationCallback; - isFullWidth?: boolean; - onToggleFullWidth?: () => void; }) { const [bodyExpanded, setBodyExpanded] = React.useState(false); - const [shortcutsOpen, setShortcutsOpen] = useState(false); - const { generateSearchUrl } = useContext(RowSidePanelContext); const isContentTruncated = mainContent.length > MAX_MAIN_CONTENT_LENGTH; const mainContentDisplayed = React.useMemo( @@ -200,74 +72,19 @@ export default function DBRowSidePanelHeader({ const { expandSidebarHeader } = userPreferences; const maxBoxHeight = 120; - const _generateSearchUrl = useCallback( - (query?: string, queryLanguage?: 'sql' | 'lucene') => { - return ( - generateSearchUrl?.({ - where: query, - whereLanguage: queryLanguage, - }) ?? '/' - ); - }, - [generateSearchUrl], - ); - - const breadCrumbPathWithDefault = useMemo(() => { - return breadcrumbPath ?? []; - }, [breadcrumbPath]); - const attributesWithDefault = useMemo(() => { return attributes ?? []; }, [attributes]); + const toggleExpandSidebarHeader = useCallback(() => { + setUserPreference({ + ...userPreferences, + expandSidebarHeader: !expandSidebarHeader, + }); + }, [expandSidebarHeader, setUserPreference, userPreferences]); + return ( <> - {/* Breadcrumb navigation */} - - - {/* Event timestamp and severity */} - - - {severityText && } - {severityText && isValidDate(date) && ( - - · - - )} - {isValidDate(date) && ( - - ·{' '} - {formatDistanceToNowStrictShort(date)} ago - - )} - - - - setShortcutsOpen(true)} - aria-label="Keyboard shortcuts" - > - - - - {onToggleFullWidth && ( - - )} - - - setShortcutsOpen(false)} - /> {!bodyConfigured ? null : mainContent ? ( - setUserPreference({ - ...userPreferences, - expandSidebarHeader: !expandSidebarHeader, - }) - } + onClick={toggleExpandSidebarHeader} > - {/* TODO: Only show expand button when maxHeight = 120? */} {expandSidebarHeader ? ( ) : ( diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 89b1ae768b..4aa39c5b76 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1524,7 +1524,10 @@ function DBSqlRowTableComponent({ }: { config: BuilderChartConfigWithDateRange; sourceId?: string; - onRowDetailsClick?: (rowWhere: RowWhereResult) => void; + onRowDetailsClick?: ( + rowWhere: RowWhereResult, + row: Record, + ) => void; highlightedLineId?: string; queryKeyPrefix?: string; enabled?: boolean; @@ -1707,7 +1710,7 @@ function DBSqlRowTableComponent({ const _onRowDetailsClick = useCallback( (row: Record) => { - return onRowDetailsClick?.(getRowWhere(row)); + return onRowDetailsClick?.(getRowWhere(row), row); }, [onRowDetailsClick, getRowWhere], ); diff --git a/packages/app/src/components/DBSessionPanel.tsx b/packages/app/src/components/DBSessionPanel.tsx index b248fd35f3..c9d84a20e2 100644 --- a/packages/app/src/components/DBSessionPanel.tsx +++ b/packages/app/src/components/DBSessionPanel.tsx @@ -4,6 +4,7 @@ import { isTraceSource, SourceKind } from '@hyperdx/common-utils/dist/types'; import { Loader } from '@mantine/core'; import useFieldExpressionGenerator from '@/hooks/useFieldExpressionGenerator'; +import { WithClause } from '@/hooks/useRowWhere'; import SessionSubpanel from '@/SessionSubpanel'; import { useSource } from '@/source'; @@ -93,14 +94,14 @@ export const DBSessionPanel = ({ dateRange, focusDate, serviceName, - setSubDrawerOpen, + onEventNavigate, }: { traceSourceId?: string; rumSessionId: string; dateRange: [Date, Date]; focusDate: Date; serviceName: string; - setSubDrawerOpen: (open: boolean) => void; + onEventNavigate?: (rowId: string, aliasWith: WithClause[]) => void; }) => { const { data: traceSource } = useSource({ id: traceSourceId, @@ -136,7 +137,7 @@ export const DBSessionPanel = ({ session={{ serviceName }} sessionSource={sessionSource} rumSessionId={rumSessionId} - setDrawerOpen={setSubDrawerOpen} + onEventNavigate={onEventNavigate} initialTs={focusDate.getTime()} /> ) : ( diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index b197486669..b1a4ab6983 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -17,14 +17,12 @@ import { useLocalStorage } from '@/utils'; import { parseAsStringEncoded } from '@/utils/queryParsers'; import { ChartErrorStateVariant } from './charts/ChartErrorState'; -import { useNestedPanelState } from './ContextSidePanel'; import { RowDataPanel } from './DBRowDataPanel'; import { RowOverviewPanel } from './DBRowOverviewPanel'; import DBRowSidePanel, { RowSidePanelContext, RowSidePanelContextProps, } from './DBRowSidePanel'; -import { BreadcrumbEntry } from './DBRowSidePanelHeader'; import { DBRowTableVariant, DBSqlRowTable } from './DBRowTable'; interface Props { @@ -41,8 +39,6 @@ interface Props { queryKeyPrefix?: string; denoiseResults?: boolean; collapseAllRows?: boolean; - isNestedPanel?: boolean; - breadcrumbPath?: BreadcrumbEntry[]; onSortingChange?: (v: SortingState | null) => void; initialSortBy?: SortingState; variant?: DBRowTableVariant; @@ -63,8 +59,6 @@ export default function DBSqlRowTableWithSideBar({ collapseAllRows, isLive, enabled, - isNestedPanel, - breadcrumbPath, onSidebarOpen, onSortingChange, initialSortBy, @@ -78,7 +72,6 @@ export default function DBSqlRowTableWithSideBar({ const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded); const [rowSource, setRowSource] = useQueryState('rowSource'); const [aliasWith, setAliasWith] = useState([]); - const { setContextRowId, setContextRowSource } = useNestedPanelState(); const onOpenSidebar = useCallback( (rowWhere: RowWhereResult) => { @@ -93,19 +86,7 @@ export default function DBSqlRowTableWithSideBar({ const onCloseSidebar = useCallback(() => { setRowId(null); setRowSource(null); - // When closing the main drawer, clear the nested panel state - // this ensures that re-opening the main drawer will not open the nested panel - if (!isNestedPanel) { - setContextRowId(null); - setContextRowSource(null); - } - }, [ - setRowId, - setRowSource, - isNestedPanel, - setContextRowId, - setContextRowSource, - ]); + }, [setRowId, setRowSource]); const renderRowDetails = useCallback( (r: { id: string; aliasWith?: WithClause[]; [key: string]: unknown }) => { if (!sourceData) { @@ -129,8 +110,6 @@ export default function DBSqlRowTableWithSideBar({ source={sourceData} rowId={rowId ?? undefined} aliasWith={aliasWith} - isNestedPanel={isNestedPanel} - breadcrumbPath={breadcrumbPath} onClose={onCloseSidebar} /> )} diff --git a/packages/app/src/components/DBTracePanel.tsx b/packages/app/src/components/DBTracePanel.tsx index 7f33c47f54..f5743f38a1 100644 --- a/packages/app/src/components/DBTracePanel.tsx +++ b/packages/app/src/components/DBTracePanel.tsx @@ -1,12 +1,14 @@ import { ReactNode, useCallback, useEffect, useMemo, useState } from 'react'; import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; +import { z } from 'zod'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { isLogSource, isTraceSource, SourceKind, TSource, + WithClauseSchema, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, @@ -43,9 +45,23 @@ type EventRowWhere = { id: string; type: string; aliasWith: WithClause[]; + // The trace this span selection was made in. Used to gate a selection left in + // the URL from a previous trace so it can't render against a different one. + traceId?: string; }; -const eventRowWhereParser = parseAsJsonEncoded(); +// Validate the persisted span selection so a stale / hand-edited `eventRowWhere` +// (valid JSON but wrong shape) resolves to null instead of feeding a malformed +// row id into the span detail query. +const eventRowWhereSchema = z.object({ + id: z.string(), + type: z.string(), + aliasWith: z.array(WithClauseSchema), + traceId: z.string().optional(), +}); +const eventRowWhereParser = parseAsJsonEncoded( + eventRowWhereSchema.parse, +); enum SpanDetailTab { Overview = 'overview', @@ -208,6 +224,25 @@ export default function DBTracePanel({ eventRowWhereParser, ); + // A persisted span selection belongs to the trace it was made in. Gate it by + // the current `traceId` at *read* time so a selection left in the URL from a + // previous trace (e.g. after "View Trace" opened a different trace, or an old + // shared link) can never render against this trace's waterfall — no matter how + // the panel was (re)mounted. + const selectedSpan = + eventRowWhere != null && eventRowWhere.traceId === traceId + ? eventRowWhere + : null; + + // Stamp the current trace onto every selection so the gate above can tell it + // apart from a stale one. + const selectSpan = useCallback( + (where: { id: string; type: string; aliasWith: WithClause[] }) => { + setEventRowWhere({ ...where, traceId }); + }, + [setEventRowWhere, traceId], + ); + const { control: traceIdControl, handleSubmit: traceIdHandleSubmit, @@ -233,13 +268,14 @@ export default function DBTracePanel({ const [showTraceIdInput, setShowTraceIdInput] = useState(false); - // Reset highlighted row when trace ID changes - // otherwise we'll show stale span details + // Tidy the URL: drop a selection that belongs to a different trace. Purely + // cosmetic — the read-time `selectedSpan` gate already prevents a foreign + // selection from rendering; this just keeps the param from lingering. useEffect(() => { - return () => { + if (eventRowWhere != null && eventRowWhere.traceId !== traceId) { setEventRowWhere(null); - }; - }, [traceId, setEventRowWhere]); + } + }, [eventRowWhere, traceId, setEventRowWhere]); const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = useState(false); @@ -254,12 +290,12 @@ export default function DBTracePanel({ }, [setEventRowWhere]); const selectedSpanSource = useMemo(() => { - if (!eventRowWhere) return null; - if (eventRowWhere.type === SourceKind.Log && logSourceData) { + if (!selectedSpan) return null; + if (selectedSpan.type === SourceKind.Log && logSourceData) { return logSourceData; } return traceSourceData; - }, [eventRowWhere, logSourceData, traceSourceData]); + }, [selectedSpan, logSourceData, traceSourceData]); return (
)}
- {eventRowWhere != null && ( + {selectedSpan != null && (
diff --git a/packages/app/src/components/PatternSidePanel.tsx b/packages/app/src/components/PatternSidePanel.tsx index 994437c78b..2982ba2b35 100644 --- a/packages/app/src/components/PatternSidePanel.tsx +++ b/packages/app/src/components/PatternSidePanel.tsx @@ -176,8 +176,6 @@ export default function PatternSidePanel({ rowId={selectedRowWhere.where} aliasWith={selectedRowWhere.aliasWith} onClose={handleCloseRowSidePanel} - isNestedPanel={true} - breadcrumbPath={[{ label: 'Pattern Overview' }]} /> )}
diff --git a/packages/app/src/components/Search/DirectTraceSidePanel.tsx b/packages/app/src/components/Search/DirectTraceSidePanel.tsx index 6e38791916..7add520964 100644 --- a/packages/app/src/components/Search/DirectTraceSidePanel.tsx +++ b/packages/app/src/components/Search/DirectTraceSidePanel.tsx @@ -94,6 +94,9 @@ export default function DirectTraceSidePanel({ onClose={onClose} position="right" size="75vw" + lockScroll={false} + withOverlay={false} + trapFocus={false} title={ diff --git a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx index 32e8823079..934d8e5f4f 100644 --- a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx +++ b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx @@ -86,8 +86,6 @@ export default function SlowestEventsTile({ expressions && ( <> void; +}; + +function SourceIcon({ kind }: { kind?: SourceKind }) { + if (kind === SourceKind.Trace) { + return ; + } + if (kind === SourceKind.Log) { + return ; + } + if (kind === SourceKind.Session) { + return ; + } + return null; +} + +function truncate(text: string, max: number) { + if (text.length <= max) return text; + return `${text.slice(0, max)}…`; +} + +function SidePanelBreadcrumbs({ + items, + onBack, +}: { + items: BreadcrumbItem[]; + onBack: () => void; +}) { + const breadcrumbElements = useMemo(() => { + return items.map((item, i) => { + const isLast = i === items.length - 1; + const isSingle = items.length === 1; + const maxLen = isSingle + ? MAX_LABEL_LENGTH_SINGLE + : isLast + ? MAX_LABEL_LENGTH_CURRENT + : MAX_LABEL_LENGTH_PREVIOUS; + const truncatedLabel = truncate(item.label, maxLen); + const needsTooltip = item.label.length > maxLen; + + const content = ( + + {i === 0 && } + + {truncatedLabel} + + + ); + + const wrapped = needsTooltip ? ( + {item.label}} + position="bottom" + multiline + maw={400} + > + {content} + + ) : ( + content + ); + + if (isLast || !item.onClick) { + return ( + + {wrapped} + + ); + } + + return ( + + + {wrapped} + + + ); + }); + }, [items]); + + return ( + + + + + + + {items.length > 0 ? ( + + {breadcrumbElements} + + ) : null} + + ); +} + +export default memo(SidePanelBreadcrumbs); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx new file mode 100644 index 0000000000..44b7f844cd --- /dev/null +++ b/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx @@ -0,0 +1,275 @@ +import React from 'react'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { MantineProvider } from '@mantine/core'; +import { fireEvent, render, screen } from '@testing-library/react'; + +// Controlled, in-memory replacement for nuqs' useQueryState so each side-panel +// URL param can be seeded and its setter inspected independently. Values are +// the already-parsed shapes the component consumes (arrays / strings), not URL +// strings. Prefixed with `mock` so jest.mock's factory may reference them. +const mockQueryStore: Record = {}; +const mockSetters: Record = {}; + +function seedParam(key: string, value: unknown) { + mockQueryStore[key] = value; +} +function setterFor(key: string) { + if (!mockSetters[key]) mockSetters[key] = jest.fn(); + return mockSetters[key]; +} +function resetQueryState() { + Object.keys(mockQueryStore).forEach(k => delete mockQueryStore[k]); + Object.keys(mockSetters).forEach(k => delete mockSetters[k]); +} + +jest.mock('nuqs', () => { + const actual = jest.requireActual('nuqs'); + return { + ...actual, + // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { + const hasValue = Object.prototype.hasOwnProperty.call( + mockQueryStore, + key, + ); + const fallback = + parser && 'defaultValue' in parser ? parser.defaultValue : null; + const value = hasValue ? mockQueryStore[key] : (fallback ?? null); + const setters = mockSetters; + if (!setters[key]) setters[key] = jest.fn(); + return [value, setters[key]]; + }, + }; +}); + +// Row data resolves off the requested rowId: when the panel skips the row query +// (leaf source loading or unresolvable) it passes `rowId: undefined`, which must +// NOT report `isLoading` — otherwise the loading branch would mask the +// missing-source branch we are asserting. +const mockUseRowData = jest.fn(); +jest.mock('../DBRowDataPanel', () => ({ + __esModule: true, + useRowData: (args: { rowId?: string }) => mockUseRowData(args), + ROW_DATA_ALIASES: { + DURATION_MS: '__hdx_duration', + SPAN_KIND: '__hdx_span_kind', + SERVICE_NAME: '__hdx_service_name', + SEVERITY_TEXT: '__hdx_severity_text', + }, + rowHasK8sContext: () => false, + RowDataPanel: () => null, +})); + +// Per-id source resolution state. An id absent from the store models the +// react-query `select` returning `undefined` for a deleted/renamed id even +// after the query has settled successfully. +const mockSourceStore: Record< + string, + { data: unknown; isLoading: boolean; isSuccess: boolean } +> = {}; +jest.mock('@/source', () => ({ + __esModule: true, + getEventBody: () => '__hdx_body', + useSource: ({ id }: { id: string | null }) => { + if (id == null) { + return { data: undefined, isLoading: false, isSuccess: false }; + } + if (Object.prototype.hasOwnProperty.call(mockSourceStore, id)) { + return mockSourceStore[id]; + } + // Settled, but the id no longer maps to a source. + return { data: undefined, isLoading: false, isSuccess: true }; + }, +})); + +jest.mock('../DBSessionPanel', () => ({ + __esModule: true, + useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }), + DBSessionPanel: () => null, +})); + +jest.mock('@/utils/highlightedAttributes', () => ({ + __esModule: true, + getHighlightedAttributesFromData: () => [], +})); + +// Heavy leaf components / chart deps that the panel imports but never renders +// under the loading / missing-source branches. Stub them so the module graph +// stays cheap to load. NOTE: SidePanelBreadcrumbs is intentionally *not* mocked +// so we can assert the Back control is present in the error state. +jest.mock('../DBTracePanel', () => ({ __esModule: true, default: () => null })); +jest.mock('../ContextSidePanel', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../DBInfraPanel', () => ({ __esModule: true, default: () => null })); +jest.mock('../DBRowOverviewPanel', () => ({ + __esModule: true, + RowOverviewPanel: () => null, +})); +jest.mock('../DBRowSidePanelErrorState', () => ({ + __esModule: true, + DBRowSidePanelErrorState: () => null, +})); +jest.mock('../DBRowSidePanelHeader', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../LogLevel', () => ({ __esModule: true, default: () => null })); +jest.mock('../ServiceMap/ServiceMapSidePanel', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../TimelineChart/utils', () => ({ + __esModule: true, + renderMs: () => '', +})); +jest.mock('../DrawerUtils', () => ({ + __esModule: true, + DrawerFullWidthToggle: () => null, + INITIAL_DRAWER_WIDTH_PERCENT: 50, +})); +jest.mock('@/LogSidePanelElements', () => ({ + __esModule: true, + KeyboardShortcutsModal: () => null, +})); +jest.mock('@/TabBar', () => ({ __esModule: true, default: () => null })); +jest.mock('@/useFormatTime', () => ({ + __esModule: true, + FormatTime: () => null, +})); + +// NOTE: this import is intentionally placed after the mock factories above, +// which close over the `mock*` helpers declared at the top of this file. +import { DBRowSidePanelInner } from '@/components/DBRowSidePanel'; +import useSidePanelStack from '@/hooks/useSidePanelStack'; + +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +const ROOT_SOURCE = { + id: 'log-src', + kind: 'log', + traceSourceId: 'trace-src', +} as TSource; + +// A cross-source frame (e.g. "View Trace") whose source id no longer resolves. +const MISSING_FRAME = { + sourceId: 'deleted-src', + rowId: 'leaf-row', + aliasWith: [], + label: 'Deleted Trace', + sourceKind: 'trace', +}; + +function InnerHarness({ rowId }: { rowId: string }) { + const sidePanelStack = useSidePanelStack({ initialRowId: rowId }); + return ( + + ); +} + +function renderInner(rowId: string) { + return render( + + + , + ); +} + +describe('DBRowSidePanelInner — unresolvable stack source (HDX-3942 permanent-loading P0)', () => { + beforeEach(() => { + resetQueryState(); + Object.keys(mockSourceStore).forEach(k => delete mockSourceStore[k]); + mockUseRowData.mockReset(); + // Not loading when there is no row to query (the missing/loading source + // branches pass rowId: undefined); loading otherwise. + mockUseRowData.mockImplementation((args: { rowId?: string }) => + args?.rowId == null + ? { + data: undefined, + isLoading: false, + isSuccess: false, + isError: false, + error: null, + } + : { + data: undefined, + isLoading: true, + isSuccess: false, + isError: false, + error: null, + }, + ); + }); + + it('renders an error state with working Back/Close instead of hanging on Loading when the leaf source id no longer resolves', () => { + // A shared URL points at a cross-source frame whose source was deleted / + // renamed / lives in another workspace. Its stackRoot matches the mounted + // row so the trail is treated as valid (not stale). + seedParam('sidePanelSourceStack', [MISSING_FRAME]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-row'); + + renderInner('root-row'); + + // Not stuck on "Loading..." — an explicit, actionable message is shown. + expect(screen.queryByText('Loading...')).not.toBeInTheDocument(); + expect( + screen.getByText(/source is no longer available/i), + ).toBeInTheDocument(); + + // The trail collapse controls remain usable. + const back = screen.getByLabelText('Back'); + const close = screen.getByLabelText('Close'); + expect(back).toBeInTheDocument(); + expect(close).toBeInTheDocument(); + + // Back pops the broken frame off the source stack. + fireEvent.click(back); + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalled(); + }); + + it('still shows Loading while the leaf source query is genuinely in flight', () => { + mockSourceStore['deleted-src'] = { + data: undefined, + isLoading: true, + isSuccess: false, + }; + seedParam('sidePanelSourceStack', [MISSING_FRAME]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-row'); + + renderInner('root-row'); + + expect(screen.getByText('Loading...')).toBeInTheDocument(); + expect( + screen.queryByText(/source is no longer available/i), + ).not.toBeInTheDocument(); + }); + + it('renders the row normally once the leaf source resolves', () => { + mockSourceStore['deleted-src'] = { + data: { id: 'deleted-src', kind: 'trace' }, + isLoading: false, + isSuccess: true, + }; + seedParam('sidePanelSourceStack', [MISSING_FRAME]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-row'); + + renderInner('root-row'); + + // Source resolved → the leaf row is queried (isLoading true → "Loading..."), + // and the missing-source error state is not shown. + expect( + screen.queryByText(/source is no longer available/i), + ).not.toBeInTheDocument(); + const resolvedRowId = mockUseRowData.mock.calls.at(-1)?.[0]?.rowId; + expect(resolvedRowId).toBe('leaf-row'); + }); +}); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx new file mode 100644 index 0000000000..471f270e14 --- /dev/null +++ b/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx @@ -0,0 +1,255 @@ +import React from 'react'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { MantineProvider } from '@mantine/core'; +import { render } from '@testing-library/react'; + +// Controlled, in-memory replacement for nuqs' useQueryState so each side-panel +// URL param can be seeded and its setter inspected independently. Values are +// the already-parsed shapes the component consumes (arrays / strings), not URL +// strings. Prefixed with `mock` so jest.mock's factory may reference them. +const mockQueryStore: Record = {}; +const mockSetters: Record = {}; + +function seedParam(key: string, value: unknown) { + mockQueryStore[key] = value; +} +function setterFor(key: string) { + if (!mockSetters[key]) mockSetters[key] = jest.fn(); + return mockSetters[key]; +} +function resetQueryState() { + Object.keys(mockQueryStore).forEach(k => delete mockQueryStore[k]); + Object.keys(mockSetters).forEach(k => delete mockSetters[k]); +} + +jest.mock('nuqs', () => { + const actual = jest.requireActual('nuqs'); + return { + ...actual, + // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { + const hasValue = Object.prototype.hasOwnProperty.call( + mockQueryStore, + key, + ); + const fallback = + parser && 'defaultValue' in parser ? parser.defaultValue : null; + const value = hasValue ? mockQueryStore[key] : (fallback ?? null); + const setters = mockSetters; + if (!setters[key]) setters[key] = jest.fn(); + return [value, setters[key]]; + }, + }; +}); + +// Capture what row the panel actually resolves to. Returning isLoading short- +// circuits the render before the heavy body/header, so those children never +// mount and the `rowId` arg is the single source of truth for "what is shown". +const mockUseRowData = jest.fn(); +jest.mock('../DBRowDataPanel', () => ({ + __esModule: true, + useRowData: (args: unknown) => mockUseRowData(args), + ROW_DATA_ALIASES: { + DURATION_MS: '__hdx_duration', + SPAN_KIND: '__hdx_span_kind', + SERVICE_NAME: '__hdx_service_name', + SEVERITY_TEXT: '__hdx_severity_text', + }, + rowHasK8sContext: () => false, + RowDataPanel: () => null, +})); + +jest.mock('@/source', () => ({ + __esModule: true, + getEventBody: () => '__hdx_body', + // A non-null id resolves to a trace source so `isResolvingSource` is false and + // the leaf frame's row id is used; a null id (no active frame) resolves to + // undefined so the panel falls back to the root source. + useSource: ({ id }: { id: string | null }) => + id ? { data: { id, kind: 'trace' } } : { data: undefined }, +})); + +jest.mock('../DBSessionPanel', () => ({ + __esModule: true, + useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }), + DBSessionPanel: () => null, +})); + +jest.mock('@/utils/highlightedAttributes', () => ({ + __esModule: true, + getHighlightedAttributesFromData: () => [], +})); + +// Heavy leaf components / chart deps that the panel imports but never renders +// under isLoading. Stub them so the module graph stays cheap to load. +jest.mock('../DBTracePanel', () => ({ __esModule: true, default: () => null })); +jest.mock('../ContextSidePanel', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../DBInfraPanel', () => ({ __esModule: true, default: () => null })); +jest.mock('../DBRowOverviewPanel', () => ({ + __esModule: true, + RowOverviewPanel: () => null, +})); +jest.mock('../DBRowSidePanelErrorState', () => ({ + __esModule: true, + DBRowSidePanelErrorState: () => null, +})); +jest.mock('../DBRowSidePanelHeader', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../SidePanelBreadcrumbs', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../LogLevel', () => ({ __esModule: true, default: () => null })); +jest.mock('../ServiceMap/ServiceMapSidePanel', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../TimelineChart/utils', () => ({ + __esModule: true, + renderMs: () => '', +})); +jest.mock('../DrawerUtils', () => ({ + __esModule: true, + DrawerFullWidthToggle: () => null, + INITIAL_DRAWER_WIDTH_PERCENT: 50, +})); +jest.mock('@/LogSidePanelElements', () => ({ + __esModule: true, + KeyboardShortcutsModal: () => null, +})); +jest.mock('@/TabBar', () => ({ __esModule: true, default: () => null })); +jest.mock('@/useFormatTime', () => ({ + __esModule: true, + FormatTime: () => null, +})); + +// NOTE: this import is intentionally placed after the mock factories above, +// which close over the `mock*` helpers declared at the top of this file. +import { DBRowSidePanelInner } from '@/components/DBRowSidePanel'; +import useSidePanelStack from '@/hooks/useSidePanelStack'; + +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +const ROOT_SOURCE = { + id: 'log-src', + kind: 'log', + traceSourceId: 'trace-src', +} as TSource; + +const TRACE_FRAME = { + sourceId: 'trace-src', + rowId: 'leaf-row', + aliasWith: [], + label: 'Trace', + sourceKind: 'trace', +}; + +function InnerHarness({ rowId }: { rowId: string }) { + const sidePanelStack = useSidePanelStack({ initialRowId: rowId }); + return ( + + ); +} + +function renderInner(rowId: string) { + return render( + + + , + ); +} + +function lastResolvedRowId() { + return mockUseRowData.mock.calls.at(-1)?.[0]?.rowId; +} + +describe('DBRowSidePanelInner — stale stack handling (HDX-3942 "Stale Stack Remains")', () => { + beforeEach(() => { + resetQueryState(); + mockUseRowData.mockReset(); + mockUseRowData.mockReturnValue({ + data: undefined, + isLoading: true, + isSuccess: false, + isError: false, + error: null, + }); + }); + + it('preserves a deep-linked trail when the stack root matches the mounted row', () => { + // A shared URL: stacks + a stackRoot that belongs to the mounted root row. + seedParam('sidePanelSourceStack', [TRACE_FRAME]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-row'); + + renderInner('root-row'); + + // Trail is honoured: the drawer resolves the leaf frame's row, not the root. + expect(lastResolvedRowId()).toBe('leaf-row'); + // ...and nothing is cleared. + expect(setterFor('sidePanelSourceStack')).not.toHaveBeenCalled(); + expect(setterFor('sidePanelNavStack')).not.toHaveBeenCalled(); + expect(setterFor('sidePanelStackRoot')).not.toHaveBeenCalled(); + }); + + it('ignores a stale trail left in the URL when a different root row mounts', () => { + // Stacks survived an unclosed drawer / cross-table switch: their stackRoot + // points at a *previous* root, but a different row is now mounted. + seedParam('sidePanelSourceStack', [TRACE_FRAME]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'old-root'); + + renderInner('new-root'); + + // The row the user actually opened wins over the stale leaf. Correctness is + // guaranteed at *read* time (the owner-gated trail is empty), so it does + // NOT depend on any clearing effect firing. + expect(lastResolvedRowId()).toBe('new-root'); + }); + + it('ignores the trail when the root row changes while the panel stays mounted', () => { + seedParam('sidePanelSourceStack', [TRACE_FRAME]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-1'); + + const { rerender } = renderInner('root-1'); + // Same-root render keeps the trail. + expect(lastResolvedRowId()).toBe('leaf-row'); + expect(setterFor('sidePanelSourceStack')).not.toHaveBeenCalled(); + + // The mounted panel now points at a different root row (e.g. the user + // clicked another row in the same table); the trail is stale and ignored. + rerender( + + + , + ); + + expect(lastResolvedRowId()).toBe('root-2'); + }); + + it('treats an ownerless trail (no recorded root) as stale and ignores it', () => { + // Malformed / truncated / pre-stackRoot URL: stacks present but no + // stackRoot token. Every real push writes stackRoot alongside the stacks + // (and shared links copy the full URL), so a stack we cannot prove owns the + // mounted row must not shadow the row the user actually clicked. + seedParam('sidePanelSourceStack', [TRACE_FRAME]); + seedParam('sidePanelNavStack', []); + // sidePanelStackRoot intentionally unseeded (null): ownerless. + + renderInner('some-row'); + + // The clicked row wins over the ownerless leaf at read time. + expect(lastResolvedRowId()).toBe('some-row'); + }); +}); diff --git a/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx index 939a1f62d9..0bd32b7f3c 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx @@ -42,7 +42,6 @@ describe('DBRowSidePanelHeader: body section (HDX-4373)', () => { it('renders the configured body content when bodyConfigured + mainContent are truthy', () => { renderWithMantine( { it('renders the softened empty state when body is configured but value is empty', () => { renderWithMantine( { it('suppresses the body paper entirely when body is not configured on the source', () => { renderWithMantine( { }); it('defaults bodyConfigured to true (back-compat with callers that do not pass it)', () => { - renderWithMantine( - , - ); + renderWithMantine(); expect(screen.queryByText('No body for this event.')).toBeInTheDocument(); }); }); diff --git a/packages/app/src/components/__tests__/SidePanelBreadcrumbs.test.tsx b/packages/app/src/components/__tests__/SidePanelBreadcrumbs.test.tsx new file mode 100644 index 0000000000..7957ef9f37 --- /dev/null +++ b/packages/app/src/components/__tests__/SidePanelBreadcrumbs.test.tsx @@ -0,0 +1,72 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { MantineProvider } from '@mantine/core'; +import { fireEvent, render, screen } from '@testing-library/react'; + +import SidePanelBreadcrumbs, { + BreadcrumbItem, +} from '@/components/SidePanelBreadcrumbs'; + +function renderCrumbs(items: BreadcrumbItem[], onBack = jest.fn()) { + const utils = render( + + + , + ); + return { ...utils, onBack }; +} + +describe('SidePanelBreadcrumbs', () => { + it('renders each item label', () => { + renderCrumbs([ + { label: 'Root', sourceKind: SourceKind.Log }, + { label: 'Leaf', sourceKind: SourceKind.Trace }, + ]); + expect(screen.getByText('Root')).toBeInTheDocument(); + expect(screen.getByText('Leaf')).toBeInTheDocument(); + }); + + it('calls onBack when the back button is clicked', () => { + const { onBack } = renderCrumbs([{ label: 'Root' }]); + fireEvent.click(screen.getByLabelText('Back')); + expect(onBack).toHaveBeenCalledTimes(1); + }); + + it('invokes onClick for a non-last (ancestor) breadcrumb', () => { + const onClick = jest.fn(); + renderCrumbs([{ label: 'Ancestor', onClick }, { label: 'Current' }]); + fireEvent.click(screen.getByText('Ancestor')); + expect(onClick).toHaveBeenCalledTimes(1); + }); + + it('does not make the last (current) item clickable', () => { + const onClick = jest.fn(); + // Even though onClick is supplied, the leaf is the current view. + renderCrumbs([{ label: 'Ancestor' }, { label: 'Current', onClick }]); + fireEvent.click(screen.getByText('Current')); + expect(onClick).not.toHaveBeenCalled(); + }); + + it('truncates an over-length single label with an ellipsis', () => { + const long = 'x'.repeat(150); // > MAX_LABEL_LENGTH_SINGLE (120) + const { container } = renderCrumbs([{ label: long }]); + // The full label is not rendered verbatim; the visible text is truncated. + expect(screen.queryByText(long)).not.toBeInTheDocument(); + expect(container.textContent).toContain('…'); + }); + + it('truncates an ancestor label more aggressively than the current one', () => { + const label = 'y'.repeat(30); // > previous(20), < current(40) + // As an ancestor (not last) it should be truncated to 20 + ellipsis. + renderCrumbs([{ label }, { label: 'Current' }]); + expect(screen.getByText(`${'y'.repeat(20)}…`)).toBeInTheDocument(); + }); + + it.each([ + [SourceKind.Log, 'tabler-icon-logs'], + [SourceKind.Trace, 'tabler-icon-connection'], + [SourceKind.Session, 'tabler-icon-device-laptop'], + ])('renders the %s source icon on the first item', (kind, iconClass) => { + const { container } = renderCrumbs([{ label: 'First', sourceKind: kind }]); + expect(container.querySelector(`.${iconClass}`)).toBeInTheDocument(); + }); +}); diff --git a/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx b/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx new file mode 100644 index 0000000000..25c55f1f4a --- /dev/null +++ b/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx @@ -0,0 +1,309 @@ +import { act, renderHook } from '@testing-library/react'; + +import { SourceFrame, Tab } from '@/components/DBRowSidePanel.types'; + +// In-memory nuqs so each side-panel param can be seeded and its setter +// inspected. Values are the already-parsed shapes the hook consumes (arrays / +// strings), not URL strings. Prefixed `mock` so jest.mock's factory can use it. +const mockQueryStore: Record = {}; +const mockSetters: Record = {}; + +function seedParam(key: string, value: unknown) { + mockQueryStore[key] = value; +} +function setterFor(key: string) { + if (!mockSetters[key]) mockSetters[key] = jest.fn(); + return mockSetters[key]; +} +function resetQueryState() { + Object.keys(mockQueryStore).forEach(k => delete mockQueryStore[k]); + Object.keys(mockSetters).forEach(k => delete mockSetters[k]); +} + +jest.mock('nuqs', () => { + const actual = jest.requireActual('nuqs'); + return { + ...actual, + // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { + const hasValue = Object.prototype.hasOwnProperty.call( + mockQueryStore, + key, + ); + const fallback = + parser && 'defaultValue' in parser ? parser.defaultValue : null; + const value = hasValue ? mockQueryStore[key] : (fallback ?? null); + if (!mockSetters[key]) mockSetters[key] = jest.fn(); + return [value, mockSetters[key]]; + }, + }; +}); + +// NOTE: imported after the mock factories above, which close over the `mock*` +// helpers declared at the top of this file. +import useSidePanelStack, { + deriveEffectiveTrail, + reconcileTab, +} from '@/hooks/useSidePanelStack'; + +const FRAME: SourceFrame = { + sourceId: 'trace-src', + rowId: 'leaf-row', + aliasWith: [], + label: 'Trace', + sourceKind: undefined, +}; + +describe('deriveEffectiveTrail', () => { + it('keeps a trail whose owner matches the mounted row', () => { + const out = deriveEffectiveTrail({ + rawSourceStack: [FRAME], + rawNavStack: [], + stackRoot: 'root-1', + initialRowId: 'root-1', + }); + expect(out.isStale).toBe(false); + expect(out.sourceStack).toEqual([FRAME]); + }); + + it('discards a trail whose owner differs from the mounted row', () => { + const out = deriveEffectiveTrail({ + rawSourceStack: [FRAME], + rawNavStack: [], + stackRoot: 'old-root', + initialRowId: 'new-root', + }); + expect(out.isStale).toBe(true); + expect(out.sourceStack).toEqual([]); + expect(out.navStack).toEqual([]); + }); + + it('treats an ownerless trail (no stackRoot) as stale', () => { + const out = deriveEffectiveTrail({ + rawSourceStack: [FRAME], + rawNavStack: [], + stackRoot: null, + initialRowId: 'some-row', + }); + expect(out.isStale).toBe(true); + expect(out.sourceStack).toEqual([]); + }); + + it('is not stale when there are no stacks at all', () => { + const out = deriveEffectiveTrail({ + rawSourceStack: [], + rawNavStack: [], + stackRoot: null, + initialRowId: 'root-1', + }); + expect(out.isStale).toBe(false); + }); +}); + +describe('reconcileTab', () => { + const available = [Tab.Overview, Tab.Parsed, Tab.Context]; + + it('keeps a persisted tab that the source offers', () => { + expect(reconcileTab(Tab.Parsed, available, Tab.Overview)).toBe(Tab.Parsed); + }); + + it('falls back to the default when the persisted tab is unavailable', () => { + expect(reconcileTab(Tab.Trace, available, Tab.Overview)).toBe(Tab.Overview); + }); + + it('falls back to the default when no tab is persisted', () => { + expect(reconcileTab(null, available, Tab.Overview)).toBe(Tab.Overview); + }); +}); + +describe('useSidePanelStack', () => { + beforeEach(resetQueryState); + + it('exposes an owner-gated empty trail when the persisted trail is stale', () => { + seedParam('sidePanelSourceStack', [FRAME]); + seedParam('sidePanelStackRoot', 'old-root'); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'new-root' }), + ); + expect(result.current.isStale).toBe(true); + expect(result.current.sourceStack).toEqual([]); + }); + + it('pushSource records the frame + owner, resets nav, and jumps to the tab', () => { + seedParam('sidePanelTab', Tab.Overview); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + + act(() => result.current.pushSource(FRAME, Tab.Trace)); + + // Frame appended, stamped with the tab active before the push (originTab). + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith([ + { ...FRAME, originTab: Tab.Overview }, + ]); + // Cross-source push clears the same-source drilldown stack... + expect(setterFor('sidePanelNavStack')).toHaveBeenCalledWith([]); + // ...records the owning root... + expect(setterFor('sidePanelStackRoot')).toHaveBeenCalledWith('root-1'); + // ...and jumps to the destination tab. + expect(setterFor('sidePanelTab')).toHaveBeenCalledWith(Tab.Trace); + }); + + it('pushSource on a stale trail starts fresh instead of extending old frames', () => { + seedParam('sidePanelSourceStack', [FRAME]); + seedParam('sidePanelStackRoot', 'old-root'); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'new-root' }), + ); + + const next: SourceFrame = { ...FRAME, rowId: 'fresh' }; + act(() => result.current.pushSource(next, Tab.Trace)); + + // The new frame is the only one (the stale leaf is not carried over). + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith([ + { ...next, originTab: undefined }, + ]); + expect(setterFor('sidePanelStackRoot')).toHaveBeenCalledWith('new-root'); + }); + + it('pushNav appends a same-source entry and records the owner', () => { + seedParam('sidePanelTab', Tab.Overview); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + + act(() => + result.current.pushNav( + { rowId: 'ctx-row', aliasWith: [], label: 'Related' }, + Tab.Parsed, + ), + ); + + expect(setterFor('sidePanelNavStack')).toHaveBeenCalledWith([ + { + rowId: 'ctx-row', + aliasWith: [], + label: 'Related', + originTab: Tab.Overview, + }, + ]); + expect(setterFor('sidePanelStackRoot')).toHaveBeenCalledWith('root-1'); + expect(setterFor('sidePanelTab')).toHaveBeenCalledWith(Tab.Parsed); + // Re-persists the effective source stack (empty here) so a nav push never + // leaves a source stack it did not own behind. + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith([]); + }); + + it('pushNav on a stale trail clears the stale source stack it revives', () => { + // A leftover source stack owned by a different row is still in the URL. + seedParam('sidePanelSourceStack', [FRAME]); + seedParam('sidePanelStackRoot', 'old-root'); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'new-root' }), + ); + + act(() => + result.current.pushNav( + { rowId: 'ctx-row', aliasWith: [], label: 'Related' }, + Tab.Parsed, + ), + ); + + // The stale source stack is overwritten with the effective (empty) stack, so + // stamping the new owner below can't revive the old source frame. + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith([]); + expect(setterFor('sidePanelNavStack')).toHaveBeenCalledWith([ + { + rowId: 'ctx-row', + aliasWith: [], + label: 'Related', + originTab: undefined, + }, + ]); + expect(setterFor('sidePanelStackRoot')).toHaveBeenCalledWith('new-root'); + }); + + it('popOne pops the nav leaf first, restoring its originTab', () => { + seedParam('sidePanelSourceStack', [{ ...FRAME, originTab: Tab.Overview }]); + seedParam('sidePanelNavStack', [ + { rowId: 'ctx', aliasWith: [], label: 'Ctx', originTab: Tab.Context }, + ]); + seedParam('sidePanelStackRoot', 'root-1'); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + + let popped: string | undefined; + act(() => { + popped = result.current.popOne(); + }); + + expect(popped).toBe('nav'); + expect(setterFor('sidePanelNavStack')).toHaveBeenCalledWith([]); + expect(setterFor('sidePanelTab')).toHaveBeenCalledWith(Tab.Context); + // The source frame is untouched while a nav entry remained. + expect(setterFor('sidePanelSourceStack')).not.toHaveBeenCalled(); + }); + + it('popOne pops the source frame when no nav entries remain', () => { + seedParam('sidePanelSourceStack', [{ ...FRAME, originTab: Tab.Parsed }]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-1'); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + + let popped: string | undefined; + act(() => { + popped = result.current.popOne(); + }); + + expect(popped).toBe('source'); + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith([]); + expect(setterFor('sidePanelTab')).toHaveBeenCalledWith(Tab.Parsed); + }); + + it('popOne returns "none" when the trail is empty (caller exits)', () => { + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + let popped: string | undefined; + act(() => { + popped = result.current.popOne(); + }); + expect(popped).toBe('none'); + expect(setterFor('sidePanelSourceStack')).not.toHaveBeenCalled(); + }); + + it('truncateTo slices both stacks and restores the dropped level tab', () => { + seedParam('sidePanelSourceStack', [ + { ...FRAME, rowId: 'a', originTab: Tab.Overview }, + { ...FRAME, rowId: 'b', originTab: Tab.Trace }, + ]); + seedParam('sidePanelNavStack', []); + seedParam('sidePanelStackRoot', 'root-1'); + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + + // Jump back to the first source frame (breadcrumb click). + act(() => result.current.truncateTo(1, 0)); + + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith([ + { ...FRAME, rowId: 'a', originTab: Tab.Overview }, + ]); + // Restores the tab active before the dropped frame (index 1) was entered. + expect(setterFor('sidePanelTab')).toHaveBeenCalledWith(Tab.Trace); + }); + + it('clearTrail nulls every nav param', () => { + const { result } = renderHook(() => + useSidePanelStack({ initialRowId: 'root-1' }), + ); + act(() => result.current.clearTrail()); + expect(setterFor('sidePanelSourceStack')).toHaveBeenCalledWith(null); + expect(setterFor('sidePanelNavStack')).toHaveBeenCalledWith(null); + expect(setterFor('sidePanelStackRoot')).toHaveBeenCalledWith(null); + expect(setterFor('sidePanelTab')).toHaveBeenCalledWith(null); + }); +}); diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 25e664698f..a5ceffdefb 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -1,18 +1,19 @@ import { useCallback, useMemo } from 'react'; import MD5 from 'crypto-js/md5'; import SqlString from 'sqlstring'; +import z from 'zod'; import { ColumnMetaType, convertCHDataTypeToJSType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; -import { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; +import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; // Type for WITH clause entries, derived from ChartConfig's with property -export type WithClause = NonNullable[number]; +export type WithClause = z.infer; // Internal row field names used by the table component for row tracking export const INTERNAL_ROW_FIELDS = { diff --git a/packages/app/src/hooks/useSidePanelStack.ts b/packages/app/src/hooks/useSidePanelStack.ts new file mode 100644 index 0000000000..9ffd68a496 --- /dev/null +++ b/packages/app/src/hooks/useSidePanelStack.ts @@ -0,0 +1,237 @@ +import { useCallback, useMemo } from 'react'; +import { parseAsStringEnum, useQueryState } from 'nuqs'; + +import { + NavEntry, + parseNavStack, + parseSourceStack, + SourceFrame, + Tab, +} from '@/components/DBRowSidePanel.types'; +import { parseAsJsonEncoded, parseAsStringEncoded } from '@/utils/queryParsers'; + +const EMPTY_SOURCE_STACK: SourceFrame[] = []; +const EMPTY_NAV_STACK: NavEntry[] = []; + +export function deriveEffectiveTrail(params: { + rawSourceStack: SourceFrame[]; + rawNavStack: NavEntry[]; + stackRoot: string | null; + initialRowId: string | undefined; +}): { + sourceStack: SourceFrame[]; + navStack: NavEntry[]; + isStale: boolean; +} { + const { rawSourceStack, rawNavStack, stackRoot, initialRowId } = params; + const hasStacks = rawSourceStack.length > 0 || rawNavStack.length > 0; + const isStale = hasStacks && (stackRoot ?? null) !== (initialRowId ?? null); + return { + sourceStack: isStale ? EMPTY_SOURCE_STACK : rawSourceStack, + navStack: isStale ? EMPTY_NAV_STACK : rawNavStack, + isStale, + }; +} + +export function reconcileTab( + tab: Tab | null, + availableTabs: readonly Tab[], + defaultTab: Tab, +): Tab { + if (tab != null && availableTabs.includes(tab)) { + return tab; + } + return defaultTab; +} + +export type SidePanelStack = { + /** Owner-gated cross-source frames (empty if the persisted trail is stale). */ + sourceStack: SourceFrame[]; + /** Owner-gated same-source drilldown entries. */ + navStack: NavEntry[]; + /** True when a persisted trail was discarded because it belongs to another row. */ + isStale: boolean; + /** Persisted tab, or null when unset. Reconcile against the source's tabs. */ + tab: Tab | null; + + /** Push a cross-source frame (e.g. View Trace) and jump to its default tab. */ + pushSource: (frame: SourceFrame, destinationTab: Tab) => void; + /** Push a same-source drilldown (e.g. surrounding context) + destination tab. */ + pushNav: (entry: NavEntry, destinationTab: Tab) => void; + /** + * Pop one level (nav → source → nothing), restoring the tab that was active + * before the popped level was entered. Returns what was popped so the caller + * can route "nothing left" to parent/close. + */ + popOne: () => 'nav' | 'source' | 'none'; + /** Truncate both stacks to the given lengths (breadcrumb jump), restoring tab. */ + truncateTo: (sourceLevel: number, navLevel: number) => void; + /** Set the active tab. */ + setTab: (tab: Tab) => void; + /** Clear the whole trail + owner token + tab (panel close / new selection). */ + clearTrail: () => void; +}; + +/** + * Owns all side-panel navigation URL params behind a single read-time owner + * gate. Every consumer reads the *effective* trail from here rather than the raw + * params, so a stale value left in the URL by some un-cleared write path can + * never be rendered. + */ +export default function useSidePanelStack({ + initialRowId, +}: { + initialRowId: string | undefined; +}): SidePanelStack { + const [rawSourceStack, setSourceStack] = useQueryState( + 'sidePanelSourceStack', + parseAsJsonEncoded(parseSourceStack).withDefault( + EMPTY_SOURCE_STACK, + ), + ); + + const [rawNavStack, setNavStack] = useQueryState( + 'sidePanelNavStack', + parseAsJsonEncoded(parseNavStack).withDefault(EMPTY_NAV_STACK), + ); + + const [stackRoot, setStackRoot] = useQueryState( + 'sidePanelStackRoot', + parseAsStringEncoded, + ); + + const [tab, setQueryTab] = useQueryState( + 'sidePanelTab', + parseAsStringEnum(Object.values(Tab)), + ); + + const { sourceStack, navStack, isStale } = useMemo( + () => + deriveEffectiveTrail({ + rawSourceStack, + rawNavStack, + stackRoot, + initialRowId, + }), + [rawSourceStack, rawNavStack, stackRoot, initialRowId], + ); + + const pushSource = useCallback( + (frame: SourceFrame, destinationTab: Tab) => { + // Compute from the *effective* stack (empty when stale) so a push from a + // stale URL starts a fresh, owned trail instead of extending old frames. + setSourceStack([ + ...sourceStack, + { ...frame, originTab: tab ?? undefined }, + ]); + setNavStack([]); + setStackRoot(initialRowId ?? null); + setQueryTab(destinationTab); + }, + [ + sourceStack, + tab, + initialRowId, + setSourceStack, + setNavStack, + setStackRoot, + setQueryTab, + ], + ); + + const pushNav = useCallback( + (entry: NavEntry, destinationTab: Tab) => { + setSourceStack(sourceStack); + setNavStack([...navStack, { ...entry, originTab: tab ?? undefined }]); + setStackRoot(initialRowId ?? null); + setQueryTab(destinationTab); + }, + [ + sourceStack, + navStack, + tab, + initialRowId, + setSourceStack, + setNavStack, + setStackRoot, + setQueryTab, + ], + ); + + const popOne = useCallback((): 'nav' | 'source' | 'none' => { + if (navStack.length > 0) { + const restoreTab = navStack[navStack.length - 1]?.originTab; + setNavStack(navStack.slice(0, -1)); + if (restoreTab) { + setQueryTab(restoreTab); + } + return 'nav'; + } + if (sourceStack.length > 0) { + const restoreTab = sourceStack[sourceStack.length - 1]?.originTab; + setSourceStack(sourceStack.slice(0, -1)); + setNavStack([]); + if (restoreTab) { + setQueryTab(restoreTab); + } + return 'source'; + } + return 'none'; + }, [navStack, sourceStack, setNavStack, setSourceStack, setQueryTab]); + + const truncateTo = useCallback( + (sourceLevel: number, navLevel: number) => { + // Restore the tab active at the level we're returning to: the first source + // frame being dropped, or (staying within the same source) the first nav + // entry being dropped. + let restoreTab: Tab | undefined; + if (sourceLevel < sourceStack.length) { + restoreTab = sourceStack[sourceLevel]?.originTab; + } else if (navLevel < navStack.length) { + restoreTab = navStack[navLevel]?.originTab; + } + setSourceStack(sourceStack.slice(0, sourceLevel)); + setNavStack(navStack.slice(0, navLevel)); + if (restoreTab) { + setQueryTab(restoreTab); + } + }, + [sourceStack, navStack, setSourceStack, setNavStack, setQueryTab], + ); + + const setTab = useCallback((next: Tab) => setQueryTab(next), [setQueryTab]); + + const clearTrail = useCallback(() => { + setSourceStack(null); // sidePanelSourceStack + setNavStack(null); // sidePanelNavStack + setStackRoot(null); // sidePanelStackRoot + setQueryTab(null); // sidePanelTab + }, [setSourceStack, setNavStack, setStackRoot, setQueryTab]); + + return useMemo( + () => ({ + sourceStack, + navStack, + isStale, + tab, + pushSource, + pushNav, + popOne, + truncateTo, + setTab, + clearTrail, + }), + [ + clearTrail, + isStale, + navStack, + popOne, + pushNav, + pushSource, + setTab, + sourceStack, + tab, + truncateTo, + ], + ); +} diff --git a/packages/app/src/utils/__tests__/queryParsers.test.ts b/packages/app/src/utils/__tests__/queryParsers.test.ts index 0508450771..ff1bf66873 100644 --- a/packages/app/src/utils/__tests__/queryParsers.test.ts +++ b/packages/app/src/utils/__tests__/queryParsers.test.ts @@ -102,4 +102,43 @@ describe('parseAsJsonEncoded', () => { ); }); }); + + describe('with a validator', () => { + // Accepts only an array of numbers; throws otherwise (mirrors zod.parse). + const numberArrayParser = parseAsJsonEncoded(value => { + if (!Array.isArray(value) || !value.every(n => typeof n === 'number')) { + throw new Error('expected number[]'); + } + return value; + }); + + it('returns a value that satisfies the validator', () => { + expect( + numberArrayParser.parse(numberArrayParser.serialize([1, 2, 3])), + ).toEqual([1, 2, 3]); + }); + + it.each([ + ['an object', {}], + ['a bare number', 5], + ['a bare string', 'x'], + ['a null literal', null], + ['an array of the wrong element type', ['a', 'b']], + ])('returns null for %s (valid JSON, wrong shape)', (_label, input) => { + const serialized = encodeURIComponent(JSON.stringify(input)); + expect(numberArrayParser.parse(serialized)).toBeNull(); + }); + + it('returns null when the validator returns null instead of throwing', () => { + const nullingParser = parseAsJsonEncoded(() => null as never); + expect(nullingParser.parse(encodeURIComponent('[1,2,3]'))).toBeNull(); + }); + + it('still returns null for non-JSON input before validation runs', () => { + const validate = jest.fn(); + const parser = parseAsJsonEncoded(validate); + expect(parser.parse('not-json')).toBeNull(); + expect(validate).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/app/src/utils/queryParsers.ts b/packages/app/src/utils/queryParsers.ts index e6486465b3..5e165eda2c 100644 --- a/packages/app/src/utils/queryParsers.ts +++ b/packages/app/src/utils/queryParsers.ts @@ -40,8 +40,31 @@ export const parseAsStringEncoded = createParser({ * spaces, unencoded '[', ']', etc.) are handled via a fallback to plain * JSON.parse after the decodeURIComponent step naturally resolves '%22' → * '"', '+' → ' ', etc. via URLSearchParams.get(). + * + * Optional `validate`: a guard that either returns the (narrowed) value or + * throws / returns `null` for a shape mismatch. A stale, hand-edited, or + * cross-version URL whose param is valid JSON but the wrong shape (`{}`, `5`, + * `"x"`, an array of malformed frames) then resolves to `null` instead of a + * structurally-invalid object. Combined with `.withDefault(...)`, callers get + * the safe default rather than a value that throws downstream during render. + * Zod schemas satisfy this signature via `schema.parse`. */ -export function parseAsJsonEncoded() { +export function parseAsJsonEncoded(validate?: (value: unknown) => T) { + const finalize = (parsed: unknown): T | null => { + if (!validate) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return parsed as T; + } + try { + const validated = validate(parsed); + // A validator may signal rejection by returning null/undefined instead + // of throwing; treat both the same as a thrown mismatch. + return validated == null ? null : validated; + } catch { + return null; + } + }; + return createParser({ parse: value => { let decoded: string; @@ -50,7 +73,7 @@ export function parseAsJsonEncoded() { } catch { // Malformed URI sequence — value is likely old-format plain JSON. try { - return JSON.parse(value); + return finalize(JSON.parse(value)); } catch { return null; } @@ -59,7 +82,7 @@ export function parseAsJsonEncoded() { // This handles both new-format (double-encoded) and old-format URLs, // since decodeURIComponent is a no-op on plain JSON strings. try { - return JSON.parse(decoded); + return finalize(JSON.parse(decoded)); } catch { return null; } diff --git a/packages/app/styles/LogSidePanel.module.scss b/packages/app/styles/LogSidePanel.module.scss index e899b6b223..6a3e3d1c43 100644 --- a/packages/app/styles/LogSidePanel.module.scss +++ b/packages/app/styles/LogSidePanel.module.scss @@ -4,7 +4,6 @@ font-size: 12px; height: 100%; background: var(--color-bg-body); - box-shadow: 0 0 100px 40px rgb(0 0 0 / 70%); border-left: 1px solid var(--color-border); } diff --git a/packages/app/tests/e2e/features/search/search.spec.ts b/packages/app/tests/e2e/features/search/search.spec.ts index 5ca57692ff..3c282ca1a6 100644 --- a/packages/app/tests/e2e/features/search/search.spec.ts +++ b/packages/app/tests/e2e/features/search/search.spec.ts @@ -53,7 +53,7 @@ test.describe('Search', { tag: '@search' }, () => { }); await test.step('Navigate through all side panel tabs', async () => { - const tabs = ['parsed', 'trace', 'context', 'overview']; + const tabs = ['parsed', 'context', 'overview']; // Use side panel component to navigate tabs for (const tabName of tabs) { @@ -104,7 +104,11 @@ test.describe('Search', { tag: '@search' }, () => { }); await test.step('Navigate through all side panel tabs', async () => { - const tabs = ['trace', 'context', 'infrastructure', 'overview']; + // Logs sources no longer render a Trace tab (it's gated behind + // source.kind === Trace). For a Kubernetes log row the available tabs + // are Surrounding Context (always present), Infrastructure (k8s + // context) and Overview. + const tabs = ['context', 'infrastructure', 'overview']; // Use side panel component with proper waiting for (const tabName of tabs) { @@ -154,10 +158,10 @@ test.describe('Search', { tag: '@search' }, () => { }); await test.step('Infrastructure tab is not offered', async () => { - // The tab bar rendered (an always-present tab is visible), but the gate - // omits Infrastructure because the row carries no k8s correlation - // attributes. - await expect(searchPage.sidePanel.getTab('trace')).toBeVisible(); + // The tab bar rendered (the always-present Column Values tab is + // visible), but the gate omits Infrastructure because the row carries + // no k8s correlation attributes. + await expect(searchPage.sidePanel.getTab('parsed')).toBeVisible(); await expect(searchPage.sidePanel.getTab('infrastructure')).toHaveCount( 0, ); diff --git a/packages/app/tests/e2e/features/sessions.spec.ts b/packages/app/tests/e2e/features/sessions.spec.ts index aec7b09f9a..45cd1b6f7e 100644 --- a/packages/app/tests/e2e/features/sessions.spec.ts +++ b/packages/app/tests/e2e/features/sessions.spec.ts @@ -46,8 +46,9 @@ test.describe('Client Sessions Functionality', { tag: ['@sessions'] }, () => { async ({ page }) => { await test.step('Navigate and open a session (with sidePanelTab=replay pre-set in URL to simulate search-page flow)', async () => { // Pre-set sidePanelTab=replay in the URL to simulate navigating from a search page - // row detail panel that had the Session Replay tab open. Without isNestedPanel=true, - // the inner DBRowSidePanel would inherit this URL param and open to the Replay tab again. + // row detail panel that had the Session Replay tab open. Selecting a session event + // must clear this param (clearInnerNavigation) so the in-place event detail opens to + // its default tab instead of re-rendering the Session Replay tab. await page.goto('/search'); await sessionsPage.goto(); await sessionsPage.selectDataSource(); @@ -75,43 +76,49 @@ test.describe('Client Sessions Functionality', { tag: ['@sessions'] }, () => { await sessionsPage.clickFirstSessionEvent(); }); - await test.step('Event detail panel opens alongside the session replay — not replacing it', async () => { - // The row-side-panel must be visible (event detail drawer opened on top of session replay) - await expect(sessionsPage.rowSidePanel).toBeVisible(); - - // The original session replay panel must still be open (not replaced/closed) + await test.step('Event detail opens in place inside the single session drawer — not as a second drawer', async () => { + // Single-drawer navigation: the event detail renders in place inside the + // existing session-side-panel (via DBRowSidePanelInner). No second + // row-side-panel Drawer is opened. await expect(sessionsPage.sessionSidePanel).toBeVisible(); - // Only one session-side-panel must exist (not a second replay opened inside the detail panel) + // Exactly one session drawer exists, and no separate row-side-panel + // drawer was stacked on top of it. await expect(page.getByTestId('session-side-panel')).toHaveCount(1); + await expect(page.getByTestId('row-side-panel')).toHaveCount(0); - // The row-side-panel must show the event detail TabBar (Overview, Trace, etc.) - // This guards against the regression where the inner panel re-opened session replay - // instead of showing event details (which has no TabBar, just the replay player) + // The session drawer now shows the event detail TabBar (Overview, Trace, + // etc.). This guards against the regression where selecting an event + // re-opened session replay instead of showing event details (which has + // no TabBar, just the replay player). await expect( - sessionsPage.rowSidePanel.getByTestId('side-panel-tabs'), + sessionsPage.sessionSidePanel.getByTestId('side-panel-tabs'), ).toBeVisible(); - // The inner panel must NOT be showing the Session Replay tab content. - // Without isNestedPanel=true (broken), the inner DBRowSidePanel reads sidePanelTab=replay - // from the URL (injected above) and renders the Session Replay tab content (side-panel-tab-replay). - // With isNestedPanel=true (fixed), the inner panel uses local state and ignores the URL, - // opening to its default tab (Trace/Overview) instead. + // The event detail must NOT land on the Session Replay tab. Selecting an + // event clears the sidePanelTab=replay param injected above + // (clearInnerNavigation), so the inner panel opens to its default tab + // (Trace/Overview) and the replay tab content is not rendered. await expect( - sessionsPage.rowSidePanel.getByTestId('side-panel-tab-replay'), + sessionsPage.sessionSidePanel.getByTestId('side-panel-tab-replay'), ).toHaveCount(0); }); - await test.step('Clicking the overlay closes the event detail panel but keeps the session replay open', async () => { - // Without the fix, withOverlay={!isNestedPanel} removed the overlay on nested panels, - // so there was nothing to click to close the panel (it had to be ESC only). - // With the fix (withOverlay always true), clicking the Mantine overlay dismisses the inner panel. - await sessionsPage.clickTopmostDrawerOverlay(); + await test.step('Pressing Escape returns to the session event list within the same drawer', async () => { + // With an event open, the embedded DBRowSidePanelInner owns Esc and pops + // one level at a time; at the event root it hands back to the session + // (onNavigateToParent), so a single Esc returns to the session event + // list instead of closing the whole drawer. (The parent's own Esc is + // disabled while an event is selected to avoid a double-fire.) + await page.keyboard.press('Escape'); - // The event detail panel must close - await expect(sessionsPage.rowSidePanel).toBeHidden(); + // The event detail TabBar collapses back to the session event list. + await expect( + sessionsPage.sessionSidePanel.getByTestId('side-panel-tabs'), + ).toBeHidden(); + await expect(sessionsPage.getSessionEventRows().first()).toBeVisible(); - // The session replay drawer must still be open + // The session drawer itself stays open. await expect(sessionsPage.sessionSidePanel).toBeVisible(); }); }, diff --git a/packages/app/tests/e2e/page-objects/SessionsPage.ts b/packages/app/tests/e2e/page-objects/SessionsPage.ts index 274f563a10..9d13088a63 100644 --- a/packages/app/tests/e2e/page-objects/SessionsPage.ts +++ b/packages/app/tests/e2e/page-objects/SessionsPage.ts @@ -89,25 +89,6 @@ export class SessionsPage { await this.getSessionEventRows().first().click(); } - /** - * Get the row side panel (event detail drawer opened from within session replay) - */ - get rowSidePanel() { - return this.page.getByTestId('row-side-panel'); - } - - /** - * Click the Mantine overlay of the topmost open drawer to close it. - * Mantine renders one overlay per open Drawer. The last one belongs to - * the innermost (topmost) drawer. - */ - async clickTopmostDrawerOverlay() { - // Mantine overlays are siblings of the drawer content inside the portal root. - // Use the last one since the inner panel's overlay is rendered on top. - const overlay = this.page.locator('.mantine-Drawer-overlay').last(); - await overlay.click({ position: { x: 10, y: 10 } }); - } - // Getters for assertions get form() { diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index e8c597fb03..067ab8f267 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1221,34 +1221,30 @@ export const CteChartConfigSchema = z.intersection( export type CteChartConfig = z.infer; +export const WithClauseSchema = z.object({ + name: z.string(), + + // Need to specify either a sql or chartConfig instance. To avoid + // the schema falling into an any type, the fields are separate + // and listed as optional. + sql: ChSqlSchema.optional(), + chartConfig: CteChartConfigSchema.optional(), + + // If true, it'll render as WITH ident AS (subquery) + // If false, it'll be a "variable" ex. WITH (sql) AS ident + // where sql can be any expression, ex. a constant string + // see: https://clickhouse.com/docs/sql-reference/statements/select/with#syntax + // default assume true + isSubquery: z.boolean().optional(), +}); + // The `with` CTE property needs to be defined at this level, just above the // non-recursive chart config so that it can reference a complete chart config // schema. This structure does mean that we cannot nest `with` clauses but does // ensure the type system can catch more issues in the build pipeline. const BuilderChartConfigSchema = z.intersection( z.intersection(_ChartConfigSchema, SelectSQLStatementSchema), - z - .object({ - with: z.array( - z.object({ - name: z.string(), - - // Need to specify either a sql or chartConfig instance. To avoid - // the schema falling into an any type, the fields are separate - // and listed as optional. - sql: ChSqlSchema.optional(), - chartConfig: CteChartConfigSchema.optional(), - - // If true, it'll render as WITH ident AS (subquery) - // If false, it'll be a "variable" ex. WITH (sql) AS ident - // where sql can be any expression, ex. a constant string - // see: https://clickhouse.com/docs/sql-reference/statements/select/with#syntax - // default assume true - isSubquery: z.boolean().optional(), - }), - ), - }) - .partial(), + z.object({ with: z.array(WithClauseSchema) }).partial(), ); export type BuilderChartConfig = z.infer; From 328e7b437eab890edc8f850a159c30f64664e42c Mon Sep 17 00:00:00 2001 From: Aryan Inguz <117257768+Aryainguz@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:55:16 +0530 Subject: [PATCH 159/728] Fix/webhook cascade (#2591) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .changeset/fix-webhook-cascade-delete.md | 5 ++ .../routers/api/__tests__/webhooks.test.ts | 70 +++++++++++++++++++ packages/api/src/routers/api/webhooks.ts | 15 +++- 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 .changeset/fix-webhook-cascade-delete.md diff --git a/.changeset/fix-webhook-cascade-delete.md b/.changeset/fix-webhook-cascade-delete.md new file mode 100644 index 0000000000..400aef61fb --- /dev/null +++ b/.changeset/fix-webhook-cascade-delete.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +Fix: Block webhook deletion when one or more alerts still reference it, prompting the user to reassign or remove those alerts first. diff --git a/packages/api/src/routers/api/__tests__/webhooks.test.ts b/packages/api/src/routers/api/__tests__/webhooks.test.ts index bc13c2eee2..50904ee0a9 100644 --- a/packages/api/src/routers/api/__tests__/webhooks.test.ts +++ b/packages/api/src/routers/api/__tests__/webhooks.test.ts @@ -1,6 +1,7 @@ import { Types } from 'mongoose'; import { getLoggedInAgent, getServer } from '@/fixtures'; +import Alert from '@/models/alert'; import Webhook, { WebhookService } from '@/models/webhook'; import * as template from '@/tasks/checkAlerts/template'; @@ -186,6 +187,75 @@ describe('webhooks router', () => { expect(deletedWebhook).toBeNull(); }); + it('DELETE /:id - returns 409 when alerts still reference the webhook', async () => { + const { agent, team } = await getLoggedInAgent(server); + + // 1. Create a webhook + const webhook = await Webhook.create({ + ...MOCK_WEBHOOK, + team: team._id, + }); + + // 2. Create an alert pointing to that webhook + await Alert.create({ + team: team._id, + name: 'Test Alert', + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + source: 'logs', + groupBy: 'test', + interval: '5m', + threshold: 1, + thresholdType: 'above', + }); + + // 3. Attempt to delete — should be blocked + const response = await agent.delete(`/webhooks/${webhook._id}`).expect(409); + + expect(response.body.message).toContain('1 alert(s) still reference it'); + + // 4. Webhook should still exist + const stillExists = await Webhook.findById(webhook._id); + expect(stillExists).not.toBeNull(); + }); + + it('DELETE /:id - succeeds after referencing alerts are removed', async () => { + const { agent, team } = await getLoggedInAgent(server); + + const webhook = await Webhook.create({ + ...MOCK_WEBHOOK, + team: team._id, + }); + + const alert = await Alert.create({ + team: team._id, + name: 'Test Alert', + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + source: 'logs', + groupBy: 'test', + interval: '5m', + threshold: 1, + thresholdType: 'above', + }); + + // First attempt should fail + await agent.delete(`/webhooks/${webhook._id}`).expect(409); + + // Remove the alert + await Alert.findByIdAndDelete(alert._id); + + // Now deletion should succeed + await agent.delete(`/webhooks/${webhook._id}`).expect(200); + + const deleted = await Webhook.findById(webhook._id); + expect(deleted).toBeNull(); + }); + it('DELETE /:id - returns 200 when webhook does not exist', async () => { const { agent } = await getLoggedInAgent(server); diff --git a/packages/api/src/routers/api/webhooks.ts b/packages/api/src/routers/api/webhooks.ts index d2e4191793..e086664bd4 100644 --- a/packages/api/src/routers/api/webhooks.ts +++ b/packages/api/src/routers/api/webhooks.ts @@ -11,7 +11,7 @@ import mongoose from 'mongoose'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; -import { AlertState } from '@/models/alert'; +import Alert, { AlertState } from '@/models/alert'; import Webhook, { WebhookService } from '@/models/webhook'; import { handleSendGenericWebhook, @@ -354,6 +354,19 @@ router.delete( if (teamId == null) { return res.sendStatus(403); } + + // Block deletion when alerts still reference this webhook. + // The user must reassign or delete those alerts first. + const referencingAlertCount = await Alert.countDocuments({ + 'channel.webhookId': req.params.id, + team: teamId, + }); + if (referencingAlertCount > 0) { + return res.status(409).json({ + message: `Cannot delete webhook: ${referencingAlertCount} alert(s) still reference it. Please update or remove those alerts first.`, + }); + } + await Webhook.findOneAndDelete({ _id: req.params.id, team: teamId }); res.json({}); } catch (err) { From 1aaa9388adf1e01789ac40af0f641c4ac7243316 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 7 Jul 2026 17:29:24 +0100 Subject: [PATCH 160/728] fix: honor INGESTION_API_KEY in all-in-one auth image (#2528) The all-in-one entrypoint set HYPERDX_IMAGE to values (all-in-one / all-in-one-noauth) that did not match the strings config.ts compares against (all-in-one-auth / all-in-one-noauth). As a result IS_ALL_IN_ONE_IMAGE was never true in the auth image, so INGESTION_API_KEY was silently ignored and the OpAMP-delivered collector config only accepted the team's UI-generated key. Report all-in-one-auth for the auth variant and all-in-one-noauth for the no-auth variant so a pre-shared INGESTION_API_KEY is added to the collector's accepted bearer tokens, enabling demo/bootstrap stacks to specify a known ingestion key without retrieving it from the UI. --- .../fix-all-in-one-ingestion-api-key.md | 19 +++++++++++++++++++ docker/hyperdx/entry.local.base.sh | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .changeset/fix-all-in-one-ingestion-api-key.md diff --git a/.changeset/fix-all-in-one-ingestion-api-key.md b/.changeset/fix-all-in-one-ingestion-api-key.md new file mode 100644 index 0000000000..a6ac6e73a0 --- /dev/null +++ b/.changeset/fix-all-in-one-ingestion-api-key.md @@ -0,0 +1,19 @@ +--- +'@hyperdx/api': patch +--- + +fix: honor INGESTION_API_KEY in the all-in-one auth image + +The all-in-one entrypoint set `HYPERDX_IMAGE` to values (`all-in-one` / +`all-in-one-noauth`) that did not match the strings `config.ts` compares against +(`all-in-one-auth` / `all-in-one-noauth`). As a result `IS_ALL_IN_ONE_IMAGE` was +never true in the auth image, so the `INGESTION_API_KEY` env var was silently +ignored and the OpAMP-delivered collector config only accepted the team's +UI-generated key. + +The entrypoint now reports `all-in-one-auth` for the auth variant and +`all-in-one-noauth` for the no-auth variant, so a pre-shared `INGESTION_API_KEY` +is added to the collector's accepted bearer tokens. This lets demo/bootstrap +stacks specify a known ingestion key up front instead of retrieving the +generated key from the UI — the all-in-one equivalent of the standalone +collector's `OTLP_AUTH_TOKEN`. diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 3b612f9451..c0ad13b416 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -18,7 +18,10 @@ export MONGO_URI="mongodb://db:27017/hyperdx" export OPAMP_SERVER_URL="http://127.0.0.1:${OPAMP_PORT}" export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT:-ch-server:9363}" -export HYPERDX_IMAGE=$([[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_mode💀" ]] && echo "all-in-one" || echo "all-in-one-noauth") +# These values must match the strings compared against in packages/api/src/config.ts +# (IS_LOCAL_IMAGE / IS_ALL_IN_ONE_IMAGE). The auth variant must report +# "all-in-one-auth" so that the INGESTION_API_KEY bootstrap path is enabled. +export HYPERDX_IMAGE=$([[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_mode💀" ]] && echo "all-in-one-noauth" || echo "all-in-one-auth") export EXPRESS_SESSION_SECRET="hyperdx is cool 👋" # IS_LOCAL_APP_MODE should be set by the calling script # Default to dangerous mode if not set From fbef83b801495576e361a96c5a21cfdea3edae2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:40:54 +0000 Subject: [PATCH 161/728] chore(deps): bump fast-xml-parser from 4.5.4 to 4.5.6 (#2152) Bumps [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) from 4.5.4 to 4.5.6.
Release notes

Sourced from fast-xml-parser's releases.

Summary update on all the previous releases from v4.2.4

  • Multiple minor fixes provided in the validator and parser
  • v6 is added for experimental use.
  • ignoreAttributes support function, and array of string or regex
  • Add support for parsing HTML numeric entities
  • v5 of the application is ESM module now. However, JS is also supported

Note: Release section in not updated frequently. Please check CHANGELOG or Tags for latest release information.

Commits

Co-authored-by: Drew Davis <6097246+pulpdrew@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8d0feaa433..584381bbed 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "brace-expansion": "^2.0.2", "diff": "^5.2.2", "on-headers": "^1.1.0", - "fast-xml-parser": "^4.5.4", + "fast-xml-parser": "^4.5.6", "systeminformation": "^5.24.0" } } diff --git a/yarn.lock b/yarn.lock index af1744fdf2..db57f469fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16372,14 +16372,14 @@ __metadata: languageName: node linkType: hard -"fast-xml-parser@npm:^4.5.4": - version: 4.5.4 - resolution: "fast-xml-parser@npm:4.5.4" +"fast-xml-parser@npm:^4.5.6": + version: 4.5.6 + resolution: "fast-xml-parser@npm:4.5.6" dependencies: strnum: "npm:^1.0.5" bin: fxparser: src/cli/cli.js - checksum: 10c0/7989148650fc1fce988798b62467f7dee0fd5a7ad049373e00e65fbc68f689c119975d03da32c427f0a1f5aad01de2efbd783a48dcdaf3ca41817a8b161ad3e8 + checksum: 10c0/1c19e183b5ee93bea9b24e1ddb0aed8564b273c6106af622b1c11ff8eb1fc8d2033cd7a0cb68976a5f3e05d1cbf0a0026e6f300f904be0bc854ff896dbdf38d2 languageName: node linkType: hard From dfa1f648b64d3650199adb71217bc3f142bb0f34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:51:39 +0000 Subject: [PATCH 162/728] chore(deps): bump brace-expansion from 2.0.2 to 2.1.1 (#2001) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 2.0.2 to 2.1.1.
Release notes

Sourced from brace-expansion's releases.

v2.1.1

  • Backport v5.0.6 change to v2 (#109) c3a817c

https://github.com/juliangruber/brace-expansion/compare/v2.1.0...v2.1.1

Commits

Co-authored-by: Drew Davis <6097246+pulpdrew@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 584381bbed..8135d28760 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "send": "^0.19.0", "serve-static": "^1.16.0", "cookie": "^0.7.0", - "brace-expansion": "^2.0.2", + "brace-expansion": "^2.1.1", "diff": "^5.2.2", "on-headers": "^1.1.0", "fast-xml-parser": "^4.5.6", diff --git a/yarn.lock b/yarn.lock index db57f469fc..036cefdf87 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12299,12 +12299,12 @@ __metadata: languageName: node linkType: hard -"brace-expansion@npm:^2.0.2": - version: 2.0.2 - resolution: "brace-expansion@npm:2.0.2" +"brace-expansion@npm:^2.1.1": + version: 2.1.1 + resolution: "brace-expansion@npm:2.1.1" dependencies: balanced-match: "npm:^1.0.0" - checksum: 10c0/6d117a4c793488af86b83172deb6af143e94c17bc53b0b3cec259733923b4ca84679d506ac261f4ba3c7ed37c46018e2ff442f9ce453af8643ecd64f4a54e6cf + checksum: 10c0/63b5ddce608b70b50a76817c0526faf8ea67a9180073d88bb402f6bbc22a22da6b1dfac4f65efc53e5faa80222fb7d44bbf2fc638c3f55365975573f671d0ccb languageName: node linkType: hard From 803f824f4fb91408b3c43fb22f043f0eca265b2c Mon Sep 17 00:00:00 2001 From: Vineet Ahirkar Date: Wed, 8 Jul 2026 00:12:54 +0530 Subject: [PATCH 163/728] Add query syntax reference to search input (#2381) ## Summary Introduce a quick-reference guide for SQL and Lucene query syntax directly within the search interface. Users can now click a help icon next to the language selector to view common query examples, operators, and formatting rules tailored to the active query language. This improves usability and helps users construct valid search queries more easily. ### Screenshots or video #### Lucene image #### SQL image ### How to test on Vercel preview ### References - Linear Issue: n/a - Related PRs: n/a Co-authored-by: Drew Davis <6097246+pulpdrew@users.noreply.github.com> --- .../SearchInput/SearchWhereInput.tsx | 146 +++++--- .../SearchInput/SyntaxReferenceModal.tsx | 350 ++++++++++++++++++ 2 files changed, 435 insertions(+), 61 deletions(-) create mode 100644 packages/app/src/components/SearchInput/SyntaxReferenceModal.tsx diff --git a/packages/app/src/components/SearchInput/SearchWhereInput.tsx b/packages/app/src/components/SearchInput/SearchWhereInput.tsx index f4fbd4be3d..296a23cb76 100644 --- a/packages/app/src/components/SearchInput/SearchWhereInput.tsx +++ b/packages/app/src/components/SearchInput/SearchWhereInput.tsx @@ -1,11 +1,14 @@ import { FieldPath, useController, UseControllerProps } from 'react-hook-form'; import { TableConnectionChoice } from '@hyperdx/common-utils/dist/core/metadata'; -import { Box, Flex, Kbd } from '@mantine/core'; +import { ActionIcon, Box, Flex, Kbd, Tooltip } from '@mantine/core'; +import { useDisclosure } from '@mantine/hooks'; +import { IconHelp } from '@tabler/icons-react'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import InputLanguageSwitch from './InputLanguageSwitch'; import SearchInputV2 from './SearchInputV2'; +import SyntaxReferenceModal from './SyntaxReferenceModal'; import styles from './SearchWhereInput.module.scss'; @@ -164,6 +167,9 @@ export default function SearchWhereInput({ languageName = `${name}Language`, sourceId, }: SearchWhereInputProps) { + const [syntaxRefOpened, { open: openSyntaxRef, close: closeSyntaxRef }] = + useDisclosure(false); + const { field: languageField } = useController({ control, name: languageName as FieldPath, @@ -182,68 +188,86 @@ export default function SearchWhereInput({ const sizeClass = size === 'xs' ? styles.sizeXs : styles.sizeSm; return ( - - e.preventDefault()} + <> + + - - - - {isSql ? ( - - ) : ( - e.preventDefault()} + > + - )} - {enableHotkey && ( - - / - - )} + + + + + + + + {isSql ? ( + + ) : ( + + )} + {enableHotkey && ( + + / + + )} + - + ); } diff --git a/packages/app/src/components/SearchInput/SyntaxReferenceModal.tsx b/packages/app/src/components/SearchInput/SyntaxReferenceModal.tsx new file mode 100644 index 0000000000..1f9b5097c5 --- /dev/null +++ b/packages/app/src/components/SearchInput/SyntaxReferenceModal.tsx @@ -0,0 +1,350 @@ +import { useEffect, useMemo, useState } from 'react'; +import { + Box, + Code, + Divider, + Group, + Modal, + ScrollArea, + SegmentedControl, + Stack, + Table, + Text, + TextInput, + Title, + Tooltip, +} from '@mantine/core'; +import { IconExternalLink, IconSearch } from '@tabler/icons-react'; + +type Language = 'sql' | 'lucene'; + +type Row = { expr: string; desc: string }; +type Section = { title: string; rows: Row[] }; + +const SQL_SECTIONS: Section[] = [ + { + title: 'String matching', + rows: [ + { expr: "ServiceName = 'api'", desc: 'Exact match' }, + { expr: "Body = 'connection refused'", desc: 'Exact phrase match' }, + { + expr: "Body ILIKE '%timeout%'", + desc: 'Substring search (case-insensitive)', + }, + { + expr: "Body LIKE '%timeout%'", + desc: 'Substring match (case-sensitive)', + }, + { + expr: "hasAllTokens(Body, 'connection timeout')", + desc: 'Full-text search (requires text index)', + }, + { + expr: "ServiceName LIKE 'auth-%'", + desc: 'Prefix wildcard (case-sensitive)', + }, + { + expr: "match(SpanName, '^/api/(checkout|payment)/.*')", + desc: 'Regular expression', + }, + ], + }, + { + title: 'Boolean operators', + rows: [ + { + expr: "ServiceName = 'api' AND SpanName = 'checkout'", + desc: 'Both must match', + }, + { + expr: "ServiceName = 'api' OR ServiceName = 'worker'", + desc: 'Either matches', + }, + { + expr: "ServiceName IN ('api', 'worker')", + desc: 'Match one of multiple values', + }, + { expr: "ServiceName != 'healthcheck'", desc: 'Exclude a value' }, + { + expr: "(StatusCode = 500 OR StatusCode = 503) AND ServiceName = 'api'", + desc: 'Nested boolean logic', + }, + { expr: 'Duration > 1000000', desc: 'Numeric comparison' }, + { expr: 'Duration BETWEEN 100 AND 1000', desc: 'Range (inclusive)' }, + { expr: 'Duration / 1e6 > 100', desc: 'Math expression' }, + ], + }, + { + title: 'Existence & absence', + rows: [ + { expr: 'notEmpty(StatusCode)', desc: 'Field exists / is not null' }, + { expr: 'empty(Body)', desc: 'Field is absent / null' }, + ], + }, + { + title: 'Map', + rows: [ + { + expr: "LogAttributes['http.method'] = 'POST'", + desc: 'Map attribute access', + }, + { + expr: "LogAttributes.http.method = 'POST'", + desc: 'JSON attribute access', + }, + ], + }, + { + title: 'Arrays', + rows: [ + { + expr: "has(Events.Name, 'exception')", + desc: 'Array column contains value', + }, + ], + }, +]; + +const LUCENE_SECTIONS: Section[] = [ + { + title: 'String matching', + rows: [ + { expr: 'ServiceName:"api"', desc: 'Field exact match' }, + { expr: 'ServiceName:api', desc: 'Field substring match' }, + { + expr: '"connection refused"', + desc: 'Phrase match against implicit column', + }, + { expr: 'timeout', desc: 'Token match against implicit column' }, + { expr: 'auth-*', desc: 'Prefix match against implicit column' }, + { expr: '*-auth', desc: 'Suffix match against implicit column' }, + { expr: '*checkout*', desc: 'Substring match against implicit column' }, + { expr: 'Duration:[100 TO 500]', desc: 'Numeric range (inclusive)' }, + { expr: 'Duration:>1000000', desc: 'Greater-than comparison' }, + ], + }, + { + title: 'Boolean operators', + rows: [ + { + expr: 'ServiceName:api AND SpanName:checkout', + desc: 'Both conditions must match', + }, + { + expr: 'ServiceName:api OR ServiceName:worker', + desc: 'Either condition matches', + }, + { + expr: 'ServiceName:(api OR worker)', + desc: 'Match multiple values for one field', + }, + { expr: 'NOT ServiceName:healthcheck', desc: 'Exclude matches' }, + { expr: '-ServiceName:healthcheck', desc: 'Shorthand for NOT' }, + { + expr: '(ServiceName:api OR ServiceName:worker) AND StatusCode:500', + desc: 'Nested boolean logic', + }, + ], + }, + { + title: 'Existence & absence', + rows: [ + { expr: 'StatusCode:*', desc: 'Field exists (not null nor empty)' }, + { expr: '-Body:*', desc: 'Field is absent / null / empty' }, + ], + }, + { + title: 'Map', + rows: [ + { + expr: 'LogAttributes.http.method:POST', + desc: 'Access map/attribute column by key', + }, + { + expr: 'ResourceAttributes.service.env:prod', + desc: 'Map / JSON attribute filter', + }, + ], + }, + { + title: 'Arrays', + rows: [ + { + expr: 'Events.Name:"exception"', + desc: 'Array column contains value', + }, + ], + }, +]; + +function filterSections(sections: Section[], query: string): Section[] { + if (!query.trim()) return sections; + const q = query.toLowerCase(); + return sections + .map(section => ({ + ...section, + rows: section.rows.filter( + row => + row.expr.toLowerCase().includes(q) || + row.desc.toLowerCase().includes(q), + ), + })) + .filter(section => section.rows.length > 0); +} + +function Highlight({ text, query }: { text: string; query: string }) { + if (!query.trim()) return <>{text}; + const idx = text.toLowerCase().indexOf(query.toLowerCase()); + if (idx === -1) return <>{text}; + return ( + <> + {text.slice(0, idx)} + + {text.slice(idx, idx + query.length)} + + {text.slice(idx + query.length)} + + ); +} + +function SyntaxTable({ + sections, + query, +}: { + sections: Section[]; + query: string; +}) { + const filtered = useMemo( + () => filterSections(sections, query), + [sections, query], + ); + + if (filtered.length === 0) { + return ( + + No results for “{query}” + + ); + } + + return ( + + {filtered.map((section, si) => ( + + {si > 0 && } + + {section.title} + + + + {section.rows.map(row => ( + + + + + + + + + + + + + ))} + +
+ + ))} + + ); +} + +export default function SyntaxReferenceModal({ + opened, + onClose, + language: initialLanguage, +}: { + opened: boolean; + onClose: () => void; + language: Language; +}) { + const [language, setLanguage] = useState(initialLanguage); + const [query, setQuery] = useState(''); + + // Sync tab when the modal opens or caller switches language externally + useEffect(() => { + if (opened) setLanguage(initialLanguage); + }, [opened, initialLanguage]); + + const sections = language === 'sql' ? SQL_SECTIONS : LUCENE_SECTIONS; + + return ( + { + setQuery(''); + onClose(); + }} + title={Search Syntax Reference} + size="xl" + scrollAreaComponent={ScrollArea.Autosize} + > + + + { + setLanguage(val as Language); + setQuery(''); + }} + data={[ + { value: 'lucene', label: 'Lucene' }, + { value: 'sql', label: 'SQL' }, + ]} + /> + } + value={query} + onChange={e => setQuery(e.currentTarget.value)} + autoFocus + size="xs" + style={{ flex: 1 }} + /> + + + + + + + + + + ); +} From 906edeb91740fab247e094204820404c3b84012f Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 7 Jul 2026 15:10:04 -0600 Subject: [PATCH 164/728] fix: track display/heatmap settings as dirty for unsaved changes modal (#2597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Editing display settings or heatmap settings in the tile editor and then closing without saving no longer silently discards changes — the "unsaved changes" confirmation modal now fires as expected. **Root cause:** Both `handleUpdateDisplaySettings` and `handleUpdateHeatmapSettings` live in sub-forms (the display settings drawer / heatmap settings drawer) with their own independent React Hook Form instances. The main tile editor's RHF `isDirty` flag never sees these changes, so `EditTileModal.handleClose` never triggered the confirmation. **Approach:** Rather than fighting RHF's `shouldDirty` (which has normalization mismatches between `undefined` defaults and the drawer's concrete defaults like `false`/`0`/`true`), dirty state is signaled directly: - The display settings drawer now passes its own `isDirty` flag through `onChange`, so `handleUpdateDisplaySettings` only latches dirty state when the user actually changed a setting. A no-op Apply does not trigger the modal. - A `subFormDirty` ref in `EditTimeChartForm` prevents RHF's `isDirty=false` resets (from `values` prop re-syncs after `onSubmit`) from clearing the flag. **Heatmap settings** always mark dirty on Apply (no `isDirty` gating). Unlike display settings, the heatmap drawer doesn't have a no-op Apply concern — it only opens for heatmap charts where the user is actively configuring value/count expressions and scale type, and any Apply is intentional. --- .changeset/tnjv-sern-hwcq.md | 5 ++ packages/app/src/DBDashboardPage.tsx | 8 ++- .../components/ChartDisplaySettingsDrawer.tsx | 27 ++++++---- .../ChartEditorControls.tsx | 1 + .../DBEditTimeChartForm/EditTimeChartForm.tsx | 49 +++++++++++++------ .../app/tests/e2e/features/dashboard.spec.ts | 26 ++++++++++ 6 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 .changeset/tnjv-sern-hwcq.md diff --git a/.changeset/tnjv-sern-hwcq.md b/.changeset/tnjv-sern-hwcq.md new file mode 100644 index 0000000000..d4f163d850 --- /dev/null +++ b/.changeset/tnjv-sern-hwcq.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: display/heatmap settings changes now trigger the unsaved changes modal when closing the tile editor diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 1ed9cf72ad..2958243804 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -1188,11 +1188,15 @@ const EditTileModal = ({ const confirm = useConfirm(); const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); + // Reset dirty state only when a *different* tile is opened, not on every + // chart-object reference change (onSubmit recreates the chart object with + // the same id, which would clear dirty state after display-settings Apply). + const chartId = chart?.id; useEffect(() => { - if (chart != null) { + if (chartId != null) { setHasUnsavedChanges(false); } - }, [chart]); + }, [chartId]); const handleClose = useCallback(() => { if (isSaving) return; diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index bc3f05d461..e163b519c3 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -70,7 +70,7 @@ interface ChartDisplaySettingsDrawerProps { /** 'sql' for raw SQL chart configs; anything else is treated as a builder config. */ configType?: 'sql' | 'builder' | 'promql'; previousDateRange?: [Date, Date]; - onChange: (settings: ChartConfigDisplaySettings) => void; + onChange: (settings: ChartConfigDisplaySettings, isDirty: boolean) => void; onClose: () => void; isPerSeriesNumberFormatAllowed?: boolean; } @@ -150,13 +150,17 @@ export default function ChartDisplaySettingsDrawer({ // instead of freezing the drawer's inferred fallback into the config. const numberFormatExplicit = settings.numberFormat != null || dirtyFields.numberFormat != null; - onChange({ - ...rest, - numberFormat: numberFormatExplicit - ? formValues.numberFormat - : undefined, - colorRules: colorRules ? stripLocalIds(colorRules) : undefined, - }); + const hasDirtyFields = Object.keys(dirtyFields).length > 0; + onChange( + { + ...rest, + numberFormat: numberFormatExplicit + ? formValues.numberFormat + : undefined, + colorRules: colorRules ? stripLocalIds(colorRules) : undefined, + }, + hasDirtyFields, + ); })(); onClose(); }, [onChange, handleSubmit, onClose, settings.numberFormat, dirtyFields]); @@ -349,7 +353,12 @@ export default function ChartDisplaySettingsDrawer({ - diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx index ce28177822..26c0212c75 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx @@ -315,6 +315,7 @@ export function ChartEditorControls({ onClick={openDisplaySettings} size="compact-sm" variant="secondary" + data-testid="display-settings-button" > Display Settings diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 889c5df4c3..4bb3ff0452 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -192,8 +192,15 @@ export default function EditTimeChartForm({ [insertSeries, getValues], ); + // Track whether sub-form changes (display settings, heatmap settings) have + // been applied. These bypass RHF's dirty tracking, so we latch here: once + // set, only a parent reset (new tile opened) clears it via onDirtyChange. + const subFormDirty = useRef(false); + useEffect(() => { - onDirtyChange?.(isDirty); + // Don't let RHF's isDirty=false clear the flag after sub-form changes + // were applied (RHF resets isDirty when its `values` prop re-syncs). + onDirtyChange?.(isDirty || subFormDirty.current); }, [isDirty, onDirtyChange]); const select = useWatch({ control, name: 'select' }); @@ -588,18 +595,21 @@ export default function EditTimeChartForm({ const [parentRef, setParentRef] = useState(null); const handleUpdateDisplaySettings = useCallback( - ({ - numberFormat, - alignDateRangeToGranularity, - fillNulls, - compareToPreviousPeriod, - fitYAxisToData, - groupByColumnsOnLeft, - seriesLimit, - color, - colorRules, - backgroundChart, - }: ChartConfigDisplaySettings) => { + ( + { + numberFormat, + alignDateRangeToGranularity, + fillNulls, + compareToPreviousPeriod, + fitYAxisToData, + groupByColumnsOnLeft, + seriesLimit, + color, + colorRules, + backgroundChart, + }: ChartConfigDisplaySettings, + isDirty: boolean, + ) => { // Only persist an explicit numberFormat. When the drawer emits undefined // (the user never chose a format), leave it unset so render-time // auto-detection keeps driving the format from the datasource. @@ -618,9 +628,15 @@ export default function EditTimeChartForm({ setValue('color', color); setValue('colorRules', colorRules); setValue('backgroundChart', backgroundChart); + // Display settings live in a separate drawer form, so RHF can't track + // them. Latch dirty state only when the drawer reports actual changes. + if (isDirty) { + subFormDirty.current = true; + onDirtyChange?.(true); + } onSubmit(); }, - [setValue, onSubmit], + [setValue, onDirtyChange, onSubmit], ); const handleUpdateHeatmapSettings = useCallback( @@ -628,10 +644,13 @@ export default function EditTimeChartForm({ setValue('series.0.valueExpression', data.value); setValue('series.0.countExpression', data.count || 'count()'); setValue('series.0.heatmapScaleType', data.scaleType); + // Heatmap settings are applied outside RHF's change tracking. + subFormDirty.current = true; + onDirtyChange?.(true); onSubmit(); closeHeatmapSettings(); }, - [setValue, onSubmit, closeHeatmapSettings], + [setValue, onDirtyChange, onSubmit, closeHeatmapSettings], ); const heatmapValueExpression = useWatch({ diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 1dfb96ea28..9a37e55867 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -292,6 +292,32 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { }); }); + test('should warn when closing tile editor with unsaved display settings changes', async () => { + await dashboardPage.openNewTileEditor(); + + // Open the Display Settings drawer + await dashboardPage.page.getByTestId('display-settings-button').click(); + const applyButton = dashboardPage.page.getByTestId( + 'display-settings-apply-button', + ); + await expect(applyButton).toBeVisible({ timeout: 5000 }); + + // Toggle a checkbox via its label + await dashboardPage.page + .locator('label', { hasText: 'Compare to Previous Period' }) + .click(); + + // Apply and wait for the drawer to close + await applyButton.click(); + await expect(applyButton).toBeHidden({ timeout: 5000 }); + + // Try to close — should show unsaved changes confirm + await dashboardPage.page.keyboard.press('Escape'); + await expect(dashboardPage.unsavedChangesConfirmModal).toBeAttached({ + timeout: 5000, + }); + }); + test('should add and remove alert on Number type chart', async () => { test.setTimeout(60000); const ts = Date.now(); From a34b7fb39bf0d3d30a9db2912f9d4833fe71bef9 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 7 Jul 2026 15:57:29 -0600 Subject: [PATCH 165/728] fix(charts): use multi-line ellipsis for dashboard tile titles (#2596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes dashboard tile titles getting clipped unpredictably when tiles are resized small, and collapses toolbar actions into a compact popover at narrow widths. ## Changes **Title truncation** — Apply `-webkit-line-clamp: 4` with `overflow-wrap: break-word` to the tile title wrapper so titles wrap naturally across up to 4 lines, truncate cleanly with "..." if needed, and only break mid-word when a single word is wider than the tile. **Toolbar collapse** — Use a CSS container query (`@container (max-width: 200px)`) to detect narrow tiles. At that breakpoint the normal toolbar icons are hidden and replaced with a single ⋮ icon that opens a popover containing the same toolbar actions. --- .changeset/tile-title-ellipsis.md | 5 + packages/app/src/DBDashboardPage.tsx | 142 +++++++++++++++++- .../components/DBSearchPageFilters/hooks.ts | 2 +- .../charts/ChartContainer.module.scss | 38 +++++ .../src/components/charts/ChartContainer.tsx | 98 +++++++++++- 5 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 .changeset/tile-title-ellipsis.md create mode 100644 packages/app/src/components/charts/ChartContainer.module.scss diff --git a/.changeset/tile-title-ellipsis.md b/.changeset/tile-title-ellipsis.md new file mode 100644 index 0000000000..fb02764e2c --- /dev/null +++ b/.changeset/tile-title-ellipsis.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +Fix dashboard tile titles getting clipped unpredictably when tiles are resized small by applying multi-line ellipsis truncation. diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 2958243804..138b04c963 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -136,6 +136,7 @@ import { calculateNextTilePosition, makeId } from '@/utils/tilePositioning'; import ChartContainer, { ChartContainerCardHeaderProvider, + CollapsedToolbarProvider, DASHBOARD_TILE_PADDING_INLINE, } from './components/charts/ChartContainer'; import DBHeatmapChart, { @@ -838,6 +839,136 @@ const Tile = forwardRef( openFullscreen, ]); + // Flat Menu.Item list for the collapsed (narrow-tile) toolbar. + // Merges the alert action + all kebab items into a single flat list + // so ChartContainer can render them without nested menus. + const collapsedMenuItems = useMemo(() => { + const isRawSql = isRawSqlSavedChartConfig(chart.config); + const isPromQL = isPromqlSavedChartConfig(chart.config); + const showAlerts = isRawSql + ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) + : isPromQL + ? displayTypeSupportsPromQLAlerts(chart.config.displayType) + : displayTypeSupportsBuilderAlerts(chart.config.displayType); + const canMoveToGroup = + onMoveToGroup && moveTargets && moveTargets.length > 0; + return ( + <> + {showAlerts && ( + <> + : + } + onClick={onEditClick} + > + {alertTooltip} + + + + )} + } + onClick={onDuplicateClick} + > + Duplicate + + } + onClick={() => openFullscreen()} + > + View fullscreen + + } + onClick={onEditClick} + > + Edit + + {canMoveToGroup && ( + <> + + Move to Group + {chart.containerId && ( + } + onClick={() => onMoveToGroup(undefined)} + > + (Ungrouped) + + )} + {moveTargets + .filter( + t => + !( + t.containerId === chart.containerId && + t.tabId === chart.tabId + ), + ) + .map(t => ( + } + onClick={() => onMoveToGroup(t.containerId, t.tabId)} + > + {t.allTabs ? ( + + {t.allTabs.map((tab, i) => ( + + {i > 0 && ( + + {' | '} + + )} + + {tab.title} + + + ))} + + ) : ( + t.label + )} + + ))} + + )} + + } + onClick={onDeleteClick} + > + Delete + + + ); + }, [ + alert, + alertTooltip, + moveTargets, + chart.config, + chart.containerId, + chart.tabId, + onDeleteClick, + onDuplicateClick, + onEditClick, + onMoveToGroup, + openFullscreen, + ]); + const title = useMemo( () => chart.config.name ? ( @@ -1132,9 +1263,14 @@ const Tile = forwardRef( style={{ paddingInline: DASHBOARD_TILE_PADDING_INLINE }} onMouseDown={e => e.stopPropagation()} > - - {renderChartContent()} - + + + {renderChartContent()} + +
{children}
diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index fa0ccfd70e..84fbe8b245 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -397,7 +397,7 @@ export function useFetchFacets({ useEffect(() => { // eslint-disable-next-line react-hooks/set-state-in-effect setExtraFacets(null); - // eslint-disable-next-line react-hooks/set-state-in-effect + setExtraFacetKeys(new Set()); }, [ sourceId, diff --git a/packages/app/src/components/charts/ChartContainer.module.scss b/packages/app/src/components/charts/ChartContainer.module.scss new file mode 100644 index 0000000000..079d72f987 --- /dev/null +++ b/packages/app/src/components/charts/ChartContainer.module.scss @@ -0,0 +1,38 @@ +.root { + container-type: inline-size; +} + +/* Normal toolbar: visible by default, hidden when narrow */ +.toolbarNormal { + display: flex; +} + +/* Collapsed toolbar: hidden by default, shown when narrow */ +.toolbarCollapsed { + display: none; +} + +/* Hide the inline section (row + divider) when the row has no visible + child elements — e.g. Number/Table tiles where all prefix items are + null and no chart-specific indicators apply. */ +.inlineSection:not(:has(.inlineRow > *)) { + display: none; +} + +/* Inline row inside the collapsed dropdown: indicators stay compact, + the last child (typically the DisplaySwitcher toggle group) stretches + to fill remaining width with icons spread evenly. */ +.inlineRow > :last-child { + flex: 1; + justify-content: space-evenly; +} + +@container (max-width: 200px) { + .toolbarNormal { + display: none; + } + + .toolbarCollapsed { + display: flex; + } +} diff --git a/packages/app/src/components/charts/ChartContainer.tsx b/packages/app/src/components/charts/ChartContainer.tsx index 58f13e6832..d92ad1c168 100644 --- a/packages/app/src/components/charts/ChartContainer.tsx +++ b/packages/app/src/components/charts/ChartContainer.tsx @@ -1,5 +1,8 @@ import { createContext, use } from 'react'; -import { Group, Stack } from '@mantine/core'; +import { ActionIcon, Group, Menu, Stack, Tooltip } from '@mantine/core'; +import { IconDotsVertical } from '@tabler/icons-react'; + +import styles from './ChartContainer.module.scss'; interface ChartContainerProps { title?: React.ReactNode; @@ -27,6 +30,33 @@ export function ChartContainerCardHeaderProvider({ ); } +// Collapsed toolbar context for narrow dashboard tiles. +// `menuItems` contains pre-built Menu.Item elements (alert + kebab actions) +// that render as a flat list in the collapsed dropdown. `suffixCount` tells +// ChartContainer how many items at the end of toolbarItems[] are "suffix" +// (i.e. the hoverToolbar) and should be excluded from the inline row in the +// collapsed view — they are replaced by the flat menuItems instead. +interface CollapsedToolbarData { + menuItems: React.ReactNode; + suffixCount: number; +} + +const CollapsedToolbarContext = createContext( + null, +); + +export function CollapsedToolbarProvider({ + menuItems, + suffixCount, + children, +}: CollapsedToolbarData & { children: React.ReactNode }) { + return ( + + {children} + + ); +} + // Horizontal padding a dashboard tile applies to its content. The card header // bleeds its separator to the tile edge by cancelling exactly this inset, so // the tile must consume the same value (see DASHBOARD_TILE_PADDING_INLINE usage @@ -43,9 +73,21 @@ function ChartContainer({ disableReactiveContainer, }: ChartContainerProps) { const cardHeader = use(ChartContainerCardHeaderContext); + const collapsedToolbar = use(CollapsedToolbarContext); const hasToolbar = !!toolbarItems?.length; const showHeader = !!title || hasToolbar; + // In the collapsed view, split toolbarItems into inline items (indicators, + // toggles) and suffix items (hoverToolbar) which get replaced by flat + // Menu.Items from context. Filter out null/undefined entries (e.g. + // filterWarning is null when no warning applies). + const inlineItems = collapsedToolbar + ? toolbarItems + ?.slice(0, -collapsedToolbar.suffixCount || undefined) + .filter(Boolean) + : undefined; + const hasInlineItems = !!inlineItems?.length; + return ( // Reset the context so nested ChartContainers don't inherit the card // header styling; only the top-most container under a tile gets it. @@ -55,6 +97,7 @@ function ChartContainer({ w="100%" gap={cardHeader ? 'xs' : undefined} style={{ flexGrow: 1 }} + className={styles.root} > {showHeader && ( {title} {toolbarItems && ( - - {toolbarItems} - + <> + {/* Normal toolbar: visible at wider sizes */} + + {toolbarItems} + + {/* Collapsed toolbar: single ⋮ dropdown at narrow sizes. + Row 1 shows inline items (indicators, toggles) as-is, + then a divider, then flattened action Menu.Items. */} + {collapsedToolbar && ( +
+ + + + + + + + + e.stopPropagation()}> + {hasInlineItems && ( +
+ + {inlineItems} + + +
+ )} + {collapsedToolbar.menuItems} + + +
+ )} + )} )} From 2b209d3ad3c554742685c06edcb8fa0db384d74d Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 8 Jul 2026 13:56:05 +0100 Subject: [PATCH 166/728] fix(TimePicker): stabilize flaky time-picker e2e tests (#2516) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The `custom-time-range` and `relative-time` search e2e specs are flaky on CI — `custom-time-range` even hard-fails on all retries on some runs (e.g. [this Shard 3 run](https://github.com/hyperdxio/hyperdx/actions/runs/28083170606/job/83142681211)). The flakiness is unrelated to the PR that surfaced it; it's a latent bug in the `TimePicker` plus non-deterministic Playwright helpers. ## Root causes 1. **Stale derived state (app bug).** `isRelative` was seeded from `defaultRelativeTimeMode` only at mount, so the relative/absolute toggle silently diverged from URL state when the live interval changed — the picker rendered in a mode that no longer matched the URL. 2. **Non-idempotent `open()` helper.** Clicking the input *toggles* the popover. Right after a programmatic close (apply/select), a click could be swallowed by a re-render (popover stays shut) or land while the popover was still detaching (flicker open→closed), so the date inputs / interval buttons / relative switch were "not found". 3. **Cross-test state leak.** The picker mode is persisted in `localStorage` via `atomWithStorage('hdx-time-picker-mode')`, so a prior test that switched to "Around a time" leaked into the reopen test (only one date input rendered → `endDateInput` never resolved). This is what made `custom-time-range` hard-fail. 4. **Helpers that hung to the 60s timeout** instead of failing fast (reading `isChecked()` / clicking a re-rendering button). ## What changed **App** — `TimePicker.tsx` - Sync `isRelative` to `defaultRelativeTimeMode` when the prop becomes true. Force **on only**: the prop is a lossy signal (also `false` for relative mode at the default live-tail interval), so syncing back to `false` would wrongly drop the user out of relative mode when they pick the default interval. **Tests** — `TimePickerComponent.ts` - `open()` now ends in a *stably-open* state via a settle+retry loop (handles already-open, closed, mid-close, and swallowed-click cases). - `apply()` waits for the popover to close + network to settle so a following `open()` starts settled. - `toggleRelativeTimeSwitch` / `selectTimeInterval` / `isRelativeTimeEnabled` retry or fail fast instead of hanging. **Tests** — specs - `custom-time-range`: reset the persisted `hdx-time-picker-mode` atom in `beforeEach` for isolation. - `relative-time`: open the popover at the start of each loop iteration instead of relying on trailing state. ## Verification (local, full-stack e2e) - `custom-time-range`: 4 consecutive clean runs - `relative-time`: 4 consecutive clean runs - Both together at `--workers=2` (CI parity): 4 consecutive clean runs (16/16) - Entire `search/` e2e directory at `--workers=2`: 65/65 passing (covers every spec using the shared `TimePickerComponent`) - `tsc --noEmit` clean; eslint clean A changeset is included (`@hyperdx/app` patch). --- .changeset/timepicker-relative-mode-sync.md | 11 ++ .../src/components/TimePicker/TimePicker.tsx | 17 +++ .../e2e/components/TimePickerComponent.ts | 138 +++++++++++++++--- .../features/search/custom-time-range.spec.ts | 16 ++ .../e2e/features/search/relative-time.spec.ts | 8 +- 5 files changed, 169 insertions(+), 21 deletions(-) create mode 100644 .changeset/timepicker-relative-mode-sync.md diff --git a/.changeset/timepicker-relative-mode-sync.md b/.changeset/timepicker-relative-mode-sync.md new file mode 100644 index 0000000000..6fa5f24d92 --- /dev/null +++ b/.changeset/timepicker-relative-mode-sync.md @@ -0,0 +1,11 @@ +--- +"@hyperdx/app": patch +--- + +fix(TimePicker): keep relative/absolute toggle in sync with URL state + +The time picker's relative-time toggle was only seeded from +`defaultRelativeTimeMode` at mount and never re-synced when the prop changed +(e.g. after switching live intervals via the URL). This left the picker +rendering in a mode that no longer matched the URL, causing nondeterministic +behavior. The toggle now follows `defaultRelativeTimeMode` whenever it changes. diff --git a/packages/app/src/components/TimePicker/TimePicker.tsx b/packages/app/src/components/TimePicker/TimePicker.tsx index 6e5002495e..18f14f6e54 100644 --- a/packages/app/src/components/TimePicker/TimePicker.tsx +++ b/packages/app/src/components/TimePicker/TimePicker.tsx @@ -234,6 +234,23 @@ const TimePickerComponent = ({ ); const [isRelative, setIsRelative] = useState(defaultRelativeTimeMode); + // Promote the toggle into relative mode when the URL-derived + // `defaultRelativeTimeMode` prop becomes true (e.g. after a reload while a + // non-default live interval is active). Previously `isRelative` was only + // seeded at mount, so this case left the picker in absolute mode even though + // the URL described a relative range. + // + // We intentionally only force the toggle ON, never OFF. The prop is a lossy + // signal — it is false both for "absolute mode" and for "relative mode at + // the default live-tail interval" (e.g. Live Tail / Last 15 minutes), so + // syncing it back to false would incorrectly drop the user out of relative + // mode whenever they pick the default interval. Turning relative mode off is + // left to the user's explicit toggle. + React.useEffect(() => { + if (defaultRelativeTimeMode) { + setIsRelative(true); + } + }, [defaultRelativeTimeMode]); // Must be state to ensure rerenders occur when ref changes const [containerRef, setContainerRef] = useState(null); const dateComponentPopoverProps = useMemo( diff --git a/packages/app/tests/e2e/components/TimePickerComponent.ts b/packages/app/tests/e2e/components/TimePickerComponent.ts index 4a2a9fd341..860689f9d2 100644 --- a/packages/app/tests/e2e/components/TimePickerComponent.ts +++ b/packages/app/tests/e2e/components/TimePickerComponent.ts @@ -2,7 +2,7 @@ * TimePickerComponent - Reusable component for time range selection * Used across Search, Dashboard, Logs, Traces, and other time-filtered pages */ -import { Locator, Page } from '@playwright/test'; +import { expect, Locator, Page } from '@playwright/test'; export class TimePickerComponent { readonly page: Page; @@ -67,14 +67,62 @@ export class TimePickerComponent { * Clicking the input when the popover is already open would close it. */ async open() { - const isOpen = await this.pickerPopover.isVisible(); - if (isOpen) return; await this.page.waitForLoadState('networkidle'); - await this.page.keyboard.press('Escape'); - await this.pickerInput.click(); + + // The input toggles the popover, so this method must end in a *stably + // open* state. The tricky cases this loop handles: + // - Already open and stable → nothing to do. + // - Closed → one click opens it. + // - Mid-close (still visible but about to detach, e.g. right after + // selectTimeInterval()'s handleRelativeSearch → close()) → an early + // return here would hand back a popover that then vanishes. We instead + // wait for it to settle to a stable hidden state, then click to open. + // - Click swallowed by a React re-render → retry. + for (let attempt = 0; attempt < 5; attempt++) { + if (await this.isPopoverStablyOpen()) return; + + // Settle to a clean hidden baseline before toggling open. If it's + // genuinely open-and-stable the check above already returned; if it's + // closing, wait for hidden so our click reopens rather than re-closes. + await this.pickerPopover + .waitFor({ state: 'hidden', timeout: 2000 }) + .catch(() => { + // Either already hidden or still finishing an open; the stability + // check at the top of the next iteration will sort it out. + }); + + // The popover is closed here, so it is safe to press Escape to dismiss + // any overlay that would intercept the toggle click — most importantly + // the search input's autocomplete dropdown ("Searching for:" portal), + // which renders on top of the time-picker input after typing a query + // (e.g. traces-workflow types "Order" then opens the picker). Escape is + // only pressed from this closed baseline, so it can't race a just-opened + // popover shut. + await this.page.keyboard.press('Escape'); + + await this.pickerInput.click(); + if (await this.isPopoverStablyOpen()) return; + } + // Surface a clear failure if it still hasn't opened. await this.pickerPopover.waitFor({ state: 'visible', timeout: 5000 }); } + /** + * True only if the popover is visible and *stays* visible briefly afterwards + * — i.e. it is not in the middle of a close transition. Used by open() to + * avoid returning a popover that is about to detach. + */ + private async isPopoverStablyOpen(): Promise { + if (!(await this.pickerPopover.isVisible())) return false; + // Dwell long enough to outlast the post-apply / post-select re-render + // cascade (URL update + results reload) that can close a just-opened + // popover. Sample twice so a mid-cascade flicker is caught. + await this.page.waitForTimeout(250); + if (!(await this.pickerPopover.isVisible())) return false; + await this.page.waitForTimeout(250); + return await this.pickerPopover.isVisible(); + } + /** * Close the time picker dropdown */ @@ -83,17 +131,45 @@ export class TimePickerComponent { } /** - * Toggle the relative time switch + * Toggle the relative time switch and wait until its checked state actually + * flips. + * + * Mantine renders the switch as a visually-hidden whose immediate + * parent (`..`) is the clickable track. Clicking that toggles the switch. + * Immediately after the popover re-opens the track can still be settling, so + * a single click occasionally doesn't register — retry until the checked + * state changes (re-opening the popover between attempts if it closed). */ async toggleRelativeTimeSwitch() { - // Click parent element to trigger the switch - await this.relativeTimeSwitch.locator('..').click({ timeout: 5000 }); + await this.relativeTimeSwitch.waitFor({ state: 'attached', timeout: 5000 }); + const before = await this.relativeTimeSwitch.isChecked(); + const track = this.relativeTimeSwitch.locator('..'); + for (let attempt = 0; attempt < 4; attempt++) { + await track.click({ timeout: 5000 }); + try { + await expect(this.relativeTimeSwitch).toBeChecked({ + checked: !before, + timeout: 2000, + }); + return; + } catch { + // Click didn't register (popover re-render); ensure it's open and retry. + await this.open(); + } + } + await expect(this.relativeTimeSwitch).toBeChecked({ checked: !before }); } /** - * Check if relative time mode is enabled + * Check if relative time mode is enabled. + * + * The relative-time switch only renders while the popover is open. Wait for + * it to attach before reading `isChecked()` so a closed/transitioning + * popover fails fast with a clear error instead of hanging until the 60s + * test timeout. */ async isRelativeTimeEnabled(): Promise { + await this.relativeTimeSwitch.waitFor({ state: 'attached', timeout: 5000 }); return await this.relativeTimeSwitch.isChecked(); } @@ -126,15 +202,25 @@ export class TimePickerComponent { const intervalButton = this.pickerPopover.getByRole('button', { name: label, }); - // Wait for the specific button to be visible before clicking. - // Avoid calling waitForLoadState('networkidle') here — the popover is - // already open from open(), and waiting for network idle can coincide - // with React re-renders of the popover content, causing the button to - // briefly detach from the DOM right before the click. - await intervalButton.waitFor({ state: 'visible', timeout: 5000 }); - // Use a longer click timeout so Playwright can retry if the element - // briefly detaches due to an ongoing render cycle. - await intervalButton.click({ timeout: 10000 }); + // The popover content re-renders (the previously-selected option flips its + // Mantine variant, the whole list reconciles), which can detach the target + // button mid-click. A single click() with a long timeout still fails if + // the element it resolved goes stale during the actionability wait, so + // retry the visible-wait + click as a unit until it lands. + let lastError: unknown; + for (let attempt = 0; attempt < 4; attempt++) { + try { + await intervalButton.waitFor({ state: 'visible', timeout: 5000 }); + await intervalButton.click({ timeout: 5000 }); + return; + } catch (error) { + lastError = error; + // The popover may have closed (e.g. a swallowed render); make sure it + // is open again before the next attempt. + await this.open(); + } + } + throw lastError; } /** @@ -170,10 +256,24 @@ export class TimePickerComponent { } /** - * Apply the selected time range + * Apply the selected time range. + * + * handleApply() runs the search and then closes the popover, which kicks off + * a re-render cascade (URL update, results reload). Wait for the popover to + * actually close so callers that immediately re-open it (e.g. the + * close/reopen regression test) start from a settled, closed state instead + * of racing the in-flight close. */ async apply() { await this.pickerApplyButton.click({ timeout: 5000 }); + await this.pickerPopover + .waitFor({ state: 'hidden', timeout: 5000 }) + .catch(() => { + // If it never reports hidden, open()'s own settling logic will cope. + }); + await this.page.waitForLoadState('networkidle').catch(() => { + // Network may stay busy (live tail); proceeding is safe. + }); } /** diff --git a/packages/app/tests/e2e/features/search/custom-time-range.spec.ts b/packages/app/tests/e2e/features/search/custom-time-range.spec.ts index 4d14276734..34f418aba2 100644 --- a/packages/app/tests/e2e/features/search/custom-time-range.spec.ts +++ b/packages/app/tests/e2e/features/search/custom-time-range.spec.ts @@ -18,6 +18,18 @@ test.describe('Custom Time Range', { tag: '@custom-time-range' }, () => { test.beforeEach(async ({ page }) => { searchPage = new SearchPage(page); + // The picker mode is persisted in localStorage via an atomWithStorage + // ('hdx-time-picker-mode'), so a prior test that switched to "Around a + // time" can leak into this one and leave the picker mounted with a single + // date input. Reset the key before any page script runs so the picker + // always mounts in the default Range mode. + await page.addInitScript(() => { + try { + window.localStorage.removeItem('hdx-time-picker-mode'); + } catch { + // localStorage may be unavailable in some contexts; ignore. + } + }); await searchPage.goto(); await expect(searchPage.form).toBeVisible(); await searchPage.timePicker.open(); @@ -73,6 +85,10 @@ test.describe('Custom Time Range', { tag: '@custom-time-range' }, () => { await test.step('Reopen picker and verify inputs still show typed times', async () => { await searchPage.timePicker.open(); + // Range mode (two date inputs) is guaranteed because the beforeEach + // resets the persisted 'hdx-time-picker-mode' atom and selects Range + // mode, and applying an absolute range leaves isLive=false (absolute + // mode). Both Start and End inputs are therefore present here. await expect(searchPage.timePicker.startDateInput).toHaveValue(start); await expect(searchPage.timePicker.endDateInput).toHaveValue(end); }); diff --git a/packages/app/tests/e2e/features/search/relative-time.spec.ts b/packages/app/tests/e2e/features/search/relative-time.spec.ts index 13a37f8f66..c719df278f 100644 --- a/packages/app/tests/e2e/features/search/relative-time.spec.ts +++ b/packages/app/tests/e2e/features/search/relative-time.spec.ts @@ -62,6 +62,12 @@ test.describe('Relative Time Picker', { tag: '@relative-time' }, () => { for (const interval of intervals) { await test.step(`Select ${interval.label}`, async () => { + // Make sure the popover is open at the start of every iteration + // rather than relying on the previous iteration leaving it open. + // selectTimeInterval() closes the popover, so each iteration must + // re-open it independently. open() is idempotent. + await searchPage.timePicker.open(); + // Ensure relative time mode is enabled await searchPage.timePicker.enableRelativeTime(); @@ -83,8 +89,6 @@ test.describe('Relative Time Picker', { tag: '@relative-time' }, () => { // Verify the time picker input displays the selected interval await expect(searchPage.timePicker.input).toHaveValue(interval.label); - - await searchPage.timePicker.open(); }); } }); From 6e25c1d5bba92d73019699eeb77d2067f74998ca Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:33:43 +0200 Subject: [PATCH 167/728] feat: trace waterfall redesign (#2565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Redesigns the trace waterfall chart and cleans up the surrounding trace panel layout. ### Waterfall redesign (HDX-3945) - **Per-service colors**: each service gets a stable color (sorted for render-order stability). Green is reserved for correlated log rows and red for error spans, so a service color is never confused with a log or error. - **Bar & label layout**: no text inside bars — the operation/service label sits beside a vertical service color bar, with the span duration shown as muted text and the span body revealed on hover, placed on whichever side has more room. - **Depth controls**: expand/collapse one level and expand/collapse all, only shown when the trace has collapsible nodes. ### Trace panel layout (HDX-3952) - Removed the duplicated Trace ID from the panel body (it lives in the side panel header). The Trace ID Expression editor now only appears as a fallback when no trace id resolves for the event. - Moved the Correlated Logs source selector into the waterfall controls bar as a compact selector. ### Collapse hint bug (HDX-4444) - Removed the flaky tooltip-based Alt+click collapse hint (`CollapseTooltipLabel` / `useDisclosure`) that could render multiple times when chevrons were highlighted in quick succession. Collapse is now an always-present chevron control. ### Supporting changes - `useWaterfallSearchState` now persists the filter language (`whereLanguage`) in the URL so shared links / reloads rebuild the query in the language it was written in. - `SearchWhereInput` accepts a `parentRef` for popover positioning. ## Testing - Added `DBTraceWaterfallChart.test.tsx` covering the depth-control logic. - Updated `DBTracePanel.test.tsx` for the new layout. ### Screenshots or video Screenshot 2026-07-01 at 17 12 42 ### How to test on Vercel preview **Preview routes:** `/search` **Steps:** 1. Open the trace timeline and explore the new features described above. ### References - Linear Issue: Closes HDX-3945, HDX-3952, HDX-4443, and HDX-4444. - Related PRs: https://github.com/hyperdxio/hyperdx/pull/1970 --- .changeset/trace-waterfall-redesign.md | 5 + packages/app/src/components/DBTracePanel.tsx | 91 +-- .../src/components/DBTraceWaterfallChart.tsx | 651 +++++++++++++----- .../SearchInput/SearchWhereInput.tsx | 3 + .../TimelineChart/TimelineChart.module.scss | 47 +- .../TimelineChart/TimelineChart.stories.tsx | 46 ++ .../TimelineChart/TimelineChart.tsx | 6 +- .../TimelineChart/TimelineChartRowEvents.tsx | 36 +- .../TimelineChart/TimelineXAxis.tsx | 78 ++- .../__tests__/DBTracePanel.test.tsx | 40 +- .../__tests__/DBTraceWaterfallChart.test.tsx | 442 +++++++++++- .../app/src/hooks/useWaterfallSearchState.tsx | 25 +- packages/app/src/utils.ts | 14 - 13 files changed, 1183 insertions(+), 301 deletions(-) create mode 100644 .changeset/trace-waterfall-redesign.md diff --git a/.changeset/trace-waterfall-redesign.md b/.changeset/trace-waterfall-redesign.md new file mode 100644 index 0000000000..e5d9f09894 --- /dev/null +++ b/.changeset/trace-waterfall-redesign.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: redesign the trace waterfall — per-service span colors, vertical service color bar, child counts, duration outside the bar with the span body on hover, expand/collapse depth controls. diff --git a/packages/app/src/components/DBTracePanel.tsx b/packages/app/src/components/DBTracePanel.tsx index f5743f38a1..db5668fe5d 100644 --- a/packages/app/src/components/DBTracePanel.tsx +++ b/packages/app/src/components/DBTracePanel.tsx @@ -14,14 +14,13 @@ import { ActionIcon, Box, Button, - Divider, Flex, Group, Stack, Text, Tooltip, } from '@mantine/core'; -import { IconPencil, IconX } from '@tabler/icons-react'; +import { IconX } from '@tabler/icons-react'; import { DBTraceWaterfallChartContainer } from '@/components/DBTraceWaterfallChart'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; @@ -266,11 +265,8 @@ export default function DBTracePanel({ } }, [parentSourceData, traceIdSetValue]); - const [showTraceIdInput, setShowTraceIdInput] = useState(false); - - // Tidy the URL: drop a selection that belongs to a different trace. Purely - // cosmetic — the read-time `selectedSpan` gate already prevents a foreign - // selection from rendering; this just keeps the param from lingering. + // Reset highlighted row when trace ID changes + // otherwise we'll show stale span details useEffect(() => { if (eventRowWhere != null && eventRowWhere.traceId !== traceId) { setEventRowWhere(null); @@ -307,50 +303,11 @@ export default function DBTracePanel({ minHeight: 0, }} > - - - - {parentSourceData && - (isLogSource(parentSourceData) || isTraceSource(parentSourceData)) - ? parentSourceData.traceIdExpression - : ''} - : {traceId || 'No trace id found for event'} - - {traceId != null && ( - - )} - - - - {parentSourceData?.kind === SourceKind.Log - ? 'Trace Source' - : 'Correlated Log Source'} - - setIsSourceSchemaPreviewOpen(true)} - isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( - childSourceData, - )} - /> - setIsSourceSchemaPreviewOpen(false)} - /> - - - {(showTraceIdInput || !traceId) && parentSourceId != null && ( - + {/* Fallback Trace ID Expression editor: only surfaced when no trace id + resolved for the event. The trace id itself now lives in the side + panel header (Copy Trace ID), so it's not duplicated here. */} + {!traceId && parentSourceId != null && ( + Trace ID Expression Save - )} - {/* Inline resizable split view: waterfall (left) + span detail (right) */}
+ + Correlated logs + + setIsSourceSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( + childSourceData, + )} + /> + setIsSourceSchemaPreviewOpen(false)} + /> + + } /> )}
diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index d907140950..5ab9067317 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -21,25 +21,33 @@ import { TTraceSource, } from '@hyperdx/common-utils/dist/types'; import { + ActionIcon, Anchor, Box, Center, Chip, Code, Group, - Kbd, + Stack, Text, Tooltip, } from '@mantine/core'; -import { useDisclosure, useElementSize } from '@mantine/hooks'; +import { useElementSize } from '@mantine/hooks'; import { + IconAlertCircleFilled, + IconAlertTriangleFilled, IconChevronDown, IconChevronRight, + IconChevronsDown, + IconChevronsRight, IconLogs, } from '@tabler/icons-react'; import { ContactSupportText } from '@/components/ContactSupportText'; -import SearchInputV2 from '@/components/SearchInput/SearchInputV2'; +import { ErrorCollapse } from '@/components/Error/ErrorCollapse'; +import SearchWhereInput, { + getStoredLanguage, +} from '@/components/SearchInput/SearchWhereInput'; import { TimelineChart, TimelineMinimap, @@ -56,12 +64,12 @@ import { } from '@/source'; import { useFormatTime } from '@/useFormatTime'; import { + CATEGORICAL_PALETTE_TOKENS, + COLORS, getChartColorError, - getChartColorErrorHighlight, getChartColorSuccess, getChartColorSuccessHighlight, getChartColorWarning, - getChartColorWarningHighlight, parseTimestampToMs, } from '@/utils'; import { @@ -97,28 +105,15 @@ type TimestampedRow = { Timestamp: string; }; -function textColor(condition: { isError: boolean; isWarn: boolean }): string { - const { isError, isWarn } = condition; - if (isError) return 'text-danger'; - if (isWarn) return 'text-warning'; - return ''; -} - +// Bar background color. Correlated log rows are always green (success), error +// spans are always red, and every other span takes its per-service color. function barColor(condition: { - isError: boolean; - isWarn: boolean; isHighlighted: boolean; + isError?: boolean; type: string | undefined; + serviceColor?: string; }) { - const { isError, isWarn, isHighlighted, type } = condition; - - if (isError) - return isHighlighted ? getChartColorErrorHighlight() : getChartColorError(); - - if (isWarn) - return isHighlighted - ? getChartColorWarningHighlight() - : getChartColorWarning(); + const { isHighlighted, isError, type, serviceColor } = condition; if (type === SourceKind.Log) { return isHighlighted @@ -126,9 +121,31 @@ function barColor(condition: { : getChartColorSuccess(); } + if (isError) { + return isHighlighted + ? `color-mix(in srgb, ${getChartColorError()} 60%, white)` + : getChartColorError(); + } + + if (serviceColor) { + return isHighlighted + ? `color-mix(in srgb, ${serviceColor} 60%, white)` + : serviceColor; + } + return isHighlighted ? '#A9AFB7' : '#6A7077'; } +// Per-service span palette: the full categorical palette minus the two hues we +// reserve for semantics — green (correlated log rows) and red (error spans) — +// so a service color is never confused with a log or an error. Index-aligned +// with CATEGORICAL_PALETTE_TOKENS, so filtering by token survives reordering. +const SERVICE_COLORS = COLORS.filter( + (_color, i) => + CATEGORICAL_PALETTE_TOKENS[i] !== 'chart-green' && + CATEGORICAL_PALETTE_TOKENS[i] !== 'chart-red', +); + function getTableBody(tableModel: TSource) { if (tableModel?.kind === SourceKind.Trace) { return getSpanEventBody(tableModel) ?? ''; @@ -143,6 +160,7 @@ function getConfig( source: TTraceSource | TLogSource, traceId: string, hiddenRowExpression?: string, + hiddenRowExpressionLanguage: 'lucene' | 'sql' = 'lucene', ) { const alias: Record = { Body: getTableBody(source), @@ -208,7 +226,7 @@ function getConfig( ? [ { valueExpression: hiddenRowExpression, - valueExpressionLanguage: 'lucene' as const, + valueExpressionLanguage: hiddenRowExpressionLanguage, alias: '__hdx_hidden', }, ] @@ -318,18 +336,26 @@ export function useEventsAroundFocus({ traceId, enabled, hiddenRowExpression, + hiddenRowExpressionLanguage = 'lucene', }: { tableSource: TTraceSource | TLogSource; focusDate: Date; dateRange: [Date, Date]; traceId: string; enabled: boolean; - /** A lucene expression that identifies rows to be hidden. Hidden rows will be returned with a `__hdx_hidden: true` column. */ + /** An expression (in `hiddenRowExpressionLanguage`) that identifies rows to be hidden. Hidden rows will be returned with a `__hdx_hidden: true` column. */ hiddenRowExpression?: string; + hiddenRowExpressionLanguage?: 'lucene' | 'sql'; }) { const { config, alias, type } = useMemo( - () => getConfig(tableSource, traceId, hiddenRowExpression), - [tableSource, traceId, hiddenRowExpression], + () => + getConfig( + tableSource, + traceId, + hiddenRowExpression, + hiddenRowExpressionLanguage, + ), + [tableSource, traceId, hiddenRowExpression, hiddenRowExpressionLanguage], ); const { @@ -398,6 +424,39 @@ export function useEventsAroundFocus({ }; } +function useFilteredEventsAroundFocus( + args: Parameters[0], +) { + const filtered = useEventsAroundFocus(args); + + const filterFailed = + args.enabled && !!args.hiddenRowExpression && !!filtered.error; + + const fallback = useEventsAroundFocus({ + ...args, + hiddenRowExpression: undefined, + enabled: filterFailed, + }); + + if (filterFailed) { + return { + rows: fallback.rows, + meta: fallback.meta, + isFetching: filtered.isFetching || fallback.isFetching, + filterError: fallback.error ? undefined : filtered.error, + fatalError: fallback.error, + }; + } + + return { + rows: filtered.rows, + meta: filtered.meta, + isFetching: filtered.isFetching, + filterError: undefined, + fatalError: filtered.error, + }; +} + export function getDescendantIds(node: { id?: string; children?: Array<{ id?: string; children?: any[] }>; @@ -419,14 +478,86 @@ export function getDescendantIds(node: { return ids; } -function CollapseTooltipLabel({ onShown }: { onShown: () => void }) { - useEffect(() => onShown, [onShown]); +/** "Collapse all": every collapsible parent, at every level, becomes collapsed. */ +export function computeCollapseAll( + parentIdsByLevel: Map>, +): Set { + const allParentIds = new Set(); + parentIdsByLevel.forEach(ids => { + ids.forEach(id => allParentIds.add(id)); + }); + return allParentIds; +} + +/** + * "Expand one level": expand the shallowest level that still has any collapsed + * parents. + */ +export function computeExpandOneLevel( + collapsedIds: Set, + parentIdsByLevel: Map>, +): Set { + if (collapsedIds.size === 0) return collapsedIds; + const newSet = new Set(collapsedIds); + // Shallowest first: expanding starts at the top of the tree. + const sortedLevels = [...parentIdsByLevel.keys()].sort((a, b) => a - b); + for (const level of sortedLevels) { + const ids = parentIdsByLevel.get(level)!; + const collapsedAtLevel = [...ids].filter(id => newSet.has(id)); + if (collapsedAtLevel.length > 0) { + collapsedAtLevel.forEach(id => newSet.delete(id)); + break; + } + } + return newSet; +} - return ( - <> - ⌥/Alt + click to collapse children - - ); +export function computeCollapseOneLevel( + collapsedIds: Set, + parentIdsByLevel: Map>, +): Set { + const newSet = new Set(collapsedIds); + // Deepest first: collapsing starts at the bottom of the tree. + const sortedLevels = [...parentIdsByLevel.keys()].sort((a, b) => b - a); + for (const level of sortedLevels) { + const ids = parentIdsByLevel.get(level)!; + const expandedAtLevel = [...ids].filter(id => !newSet.has(id)); + if (expandedAtLevel.length > 0) { + expandedAtLevel.forEach(id => newSet.add(id)); + break; + } + } + return newSet; +} + +export function computeToggleCollapse( + collapsedIds: Set, + id: string, + node: + | { id?: string; children?: Array<{ id?: string; children?: any[] }> } + | undefined, + includeDescendants: boolean, +): Set { + const next = new Set(collapsedIds); + const wasCollapsed = next.has(id); + + if (wasCollapsed) { + next.delete(id); + } else { + next.add(id); + } + + if (includeDescendants && node?.children?.length) { + // Match every descendant to the node's new state. + const descendantIds = getDescendantIds(node); + if (wasCollapsed) { + descendantIds.forEach(descId => next.delete(descId)); + } else { + descendantIds.forEach(descId => next.add(descId)); + } + } + + return next; } // TODO: Optimize with ts lookup tables @@ -440,6 +571,7 @@ export function DBTraceWaterfallChartContainer({ highlightedRowWhere, initialRowHighlightHint, emptyState, + controlsExtra, }: { traceTableSource: TTraceSource; logTableSource: TLogSource | null; @@ -458,31 +590,73 @@ export function DBTraceWaterfallChartContainer({ body: string; }; emptyState?: ReactNode; + /** Extra controls rendered in the waterfall controls bar (e.g. the correlated logs source selector). */ + controlsExtra?: ReactNode; }) { const formatTime = useFormatTime(); const { traceWhere, logWhere, + traceWhereLanguage, + logWhereLanguage, clear: clearFilters, isFilterActive, isFilterExpanded, setIsFilterExpanded, - onSubmit: onSubmitFilters, + onSubmit: submitFilters, } = useWaterfallSearchState({ hasLogSource: !!logTableSource, }); + // Each filter runs in its own language. Defaults come from the URL (shared + // link / reload fidelity), falling back to the stored preference. + const traceFilterLanguage: 'lucene' | 'sql' = + traceWhereLanguage === 'sql' ? 'sql' : 'lucene'; + const logFilterLanguage: 'lucene' | 'sql' = + logWhereLanguage === 'sql' ? 'sql' : 'lucene'; + const { control, handleSubmit, setValue } = useForm({ defaultValues: { traceWhere: traceWhere ?? '', logWhere: logWhere ?? '', + // Prefer each input's URL language so a shared link / reload shows the + // same language the filter runs in; fall back to the stored preference. + traceWhereLanguage: + traceWhereLanguage === 'sql' || traceWhereLanguage === 'lucene' + ? traceWhereLanguage + : (getStoredLanguage() ?? 'lucene'), + logWhereLanguage: + logWhereLanguage === 'sql' || logWhereLanguage === 'lucene' + ? logWhereLanguage + : (getStoredLanguage() ?? 'lucene'), }, }); + const onSubmitFilters = useCallback( + (data: { + traceWhere: string; + logWhere: string; + traceWhereLanguage: string; + logWhereLanguage: string; + }) => { + submitFilters({ + traceWhere: data.traceWhere, + logWhere: data.logWhere, + traceWhereLanguage: data.traceWhereLanguage, + logWhereLanguage: data.logWhereLanguage, + }); + }, + [submitFilters], + ); + const onClearFilters = useCallback(() => { setValue('traceWhere', ''); setValue('logWhere', ''); + // Reset the language toggles too, otherwise stale values get re-serialized + // into the URL on the next submit, undoing the cleared state. + setValue('traceWhereLanguage', getStoredLanguage() ?? 'lucene'); + setValue('logWhereLanguage', getStoredLanguage() ?? 'lucene'); clearFilters(); }, [clearFilters, setValue]); @@ -490,21 +664,24 @@ export function DBTraceWaterfallChartContainer({ rows: traceRowsData, isFetching: traceIsFetching, meta: traceRowsMeta, - error: traceError, - } = useEventsAroundFocus({ + filterError: traceFilterError, + fatalError: traceError, + } = useFilteredEventsAroundFocus({ tableSource: traceTableSource, focusDate, dateRange, traceId, hiddenRowExpression: traceWhere ? `NOT (${traceWhere})` : undefined, + hiddenRowExpressionLanguage: traceFilterLanguage, enabled: true, }); const { rows: logRowsData, isFetching: logIsFetching, meta: logRowsMeta, - error: logError, - } = useEventsAroundFocus({ + filterError: logFilterError, + fatalError: logFatalError, + } = useFilteredEventsAroundFocus({ // search data if logTableModel exist // search invalid date range if no logTableModel(react hook need execute no matter what) tableSource: logTableSource ? logTableSource : traceTableSource, @@ -512,11 +689,20 @@ export function DBTraceWaterfallChartContainer({ dateRange: logTableSource ? dateRange : [dateRange[1], dateRange[0]], // different query to prevent cache traceId, hiddenRowExpression: logWhere ? `NOT (${logWhere})` : undefined, + hiddenRowExpressionLanguage: logFilterLanguage, enabled: logTableSource ? true : false, // disable fire query if logSource is not exist }); const isFetching = traceIsFetching || logIsFetching; - const error = traceError || logError; + // Only a fatal trace failure (bad source / connection) blanks the waterfall. + // A *filter* error on either side is non-fatal: the offending query falls back + // to unfiltered data and the error is surfaced inline next to its input, so a + // bad filter can never hide valid spans or logs. The correlated-log source is + // secondary, so even a fatal log failure only drops logs — never the chart. + const error = traceError; + // Log-side error to surface inline: prefer the filter error (fallback data is + // shown), otherwise the fatal error (no logs could be loaded at all). + const logError = logFilterError ?? logFatalError; const rows: any[] = useMemo(() => { const nextRows: Array<(typeof traceRowsData)[number] & TimestampedRow> = [ @@ -537,6 +723,29 @@ export function DBTraceWaterfallChartContainer({ return nextRows; }, [traceRowsData, logRowsData]); + // Map each distinct span service to a stable color. Sorting the names first + // keeps a service's color stable across renders regardless of row ordering. + const serviceColorMap = useMemo(() => { + const serviceNames = [ + ...new Set( + rows + .filter( + r => + r.ServiceName && + r.type !== SourceKind.Log && + typeof r.ServiceName === 'string', + ) + .map(r => r.ServiceName), + ), + ].sort(); + + const map = new Map(); + serviceNames.forEach((name, i) => { + map.set(name, SERVICE_COLORS[i % SERVICE_COLORS.length]); + }); + return map; + }, [rows]); + const highlightedAttributeValues = useMemo(() => { const visibleTraceRowsData = traceRowsData?.filter( row => !row.__hdx_hidden, @@ -644,7 +853,7 @@ export function DBTraceWaterfallChartContainer({ const [showSpans, setShowSpans] = useState(true); const [showLogs, setShowLogs] = useState(true); - const { nodesMap, flattenedNodes } = useMemo(() => { + const { nodesMap, flattenedNodes, parentIdsByLevel } = useMemo(() => { const rootNodes: Node[] = []; const nodesMap = new Map(); // Maps result.id (or placeholder id) -> Node const spanIdMap = new Map(); // Maps SpanId -> result.id of FIRST node with that SpanId @@ -712,6 +921,20 @@ export function DBTraceWaterfallChartContainer({ } } + // Build a map of level → parent node IDs (nodes that have children) so the + // depth controls can expand/collapse a whole level at a time. + const parentIdsByLevel = new Map>(); + const collectParents = (node: any, level: number) => { + if (node.children?.length > 0 && node.id) { + if (!parentIdsByLevel.has(level)) { + parentIdsByLevel.set(level, new Set()); + } + parentIdsByLevel.get(level)!.add(node.id); + } + node.children?.forEach((child: any) => collectParents(child, level + 1)); + }; + rootNodes.forEach(root => collectParents(root, 0)); + type NodeWithLevel = Node & { level: number }; // flatten the rootnode dag into an array via in-order traversal const traverse = (node: Node, arr: NodeWithLevel[], level = 0) => { @@ -735,40 +958,37 @@ export function DBTraceWaterfallChartContainer({ rootNodes.forEach(rootNode => traverse(rootNode, flattenedNodes)); } - return { nodesMap, flattenedNodes }; + return { nodesMap, flattenedNodes, parentIdsByLevel }; }, [collapsedIds, rows, validSpanIDs]); const toggleCollapse = useCallback( (id: string, event: React.MouseEvent) => { event.stopPropagation(); // prevent collapsing from selecting row + // Alt/Option-click toggles the whole subtree along with the node. + setCollapsedIds(prev => + computeToggleCollapse(prev, id, nodesMap.get(id), event.altKey), + ); + }, + [nodesMap], + ); - setCollapsedIds(prev => { - const newSet = new Set(prev); - const isCollapsed = newSet.has(id); + const expandAll = useCallback(() => { + setCollapsedIds(new Set()); + }, []); - if (isCollapsed) { - newSet.delete(id); - } else { - newSet.add(id); - } + const collapseAll = useCallback(() => { + setCollapsedIds(computeCollapseAll(parentIdsByLevel)); + }, [parentIdsByLevel]); - if (event.altKey) { - const node = nodesMap.get(id); - if (node?.children?.length) { - const descendantIds = getDescendantIds(node); - if (isCollapsed) { - descendantIds.forEach(descId => newSet.delete(descId)); - } else { - descendantIds.forEach(descId => newSet.add(descId)); - } - } - } + const expandOneLevel = useCallback(() => { + setCollapsedIds(prev => computeExpandOneLevel(prev, parentIdsByLevel)); + }, [parentIdsByLevel]); - return newSet; - }); - }, - [nodesMap], - ); + const collapseOneLevel = useCallback(() => { + setCollapsedIds(prev => computeCollapseOneLevel(prev, parentIdsByLevel)); + }, [parentIdsByLevel]); + + const hasCollapsibleNodes = parentIdsByLevel.size > 0; const visibleNodes = useMemo(() => { if (showSpans && showLogs) return flattenedNodes; @@ -811,9 +1031,6 @@ export function DBTraceWaterfallChartContainer({ const minOffset = foundMinOffset === Number.MAX_SAFE_INTEGER ? 0 : foundMinOffset; - const [collapseTooltipShown, { open: setCollapseTooltipShown }] = - useDisclosure(false); - const timelineRows = useMemo( () => visibleNodes.map(result => { @@ -867,13 +1084,20 @@ export function DBTraceWaterfallChartContainer({ const isWarn = result.SeverityText === 'warn'; const isHighlighted = highlightedRowWhere === id; + const barBackgroundColor = + type === SourceKind.Log + ? getChartColorSuccess() + : serviceName + ? (serviceColorMap.get(serviceName) ?? '#6A7077') + : '#6A7077'; + return { id, type, aliasWith, label: (
))} - +
0 ? 1 : 0, + }} + onClick={ + result.children.length > 0 + ? e => { + toggleCollapse(id, e); + } + : undefined } - withArrow > -
0 ? 1 : 0, - }} - onClick={ - result.children.length > 0 - ? e => { - toggleCollapse(id, e); - } - : undefined - } + {collapsedIds.has(id) ? ( + + ) : ( + + )}{' '} +
+ +
+ + {result.children.length > 0 && ( + - {collapsedIds.has(id) ? ( - - ) : ( - - )}{' '} -
- - - {!isFilterActive && ( - - {result.children.length > 0 - ? `(${result.children.length})` - : ''} + ({result.children.length}) )} + {isError && ( + + )} + {isWarn && !isError && ( + + )} + {type === SourceKind.Log ? ( - {serviceName ? `${serviceName} | ` : ''} - {displayText} + {serviceName && <>{serviceName} } + + {displayText} +