feat: allow applying session settings to queries (#1609)

Closes HDX-3154

This PR adds a feature that allows the user to add settings to a source. These settings are then added to the end of every query that is rendered through the `renderChartConfig` function, along with any other chart specific settings. 

See: https://clickhouse.com/docs/sql-reference/statements/select#settings-in-select-query

Most of the work was to pass the `source` or `source.querySettings` value through the code to the `renderChartConfig` calls and to update the related tests. There are also some UI changes in the `SourceForm` components.

`SQLParser.Parser` from the `node-sql-parser` throws an error when it encounters a SETTINGS clause in a sql string, so a function was added to remove that clause from any sql that is passed to the parser. It assumes that the SETTINGS clause will always be at the end of the sql string, it removes any part of the string including and after the SETTINGS clause.


https://github.com/user-attachments/assets/7ac3b852-2c86-4431-88bc-106f982343bb

Author: karl-power

.changeset/hip-icons-build.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+feat: allow applying session settings to queries

packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -473,6 +473,17 @@ Array [
 ]
 `;
 
+exports[`renderChartConfig Query settings handles the the query settings 1`] = `
+Array [
+  Object {
+    "Body": "Oh no! Something went wrong!",
+  },
+  Object {
+    "Body": "This is a test message.",
+  },
+]
+`;
+
 exports[`renderChartConfig aggFn numeric agg functions should handle numeric values as strings 1`] = `
 Array [
   Object {

packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts

@@ -1,13 +1,12 @@
 // TODO: we might want to move this test file to common-utils package
 
-import { ChSql } from '@hyperdx/common-utils/dist/clickhouse';
+import { ChSql, chSql } from '@hyperdx/common-utils/dist/clickhouse';
 import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
 import {
-  AggregateFunctionSchema,
-  DerivedColumn,
   MetricsDataType,
+  QuerySettings,
 } from '@hyperdx/common-utils/dist/types';
 import _ from 'lodash';
 import ms from 'ms';
@@ -36,6 +35,13 @@ const TEST_METRIC_TABLES = {
   'exponential histogram': DEFAULT_METRICS_TABLE.EXPONENTIAL_HISTOGRAM,
 };
 
+const querySettings: QuerySettings = [
+  { setting: 'optimize_read_in_order', value: '0' },
+  { setting: 'cast_keep_nullable', value: '1' },
+  { setting: 'count_distinct_implementation', value: 'uniqCombined64' },
+  { setting: 'async_insert_busy_timeout_min_ms', value: '20000' },
+];
+
 describe('renderChartConfig', () => {
   const server = getServer();
 
@@ -152,6 +158,7 @@ describe('renderChartConfig', () => {
           timestampValueExpression: 'ts',
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -190,6 +197,7 @@ describe('renderChartConfig', () => {
           timestampValueExpression: 'ts',
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -226,16 +234,10 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
-      const resp = await clickhouseClient
-        .query<'JSON'>({
-          query: query.sql,
-          query_params: query.params,
-          format: 'JSON',
-        })
-        .then(res => res.json() as any);
-      expect(resp.data).toMatchSnapshot();
+      expect(await queryData(query)).toMatchSnapshot();
     });
 
     it('simple select + group by query logs', async () => {
@@ -272,6 +274,7 @@ describe('renderChartConfig', () => {
           groupBy: 'ServiceName',
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -342,6 +345,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(avgQuery)).toMatchSnapshot();
       const maxQuery = await renderChartConfig(
@@ -363,6 +367,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(maxQuery)).toMatchSnapshot();
       const sumQuery = await renderChartConfig(
@@ -384,6 +389,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(sumQuery)).toMatchSnapshot();
     });
@@ -409,6 +415,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -434,6 +441,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -459,6 +467,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -485,6 +494,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -657,6 +667,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -681,6 +692,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -705,6 +717,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -749,6 +762,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(minQuery)).toMatchSnapshot('minSum');
 
@@ -771,6 +785,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(maxQuery)).toMatchSnapshot('maxSum');
     });
@@ -1151,6 +1166,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1200,6 +1216,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1250,6 +1267,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1294,6 +1312,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1320,6 +1339,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1347,6 +1367,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1374,6 +1395,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1444,6 +1466,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
       const res = await queryData(query);
@@ -1480,6 +1503,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
       const res = await queryData(query);
@@ -1509,6 +1533,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
       const res = await queryData(query);
@@ -1520,4 +1545,43 @@ describe('renderChartConfig', () => {
       expect(query.sql).not.toMatch(/MetricName IN /);
     });
   });
+
+  describe('Query settings', () => {
+    it('handles the the query settings', async () => {
+      const now = new Date('2023-11-16T22:12:00.000Z');
+
+      await bulkInsertLogs([
+        {
+          ServiceName: 'api',
+          Timestamp: now,
+          SeverityText: 'error',
+          Body: 'Oh no! Something went wrong!',
+        },
+        {
+          ServiceName: 'api',
+          Timestamp: now,
+          SeverityText: 'info',
+          Body: 'This is a test message.',
+        },
+      ]);
+
+      const query = await renderChartConfig(
+        {
+          select: [{ valueExpression: 'Body' }],
+          from: logSource.from,
+          where: '',
+          timestampValueExpression: 'Timestamp',
+          connection: connection.id,
+          settings: chSql`max_result_rows = 1`,
+        },
+        metadata,
+        [...querySettings, { setting: 'result_overflow_mode', value: 'break' }],
+      );
+
+      const res = await queryData(query);
+      // ensures `result_overflow_mode = break` is applied, otherwise query would error.
+      expect(res).toHaveLength(2);
+      expect(res).toMatchSnapshot();
+    });
+  });
 });

packages/api/src/models/source.ts

@@ -86,6 +86,17 @@ export const Source = mongoose.model<ISource>(
         },
         default: undefined,
       },
+
+      querySettings: {
+        type: [
+          {
+            setting: { type: String, required: true },
+            value: { type: String, required: true },
+          },
+        ],
+        default: undefined,
+        maxlength: 10,
+      },
     },
     {
       toJSON: { virtuals: true },

packages/api/src/routers/api/ai.ts

@@ -274,6 +274,7 @@ router.post(
       const keyValues = await metadata.getKeyValues({
         chartConfig: cc,
         keys: keysToFetch.map(f => f.key),
+        source,
       });
 
       const anthropic = createAnthropic({

packages/api/src/routers/external-api/v2/charts.ts

@@ -588,6 +588,7 @@ router.post(
             const result = await clickhouseClient.queryChartConfig({
               config: chartConfig,
               metadata,
+              querySettings: source.querySettings,
             });
 
             return {

packages/api/src/tasks/checkAlerts/index.ts

@@ -448,6 +448,7 @@ export const processAlert = async (
     const checksData = await clickhouseClient.queryChartConfig({
       config: optimizedChartConfig,
       metadata,
+      querySettings: source.querySettings,
     });
 
     logger.info(

packages/api/src/tasks/checkAlerts/template.ts

@@ -602,7 +602,11 @@ ${targetTemplate}`;
 
     let truncatedResults = '';
     try {
-      const query = await renderChartConfig(chartConfig, metadata);
+      const query = await renderChartConfig(
+        chartConfig,
+        metadata,
+        source.querySettings,
+      );
       const raw = await clickhouseClient
         .query<'CSV'>({
           query: query.sql,

packages/api/src/tasks/pingPongTask.ts

@@ -4,7 +4,6 @@ import logger from '@/utils/logger';
 export default class PingPongTask implements HdxTask<PingTaskArgs> {
   constructor(private args: PingTaskArgs) {}
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async execute(): Promise<void> {
     logger.info(`
                    O .

packages/app/src/TeamPage.tsx

@@ -1,9 +1,9 @@
 import { useCallback, useState } from 'react';
 import Head from 'next/head';
 import { CopyToClipboard } from 'react-copy-to-clipboard';
-import { SubmitHandler, useForm, useWatch } from 'react-hook-form';
+import { SubmitHandler, useForm } from 'react-hook-form';
 import { DEFAULT_METADATA_MAX_ROWS_TO_READ } from '@hyperdx/common-utils/dist/core/metadata';
-import { TeamClickHouseSettings } from '@hyperdx/common-utils/dist/types';
+import { type TeamClickHouseSettings } from '@hyperdx/common-utils/dist/types';
 import {
   Box,
   Button,