From 2f191581e15aa66cdde9147aafafad6dc66fce67 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 15:45:58 -0400 Subject: [PATCH 01/22] bugfix: replace MD5 with SHA256 for FIPS compatibility This patch is to resolve Issue #3390 - when running HyperDX against ClickHouse in FIPS mode the MD5 function isn't available, so viewing some logs fails, because MD5 is not FIPS-compliant. This patch was generated using Cursor, running Composer 2.5 Fast. All relevant tests still pass. Co-authored-by: Cursor --- packages/app/src/components/DBRowTable.tsx | 2 +- .../src/hooks/__tests__/useRowWhere.test.tsx | 26 +++++++++---------- packages/app/src/hooks/useRowWhere.tsx | 12 ++++----- packages/cli/src/shared/useRowWhere.ts | 10 +++---- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 69e9a15d80..c682eb823e 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1849,7 +1849,7 @@ function DBSqlRowTableComponent({ if (columnMap.get(c)?._type === JSDataType.JSON) { // special rule for json // for json {SomePath: /c}, CH will return {SomePath: \/c} - // add this to make sure md5 get correct result + // add this to make sure SHA256 get correct result newRow[c] = JSON.stringify(row[c]).replace(/\//g, '\\/'); } else { newRow[c] = JSON.stringify(row[c]); diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 90b1aed0f1..4e44f0c646 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -1,4 +1,4 @@ -import MD5 from 'crypto-js/md5'; +import SHA256 from 'crypto-js/sha256'; import { ColumnMetaType, JSDataType, @@ -7,8 +7,8 @@ import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; -// Mock crypto-js/md5 -jest.mock('crypto-js/md5'); +// Mock crypto-js/sha256 +jest.mock('crypto-js/sha256'); // Mock convertCHDataTypeToJSType jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ @@ -31,8 +31,8 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ describe('processRowToWhereClause', () => { beforeEach(() => { jest.clearAllMocks(); - (MD5 as jest.Mock).mockImplementation((value: string) => ({ - toString: () => `md5_${value}`, + (SHA256 as jest.Mock).mockImplementation((value: string) => ({ + toString: () => `sha256_${value}`, })); }); @@ -116,7 +116,7 @@ describe('processRowToWhereClause', () => { ); }); - it('should handle JSON columns with MD5', () => { + it('should handle JSON columns with SHA256', () => { const columnMap = new Map([ [ 'data', @@ -133,9 +133,9 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - 'lower(hex(MD5(toString(data))))=\'md5_{\\"key\\": \\"value\\"}\'', + 'lower(hex(SHA256(toString(data))))=\'sha256_{\\"key\\": \\"value\\"}\'', ); - expect(MD5).toHaveBeenCalledWith('{"key": "value"}'); + expect(SHA256).toHaveBeenCalledWith('{"key": "value"}'); }); it('should handle Dynamic columns with null value', () => { @@ -266,7 +266,7 @@ describe('processRowToWhereClause', () => { ); }); - it('should handle long strings with MD5', () => { + it('should handle long strings with SHA256', () => { const columnMap = new Map([ [ 'description', @@ -284,9 +284,9 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - `lower(hex(MD5(leftUTF8(description, 1000))))='md5_${'a'.repeat(600)}'`, + `lower(hex(SHA256(leftUTF8(description, 1000))))='sha256_${'a'.repeat(600)}'`, ); - expect(MD5).toHaveBeenCalledWith('a'.repeat(600)); + expect(SHA256).toHaveBeenCalledWith('a'.repeat(600)); }); it('should handle multiple columns with AND', () => { @@ -448,8 +448,8 @@ describe('processRowToWhereClause', () => { describe('useRowWhere', () => { beforeEach(() => { jest.clearAllMocks(); - (MD5 as jest.Mock).mockImplementation((value: string) => ({ - toString: () => `md5_${value}`, + (SHA256 as jest.Mock).mockImplementation((value: string) => ({ + toString: () => `sha256_${value}`, })); }); diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index ad0f5ede8e..e79ecfb7d0 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -1,5 +1,5 @@ import { useCallback, useMemo } from 'react'; -import MD5 from 'crypto-js/md5'; +import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import z from 'zod'; import { @@ -84,9 +84,9 @@ export function processRowToWhereClause( ]); case JSDataType.JSON: // Handle case for whole json object, ex: json - return SqlString.format(`lower(hex(MD5(toString(?))))=?`, [ + return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - MD5(value).toString(), + SHA256(value).toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -113,13 +113,13 @@ export function processRowToWhereClause( // Handle the case when string is too long if (value.length > MAX_STRING_LENGTH) { return SqlString.format( - // We need to slice since md5 can be slow on big payloads + // We need to slice since hashing can be slow on big payloads // which will block the main thread on search table render // UTF8 since js only slices in utf8 points, not bytes - `lower(hex(MD5(leftUTF8(?, 1000))))=?`, + `lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - MD5(value.substring(0, 1000)).toString(), + SHA256(value.substring(0, 1000)).toString(), ], ); } diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 98f2fcdddb..457cb0ec9b 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -11,7 +11,7 @@ * to common-utils later. */ -import MD5 from 'crypto-js/md5'; +import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import { ColumnMetaType, @@ -89,9 +89,9 @@ function processRowToWhereClause( ]); case JSDataType.JSON: // Handle case for whole json object, ex: json - return SqlString.format(`lower(hex(MD5(toString(?))))=?`, [ + return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - MD5(strValue ?? '').toString(), + SHA256(strValue ?? '').toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -114,9 +114,9 @@ function processRowToWhereClause( } // Handle the case when string is too long if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { - return SqlString.format(`lower(hex(MD5(leftUTF8(?, 1000))))=?`, [ + return SqlString.format(`lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - MD5((strValue ?? '').substring(0, 1000)).toString(), + SHA256((strValue ?? '').substring(0, 1000)).toString(), ]); } return SqlString.format(`?=?`, [ From a1dfe9ff96830ebd4eab6ab0c1b00943c068b77b Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 16:28:00 -0400 Subject: [PATCH 02/22] fixup tests to use real-world data --- .../src/hooks/__tests__/useRowWhere.test.tsx | 60 ++++++++++++------- packages/app/src/hooks/useRowWhere.tsx | 7 ++- packages/app/src/utils/clickHouseSha256.ts | 9 +++ packages/cli/src/shared/useRowWhere.ts | 11 +++- 4 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 packages/app/src/utils/clickHouseSha256.ts diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 4e44f0c646..7235eed19a 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -1,4 +1,3 @@ -import SHA256 from 'crypto-js/sha256'; import { ColumnMetaType, JSDataType, @@ -6,9 +5,14 @@ import { import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; +import { clickHouseSha256HexLower } from '@/utils/clickHouseSha256'; -// Mock crypto-js/sha256 -jest.mock('crypto-js/sha256'); +/** From ClickHouse: lower(hex(SHA256())) on telemetry.otel_logs. */ +const OTEL_LOG_JSON_FOR_SHA256 = + '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}'; + +const OTEL_LOG_JSON_SHA256_HEX = + '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8'; // Mock convertCHDataTypeToJSType jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ @@ -28,14 +32,15 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ }), })); -describe('processRowToWhereClause', () => { - beforeEach(() => { - jest.clearAllMocks(); - (SHA256 as jest.Mock).mockImplementation((value: string) => ({ - toString: () => `sha256_${value}`, - })); +describe('clickHouseSha256HexLower', () => { + it('matches ClickHouse lower(hex(SHA256(...))) for a real otel log JSON string', () => { + expect(clickHouseSha256HexLower(OTEL_LOG_JSON_FOR_SHA256)).toBe( + OTEL_LOG_JSON_SHA256_HEX, + ); }); +}); +describe('processRowToWhereClause', () => { it('should handle string columns', () => { const columnMap = new Map([ [ @@ -129,13 +134,34 @@ describe('processRowToWhereClause', () => { ], ]); - const row = { data: '{"key": "value"}' }; + const jsonValue = '{"key": "value"}'; + const row = { data: jsonValue }; + const result = processRowToWhereClause(row, columnMap); + + expect(result).toBe( + `lower(hex(SHA256(toString(data))))='${clickHouseSha256HexLower(jsonValue)}'`, + ); + }); + + it('should handle JSON columns with SHA256 for otel log payload', () => { + const columnMap = new Map([ + [ + 'Body', + { + name: 'Body', + type: 'JSON', + valueExpr: 'Body', + jsType: JSDataType.JSON, + }, + ], + ]); + + const row = { Body: OTEL_LOG_JSON_FOR_SHA256 }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - 'lower(hex(SHA256(toString(data))))=\'sha256_{\\"key\\": \\"value\\"}\'', + `lower(hex(SHA256(toString(Body))))='${OTEL_LOG_JSON_SHA256_HEX}'`, ); - expect(SHA256).toHaveBeenCalledWith('{"key": "value"}'); }); it('should handle Dynamic columns with null value', () => { @@ -284,9 +310,8 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='sha256_${'a'.repeat(600)}'`, + `lower(hex(SHA256(leftUTF8(description, 1000))))='${clickHouseSha256HexLower(longString)}'`, ); - expect(SHA256).toHaveBeenCalledWith('a'.repeat(600)); }); it('should handle multiple columns with AND', () => { @@ -446,13 +471,6 @@ describe('processRowToWhereClause', () => { }); describe('useRowWhere', () => { - beforeEach(() => { - jest.clearAllMocks(); - (SHA256 as jest.Mock).mockImplementation((value: string) => ({ - toString: () => `sha256_${value}`, - })); - }); - it('should return a function that processes rows', () => { const meta: ColumnMetaType[] = [ { name: 'id', type: 'String' }, diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index e79ecfb7d0..22ca2c907f 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -1,5 +1,4 @@ import { useCallback, useMemo } from 'react'; -import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import z from 'zod'; import { @@ -11,6 +10,8 @@ import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/m import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; +import { clickHouseSha256HexLower } from '@/utils/clickHouseSha256'; + const MAX_STRING_LENGTH = 512; // Type for WITH clause entries, derived from ChartConfig's with property @@ -86,7 +87,7 @@ export function processRowToWhereClause( // Handle case for whole json object, ex: json return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - SHA256(value).toString(), + clickHouseSha256HexLower(value), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -119,7 +120,7 @@ export function processRowToWhereClause( `lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - SHA256(value.substring(0, 1000)).toString(), + clickHouseSha256HexLower(value.substring(0, 1000)), ], ); } diff --git a/packages/app/src/utils/clickHouseSha256.ts b/packages/app/src/utils/clickHouseSha256.ts new file mode 100644 index 0000000000..3c560c13ae --- /dev/null +++ b/packages/app/src/utils/clickHouseSha256.ts @@ -0,0 +1,9 @@ +import SHA256 from 'crypto-js/sha256'; + +/** + * Lowercase hex SHA-256 of a UTF-8 string, matching ClickHouse + * `lower(hex(SHA256(?)))` for the same input. + */ +export function clickHouseSha256HexLower(value: string): string { + return SHA256(value).toString(); +} diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 457cb0ec9b..37d3ce8ed9 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -11,7 +11,7 @@ * to common-utils later. */ -import SHA256 from 'crypto-js/sha256'; +import { createHash } from 'node:crypto'; import SqlString from 'sqlstring'; import { ColumnMetaType, @@ -24,6 +24,11 @@ import type { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; +/** Matches ClickHouse `lower(hex(SHA256(?)))` for UTF-8 strings. */ +function clickHouseSha256HexLower(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + // Type for WITH clause entries, derived from ChartConfig's with property type WithClause = NonNullable[number]; @@ -91,7 +96,7 @@ function processRowToWhereClause( // Handle case for whole json object, ex: json return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - SHA256(strValue ?? '').toString(), + clickHouseSha256HexLower(strValue ?? ''), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -116,7 +121,7 @@ function processRowToWhereClause( if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { return SqlString.format(`lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - SHA256((strValue ?? '').substring(0, 1000)).toString(), + clickHouseSha256HexLower((strValue ?? '').substring(0, 1000)), ]); } return SqlString.format(`?=?`, [ From c1e0753b34054118ee6e0edb1576e8819131656e Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 16:28:19 -0400 Subject: [PATCH 03/22] add changeset --- .changeset/fips-row-lookup-sha256.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/fips-row-lookup-sha256.md diff --git a/.changeset/fips-row-lookup-sha256.md b/.changeset/fips-row-lookup-sha256.md new file mode 100644 index 0000000000..26efa27eb8 --- /dev/null +++ b/.changeset/fips-row-lookup-sha256.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/app': patch +'@hyperdx/cli': patch +--- + +fix: use SHA256 instead of MD5 for JSON row lookup WHERE clauses so row +details and drill-down work on FIPS-enabled ClickHouse deployments From 732cdf327cf99ea43628e7f550b6749146aedf2f Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 17:09:10 -0400 Subject: [PATCH 04/22] manually fix up tests --- packages/app/src/hooks/__tests__/useRowWhere.test.tsx | 11 +++++++---- packages/app/src/hooks/useRowWhere.tsx | 6 +++--- packages/app/src/utils/clickHouseSha256.ts | 9 --------- 3 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 packages/app/src/utils/clickHouseSha256.ts diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 7235eed19a..eccc7ce0c9 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -5,7 +5,7 @@ import { import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; -import { clickHouseSha256HexLower } from '@/utils/clickHouseSha256'; +import SHA256 from 'crypto-js/sha256'; /** From ClickHouse: lower(hex(SHA256())) on telemetry.otel_logs. */ const OTEL_LOG_JSON_FOR_SHA256 = @@ -34,7 +34,7 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ describe('clickHouseSha256HexLower', () => { it('matches ClickHouse lower(hex(SHA256(...))) for a real otel log JSON string', () => { - expect(clickHouseSha256HexLower(OTEL_LOG_JSON_FOR_SHA256)).toBe( + expect(SHA256(OTEL_LOG_JSON_FOR_SHA256).toString()).toBe( OTEL_LOG_JSON_SHA256_HEX, ); }); @@ -139,7 +139,7 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - `lower(hex(SHA256(toString(data))))='${clickHouseSha256HexLower(jsonValue)}'`, + `lower(hex(SHA256(toString(data))))='${SHA256(jsonValue).toString()}'`, ); }); @@ -309,8 +309,11 @@ describe('processRowToWhereClause', () => { const row = { description: longString }; const result = processRowToWhereClause(row, columnMap); + // in ClickHouse: SELECT lower(hex(SHA256(repeat('a', 600)))); + const expected = 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d' + expect(result).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${clickHouseSha256HexLower(longString)}'`, + `lower(hex(SHA256(leftUTF8(description, 1000))))='${expected}'`, ); }); diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 22ca2c907f..ee90ca3fd0 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -10,7 +10,7 @@ import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/m import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; -import { clickHouseSha256HexLower } from '@/utils/clickHouseSha256'; +import SHA256 from 'crypto-js/sha256'; const MAX_STRING_LENGTH = 512; @@ -87,7 +87,7 @@ export function processRowToWhereClause( // Handle case for whole json object, ex: json return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - clickHouseSha256HexLower(value), + SHA256(value).toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -120,7 +120,7 @@ export function processRowToWhereClause( `lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - clickHouseSha256HexLower(value.substring(0, 1000)), + SHA256(value.substring(0, 1000)).toString(), ], ); } diff --git a/packages/app/src/utils/clickHouseSha256.ts b/packages/app/src/utils/clickHouseSha256.ts deleted file mode 100644 index 3c560c13ae..0000000000 --- a/packages/app/src/utils/clickHouseSha256.ts +++ /dev/null @@ -1,9 +0,0 @@ -import SHA256 from 'crypto-js/sha256'; - -/** - * Lowercase hex SHA-256 of a UTF-8 string, matching ClickHouse - * `lower(hex(SHA256(?)))` for the same input. - */ -export function clickHouseSha256HexLower(value: string): string { - return SHA256(value).toString(); -} From 3a08e37aee31dd2a115fef10a61a32648d4727be Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 17:14:43 -0400 Subject: [PATCH 05/22] remove unnecessary function and imports --- packages/cli/src/shared/useRowWhere.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 37d3ce8ed9..457cb0ec9b 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -11,7 +11,7 @@ * to common-utils later. */ -import { createHash } from 'node:crypto'; +import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import { ColumnMetaType, @@ -24,11 +24,6 @@ import type { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; -/** Matches ClickHouse `lower(hex(SHA256(?)))` for UTF-8 strings. */ -function clickHouseSha256HexLower(value: string): string { - return createHash('sha256').update(value, 'utf8').digest('hex'); -} - // Type for WITH clause entries, derived from ChartConfig's with property type WithClause = NonNullable[number]; @@ -96,7 +91,7 @@ function processRowToWhereClause( // Handle case for whole json object, ex: json return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - clickHouseSha256HexLower(strValue ?? ''), + SHA256(strValue ?? '').toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -121,7 +116,7 @@ function processRowToWhereClause( if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { return SqlString.format(`lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - clickHouseSha256HexLower((strValue ?? '').substring(0, 1000)), + SHA256((strValue ?? '').substring(0, 1000)).toString(), ]); } return SqlString.format(`?=?`, [ From 89f0702bab78c8958eea736eed45154a5c5e80da Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 17:37:55 -0400 Subject: [PATCH 06/22] ran yarn lint:fix --- packages/app/src/hooks/__tests__/useRowWhere.test.tsx | 5 +++-- packages/app/src/hooks/useRowWhere.tsx | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index eccc7ce0c9..04e816f6ef 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -1,3 +1,4 @@ +import SHA256 from 'crypto-js/sha256'; import { ColumnMetaType, JSDataType, @@ -5,7 +6,6 @@ import { import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; -import SHA256 from 'crypto-js/sha256'; /** From ClickHouse: lower(hex(SHA256())) on telemetry.otel_logs. */ const OTEL_LOG_JSON_FOR_SHA256 = @@ -310,7 +310,8 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); // in ClickHouse: SELECT lower(hex(SHA256(repeat('a', 600)))); - const expected = 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d' + const expected = + 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d'; expect(result).toBe( `lower(hex(SHA256(leftUTF8(description, 1000))))='${expected}'`, diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index ee90ca3fd0..e79ecfb7d0 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -1,4 +1,5 @@ import { useCallback, useMemo } from 'react'; +import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import z from 'zod'; import { @@ -10,8 +11,6 @@ import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/m import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; -import SHA256 from 'crypto-js/sha256'; - const MAX_STRING_LENGTH = 512; // Type for WITH clause entries, derived from ChartConfig's with property From b3b864e64fb82d4a9930b9255d73a30eb404e9a5 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 17:59:03 -0400 Subject: [PATCH 07/22] improve another test --- packages/app/src/hooks/__tests__/useRowWhere.test.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 04e816f6ef..d9c88a0354 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -138,9 +138,11 @@ describe('processRowToWhereClause', () => { const row = { data: jsonValue }; const result = processRowToWhereClause(row, columnMap); - expect(result).toBe( - `lower(hex(SHA256(toString(data))))='${SHA256(jsonValue).toString()}'`, - ); + // in ClickHouse: SELECT lower(hex(SHA256(toString('{"key": "value"}'))); + const expected = + '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5'; + + expect(result).toBe(`lower(hex(SHA256(toString(data))))='${expected}'`); }); it('should handle JSON columns with SHA256 for otel log payload', () => { From 339899c2c02bc47626d6a091292cf128d9012fe1 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 18:09:30 -0400 Subject: [PATCH 08/22] add missing tests --- packages/cli/src/__tests__/rowWhere.test.ts | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 0f427d4f46..36f70ed66f 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -4,6 +4,17 @@ import type { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; import { buildColumnMap, getRowWhere } from '@/shared/useRowWhere'; +/** From ClickHouse: lower(hex(SHA256())) on telemetry.otel_logs. */ +const OTEL_LOG_JSON_FOR_SHA256 = + '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}'; + +const OTEL_LOG_JSON_SHA256_HEX = + '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8'; + +/** From ClickHouse: lower(hex(SHA256(repeat('a', 600)))). */ +const LONG_STRING_SHA256_HEX = + 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d'; + describe('buildColumnMap', () => { it('quotes a column name that is not a bare identifier', () => { const meta: ColumnMetaType[] = [{ name: 'x-host-header', type: 'String' }]; @@ -66,4 +77,32 @@ describe('buildColumnMap', () => { expect(where).toBe("ResourceAttributes['service.name']='api'"); }); + + it('hashes JSON columns with SHA256 (otel log payload)', () => { + const meta: ColumnMetaType[] = [{ name: 'Body', type: 'JSON' }]; + + const where = getRowWhere( + { Body: OTEL_LOG_JSON_FOR_SHA256 }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe( + `lower(hex(SHA256(toString(Body))))='${OTEL_LOG_JSON_SHA256_HEX}'`, + ); + }); + + it('hashes strings longer than MAX_STRING_LENGTH with SHA256', () => { + const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; + + const where = getRowWhere( + { description: 'a'.repeat(600) }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe( + `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONG_STRING_SHA256_HEX}'`, + ); + }); }); From 655d3a8f552d0cb54f25be5afdb5b7dbd8a74c8b Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 18:19:27 -0400 Subject: [PATCH 09/22] add more tests --- .../src/hooks/__tests__/useRowWhere.test.tsx | 25 +++++++++++++++++++ packages/cli/src/__tests__/rowWhere.test.ts | 18 +++++++++++++ 2 files changed, 43 insertions(+) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index d9c88a0354..87b1da9846 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -14,6 +14,10 @@ const OTEL_LOG_JSON_FOR_SHA256 = const OTEL_LOG_JSON_SHA256_HEX = '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8'; +/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000)))). */ +const LONGER_STRING_SHA256_HEX = + '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3'; + // Mock convertCHDataTypeToJSType jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ ...jest.requireActual('@hyperdx/common-utils/dist/clickhouse'), @@ -320,6 +324,27 @@ describe('processRowToWhereClause', () => { ); }); + it('should hash only the first 1000 UTF-8 code units when the string exceeds that limit', () => { + const columnMap = new Map([ + [ + 'description', + { + name: 'description', + type: 'String', + valueExpr: 'description', + jsType: JSDataType.String, + }, + ], + ]); + + const row = { description: 'a'.repeat(2000) }; + const result = processRowToWhereClause(row, columnMap); + + expect(result).toBe( + `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONGER_STRING_SHA256_HEX}'`, + ); + }); + it('should handle multiple columns with AND', () => { const columnMap = new Map([ [ diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 36f70ed66f..2102142351 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -15,6 +15,10 @@ const OTEL_LOG_JSON_SHA256_HEX = const LONG_STRING_SHA256_HEX = 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d'; +/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000)))). */ +const LONGER_STRING_SHA256_HEX = + '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3'; + describe('buildColumnMap', () => { it('quotes a column name that is not a bare identifier', () => { const meta: ColumnMetaType[] = [{ name: 'x-host-header', type: 'String' }]; @@ -105,4 +109,18 @@ describe('buildColumnMap', () => { `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONG_STRING_SHA256_HEX}'`, ); }); + + it('hashes only the first 1000 UTF-8 code units when the string exceeds that limit', () => { + const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; + + const where = getRowWhere( + { description: 'a'.repeat(2000) }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe( + `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONGER_STRING_SHA256_HEX}'`, + ); + }); }); From e3c7b2142d5506f80b8731d82c44d8784922e9ab Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 18:59:39 -0400 Subject: [PATCH 10/22] fix: slice long strings by Unicode code point to match ClickHouse leftUTF8 ClickHouse's leftUTF8(?, 1000) truncates by Unicode code points, but the client-side SHA256 was using JS substring, which counts UTF-16 code units. For strings containing non-BMP characters (e.g. emoji), this produced a different hash than the database would compute, breaking row lookups. Update both app and CLI row-where helpers to slice by code point with Array.from(value).slice(0, 1000).join(''), and add tests for emoji strings. --- .../src/hooks/__tests__/useRowWhere.test.tsx | 27 ++++++++++++++++++- packages/app/src/hooks/useRowWhere.tsx | 6 ++--- packages/cli/src/__tests__/rowWhere.test.ts | 20 +++++++++++++- packages/cli/src/shared/useRowWhere.ts | 6 ++++- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 87b1da9846..6398961d5f 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -18,6 +18,10 @@ const OTEL_LOG_JSON_SHA256_HEX = const LONGER_STRING_SHA256_HEX = '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3'; +/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000)))). */ +const EMOJI_STRING_SHA256_HEX = + '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43'; + // Mock convertCHDataTypeToJSType jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ ...jest.requireActual('@hyperdx/common-utils/dist/clickhouse'), @@ -324,7 +328,7 @@ describe('processRowToWhereClause', () => { ); }); - it('should hash only the first 1000 UTF-8 code units when the string exceeds that limit', () => { + it('should hash only the first 1000 Unicode code points when the string exceeds that limit', () => { const columnMap = new Map([ [ 'description', @@ -345,6 +349,27 @@ describe('processRowToWhereClause', () => { ); }); + it('should match ClickHouse leftUTF8 code-point truncation for non-BMP strings', () => { + const columnMap = new Map([ + [ + 'description', + { + name: 'description', + type: 'String', + valueExpr: 'description', + jsType: JSDataType.String, + }, + ], + ]); + + const row = { description: '😀'.repeat(600) }; + const result = processRowToWhereClause(row, columnMap); + + expect(result).toBe( + `lower(hex(SHA256(leftUTF8(description, 1000))))='${EMOJI_STRING_SHA256_HEX}'`, + ); + }); + it('should handle multiple columns with AND', () => { const columnMap = new Map([ [ diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index e79ecfb7d0..c616dbe1e9 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -113,13 +113,11 @@ export function processRowToWhereClause( // Handle the case when string is too long if (value.length > MAX_STRING_LENGTH) { return SqlString.format( - // We need to slice since hashing can be slow on big payloads - // which will block the main thread on search table render - // UTF8 since js only slices in utf8 points, not bytes + // Slice by code point to match ClickHouse leftUTF8(?, 1000). `lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - SHA256(value.substring(0, 1000)).toString(), + SHA256(Array.from(value).slice(0, 1000).join('')).toString(), ], ); } diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 2102142351..16c770d9d4 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -19,6 +19,10 @@ const LONG_STRING_SHA256_HEX = const LONGER_STRING_SHA256_HEX = '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3'; +/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000)))). */ +const EMOJI_STRING_SHA256_HEX = + '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43'; + describe('buildColumnMap', () => { it('quotes a column name that is not a bare identifier', () => { const meta: ColumnMetaType[] = [{ name: 'x-host-header', type: 'String' }]; @@ -110,7 +114,7 @@ describe('buildColumnMap', () => { ); }); - it('hashes only the first 1000 UTF-8 code units when the string exceeds that limit', () => { + it('hashes only the first 1000 Unicode code points when the string exceeds that limit', () => { const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; const where = getRowWhere( @@ -123,4 +127,18 @@ describe('buildColumnMap', () => { `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONGER_STRING_SHA256_HEX}'`, ); }); + + it('matches ClickHouse leftUTF8 code-point truncation for non-BMP strings', () => { + const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; + + const where = getRowWhere( + { description: '😀'.repeat(600) }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe( + `lower(hex(SHA256(leftUTF8(description, 1000))))='${EMOJI_STRING_SHA256_HEX}'`, + ); + }); }); diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 457cb0ec9b..ddc9cbd94e 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -116,7 +116,11 @@ function processRowToWhereClause( if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { return SqlString.format(`lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - SHA256((strValue ?? '').substring(0, 1000)).toString(), + SHA256( + Array.from(strValue ?? '') + .slice(0, 1000) + .join(''), + ).toString(), ]); } return SqlString.format(`?=?`, [ From f65aef8a2a6b5609d344995d05f836f80dcd1bae Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 19:10:16 -0400 Subject: [PATCH 11/22] make slicing a unicode string more efficient --- packages/app/src/hooks/useRowWhere.tsx | 18 ++++++++++++++++-- packages/cli/src/shared/useRowWhere.ts | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index c616dbe1e9..fc0eebe2ad 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -12,6 +12,18 @@ import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; +const LEFT_UTF8_HASH_CODE_POINTS = 1000; + +/** First `limit` Unicode code points, matching ClickHouse leftUTF8(s, limit). */ +function leftUtf8Prefix(value: string, limit: number): string { + let prefix = ''; + let n = 0; + for (const ch of value) { + if (n++ === limit) break; + prefix += ch; + } + return prefix; +} // Type for WITH clause entries, derived from ChartConfig's with property export type WithClause = z.infer; @@ -113,11 +125,13 @@ export function processRowToWhereClause( // Handle the case when string is too long if (value.length > MAX_STRING_LENGTH) { return SqlString.format( - // Slice by code point to match ClickHouse leftUTF8(?, 1000). + // Hash only a prefix so huge payloads do not block the main thread. `lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), - SHA256(Array.from(value).slice(0, 1000).join('')).toString(), + SHA256( + leftUtf8Prefix(value, LEFT_UTF8_HASH_CODE_POINTS), + ).toString(), ], ); } diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index ddc9cbd94e..e05dee567f 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -23,6 +23,18 @@ import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import type { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; +const LEFT_UTF8_HASH_CODE_POINTS = 1000; + +/** First `limit` Unicode code points, matching ClickHouse leftUTF8(s, limit). */ +function leftUtf8Prefix(value: string, limit: number): string { + let prefix = ''; + let n = 0; + for (const ch of value) { + if (n++ === limit) break; + prefix += ch; + } + return prefix; +} // Type for WITH clause entries, derived from ChartConfig's with property type WithClause = NonNullable[number]; @@ -117,9 +129,7 @@ function processRowToWhereClause( return SqlString.format(`lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ SqlString.raw(valueExpr), SHA256( - Array.from(strValue ?? '') - .slice(0, 1000) - .join(''), + leftUtf8Prefix(strValue ?? '', LEFT_UTF8_HASH_CODE_POINTS), ).toString(), ]); } From 5042f401c893d30958240d446a9c1932a1ff8557 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 21:08:53 -0400 Subject: [PATCH 12/22] fix: use LEFT_UTF8_HASH_CODE_POINTS constant in SQL query instead of hardcoded 1000 --- packages/app/src/hooks/useRowWhere.tsx | 2 +- packages/cli/src/shared/useRowWhere.ts | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index fc0eebe2ad..7bc64ce04d 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -126,7 +126,7 @@ export function processRowToWhereClause( if (value.length > MAX_STRING_LENGTH) { return SqlString.format( // Hash only a prefix so huge payloads do not block the main thread. - `lower(hex(SHA256(leftUTF8(?, 1000))))=?`, + `lower(hex(SHA256(leftUTF8(?, ${LEFT_UTF8_HASH_CODE_POINTS}))))=?`, [ SqlString.raw(valueExpr), SHA256( diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index e05dee567f..11b5215448 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -126,12 +126,15 @@ function processRowToWhereClause( } // Handle the case when string is too long if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { - return SqlString.format(`lower(hex(SHA256(leftUTF8(?, 1000))))=?`, [ - SqlString.raw(valueExpr), - SHA256( - leftUtf8Prefix(strValue ?? '', LEFT_UTF8_HASH_CODE_POINTS), - ).toString(), - ]); + return SqlString.format( + `lower(hex(SHA256(leftUTF8(?, ${LEFT_UTF8_HASH_CODE_POINTS}))))=?`, + [ + SqlString.raw(valueExpr), + SHA256( + leftUtf8Prefix(strValue ?? '', LEFT_UTF8_HASH_CODE_POINTS), + ).toString(), + ], + ); } return SqlString.format(`?=?`, [ SqlString.raw(valueExpr), // don't escape expressions From b50efb0d2c0b2bf407f207d5d3378c8e4239278e Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 21:17:48 -0400 Subject: [PATCH 13/22] fix nits 1 --- packages/app/src/hooks/__tests__/useRowWhere.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 6398961d5f..aa90343791 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -7,7 +7,7 @@ import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; -/** From ClickHouse: lower(hex(SHA256())) on telemetry.otel_logs. */ +/** From ClickHouse: lower(hex(SHA256())). */ const OTEL_LOG_JSON_FOR_SHA256 = '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}'; @@ -40,7 +40,7 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ }), })); -describe('clickHouseSha256HexLower', () => { +describe('SHA256 matches ClickHouse lower(hex(SHA256(...)))', () => { it('matches ClickHouse lower(hex(SHA256(...))) for a real otel log JSON string', () => { expect(SHA256(OTEL_LOG_JSON_FOR_SHA256).toString()).toBe( OTEL_LOG_JSON_SHA256_HEX, From ad6909f3d3d2f5f8e0c17b586ccbbd727fca8ec2 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 21:33:47 -0400 Subject: [PATCH 14/22] improve comment --- packages/app/src/components/DBRowTable.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index c682eb823e..587b796c47 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1849,7 +1849,8 @@ function DBSqlRowTableComponent({ if (columnMap.get(c)?._type === JSDataType.JSON) { // special rule for json // for json {SomePath: /c}, CH will return {SomePath: \/c} - // add this to make sure SHA256 get correct result + // add this to make sure SHA256 gets the correct result... slashes are escaped so the + // client-side JSON string matches ClickHouse's serialization before hashing. newRow[c] = JSON.stringify(row[c]).replace(/\//g, '\\/'); } else { newRow[c] = JSON.stringify(row[c]); From e753d45f0f66e0abc29c8dc3f65a1d7122ea8dff Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 19 Oct 2026 22:00:02 -0400 Subject: [PATCH 15/22] refactor: share leftUtf8Prefix in common-utils for row WHERE Hoist the ClickHouse leftUTF8-matching helper so app and CLI row lookups stay in sync and cannot drift silently. Co-authored-by: Cursor --- packages/app/src/hooks/useRowWhere.tsx | 18 +++++------------- packages/cli/src/shared/useRowWhere.ts | 18 +++++------------- packages/common-utils/src/core/utils.ts | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 7bc64ce04d..d7278555d6 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -8,22 +8,14 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/metadata'; -import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; +import { + aliasMapToWithClauses, + LEFT_UTF8_HASH_CODE_POINTS, + leftUtf8Prefix, +} from '@hyperdx/common-utils/dist/core/utils'; import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; -const LEFT_UTF8_HASH_CODE_POINTS = 1000; - -/** First `limit` Unicode code points, matching ClickHouse leftUTF8(s, limit). */ -function leftUtf8Prefix(value: string, limit: number): string { - let prefix = ''; - let n = 0; - for (const ch of value) { - if (n++ === limit) break; - prefix += ch; - } - return prefix; -} // Type for WITH clause entries, derived from ChartConfig's with property export type WithClause = z.infer; diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 11b5215448..7491db0392 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -19,22 +19,14 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/metadata'; -import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; +import { + aliasMapToWithClauses, + LEFT_UTF8_HASH_CODE_POINTS, + leftUtf8Prefix, +} from '@hyperdx/common-utils/dist/core/utils'; import type { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; -const LEFT_UTF8_HASH_CODE_POINTS = 1000; - -/** First `limit` Unicode code points, matching ClickHouse leftUTF8(s, limit). */ -function leftUtf8Prefix(value: string, limit: number): string { - let prefix = ''; - let n = 0; - for (const ch of value) { - if (n++ === limit) break; - prefix += ch; - } - return prefix; -} // Type for WITH clause entries, derived from ChartConfig's with property type WithClause = NonNullable[number]; diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 0042b9da81..21a3f95ef1 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -51,6 +51,20 @@ import { SkipIndexMetadata, TableMetadata } from './metadata'; /** The default maximum number of buckets setting when determining a bucket duration for 'auto' granularity */ export const DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS = 60; +/** Code-point limit for row-lookup string hashes (matches ClickHouse leftUTF8). */ +export const LEFT_UTF8_HASH_CODE_POINTS = 1000; + +/** First `limit` Unicode code points, matching ClickHouse leftUTF8(s, limit). */ +export function leftUtf8Prefix(value: string, limit: number): string { + let prefix = ''; + let n = 0; + for (const ch of value) { + if (n++ === limit) break; + prefix += ch; + } + return prefix; +} + /** * Returns the quantile level of a percentile shorthand aggFn (`p95` -> 0.95), * or undefined for any other aggFn. From b7e70ae4c885247c1a2e11e447e32c3cd3f54fc3 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Tue, 20 Oct 2026 07:40:24 -0400 Subject: [PATCH 16/22] fix(cli): use isNull for null JSON row lookup values Match the app row-WHERE builder so null JSON columns are not hashed as an empty string. Co-authored-by: Cursor --- packages/cli/src/__tests__/rowWhere.test.ts | 12 ++++++++++++ packages/cli/src/shared/useRowWhere.ts | 17 +++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 16c770d9d4..91edfe644a 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -100,6 +100,18 @@ describe('buildColumnMap', () => { ); }); + it('uses isNull for null JSON column values', () => { + const meta: ColumnMetaType[] = [{ name: 'Body', type: 'JSON' }]; + + const where = getRowWhere( + { Body: null }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe('isNull(Body)'); + }); + it('hashes strings longer than MAX_STRING_LENGTH with SHA256', () => { const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 7491db0392..2a4e3968ab 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -71,7 +71,12 @@ function processRowToWhereClause( ); } - const strValue = value != null ? String(value) : null; + // Handle nullish values for all types uniformly + if (value == null) { + return SqlString.format(`isNull(?)`, [SqlString.raw(valueExpr)]); + } + + const strValue = String(value); switch (jsType) { case JSDataType.Date: @@ -95,7 +100,7 @@ function processRowToWhereClause( // Handle case for whole json object, ex: json return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - SHA256(strValue ?? '').toString(), + SHA256(strValue).toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -112,18 +117,14 @@ function processRowToWhereClause( ); default: - // Handle nullish values - if (value == null) { - return SqlString.format(`isNull(?)`, [SqlString.raw(valueExpr)]); - } // Handle the case when string is too long - if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { + if (strValue.length > MAX_STRING_LENGTH) { return SqlString.format( `lower(hex(SHA256(leftUTF8(?, ${LEFT_UTF8_HASH_CODE_POINTS}))))=?`, [ SqlString.raw(valueExpr), SHA256( - leftUtf8Prefix(strValue ?? '', LEFT_UTF8_HASH_CODE_POINTS), + leftUtf8Prefix(strValue, LEFT_UTF8_HASH_CODE_POINTS), ).toString(), ], ); From e40192b9f13f4285dfd9b65a80d08d1a1790c647 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Tue, 20 Oct 2026 07:46:00 -0400 Subject: [PATCH 17/22] add common-utils to changeset --- .changeset/fips-row-lookup-sha256.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/fips-row-lookup-sha256.md b/.changeset/fips-row-lookup-sha256.md index 26efa27eb8..5330967a3c 100644 --- a/.changeset/fips-row-lookup-sha256.md +++ b/.changeset/fips-row-lookup-sha256.md @@ -1,4 +1,5 @@ --- +'@hyperdx/common-utils': patch '@hyperdx/app': patch '@hyperdx/cli': patch --- From 50dbe95791d20c9921e8f5dab5453631cd853d01 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Tue, 20 Oct 2026 07:48:41 -0400 Subject: [PATCH 18/22] test(cli): assert isNull for null Date row lookup values Cover the uniform null guard for a non-JSON column type alongside JSON. Co-authored-by: Cursor --- packages/cli/src/__tests__/rowWhere.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 91edfe644a..29ad5c6bf1 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -112,6 +112,20 @@ describe('buildColumnMap', () => { expect(where).toBe('isNull(Body)'); }); + it('uses isNull for null Date column values', () => { + const meta: ColumnMetaType[] = [ + { name: 'event_created', type: "Nullable(DateTime64(3, 'UTC'))" }, + ]; + + const where = getRowWhere( + { event_created: null }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe('isNull(event_created)'); + }); + it('hashes strings longer than MAX_STRING_LENGTH with SHA256', () => { const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; From dd1822976585f1889f3f514a28a76b3d8be33813 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Tue, 20 Oct 2026 08:24:32 -0400 Subject: [PATCH 19/22] test: drive row WHERE SHA256 cases from shared table Use SHA256_TEST_CASES with it.each in app and CLI suites so ClickHouse golden hashes stay aligned and duplicated constants are removed. Co-authored-by: Cursor --- .../src/hooks/__tests__/useRowWhere.test.tsx | 239 ++++++++---------- packages/cli/src/__tests__/rowWhere.test.ts | 138 +++++----- 2 files changed, 176 insertions(+), 201 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index aa90343791..95edf4a020 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -3,24 +3,96 @@ import { ColumnMetaType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; +import { + LEFT_UTF8_HASH_CODE_POINTS, + leftUtf8Prefix, +} from '@hyperdx/common-utils/dist/core/utils'; import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; -/** From ClickHouse: lower(hex(SHA256())). */ -const OTEL_LOG_JSON_FOR_SHA256 = - '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}'; - -const OTEL_LOG_JSON_SHA256_HEX = - '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8'; - -/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000)))). */ -const LONGER_STRING_SHA256_HEX = - '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3'; - -/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000)))). */ -const EMOJI_STRING_SHA256_HEX = - '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43'; +type Sha256WhereKind = 'json' | 'leftUtf8'; + +type Sha256TestCase = { + /** ClickHouse expression this hash was verified against. */ + clickHouse: string; + columnMeta: ColumnMetaType[]; + input: string; + expected: string; + whereKind: Sha256WhereKind; +}; + +const SHA256_TEST_CASES: Record = { + 'simple JSON object': { + clickHouse: 'lower(hex(SHA256(toString(\'{"key": "value"}\'))))', + columnMeta: [{ name: 'data', type: 'JSON' }], + input: '{"key": "value"}', + expected: '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5', + whereKind: 'json', + }, + 'actual log line': { + clickHouse: 'lower(hex(SHA256()))', + columnMeta: [{ name: 'Body', type: 'JSON' }], + input: + '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', + expected: '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', + whereKind: 'json', + }, + '600-char string': { + clickHouse: "lower(hex(SHA256(repeat('a', 600))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(600), + expected: 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', + whereKind: 'leftUtf8', + }, + '2000-char string truncated to 1000 code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(2000), + expected: '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', + whereKind: 'leftUtf8', + }, + '600 emoji code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: '😀'.repeat(600), + expected: '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', + whereKind: 'leftUtf8', + }, +}; + +function sha256RowWhere( + columnName: string, + whereKind: Sha256WhereKind, + hash: string, +): string { + if (whereKind === 'json') { + return `lower(hex(SHA256(toString(${columnName}))))='${hash}'`; + } + return `lower(hex(SHA256(leftUTF8(${columnName}, 1000))))='${hash}'`; +} + +function jsTypeForSha256Column(chType: string): JSDataType { + if (chType === 'JSON') { + return JSDataType.JSON; + } + return JSDataType.String; +} + +function columnMapForSha256TestCase(testCase: Sha256TestCase) { + const { name, type } = testCase.columnMeta[0]!; + return new Map([ + [ + name, + { + name, + type, + valueExpr: name, + jsType: jsTypeForSha256Column(type), + }, + ], + ]); +} // Mock convertCHDataTypeToJSType jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ @@ -41,11 +113,16 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ })); describe('SHA256 matches ClickHouse lower(hex(SHA256(...)))', () => { - it('matches ClickHouse lower(hex(SHA256(...))) for a real otel log JSON string', () => { - expect(SHA256(OTEL_LOG_JSON_FOR_SHA256).toString()).toBe( - OTEL_LOG_JSON_SHA256_HEX, - ); - }); + it.each(Object.entries(SHA256_TEST_CASES))( + 'matches ClickHouse for %s', + (_label, testCase) => { + const inputForHash = + testCase.whereKind === 'leftUtf8' + ? leftUtf8Prefix(testCase.input, LEFT_UTF8_HASH_CODE_POINTS) + : testCase.input; + expect(SHA256(inputForHash).toString()).toBe(testCase.expected); + }, + ); }); describe('processRowToWhereClause', () => { @@ -129,50 +206,20 @@ describe('processRowToWhereClause', () => { ); }); - it('should handle JSON columns with SHA256', () => { - const columnMap = new Map([ - [ - 'data', - { - name: 'data', - type: 'JSON', - valueExpr: 'data', - jsType: JSDataType.JSON, - }, - ], - ]); - - const jsonValue = '{"key": "value"}'; - const row = { data: jsonValue }; - const result = processRowToWhereClause(row, columnMap); + it.each(Object.entries(SHA256_TEST_CASES))( + 'SHA256 row WHERE matches ClickHouse for %s', + (_label, testCase) => { + const columnName = testCase.columnMeta[0]!.name; + const result = processRowToWhereClause( + { [columnName]: testCase.input }, + columnMapForSha256TestCase(testCase), + ); - // in ClickHouse: SELECT lower(hex(SHA256(toString('{"key": "value"}'))); - const expected = - '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5'; - - expect(result).toBe(`lower(hex(SHA256(toString(data))))='${expected}'`); - }); - - it('should handle JSON columns with SHA256 for otel log payload', () => { - const columnMap = new Map([ - [ - 'Body', - { - name: 'Body', - type: 'JSON', - valueExpr: 'Body', - jsType: JSDataType.JSON, - }, - ], - ]); - - const row = { Body: OTEL_LOG_JSON_FOR_SHA256 }; - const result = processRowToWhereClause(row, columnMap); - - expect(result).toBe( - `lower(hex(SHA256(toString(Body))))='${OTEL_LOG_JSON_SHA256_HEX}'`, - ); - }); + expect(result).toBe( + sha256RowWhere(columnName, testCase.whereKind, testCase.expected), + ); + }, + ); it('should handle Dynamic columns with null value', () => { const columnMap = new Map([ @@ -302,74 +349,6 @@ describe('processRowToWhereClause', () => { ); }); - it('should handle long strings with SHA256', () => { - const columnMap = new Map([ - [ - 'description', - { - name: 'description', - type: 'String', - valueExpr: 'description', - jsType: JSDataType.String, - }, - ], - ]); - - const longString = 'a'.repeat(600); - const row = { description: longString }; - const result = processRowToWhereClause(row, columnMap); - - // in ClickHouse: SELECT lower(hex(SHA256(repeat('a', 600)))); - const expected = - 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d'; - - expect(result).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${expected}'`, - ); - }); - - it('should hash only the first 1000 Unicode code points when the string exceeds that limit', () => { - const columnMap = new Map([ - [ - 'description', - { - name: 'description', - type: 'String', - valueExpr: 'description', - jsType: JSDataType.String, - }, - ], - ]); - - const row = { description: 'a'.repeat(2000) }; - const result = processRowToWhereClause(row, columnMap); - - expect(result).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONGER_STRING_SHA256_HEX}'`, - ); - }); - - it('should match ClickHouse leftUTF8 code-point truncation for non-BMP strings', () => { - const columnMap = new Map([ - [ - 'description', - { - name: 'description', - type: 'String', - valueExpr: 'description', - jsType: JSDataType.String, - }, - ], - ]); - - const row = { description: '😀'.repeat(600) }; - const result = processRowToWhereClause(row, columnMap); - - expect(result).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${EMOJI_STRING_SHA256_HEX}'`, - ); - }); - it('should handle multiple columns with AND', () => { const columnMap = new Map([ [ diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 29ad5c6bf1..4a3cb8aa37 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -4,24 +4,59 @@ import type { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; import { buildColumnMap, getRowWhere } from '@/shared/useRowWhere'; -/** From ClickHouse: lower(hex(SHA256())) on telemetry.otel_logs. */ -const OTEL_LOG_JSON_FOR_SHA256 = - '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}'; - -const OTEL_LOG_JSON_SHA256_HEX = - '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8'; - -/** From ClickHouse: lower(hex(SHA256(repeat('a', 600)))). */ -const LONG_STRING_SHA256_HEX = - 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d'; - -/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000)))). */ -const LONGER_STRING_SHA256_HEX = - '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3'; - -/** From ClickHouse: lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000)))). */ -const EMOJI_STRING_SHA256_HEX = - '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43'; +type Sha256WhereKind = 'json' | 'leftUtf8'; + +type Sha256TestCase = { + /** ClickHouse expression this hash was verified against. */ + clickHouse: string; + columnMeta: ColumnMetaType[]; + input: string; + expected: string; + whereKind: Sha256WhereKind; +}; + +const SHA256_TEST_CASES: Record = { + 'actual log line': { + clickHouse: 'lower(hex(SHA256()))', + columnMeta: [{ name: 'Body', type: 'JSON' }], + input: + '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', + expected: '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', + whereKind: 'json', + }, + '600-char string': { + clickHouse: "lower(hex(SHA256(repeat('a', 600))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(600), + expected: 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', + whereKind: 'leftUtf8', + }, + '2000-char string truncated to 1000 code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(2000), + expected: '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', + whereKind: 'leftUtf8', + }, + '600 emoji code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: '😀'.repeat(600), + expected: '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', + whereKind: 'leftUtf8', + }, +}; + +function sha256RowWhere( + columnName: string, + whereKind: Sha256WhereKind, + hash: string, +): string { + if (whereKind === 'json') { + return `lower(hex(SHA256(toString(${columnName}))))='${hash}'`; + } + return `lower(hex(SHA256(leftUTF8(${columnName}, 1000))))='${hash}'`; +} describe('buildColumnMap', () => { it('quotes a column name that is not a bare identifier', () => { @@ -86,19 +121,22 @@ describe('buildColumnMap', () => { expect(where).toBe("ResourceAttributes['service.name']='api'"); }); - it('hashes JSON columns with SHA256 (otel log payload)', () => { - const meta: ColumnMetaType[] = [{ name: 'Body', type: 'JSON' }]; + it.each(Object.entries(SHA256_TEST_CASES))( + 'SHA256 row WHERE matches ClickHouse for %s', + (_label, testCase) => { + const columnName = testCase.columnMeta[0]!.name; - const where = getRowWhere( - { Body: OTEL_LOG_JSON_FOR_SHA256 }, - buildColumnMap(meta, {}), - {}, - ).where; + const where = getRowWhere( + { [columnName]: testCase.input }, + buildColumnMap(testCase.columnMeta, {}), + {}, + ).where; - expect(where).toBe( - `lower(hex(SHA256(toString(Body))))='${OTEL_LOG_JSON_SHA256_HEX}'`, - ); - }); + expect(where).toBe( + sha256RowWhere(columnName, testCase.whereKind, testCase.expected), + ); + }, + ); it('uses isNull for null JSON column values', () => { const meta: ColumnMetaType[] = [{ name: 'Body', type: 'JSON' }]; @@ -125,46 +163,4 @@ describe('buildColumnMap', () => { expect(where).toBe('isNull(event_created)'); }); - - it('hashes strings longer than MAX_STRING_LENGTH with SHA256', () => { - const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; - - const where = getRowWhere( - { description: 'a'.repeat(600) }, - buildColumnMap(meta, {}), - {}, - ).where; - - expect(where).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONG_STRING_SHA256_HEX}'`, - ); - }); - - it('hashes only the first 1000 Unicode code points when the string exceeds that limit', () => { - const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; - - const where = getRowWhere( - { description: 'a'.repeat(2000) }, - buildColumnMap(meta, {}), - {}, - ).where; - - expect(where).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${LONGER_STRING_SHA256_HEX}'`, - ); - }); - - it('matches ClickHouse leftUTF8 code-point truncation for non-BMP strings', () => { - const meta: ColumnMetaType[] = [{ name: 'description', type: 'String' }]; - - const where = getRowWhere( - { description: '😀'.repeat(600) }, - buildColumnMap(meta, {}), - {}, - ).where; - - expect(where).toBe( - `lower(hex(SHA256(leftUTF8(description, 1000))))='${EMOJI_STRING_SHA256_HEX}'`, - ); - }); }); From d00e10dc3332d79ea1e3d5fc365ae131f26dc452 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 2 Nov 2026 09:15:16 -0500 Subject: [PATCH 20/22] address linter errors in cli --- packages/cli/src/__tests__/rowWhere.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 4a3cb8aa37..a97a3c5e8f 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -21,28 +21,32 @@ const SHA256_TEST_CASES: Record = { columnMeta: [{ name: 'Body', type: 'JSON' }], input: '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', - expected: '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', + expected: + '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', whereKind: 'json', }, '600-char string': { clickHouse: "lower(hex(SHA256(repeat('a', 600))))", columnMeta: [{ name: 'description', type: 'String' }], input: 'a'.repeat(600), - expected: 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', + expected: + 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', whereKind: 'leftUtf8', }, '2000-char string truncated to 1000 code points': { clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", columnMeta: [{ name: 'description', type: 'String' }], input: 'a'.repeat(2000), - expected: '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', + expected: + '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', whereKind: 'leftUtf8', }, '600 emoji code points': { clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", columnMeta: [{ name: 'description', type: 'String' }], input: '😀'.repeat(600), - expected: '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', + expected: + '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', whereKind: 'leftUtf8', }, }; From db9941a9dfe2616682d56fc29717da6b6dd6ad95 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 2 Nov 2026 10:00:03 -0500 Subject: [PATCH 21/22] fix linter errors in app --- .../app/src/hooks/__tests__/useRowWhere.test.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 95edf4a020..6aad3b59c3 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -35,28 +35,32 @@ const SHA256_TEST_CASES: Record = { columnMeta: [{ name: 'Body', type: 'JSON' }], input: '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', - expected: '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', + expected: + '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', whereKind: 'json', }, '600-char string': { clickHouse: "lower(hex(SHA256(repeat('a', 600))))", columnMeta: [{ name: 'description', type: 'String' }], input: 'a'.repeat(600), - expected: 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', + expected: + 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', whereKind: 'leftUtf8', }, '2000-char string truncated to 1000 code points': { clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", columnMeta: [{ name: 'description', type: 'String' }], input: 'a'.repeat(2000), - expected: '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', + expected: + '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', whereKind: 'leftUtf8', }, '600 emoji code points': { clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", columnMeta: [{ name: 'description', type: 'String' }], input: '😀'.repeat(600), - expected: '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', + expected: + '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', whereKind: 'leftUtf8', }, }; From 96d1e0ec9b06171bf58d0d9684bd4fbb6d523a91 Mon Sep 17 00:00:00 2001 From: Steve Scaffidi Date: Mon, 2 Nov 2026 10:01:32 -0500 Subject: [PATCH 22/22] fix linter errors in app (missed one) --- packages/app/src/hooks/__tests__/useRowWhere.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 6aad3b59c3..8def46be7e 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -27,7 +27,8 @@ const SHA256_TEST_CASES: Record = { clickHouse: 'lower(hex(SHA256(toString(\'{"key": "value"}\'))))', columnMeta: [{ name: 'data', type: 'JSON' }], input: '{"key": "value"}', - expected: '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5', + expected: + '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5', whereKind: 'json', }, 'actual log line': {