diff --git a/.changeset/fips-row-lookup-sha256.md b/.changeset/fips-row-lookup-sha256.md new file mode 100644 index 0000000000..5330967a3c --- /dev/null +++ b/.changeset/fips-row-lookup-sha256.md @@ -0,0 +1,8 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/cli': patch +--- + +fix: use SHA256 instead of MD5 for JSON row lookup WHERE clauses so row +details and drill-down work on FIPS-enabled ClickHouse deployments diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 69e9a15d80..587b796c47 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1849,7 +1849,8 @@ function DBSqlRowTableComponent({ if (columnMap.get(c)?._type === JSDataType.JSON) { // special rule for json // for json {SomePath: /c}, CH will return {SomePath: \/c} - // add this to make sure md5 get correct result + // add this to make sure SHA256 gets the correct result... slashes are escaped so the + // client-side JSON string matches ClickHouse's serialization before hashing. newRow[c] = JSON.stringify(row[c]).replace(/\//g, '\\/'); } else { newRow[c] = JSON.stringify(row[c]); diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 90b1aed0f1..8def46be7e 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -1,14 +1,103 @@ -import MD5 from 'crypto-js/md5'; +import SHA256 from 'crypto-js/sha256'; import { ColumnMetaType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; +import { + LEFT_UTF8_HASH_CODE_POINTS, + leftUtf8Prefix, +} from '@hyperdx/common-utils/dist/core/utils'; import { renderHook } from '@testing-library/react'; import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; -// Mock crypto-js/md5 -jest.mock('crypto-js/md5'); +type Sha256WhereKind = 'json' | 'leftUtf8'; + +type Sha256TestCase = { + /** ClickHouse expression this hash was verified against. */ + clickHouse: string; + columnMeta: ColumnMetaType[]; + input: string; + expected: string; + whereKind: Sha256WhereKind; +}; + +const SHA256_TEST_CASES: Record = { + 'simple JSON object': { + clickHouse: 'lower(hex(SHA256(toString(\'{"key": "value"}\'))))', + columnMeta: [{ name: 'data', type: 'JSON' }], + input: '{"key": "value"}', + expected: + '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5', + whereKind: 'json', + }, + 'actual log line': { + clickHouse: 'lower(hex(SHA256()))', + columnMeta: [{ name: 'Body', type: 'JSON' }], + input: + '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', + expected: + '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', + whereKind: 'json', + }, + '600-char string': { + clickHouse: "lower(hex(SHA256(repeat('a', 600))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(600), + expected: + 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', + whereKind: 'leftUtf8', + }, + '2000-char string truncated to 1000 code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(2000), + expected: + '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', + whereKind: 'leftUtf8', + }, + '600 emoji code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: '😀'.repeat(600), + expected: + '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', + whereKind: 'leftUtf8', + }, +}; + +function sha256RowWhere( + columnName: string, + whereKind: Sha256WhereKind, + hash: string, +): string { + if (whereKind === 'json') { + return `lower(hex(SHA256(toString(${columnName}))))='${hash}'`; + } + return `lower(hex(SHA256(leftUTF8(${columnName}, 1000))))='${hash}'`; +} + +function jsTypeForSha256Column(chType: string): JSDataType { + if (chType === 'JSON') { + return JSDataType.JSON; + } + return JSDataType.String; +} + +function columnMapForSha256TestCase(testCase: Sha256TestCase) { + const { name, type } = testCase.columnMeta[0]!; + return new Map([ + [ + name, + { + name, + type, + valueExpr: name, + jsType: jsTypeForSha256Column(type), + }, + ], + ]); +} // Mock convertCHDataTypeToJSType jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ @@ -28,14 +117,20 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ }), })); -describe('processRowToWhereClause', () => { - beforeEach(() => { - jest.clearAllMocks(); - (MD5 as jest.Mock).mockImplementation((value: string) => ({ - toString: () => `md5_${value}`, - })); - }); +describe('SHA256 matches ClickHouse lower(hex(SHA256(...)))', () => { + it.each(Object.entries(SHA256_TEST_CASES))( + 'matches ClickHouse for %s', + (_label, testCase) => { + const inputForHash = + testCase.whereKind === 'leftUtf8' + ? leftUtf8Prefix(testCase.input, LEFT_UTF8_HASH_CODE_POINTS) + : testCase.input; + expect(SHA256(inputForHash).toString()).toBe(testCase.expected); + }, + ); +}); +describe('processRowToWhereClause', () => { it('should handle string columns', () => { const columnMap = new Map([ [ @@ -116,27 +211,20 @@ describe('processRowToWhereClause', () => { ); }); - it('should handle JSON columns with MD5', () => { - const columnMap = new Map([ - [ - 'data', - { - name: 'data', - type: 'JSON', - valueExpr: 'data', - jsType: JSDataType.JSON, - }, - ], - ]); + it.each(Object.entries(SHA256_TEST_CASES))( + 'SHA256 row WHERE matches ClickHouse for %s', + (_label, testCase) => { + const columnName = testCase.columnMeta[0]!.name; + const result = processRowToWhereClause( + { [columnName]: testCase.input }, + columnMapForSha256TestCase(testCase), + ); - const row = { data: '{"key": "value"}' }; - const result = processRowToWhereClause(row, columnMap); - - expect(result).toBe( - 'lower(hex(MD5(toString(data))))=\'md5_{\\"key\\": \\"value\\"}\'', - ); - expect(MD5).toHaveBeenCalledWith('{"key": "value"}'); - }); + expect(result).toBe( + sha256RowWhere(columnName, testCase.whereKind, testCase.expected), + ); + }, + ); it('should handle Dynamic columns with null value', () => { const columnMap = new Map([ @@ -266,29 +354,6 @@ describe('processRowToWhereClause', () => { ); }); - it('should handle long strings with MD5', () => { - const columnMap = new Map([ - [ - 'description', - { - name: 'description', - type: 'String', - valueExpr: 'description', - jsType: JSDataType.String, - }, - ], - ]); - - const longString = 'a'.repeat(600); - const row = { description: longString }; - const result = processRowToWhereClause(row, columnMap); - - expect(result).toBe( - `lower(hex(MD5(leftUTF8(description, 1000))))='md5_${'a'.repeat(600)}'`, - ); - expect(MD5).toHaveBeenCalledWith('a'.repeat(600)); - }); - it('should handle multiple columns with AND', () => { const columnMap = new Map([ [ @@ -446,13 +511,6 @@ describe('processRowToWhereClause', () => { }); describe('useRowWhere', () => { - beforeEach(() => { - jest.clearAllMocks(); - (MD5 as jest.Mock).mockImplementation((value: string) => ({ - toString: () => `md5_${value}`, - })); - }); - it('should return a function that processes rows', () => { const meta: ColumnMetaType[] = [ { name: 'id', type: 'String' }, diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index ad0f5ede8e..d7278555d6 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -1,5 +1,5 @@ import { useCallback, useMemo } from 'react'; -import MD5 from 'crypto-js/md5'; +import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import z from 'zod'; import { @@ -8,7 +8,11 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/metadata'; -import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; +import { + aliasMapToWithClauses, + LEFT_UTF8_HASH_CODE_POINTS, + leftUtf8Prefix, +} from '@hyperdx/common-utils/dist/core/utils'; import { WithClauseSchema } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; @@ -84,9 +88,9 @@ export function processRowToWhereClause( ]); case JSDataType.JSON: // Handle case for whole json object, ex: json - return SqlString.format(`lower(hex(MD5(toString(?))))=?`, [ + return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - MD5(value).toString(), + SHA256(value).toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -113,13 +117,13 @@ export function processRowToWhereClause( // Handle the case when string is too long if (value.length > MAX_STRING_LENGTH) { return SqlString.format( - // We need to slice since md5 can be slow on big payloads - // which will block the main thread on search table render - // UTF8 since js only slices in utf8 points, not bytes - `lower(hex(MD5(leftUTF8(?, 1000))))=?`, + // Hash only a prefix so huge payloads do not block the main thread. + `lower(hex(SHA256(leftUTF8(?, ${LEFT_UTF8_HASH_CODE_POINTS}))))=?`, [ SqlString.raw(valueExpr), - MD5(value.substring(0, 1000)).toString(), + SHA256( + leftUtf8Prefix(value, LEFT_UTF8_HASH_CODE_POINTS), + ).toString(), ], ); } diff --git a/packages/cli/src/__tests__/rowWhere.test.ts b/packages/cli/src/__tests__/rowWhere.test.ts index 0f427d4f46..a97a3c5e8f 100644 --- a/packages/cli/src/__tests__/rowWhere.test.ts +++ b/packages/cli/src/__tests__/rowWhere.test.ts @@ -4,6 +4,64 @@ import type { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; import { buildColumnMap, getRowWhere } from '@/shared/useRowWhere'; +type Sha256WhereKind = 'json' | 'leftUtf8'; + +type Sha256TestCase = { + /** ClickHouse expression this hash was verified against. */ + clickHouse: string; + columnMeta: ColumnMetaType[]; + input: string; + expected: string; + whereKind: Sha256WhereKind; +}; + +const SHA256_TEST_CASES: Record = { + 'actual log line': { + clickHouse: 'lower(hex(SHA256()))', + columnMeta: [{ name: 'Body', type: 'JSON' }], + input: + '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', + expected: + '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', + whereKind: 'json', + }, + '600-char string': { + clickHouse: "lower(hex(SHA256(repeat('a', 600))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(600), + expected: + 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', + whereKind: 'leftUtf8', + }, + '2000-char string truncated to 1000 code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: 'a'.repeat(2000), + expected: + '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', + whereKind: 'leftUtf8', + }, + '600 emoji code points': { + clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", + columnMeta: [{ name: 'description', type: 'String' }], + input: '😀'.repeat(600), + expected: + '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', + whereKind: 'leftUtf8', + }, +}; + +function sha256RowWhere( + columnName: string, + whereKind: Sha256WhereKind, + hash: string, +): string { + if (whereKind === 'json') { + return `lower(hex(SHA256(toString(${columnName}))))='${hash}'`; + } + return `lower(hex(SHA256(leftUTF8(${columnName}, 1000))))='${hash}'`; +} + describe('buildColumnMap', () => { it('quotes a column name that is not a bare identifier', () => { const meta: ColumnMetaType[] = [{ name: 'x-host-header', type: 'String' }]; @@ -66,4 +124,47 @@ describe('buildColumnMap', () => { expect(where).toBe("ResourceAttributes['service.name']='api'"); }); + + it.each(Object.entries(SHA256_TEST_CASES))( + 'SHA256 row WHERE matches ClickHouse for %s', + (_label, testCase) => { + const columnName = testCase.columnMeta[0]!.name; + + const where = getRowWhere( + { [columnName]: testCase.input }, + buildColumnMap(testCase.columnMeta, {}), + {}, + ).where; + + expect(where).toBe( + sha256RowWhere(columnName, testCase.whereKind, testCase.expected), + ); + }, + ); + + it('uses isNull for null JSON column values', () => { + const meta: ColumnMetaType[] = [{ name: 'Body', type: 'JSON' }]; + + const where = getRowWhere( + { Body: null }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe('isNull(Body)'); + }); + + it('uses isNull for null Date column values', () => { + const meta: ColumnMetaType[] = [ + { name: 'event_created', type: "Nullable(DateTime64(3, 'UTC'))" }, + ]; + + const where = getRowWhere( + { event_created: null }, + buildColumnMap(meta, {}), + {}, + ).where; + + expect(where).toBe('isNull(event_created)'); + }); }); diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 98f2fcdddb..2a4e3968ab 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -11,7 +11,7 @@ * to common-utils later. */ -import MD5 from 'crypto-js/md5'; +import SHA256 from 'crypto-js/sha256'; import SqlString from 'sqlstring'; import { ColumnMetaType, @@ -19,7 +19,11 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/metadata'; -import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; +import { + aliasMapToWithClauses, + LEFT_UTF8_HASH_CODE_POINTS, + leftUtf8Prefix, +} from '@hyperdx/common-utils/dist/core/utils'; import type { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; @@ -67,7 +71,12 @@ function processRowToWhereClause( ); } - const strValue = value != null ? String(value) : null; + // Handle nullish values for all types uniformly + if (value == null) { + return SqlString.format(`isNull(?)`, [SqlString.raw(valueExpr)]); + } + + const strValue = String(value); switch (jsType) { case JSDataType.Date: @@ -89,9 +98,9 @@ function processRowToWhereClause( ]); case JSDataType.JSON: // Handle case for whole json object, ex: json - return SqlString.format(`lower(hex(MD5(toString(?))))=?`, [ + return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [ SqlString.raw(valueExpr), - MD5(strValue ?? '').toString(), + SHA256(strValue).toString(), ]); case JSDataType.Dynamic: // Handle case for json element, ex: json.c @@ -108,16 +117,17 @@ function processRowToWhereClause( ); default: - // Handle nullish values - if (value == null) { - return SqlString.format(`isNull(?)`, [SqlString.raw(valueExpr)]); - } // Handle the case when string is too long - if ((strValue?.length ?? 0) > MAX_STRING_LENGTH) { - return SqlString.format(`lower(hex(MD5(leftUTF8(?, 1000))))=?`, [ - SqlString.raw(valueExpr), - MD5((strValue ?? '').substring(0, 1000)).toString(), - ]); + if (strValue.length > MAX_STRING_LENGTH) { + return SqlString.format( + `lower(hex(SHA256(leftUTF8(?, ${LEFT_UTF8_HASH_CODE_POINTS}))))=?`, + [ + SqlString.raw(valueExpr), + SHA256( + leftUtf8Prefix(strValue, LEFT_UTF8_HASH_CODE_POINTS), + ).toString(), + ], + ); } return SqlString.format(`?=?`, [ SqlString.raw(valueExpr), // don't escape expressions diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 0042b9da81..21a3f95ef1 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -51,6 +51,20 @@ import { SkipIndexMetadata, TableMetadata } from './metadata'; /** The default maximum number of buckets setting when determining a bucket duration for 'auto' granularity */ export const DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS = 60; +/** Code-point limit for row-lookup string hashes (matches ClickHouse leftUTF8). */ +export const LEFT_UTF8_HASH_CODE_POINTS = 1000; + +/** First `limit` Unicode code points, matching ClickHouse leftUTF8(s, limit). */ +export function leftUtf8Prefix(value: string, limit: number): string { + let prefix = ''; + let n = 0; + for (const ch of value) { + if (n++ === limit) break; + prefix += ch; + } + return prefix; +} + /** * Returns the quantile level of a percentile shorthand aggFn (`p95` -> 0.95), * or undefined for any other aggFn.