Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
2f19158
bugfix: replace MD5 with SHA256 for FIPS compatibility
hercynium Oct 19, 2026
a1dfe9f
fixup tests to use real-world data
hercynium Oct 19, 2026
c1e0753
add changeset
hercynium Oct 19, 2026
732cdf3
manually fix up tests
hercynium Oct 19, 2026
3a08e37
remove unnecessary function and imports
hercynium Oct 19, 2026
89f0702
ran yarn lint:fix
hercynium Oct 19, 2026
b3b864e
improve another test
hercynium Oct 19, 2026
339899c
add missing tests
hercynium Oct 19, 2026
655d3a8
add more tests
hercynium Oct 19, 2026
e3c7b21
fix: slice long strings by Unicode code point to match ClickHouse lef…
hercynium Oct 19, 2026
f65aef8
make slicing a unicode string more efficient
hercynium Oct 19, 2026
5042f40
fix: use LEFT_UTF8_HASH_CODE_POINTS constant in SQL query instead of …
hercynium Oct 20, 2026
b50efb0
fix nits 1
hercynium Oct 20, 2026
ad6909f
improve comment
hercynium Oct 20, 2026
e753d45
refactor: share leftUtf8Prefix in common-utils for row WHERE
hercynium Oct 20, 2026
b7e70ae
fix(cli): use isNull for null JSON row lookup values
hercynium Oct 20, 2026
e40192b
add common-utils to changeset
hercynium Oct 20, 2026
50dbe95
test(cli): assert isNull for null Date row lookup values
hercynium Oct 20, 2026
dd18229
test: drive row WHERE SHA256 cases from shared table
hercynium Oct 20, 2026
882afde
Merge branch 'main' into bugfix/replace-md5-with-sha256-1
MikeShi42 Oct 23, 2026
a0c5b15
Merge branch 'main' into bugfix/replace-md5-with-sha256-1
hercynium Oct 28, 2026
d00e10d
address linter errors in cli
hercynium Nov 2, 2026
db9941a
fix linter errors in app
hercynium Nov 2, 2026
96d1e0e
fix linter errors in app (missed one)
hercynium Nov 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .changeset/fips-row-lookup-sha256.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/app': patch
Comment thread
hercynium marked this conversation as resolved.
Resolved
'@hyperdx/cli': patch
Comment thread
hercynium marked this conversation as resolved.
Resolved
---

fix: use SHA256 instead of MD5 for JSON row lookup WHERE clauses so row
details and drill-down work on FIPS-enabled ClickHouse deployments
3 changes: 2 additions & 1 deletion packages/app/src/components/DBRowTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1849,7 +1849,8 @@ function DBSqlRowTableComponent({
if (columnMap.get(c)?._type === JSDataType.JSON) {
// special rule for json
// for json {SomePath: /c}, CH will return {SomePath: \/c}
// add this to make sure md5 get correct result
// add this to make sure SHA256 gets the correct result... slashes are escaped so the
// client-side JSON string matches ClickHouse's serialization before hashing.
newRow[c] = JSON.stringify(row[c]).replace(/\//g, '\\/');
} else {
newRow[c] = JSON.stringify(row[c]);
Expand Down
178 changes: 118 additions & 60 deletions packages/app/src/hooks/__tests__/useRowWhere.test.tsx
Original file line number Diff line number Diff line change
@@ -1,14 +1,103 @@
import MD5 from 'crypto-js/md5';
import SHA256 from 'crypto-js/sha256';
import {
ColumnMetaType,
JSDataType,
} from '@hyperdx/common-utils/dist/clickhouse';
import {
LEFT_UTF8_HASH_CODE_POINTS,
leftUtf8Prefix,
} from '@hyperdx/common-utils/dist/core/utils';
import { renderHook } from '@testing-library/react';

import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere';

// Mock crypto-js/md5
jest.mock('crypto-js/md5');
type Sha256WhereKind = 'json' | 'leftUtf8';

type Sha256TestCase = {
/** ClickHouse expression this hash was verified against. */
clickHouse: string;
columnMeta: ColumnMetaType[];
input: string;
expected: string;
whereKind: Sha256WhereKind;
};

const SHA256_TEST_CASES: Record<string, Sha256TestCase> = {
'simple JSON object': {
clickHouse: 'lower(hex(SHA256(toString(\'{"key": "value"}\'))))',
columnMeta: [{ name: 'data', type: 'JSON' }],
input: '{"key": "value"}',
expected:
'9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5',
whereKind: 'json',
},
'actual log line': {
clickHouse: 'lower(hex(SHA256(<json log line>)))',
columnMeta: [{ name: 'Body', type: 'JSON' }],
input:
'{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}',
expected:
'9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8',
whereKind: 'json',
},
'600-char string': {
clickHouse: "lower(hex(SHA256(repeat('a', 600))))",
columnMeta: [{ name: 'description', type: 'String' }],
input: 'a'.repeat(600),
expected:
'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d',
whereKind: 'leftUtf8',
},
'2000-char string truncated to 1000 code points': {
clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))",
columnMeta: [{ name: 'description', type: 'String' }],
input: 'a'.repeat(2000),
expected:
'41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3',
whereKind: 'leftUtf8',
},
'600 emoji code points': {
clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))",
columnMeta: [{ name: 'description', type: 'String' }],

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — The SHA256 test fixtures are copied word-for-word between the app and CLI tests

Sha256WhereKind, Sha256TestCase, SHA256_TEST_CASES and sha256RowWhere are restated in packages/cli/src/__tests__/rowWhere.test.ts:7-63, so the verified hashes now have to be kept in sync by hand. Move the fixture table into a shared module next to leftUtf8Prefix in common-utils and import it from both tests. Also build the expected SQL with LEFT_UTF8_HASH_CODE_POINTS instead of the hardcoded 1000.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As mentioned before, this change already touches more files than I'd like so addressing this can be a different PR.

input: '😀'.repeat(600),
expected:
'7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43',
whereKind: 'leftUtf8',
},
};

function sha256RowWhere(
columnName: string,
whereKind: Sha256WhereKind,
hash: string,
): string {
if (whereKind === 'json') {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — JSON test cases say they match ClickHouse, but they hash raw literals instead of what toString(<JSON column>) actually returns

The expected hashes come from SHA256('<literal>'), so the tests only show that crypto-js and ClickHouse hash the same bytes the same way. The real risk is a serialization mismatch. toString() on a JSON column emits compact JSON. Per the comment at packages/app/src/components/DBRowTable.tsx:1848, it also escapes / as \/. So {"key": "value"} (with a space) and the 'actual log line' (with an unescaped github.com/...) would not match a stored row. Fix: build the input the way DBRowTable does (JSON.stringify(obj).replace(/\//g, '\\/')), and take the expected hash from lower(hex(SHA256(toString(Body)))) on a row inserted into a JSON column. Otherwise, drop the 'matches ClickHouse' claim and the clickHouse field.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm pretty sure the tests are now more robust than they were before. This minor finding is fixable, but seems like it should go into another PR.

return `lower(hex(SHA256(toString(${columnName}))))='${hash}'`;
}
return `lower(hex(SHA256(leftUTF8(${columnName}, 1000))))='${hash}'`;
}

function jsTypeForSha256Column(chType: string): JSDataType {
if (chType === 'JSON') {
return JSDataType.JSON;
}
return JSDataType.String;
}

function columnMapForSha256TestCase(testCase: Sha256TestCase) {
const { name, type } = testCase.columnMeta[0]!;
return new Map([
[
name,
{
name,
type,
valueExpr: name,
jsType: jsTypeForSha256Column(type),
},
],
]);
}

// Mock convertCHDataTypeToJSType
jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({
Expand All @@ -28,14 +117,20 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({
}),
}));

describe('processRowToWhereClause', () => {
beforeEach(() => {
jest.clearAllMocks();
(MD5 as jest.Mock).mockImplementation((value: string) => ({
toString: () => `md5_${value}`,
}));
});
describe('SHA256 matches ClickHouse lower(hex(SHA256(...)))', () => {
it.each(Object.entries(SHA256_TEST_CASES))(
'matches ClickHouse for %s',
(_label, testCase) => {
const inputForHash =
testCase.whereKind === 'leftUtf8'
? leftUtf8Prefix(testCase.input, LEFT_UTF8_HASH_CODE_POINTS)
: testCase.input;
expect(SHA256(inputForHash).toString()).toBe(testCase.expected);
},
);
});

describe('processRowToWhereClause', () => {
it('should handle string columns', () => {
const columnMap = new Map([
[
Expand Down Expand Up @@ -116,27 +211,20 @@ describe('processRowToWhereClause', () => {
);
});

it('should handle JSON columns with MD5', () => {
const columnMap = new Map([
[
'data',
{
name: 'data',
type: 'JSON',
valueExpr: 'data',
jsType: JSDataType.JSON,
},
],
]);
it.each(Object.entries(SHA256_TEST_CASES))(
'SHA256 row WHERE matches ClickHouse for %s',
(_label, testCase) => {
const columnName = testCase.columnMeta[0]!.name;
const result = processRowToWhereClause(
{ [columnName]: testCase.input },
columnMapForSha256TestCase(testCase),
);

const row = { data: '{"key": "value"}' };
const result = processRowToWhereClause(row, columnMap);

expect(result).toBe(
'lower(hex(MD5(toString(data))))=\'md5_{\\"key\\": \\"value\\"}\'',
);
expect(MD5).toHaveBeenCalledWith('{"key": "value"}');
});
expect(result).toBe(
sha256RowWhere(columnName, testCase.whereKind, testCase.expected),
);
},
);

it('should handle Dynamic columns with null value', () => {
const columnMap = new Map([
Expand Down Expand Up @@ -266,29 +354,6 @@ describe('processRowToWhereClause', () => {
);
});

it('should handle long strings with MD5', () => {
const columnMap = new Map([
[
'description',
{
name: 'description',
type: 'String',
valueExpr: 'description',
jsType: JSDataType.String,
},
],
]);

const longString = 'a'.repeat(600);
const row = { description: longString };
const result = processRowToWhereClause(row, columnMap);

expect(result).toBe(
`lower(hex(MD5(leftUTF8(description, 1000))))='md5_${'a'.repeat(600)}'`,
);
expect(MD5).toHaveBeenCalledWith('a'.repeat(600));
});

it('should handle multiple columns with AND', () => {
const columnMap = new Map([
[
Expand Down Expand Up @@ -446,13 +511,6 @@ describe('processRowToWhereClause', () => {
});

describe('useRowWhere', () => {
beforeEach(() => {
jest.clearAllMocks();
(MD5 as jest.Mock).mockImplementation((value: string) => ({
toString: () => `md5_${value}`,
}));
});

it('should return a function that processes rows', () => {
const meta: ColumnMetaType[] = [
{ name: 'id', type: 'String' },
Expand Down
22 changes: 13 additions & 9 deletions packages/app/src/hooks/useRowWhere.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { useCallback, useMemo } from 'react';
import MD5 from 'crypto-js/md5';
import SHA256 from 'crypto-js/sha256';
import SqlString from 'sqlstring';
import z from 'zod';
import {
Expand All @@ -8,7 +8,11 @@ import {
JSDataType,
} from '@hyperdx/common-utils/dist/clickhouse';
import { quoteResultColumnNameIfNeeded } from '@hyperdx/common-utils/dist/core/metadata';
import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils';
import {
aliasMapToWithClauses,
LEFT_UTF8_HASH_CODE_POINTS,
leftUtf8Prefix,
} from '@hyperdx/common-utils/dist/core/utils';
import { WithClauseSchema } from '@hyperdx/common-utils/dist/types';

const MAX_STRING_LENGTH = 512;
Expand Down Expand Up @@ -84,9 +88,9 @@ export function processRowToWhereClause(
]);
case JSDataType.JSON:
// Handle case for whole json object, ex: json
return SqlString.format(`lower(hex(MD5(toString(?))))=?`, [
return SqlString.format(`lower(hex(SHA256(toString(?))))=?`, [
SqlString.raw(valueExpr),
MD5(value).toString(),
SHA256(value).toString(),
]);
case JSDataType.Dynamic:
// Handle case for json element, ex: json.c
Expand All @@ -113,13 +117,13 @@ export function processRowToWhereClause(
// Handle the case when string is too long
if (value.length > MAX_STRING_LENGTH) {
return SqlString.format(
// We need to slice since md5 can be slow on big payloads
// which will block the main thread on search table render
// UTF8 since js only slices in utf8 points, not bytes
`lower(hex(MD5(leftUTF8(?, 1000))))=?`,
// Hash only a prefix so huge payloads do not block the main thread.
`lower(hex(SHA256(leftUTF8(?, ${LEFT_UTF8_HASH_CODE_POINTS}))))=?`,
[
SqlString.raw(valueExpr),
MD5(value.substring(0, 1000)).toString(),
SHA256(
leftUtf8Prefix(value, LEFT_UTF8_HASH_CODE_POINTS),
).toString(),
],
);
}
Expand Down
Loading
Loading