Repository navigation
fix: replace MD5 with SHA256 for FIPS compatibility #3409
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
2f19158
a1dfe9f
c1e0753
732cdf3
3a08e37
89f0702
b3b864e
339899c
655d3a8
e3c7b21
f65aef8
5042f40
b50efb0
ad6909f
e753d45
b7e70ae
e40192b
50dbe95
dd18229
882afde
a0c5b15
d00e10d
db9941a
96d1e0e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| --- | ||
| '@hyperdx/common-utils': patch | ||
| '@hyperdx/app': patch | ||
| '@hyperdx/cli': patch | ||
|
hercynium marked this conversation as resolved.
Resolved
|
||
| --- | ||
|
|
||
| fix: use SHA256 instead of MD5 for JSON row lookup WHERE clauses so row | ||
| details and drill-down work on FIPS-enabled ClickHouse deployments | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,14 +1,103 @@ | ||
| import MD5 from 'crypto-js/md5'; | ||
| import SHA256 from 'crypto-js/sha256'; | ||
| import { | ||
| ColumnMetaType, | ||
| JSDataType, | ||
| } from '@hyperdx/common-utils/dist/clickhouse'; | ||
| import { | ||
| LEFT_UTF8_HASH_CODE_POINTS, | ||
| leftUtf8Prefix, | ||
| } from '@hyperdx/common-utils/dist/core/utils'; | ||
| import { renderHook } from '@testing-library/react'; | ||
|
|
||
| import useRowWhere, { processRowToWhereClause } from '@/hooks/useRowWhere'; | ||
|
|
||
| // Mock crypto-js/md5 | ||
| jest.mock('crypto-js/md5'); | ||
| type Sha256WhereKind = 'json' | 'leftUtf8'; | ||
|
|
||
| type Sha256TestCase = { | ||
| /** ClickHouse expression this hash was verified against. */ | ||
| clickHouse: string; | ||
| columnMeta: ColumnMetaType[]; | ||
| input: string; | ||
| expected: string; | ||
| whereKind: Sha256WhereKind; | ||
| }; | ||
|
|
||
| const SHA256_TEST_CASES: Record<string, Sha256TestCase> = { | ||
| 'simple JSON object': { | ||
| clickHouse: 'lower(hex(SHA256(toString(\'{"key": "value"}\'))))', | ||
| columnMeta: [{ name: 'data', type: 'JSON' }], | ||
| input: '{"key": "value"}', | ||
| expected: | ||
| '9724c1e20e6e3e4d7f57ed25f9d4efb006e508590d528c90da597f6a775c13e5', | ||
| whereKind: 'json', | ||
| }, | ||
| 'actual log line': { | ||
| clickHouse: 'lower(hex(SHA256(<json log line>)))', | ||
| columnMeta: [{ name: 'Body', type: 'JSON' }], | ||
| input: | ||
| '{"caller":"github.com/cloudzero/cloudzero-agent/app/domain/shipper/shipper.go:289","level":"info","message":"Successfully ran the shipper cycle","time":"2026-10-18T00:00:03Z","version":"5d05872b1f292e4bf481c61b187494b9bafff6a8.1.2.10-2026-03-24_02:00:46PM"}', | ||
| expected: | ||
| '9bf1e18eb79cfe32fa7abb7c2b5370085a73eb299ffaaa526393084c8681c0b8', | ||
| whereKind: 'json', | ||
| }, | ||
| '600-char string': { | ||
| clickHouse: "lower(hex(SHA256(repeat('a', 600))))", | ||
| columnMeta: [{ name: 'description', type: 'String' }], | ||
| input: 'a'.repeat(600), | ||
| expected: | ||
| 'ba35c170729417f1499e0886e7e12fcdb4ab00ad411110ae1e888c766d4ed70d', | ||
| whereKind: 'leftUtf8', | ||
| }, | ||
| '2000-char string truncated to 1000 code points': { | ||
| clickHouse: "lower(hex(SHA256(leftUTF8(repeat('a', 2000), 1000))))", | ||
| columnMeta: [{ name: 'description', type: 'String' }], | ||
| input: 'a'.repeat(2000), | ||
| expected: | ||
| '41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3', | ||
| whereKind: 'leftUtf8', | ||
| }, | ||
| '600 emoji code points': { | ||
| clickHouse: "lower(hex(SHA256(leftUTF8(repeat('😀', 600), 1000))))", | ||
| columnMeta: [{ name: 'description', type: 'String' }], | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔵 minor — The SHA256 test fixtures are copied word-for-word between the app and CLI tests
Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As mentioned before, this change already touches more files than I'd like so addressing this can be a different PR. |
||
| input: '😀'.repeat(600), | ||
| expected: | ||
| '7f22fd88b289648f5f2a8f290d1010a0cefcf260777d613f34f9fb7948db3e43', | ||
| whereKind: 'leftUtf8', | ||
| }, | ||
| }; | ||
|
|
||
| function sha256RowWhere( | ||
| columnName: string, | ||
| whereKind: Sha256WhereKind, | ||
| hash: string, | ||
| ): string { | ||
| if (whereKind === 'json') { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔵 minor — JSON test cases say they match ClickHouse, but they hash raw literals instead of what The expected hashes come from Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm pretty sure the tests are now more robust than they were before. This minor finding is fixable, but seems like it should go into another PR. |
||
| return `lower(hex(SHA256(toString(${columnName}))))='${hash}'`; | ||
| } | ||
| return `lower(hex(SHA256(leftUTF8(${columnName}, 1000))))='${hash}'`; | ||
| } | ||
|
|
||
| function jsTypeForSha256Column(chType: string): JSDataType { | ||
| if (chType === 'JSON') { | ||
| return JSDataType.JSON; | ||
| } | ||
| return JSDataType.String; | ||
| } | ||
|
|
||
| function columnMapForSha256TestCase(testCase: Sha256TestCase) { | ||
| const { name, type } = testCase.columnMeta[0]!; | ||
| return new Map([ | ||
| [ | ||
| name, | ||
| { | ||
| name, | ||
| type, | ||
| valueExpr: name, | ||
| jsType: jsTypeForSha256Column(type), | ||
| }, | ||
| ], | ||
| ]); | ||
| } | ||
|
|
||
| // Mock convertCHDataTypeToJSType | ||
| jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ | ||
|
|
@@ -28,14 +117,20 @@ jest.mock('@hyperdx/common-utils/dist/clickhouse', () => ({ | |
| }), | ||
| })); | ||
|
|
||
| describe('processRowToWhereClause', () => { | ||
| beforeEach(() => { | ||
| jest.clearAllMocks(); | ||
| (MD5 as jest.Mock).mockImplementation((value: string) => ({ | ||
| toString: () => `md5_${value}`, | ||
| })); | ||
| }); | ||
| describe('SHA256 matches ClickHouse lower(hex(SHA256(...)))', () => { | ||
| it.each(Object.entries(SHA256_TEST_CASES))( | ||
| 'matches ClickHouse for %s', | ||
| (_label, testCase) => { | ||
| const inputForHash = | ||
| testCase.whereKind === 'leftUtf8' | ||
| ? leftUtf8Prefix(testCase.input, LEFT_UTF8_HASH_CODE_POINTS) | ||
| : testCase.input; | ||
| expect(SHA256(inputForHash).toString()).toBe(testCase.expected); | ||
| }, | ||
| ); | ||
| }); | ||
|
|
||
| describe('processRowToWhereClause', () => { | ||
| it('should handle string columns', () => { | ||
| const columnMap = new Map([ | ||
| [ | ||
|
|
@@ -116,27 +211,20 @@ describe('processRowToWhereClause', () => { | |
| ); | ||
| }); | ||
|
|
||
| it('should handle JSON columns with MD5', () => { | ||
| const columnMap = new Map([ | ||
| [ | ||
| 'data', | ||
| { | ||
| name: 'data', | ||
| type: 'JSON', | ||
| valueExpr: 'data', | ||
| jsType: JSDataType.JSON, | ||
| }, | ||
| ], | ||
| ]); | ||
| it.each(Object.entries(SHA256_TEST_CASES))( | ||
| 'SHA256 row WHERE matches ClickHouse for %s', | ||
| (_label, testCase) => { | ||
| const columnName = testCase.columnMeta[0]!.name; | ||
| const result = processRowToWhereClause( | ||
| { [columnName]: testCase.input }, | ||
| columnMapForSha256TestCase(testCase), | ||
| ); | ||
|
|
||
| const row = { data: '{"key": "value"}' }; | ||
| const result = processRowToWhereClause(row, columnMap); | ||
|
|
||
| expect(result).toBe( | ||
| 'lower(hex(MD5(toString(data))))=\'md5_{\\"key\\": \\"value\\"}\'', | ||
| ); | ||
| expect(MD5).toHaveBeenCalledWith('{"key": "value"}'); | ||
| }); | ||
| expect(result).toBe( | ||
| sha256RowWhere(columnName, testCase.whereKind, testCase.expected), | ||
| ); | ||
| }, | ||
| ); | ||
|
|
||
| it('should handle Dynamic columns with null value', () => { | ||
| const columnMap = new Map([ | ||
|
|
@@ -266,29 +354,6 @@ describe('processRowToWhereClause', () => { | |
| ); | ||
| }); | ||
|
|
||
| it('should handle long strings with MD5', () => { | ||
| const columnMap = new Map([ | ||
| [ | ||
| 'description', | ||
| { | ||
| name: 'description', | ||
| type: 'String', | ||
| valueExpr: 'description', | ||
| jsType: JSDataType.String, | ||
| }, | ||
| ], | ||
| ]); | ||
|
|
||
| const longString = 'a'.repeat(600); | ||
| const row = { description: longString }; | ||
| const result = processRowToWhereClause(row, columnMap); | ||
|
|
||
| expect(result).toBe( | ||
| `lower(hex(MD5(leftUTF8(description, 1000))))='md5_${'a'.repeat(600)}'`, | ||
| ); | ||
| expect(MD5).toHaveBeenCalledWith('a'.repeat(600)); | ||
| }); | ||
|
|
||
| it('should handle multiple columns with AND', () => { | ||
| const columnMap = new Map([ | ||
| [ | ||
|
|
@@ -446,13 +511,6 @@ describe('processRowToWhereClause', () => { | |
| }); | ||
|
|
||
| describe('useRowWhere', () => { | ||
| beforeEach(() => { | ||
| jest.clearAllMocks(); | ||
| (MD5 as jest.Mock).mockImplementation((value: string) => ({ | ||
| toString: () => `md5_${value}`, | ||
| })); | ||
| }); | ||
|
|
||
| it('should return a function that processes rows', () => { | ||
| const meta: ColumnMetaType[] = [ | ||
| { name: 'id', type: 'String' }, | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.