From 1028a895ad8d545242534a983d7b8e7f806bc3ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 07:09:23 +0900 Subject: [PATCH 1/4] test(etl): reproduce invalid amount coercion on current develop --- .../EtlServiceAmountIntegrityTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceAmountIntegrityTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceAmountIntegrityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceAmountIntegrityTest.java new file mode 100644 index 00000000..d905af5d --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceAmountIntegrityTest.java @@ -0,0 +1,71 @@ +package com.xtrmetl.etl.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.jdbc.core.JdbcTemplate; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +/** + * Proves that invalid monetary input is rejected rather than converted into a legitimate zero. + */ +class EtlServiceAmountIntegrityTest { + + private JdbcTemplate jdbcTemplate; + private EtlService etlService; + + @BeforeEach + void setUp() { + jdbcTemplate = mock(JdbcTemplate.class); + EtlBatchProperties properties = new EtlBatchProperties(); + properties.setMaxPayloadBytes(65_536); + properties.setMaxBatchRecords(100); + etlService = new EtlService(jdbcTemplate, new ObjectMapper(), properties); + } + + @ParameterizedTest + @ValueSource(strings = { + "", + "not-a-number", + "123456789012345678901234567890123456789", + "0.0000000000000000001", + "1E+19" + }) + void rejectsInvalidOrUnsupportedAmountsBeforeJdbc(String amount) { + EtlRequestException exception = assertThrows( + EtlRequestException.class, + () -> etlService.processData(jsonRecord("record_alpha", amount)) + ); + + assertSame(EtlRequestError.INVALID_RECORD, exception.error()); + verifyNoInteractions(jdbcTemplate); + } + + @Test + void invalidAmountMakesTheWholeBatchFailBeforeAnyJdbcWrite() { + String payload = """ + [ + {"id":"record_alpha","amount":"10.00"}, + {"id":"record_beta","amount":"not-a-number"} + ] + """; + + EtlRequestException exception = assertThrows( + EtlRequestException.class, + () -> etlService.processData(payload) + ); + + assertSame(EtlRequestError.INVALID_RECORD, exception.error()); + verifyNoInteractions(jdbcTemplate); + } + + private static String jsonRecord(String id, String amount) { + return "[{\"id\":\"" + id + "\",\"amount\":\"" + amount + "\"}]"; + } +} From f211d818a8e7e7ead207620b7c4a0679666e6be3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 09:47:58 +0900 Subject: [PATCH 2/4] fix(etl): reject invalid amounts before persistence --- .../com/xtrmetl/etl/service/EtlService.java | 19 ++++++++++--------- .../service/EtlServiceBatchSafetyTest.java | 13 +++++++------ .../xtrmetl/etl/service/EtlServiceTest.java | 19 +++++++++++-------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/etl-service/src/main/java/com/xtrmetl/etl/service/EtlService.java b/etl-service/src/main/java/com/xtrmetl/etl/service/EtlService.java index 6bb32a15..2557ec04 100644 --- a/etl-service/src/main/java/com/xtrmetl/etl/service/EtlService.java +++ b/etl-service/src/main/java/com/xtrmetl/etl/service/EtlService.java @@ -400,18 +400,19 @@ private String transformValue(String key, @Nullable JsonNode valueNode) { } private String formatAmount(String value) { + final BigDecimal amount; try { - BigDecimal amount = new BigDecimal(value.trim()); - int scale = amount.scale(); - if (amount.precision() > MAX_AMOUNT_PRECISION - || scale < -MAX_AMOUNT_ABSOLUTE_SCALE - || scale > MAX_AMOUNT_ABSOLUTE_SCALE) { - return "0.00"; - } - return amount.setScale(2, RoundingMode.HALF_UP).toPlainString(); + amount = new BigDecimal(value.trim()); } catch (NumberFormatException exception) { - return "0.00"; + throw invalidRecord(); + } + int scale = amount.scale(); + if (amount.precision() > MAX_AMOUNT_PRECISION + || scale < -MAX_AMOUNT_ABSOLUTE_SCALE + || scale > MAX_AMOUNT_ABSOLUTE_SCALE) { + throw invalidRecord(); } + return amount.setScale(2, RoundingMode.HALF_UP).toPlainString(); } private record StoredIdempotencyRecord(String requestDigest, String responseBody) { diff --git a/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceBatchSafetyTest.java b/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceBatchSafetyTest.java index b32e9a66..eaf9cc3b 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceBatchSafetyTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceBatchSafetyTest.java @@ -274,15 +274,16 @@ void usesLocaleIndependentTextAndDecimalTransformations() { } @Test - void boundsExtremeDecimalInputsWithoutHugePlainStringExpansion() { + void rejectsExtremeDecimalInputsWithoutHugePlainStringExpansion() { EtlService service = service(); - service.processData("[{\"id\":\"record_alpha\",\"amount\":\"1E+1000000\"}]"); - - verify(jdbcTemplate).update( - "INSERT INTO processed_data (data) VALUES (?)", - "ID:record_alpha,AMOUNT:0.00," + EtlRequestException exception = assertThrows( + EtlRequestException.class, + () -> service.processData("[{\"id\":\"record_alpha\",\"amount\":\"1E+1000000\"}]") ); + + assertSame(EtlRequestError.INVALID_RECORD, exception.error()); + verifyNoInteractions(jdbcTemplate); } @Test diff --git a/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceTest.java b/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceTest.java index 5cc4ca06..f386131c 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/service/EtlServiceTest.java @@ -129,12 +129,16 @@ void formatsAmountsDeterministically(String input, String expected) { } @Test - void fallsBackToZeroForInvalidAmount() { - etlService.processData( - "[{\"id\":\"record_alpha\",\"amount\":\"not-a-number\"}]" + void rejectsInvalidAmountInsteadOfManufacturingZero() { + EtlRequestException exception = assertThrows( + EtlRequestException.class, + () -> etlService.processData( + "[{\"id\":\"record_alpha\",\"amount\":\"not-a-number\"}]" + ) ); - verify(jdbcTemplate).update(anyString(), contains("AMOUNT:0.00")); + assertSame(EtlRequestError.INVALID_RECORD, exception.error()); + verifyNoInteractions(jdbcTemplate); } @Test @@ -154,19 +158,18 @@ void retainsUnknownAndUnicodeFields() { } @Test - void preservesEmptyOptionalValues() { + void preservesEmptyOptionalNameAndEmailValues() { etlService.processData(""" [{ "id":"record_alpha", "name":"", - "email":"", - "amount":"" + "email":"" }] """); verify(jdbcTemplate).update( anyString(), - eq("ID:record_alpha,NAME:,EMAIL:,AMOUNT:0.00,") + eq("ID:record_alpha,NAME:,EMAIL:,") ); } } From 740fd2113b764081e88a12ec1c4de7905a054fb8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 10:16:47 +0900 Subject: [PATCH 3/4] test(docs): require amount-integrity operator truth --- .../EtlBatchDocsAlignmentTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlBatchDocsAlignmentTest.java b/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlBatchDocsAlignmentTest.java index 6ffb4cdd..6a0cfecd 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlBatchDocsAlignmentTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlBatchDocsAlignmentTest.java @@ -45,6 +45,17 @@ void runbookStatesAdmissionRollbackAndIngressBoundaries() throws IOException { assertTrue(runbook.contains("Duplicate field names are rejected")); } + @Test + void runbookRejectsInvalidAmountsInsteadOfManufacturingZero() throws IOException { + String runbook = read("docs/etl/bounded-atomic-batches.md"); + + assertTrue(runbook.contains( + "Malformed, blank, excessive-precision, or extreme-scale `AMOUNT` values are rejected" + )); + assertTrue(runbook.contains("before the first JDBC call")); + assertFalse(runbook.contains("legacy fallback `0.00`")); + } + @Test void configurationAndChangelogUseTheSameLimits() throws IOException { String application = read("etl-service/src/main/resources/application.yml"); @@ -56,6 +67,15 @@ void configurationAndChangelogUseTheSameLimits() throws IOException { assertTrue(changelog.contains("retry only transient Spring data-access failures")); } + @Test + void changelogRecordsFailClosedAmountIntegrity() throws IOException { + String changelog = read("CHANGELOG.md"); + + assertTrue(changelog.contains("invalid `AMOUNT` values fail closed")); + assertTrue(changelog.contains("genuine zero")); + assertFalse(changelog.contains("invalid `AMOUNT` values to `0.00`")); + } + private static String read(String relativePath) throws IOException { return Files.readString(projectRoot().resolve(relativePath), StandardCharsets.UTF_8); } From 0757cee58f950d1f0f20305119cdf9ae9656c026 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 10:23:19 +0900 Subject: [PATCH 4/4] docs(etl): align amount-integrity operator contract --- CHANGELOG.md | 1 + docs/etl/bounded-atomic-batches.md | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8c3e4db..a04550c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ETL request errors now use RFC 9457 `application/problem+json` responses with a stable `errorCode`, fixed type URI, explicit 400/401/404/409/413/422/503/500 taxonomy, and no internal exception text in client responses. - ETL requests now enforce bounded UTF-8 payload and record-count limits, prevalidate and transform the complete batch before the first JDBC call, and commit accepted records inside one Spring transaction. - ETL transformations now preserve comma/colon-bearing values, use locale-independent text conversion and deterministic `BigDecimal` amount formatting, and retry only transient Spring data-access failures. +- ETL amount integrity: invalid `AMOUNT` values fail closed before persistence instead of being rewritten to `0.00`, preserving the distinction from genuine zero values. - Product branding: user-facing docs and suggested image tags use **mightyETL** (formerly xtrmETL). - Legacy Java packages (`com.xtrmetl.*`), Maven `artifactId` `xtrmETL`, and some env/topic defaults remain for compatibility. - See `docs/rebrand-name-matrix.md`. diff --git a/docs/etl/bounded-atomic-batches.md b/docs/etl/bounded-atomic-batches.md index f9cf62db..4419ea0d 100644 --- a/docs/etl/bounded-atomic-batches.md +++ b/docs/etl/bounded-atomic-batches.md @@ -44,10 +44,13 @@ Fields are transformed directly from the Jackson JSON tree; values are not split - `NAME` values use locale-independent uppercase conversion. - `EMAIL` values use locale-independent lowercase conversion. - `AMOUNT` values use `BigDecimal`, `HALF_UP`, scale `2`, and `toPlainString()`. -- Invalid, excessive-precision, or extreme-scale amounts retain the legacy fallback `0.00` without expanding attacker-controlled exponents into huge strings. +- Malformed, blank, excessive-precision, or extreme-scale `AMOUNT` values are rejected as invalid records before the first JDBC call instead of being rewritten to a valid-looking zero. +- Batch admission remains all-or-nothing: if any record has an invalid `AMOUNT`, the request performs no database writes. - Nested arrays and objects are retained as compact JSON instead of collapsing to empty text. - Response lines remain `Processed: ` in input order. Identifier whitespace, ISO control, Unicode format-control, Unicode line-separator characters, and length are bounded so one record cannot inject, visually reorder, conceal, or amplify response lines. +Rows created by historic releases that rewrote invalid amounts to zero cannot be distinguished from genuine zero values after the fact without independent source-system evidence. Reconcile affected historical data from an authoritative source before making accounting or compliance decisions. + ## Configuration Preferred keys use the mightyETL product namespace; compatibility keys remain available: @@ -73,6 +76,8 @@ Values outside the supported range fail configuration binding instead of silentl - Keep the payload limit aligned with gateway and ingress body-size limits. The service-level check occurs after the MVC stack has materialized the request string and is not a substitute for edge enforcement. - Keep the record limit below the transaction size that the target database can commit within the request timeout and lock budget. - Monitor request latency, transaction duration, rollback rate, database pool wait time, and rejected payload/record-limit errors before raising either limit. +- Monitor invalid-record rejection rates and reconcile upstream amount-format changes before retrying rejected inputs. +- Do not log raw amount values or request payloads when diagnosing rejections; use bounded request/error metadata instead. - Use descriptive string identifiers. Numeric JSON identifier types are rejected to keep identifier contracts explicit and stable across systems. ## Remaining boundary