From c96f0978557ca2172a601a24cc223d104dbea69a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 20:13:34 +0900 Subject: [PATCH 1/4] test(stock-data): define bounded FSC acquisition and hostile-input contracts Capture missing-source RED and explicit cancellation/formatting regressions before provider implementation. Synthetic payloads are unit-test-only. No source/ref from an existing writer is overwritten. --- .../stock_data/FscStockDataSourceTest.java | 12 + .../stock_data/StockDataContractChecks.java | 249 ++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/stock_data/FscStockDataSourceTest.java create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/stock_data/FscStockDataSourceTest.java b/etl-service/src/test/java/com/xtrmetl/etl/stock_data/FscStockDataSourceTest.java new file mode 100644 index 00000000..953f1980 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/stock_data/FscStockDataSourceTest.java @@ -0,0 +1,12 @@ +package com.xtrmetl.etl.stock_data; + +import org.junit.jupiter.api.Test; + +/** Runs the identical source-acquisition contracts in the existing Maven reactor. */ +class FscStockDataSourceTest { + /** No new workflow or alternative success path replaces existing repository CI. */ + @Test + void validatesStockAcquisitionContracts() throws Exception { + StockDataContractChecks.verifyAll(); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java b/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java new file mode 100644 index 00000000..43b5c0a7 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java @@ -0,0 +1,249 @@ +package com.xtrmetl.etl.stock_data; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** Dependency-free unit contracts, also executed by the repository's JUnit suite. */ +public final class StockDataContractChecks { + private static final Clock FIXED_CLOCK = Clock.fixed(Instant.parse("2026-09-07T07:00:00Z"), ZoneOffset.UTC); + private static final LocalDate SOURCE_DATE = LocalDate.of(2026, 9, 4); + private static int assertionCount; + + private StockDataContractChecks() { } + + /** Run the same tests without downloading a second test framework. */ + public static void main(String[] commandArguments) throws Exception { + verifyAll(); + System.out.println("stock-data assertions passed: " + assertionCount); + } + + /** Exercise production query, transport-boundary, parser, and completeness behavior. */ + public static void verifyAll() throws Exception { + assertionCount = 0; + verifyCompleteCollection(); + verifyEmptyAndSingleton(); + verifyQueryRejection(); + verifyInvalidPages(); + verifyInvalidRecords(); + verifyHostileXml(); + verifyTransportFailures(); + verifyResourceBounds(); + verifyLateCancellationAndSafeFormatting(); + } + + private static FscStockDataSource.StockQuery sourceQuery(int pageSize) { + return new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, pageSize, 100, 10000); + } + + private static FscStockDataSource sourceFor(List responseBodies, List observedRequests) { + return new FscStockDataSource(requestValue -> { + observedRequests.add(requestValue); + return new StockDataTransport.PageResponse(200, "application/xml; charset=UTF-8", + new ByteArrayInputStream(responseBodies.get(requestValue.pageNumber() - 1).getBytes(StandardCharsets.UTF_8))); + }, "fsc_stock_key", FIXED_CLOCK); + } + + private static void verifyCompleteCollection() { + List observedRequests = new ArrayList<>(); + String firstBody = pageXml(1, 1, 2, itemXml("005930", "KR7005930003", "20260904")); + String secondBody = pageXml(2, 1, 2, itemXml("000660", "KR7000660001", "20260904")); + var sourceBatch = sourceFor(List.of(firstBody, secondBody), observedRequests).collectStockData(sourceQuery(1)); + requireEqual(2, sourceBatch.priceRecords().size(), "all pages are collected"); + requireEqual(2, observedRequests.size(), "exact request count"); + requireEqual("005930", sourceBatch.priceRecords().get(0).shortCode(), "leading zero preserved"); + requireEqual(new BigDecimal("70000.25"), sourceBatch.priceRecords().get(0).closePrice(), "decimal is exact"); + requireEqual("9007199254740993", sourceBatch.priceRecords().get(0).tradingValue().toString(), "large values are exact"); + requireEqual(FIXED_CLOCK.instant(), sourceBatch.rawPages().get(0).collectedAt(), "observation time preserved"); + requireEqual(firstBody, new String(sourceBatch.rawPages().get(0).rawBody(), StandardCharsets.UTF_8), "raw body preserved"); + requireEqual(64, sourceBatch.rawPages().get(0).sha256Digest().length(), "digest exists"); + requireEqual("provider_unspecified", sourceBatch.adjustmentBasis(), "adjustment not invented"); + requireEqual("delayed_daily", sourceBatch.freshnessClass(), "not realtime"); + requireEqual("xml", observedRequests.get(0).publicParameters().get("resultType"), "XML selected"); + requireEqual("20260904", observedRequests.get(0).publicParameters().get("basDt"), "explicit date"); + requireEqual(null, observedRequests.get(0).sourceEndpoint().getRawQuery(), "endpoint contains no credentials"); + requireEqual(false, observedRequests.get(0).publicParameters().containsKey("serviceKey"), "no key materialization"); + byte[] exposedBytes = sourceBatch.rawPages().get(0).rawBody(); + exposedBytes[0] = 0; + requireEqual(firstBody, new String(sourceBatch.rawPages().get(0).rawBody(), StandardCharsets.UTF_8), "raw body immutable"); + expectException(UnsupportedOperationException.class, () -> sourceBatch.priceRecords().clear()); + expectException(UnsupportedOperationException.class, () -> observedRequests.get(0).publicParameters().put("serviceKey", "secret")); + List filteredRequests = new ArrayList<>(); + var filteredQuery = new FscStockDataSource.StockQuery(SOURCE_DATE.minusDays(1), SOURCE_DATE, "KR7005930003", 1, 1, 1); + sourceFor(List.of(pageXml(1, 1, 1, itemXml("005930", "KR7005930003", "20260904"))), filteredRequests).collectStockData(filteredQuery); + requireEqual("20260903", filteredRequests.get(0).publicParameters().get("beginBasDt"), "range lower bound"); + requireEqual("20260904", filteredRequests.get(0).publicParameters().get("endBasDt"), "range upper bound"); + requireEqual("KR7005930003", filteredRequests.get(0).publicParameters().get("isinCd"), "exact ISIN filter"); + } + + private static void verifyEmptyAndSingleton() { + var emptyBatch = sourceFor(List.of(pageXml(1, 10, 0, "")), new ArrayList<>()).collectStockData(sourceQuery(10)); + requireEqual(0, emptyBatch.priceRecords().size(), "empty is not a fabricated candle"); + requireEqual("empty_source_result", emptyBatch.resultState(), "empty does not imply holiday"); + var oneBatch = sourceFor(List.of(pageXml(1, 10, 1, itemXml("005930", "KR7005930003", "20260904"))), new ArrayList<>()).collectStockData(sourceQuery(10)); + requireEqual(1, oneBatch.priceRecords().size(), "singleton retained"); + String noTrades = itemXml("005930", "KR7005930003", "20260904") + .replace("70000", "0").replace("71000", "0") + .replace("69000", "0").replace("1000", "0") + .replace("9007199254740993", "0"); + requireEqual(BigDecimal.ZERO, sourceFor(List.of(pageXml(1, 10, 1, noTrades)), new ArrayList<>()).collectStockData(sourceQuery(10)).priceRecords().get(0).openPrice(), "source zero with no trades preserved"); + } + + private static void verifyQueryRejection() { + for (int invalidSize : new int[]{0, -1, 1001}) { + expectCode("invalid_query", () -> sourceQuery(invalidSize)); + } + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(null, SOURCE_DATE, null, 10, 100, 10000)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE.minusDays(1), null, 10, 100, 10000)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE.minusDays(366), SOURCE_DATE, null, 10, 100, 10000)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, "bad&serviceKey=secret", 10, 100, 10000)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 10, 0, 10000)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 10, 101, 10000)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 10, 100, 0)); + expectCode("invalid_query", () -> new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 10, 100, 10001)); + expectCode("invalid_query", () -> new FscStockDataSource(null, "fsc_stock_key", FIXED_CLOCK)); + expectCode("invalid_query", () -> new FscStockDataSource(requestValue -> null, "raw/key+material=", FIXED_CLOCK)); + expectCode("invalid_query", () -> sourceFor(List.of(), new ArrayList<>()).collectStockData(null)); + } + + private static void verifyInvalidPages() { + String validItem = itemXml("005930", "KR7005930003", "20260904"); + for (String invalidBody : List.of( + pageXml(2, 10, 1, validItem), pageXml(1, 9, 1, validItem), + pageXml(1, 10, 2, validItem), pageXml(1, 10, 0, validItem), + pageXml(1, 10, -1, ""), pageXml(1, 10, 1, validItem).replace("1", "11"))) { + expectFailure(() -> sourceFor(List.of(invalidBody), new ArrayList<>()).collectStockData(sourceQuery(10))); + } + expectCode("incomplete_result", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem), pageXml(2, 1, 3, validItem)), new ArrayList<>()).collectStockData(sourceQuery(1))); + expectCode("duplicate_record", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem), pageXml(2, 1, 2, validItem)), new ArrayList<>()).collectStockData(sourceQuery(1))); + expectCode("incomplete_result", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem)), new ArrayList<>()).collectStockData(new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 1, 1, 2))); + expectCode("incomplete_result", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem)), new ArrayList<>()).collectStockData(new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 1, 2, 1))); + expectCode("provider_rejected", () -> sourceFor(List.of("
30secret
"), new ArrayList<>()).collectStockData(sourceQuery(10))); + } + + private static void verifyInvalidRecords() { + String validItem = itemXml("005930", "KR7005930003", "20260904"); + for (String invalidItem : List.of( + validItem.replace("70000.25", "-"), validItem.replace("70000.25", "NaN"), validItem.replace("70000.25", "1e1000"), + validItem.replace("70000.25", "-1"), validItem.replace("70000.25", "99999"), + validItem.replace("1000", "-1"), + validItem.replace("1000", "1.5"), + validItem.replace("20260904", "20260905"), validItem.replace("20260904", "20260230"), + validItem.replace("70000.25", ""), + validItem.replace("70000.25", "12"), + validItem.replace("단위시험종목", ""), + validItem.replace("KR7005930003", "not-an-isin"))) { + expectFailure(() -> sourceFor(List.of(pageXml(1, 10, 1, invalidItem)), new ArrayList<>()).collectStockData(sourceQuery(10))); + } + var filteredQuery = new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, "KR7000660001", 10, 100, 10000); + expectCode("invalid_record", () -> sourceFor(List.of(pageXml(1, 10, 1, validItem)), new ArrayList<>()).collectStockData(filteredQuery)); + } + + private static void verifyHostileXml() { + for (String hostileBody : List.of("login", "", + "]>&payload;", + "
", + "" + "".repeat(40) + "".repeat(40) + "")) { + expectFailure(() -> sourceFor(List.of(hostileBody), new ArrayList<>()).collectStockData(sourceQuery(10))); + } + } + + private static void verifyTransportFailures() { + expectCode("transport_failure", () -> new FscStockDataSource(requestValue -> { throw new IOException("https://provider.invalid/?serviceKey=secret"); }, "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + expectCode("transport_failure", () -> new FscStockDataSource(requestValue -> { throw new IllegalStateException("secret"); }, "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + expectCode("transport_failure", () -> new FscStockDataSource(requestValue -> null, "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + for (int statusCode : new int[]{301, 401, 403, 500}) { + boolean[] closedBody = {false}; + expectCode("provider_rejected", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(statusCode, "application/xml", trackedBody("secret", closedBody)), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + requireEqual(true, closedBody[0], "rejected HTTP body closed"); + } + expectCode("rate_limited", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(429, "application/xml", new ByteArrayInputStream(new byte[0])), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + for (String contentType : new String[]{null, "text/html", "application/xml; charset=ISO-8859-1"}) { + expectCode("invalid_content_type", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(200, contentType, new ByteArrayInputStream(new byte[0])), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + } + expectCode("cancelled", () -> new FscStockDataSource(requestValue -> { throw new InterruptedException("secret"); }, "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + requireEqual(true, Thread.interrupted(), "interrupt restored and cleared by test"); + expectCode("invalid_xml", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(200, "application/xml", new ByteArrayInputStream(new byte[]{(byte)0xff})), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + } + + private static void verifyLateCancellationAndSafeFormatting() { + var requestValue = new StockDataTransport.PageRequest(FscStockDataSource.SOURCE_ENDPOINT, Map.of(), "fsc_stock_key", 1); + requireEqual(false, requestValue.toString().contains("fsc_stock_key"), "credential reference not formatted"); + boolean[] closedBody = {false}; + try { + expectCode("cancelled", () -> new FscStockDataSource(pageRequest -> { + Thread.currentThread().interrupt(); + return new StockDataTransport.PageResponse(200, "application/xml", trackedBody(pageXml(1, 10, 0, ""), closedBody)); + }, "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + } finally { + requireEqual(true, Thread.interrupted(), "late cancellation remains signalled"); + } + requireEqual(true, closedBody[0], "late cancelled body closed"); + } + + private static void verifyResourceBounds() { + boolean[] closedBody = {false}; + expectCode("body_too_large", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(200, "application/xml", trackedBody(" ".repeat(2 * 1024 * 1024 + 1), closedBody)), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + requireEqual(true, closedBody[0], "oversized body closed"); + } + + private static InputStream trackedBody(String bodyText, boolean[] closedBody) { + return new ByteArrayInputStream(bodyText.getBytes(StandardCharsets.UTF_8)) { + @Override public void close() throws IOException { closedBody[0] = true; super.close(); } + }; + } + + private static String pageXml(int pageNumber, int pageSize, int totalCount, String itemContent) { + return "
00NORMAL SERVICE.
" + pageSize + "" + pageNumber + "" + totalCount + "" + itemContent + ""; + } + + private static String itemXml(String shortCode, String isinCode, String sourceDate) { + return "" + sourceDate + "" + shortCode + "" + isinCode + "단위시험종목KOSPI70000.25-8-4.577000071000690001000900719925474099310000700002500"; + } + + private static void requireEqual(Object expectedValue, Object actualValue, String assertionName) { + assertionCount++; + if (!java.util.Objects.equals(expectedValue, actualValue)) { + throw new AssertionError(assertionName + ": expected " + expectedValue + " got " + actualValue); + } + } + + private static void expectFailure(Runnable operationCall) { + assertionCount++; + try { operationCall.run(); } catch (StockDataException failureValue) { + requireEqual(null, failureValue.getCause(), "no unsafe cause"); + requireEqual(false, failureValue.toString().contains("secret"), "no leaked secret"); + return; + } + throw new AssertionError("expected a stock-data failure"); + } + + private static void expectCode(String expectedCode, Runnable operationCall) { + assertionCount++; + try { operationCall.run(); } catch (StockDataException failureValue) { + requireEqual(expectedCode, failureValue.errorCode(), "stable error code"); + requireEqual(null, failureValue.getCause(), "no unsafe cause"); + requireEqual(false, failureValue.toString().contains("secret"), "no leaked secret"); + return; + } + throw new AssertionError("expected " + expectedCode); + } + + private static void expectException(Class expectedType, Runnable operationCall) { + assertionCount++; + try { operationCall.run(); } catch (RuntimeException failureValue) { + if (expectedType.isInstance(failureValue)) { return; } + throw failureValue; + } + throw new AssertionError("expected " + expectedType.getSimpleName()); + } +} From 06ccc7f688eef135ff13bf105076f62f2879abfd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 20:17:02 +0900 Subject: [PATCH 2/4] feat(stock-data): add bounded FSC provider adapter and source evidence Implement explicit-query collection, all-or-error pagination, exact source values, bounded XML decoding, raw-page SHA-256 evidence, credential-free diagnostics and cancellation/close handling. Add proposed ADR, scoped PRD/TRD/UML/doctoring and gap baseline. Local focused evidence: OpenJDK 21.0.11, javac -Xlint:all -Werror, 245 synthetic unit assertions, Javadoc -Werror -Xdoclint:all. Full Java 25/Maven, coverage and live-provider conformance are not claimed. No default network client or provider key. EgressWeave #246 owns the missing released cross-language transport. Preserve canonical root-document writer #149 and all existing source/CI paths; no force push, closure or release. --- docs/adr/stock_data_source_boundary.md | 46 +++ docs/changes/stock_data_source.md | 7 + docs/doctoring/fsc_stock_data_sources.md | 32 ++ docs/product-technical-gap-baseline.md | 18 ++ docs/stock_data/stock_source_specification.md | 81 +++++ .../plans/2026-09-07-stock-data-source.md | 45 +++ .../etl/stock_data/FscStockDataSource.java | 278 ++++++++++++++++++ .../etl/stock_data/StockDataException.java | 20 ++ .../etl/stock_data/StockDataTransport.java | 82 ++++++ .../etl/stock_data/StockPageDecoder.java | 231 +++++++++++++++ .../etl/stock_data/StockPriceRecord.java | 40 +++ scripts/verify_stock_data_source.sh | 12 + 12 files changed, 892 insertions(+) create mode 100644 docs/adr/stock_data_source_boundary.md create mode 100644 docs/changes/stock_data_source.md create mode 100644 docs/doctoring/fsc_stock_data_sources.md create mode 100644 docs/product-technical-gap-baseline.md create mode 100644 docs/stock_data/stock_source_specification.md create mode 100644 docs/superpowers/plans/2026-09-07-stock-data-source.md create mode 100644 etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java create mode 100644 etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataException.java create mode 100644 etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataTransport.java create mode 100644 etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java create mode 100644 etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPriceRecord.java create mode 100644 scripts/verify_stock_data_source.sh diff --git a/docs/adr/stock_data_source_boundary.md b/docs/adr/stock_data_source_boundary.md new file mode 100644 index 00000000..3045b2f9 --- /dev/null +++ b/docs/adr/stock_data_source_boundary.md @@ -0,0 +1,46 @@ +# ADR: Stock-data source acquisition and transport ownership + +Status: Proposed. Date: 2026-09-07. Canonical identity: `stock_data_source_boundary` (semantic filename avoids collisions with the numbered ADR stack in PR #149). + +## Problem and evidence + +The user requested stock-data collection through CWL libraries. At protected `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb`, mightyETL has a Java ETL host and database/CDC connectors, but no discovered stock-source implementation. An organization code search and a focused open-PR search did not identify an existing stock collector; this is an inventory observation, not proof about private deployments. EgressWeave's protected README describes a Python HTTPX library. OriginWeave's protected README explicitly says its HTTP adapters are not shipped; its release listing returned no releases. + +The FSC public-data portal describes daily stock-price/volume data and explicitly states that reference-day data is supplied after 13:00 on the following business day. The introductory marketing phrase about realtime information must not override that operational notice. A Friday result may arrive on Monday or later when holidays intervene. An empty response is not proof of a holiday or a completed publication. + +## Alternatives and choice + +1. HTML scraping in a product consumer: rejected. It would duplicate acquisition policy, depend on page markup and tempt unsupported fallback after authorization failures. +2. Python/yfinance inside the Java service: rejected. It adds a second consumer runtime and an unverified provider/licensing path for convenience. +3. A new market-data service and financial database: deferred. No independent domain-truth or transaction boundary is needed just to retrieve provider records. +4. A source adapter in mightyETL, behind an explicit governed transport: chosen. The Java code is provider anti-corruption/host glue, not a new numerical analytics or security runtime. It uses JDK exact-value types without rounding or financial inference. Rust remains the implementation policy for new shared numerical and security/performance runtimes. A future Rust network binding belongs to EgressWeave, not a copied consumer client. + +## Implemented candidate contract + +`FscStockDataSource.collectStockData` accepts an inclusive reference-date range, optional exact ISIN, and bounded pagination budgets. It submits secret-free page requests through `StockDataTransport`; the host transport must resolve a deployment-owned credential reference and materialize `serviceKey` once. There is no built-in HTTP client, network fallback, Spring registration, scheduler, REST endpoint, SQL write or LLM call. + +The adapter validates the UTF-8 XML envelope and success code, response page number and size, stable total count, exact expected row count, date/filter membership, identifier grammar, duplicate date/ISIN identity, bounded exact numeric fields and OHLC ranges. It retains source zeroes only when consistent with the range rules; absent/invalid values never become zero. This is not full exchange-rule validation or ISIN-checksum certification. + +A complete batch is returned only after every page passes. Raw transfer-decoded XML bytes and SHA-256 digests are retained with per-page `collectedAt`. The provider reference date, observation time, `Asia/Seoul`, `KRW`, `delayed_daily`, and `provider_unspecified` adjustment basis remain distinct. No holiday, publication timestamp, corporate-action adjustment or realtime quote is inferred. + +Bounds are consumer safety budgets, not claimed provider quotas: 366 inclusive calendar days, 1,000 rows/page, 100 pages, 10,000 records, 2 MiB/page and 16 MiB raw bytes/batch. Exceeding a budget fails the request instead of silently truncating. Large backfills must be partitioned explicitly by the host and retain separate collection receipts. + +XML processing denies DTDs, external entities, schemas and XInclude, limits element depth, and suppresses provider/parser diagnostics. Transport errors, including close/suppressed failures, do not expose URLs, keys or payloads. Cancellation is checked before acquisition and after body delivery/read; response ownership is closed on every outcome. + +## Ownership and interoperability + +mightyETL owns provider query/field mapping and collection receipts. EgressWeave owns destination authorization, actual socket/DNS pinning, TLS, credentials, transport deadlines, rate/concurrency limits and response framing. The interface is a port, not evidence that those controls have run. Owner issue [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246) specifies the missing released cross-language binding and hostile/live conformance gates. + +A financial product owns market-data revisions, trading decisions and its database. This adapter does not write `processed_data` or route stock prices through the existing generic AMOUNT transformation. Context Graph Contracts and Enterprise Architecture Core remain read-only dependencies for this slice; no domain schema is copied into their repositories. A future catalog publication must reference a released source contract rather than copy observations into a catalog truth store. + +## Risks, acceptance and rollback + +Stable counts and duplicate checks detect common pagination corruption but do not prove an upstream transactional snapshot. The provider can revise data without changing its total count. Preserve raw pages and separate per-page observation times; do not advertise snapshot isolation or point-in-time backtest safety. + +Local verification covers synthetic unit inputs only. A provider key and real source response were not available. The official portal was inspected, but the complete primary wire guide was not retrieved; wire-profile approval and actual keyed retrieval remain release gates. Full Java 25 Maven CI, measured 100% production coverage, independent review, security/provenance and immutable release evidence are still required. Java 21 focused compilation is development evidence, not a replacement for the repository's Java 25 support gate. + +Before production use, complete #246, adopt its immutable release, verify the primary wire profile and run a known-day FSC request plus rate-limit/credential/close-path conformance. There is no automatic activation to roll back. Removing this package removes only the new source capability; existing ETL/CDC behavior and stored data are unchanged. + +Root PRD/TRD/README/AGENTS/CLAUDE/CHANGELOG remain in canonical documentation PR #149's ownership. This path-disjoint ADR and the linked stock-specific specification supply the feature delta for ordinary later integration, not a competing whole-file rewrite. + +References and source-to-test mapping: [doctoring](../doctoring/fsc_stock_data_sources.md). Usage/specification: [stock source](../stock_data/stock_source_specification.md). Current scope: [gap baseline](../product-technical-gap-baseline.md). diff --git a/docs/changes/stock_data_source.md b/docs/changes/stock_data_source.md new file mode 100644 index 00000000..252b6661 --- /dev/null +++ b/docs/changes/stock_data_source.md @@ -0,0 +1,7 @@ +# Unreleased candidate: FSC stock-source acquisition + +Add a Java provider adapter in the existing mightyETL ETL host for bounded FSC stock queries, typed exact values, complete-result pagination validation, immutable raw-page evidence and safe cancellation/error/stream handling. No new runtime dependency or workflow is introduced; JUnit invokes the same focused executable contracts. + +The source requires an explicit approved transport. It is not yet a released/live-provider-verified capability. EgressWeave #246 owns the missing immutable cross-language transport binding. The full primary wire guide, real keyed retrieval, Java 25 reactor, coverage, security and independent review remain release gates. + +This fragment is supplied to canonical documentation PR #149 rather than rewriting its concurrently owned root CHANGELOG/PRD/TRD/README/AGENTS/CLAUDE files. Merge it into the root changelog only with the actual integrated feature and its evidence; do not backdate a release or mark a Proposed ADR Accepted solely because code exists. diff --git a/docs/doctoring/fsc_stock_data_sources.md b/docs/doctoring/fsc_stock_data_sources.md new file mode 100644 index 00000000..f095101e --- /dev/null +++ b/docs/doctoring/fsc_stock_data_sources.md @@ -0,0 +1,32 @@ +# FSC stock-source evidence and decision traceability + +Access date: 2026-09-07. Only the following primary documents are cited as authorities. Operational and source-code observations are separate from future acceptance requirements. + +## References (APA 7) + +Financial Services Commission. (n.d.). *금융위원회_주식시세정보*. Public Data Portal. https://www.data.go.kr/data/15094808/openapi.do + +Oracle. (n.d.). *Java API for XML Processing (JAXP) security guide*. Java Platform, Standard Edition 25 Security Developer's Guide. https://docs.oracle.com/en/java/javase/25/security/java-api-xml-processing-jaxp-security-guide.html + +Oracle. (n.d.). *Class BigDecimal*. Java Platform, Standard Edition 25 API specification. https://docs.oracle.com/en/java/javase/25/docs/api/java.base/java/math/BigDecimal.html + +## Evidence boundaries + +The FSC portal explicitly says daily reference data is available after 13:00 on the next business day, offers XML/JSON REST data and requires an API application/key. This supports the delayed-daily classification and explicit credential boundary. It does not prove a particular local credential is approved, that a given day is complete, or that a transport implementation has passed conformance. The full attached primary wire guide was not retrieved in this run. Parameter/field mapping remains a candidate profile until the guide and a real keyed request are verified; third-party examples are not promoted to primary authority. + +The JAXP guide explains why default secure-processing settings alone should not be treated as external-resource denial. The decoder explicitly disables DTDs, external entity/schema access and XInclude, sets a depth limit, selects the JDK factory and uses strict UTF-8 decoding. Hostile XML contracts exercise the source boundary. They do not prove an upstream socket is governed; that belongs to EgressWeave. + +BigDecimal supports exact decimal representation. The adapter constructs from validated source text and performs no rounding or return/risk calculations. BigInteger retains whole-number quantities without IEEE-754 precision loss. These are provider type conversions, not a new financial analytics engine. + +## Traceability + +| Authority / concern | Source implementation | Executable evidence | +|---|---|---| +| FSC publication timing | `StockBatch.freshnessClass`, raw-page timestamps | `verifyCompleteCollection`, `verifyEmptyAndSingleton` | +| Query/pagination integrity | `collectStockData`, `StockPageDecoder.decodePage` | `verifyInvalidPages`, `verifyQueryRejection` | +| Exact source values | `StockPriceRecord`, decoder numeric functions | `verifyCompleteCollection`, `verifyInvalidRecords` | +| JAXP external resource limits | `StockPageDecoder.readDocument` | `verifyHostileXml`, `verifyTransportFailures` | +| No key/diagnostic export | `fetchBody`, request/response formatting, finite exception | `verifyTransportFailures`, `verifyLateCancellationAndSafeFormatting` | +| Response lifecycle | `PageResponse.close`, `fetchBody` | rejected/oversized/cancelled-body close assertions | + +The local compiler is OpenJDK 21.0.11. The focused suite reached 245 passing assertions, and warning-as-error compilation and Javadoc passed. This is not 245 independent JUnit test methods and not a 100% coverage measurement. The repository's Java 25 Maven reactor was not run locally; its full CI and security/review gates remain mandatory. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md new file mode 100644 index 00000000..d317780c --- /dev/null +++ b/docs/product-technical-gap-baseline.md @@ -0,0 +1,18 @@ +# Product / technical gap baseline + +Scope: stock-data source acquisition delta only, observed 2026-09-07. This is not a complete mightyETL readiness assessment. Protected baseline: `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb`; canonical broader documentation remains in open Draft PR #149. Open work is candidate/proposed, not shipped truth. + +| Buyer gap | Candidate action | Evidence / remaining gate | +|---|---|---| +| No discovered stock-source adapter | `FscStockDataSource.collectStockData` and typed provider ACL | Added source and executable unit contracts; not protected/released until PR integration | +| A partial history can look complete | Validate page identity, total, row count and duplicate date/ISIN; fail whole call | Focused synthetic unit tests; no upstream snapshot-isolation claim | +| Lost leading zeroes or numeric precision | Strings, `BigDecimal`, `BigInteger`; retain raw pages | Leading-zero, fractional-price and >2^53 assertions | +| Delayed/empty data confused with live trading | `delayed_daily`, `empty_source_result`, source date separate from collection time | Official FSC portal notice; no market-calendar inference | +| No verified released cross-language HTTP authority | Explicit transport port, no automatic network implementation | [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246): owner runtime, release and consumer conformance required | +| Provider wire/profile not fully verified | Bound the candidate mapping and preserve unknown fields/raw bytes | Official portal inspected; full primary guide and actual keyed known-day retrieval still required | +| Test/release acceptance incomplete | Existing JUnit entrypoint; warning-free local compile and Javadoc | Local Java 21 subset only; full Java 25 Maven, coverage, security, review, immutable release not proven | +| Durable stock store / revision history absent | Do not mutate generic `processed_data` or create cross-service SQL | Market-data domain owner, archive/revision API, migrations and real DB tests remain separate work | + +Source documentation: [ADR](adr/stock_data_source_boundary.md), [PRD/TRD/API/UML slice](stock_data/stock_source_specification.md), [doctoring](doctoring/fsc_stock_data_sources.md), [change fragment](changes/stock_data_source.md), [implementation plan](superpowers/plans/2026-09-07-stock-data-source.md). + +No release, live data capture, new schedule, independent approval or 100% coverage is asserted by this document. No existing PR was closed, superseded, force-pushed or stripped of valid delta for this feature. diff --git a/docs/stock_data/stock_source_specification.md b/docs/stock_data/stock_source_specification.md new file mode 100644 index 00000000..37179611 --- /dev/null +++ b/docs/stock_data/stock_source_specification.md @@ -0,0 +1,81 @@ +# Stock-data source specification and usage + +Status: implemented candidate, not released or live-provider verified. See [ADR](../adr/stock_data_source_boundary.md) and [gap baseline](../product-technical-gap-baseline.md). + +## Product requirement + +An ETL host can request published Korean stock observations for an explicit reference day or date range, optionally limited to one ISIN, and receive a complete typed batch with original response evidence. A failed or oversized page must not look like a successful partial historical dataset. The user must be able to distinguish reference date, collection time, empty source output and realtime data. + +This first provider is FSC `GetStockSecuritiesInfoService/getStockPriceInfo`. ETF, index, fundamentals, tick/order-book, trading/order APIs, adjusted total-return series and other countries are not implemented. New provider adapters must preserve their own publication and adjustment semantics rather than silently substitute feeds. + +## Java integration + +The package is `com.xtrmetl.etl.stock_data` in the existing `etl-service` module. It is not a separately published Maven artifact yet. + +```java +public FscStockDataSource.StockBatch loadPublishedStockDay( + StockDataTransport approvedTransport, Clock observationClock) { + var stockSource = new FscStockDataSource( + approvedTransport, "fsc_stock_key", observationClock); + var stockQuery = new FscStockDataSource.StockQuery( + LocalDate.of(2026, 9, 4), LocalDate.of(2026, 9, 4), + "KR7005930003", 1000, 100, 10000); + return stockSource.collectStockData(stockQuery); +} +``` + +This is host integration code, not a self-contained live example: `approvedTransport` must be an actual reviewed, released EgressWeave binding. `fsc_stock_key` is an opaque deployment reference, not an API key. No such concrete Java/Rust binding is claimed shipped here; [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246) owns it. Never replace it with arbitrary Java HTTP/curl/Python fetching in the consumer. The model/agent never receives key material. + +Public page parameters use `basDt` for one day or `beginBasDt`/`endBasDt` for a range, optional `isinCd`, plus `pageNo`, `numOfRows` and `resultType=xml`. The endpoint has no query string. XML is selected to use the existing JDK without a new parser dependency. The transport owns key encoding, HTTP framing/decompression, inactivity budgets, retries at the job boundary and shared provider quotas. The collector performs sequential requests and no retry; `rate_limited` requires the host to respect provider policy before a new collection attempt. + +## Field and provenance contract + +| FSC field | Typed result | Meaning retained | +|---|---|---| +| `basDt` | `referenceDate` | provider reference date, not publication/collection time | +| `srtnCd` | `shortCode` | exact text including leading zeroes | +| `isinCd` | `isinCode` | provider ISIN, grammar-checked, not checksum-certified | +| `itmsNm`, `mrktCtg` | `instrumentName`, `marketCategory` | provider labels, not guessed MICs | +| `mkp`, `hipr`, `lopr`, `clpr` | open/high/low/close `BigDecimal` | no binary floating point or two-decimal rounding | +| `trqu`, `trPrc` | `BigInteger` volume/value | exact integers, including values above 2^53 | +| remaining row fields | immutable `sourceFields` | bounded XML text with surrounding whitespace stripped; exact bytes remain in raw pages | + +Every batch retains the request, immutable observations and raw pages. Each raw page has `pageNumber`, `collectedAt`, exact bytes and their SHA-256. Preserve those bytes in an authorized archive before discarding the batch when durable replay is required. Returning a digest does not itself create durable storage or lineage publication. No request URL with `serviceKey` enters this evidence. + +## Failure and operating behavior + +`StockDataException.errorCode()` is one of `invalid_query`, `transport_failure`, `cancelled`, `rate_limited`, `provider_rejected`, `invalid_content_type`, `invalid_xml`, `invalid_page`, `invalid_record`, `incomplete_result`, `duplicate_record`, `body_too_large` or `digest_unavailable`. Errors contain no provider message, source row, request credential, cause or suppressed exception. + +A zero-row successful response produces `empty_source_result`; it does not certify a holiday, a future date, an unpublished day or nonexistent instrument. Populated results produce `complete`, meaning the response count/pagination contract passed. It does not certify source correctness, transactional snapshot consistency or exchange completeness. A single inconsistent row causes whole-call failure; archive/reconciliation policy belongs to the host, not silent row dropping. + +The adapter retains provider zero values and does not invent a trading-halt status. A zero-trade row whose open/high/low are all zero is allowed even when the provider carries forward a nonzero close. Corporate actions, split adjustments, suspension calendars, survivorship bias and historical availability remain separate domain responsibilities. + +## Sequence and acceptance + +```mermaid +sequenceDiagram + participant Host as ETL host + participant Source as FSC source adapter + participant Transport as Released EgressWeave binding + participant Provider as FSC API + Host->>Source: explicit bounded query + loop each required page + Source->>Transport: public params + credential reference + Transport->>Provider: approved GET with secret resolved privately + Provider-->>Transport: status, XML response + Transport-->>Source: owned bounded response stream + Source->>Source: read, close, decode, validate, hash + end + Source-->>Host: complete records + original-page evidence +``` + +The transport participant is the required owner contract, not an implemented service in this PR. There is no database migration or ERD delta. Each invocation's batch is the minimum all-or-error application boundary. + +Focused verification uses the same dependency-free contract suite as JUnit: + +```sh +sh scripts/verify_stock_data_source.sh +./mvnw -B -pl etl-service -am test +``` + +The first command compiles the new source with warnings as errors, runs synthetic unit assertions and generates Javadoc with warnings as errors. The second is the existing Maven reactor integration gate. Neither command makes a live API call. Do not replace the full repository CI or claim a fixture test is live conformance. diff --git a/docs/superpowers/plans/2026-09-07-stock-data-source.md b/docs/superpowers/plans/2026-09-07-stock-data-source.md new file mode 100644 index 00000000..48721d50 --- /dev/null +++ b/docs/superpowers/plans/2026-09-07-stock-data-source.md @@ -0,0 +1,45 @@ +# FSC Stock Data Source Implementation Plan + +> For agentic workers: use the execution and verification skills for each source change. + +**Goal:** Allow mightyETL callers to collect a bounded, complete FSC daily-stock result through an explicitly supplied governed transport, without importing a browser or giving the collector credentials. + +**Architecture:** A Java provider anti-corruption adapter fits the existing Java ETL host. It owns query construction, XML decoding, exact decimal conversion, complete-result validation and raw-response provenance. EgressWeave remains outbound-policy and transport authority; no HTTP client or Python runtime is embedded here. This slice is not a stock exchange, trading engine, numerical analytics core or a market-data system of record. + +**Tech Stack:** Existing Java host; JDK XML, date, decimal and digest APIs; existing JUnit Jupiter test dependency. No dependency or runtime-version change. + +**Spec:** `docs/adr/stock_data_source_boundary.md`. + +## Global constraints + +- Start from `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb` and preserve its tree and other writers. +- No HTTP client, credential materialization, redirect following, service registration or scheduled network work in this feature. +- No fabricated prices or fixture-based live-support claim. Synthetic inputs are unit-test-only. +- Java is provider/host glue, not a new numerical or security runtime. Future Rust transport belongs to EgressWeave and must be released before adoption. +- All-or-error within 366 calendar days, 100 pages, 10,000 records and 16 MiB aggregate XML; 2 MiB per response. +- No endpoint, provider exception, service key, XML fragment or input value in ordinary errors. + +## Task 1 — Executable contract and RED + +Create `etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java` with real source calls, synthetic XML and a close-tracking transport. Compile it against the absent implementation and retain the missing-source diagnostics as interface RED, not a passing behavior test. + +```sh +javac -d /tmp/stock_classes etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java +``` + +## Task 2 — Provider implementation and GREEN + +Create `StockDataException`, `StockDataTransport`, `StockPriceRecord`, `StockPageDecoder`, and `FscStockDataSource` in the matching production package. Validate shape, field multiplicity, UTF-8, DTD/entity prohibition, page identity/count, stable totals, bounds, duplicate instrument/date identities, and OHLC ranges without changing source values. Retain raw pages and SHA-256 digests. Catch transport diagnostics only at the external boundary and preserve interruption. + +```sh +javac -Xlint:all -Werror -d /tmp/stock_classes etl-service/src/main/java/com/xtrmetl/etl/stock_data/*.java etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java +java -cp /tmp/stock_classes com.xtrmetl.etl.stock_data.StockDataContractChecks +``` + +## Task 3 — Existing CI integration and disclosure + +Add `FscStockDataSourceTest` as a JUnit entrypoint to the same executable contracts, without replacing other tests or changing CI. Add scoped ADR, usage/field mapping, gap evidence and changelog fragment. Run the commands above, Javadoc with errors on warnings, and the existing full Maven test command where the Java 25 toolchain and dependencies are available. Open one Draft PR; local Java 21 subset success does not establish Java 25/Maven/hosted or live-provider acceptance. + +## Required continuation + +A released EgressWeave transport binding, provider-approved credential, primary wire-guide receipt and a real keyed FSC retrieval are release gates, not preconditions to writing the provider adapter. Record the owner issue and precise conformance expectations rather than shipping an unrestricted network fallback. diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java new file mode 100644 index 00000000..e62b5206 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java @@ -0,0 +1,278 @@ +package com.xtrmetl.etl.stock_data; + +import java.io.IOException; +import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Clock; +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Complete-result FSC stock collector with a host-owned governed transport. + * No network access, credentials, database writes or background tasks are implicit. + */ +public final class FscStockDataSource { + /** Secret-free official operation identity; never append a credential here. */ + public static final URI SOURCE_ENDPOINT = URI.create("https://apis.data.go.kr/1160100/service/GetStockSecuritiesInfoService/getStockPriceInfo"); + private static final int MAX_PAGE_BYTES = 2 * 1024 * 1024; + private static final int MAX_BATCH_BYTES = 16 * 1024 * 1024; + private final StockDataTransport sourceTransport; + private final String credentialReference; + private final Clock observationClock; + + /** + * Construct an explicit source adapter without fetching or resolving secrets. + * + * @param sourceTransport deployment-approved transport; no default is provided + * @param credentialReference opaque local secret reference, not the actual service key + * @param observationClock injected clock for collection evidence + */ + public FscStockDataSource(StockDataTransport sourceTransport, String credentialReference, Clock observationClock) { + if (sourceTransport == null || observationClock == null || credentialReference == null + || !credentialReference.matches("[a-z][a-z0-9_]{2,63}")) { + throw new StockDataException("invalid_query"); + } + this.sourceTransport = sourceTransport; + this.credentialReference = credentialReference; + this.observationClock = observationClock; + } + + /** + * Fetch and validate every page before exposing any result. The operation does + * not retry, publish, write a database, adjust prices or infer trading calendars. + * + * @param sourceQuery explicit bounded date range, optional ISIN and budgets + * @return complete records and immutable original-page evidence, or an explicit empty result + * @throws StockDataException on any transport, provider, record or completeness failure + */ + public StockBatch collectStockData(StockQuery sourceQuery) { + if (sourceQuery == null) { + throw new StockDataException("invalid_query"); + } + List priceRecords = new ArrayList<>(); + List rawPages = new ArrayList<>(); + Set recordIdentities = new HashSet<>(); + int expectedTotal = -1; + int batchBytes = 0; + for (int pageNumber = 1; pageNumber <= sourceQuery.maximumPages(); pageNumber++) { + byte[] rawBody = fetchBody(pageRequest(sourceQuery, pageNumber)); + batchBytes += rawBody.length; + if (batchBytes > MAX_BATCH_BYTES) { + throw new StockDataException("body_too_large"); + } + Instant collectedAt = observationClock.instant(); + var decodedPage = StockPageDecoder.decodePage(rawBody, sourceQuery, pageNumber); + if (expectedTotal == -1) { + expectedTotal = decodedPage.totalCount(); + if (expectedTotal > sourceQuery.maximumRecords() + || expectedTotal > (long) sourceQuery.pageSize() * sourceQuery.maximumPages()) { + throw new StockDataException("incomplete_result"); + } + } + if (decodedPage.totalCount() != expectedTotal + || decodedPage.priceRecords().size() != Math.min(sourceQuery.pageSize(), expectedTotal - priceRecords.size())) { + throw new StockDataException("incomplete_result"); + } + for (StockPriceRecord priceRecord : decodedPage.priceRecords()) { + String recordIdentity = priceRecord.referenceDate() + ":" + priceRecord.isinCode(); + if (!recordIdentities.add(recordIdentity)) { + throw new StockDataException("duplicate_record"); + } + priceRecords.add(priceRecord); + } + rawPages.add(new RawStockPage(pageNumber, collectedAt, rawBody)); + if (priceRecords.size() == expectedTotal) { + return new StockBatch(sourceQuery, priceRecords, rawPages); + } + } + throw new StockDataException("incomplete_result"); + } + + private StockDataTransport.PageRequest pageRequest(StockQuery sourceQuery, int pageNumber) { + Map publicParameters = new LinkedHashMap<>(); + publicParameters.put("resultType", "xml"); + publicParameters.put("numOfRows", Integer.toString(sourceQuery.pageSize())); + publicParameters.put("pageNo", Integer.toString(pageNumber)); + if (sourceQuery.fromDate().equals(sourceQuery.toDate())) { + publicParameters.put("basDt", sourceQuery.fromDate().format(DateTimeFormatter.BASIC_ISO_DATE)); + } else { + publicParameters.put("beginBasDt", sourceQuery.fromDate().format(DateTimeFormatter.BASIC_ISO_DATE)); + publicParameters.put("endBasDt", sourceQuery.toDate().format(DateTimeFormatter.BASIC_ISO_DATE)); + } + if (sourceQuery.isinCode() != null) { + publicParameters.put("isinCd", sourceQuery.isinCode()); + } + return new StockDataTransport.PageRequest(SOURCE_ENDPOINT, publicParameters, credentialReference, pageNumber); + } + + private byte[] fetchBody(StockDataTransport.PageRequest pageRequest) { + if (Thread.currentThread().isInterrupted()) { + throw new StockDataException("cancelled"); + } + StockDataTransport.PageResponse pageResponse; + try { + pageResponse = sourceTransport.fetchPage(pageRequest); + } catch (InterruptedException failureValue) { + Thread.currentThread().interrupt(); + throw new StockDataException("cancelled"); + } catch (IOException | RuntimeException failureValue) { + throw new StockDataException("transport_failure"); + } + if (pageResponse == null) { + throw new StockDataException("transport_failure"); + } + try (pageResponse) { + if (Thread.currentThread().isInterrupted()) { + throw new StockDataException("cancelled"); + } + if (pageResponse.statusCode() == 429) { + throw new StockDataException("rate_limited"); + } + if (pageResponse.statusCode() != 200) { + throw new StockDataException("provider_rejected"); + } + requireXmlContentType(pageResponse.contentType()); + byte[] rawBody = pageResponse.bodyStream().readNBytes(MAX_PAGE_BYTES + 1); + if (rawBody.length > MAX_PAGE_BYTES) { + throw new StockDataException("body_too_large"); + } + if (Thread.currentThread().isInterrupted()) { + throw new StockDataException("cancelled"); + } + return rawBody; + } catch (StockDataException failureValue) { + // Try-with-resources can attach a credential-bearing close failure. + throw new StockDataException(failureValue.errorCode()); + } catch (IOException | RuntimeException failureValue) { + throw new StockDataException("transport_failure"); + } + } + + private static void requireXmlContentType(String contentType) { + if (contentType == null || contentType.length() > 256) { + throw new StockDataException("invalid_content_type"); + } + String normalizedType = contentType.toLowerCase(Locale.ROOT); + if (!normalizedType.matches("(?:application|text)/xml(?:\\s*;\\s*charset\\s*=\\s*(?:utf-8|\"utf-8\"))?\\s*")) { + throw new StockDataException("invalid_content_type"); + } + } + + /** + * Inclusive, bounded source query. Empty provider responses are not holiday evidence. + * + * @param fromDate inclusive reference-date lower bound + * @param toDate inclusive upper bound, at most 365 days after the lower bound + * @param isinCode optional exact ISIN; null collects all returned instruments + * @param pageSize between 1 and 1,000 rows + * @param maximumPages between 1 and 100 pages, with no truncation on excess + * @param maximumRecords between 1 and 10,000 records, with no truncation on excess + */ + public record StockQuery(LocalDate fromDate, LocalDate toDate, String isinCode, + int pageSize, int maximumPages, int maximumRecords) { + /** Validate all query bounds before the first transport call. */ + public StockQuery { + if (fromDate == null || toDate == null || fromDate.isAfter(toDate) + || fromDate.getYear() < 1900 || toDate.getYear() > 9999 + || ChronoUnit.DAYS.between(fromDate, toDate) > 365 + || (isinCode != null && !isinCode.matches("[A-Z]{2}[A-Z0-9]{9}[0-9]")) + || pageSize < 1 || pageSize > 1000 || maximumPages < 1 || maximumPages > 100 + || maximumRecords < 1 || maximumRecords > 10000) { + throw new StockDataException("invalid_query"); + } + } + } + + /** Immutable raw response evidence; it deliberately contains no request credential. */ + public static final class RawStockPage { + private final int pageNumber; + private final Instant collectedAt; + private final byte[] rawBody; + private final String sha256Digest; + + private RawStockPage(int pageNumber, Instant collectedAt, byte[] rawBody) { + this.pageNumber = pageNumber; + this.collectedAt = collectedAt; + this.rawBody = rawBody.clone(); + try { + this.sha256Digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(rawBody)); + } catch (NoSuchAlgorithmException failureValue) { + throw new StockDataException("digest_unavailable"); + } + } + + /** Identify the original page within the collection. + * @return provider page number + */ + public int pageNumber() { return pageNumber; } + /** Keep observation time separate from the provider reference date. + * @return complete-response observation time + */ + public Instant collectedAt() { return collectedAt; } + /** Read the original response without exposing mutable internal storage. + * @return a defensive copy of the exact response bytes + */ + public byte[] rawBody() { return rawBody.clone(); } + /** Identify the original page independently of later field normalization. + * @return lowercase SHA-256 of the raw body + */ + public String sha256Digest() { return sha256Digest; } + } + + /** A complete source result, created only after every requested page is validated. */ + public static final class StockBatch { + private final StockQuery sourceQuery; + private final List priceRecords; + private final List rawPages; + + private StockBatch(StockQuery sourceQuery, List priceRecords, List rawPages) { + this.sourceQuery = sourceQuery; + this.priceRecords = List.copyOf(priceRecords); + this.rawPages = List.copyOf(rawPages); + } + + /** Recover the exact query whose pagination completed. + * @return completed query + */ + public StockQuery sourceQuery() { return sourceQuery; } + /** Read validated observations without fabricated gap-fill. + * @return immutable source records + */ + public List priceRecords() { return priceRecords; } + /** Access raw evidence for authorized archival or reprocessing. + * @return immutable complete-page evidence + */ + public List rawPages() { return rawPages; } + /** Distinguish this daily publication from a realtime feed. + * @return delayed-daily classification + */ + public String freshnessClass() { return "delayed_daily"; } + /** Avoid asserting corporate-action adjustment without provider evidence. + * @return provider-unspecified adjustment semantics + */ + public String adjustmentBasis() { return "provider_unspecified"; } + /** Expose the currency of this provider profile. + * @return KRW currency code + */ + public String currencyCode() { return "KRW"; } + /** Interpret reference dates in the source market time zone. + * @return Asia/Seoul reference zone + */ + public String referenceZone() { return "Asia/Seoul"; } + /** Distinguish an empty publication from a populated completed result. + * @return source-result state, not market-calendar status + */ + public String resultState() { return priceRecords.isEmpty() ? "empty_source_result" : "complete"; } + } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataException.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataException.java new file mode 100644 index 00000000..7bc07e19 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataException.java @@ -0,0 +1,20 @@ +package com.xtrmetl.etl.stock_data; + +/** A finite, credential-free failure at the stock acquisition boundary. */ +public final class StockDataException extends RuntimeException { + private static final long serialVersionUID = 1L; + /** Finite classification retained by this serializable failure. */ + private final String errorCode; + + StockDataException(String errorCode) { + super("Stock data acquisition failed: " + errorCode, null, false, true); + this.errorCode = errorCode; + } + + /** Classify the failure without exporting source diagnostics. + * @return stable machine error code + */ + public String errorCode() { + return errorCode; + } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataTransport.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataTransport.java new file mode 100644 index 00000000..87194645 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockDataTransport.java @@ -0,0 +1,82 @@ +package com.xtrmetl.etl.stock_data; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.Map; + +/** + * Host-supplied acquisition port. Implementations must use an approved, released + * EgressWeave binding for exact destination/TLS policy, credential resolution, + * transport budgets and provider-wide throttling. This interface itself proves + * none of those controls and deliberately has no unrestricted default client. + */ +@FunctionalInterface +public interface StockDataTransport { + /** + * Fetch exactly one page without redirects or retrying provider rejections. + * + * @param pageRequest immutable public query and opaque credential reference + * @return response body whose ownership transfers to the collector + * @throws IOException when transport cannot complete the request + * @throws InterruptedException when the caller cancels acquisition + */ + PageResponse fetchPage(PageRequest pageRequest) throws IOException, InterruptedException; + + /** + * A credential-free request description; the transport materializes serviceKey. + * + * @param sourceEndpoint the fixed FSC HTTPS endpoint, never a user-supplied URL + * @param publicParameters query values that exclude credentials + * @param credentialReference deployment-owned secret reference, not key material + * @param pageNumber expected response page number + */ + record PageRequest(URI sourceEndpoint, Map publicParameters, + String credentialReference, int pageNumber) { + /** Create an immutable request and retain the secret-free parameter map. */ + public PageRequest { + publicParameters = Map.copyOf(publicParameters); + } + + /** Describe the page without credential references or caller parameters. + * @return finite page identity only + */ + @Override + public String toString() { + return "PageRequest[pageNumber=" + pageNumber + "]"; + } + } + + /** + * Transfer-decoded, identity-content-coded response from the governed transport. + * + * @param statusCode HTTP response status + * @param contentType original media type, with optional UTF-8 charset + * @param bodyStream unconsumed response body; close on every outcome + */ + record PageResponse(int statusCode, String contentType, InputStream bodyStream) implements AutoCloseable { + /** Validate the owned stream without exposing it to diagnostic formatting. */ + public PageResponse { + if (bodyStream == null) { + throw new StockDataException("transport_failure"); + } + } + + /** + * Release the underlying connection/body on success and on failure. + * @throws IOException if the supplied stream fails to close + */ + @Override + public void close() throws IOException { + bodyStream.close(); + } + + /** Describe the response without formatting its stream. + * @return bounded HTTP status metadata + */ + @Override + public String toString() { + return "PageResponse[statusCode=" + statusCode + "]"; + } + } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java new file mode 100644 index 00000000..ebc715a1 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java @@ -0,0 +1,231 @@ +package com.xtrmetl.etl.stock_data; + +import java.io.StringReader; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import org.xml.sax.helpers.DefaultHandler; + +/** FSC-specific wire anti-corruption boundary; XML never grants network authority. */ +final class StockPageDecoder { + private StockPageDecoder() { } + + /** Decode one size-bounded UTF-8 response without resolving external entities. */ + static DecodedPage decodePage(byte[] rawBody, FscStockDataSource.StockQuery sourceQuery, int pageNumber) { + Element rootElement = readDocument(rawBody); + requireName(rootElement, "response"); + Element headerElement = onlyChild(rootElement, "header"); + if (!"00".equals(textValue(onlyChild(headerElement, "resultCode")))) { + throw new StockDataException("provider_rejected"); + } + Element bodyElement = onlyChild(rootElement, "body"); + int responsePage = integerValue(textValue(onlyChild(bodyElement, "pageNo"))); + int responseSize = integerValue(textValue(onlyChild(bodyElement, "numOfRows"))); + int totalCount = integerValue(textValue(onlyChild(bodyElement, "totalCount"))); + if (responsePage != pageNumber || responseSize != sourceQuery.pageSize()) { + throw new StockDataException("invalid_page"); + } + List priceRecords = new ArrayList<>(); + Element itemsElement = onlyChild(bodyElement, "items"); + for (Element itemElement : elementChildren(itemsElement)) { + requireName(itemElement, "item"); + if (priceRecords.size() >= sourceQuery.pageSize()) { + throw new StockDataException("invalid_page"); + } + priceRecords.add(decodeRecord(itemElement, sourceQuery, pageNumber)); + } + return new DecodedPage(totalCount, List.copyOf(priceRecords)); + } + + private static Element readDocument(byte[] rawBody) { + try { + String xmlText = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(rawBody)).toString(); + if (xmlText.startsWith("\ufeff")) { + xmlText = xmlText.substring(1); + } + DocumentBuilderFactory xmlFactory = DocumentBuilderFactory.newDefaultInstance(); + xmlFactory.setNamespaceAware(true); + xmlFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + xmlFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + xmlFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); + xmlFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + xmlFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + xmlFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + xmlFactory.setAttribute("http://www.oracle.com/xml/jaxp/properties/maxElementDepth", "8"); + xmlFactory.setXIncludeAware(false); + xmlFactory.setExpandEntityReferences(false); + var documentBuilder = xmlFactory.newDocumentBuilder(); + documentBuilder.setEntityResolver((publicId, systemId) -> { throw new SAXException("External entity refused"); }); + documentBuilder.setErrorHandler(new DefaultHandler() { + @Override public void error(SAXParseException failureValue) throws SAXException { throw failureValue; } + @Override public void fatalError(SAXParseException failureValue) throws SAXException { throw failureValue; } + }); + return documentBuilder.parse(new InputSource(new StringReader(xmlText))).getDocumentElement(); + } catch (CharacterCodingException failureValue) { + throw new StockDataException("invalid_xml"); + } catch (Exception failureValue) { + // XML/provider diagnostics may contain original values; they never cross this ACL. + throw new StockDataException("invalid_xml"); + } + } + + private static StockPriceRecord decodeRecord(Element itemElement, FscStockDataSource.StockQuery sourceQuery, int pageNumber) { + Map sourceFields = new LinkedHashMap<>(); + for (Element fieldElement : elementChildren(itemElement)) { + requirePlainElement(fieldElement); + String fieldName = fieldElement.getTagName(); + if (!fieldName.matches("[A-Za-z][A-Za-z0-9]{0,63}") || sourceFields.size() >= 64 + || sourceFields.putIfAbsent(fieldName, textValue(fieldElement)) != null) { + throw new StockDataException("invalid_record"); + } + } + try { + String dateText = requiredField(sourceFields, "basDt"); + if (!dateText.matches("[0-9]{8}")) { + throw new IllegalArgumentException(); + } + LocalDate referenceDate = LocalDate.parse(dateText, DateTimeFormatter.BASIC_ISO_DATE); + String shortCode = requiredField(sourceFields, "srtnCd"); + String isinCode = requiredField(sourceFields, "isinCd"); + if (!shortCode.matches("[A-Z0-9]{6,12}") || !isinCode.matches("[A-Z]{2}[A-Z0-9]{9}[0-9]") + || referenceDate.isBefore(sourceQuery.fromDate()) || referenceDate.isAfter(sourceQuery.toDate()) + || (sourceQuery.isinCode() != null && !sourceQuery.isinCode().equals(isinCode))) { + throw new IllegalArgumentException(); + } + BigDecimal openPrice = decimalValue(requiredField(sourceFields, "mkp")); + BigDecimal highPrice = decimalValue(requiredField(sourceFields, "hipr")); + BigDecimal lowPrice = decimalValue(requiredField(sourceFields, "lopr")); + BigDecimal closePrice = decimalValue(requiredField(sourceFields, "clpr")); + BigInteger tradingVolume = wholeValue(requiredField(sourceFields, "trqu")); + BigInteger tradingValue = wholeValue(requiredField(sourceFields, "trPrc")); + boolean noTradeRange = tradingVolume.signum() == 0 && openPrice.signum() == 0 + && highPrice.signum() == 0 && lowPrice.signum() == 0; + if (!noTradeRange && (lowPrice.compareTo(highPrice) > 0 || openPrice.compareTo(lowPrice) < 0 + || openPrice.compareTo(highPrice) > 0 || closePrice.compareTo(lowPrice) < 0 + || closePrice.compareTo(highPrice) > 0)) { + throw new IllegalArgumentException(); + } + if (tradingVolume.signum() > 0 && (openPrice.signum() == 0 || highPrice.signum() == 0 + || lowPrice.signum() == 0 || closePrice.signum() == 0)) { + throw new IllegalArgumentException(); + } + return new StockPriceRecord(referenceDate, shortCode, isinCode, + requiredField(sourceFields, "itmsNm"), requiredField(sourceFields, "mrktCtg"), + openPrice, highPrice, lowPrice, closePrice, tradingVolume, tradingValue, sourceFields, pageNumber); + } catch (RuntimeException failureValue) { + throw new StockDataException("invalid_record"); + } + } + + private static String requiredField(Map sourceFields, String fieldName) { + String fieldValue = sourceFields.get(fieldName); + if (fieldValue == null || fieldValue.isBlank()) { + throw new StockDataException("invalid_record"); + } + return fieldValue; + } + + private static BigDecimal decimalValue(String sourceValue) { + if (sourceValue.length() > 38 || !sourceValue.matches("[0-9]+(?:\\.[0-9]+)?")) { + throw new StockDataException("invalid_record"); + } + return new BigDecimal(sourceValue); + } + + private static BigInteger wholeValue(String sourceValue) { + if (sourceValue.length() > 38 || !sourceValue.matches("[0-9]+")) { + throw new StockDataException("invalid_record"); + } + return new BigInteger(sourceValue); + } + + private static int integerValue(String sourceValue) { + try { + if (!sourceValue.matches("[0-9]{1,9}")) { + throw new NumberFormatException(); + } + return Integer.parseInt(sourceValue); + } catch (NumberFormatException failureValue) { + throw new StockDataException("invalid_page"); + } + } + + private static Element onlyChild(Element parentElement, String childName) { + Element selectedElement = null; + for (Element childElement : elementChildren(parentElement)) { + if (childName.equals(childElement.getTagName())) { + requirePlainElement(childElement); + if (selectedElement != null) { + throw new StockDataException("invalid_xml"); + } + selectedElement = childElement; + } + } + if (selectedElement == null) { + throw new StockDataException("invalid_xml"); + } + return selectedElement; + } + + private static List elementChildren(Element parentElement) { + List childElements = new ArrayList<>(); + for (Node childNode = parentElement.getFirstChild(); childNode != null; childNode = childNode.getNextSibling()) { + if (childNode instanceof Element childElement) { + childElements.add(childElement); + } else if ((childNode.getNodeType() == Node.TEXT_NODE || childNode.getNodeType() == Node.CDATA_SECTION_NODE) + && !childNode.getTextContent().isBlank()) { + throw new StockDataException("invalid_xml"); + } + } + return childElements; + } + + private static String textValue(Element fieldElement) { + StringBuilder fieldText = new StringBuilder(); + for (Node childNode = fieldElement.getFirstChild(); childNode != null; childNode = childNode.getNextSibling()) { + if (childNode.getNodeType() != Node.TEXT_NODE && childNode.getNodeType() != Node.CDATA_SECTION_NODE) { + throw new StockDataException("invalid_xml"); + } + fieldText.append(childNode.getNodeValue()); + if (fieldText.length() > 1024) { + throw new StockDataException("invalid_record"); + } + } + return fieldText.toString().strip(); + } + + private static void requireName(Element sourceElement, String expectedName) { + requirePlainElement(sourceElement); + if (!expectedName.equals(sourceElement.getTagName())) { + throw new StockDataException("invalid_xml"); + } + } + + private static void requirePlainElement(Element sourceElement) { + if (sourceElement.getNamespaceURI() != null || sourceElement.hasAttributes()) { + throw new StockDataException("invalid_xml"); + } + } + + /** Internal decoded page; no completeness claim until collection finishes. */ + record DecodedPage(int totalCount, List priceRecords) { } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPriceRecord.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPriceRecord.java new file mode 100644 index 00000000..a9e14993 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPriceRecord.java @@ -0,0 +1,40 @@ +package com.xtrmetl.etl.stock_data; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.util.Map; + +/** + * Source-preserving stock observation, not an adjusted-price or trading signal. + * + * @param referenceDate provider business/reference date, not observation time + * @param shortCode exact short code, retaining leading zeroes + * @param isinCode exact provider ISIN text + * @param instrumentName provider instrument label + * @param marketCategory provider market label, not a guessed exchange MIC + * @param openPrice exact provider open value + * @param highPrice exact provider high value + * @param lowPrice exact provider low value + * @param closePrice exact provider close value + * @param tradingVolume exact share count + * @param tradingValue exact KRW trading value + * @param sourceFields all bounded provider row fields, including unmapped fields + * @param sourcePageNumber page containing the original observation + */ +public record StockPriceRecord(LocalDate referenceDate, String shortCode, String isinCode, + String instrumentName, String marketCategory, BigDecimal openPrice, + BigDecimal highPrice, BigDecimal lowPrice, BigDecimal closePrice, + BigInteger tradingVolume, BigInteger tradingValue, Map sourceFields, + int sourcePageNumber) { + /** Retain an immutable copy of the provider fields. */ + public StockPriceRecord { + sourceFields = Map.copyOf(sourceFields); + } + + /** @return a credential-free identity summary, not the provider field payload */ + @Override + public String toString() { + return "StockPriceRecord[referenceDate=" + referenceDate + ", sourcePageNumber=" + sourcePageNumber + "]"; + } +} diff --git a/scripts/verify_stock_data_source.sh b/scripts/verify_stock_data_source.sh new file mode 100644 index 00000000..3f826b0b --- /dev/null +++ b/scripts/verify_stock_data_source.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +verification_root=$(mktemp -d) +trap 'rm -rf "$verification_root"' EXIT HUP INT TERM +source_root="$repository_root/etl-service/src/main/java/com/xtrmetl/etl/stock_data" +test_root="$repository_root/etl-service/src/test/java/com/xtrmetl/etl/stock_data" +java -version +javac -Xlint:all -Werror -d "$verification_root/classes" \ + "$source_root"/*.java "$test_root/StockDataContractChecks.java" +java -cp "$verification_root/classes" com.xtrmetl.etl.stock_data.StockDataContractChecks +javadoc -quiet -Werror -Xdoclint:all -d "$verification_root/javadoc" "$source_root"/*.java From 72462610962a341a793d925ff0e7d1cfdf99b689 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 14:33:19 +0900 Subject: [PATCH 3/4] fix(stock-data): reject undeclared envelope children and late-cancel success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check interruption after XML decode and before returning a completed batch. Require declared response/header/body children; keep provider fields only on items. Extend unit contracts for cumulative body limits, read/close failures, and exact SHA-256. Refresh the product/technical gap baseline from current evidence. No HTTP client added. Local OpenJDK 21.0.11: sh scripts/verify_stock_data_source.sh — 299 assertions, javac -Werror, javadoc -Werror. --- docs/adr/stock_data_source_boundary.md | 2 +- docs/changes/stock_data_source.md | 2 + docs/doctoring/fsc_stock_data_sources.md | 6 +- docs/product-technical-gap-baseline.md | 54 ++++++++++- docs/stock_data/stock_source_specification.md | 2 +- .../etl/stock_data/FscStockDataSource.java | 16 +-- .../etl/stock_data/StockPageDecoder.java | 45 +++++---- .../stock_data/StockDataContractChecks.java | 97 ++++++++++++++++++- 8 files changed, 187 insertions(+), 37 deletions(-) diff --git a/docs/adr/stock_data_source_boundary.md b/docs/adr/stock_data_source_boundary.md index 3045b2f9..dad00c72 100644 --- a/docs/adr/stock_data_source_boundary.md +++ b/docs/adr/stock_data_source_boundary.md @@ -25,7 +25,7 @@ A complete batch is returned only after every page passes. Raw transfer-decoded Bounds are consumer safety budgets, not claimed provider quotas: 366 inclusive calendar days, 1,000 rows/page, 100 pages, 10,000 records, 2 MiB/page and 16 MiB raw bytes/batch. Exceeding a budget fails the request instead of silently truncating. Large backfills must be partitioned explicitly by the host and retain separate collection receipts. -XML processing denies DTDs, external entities, schemas and XInclude, limits element depth, and suppresses provider/parser diagnostics. Transport errors, including close/suppressed failures, do not expose URLs, keys or payloads. Cancellation is checked before acquisition and after body delivery/read; response ownership is closed on every outcome. +XML processing denies DTDs, external entities, schemas and XInclude, limits element depth, and suppresses provider/parser diagnostics. Structural `response`, `header`, and `body` children must match the declared envelope; unknown or duplicate structural elements fail the page. Provider-defined fields remain allowed only inside `item`. Transport errors, including close/suppressed failures, do not expose URLs, keys or payloads. Cancellation is checked before acquisition, after body delivery/read, after XML decode/validation, and immediately before a completed batch can escape; response ownership is closed on every outcome. ## Ownership and interoperability diff --git a/docs/changes/stock_data_source.md b/docs/changes/stock_data_source.md index 252b6661..4c6bd604 100644 --- a/docs/changes/stock_data_source.md +++ b/docs/changes/stock_data_source.md @@ -4,4 +4,6 @@ Add a Java provider adapter in the existing mightyETL ETL host for bounded FSC s The source requires an explicit approved transport. It is not yet a released/live-provider-verified capability. EgressWeave #246 owns the missing immutable cross-language transport binding. The full primary wire guide, real keyed retrieval, Java 25 reactor, coverage, security and independent review remain release gates. +Hourly fire 2026-09-08: review findings on Draft #333 were verified against `06ccc7f`. Late cancellation after body read can no longer return a completed batch; undeclared structural XML children are rejected; contract checks now cover cumulative batch-byte overflow, read/close failures, and an exact SHA-256 digest. The code-quality comment on `StockBatch.priceRecords()` remains a false positive: the private constructor already stores `List.copyOf`. No HTTP client was added. + This fragment is supplied to canonical documentation PR #149 rather than rewriting its concurrently owned root CHANGELOG/PRD/TRD/README/AGENTS/CLAUDE files. Merge it into the root changelog only with the actual integrated feature and its evidence; do not backdate a release or mark a Proposed ADR Accepted solely because code exists. diff --git a/docs/doctoring/fsc_stock_data_sources.md b/docs/doctoring/fsc_stock_data_sources.md index f095101e..e08474b8 100644 --- a/docs/doctoring/fsc_stock_data_sources.md +++ b/docs/doctoring/fsc_stock_data_sources.md @@ -26,7 +26,9 @@ BigDecimal supports exact decimal representation. The adapter constructs from va | Query/pagination integrity | `collectStockData`, `StockPageDecoder.decodePage` | `verifyInvalidPages`, `verifyQueryRejection` | | Exact source values | `StockPriceRecord`, decoder numeric functions | `verifyCompleteCollection`, `verifyInvalidRecords` | | JAXP external resource limits | `StockPageDecoder.readDocument` | `verifyHostileXml`, `verifyTransportFailures` | +| Declared envelope cardinality | `StockPageDecoder.requireChildren` | extra/duplicate `response`/`header`/`body`/`items` cases in `verifyHostileXml` | | No key/diagnostic export | `fetchBody`, request/response formatting, finite exception | `verifyTransportFailures`, `verifyLateCancellationAndSafeFormatting` | -| Response lifecycle | `PageResponse.close`, `fetchBody` | rejected/oversized/cancelled-body close assertions | +| Response lifecycle | `PageResponse.close`, `fetchBody` | rejected/oversized/cancelled-body close assertions; read/close and close-with-primary failures | +| Late cancellation | `requireNotCancelled` after decode and before batch return | Clock.instant() interrupt in `verifyLateCancellationAndSafeFormatting` | -The local compiler is OpenJDK 21.0.11. The focused suite reached 245 passing assertions, and warning-as-error compilation and Javadoc passed. This is not 245 independent JUnit test methods and not a 100% coverage measurement. The repository's Java 25 Maven reactor was not run locally; its full CI and security/review gates remain mandatory. +The local compiler is OpenJDK 21.0.11. After the 2026-09-08 review repairs, `sh scripts/verify_stock_data_source.sh` passed 299 synthetic assertions plus `javac -Xlint:all -Werror` and `javadoc -Werror -Xdoclint:all`. This is not 299 independent JUnit methods and not a 100% coverage measurement. The repository's Java 25 Maven reactor, security checks, independent review and immutable release remain mandatory. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d317780c..734c633f 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,48 @@ # Product / technical gap baseline -Scope: stock-data source acquisition delta only, observed 2026-09-07. This is not a complete mightyETL readiness assessment. Protected baseline: `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb`; canonical broader documentation remains in open Draft PR #149. Open work is candidate/proposed, not shipped truth. +Observed 2026-09-08. Protected baseline: `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb`. Canonical broader documentation remains in open Draft PR #149. Open work is candidate/proposed, not shipped truth. Keep Product Requirements Document names and case: Change Data Capture (CDC), Extract-Transform-Load (ETL), JWT, Eureka, Config Server. + +## PRD vs evidence + +| PRD capability | Current evidence | Remaining gap | +|---|---|---| +| Change Data Capture from PostgreSQL | Live path Postgres→Kafka; slot probe, replica allow-list, ops docs | Multi-source Debezium engines; MySQL/SQL Server are discovery scaffolds | +| Extract-Transform-Load JSON pipeline | Bounded batches, idempotent retries, RFC 9457 errors, Flyway schema authority | Durable job worker/replay/cancellation still stacked as drafts; amount integrity refresh is draft #316 | +| JWT authentication and RBAC | Gateway placeholder-token repair is draft #142; ETL JWT fail-closed is draft #287 | Production JWT resource-server path not on `develop` | +| Qlik Sense / Databricks / Snowflake | SPI + YAML + required-key validation + catalog; writes refused | Live SaaS loaders need credentials and real clients | +| Any-to-any CDC | Source/target SPI; live Postgres→Kafka only | Do not market warehouse loaders as supported | +| Stock observations (this fire) | Candidate `FscStockDataSource` on Draft #333 | No released transport; no live keyed FSC retrieval | + +## TRD / UML + +Ports remain: Gateway 8080, ETL 8000, CDC 8001, Eureka 8761, Zipkin 9412. Java 25 / Spring Boot 3.5.9 / Debezium 3.4.0.Final. UML for stock collection is the sequence in `docs/stock_data/stock_source_specification.md`; the transport participant is an owner port, not a shipped service. Broader architecture diagrams stay with Draft #149. + +## Connector / owner linkage + +Chicken-and-egg is broken with a minimum port, not a consumer HTTP clone. mightyETL owns `StockDataTransport` and FSC field ACL. [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246) owns destination authorization, TLS, credentials, deadlines, rate limits, and the missing released cross-language binding. OriginWeave HTTP adapters remain unshipped. No default Java HTTP, curl, or Python client is added in this consumer. + +Warehouse connectors stay scaffolds. CDC registry snapshot immutability is issue #246 in this repo (distinct from EgressWeave #246). + +## Open actions (2026-09-08) + +Non-draft PRs targeting `develop` are all `mergeable_state=blocked`; none merged this fire. + +| ID | State | Note | +|---|---|---| +| PR #333 | draft | Stock-data candidate; this fire repairs verified review findings | +| PR #330 | blocked | Reusable dependency-review caller; waits on ContextualWisdomLab/.github#1724 | +| PR #328 | needs-review | Config Server authority successor of #327/#322; not merged, so predecessors stay open | +| PR #327 / #322 | needs-review | Do not merge; #328 claims succession but is not protected-integrated | +| PR #326 | needs-review | Hourly central PR maintenance; stale base `d6c6665` | +| PR #321 | needs-review | Replication-probe confidentiality; stale base, checks from 2026-08-15 | +| PR #329 | draft | CDC semantic identifiers | +| Draft stack #254/#256 and older durable-job PRs | draft | Stacked on non-`develop` bases; restack later, do not close | +| Issue #247 | open | ETL request-size limits before full body materialization | +| Issue #252 | open | Fail closed before protected merges on non-qualifying evidence | + +Displayed closed is not done. No PR was closed this fire. + +## Stock-data candidate (Draft #333) | Buyer gap | Candidate action | Evidence / remaining gate | |---|---|---| @@ -8,11 +50,13 @@ Scope: stock-data source acquisition delta only, observed 2026-09-07. This is no | A partial history can look complete | Validate page identity, total, row count and duplicate date/ISIN; fail whole call | Focused synthetic unit tests; no upstream snapshot-isolation claim | | Lost leading zeroes or numeric precision | Strings, `BigDecimal`, `BigInteger`; retain raw pages | Leading-zero, fractional-price and >2^53 assertions | | Delayed/empty data confused with live trading | `delayed_daily`, `empty_source_result`, source date separate from collection time | Official FSC portal notice; no market-calendar inference | -| No verified released cross-language HTTP authority | Explicit transport port, no automatic network implementation | [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246): owner runtime, release and consumer conformance required | +| Malformed provider envelope accepted | Reject undeclared/duplicate structural XML children | Review P1 on #333; envelope tests added this fire | +| Late cancel returned success | Check interrupt after decode and before batch return | Clock.instant() interrupt contract added this fire | +| No verified released cross-language HTTP authority | Explicit transport port, no automatic network implementation | EgressWeave #246: owner runtime, release and consumer conformance required | | Provider wire/profile not fully verified | Bound the candidate mapping and preserve unknown fields/raw bytes | Official portal inspected; full primary guide and actual keyed known-day retrieval still required | -| Test/release acceptance incomplete | Existing JUnit entrypoint; warning-free local compile and Javadoc | Local Java 21 subset only; full Java 25 Maven, coverage, security, review, immutable release not proven | +| Test/release acceptance incomplete | Existing JUnit entrypoint; warning-free local compile and Javadoc | Local Java 21 subset; full Java 25 Maven, coverage, security, review, immutable release not proven | | Durable stock store / revision history absent | Do not mutate generic `processed_data` or create cross-service SQL | Market-data domain owner, archive/revision API, migrations and real DB tests remain separate work | -Source documentation: [ADR](adr/stock_data_source_boundary.md), [PRD/TRD/API/UML slice](stock_data/stock_source_specification.md), [doctoring](doctoring/fsc_stock_data_sources.md), [change fragment](changes/stock_data_source.md), [implementation plan](superpowers/plans/2026-09-07-stock-data-source.md). +Source documentation: [ADR](adr/stock_data_source_boundary.md) (Proposed, not Accepted), [PRD/TRD/API/UML slice](stock_data/stock_source_specification.md), [doctoring](doctoring/fsc_stock_data_sources.md), [change fragment](changes/stock_data_source.md), [implementation plan](superpowers/plans/2026-09-07-stock-data-source.md). -No release, live data capture, new schedule, independent approval or 100% coverage is asserted by this document. No existing PR was closed, superseded, force-pushed or stripped of valid delta for this feature. +This fire does not mark the ADR Accepted, claim UI completeness, force-push, or advertise live stock crawling. No existing PR was closed, superseded, or stripped of valid delta. diff --git a/docs/stock_data/stock_source_specification.md b/docs/stock_data/stock_source_specification.md index 37179611..31f33dd1 100644 --- a/docs/stock_data/stock_source_specification.md +++ b/docs/stock_data/stock_source_specification.md @@ -40,7 +40,7 @@ Public page parameters use `basDt` for one day or `beginBasDt`/`endBasDt` for a | `trqu`, `trPrc` | `BigInteger` volume/value | exact integers, including values above 2^53 | | remaining row fields | immutable `sourceFields` | bounded XML text with surrounding whitespace stripped; exact bytes remain in raw pages | -Every batch retains the request, immutable observations and raw pages. Each raw page has `pageNumber`, `collectedAt`, exact bytes and their SHA-256. Preserve those bytes in an authorized archive before discarding the batch when durable replay is required. Returning a digest does not itself create durable storage or lineage publication. No request URL with `serviceKey` enters this evidence. +Every batch retains the request, immutable observations and raw pages. Each raw page has `pageNumber`, `collectedAt`, exact bytes and their SHA-256. Preserve those bytes in an authorized archive before discarding the batch when durable replay is required. Returning a digest does not itself create durable storage or lineage publication. No request URL with `serviceKey` enters this evidence. Undeclared or duplicate children in `response`, `header`, or `body` fail the page; `item` fields may still carry provider-defined names. Cancellation after body read, after decode, or immediately before returning a batch yields `cancelled` rather than a completed result. ## Failure and operating behavior diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java index e62b5206..7dcadcfa 100644 --- a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/FscStockDataSource.java @@ -73,6 +73,7 @@ public StockBatch collectStockData(StockQuery sourceQuery) { } Instant collectedAt = observationClock.instant(); var decodedPage = StockPageDecoder.decodePage(rawBody, sourceQuery, pageNumber); + requireNotCancelled(); if (expectedTotal == -1) { expectedTotal = decodedPage.totalCount(); if (expectedTotal > sourceQuery.maximumRecords() @@ -93,6 +94,7 @@ public StockBatch collectStockData(StockQuery sourceQuery) { } rawPages.add(new RawStockPage(pageNumber, collectedAt, rawBody)); if (priceRecords.size() == expectedTotal) { + requireNotCancelled(); return new StockBatch(sourceQuery, priceRecords, rawPages); } } @@ -116,10 +118,14 @@ private StockDataTransport.PageRequest pageRequest(StockQuery sourceQuery, int p return new StockDataTransport.PageRequest(SOURCE_ENDPOINT, publicParameters, credentialReference, pageNumber); } - private byte[] fetchBody(StockDataTransport.PageRequest pageRequest) { + private static void requireNotCancelled() { if (Thread.currentThread().isInterrupted()) { throw new StockDataException("cancelled"); } + } + + private byte[] fetchBody(StockDataTransport.PageRequest pageRequest) { + requireNotCancelled(); StockDataTransport.PageResponse pageResponse; try { pageResponse = sourceTransport.fetchPage(pageRequest); @@ -133,9 +139,7 @@ private byte[] fetchBody(StockDataTransport.PageRequest pageRequest) { throw new StockDataException("transport_failure"); } try (pageResponse) { - if (Thread.currentThread().isInterrupted()) { - throw new StockDataException("cancelled"); - } + requireNotCancelled(); if (pageResponse.statusCode() == 429) { throw new StockDataException("rate_limited"); } @@ -147,9 +151,7 @@ private byte[] fetchBody(StockDataTransport.PageRequest pageRequest) { if (rawBody.length > MAX_PAGE_BYTES) { throw new StockDataException("body_too_large"); } - if (Thread.currentThread().isInterrupted()) { - throw new StockDataException("cancelled"); - } + requireNotCancelled(); return rawBody; } catch (StockDataException failureValue) { // Try-with-resources can attach a credential-bearing close failure. diff --git a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java index ebc715a1..23428a93 100644 --- a/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java +++ b/etl-service/src/main/java/com/xtrmetl/etl/stock_data/StockPageDecoder.java @@ -13,6 +13,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilderFactory; import org.w3c.dom.Element; @@ -30,19 +31,23 @@ private StockPageDecoder() { } static DecodedPage decodePage(byte[] rawBody, FscStockDataSource.StockQuery sourceQuery, int pageNumber) { Element rootElement = readDocument(rawBody); requireName(rootElement, "response"); - Element headerElement = onlyChild(rootElement, "header"); - if (!"00".equals(textValue(onlyChild(headerElement, "resultCode")))) { + Map responseChildren = requireChildren(rootElement, Set.of("header", "body"), Set.of()); + Element headerElement = responseChildren.get("header"); + Map headerChildren = requireChildren(headerElement, Set.of("resultCode"), Set.of("resultMsg")); + if (!"00".equals(textValue(headerChildren.get("resultCode")))) { throw new StockDataException("provider_rejected"); } - Element bodyElement = onlyChild(rootElement, "body"); - int responsePage = integerValue(textValue(onlyChild(bodyElement, "pageNo"))); - int responseSize = integerValue(textValue(onlyChild(bodyElement, "numOfRows"))); - int totalCount = integerValue(textValue(onlyChild(bodyElement, "totalCount"))); + Element bodyElement = responseChildren.get("body"); + Map bodyChildren = requireChildren( + bodyElement, Set.of("pageNo", "numOfRows", "totalCount", "items"), Set.of()); + int responsePage = integerValue(textValue(bodyChildren.get("pageNo"))); + int responseSize = integerValue(textValue(bodyChildren.get("numOfRows"))); + int totalCount = integerValue(textValue(bodyChildren.get("totalCount"))); if (responsePage != pageNumber || responseSize != sourceQuery.pageSize()) { throw new StockDataException("invalid_page"); } List priceRecords = new ArrayList<>(); - Element itemsElement = onlyChild(bodyElement, "items"); + Element itemsElement = bodyChildren.get("items"); for (Element itemElement : elementChildren(itemsElement)) { requireName(itemElement, "item"); if (priceRecords.size() >= sourceQuery.pageSize()) { @@ -169,21 +174,25 @@ private static int integerValue(String sourceValue) { } } - private static Element onlyChild(Element parentElement, String childName) { - Element selectedElement = null; + private static Map requireChildren( + Element parentElement, Set requiredNames, Set optionalNames) { + Map selectedChildren = new LinkedHashMap<>(); for (Element childElement : elementChildren(parentElement)) { - if (childName.equals(childElement.getTagName())) { - requirePlainElement(childElement); - if (selectedElement != null) { - throw new StockDataException("invalid_xml"); - } - selectedElement = childElement; + requirePlainElement(childElement); + String childName = childElement.getTagName(); + if (!requiredNames.contains(childName) && !optionalNames.contains(childName)) { + throw new StockDataException("invalid_xml"); + } + if (selectedChildren.putIfAbsent(childName, childElement) != null) { + throw new StockDataException("invalid_xml"); } } - if (selectedElement == null) { - throw new StockDataException("invalid_xml"); + for (String requiredName : requiredNames) { + if (!selectedChildren.containsKey(requiredName)) { + throw new StockDataException("invalid_xml"); + } } - return selectedElement; + return selectedChildren; } private static List elementChildren(Element parentElement) { diff --git a/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java b/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java index 43b5c0a7..dde11a24 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/stock_data/StockDataContractChecks.java @@ -5,11 +5,14 @@ import java.io.InputStream; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.time.Clock; import java.time.Instant; import java.time.LocalDate; +import java.time.ZoneId; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.HexFormat; import java.util.List; import java.util.Map; @@ -66,6 +69,7 @@ private static void verifyCompleteCollection() { requireEqual(FIXED_CLOCK.instant(), sourceBatch.rawPages().get(0).collectedAt(), "observation time preserved"); requireEqual(firstBody, new String(sourceBatch.rawPages().get(0).rawBody(), StandardCharsets.UTF_8), "raw body preserved"); requireEqual(64, sourceBatch.rawPages().get(0).sha256Digest().length(), "digest exists"); + requireEqual(knownSha256(firstBody), sourceBatch.rawPages().get(0).sha256Digest(), "digest matches known bytes"); requireEqual("provider_unspecified", sourceBatch.adjustmentBasis(), "adjustment not invented"); requireEqual("delayed_daily", sourceBatch.freshnessClass(), "not realtime"); requireEqual("xml", observedRequests.get(0).publicParameters().get("resultType"), "XML selected"); @@ -127,7 +131,7 @@ private static void verifyInvalidPages() { expectCode("duplicate_record", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem), pageXml(2, 1, 2, validItem)), new ArrayList<>()).collectStockData(sourceQuery(1))); expectCode("incomplete_result", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem)), new ArrayList<>()).collectStockData(new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 1, 1, 2))); expectCode("incomplete_result", () -> sourceFor(List.of(pageXml(1, 1, 2, validItem)), new ArrayList<>()).collectStockData(new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 1, 2, 1))); - expectCode("provider_rejected", () -> sourceFor(List.of("
30secret
"), new ArrayList<>()).collectStockData(sourceQuery(10))); + expectCode("provider_rejected", () -> sourceFor(List.of("
30secret
1010"), new ArrayList<>()).collectStockData(sourceQuery(10))); } private static void verifyInvalidRecords() { @@ -149,11 +153,18 @@ private static void verifyInvalidRecords() { } private static void verifyHostileXml() { + String validPage = pageXml(1, 10, 0, ""); for (String hostileBody : List.of("login", "", "]>&payload;", "
", - "" + "".repeat(40) + "".repeat(40) + "")) { - expectFailure(() -> sourceFor(List.of(hostileBody), new ArrayList<>()).collectStockData(sourceQuery(10))); + "" + "".repeat(40) + "".repeat(40) + "", + validPage.replace("", ""), + validPage.replace("
", "x
"), + validPage.replace("", ""), + validPage.replace("
", "
"), + validPage.replace("", ""), + validPage.replace("", ""))) { + expectCode("invalid_xml", () -> sourceFor(List.of(hostileBody), new ArrayList<>()).collectStockData(sourceQuery(10))); } } @@ -188,12 +199,45 @@ private static void verifyLateCancellationAndSafeFormatting() { requireEqual(true, Thread.interrupted(), "late cancellation remains signalled"); } requireEqual(true, closedBody[0], "late cancelled body closed"); + boolean[] closedAfterRead = {false}; + try { + expectCode("cancelled", () -> new FscStockDataSource( + pageRequest -> new StockDataTransport.PageResponse( + 200, "application/xml", trackedBody(pageXml(1, 10, 0, ""), closedAfterRead)), + "fsc_stock_key", + interruptingClock()).collectStockData(sourceQuery(10))); + } finally { + requireEqual(true, Thread.interrupted(), "post-read cancellation remains signalled"); + } + requireEqual(true, closedAfterRead[0], "post-read cancelled body closed"); } private static void verifyResourceBounds() { boolean[] closedBody = {false}; expectCode("body_too_large", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(200, "application/xml", trackedBody(" ".repeat(2 * 1024 * 1024 + 1), closedBody)), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); requireEqual(true, closedBody[0], "oversized body closed"); + List cumulativePages = new ArrayList<>(); + int pageBytes = 2 * 1024 * 1024; + for (int pageNumber = 1; pageNumber <= 8; pageNumber++) { + cumulativePages.add(paddedPageXml(pageNumber, 1, 9, uniqueItem(pageNumber), pageBytes)); + } + cumulativePages.add(paddedPageXml(9, 1, 9, uniqueItem(9), 16)); + expectCode("body_too_large", () -> sourceFor(cumulativePages, new ArrayList<>()) + .collectStockData(new FscStockDataSource.StockQuery(SOURCE_DATE, SOURCE_DATE, null, 1, 100, 10000))); + boolean[] closedRead = {false}; + expectCode("transport_failure", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse(200, "application/xml", new InputStream() { + @Override public int read() throws IOException { throw new IOException("secret"); } + @Override public void close() { closedRead[0] = true; } + }), "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + requireEqual(true, closedRead[0], "failed read body closed"); + expectCode("transport_failure", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse( + 200, "application/xml", throwingCloseBody(pageXml(1, 10, 0, ""))), + "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + boolean[] closedBoth = {false}; + expectCode("provider_rejected", () -> new FscStockDataSource(requestValue -> new StockDataTransport.PageResponse( + 401, "application/xml", throwingCloseBody("secret", closedBoth)), + "fsc_stock_key", FIXED_CLOCK).collectStockData(sourceQuery(10))); + requireEqual(true, closedBoth[0], "primary and close failures still close"); } private static InputStream trackedBody(String bodyText, boolean[] closedBody) { @@ -202,6 +246,53 @@ private static InputStream trackedBody(String bodyText, boolean[] closedBody) { }; } + private static InputStream throwingCloseBody(String bodyText) { + return throwingCloseBody(bodyText, null); + } + + private static InputStream throwingCloseBody(String bodyText, boolean[] closedBody) { + return new ByteArrayInputStream(bodyText.getBytes(StandardCharsets.UTF_8)) { + @Override public void close() throws IOException { + if (closedBody != null) { + closedBody[0] = true; + } + throw new IOException("secret"); + } + }; + } + + private static Clock interruptingClock() { + return new Clock() { + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return Clock.fixed(FIXED_CLOCK.instant(), zone); } + @Override public Instant instant() { + Thread.currentThread().interrupt(); + return FIXED_CLOCK.instant(); + } + }; + } + + private static String uniqueItem(int itemIndex) { + return itemXml("005930", String.format("KR7%08d0", itemIndex), "20260904"); + } + + private static String paddedPageXml(int pageNumber, int pageSize, int totalCount, String itemContent, int minimumBytes) { + String xml = pageXml(pageNumber, pageSize, totalCount, itemContent); + if (xml.length() >= minimumBytes) { + return xml; + } + return "" + " ".repeat(minimumBytes - xml.length()) + xml.substring("".length()); + } + + private static String knownSha256(String bodyText) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(bodyText.getBytes(StandardCharsets.UTF_8))); + } catch (Exception failureValue) { + throw new AssertionError("SHA-256 unavailable", failureValue); + } + } + private static String pageXml(int pageNumber, int pageSize, int totalCount, String itemContent) { return "
00NORMAL SERVICE.
" + pageSize + "" + pageNumber + "" + totalCount + "" + itemContent + ""; } From 6c6b89282fba7468666032c98a1efc81d0b45482 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 16:37:41 +0900 Subject: [PATCH 4/4] docs(gap): refresh product-technical baseline for the hourly fire Record Draft #334 as the job-intake transport successor for issue #247, keep #333 as the stock-data candidate, and note that non-draft PRs remain blocked on review rather than product-check failures. --- docs/product-technical-gap-baseline.md | 29 +++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 734c633f..0ad8bae5 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,43 +1,44 @@ # Product / technical gap baseline -Observed 2026-09-08. Protected baseline: `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb`. Canonical broader documentation remains in open Draft PR #149. Open work is candidate/proposed, not shipped truth. Keep Product Requirements Document names and case: Change Data Capture (CDC), Extract-Transform-Load (ETL), JWT, Eureka, Config Server. +Observed 2026-09-08 (hourly fire). Protected baseline: `develop@e550688c80f0dcf4677c0fbe50bd3341429106fb`. Canonical broader documentation remains in open Draft PR #149. Open work is candidate/proposed, not shipped truth. Keep Product Requirements Document names and case: Change Data Capture (CDC), Extract-Transform-Load (ETL), JWT, Eureka, Config Server. ## PRD vs evidence | PRD capability | Current evidence | Remaining gap | |---|---|---| | Change Data Capture from PostgreSQL | Live path Postgres→Kafka; slot probe, replica allow-list, ops docs | Multi-source Debezium engines; MySQL/SQL Server are discovery scaffolds | -| Extract-Transform-Load JSON pipeline | Bounded batches, idempotent retries, RFC 9457 errors, Flyway schema authority | Durable job worker/replay/cancellation still stacked as drafts; amount integrity refresh is draft #316 | +| Extract-Transform-Load JSON pipeline | Bounded batches, idempotent retries, RFC 9457 errors, Flyway schema authority; process-path transport admission is on `develop` | Durable job worker/replay/cancellation still stacked as drafts; amount integrity refresh is draft #316; job-intake transport admission is Draft #334, not protected | | JWT authentication and RBAC | Gateway placeholder-token repair is draft #142; ETL JWT fail-closed is draft #287 | Production JWT resource-server path not on `develop` | | Qlik Sense / Databricks / Snowflake | SPI + YAML + required-key validation + catalog; writes refused | Live SaaS loaders need credentials and real clients | | Any-to-any CDC | Source/target SPI; live Postgres→Kafka only | Do not market warehouse loaders as supported | -| Stock observations (this fire) | Candidate `FscStockDataSource` on Draft #333 | No released transport; no live keyed FSC retrieval | +| Stock observations | Candidate `FscStockDataSource` on Draft #333 | No released transport; no live keyed FSC retrieval | ## TRD / UML -Ports remain: Gateway 8080, ETL 8000, CDC 8001, Eureka 8761, Zipkin 9412. Java 25 / Spring Boot 3.5.9 / Debezium 3.4.0.Final. UML for stock collection is the sequence in `docs/stock_data/stock_source_specification.md`; the transport participant is an owner port, not a shipped service. Broader architecture diagrams stay with Draft #149. +Ports remain: Gateway 8080, ETL 8000, CDC 8001, Eureka 8761, Zipkin 9412. Root `pom.xml` on `develop` pins Java 25, Spring Boot 3.5.16, Spring Cloud 2025.0.3, Debezium 3.4.0.Final. `TRD.md` still names Spring Boot 3.5.9 / Spring Cloud 2025.0.1; that document lag is a remaining gap, not a license to claim the older matrix. UML for stock collection is the sequence in `docs/stock_data/stock_source_specification.md`; the transport participant is an owner port, not a shipped service. Broader architecture diagrams stay with Draft #149. ## Connector / owner linkage -Chicken-and-egg is broken with a minimum port, not a consumer HTTP clone. mightyETL owns `StockDataTransport` and FSC field ACL. [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246) owns destination authorization, TLS, credentials, deadlines, rate limits, and the missing released cross-language binding. OriginWeave HTTP adapters remain unshipped. No default Java HTTP, curl, or Python client is added in this consumer. +Chicken-and-egg is broken with a minimum port, not a consumer HTTP clone. mightyETL owns `StockDataTransport` and FSC field ACL. [EgressWeave #246](https://github.com/ContextualWisdomLab/EgressWeave/issues/246) remains OPEN and owns destination authorization, TLS, credentials, deadlines, rate limits, and the missing released cross-language binding. OriginWeave HTTP adapters remain unshipped. No default Java HTTP, curl, or Python client is added in this consumer. Warehouse connectors stay scaffolds. CDC registry snapshot immutability is issue #246 in this repo (distinct from EgressWeave #246). -## Open actions (2026-09-08) +## Open actions (2026-09-08 hourly) -Non-draft PRs targeting `develop` are all `mergeable_state=blocked`; none merged this fire. +Non-draft PRs targeting `develop` remain `mergeStateStatus=BLOCKED`; none merged this fire. CodeQL compatibility jobs on #333 failed with `VERDICT_STATE=pending` after a successful dispatch (handshake, not a product defect). Do not stall the loop on that rerun. | ID | State | Note | |---|---|---| -| PR #333 | draft | Stock-data candidate; this fire repairs verified review findings | +| PR #334 | draft | New this fire: job-intake transport admission for #247; local Temurin 25 `EtlJob*` BUILD SUCCESS | +| PR #333 | draft | Stock-data candidate at `72462610`; prior fire repaired envelope/cancel findings; leave Draft | | PR #330 | blocked | Reusable dependency-review caller; waits on ContextualWisdomLab/.github#1724 | -| PR #328 | needs-review | Config Server authority successor of #327/#322; not merged, so predecessors stay open | -| PR #327 / #322 | needs-review | Do not merge; #328 claims succession but is not protected-integrated | -| PR #326 | needs-review | Hourly central PR maintenance; stale base `d6c6665` | -| PR #321 | needs-review | Replication-probe confidentiality; stale base, checks from 2026-08-15 | +| PR #328 | needs-review | Config Server authority successor of #327/#322; Strix failed; not merged, so predecessors stay open | +| PR #327 / #322 | needs-review | CHANGES_REQUESTED; do not merge or close while #328 is not protected-integrated | +| PR #326 | needs-review | Hourly central PR maintenance; stale base `d6c6665`; Scorecard failed | +| PR #321 | needs-review | Replication-probe confidentiality; all listed checks SUCCESS, still blocked on review; one CodeRabbit minor on log assertion | | PR #329 | draft | CDC semantic identifiers | | Draft stack #254/#256 and older durable-job PRs | draft | Stacked on non-`develop` bases; restack later, do not close | -| Issue #247 | open | ETL request-size limits before full body materialization | +| Issue #247 | open | Process-path admission is on `develop`; jobs path successor is Draft #334 | | Issue #252 | open | Fail closed before protected merges on non-qualifying evidence | Displayed closed is not done. No PR was closed this fire. @@ -59,4 +60,4 @@ Displayed closed is not done. No PR was closed this fire. Source documentation: [ADR](adr/stock_data_source_boundary.md) (Proposed, not Accepted), [PRD/TRD/API/UML slice](stock_data/stock_source_specification.md), [doctoring](doctoring/fsc_stock_data_sources.md), [change fragment](changes/stock_data_source.md), [implementation plan](superpowers/plans/2026-09-07-stock-data-source.md). -This fire does not mark the ADR Accepted, claim UI completeness, force-push, or advertise live stock crawling. No existing PR was closed, superseded, or stripped of valid delta. +This fire does not mark the ADR Accepted, claim UI completeness, force-push, or advertise live stock crawling. No existing PR was closed, superseded, or stripped of valid delta. Draft #334 is an independent Extract-Transform-Load availability successor, not a replacement for #333.