From 600808181d99e3f564c108389b7df644360e9dc1 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Mon, 27 Jul 2026 18:53:50 -0400 Subject: [PATCH] Fix stale databento.py docstring; derive _DATABENTO_UNSUPPORTED from the registry The module docstring still said "DORMANT. NOT in the live EOD path", but the file now hosts the live ADR-0006 two-stage producer (ingest/build) that update.py drives. Rewrote it to describe both parts: the LIVE producer (databento as a validated alternative to Norgate) and the genuinely dormant older EOD helpers kept for the intraday work. _DATABENTO_UNSUPPORTED was a hardcoded set (CC/OJ/SB/KC/LBR/CT) that duplicated and had drifted from the registry's authoritative `databento: null` mapping (it missed DX, MME, MFS). Derive it from the registry so it can't go stale. Co-Authored-By: Claude Opus 4.8 --- src/cotdata/providers/databento.py | 49 ++++++++++++++++++------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/cotdata/providers/databento.py b/src/cotdata/providers/databento.py index a9adddc..87c8aa6 100644 --- a/src/cotdata/providers/databento.py +++ b/src/cotdata/providers/databento.py @@ -1,20 +1,29 @@ -"""Databento provider — DORMANT. NOT in the live EOD path (Norgate replaced it). - -Retained deliberately for: - 1. the intraday news-failure work (Norgate has no intraday; Databento is the - source for the release-window reaction refinement of the CMR trigger), and - 2. cross-checking Norgate's settlement close against Databento statistics. - -Ported from cot-analyzer/src/core/market_data.py (GLBX.MDP3 ohlcv-1d + statistics). -The hard-won part preserved here is the STATISTICS extraction — Open Interest -(stat_type 9) and the settlement fix: use StatType.SETTLEMENT_PRICE == 3 (NOT 7 = -LOWEST_OFFER, which overwrote Close with the day's lowest offer), dated by ts_ref -(the session it applies to) not ts_event (final settle is disseminated next morning). - -Standalone-adapted for cotdata: lazy `import databento` (behind the [databento] -extra), symbols from the registry, cache under $COTDATA_STORE/_cache/databento. -Requires DATABENTO_API_KEY. The daily path below is superseded by Norgate; the -intraday work will reuse this statistics logic against ohlcv-1h / trades schemas. +"""Databento price provider for cotdata (GLBX.MDP3). Two parts live in this file. + +LIVE, the ADR-0006 two-stage producer (an alternative to Norgate, selected per deployment +via COTDATA_PRICE_SOURCE): + * ingest() fetches raw .n.0/.n.1 ohlcv-1d + statistics into an append-only raw bronze + store ($COTDATA_DATABENTO_RAW). The only paid step, resumable, paged by year. + * build() reads only the raw store and writes back-adjusted prices to the cotdata store. + Free and re-runnable. +Driven by `cotdata-update --ingest-databento` / `--build-databento`. databento is a +validated ALTERNATIVE provider (ADR-0006 Accepted), producing provider-different series +from Norgate, so it is not the default source. See crucible-stack ADR-0006 and +docs/databento_norgate_parity.md. + +DORMANT, an older per-symbol EOD path (fetch_daily_ohlc, run_batch_backfill, +update_all_daily_prices) that is on NO live path. Kept deliberately for (1) the intraday +news-failure work (Norgate has no intraday, so databento would source the release-window +reaction refinement of the CMR trigger) and (2) cross-checking Norgate's settlement close +against databento statistics. Remove it if that intraday work is abandoned. + +The hard-won piece both parts share is the STATISTICS extraction: Open Interest is +stat_type 9, and settlement is StatType.SETTLEMENT_PRICE == 3 (NOT 7 = LOWEST_OFFER, which +overwrote Close with the day's lowest offer), dated by ts_ref (the session it applies to), +not ts_event (the final settle is disseminated the next morning). + +Lazy `import databento` (behind the [databento] extra), symbols from the registry, +requires DATABENTO_API_KEY for the paid fetches. """ import datetime as dt import json @@ -33,8 +42,10 @@ logger = logging.getLogger(__name__) -# Symbols Databento GLBX.MDP3 doesn't carry as .n.0 continuous (fall back to yfinance). -_DATABENTO_UNSUPPORTED = {"CC", "OJ", "SB", "KC", "LBR", "CT"} +# Symbols Databento GLBX.MDP3 doesn't carry as a .n.0 continuous (fall back to yfinance). +# Derived from the registry (`databento: null`), the authoritative capability mapping, so +# it can't drift from it the way the old hardcoded list did (it missed DX/MME/MFS). +_DATABENTO_UNSUPPORTED = frozenset(s.internal for s in all_symbols() if s.databento is None) _API_LAST_CHECKED = {} # in-memory throttle to avoid hammering the API