diff --git a/executor/eod_reconcile.py b/executor/eod_reconcile.py index 0cc1d38..9e9542f 100644 --- a/executor/eod_reconcile.py +++ b/executor/eod_reconcile.py @@ -563,12 +563,15 @@ def run(run_date: str | None = None) -> None: # Hard-fails on any miss: EOD reconcile must reconcile against an # authoritative price source, not IB Gateway's delayed intraday data. # - # Macro-routed held positions (SPY/sector ETFs/etc.) live in the - # `macro` library, NOT `universe`. The portfolio-optimizer cutover + # Macro-routed held positions (sector ETFs / VIX / TNX / etc.) live in + # the `macro` library, NOT `universe`. The portfolio-optimizer cutover # (2026-05-13) made SPY a held core position; its first EOD on # 2026-05-14 raised NoSuchVersionException because reconcile was - # universe-only. Mirror price_cache.load_price_histories' macro-aware - # dispatch (executor/price_cache.py:128-145, _MACRO_SYMBOLS). + # universe-only. SPY-as-held is now read from `universe` directly + # (alpha-engine-data #245 lifted SPY to a full universe member via + # `_UNIVERSE_EXTRA`); only the remaining macro-only-Close symbols still + # need the macro-lib dispatch. Mirror `price_cache.load_price_histories` + # (executor/price_cache.py, `_MACRO_SYMBOLS`). from executor.price_cache import ( _open_universe_library, _open_macro_library, @@ -579,13 +582,8 @@ def run(run_date: str | None = None) -> None: target_ts = pd.Timestamp(run_date).normalize() closing_prices: dict[str, float] = {} missing: list[str] = [] - # L1346 (c) second-half routing post-#245: SPY removed from macro-routed - # set since universe.SPY now carries full OHLCV. Defensive macro fallback - # for SPY preserves backwards compat — mirrors price_cache.py + predictor - # #196 pattern. - _MACRO_SYMBOLS_NO_OHLCV = _MACRO_SYMBOLS - {"SPY"} for ticker in positions.keys(): - if ticker in _MACRO_SYMBOLS_NO_OHLCV: + if ticker in _MACRO_SYMBOLS: if macro_lib is None: macro_lib = _open_macro_library(trades_bucket) lib = macro_lib @@ -594,22 +592,8 @@ def run(run_date: str | None = None) -> None: try: df = lib.read(ticker).data except Exception as e: - # SPY-specific defensive fallback to macro.SPY if universe.SPY - # unreadable. Removed once L1346 (b)+(c) soak clean ≥1 cycle. - if ticker == "SPY" and lib is universe_lib: - if macro_lib is None: - macro_lib = _open_macro_library(trades_bucket) - try: - df = macro_lib.read(ticker).data - except Exception as e2: - missing.append( - f"{ticker} (universe={e.__class__.__name__}, " - f"macro={e2.__class__.__name__})" - ) - continue - else: - missing.append(f"{ticker} ({e.__class__.__name__})") - continue + missing.append(f"{ticker} ({e.__class__.__name__})") + continue if df.empty or "Close" not in df.columns: missing.append(f"{ticker} (no Close column)") continue diff --git a/executor/main.py b/executor/main.py index 16b26ce..a099461 100644 --- a/executor/main.py +++ b/executor/main.py @@ -1197,33 +1197,28 @@ def run( # missing ticker or stale data — feedback_hard_fail_until_stable. # Scope: signal tickers (ENTER) + held positions (for trailing stops). # - # Macro-routed tickers (sector ETFs/VIX/TNX/etc., see _MACRO_SYMBOLS) - # are intentionally excluded. They're used for sector-relative exit - # veto via price_histories, not ATR-based execution, and they live in - # the Close-only `macro` ArcticDB library which has no atr_14_pct - # feature column. + # Macro-routed tickers (sector ETFs / VIX / TNX / etc., see + # _MACRO_SYMBOLS) are intentionally excluded. They live in the + # Close-only `macro` ArcticDB library which has no atr_14_pct + # feature column and are used for sector-relative exit veto via + # price_histories, not ATR-based execution. # - # SPY is NO LONGER excluded as of L1346 (c) (2026-05-24): alpha-engine - # -data #245 promoted SPY to a full ``universe`` ArcticDB member - # (_UNIVERSE_EXTRA = frozenset({"SPY"}), written by both - # builders/backfill.py and builders/daily_append.py), so SPY now has - # the full OHLCV + atr_14_pct feature column that load_atr_14_pct - # needs. The pre-L1346 #185 fix excluded SPY because the macro lib - # was Close-only; with universe.SPY now carrying ATR, the exclusion - # is dead defense. Gate (a) verified via 2026-05-24 DataPhase1 SSM - # log (`Backfill write complete: 904 ok` = 903 constituents + SPY). - # _MACRO_SYMBOLS_NO_OHLCV is the post-L1346 subset of macro-only - # symbols that still need to be excluded — everything in macro EXCEPT - # SPY (which now has full OHLCV via universe write). + # SPY is NOT excluded — alpha-engine-data #245 (2026-05-15) lifted + # SPY to a full `universe` ArcticDB member (`_UNIVERSE_EXTRA = + # frozenset({"SPY"})`, written by both builders/backfill.py and + # builders/daily_append.py), so `universe.SPY` carries the full + # OHLCV + atr_14_pct feature column that `load_atr_14_pct` needs. + # SPY is removed from `_MACRO_SYMBOLS` (price_cache.py) for the + # same reason — the executor reads SPY from `universe` like any + # other held ticker. # # ``atr_map`` kwarg mirrors the vwap_map injection pattern — backtester # precomputes once per simulate pipeline, skipping millions of # per-call universe.read(ticker) round-trips. Live trading passes # atr_map=None and takes the load_atr_14_pct path unchanged. - _MACRO_SYMBOLS_NO_OHLCV = _MACRO_SYMBOLS - {"SPY"} atr_tickers = [s["ticker"] for s in signals.get("enter", [])] atr_tickers += list(current_positions.keys()) - atr_tickers = sorted(set(atr_tickers) - _MACRO_SYMBOLS_NO_OHLCV) + atr_tickers = sorted(set(atr_tickers) - _MACRO_SYMBOLS) if atr_map is None: if atr_tickers: atr_map = load_atr_14_pct( diff --git a/executor/price_cache.py b/executor/price_cache.py index 9371b0b..2a83d42 100644 --- a/executor/price_cache.py +++ b/executor/price_cache.py @@ -39,12 +39,17 @@ # predictor's own DailyData dependency expectation. _ATR_MAX_STALENESS_TRADING_DAYS = 1 -# Symbols that live in the ArcticDB `macro` library rather than `universe`. -# Mirrors the canonical writer list in alpha-engine-data's -# ``builders/daily_append.py`` (macro_keys + sector_etfs). Kept in sync -# manually; any additions there need matching updates here. +# Symbols the executor reads from the ArcticDB `macro` library (Close-only). +# Subset of what alpha-engine-data writes to macro — SPY is dual-written +# there but read from `universe` for full OHLCV + atr_14_pct (L1346 (c) +# retirement, 2026-05-28; SPY became a full `universe` member via +# alpha-engine-data #245's `_UNIVERSE_EXTRA = frozenset({"SPY"})` write +# path on 2026-05-15). VIX/VIX3M/TNX/IRX/GLD/USO + XL* sector ETFs stay +# macro-routed because they have no `universe` counterpart and their +# use cases (sector-relative exit veto + macro features) consume Close +# only. _MACRO_SYMBOLS = frozenset({ - "SPY", "VIX", "VIX3M", "TNX", "IRX", "GLD", "USO", + "VIX", "VIX3M", "TNX", "IRX", "GLD", "USO", "XLB", "XLC", "XLE", "XLF", "XLI", "XLK", "XLP", "XLRE", "XLU", "XLV", "XLY", }) @@ -131,17 +136,8 @@ def load_price_histories( read_errors: list[str] = [] empty: list[str] = [] - # L1346 (c) second-half routing post-#245: - # SPY is no longer in `_MACRO_SYMBOLS_NO_OHLCV` because universe.SPY - # now carries full OHLCV (alpha-engine-data #245 _UNIVERSE_EXTRA - # write path; gate (a) verified via 5/24 DataPhase1 SSM log). - # Defensive fallback to macro.SPY preserves backwards compat during - # the cross-repo retirement — mirrors alpha-engine-predictor #196's - # universe-preferred + macro-fallback shape. Non-SPY macro symbols - # (VIX/TNX/IRX/sector ETFs) remain macro-routed (still Close-only). - _MACRO_SYMBOLS_NO_OHLCV = _MACRO_SYMBOLS - {"SPY"} for ticker in tickers: - if ticker in _MACRO_SYMBOLS_NO_OHLCV: + if ticker in _MACRO_SYMBOLS: if macro is None: macro = _open_macro_library(signals_bucket) lib = macro @@ -150,24 +146,8 @@ def load_price_histories( try: df = lib.read(ticker).data except Exception as e: - # L1346 (c) SPY-specific fallback: if SPY isn't in universe - # (e.g. universe.SPY backfill hadn't run yet), fall through - # to macro.SPY. Removed once L1346 (b) + (c) soak clean for - # ≥1 Saturday cycle on production. - if ticker == "SPY" and lib is universe: - if macro is None: - macro = _open_macro_library(signals_bucket) - try: - df = macro.read(ticker).data - except Exception as e2: - read_errors.append( - f"{ticker} (universe={e.__class__.__name__}, " - f"macro={e2.__class__.__name__})" - ) - continue - else: - read_errors.append(f"{ticker} ({e.__class__.__name__})") - continue + read_errors.append(f"{ticker} ({e.__class__.__name__})") + continue if df.empty: empty.append(ticker) continue diff --git a/tests/test_executor_run_precompute_kwargs.py b/tests/test_executor_run_precompute_kwargs.py index c426091..f7d7159 100644 --- a/tests/test_executor_run_precompute_kwargs.py +++ b/tests/test_executor_run_precompute_kwargs.py @@ -290,10 +290,28 @@ def test_main_imports_macro_symbols_from_price_cache(): tickers (defined in price_cache.py, also consumed by load_price_histories + eod_reconcile #181). Pin the import so the exclusion set never drifts from the dispatch set. + + Post-L1346 (c) (2026-05-28): SPY is excluded from `_MACRO_SYMBOLS` + because `universe.SPY` now carries full OHLCV + atr_14_pct via + alpha-engine-data #245's `_UNIVERSE_EXTRA` write path. SPY is read + from `universe` like any other held ticker. VIX/VIX3M/TNX/IRX/GLD/ + USO + XL* sector ETFs remain macro-routed (no `universe` counterpart; + Close-only). """ from executor.price_cache import _MACRO_SYMBOLS - assert "SPY" in _MACRO_SYMBOLS + assert "SPY" not in _MACRO_SYMBOLS, ( + "SPY must NOT be in _MACRO_SYMBOLS post-L1346 (c) retirement — " + "universe.SPY now has full OHLCV (alpha-engine-data #245). " + "Re-adding SPY here would re-introduce the dead-defense pattern " + "that excluded SPY from ATR computation." + ) + # Sanity: remaining macro-only symbols still routed to macro lib. + for sym in ("VIX", "TNX", "IRX", "XLK", "XLF"): + assert sym in _MACRO_SYMBOLS, ( + f"{sym} must remain in _MACRO_SYMBOLS — it lives only in the " + f"`macro` ArcticDB library (Close-only)." + ) assert executor_main._MACRO_SYMBOLS is _MACRO_SYMBOLS, ( "executor.main must reuse price_cache._MACRO_SYMBOLS, not " "redefine its own macro set (drift risk)." diff --git a/tests/test_l1346_spy_atr_exclusion_retired.py b/tests/test_l1346_spy_atr_exclusion_retired.py index 2b5f4b0..159d4a4 100644 --- a/tests/test_l1346_spy_atr_exclusion_retired.py +++ b/tests/test_l1346_spy_atr_exclusion_retired.py @@ -1,57 +1,93 @@ -"""Pin L1346 (c) — SPY no longer excluded from ATR ticker set. +"""Pin L1346 (c) — SPY routes to `universe` ArcticDB library, not `macro`. Pre-fix: alpha-engine #185 excluded ALL `_MACRO_SYMBOLS` (incl. SPY) from -the ATR ticker list because macro lib was Close-only. Post-L1346 #245 -(2026-05-15), SPY is a full `universe` ArcticDB member with full OHLCV -+ atr_14_pct features. The pre-L1346 exclusion is now dead defense. +the ATR ticker list because macro lib was Close-only. The 2026-05-24 +transition shipped a `_MACRO_SYMBOLS_NO_OHLCV = _MACRO_SYMBOLS - {"SPY"}` +defensive carve-out + a SPY-specific macro-fallback in `load_price_histories` +to bridge the cross-repo soak window after alpha-engine-data #245 +(2026-05-15) lifted SPY to a full `universe` member. -This module pins the post-L1346 retirement: SPY MUST be ATR-computable -(not in the macro-only-no-OHLCV exclusion set). +Post-retirement (2026-05-28): the carve-out + defensive fallback are gone. +SPY is removed from `_MACRO_SYMBOLS` entirely — the executor reads SPY +from `universe` like any other held ticker, and the ATR-exclusion line +subtracts `_MACRO_SYMBOLS` directly (no `_NO_OHLCV` derivation). + +This module pins the structural retirement so a refactor can't silently +re-introduce the dead defense. """ from __future__ import annotations import inspect -import pytest -def test_atr_exclusion_set_does_not_include_spy(): - """Read the embedded `_MACRO_SYMBOLS_NO_OHLCV` constant from - `executor.main` and assert SPY is NOT in it.""" - from executor.main import _MACRO_SYMBOLS +def test_spy_not_in_macro_symbols(): + """`_MACRO_SYMBOLS` must NOT contain SPY post-L1346 (c) retirement.""" + from executor.price_cache import _MACRO_SYMBOLS - # Mirror the post-fix derivation: post-L1346 the exclusion subtracts SPY - # because universe.SPY now carries the atr_14_pct feature column. - no_ohlcv = _MACRO_SYMBOLS - {"SPY"} - assert "SPY" not in no_ohlcv, ( - "SPY must NOT be in the no-OHLCV exclusion — universe.SPY now has " - "full OHLCV + atr_14_pct (post L1346 #245 _UNIVERSE_EXTRA write path). " - "If this assertion fails, the L1346 (c) retirement has regressed." + assert "SPY" not in _MACRO_SYMBOLS, ( + "SPY must NOT be in `_MACRO_SYMBOLS` — universe.SPY now has full " + "OHLCV + atr_14_pct (post L1346 #245 _UNIVERSE_EXTRA write path). " + "Re-adding SPY here would route reads to macro (Close-only) and " + "re-break the morning planner's ATR computation for held SPY." ) # Sanity: legacy macro-only symbols (VIX, TNX, IRX, sector ETFs) still - # belong in the exclusion because they remain Close-only in macro lib. + # belong in the routing list because they remain Close-only in macro lib. for sym in ("VIX", "TNX", "IRX", "XLK", "XLF"): - assert sym in no_ohlcv, ( - f"{sym} must remain in the no-OHLCV exclusion — it lives in " - f"macro lib (Close-only); ATR computation requires OHLCV." + assert sym in _MACRO_SYMBOLS, ( + f"{sym} must remain in `_MACRO_SYMBOLS` — it lives only in the " + f"`macro` ArcticDB library (Close-only); ATR computation " + f"requires OHLCV." ) -def test_main_atr_tickers_set_does_not_subtract_spy(): +def test_main_atr_tickers_subtracts_macro_symbols_directly(): """White-box source-level pin: locate the atr_tickers derivation in - `executor/main.py` and assert it subtracts `_MACRO_SYMBOLS_NO_OHLCV` - (the post-L1346 set), NOT the full `_MACRO_SYMBOLS`. + `executor/main.py` and assert it subtracts `_MACRO_SYMBOLS` directly, + NOT a `_MACRO_SYMBOLS_NO_OHLCV = _MACRO_SYMBOLS - {"SPY"}` carve-out + (which would silently re-introduce the L1346 transitional defense). Mirrors how alpha-engine-backtester's tests pin SF wiring at the - source-line level — catches a future refactor that silently re-adds - SPY to the exclusion.""" + source-line level — catches a future refactor that re-adds the + SPY-specific exclusion.""" from executor import main src = inspect.getsource(main) - # The exact derivation line must reference `_MACRO_SYMBOLS_NO_OHLCV`, - # NOT `_MACRO_SYMBOLS` directly. - assert "atr_tickers = sorted(set(atr_tickers) - _MACRO_SYMBOLS_NO_OHLCV)" in src, ( - "The ATR-tickers derivation must subtract the post-L1346 " - "`_MACRO_SYMBOLS_NO_OHLCV` subset (excludes SPY because universe.SPY " - "now has OHLCV), NOT the full `_MACRO_SYMBOLS` set. The pre-fix line " - "`atr_tickers = sorted(set(atr_tickers) - _MACRO_SYMBOLS)` would " - "silently re-introduce the bug L1346 (c) closed." + assert "atr_tickers = sorted(set(atr_tickers) - _MACRO_SYMBOLS)" in src, ( + "The ATR-tickers derivation must subtract `_MACRO_SYMBOLS` " + "directly. The pre-retirement form " + "`atr_tickers = sorted(set(atr_tickers) - _MACRO_SYMBOLS_NO_OHLCV)` " + "with a separate `_MACRO_SYMBOLS_NO_OHLCV = _MACRO_SYMBOLS - {'SPY'}` " + "derivation is the dead-defense pattern L1346 (c) retired." + ) + assert "_MACRO_SYMBOLS_NO_OHLCV" not in src, ( + "`_MACRO_SYMBOLS_NO_OHLCV` derivation must not re-appear in " + "`executor/main.py` — it was a transitional carve-out retired " + "post-soak when SPY became a full `universe` member." + ) + + +def test_price_cache_no_spy_specific_fallback(): + """`load_price_histories` must not carry a SPY-specific macro + fallback. The defensive fallback was a transition device while + universe.SPY's write path soaked; after retirement reads go straight + to `universe` like any other ticker.""" + from executor import price_cache + src = inspect.getsource(price_cache) + assert 'if ticker == "SPY" and lib is universe' not in src, ( + "Defensive SPY-specific macro fallback must be retired — " + "universe.SPY is the canonical source post-L1346 (c). If " + "universe.SPY ever fails to read, the right surface is a hard " + "failure (per feedback_no_silent_fails), not a silent macro " + "fallback that masks the upstream gap." + ) + + +def test_eod_reconcile_no_spy_specific_fallback(): + """`eod_reconcile.run` must not carry a SPY-specific macro fallback. + Same rationale as `price_cache` above.""" + from executor import eod_reconcile + src = inspect.getsource(eod_reconcile) + assert 'if ticker == "SPY" and lib is universe_lib' not in src, ( + "Defensive SPY-specific macro fallback must be retired from " + "eod_reconcile — universe.SPY is the canonical source post-L1346 " + "(c). Silent macro fallback would mask universe.SPY gaps." )