From 0f7d2421da3300cfa980d63a162e419e7dab0f02 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 30 Jul 2026 21:33:36 -0400 Subject: [PATCH] chore: lint scripts/, and put scripts/ under CI lint scripts/ had never been linted -- CI ran 'ruff check src tests' -- and had quietly accumulated 26 errors, all in reconcile_volume.py: 22 W293 blank-line-with-whitespace, 1 W291 trailing whitespace, 1 I001 unsorted imports, 2 F541 f-strings with no placeholders. All autofixable, all applied with 'ruff check --fix'. No behaviour changed, and that is verified rather than asserted: the significant-token stream is 575 tokens before and after, and the only differences are the import block regrouped (identical tokens, moved) and two 'f' prefixes dropped from strings containing no placeholders. f"text" and "text" evaluate identically. All five scripts still py_compile. reconcile_volume.py needs Norgate/Windows so it cannot be executed here, which is why the check is token-level rather than behavioural. CI now lints scripts/ too. The selected rules are mostly not style: pyflakes F catches undefined names and unused imports, which are crashes waiting to happen, and scripts/vintage_alert_selftest.py is a diagnostic users are told to run, so a broken one is worse than none. Leaving a source root unlinted also fails open, which is exactly how 26 errors accumulated unnoticed. The trade-off, stated plainly: a future throwaway investigation script now has to pass lint before CI goes green. E501 is already ignored so there is no line-length nagging, and --fix handles nearly everything. To revert, drop 'scripts' from the ruff step; nothing else depends on it. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/python-test.yml | 8 ++++- scripts/reconcile_volume.py | 56 ++++++++++++++++--------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 154cf33..23d4a5f 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -25,7 +25,13 @@ jobs: python -m pip install --upgrade pip python -m pip install -e ".[dev]" - name: Run linting - run: python -m ruff check src tests + # scripts/ is included deliberately. The selected rules (E/F/W/I, E501 ignored) are + # mostly not style: pyflakes F catches undefined names and unused imports, which are + # crashes waiting to happen, and scripts/vintage_alert_selftest.py is a diagnostic + # users are told to run -- a broken one is worse than none. Leaving a source root + # unlinted also fails open: scripts/ had quietly accumulated 26 errors before anyone + # looked. To revert, drop `scripts` here; nothing else depends on it. + run: python -m ruff check src tests scripts - name: Run tests run: | # Create a dummy store for the test environment to prevent COTDATA_STORE missing errors diff --git a/scripts/reconcile_volume.py b/scripts/reconcile_volume.py index a02cc5f..0fba12b 100644 --- a/scripts/reconcile_volume.py +++ b/scripts/reconcile_volume.py @@ -1,18 +1,20 @@ import argparse -import pandas as pd + import numpy as np +import pandas as pd -from cotdata.providers import norgate from cotdata import store +from cotdata.providers import norgate + def get_roll_windows(df: pd.DataFrame, window_days: int = 5) -> pd.DataFrame: """Find dates where the contract rolled (Delivery Month changed) and return a mask of those dates +/- window_days.""" if "Delivery Month" not in df.columns: return pd.Series(False, index=df.index) - + dm = df["Delivery Month"] roll = dm.ne(dm.shift()) & dm.shift().notna() - + # Expand window roll_dates = df.index[roll] mask = pd.Series(False, index=df.index) @@ -20,18 +22,18 @@ def get_roll_windows(df: pd.DataFrame, window_days: int = 5) -> pd.DataFrame: start = rd - pd.Timedelta(days=window_days) end = rd + pd.Timedelta(days=window_days) mask.loc[start:end] = True - + return mask def verify_symbol(symbol: str): print(f"\n{'='*50}\nReconciling {symbol}\n{'='*50}") - + print(f"1. Fetching & Reconstructing {symbol} via norgate.py ...") norgate.update(symbols=[symbol]) - + print(f"2. Reading updated dataframe for {symbol} ...") df = store.read_prices(symbol, "backadj") - + if df.empty: print(f"Error: No data found for {symbol}.") return @@ -42,65 +44,65 @@ def verify_symbol(symbol: str): if col not in df.columns: print(f"❌ Missing expected column: {col}") return - + print("✅ All expected additive columns are present.") - + # Check Fallback source = df["Volume_Source"].iloc[-1] print(f"Volume Source: {source}") - + if source == "raw": print(f"✅ Fallback applied correctly. Volume_Reconstructed == Volume for all rows: {(df['Volume_Reconstructed'] == df['Volume']).all()}") return - + # Check bounds print("3. Running Sanity Checks...") - + # a. First + Second = Reconstructed valid = df.dropna(subset=["FirstVolume", "SecondVolume"]) reconstructed_match = np.isclose( - valid["FirstVolume"] + valid["SecondVolume"], + valid["FirstVolume"] + valid["SecondVolume"], valid["Volume_Reconstructed"] ).all() print(f"{'✅' if reconstructed_match else '❌'} FirstVolume + SecondVolume == Volume_Reconstructed") - + # b. Reconstructed >= Volume (raw) # The true combined volume should be roughly >= the single front-month volume # Some minor exceptions can happen due to Norgate raw volume inclusions, but generally true. vol_diff = (valid["Volume_Reconstructed"] >= valid["Volume"] * 0.95).mean() print(f"{'✅' if vol_diff > 0.99 else '❌'} Volume_Reconstructed is >= Raw Volume (in {vol_diff*100:.1f}% of days)") - + # c. No NaNs introduced into default Volume nans_in_raw = df["Volume"].isna().sum() print(f"{'✅' if nans_in_raw == 0 else '❌'} Default 'Volume' has 0 NaNs ({nans_in_raw} found)") - + print("\n4. Roll Window Drop-off Analysis") # Compare average volume during roll windows vs non-roll windows roll_mask = get_roll_windows(df, window_days=5) - + if roll_mask.sum() == 0: print("No roll windows detected.") return - + roll_raw = df.loc[roll_mask, "Volume"].mean() nonroll_raw = df.loc[~roll_mask, "Volume"].mean() - + roll_rec = df.loc[roll_mask, "Volume_Reconstructed"].mean() nonroll_rec = df.loc[~roll_mask, "Volume_Reconstructed"].mean() - + raw_drop = (nonroll_raw - roll_raw) / nonroll_raw * 100 if nonroll_raw else 0 rec_drop = (nonroll_rec - roll_rec) / nonroll_rec * 100 if nonroll_rec else 0 - - print(f" Raw Volume:") + + print(" Raw Volume:") print(f" Non-Roll Avg: {nonroll_raw:,.0f}") print(f" Roll Avg: {roll_raw:,.0f}") print(f" Drop-off: {raw_drop:.1f}%") - - print(f" Reconstructed Volume:") + + print(" Reconstructed Volume:") print(f" Non-Roll Avg: {nonroll_rec:,.0f}") print(f" Roll Avg: {roll_rec:,.0f}") print(f" Drop-off: {rec_drop:.1f}%") - + if rec_drop < raw_drop: print("✅ Reconstructed volume successfully smoothed the roll drop-off!") else: @@ -110,6 +112,6 @@ def verify_symbol(symbol: str): parser = argparse.ArgumentParser(description="Reconcile Volume Reconstruction") parser.add_argument("--symbols", nargs="+", default=["ES", "BTC"], help="Internal symbols to test") args = parser.parse_args() - + for sym in args.symbols: verify_symbol(sym)