chore: lint scripts/, and put scripts/ under CI lint

.github/workflows/python-test.yml

@@ -25,7 +25,13 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install -e ".[dev]"
     - name: Run linting
-      run: python -m ruff check src tests
+      # scripts/ is included deliberately. The selected rules (E/F/W/I, E501 ignored) are
+      # mostly not style: pyflakes F catches undefined names and unused imports, which are
+      # crashes waiting to happen, and scripts/vintage_alert_selftest.py is a diagnostic
+      # users are told to run -- a broken one is worse than none. Leaving a source root
+      # unlinted also fails open: scripts/ had quietly accumulated 26 errors before anyone
+      # looked. To revert, drop `scripts` here; nothing else depends on it.
+      run: python -m ruff check src tests scripts
     - name: Run tests
       run: |
         # Create a dummy store for the test environment to prevent COTDATA_STORE missing errors

scripts/reconcile_volume.py

@@ -1,37 +1,39 @@
 import argparse
-import pandas as pd
+
 import numpy as np
+import pandas as pd
 
-from cotdata.providers import norgate
 from cotdata import store
+from cotdata.providers import norgate
+
 
 def get_roll_windows(df: pd.DataFrame, window_days: int = 5) -> pd.DataFrame:
     """Find dates where the contract rolled (Delivery Month changed) and return a mask of those dates +/- window_days."""
     if "Delivery Month" not in df.columns:
         return pd.Series(False, index=df.index)
-        
+
     dm = df["Delivery Month"]
     roll = dm.ne(dm.shift()) & dm.shift().notna()
-    
+
     # Expand window
     roll_dates = df.index[roll]
     mask = pd.Series(False, index=df.index)
     for rd in roll_dates:
         start = rd - pd.Timedelta(days=window_days)
         end = rd + pd.Timedelta(days=window_days)
         mask.loc[start:end] = True
-        
+
     return mask
 
 def verify_symbol(symbol: str):
     print(f"\n{'='*50}\nReconciling {symbol}\n{'='*50}")
-    
+
     print(f"1. Fetching & Reconstructing {symbol} via norgate.py ...")
     norgate.update(symbols=[symbol])
-    
+
     print(f"2. Reading updated dataframe for {symbol} ...")
     df = store.read_prices(symbol, "backadj")
-    
+
     if df.empty:
         print(f"Error: No data found for {symbol}.")
         return
@@ -42,65 +44,65 @@ def verify_symbol(symbol: str):
         if col not in df.columns:
             print(f"❌ Missing expected column: {col}")
             return
-            
+
     print("✅ All expected additive columns are present.")
-    
+
     # Check Fallback
     source = df["Volume_Source"].iloc[-1]
     print(f"Volume Source: {source}")
-    
+
     if source == "raw":
         print(f"✅ Fallback applied correctly. Volume_Reconstructed == Volume for all rows: {(df['Volume_Reconstructed'] == df['Volume']).all()}")
         return
-        
+
     # Check bounds
     print("3. Running Sanity Checks...")
-    
+
     # a. First + Second = Reconstructed
     valid = df.dropna(subset=["FirstVolume", "SecondVolume"])
     reconstructed_match = np.isclose(
-        valid["FirstVolume"] + valid["SecondVolume"], 
+        valid["FirstVolume"] + valid["SecondVolume"],
         valid["Volume_Reconstructed"]
     ).all()
     print(f"{'✅' if reconstructed_match else '❌'} FirstVolume + SecondVolume == Volume_Reconstructed")
-    
+
     # b. Reconstructed >= Volume (raw)
     # The true combined volume should be roughly >= the single front-month volume
     # Some minor exceptions can happen due to Norgate raw volume inclusions, but generally true.
     vol_diff = (valid["Volume_Reconstructed"] >= valid["Volume"] * 0.95).mean()
     print(f"{'✅' if vol_diff > 0.99 else '❌'} Volume_Reconstructed is >= Raw Volume (in {vol_diff*100:.1f}% of days)")
-    
+
     # c. No NaNs introduced into default Volume
     nans_in_raw = df["Volume"].isna().sum()
     print(f"{'✅' if nans_in_raw == 0 else '❌'} Default 'Volume' has 0 NaNs ({nans_in_raw} found)")
-    
+
     print("\n4. Roll Window Drop-off Analysis")
     # Compare average volume during roll windows vs non-roll windows
     roll_mask = get_roll_windows(df, window_days=5)
-    
+
     if roll_mask.sum() == 0:
         print("No roll windows detected.")
         return
-        
+
     roll_raw = df.loc[roll_mask, "Volume"].mean()
     nonroll_raw = df.loc[~roll_mask, "Volume"].mean()
-    
+
     roll_rec = df.loc[roll_mask, "Volume_Reconstructed"].mean()
     nonroll_rec = df.loc[~roll_mask, "Volume_Reconstructed"].mean()
-    
+
     raw_drop = (nonroll_raw - roll_raw) / nonroll_raw * 100 if nonroll_raw else 0
     rec_drop = (nonroll_rec - roll_rec) / nonroll_rec * 100 if nonroll_rec else 0
-    
-    print(f"  Raw Volume:")
+
+    print("  Raw Volume:")
     print(f"    Non-Roll Avg: {nonroll_raw:,.0f}")
     print(f"    Roll Avg:     {roll_raw:,.0f}")
     print(f"    Drop-off:     {raw_drop:.1f}%")
-    
-    print(f"  Reconstructed Volume:")
+
+    print("  Reconstructed Volume:")
     print(f"    Non-Roll Avg: {nonroll_rec:,.0f}")
     print(f"    Roll Avg:     {roll_rec:,.0f}")
     print(f"    Drop-off:     {rec_drop:.1f}%")
-    
+
     if rec_drop < raw_drop:
         print("✅ Reconstructed volume successfully smoothed the roll drop-off!")
     else:
@@ -110,6 +112,6 @@ def verify_symbol(symbol: str):
     parser = argparse.ArgumentParser(description="Reconcile Volume Reconstruction")
     parser.add_argument("--symbols", nargs="+", default=["ES", "BTC"], help="Internal symbols to test")
     args = parser.parse_args()
-    
+
     for sym in args.symbols:
         verify_symbol(sym)