From 5738822ca08cb108160e07693dcd76bf03e60812 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 30 Jul 2026 20:57:06 -0400 Subject: [PATCH] feat(vintage): write a revision marker file, and a selftest that proves it fires Task Scheduler's own 'Send an e-mail' / 'Display a message' actions are DEPRECATED and non-functional on Windows 8 / Server 2012+, so a non-zero exit alone is only a passive Last Run Result 0x1 that someone has to go and look at. An earlier comment in this file implied otherwise; corrected. run-vintage.cmd now notifies itself. On any run that records a revision it appends that run's output to \vintage\REVISIONS_.txt, which rides the existing robocopy /MIR push to the Mac, so the alert lands on the machine the work happens on. - The marker is written IMMEDIATELY after ingest, not at the end: the revision is already committed at that point, so a later unrelated failure must not be able to suppress it. - Appends rather than overwrites, so two revision runs in one day both survive. - Date comes from PowerShell, not %DATE%, which is formatted per regional settings and is not safe in a filename. - Ingest output goes to a .tmp scratch file so the marker holds only THIS run rather than the whole appended history, and the existing *.tmp sync exclusions skip it. scripts/vintage_alert_selftest.py forces a revision in a throwaway store and drives the real CLI, so the loud path can be verified now rather than trusted until CFTC eventually restates something. An alert nobody has watched fire is one nobody should rely on. Co-Authored-By: Claude Opus 4.8 --- docs/examples/windows/run-vintage.cmd | 47 ++++++++-- scripts/vintage_alert_selftest.py | 125 ++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 scripts/vintage_alert_selftest.py diff --git a/docs/examples/windows/run-vintage.cmd b/docs/examples/windows/run-vintage.cmd index 7ea047d..9880348 100644 --- a/docs/examples/windows/run-vintage.cmd +++ b/docs/examples/windows/run-vintage.cmd @@ -28,11 +28,29 @@ REM single sync carries both the current-state update and the new vintage snapsh REM NOTIFICATION: Task Scheduler discards stdout, so everything below is ALSO appended to REM a log file, and `ingest` exits NON-ZERO whenever it records a revision or spots a REM closed-year restatement. That is a notification, not a failure -- the data is already -REM committed. Point Task Scheduler's "send mail"/alert at this task, or just read the log. -REM Without it a retroactive restatement would be detected and silently swallowed. +REM committed. +REM +REM Task Scheduler's own "Send an e-mail" / "Display a message" actions are DEPRECATED and +REM non-functional on Windows 8 / Server 2012 and later, so a non-zero exit on its own only +REM shows up passively as Last Run Result 0x1 in the Task Scheduler UI. This script therefore +REM does its own notifying: on any run that records a revision it writes a MARKER FILE, +REM +REM \vintage\REVISIONS_.txt +REM +REM holding that run's output. It lives inside vintage\, so the existing robocopy /MIR push +REM carries it to the Mac and the alert shows up on the machine you actually work on. setlocal set COTDATA_STORE=REPLACE_WITH_STORE_PATH set VINTAGE_LOG=REPLACE_WITH_STORE_PATH\vintage\run.log +set VINTAGE_RUNOUT=REPLACE_WITH_STORE_PATH\vintage\.last_ingest.tmp + +REM Locale-independent date. %DATE% is formatted per the machine's regional settings, so it +REM is not safe in a filename; PowerShell gives a stable yyyy-MM-dd on any box. +for /f %%i in ('powershell -NoProfile -Command "Get-Date -Format yyyy-MM-dd"') do set TODAY=%%i +set VINTAGE_MARKER=REPLACE_WITH_STORE_PATH\vintage\REVISIONS_%TODAY%.txt + +REM The store may not exist yet on a first run; the vintage dir must, to hold the log. +if not exist "REPLACE_WITH_STORE_PATH\vintage" mkdir "REPLACE_WITH_STORE_PATH\vintage" REM Optional: identify yourself to CFTC. Defaults to the repo URL if unset. REM set COTDATA_USER_AGENT=cotdata-vintage/0.1 (+contact you@example.com) @@ -47,11 +65,26 @@ if %ERRORLEVEL% NEQ 0 ( REM Parse whatever was just retained into change-only observations + revisions. REM Safe to re-run: re-ingesting identical bytes writes zero rows. -"REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ingest --pending >> "%VINTAGE_LOG%" 2>&1 +REM Captured to its OWN file first, so the marker can hold just THIS run's output rather +REM than the whole appended history, then folded into the running log. +"REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ingest --pending > "%VINTAGE_RUNOUT%" 2>&1 REM NON-ZERO HERE MEANS "revisions were recorded", not "the run broke". The data is REM already written. Remember it, keep going, and re-raise at the end so the scheduler REM shows the task as attention-needed. set VINTAGE_REVISED=%ERRORLEVEL% +type "%VINTAGE_RUNOUT%" >> "%VINTAGE_LOG%" + +REM Write the marker HERE, not at the end. The revision is already committed at this +REM point, so an unrelated failure in a later step must not be able to suppress the +REM alert. Appends rather than overwrites, so two revision runs on one day both survive. +if NOT "%VINTAGE_REVISED%"=="0" ( + echo ================================================================ >> "%VINTAGE_MARKER%" + echo cotdata vintage: REVISIONS RECORDED %TODAY% >> "%VINTAGE_MARKER%" + echo Expand with: cotdata-vintage diff >> "%VINTAGE_MARKER%" + echo ================================================================ >> "%VINTAGE_MARKER%" + type "%VINTAGE_RUNOUT%" >> "%VINTAGE_MARKER%" + echo. >> "%VINTAGE_MARKER%" +) REM Resolve release dates. `published` reads the true publication timestamp out of the REM weekly static just captured (its HTTP Last-Modified), which beats a poll-derived @@ -70,9 +103,13 @@ if %ERRORLEVEL% NEQ 0 ( ) if NOT "%VINTAGE_REVISED%"=="0" ( - echo vintage ok, but REVISIONS WERE RECORDED -- see "%VINTAGE_LOG%" and run: cotdata-vintage diff - type "%VINTAGE_LOG%" + echo vintage ok, but REVISIONS WERE RECORDED + echo marker: "%VINTAGE_MARKER%" ^(syncs to the Mac with the store^) + echo log: "%VINTAGE_LOG%" + type "%VINTAGE_RUNOUT%" + del "%VINTAGE_RUNOUT%" 2>nul exit /b %VINTAGE_REVISED% ) +del "%VINTAGE_RUNOUT%" 2>nul echo vintage ok, no revisions exit /b 0 diff --git a/scripts/vintage_alert_selftest.py b/scripts/vintage_alert_selftest.py new file mode 100644 index 0000000..d8cce10 --- /dev/null +++ b/scripts/vintage_alert_selftest.py @@ -0,0 +1,125 @@ +"""Prove the vintage revision ALERT fires, without waiting for CFTC to revise anything. + +The quiet path is easy to verify (run the task, it exits 0). The loud path is not: it +only triggers when CFTC actually restates something, which may be months away — and an +alert nobody has ever seen fire is an alert nobody should trust. + +This forces a revision in a THROWAWAY store and runs the real CLI path over it, so you +can confirm the exit code and the marker-file contents your scheduled task depends on. +It never touches $COTDATA_STORE. + + python scripts/vintage_alert_selftest.py + +Expected: "SELFTEST PASSED", exit 0. Any other outcome means the notification wiring in +run-vintage.cmd would not have alerted you to a real restatement. +""" +from __future__ import annotations + +import datetime as dt +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +WIDE_COLS = { + "Market_and_Exchange_Names": ["GOLD"], + "CFTC_Contract_Market_Code": ["088691"], + "Open_Interest_All": [500000], + "Comm_Positions_Long_All": [200000], + "NonComm_Positions_Long_All": [150000], + "NonComm_Positions_Short_All": [90000], + "NonRept_Positions_Long_All": [40000], + "NonRept_Positions_Short_All": [30000], + "Traders_Comm_Long_All": [50], "Traders_Comm_Short_All": [55], + "Traders_NonComm_Long_All": [60], "Traders_NonComm_Short_All": [45], +} + + +def _wide(report_date: str, comm_short: int): + import pandas as pd + idx = pd.to_datetime([report_date]) + idx.name = "Report_Date_as_MM_DD_YYYY" + return pd.DataFrame({**WIDE_COLS, "Comm_Positions_Short_All": [comm_short]}, index=idx) + + +def main() -> int: + store = Path(tempfile.mkdtemp(prefix="cotdata_vintage_selftest_")) + os.environ["COTDATA_STORE"] = str(store) + print(f"scratch store: {store}\n(your real $COTDATA_STORE is untouched)\n") + + from cotdata import vintage + from cotdata import vintage_ingest as vi + + # Week 1: the report as first published. + vi.ingest_canonical(vi.canonicalize_legacy(_wide("2026-07-21", 250_000)), + snapshot_id="selftest-week1", + observed_at=dt.datetime(2026, 7, 24, tzinfo=dt.timezone.utc)) + # Week 2: CFTC restates one field. This is the event the alert exists for. + vi.ingest_canonical(vi.canonicalize_legacy(_wide("2026-07-21", 251_000)), + snapshot_id="selftest-week2", + observed_at=dt.datetime(2026, 7, 31, tzinfo=dt.timezone.utc)) + + revisions = vi.read_revisions() + if revisions.empty: + print("FAIL: no revision was recorded from a changed field.") + return 1 + print(f"forced {len(revisions)} revision row(s):") + cols = [c for c in ["report_date", "market_code", "category", "field", + "old_value", "new_value", "age_days"] if c in revisions.columns] + print(revisions[cols].to_string(index=False)) + + print("\n--- what 'cotdata-vintage diff' shows you ---") + sys.stdout.flush() # subprocesses write straight to the terminal; keep ordering sane + subprocess.run( + [sys.executable, "-c", + "from cotdata import vintage_cli; vintage_cli.main(['diff'])"], + env={**os.environ, "COTDATA_STORE": str(store)}, + ) + + # Drive the same CLI entry point run-vintage.cmd calls, so the exit code proves what + # the .cmd would actually observe. source_kind is weekly_static deliberately: ingest + # only parses annual zips, so this skips the parse step and keeps the demonstration + # about the ALERT rather than about a fake fixture failing to unzip. A real suspect + # would sit on an annual zip; the notification path is identical either way. + vintage._write_manifest({"schema_version": 1, "snapshots": [{ + "snapshot_id": "selftest-week2", "report_type": "legacy", + "source_kind": "weekly_static", "report_year": 2025, + "local_path": "vintage/raw/weekly_static/2026/selftest.txt", + "parse_status": "pending", "restatement_suspect": True, + "retrieved_at": "2026-07-31T21:00:00Z", + }]}) + + sys.stdout.flush() + proc = subprocess.run( + [sys.executable, "-c", + "from cotdata import vintage_cli; import sys; sys.exit(vintage_cli.main(['ingest','--pending']))"], + capture_output=True, text=True, env={**os.environ, "COTDATA_STORE": str(store)}, + ) + output = proc.stdout + proc.stderr + print("\n--- what your marker file would contain ---") + print(output.strip()) + print("--- end ---\n") + + problems = [] + if proc.returncode == 0: + problems.append("ingest exited 0; the scheduled task would have looked clean") + if "RESTATEMENT SUSPECT" not in output: + problems.append("closed-year restatement was not announced in the output") + if "cotdata-vintage diff" not in output: + problems.append("output does not tell you how to inspect the revision") + + if problems: + print("SELFTEST FAILED:") + for p in problems: + print(f" - {p}") + return 1 + + print(f"SELFTEST PASSED — ingest exited {proc.returncode} (non-zero), and the output " + f"names the restatement.\nrun-vintage.cmd writes exactly that text to " + f"vintage\\REVISIONS_.txt and exits non-zero.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())