diff --git a/docs/examples/windows/run-vintage.cmd b/docs/examples/windows/run-vintage.cmd index 7ea047d..9880348 100644 --- a/docs/examples/windows/run-vintage.cmd +++ b/docs/examples/windows/run-vintage.cmd @@ -28,11 +28,29 @@ REM single sync carries both the current-state update and the new vintage snapsh REM NOTIFICATION: Task Scheduler discards stdout, so everything below is ALSO appended to REM a log file, and `ingest` exits NON-ZERO whenever it records a revision or spots a REM closed-year restatement. That is a notification, not a failure -- the data is already -REM committed. Point Task Scheduler's "send mail"/alert at this task, or just read the log. -REM Without it a retroactive restatement would be detected and silently swallowed. +REM committed. +REM +REM Task Scheduler's own "Send an e-mail" / "Display a message" actions are DEPRECATED and +REM non-functional on Windows 8 / Server 2012 and later, so a non-zero exit on its own only +REM shows up passively as Last Run Result 0x1 in the Task Scheduler UI. This script therefore +REM does its own notifying: on any run that records a revision it writes a MARKER FILE, +REM +REM \vintage\REVISIONS_.txt +REM +REM holding that run's output. It lives inside vintage\, so the existing robocopy /MIR push +REM carries it to the Mac and the alert shows up on the machine you actually work on. setlocal set COTDATA_STORE=REPLACE_WITH_STORE_PATH set VINTAGE_LOG=REPLACE_WITH_STORE_PATH\vintage\run.log +set VINTAGE_RUNOUT=REPLACE_WITH_STORE_PATH\vintage\.last_ingest.tmp + +REM Locale-independent date. %DATE% is formatted per the machine's regional settings, so it +REM is not safe in a filename; PowerShell gives a stable yyyy-MM-dd on any box. +for /f %%i in ('powershell -NoProfile -Command "Get-Date -Format yyyy-MM-dd"') do set TODAY=%%i +set VINTAGE_MARKER=REPLACE_WITH_STORE_PATH\vintage\REVISIONS_%TODAY%.txt + +REM The store may not exist yet on a first run; the vintage dir must, to hold the log. +if not exist "REPLACE_WITH_STORE_PATH\vintage" mkdir "REPLACE_WITH_STORE_PATH\vintage" REM Optional: identify yourself to CFTC. Defaults to the repo URL if unset. REM set COTDATA_USER_AGENT=cotdata-vintage/0.1 (+contact you@example.com) @@ -47,11 +65,26 @@ if %ERRORLEVEL% NEQ 0 ( REM Parse whatever was just retained into change-only observations + revisions. REM Safe to re-run: re-ingesting identical bytes writes zero rows. -"REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ingest --pending >> "%VINTAGE_LOG%" 2>&1 +REM Captured to its OWN file first, so the marker can hold just THIS run's output rather +REM than the whole appended history, then folded into the running log. +"REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ingest --pending > "%VINTAGE_RUNOUT%" 2>&1 REM NON-ZERO HERE MEANS "revisions were recorded", not "the run broke". The data is REM already written. Remember it, keep going, and re-raise at the end so the scheduler REM shows the task as attention-needed. set VINTAGE_REVISED=%ERRORLEVEL% +type "%VINTAGE_RUNOUT%" >> "%VINTAGE_LOG%" + +REM Write the marker HERE, not at the end. The revision is already committed at this +REM point, so an unrelated failure in a later step must not be able to suppress the +REM alert. Appends rather than overwrites, so two revision runs on one day both survive. +if NOT "%VINTAGE_REVISED%"=="0" ( + echo ================================================================ >> "%VINTAGE_MARKER%" + echo cotdata vintage: REVISIONS RECORDED %TODAY% >> "%VINTAGE_MARKER%" + echo Expand with: cotdata-vintage diff >> "%VINTAGE_MARKER%" + echo ================================================================ >> "%VINTAGE_MARKER%" + type "%VINTAGE_RUNOUT%" >> "%VINTAGE_MARKER%" + echo. >> "%VINTAGE_MARKER%" +) REM Resolve release dates. `published` reads the true publication timestamp out of the REM weekly static just captured (its HTTP Last-Modified), which beats a poll-derived @@ -70,9 +103,13 @@ if %ERRORLEVEL% NEQ 0 ( ) if NOT "%VINTAGE_REVISED%"=="0" ( - echo vintage ok, but REVISIONS WERE RECORDED -- see "%VINTAGE_LOG%" and run: cotdata-vintage diff - type "%VINTAGE_LOG%" + echo vintage ok, but REVISIONS WERE RECORDED + echo marker: "%VINTAGE_MARKER%" ^(syncs to the Mac with the store^) + echo log: "%VINTAGE_LOG%" + type "%VINTAGE_RUNOUT%" + del "%VINTAGE_RUNOUT%" 2>nul exit /b %VINTAGE_REVISED% ) +del "%VINTAGE_RUNOUT%" 2>nul echo vintage ok, no revisions exit /b 0 diff --git a/scripts/vintage_alert_selftest.py b/scripts/vintage_alert_selftest.py new file mode 100644 index 0000000..d8cce10 --- /dev/null +++ b/scripts/vintage_alert_selftest.py @@ -0,0 +1,125 @@ +"""Prove the vintage revision ALERT fires, without waiting for CFTC to revise anything. + +The quiet path is easy to verify (run the task, it exits 0). The loud path is not: it +only triggers when CFTC actually restates something, which may be months away — and an +alert nobody has ever seen fire is an alert nobody should trust. + +This forces a revision in a THROWAWAY store and runs the real CLI path over it, so you +can confirm the exit code and the marker-file contents your scheduled task depends on. +It never touches $COTDATA_STORE. + + python scripts/vintage_alert_selftest.py + +Expected: "SELFTEST PASSED", exit 0. Any other outcome means the notification wiring in +run-vintage.cmd would not have alerted you to a real restatement. +""" +from __future__ import annotations + +import datetime as dt +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +WIDE_COLS = { + "Market_and_Exchange_Names": ["GOLD"], + "CFTC_Contract_Market_Code": ["088691"], + "Open_Interest_All": [500000], + "Comm_Positions_Long_All": [200000], + "NonComm_Positions_Long_All": [150000], + "NonComm_Positions_Short_All": [90000], + "NonRept_Positions_Long_All": [40000], + "NonRept_Positions_Short_All": [30000], + "Traders_Comm_Long_All": [50], "Traders_Comm_Short_All": [55], + "Traders_NonComm_Long_All": [60], "Traders_NonComm_Short_All": [45], +} + + +def _wide(report_date: str, comm_short: int): + import pandas as pd + idx = pd.to_datetime([report_date]) + idx.name = "Report_Date_as_MM_DD_YYYY" + return pd.DataFrame({**WIDE_COLS, "Comm_Positions_Short_All": [comm_short]}, index=idx) + + +def main() -> int: + store = Path(tempfile.mkdtemp(prefix="cotdata_vintage_selftest_")) + os.environ["COTDATA_STORE"] = str(store) + print(f"scratch store: {store}\n(your real $COTDATA_STORE is untouched)\n") + + from cotdata import vintage + from cotdata import vintage_ingest as vi + + # Week 1: the report as first published. + vi.ingest_canonical(vi.canonicalize_legacy(_wide("2026-07-21", 250_000)), + snapshot_id="selftest-week1", + observed_at=dt.datetime(2026, 7, 24, tzinfo=dt.timezone.utc)) + # Week 2: CFTC restates one field. This is the event the alert exists for. + vi.ingest_canonical(vi.canonicalize_legacy(_wide("2026-07-21", 251_000)), + snapshot_id="selftest-week2", + observed_at=dt.datetime(2026, 7, 31, tzinfo=dt.timezone.utc)) + + revisions = vi.read_revisions() + if revisions.empty: + print("FAIL: no revision was recorded from a changed field.") + return 1 + print(f"forced {len(revisions)} revision row(s):") + cols = [c for c in ["report_date", "market_code", "category", "field", + "old_value", "new_value", "age_days"] if c in revisions.columns] + print(revisions[cols].to_string(index=False)) + + print("\n--- what 'cotdata-vintage diff' shows you ---") + sys.stdout.flush() # subprocesses write straight to the terminal; keep ordering sane + subprocess.run( + [sys.executable, "-c", + "from cotdata import vintage_cli; vintage_cli.main(['diff'])"], + env={**os.environ, "COTDATA_STORE": str(store)}, + ) + + # Drive the same CLI entry point run-vintage.cmd calls, so the exit code proves what + # the .cmd would actually observe. source_kind is weekly_static deliberately: ingest + # only parses annual zips, so this skips the parse step and keeps the demonstration + # about the ALERT rather than about a fake fixture failing to unzip. A real suspect + # would sit on an annual zip; the notification path is identical either way. + vintage._write_manifest({"schema_version": 1, "snapshots": [{ + "snapshot_id": "selftest-week2", "report_type": "legacy", + "source_kind": "weekly_static", "report_year": 2025, + "local_path": "vintage/raw/weekly_static/2026/selftest.txt", + "parse_status": "pending", "restatement_suspect": True, + "retrieved_at": "2026-07-31T21:00:00Z", + }]}) + + sys.stdout.flush() + proc = subprocess.run( + [sys.executable, "-c", + "from cotdata import vintage_cli; import sys; sys.exit(vintage_cli.main(['ingest','--pending']))"], + capture_output=True, text=True, env={**os.environ, "COTDATA_STORE": str(store)}, + ) + output = proc.stdout + proc.stderr + print("\n--- what your marker file would contain ---") + print(output.strip()) + print("--- end ---\n") + + problems = [] + if proc.returncode == 0: + problems.append("ingest exited 0; the scheduled task would have looked clean") + if "RESTATEMENT SUSPECT" not in output: + problems.append("closed-year restatement was not announced in the output") + if "cotdata-vintage diff" not in output: + problems.append("output does not tell you how to inspect the revision") + + if problems: + print("SELFTEST FAILED:") + for p in problems: + print(f" - {p}") + return 1 + + print(f"SELFTEST PASSED — ingest exited {proc.returncode} (non-zero), and the output " + f"names the restatement.\nrun-vintage.cmd writes exactly that text to " + f"vintage\\REVISIONS_.txt and exits non-zero.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())