feat(vintage): revision marker file + alert selftest

docs/examples/windows/run-vintage.cmd

@@ -28,11 +28,29 @@ REM single sync carries both the current-state update and the new vintage snapsh
 REM NOTIFICATION: Task Scheduler discards stdout, so everything below is ALSO appended to
 REM a log file, and `ingest` exits NON-ZERO whenever it records a revision or spots a
 REM closed-year restatement. That is a notification, not a failure -- the data is already
-REM committed. Point Task Scheduler's "send mail"/alert at this task, or just read the log.
-REM Without it a retroactive restatement would be detected and silently swallowed.
+REM committed.
+REM
+REM Task Scheduler's own "Send an e-mail" / "Display a message" actions are DEPRECATED and
+REM non-functional on Windows 8 / Server 2012 and later, so a non-zero exit on its own only
+REM shows up passively as Last Run Result 0x1 in the Task Scheduler UI. This script therefore
+REM does its own notifying: on any run that records a revision it writes a MARKER FILE,
+REM
+REM     <store>\vintage\REVISIONS_<yyyy-MM-dd>.txt
+REM
+REM holding that run's output. It lives inside vintage\, so the existing robocopy /MIR push
+REM carries it to the Mac and the alert shows up on the machine you actually work on.
 setlocal
 set COTDATA_STORE=REPLACE_WITH_STORE_PATH
 set VINTAGE_LOG=REPLACE_WITH_STORE_PATH\vintage\run.log
+set VINTAGE_RUNOUT=REPLACE_WITH_STORE_PATH\vintage\.last_ingest.tmp
+
+REM Locale-independent date. %DATE% is formatted per the machine's regional settings, so it
+REM is not safe in a filename; PowerShell gives a stable yyyy-MM-dd on any box.
+for /f %%i in ('powershell -NoProfile -Command "Get-Date -Format yyyy-MM-dd"') do set TODAY=%%i
+set VINTAGE_MARKER=REPLACE_WITH_STORE_PATH\vintage\REVISIONS_%TODAY%.txt
+
+REM The store may not exist yet on a first run; the vintage dir must, to hold the log.
+if not exist "REPLACE_WITH_STORE_PATH\vintage" mkdir "REPLACE_WITH_STORE_PATH\vintage"
 REM Optional: identify yourself to CFTC. Defaults to the repo URL if unset.
 REM set COTDATA_USER_AGENT=cotdata-vintage/0.1 (+contact you@example.com)
 
@@ -47,11 +65,26 @@ if %ERRORLEVEL% NEQ 0 (
 
 REM Parse whatever was just retained into change-only observations + revisions.
 REM Safe to re-run: re-ingesting identical bytes writes zero rows.
-"REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ingest --pending >> "%VINTAGE_LOG%" 2>&1
+REM Captured to its OWN file first, so the marker can hold just THIS run's output rather
+REM than the whole appended history, then folded into the running log.
+"REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ingest --pending > "%VINTAGE_RUNOUT%" 2>&1
 REM NON-ZERO HERE MEANS "revisions were recorded", not "the run broke". The data is
 REM already written. Remember it, keep going, and re-raise at the end so the scheduler
 REM shows the task as attention-needed.
 set VINTAGE_REVISED=%ERRORLEVEL%
+type "%VINTAGE_RUNOUT%" >> "%VINTAGE_LOG%"
+
+REM Write the marker HERE, not at the end. The revision is already committed at this
+REM point, so an unrelated failure in a later step must not be able to suppress the
+REM alert. Appends rather than overwrites, so two revision runs on one day both survive.
+if NOT "%VINTAGE_REVISED%"=="0" (
+  echo ================================================================ >> "%VINTAGE_MARKER%"
+  echo cotdata vintage: REVISIONS RECORDED %TODAY% >> "%VINTAGE_MARKER%"
+  echo Expand with:  cotdata-vintage diff >> "%VINTAGE_MARKER%"
+  echo ================================================================ >> "%VINTAGE_MARKER%"
+  type "%VINTAGE_RUNOUT%" >> "%VINTAGE_MARKER%"
+  echo. >> "%VINTAGE_MARKER%"
+)
 
 REM Resolve release dates. `published` reads the true publication timestamp out of the
 REM weekly static just captured (its HTTP Last-Modified), which beats a poll-derived
@@ -70,9 +103,13 @@ if %ERRORLEVEL% NEQ 0 (
 )
 
 if NOT "%VINTAGE_REVISED%"=="0" (
-  echo vintage ok, but REVISIONS WERE RECORDED -- see "%VINTAGE_LOG%" and run: cotdata-vintage diff
-  type "%VINTAGE_LOG%"
+  echo vintage ok, but REVISIONS WERE RECORDED
+  echo   marker: "%VINTAGE_MARKER%"   ^(syncs to the Mac with the store^)
+  echo   log:    "%VINTAGE_LOG%"
+  type "%VINTAGE_RUNOUT%"
+  del "%VINTAGE_RUNOUT%" 2>nul
   exit /b %VINTAGE_REVISED%
 )
+del "%VINTAGE_RUNOUT%" 2>nul
 echo vintage ok, no revisions
 exit /b 0

scripts/vintage_alert_selftest.py

@@ -0,0 +1,125 @@
+"""Prove the vintage revision ALERT fires, without waiting for CFTC to revise anything.
+
+The quiet path is easy to verify (run the task, it exits 0). The loud path is not: it
+only triggers when CFTC actually restates something, which may be months away — and an
+alert nobody has ever seen fire is an alert nobody should trust.
+
+This forces a revision in a THROWAWAY store and runs the real CLI path over it, so you
+can confirm the exit code and the marker-file contents your scheduled task depends on.
+It never touches $COTDATA_STORE.
+
+    python scripts/vintage_alert_selftest.py
+
+Expected: "SELFTEST PASSED", exit 0. Any other outcome means the notification wiring in
+run-vintage.cmd would not have alerted you to a real restatement.
+"""
+from __future__ import annotations
+
+import datetime as dt
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+WIDE_COLS = {
+    "Market_and_Exchange_Names": ["GOLD"],
+    "CFTC_Contract_Market_Code": ["088691"],
+    "Open_Interest_All": [500000],
+    "Comm_Positions_Long_All": [200000],
+    "NonComm_Positions_Long_All": [150000],
+    "NonComm_Positions_Short_All": [90000],
+    "NonRept_Positions_Long_All": [40000],
+    "NonRept_Positions_Short_All": [30000],
+    "Traders_Comm_Long_All": [50], "Traders_Comm_Short_All": [55],
+    "Traders_NonComm_Long_All": [60], "Traders_NonComm_Short_All": [45],
+}
+
+
+def _wide(report_date: str, comm_short: int):
+    import pandas as pd
+    idx = pd.to_datetime([report_date])
+    idx.name = "Report_Date_as_MM_DD_YYYY"
+    return pd.DataFrame({**WIDE_COLS, "Comm_Positions_Short_All": [comm_short]}, index=idx)
+
+
+def main() -> int:
+    store = Path(tempfile.mkdtemp(prefix="cotdata_vintage_selftest_"))
+    os.environ["COTDATA_STORE"] = str(store)
+    print(f"scratch store: {store}\n(your real $COTDATA_STORE is untouched)\n")
+
+    from cotdata import vintage
+    from cotdata import vintage_ingest as vi
+
+    # Week 1: the report as first published.
+    vi.ingest_canonical(vi.canonicalize_legacy(_wide("2026-07-21", 250_000)),
+                        snapshot_id="selftest-week1",
+                        observed_at=dt.datetime(2026, 7, 24, tzinfo=dt.timezone.utc))
+    # Week 2: CFTC restates one field. This is the event the alert exists for.
+    vi.ingest_canonical(vi.canonicalize_legacy(_wide("2026-07-21", 251_000)),
+                        snapshot_id="selftest-week2",
+                        observed_at=dt.datetime(2026, 7, 31, tzinfo=dt.timezone.utc))
+
+    revisions = vi.read_revisions()
+    if revisions.empty:
+        print("FAIL: no revision was recorded from a changed field.")
+        return 1
+    print(f"forced {len(revisions)} revision row(s):")
+    cols = [c for c in ["report_date", "market_code", "category", "field",
+                        "old_value", "new_value", "age_days"] if c in revisions.columns]
+    print(revisions[cols].to_string(index=False))
+
+    print("\n--- what 'cotdata-vintage diff' shows you ---")
+    sys.stdout.flush()  # subprocesses write straight to the terminal; keep ordering sane
+    subprocess.run(
+        [sys.executable, "-c",
+         "from cotdata import vintage_cli; vintage_cli.main(['diff'])"],
+        env={**os.environ, "COTDATA_STORE": str(store)},
+    )
+
+    # Drive the same CLI entry point run-vintage.cmd calls, so the exit code proves what
+    # the .cmd would actually observe. source_kind is weekly_static deliberately: ingest
+    # only parses annual zips, so this skips the parse step and keeps the demonstration
+    # about the ALERT rather than about a fake fixture failing to unzip. A real suspect
+    # would sit on an annual zip; the notification path is identical either way.
+    vintage._write_manifest({"schema_version": 1, "snapshots": [{
+        "snapshot_id": "selftest-week2", "report_type": "legacy",
+        "source_kind": "weekly_static", "report_year": 2025,
+        "local_path": "vintage/raw/weekly_static/2026/selftest.txt",
+        "parse_status": "pending", "restatement_suspect": True,
+        "retrieved_at": "2026-07-31T21:00:00Z",
+    }]})
+
+    sys.stdout.flush()
+    proc = subprocess.run(
+        [sys.executable, "-c",
+         "from cotdata import vintage_cli; import sys; sys.exit(vintage_cli.main(['ingest','--pending']))"],
+        capture_output=True, text=True, env={**os.environ, "COTDATA_STORE": str(store)},
+    )
+    output = proc.stdout + proc.stderr
+    print("\n--- what your marker file would contain ---")
+    print(output.strip())
+    print("--- end ---\n")
+
+    problems = []
+    if proc.returncode == 0:
+        problems.append("ingest exited 0; the scheduled task would have looked clean")
+    if "RESTATEMENT SUSPECT" not in output:
+        problems.append("closed-year restatement was not announced in the output")
+    if "cotdata-vintage diff" not in output:
+        problems.append("output does not tell you how to inspect the revision")
+
+    if problems:
+        print("SELFTEST FAILED:")
+        for p in problems:
+            print(f"  - {p}")
+        return 1
+
+    print(f"SELFTEST PASSED — ingest exited {proc.returncode} (non-zero), and the output "
+          f"names the restatement.\nrun-vintage.cmd writes exactly that text to "
+          f"vintage\\REVISIONS_<date>.txt and exits non-zero.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())