fix(vintage): drain unparseable snapshots; add wrapper preflight

docs/examples/windows/run-vintage.cmd

@@ -49,8 +49,34 @@ REM is not safe in a filename; PowerShell gives a stable yyyy-MM-dd on any box.
 for /f %%i in ('powershell -NoProfile -Command "Get-Date -Format yyyy-MM-dd"') do set TODAY=%%i
 set VINTAGE_MARKER=REPLACE_WITH_STORE_PATH\vintage\REVISIONS_%TODAY%.txt
 
-REM The store may not exist yet on a first run; the vintage dir must, to hold the log.
+REM The vintage dir must exist BEFORE the first redirect below. cmd opens a >> target
+REM before running the command, so without this the very first line fails with "The system
+REM cannot find the path specified", the program never runs, and the task exits having
+REM created nothing -- which looks identical to "the task never fired".
 if not exist "REPLACE_WITH_STORE_PATH\vintage" mkdir "REPLACE_WITH_STORE_PATH\vintage"
+
+REM ---- Preflight -------------------------------------------------------------------
+REM cotdata-vintage and cotdata-schedule are NEWER than the rest of the CLI, so a venv
+REM installed before they existed will not have them and every step below would fail with
+REM an unhelpful "cannot find path". Check once, up front, and say exactly what to do.
+if not exist "REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" (
+  echo PREFLIGHT FAILED: cotdata-vintage.exe not found.
+  echo   looked in: REPLACE_WITH_VENV_PATH\Scripts\
+  echo   fix:       cd to your cotdata checkout, then:  git pull ^&^& .venv\Scripts\pip install -e .
+  exit /b 9009
+)
+if not exist "REPLACE_WITH_VENV_PATH\Scripts\cotdata-schedule.exe" (
+  echo PREFLIGHT FAILED: cotdata-schedule.exe not found.
+  echo   looked in: REPLACE_WITH_VENV_PATH\Scripts\
+  echo   fix:       cd to your cotdata checkout, then:  git pull ^&^& .venv\Scripts\pip install -e .
+  exit /b 9009
+)
+if not exist "REPLACE_WITH_STORE_PATH" (
+  echo PREFLIGHT FAILED: store path does not exist: REPLACE_WITH_STORE_PATH
+  echo   Did you replace REPLACE_WITH_STORE_PATH in this file with your real store path?
+  exit /b 3
+)
+REM ----------------------------------------------------------------------------------
 REM Optional: identify yourself to CFTC. Defaults to the repo URL if unset.
 REM set COTDATA_USER_AGENT=cotdata-vintage/0.1 (+contact you@example.com)
 

src/cotdata/vintage_cli.py

@@ -42,6 +42,17 @@ def _cmd_ingest(args) -> int:
     total_obs = total_rev = 0
     for s in snaps:
         if s.get("report_type") != "legacy" or s.get("source_kind") != "annual_zip":
+            # No canonicaliser for this report type yet. Mark it SKIPPED rather than
+            # leaving it pending: a snapshot that never drains is re-selected by
+            # --pending on every future run, and if it ever carried restatement_suspect
+            # the alert below would then re-fire forever, which is how an alert gets
+            # ignored. Skipped snapshots surface once and then go quiet.
+            # Raw bytes are retained, so adding a canonicaliser later just means
+            # re-marking these pending and re-running ingest — nothing is lost.
+            vintage.update_snapshot(
+                s["snapshot_id"], parse_status="skipped",
+                parse_error=f"no canonicaliser for {s.get('report_type')}/"
+                            f"{s.get('source_kind')} yet")
             continue
         path = config.store_root() / s["local_path"]
         try:

tests/test_vintage_ingest.py

@@ -225,6 +225,41 @@ def test_cli_ingest_exits_nonzero_when_revisions_are_recorded(store_env, capsys)
     assert "RESTATEMENT SUSPECT" in capsys.readouterr().out
 
 
+def test_unparseable_report_types_drain_out_of_pending(store_env):
+    """disagg/TFF/weekly-static have no canonicaliser yet. They must be marked SKIPPED,
+    not left pending: a snapshot that never drains is re-selected forever, and a suspect
+    one would then re-alert on every future run."""
+    from cotdata import vintage, vintage_cli
+    vintage._write_manifest({"schema_version": 1, "snapshots": [
+        {"snapshot_id": "d1", "report_type": "disaggregated", "source_kind": "annual_zip",
+         "local_path": "x", "parse_status": "pending", "retrieved_at": "2026-07-31T00:00:00Z"},
+        {"snapshot_id": "w1", "report_type": "legacy", "source_kind": "weekly_static",
+         "local_path": "y", "parse_status": "pending", "retrieved_at": "2026-07-31T00:00:00Z"},
+    ]})
+    assert vintage_cli.main(["ingest", "--pending"]) == 0
+
+    after = {s["snapshot_id"]: s for s in vintage.read_snapshots()}
+    assert after["d1"]["parse_status"] == "skipped"
+    assert after["w1"]["parse_status"] == "skipped"
+    assert "no canonicaliser" in after["d1"]["parse_error"]
+    # and a second run selects nothing, so it stays quiet
+    assert vintage_cli.main(["ingest", "--pending"]) == 0
+
+
+def test_suspect_on_an_unparseable_type_alerts_once_then_drains(store_env):
+    """The alert-fatigue path this fix closes: a suspect that never drains re-fires
+    every run forever."""
+    from cotdata import vintage, vintage_cli
+    vintage._write_manifest({"schema_version": 1, "snapshots": [
+        {"snapshot_id": "d1", "report_type": "disaggregated", "source_kind": "annual_zip",
+         "local_path": "x", "parse_status": "pending", "restatement_suspect": True,
+         "report_year": 2025, "retrieved_at": "2026-07-31T00:00:00Z"},
+    ]})
+    with pytest.raises(SystemExit):      # fires once
+        vintage_cli.main(["ingest", "--pending"])
+    assert vintage_cli.main(["ingest", "--pending"]) == 0   # and then goes quiet
+
+
 def test_restatement_alert_does_not_fire_forever(store_env):
     """A suspect recorded in an EARLIER run must not keep failing every later run —
     an alert that never clears is one that gets switched off."""