From 0f9588a5876af27361d6fd72d9d5569b0c9215e0 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 30 Jul 2026 21:24:02 -0400 Subject: [PATCH] fix(vintage): drain unparseable snapshots; add wrapper preflight Both found from the first real production capture, which retained 4 snapshots but ingested only the Legacy annual zip. DRAIN BUG. disagg/TFF/weekly-static have no canonicaliser yet, so ingest 'continue'd past them and left them parse_status=pending forever. Two consequences: --pending never drains, and -- the real problem -- a pending snapshot carrying restatement_suspect would re-fire the alert on EVERY subsequent run, forever. Scoping suspects to the snapshots a run processed only works if snapshots actually drain out of pending, which these never did. They are now marked 'skipped' with a reason, so they surface once and go quiet. Raw bytes are retained, so adding a canonicaliser later just means re-marking them pending. PREFLIGHT. run-vintage.cmd now checks up front that cotdata-vintage.exe and cotdata-schedule.exe exist and that the store path is real, naming the fix (git pull && pip install -e .) instead of failing four times with 'cannot find path'. Those two entry points are newer than the rest of the CLI, so a venv installed before they existed has neither. Also expands the comment on why the vintage dir must be created BEFORE the first redirect: cmd opens a >> target before running the command, so without it the first line fails, nothing runs, and the task looks like it never fired -- which is exactly what happened on the first real setup. Co-Authored-By: Claude Opus 4.8 --- docs/examples/windows/run-vintage.cmd | 28 ++++++++++++++++++++- src/cotdata/vintage_cli.py | 11 +++++++++ tests/test_vintage_ingest.py | 35 +++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/examples/windows/run-vintage.cmd b/docs/examples/windows/run-vintage.cmd index 9880348..f17a60a 100644 --- a/docs/examples/windows/run-vintage.cmd +++ b/docs/examples/windows/run-vintage.cmd @@ -49,8 +49,34 @@ REM is not safe in a filename; PowerShell gives a stable yyyy-MM-dd on any box. for /f %%i in ('powershell -NoProfile -Command "Get-Date -Format yyyy-MM-dd"') do set TODAY=%%i set VINTAGE_MARKER=REPLACE_WITH_STORE_PATH\vintage\REVISIONS_%TODAY%.txt -REM The store may not exist yet on a first run; the vintage dir must, to hold the log. +REM The vintage dir must exist BEFORE the first redirect below. cmd opens a >> target +REM before running the command, so without this the very first line fails with "The system +REM cannot find the path specified", the program never runs, and the task exits having +REM created nothing -- which looks identical to "the task never fired". if not exist "REPLACE_WITH_STORE_PATH\vintage" mkdir "REPLACE_WITH_STORE_PATH\vintage" + +REM ---- Preflight ------------------------------------------------------------------- +REM cotdata-vintage and cotdata-schedule are NEWER than the rest of the CLI, so a venv +REM installed before they existed will not have them and every step below would fail with +REM an unhelpful "cannot find path". Check once, up front, and say exactly what to do. +if not exist "REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ( + echo PREFLIGHT FAILED: cotdata-vintage.exe not found. + echo looked in: REPLACE_WITH_VENV_PATH\Scripts\ + echo fix: cd to your cotdata checkout, then: git pull ^&^& .venv\Scripts\pip install -e . + exit /b 9009 +) +if not exist "REPLACE_WITH_VENV_PATH\Scripts\cotdata-schedule.exe" ( + echo PREFLIGHT FAILED: cotdata-schedule.exe not found. + echo looked in: REPLACE_WITH_VENV_PATH\Scripts\ + echo fix: cd to your cotdata checkout, then: git pull ^&^& .venv\Scripts\pip install -e . + exit /b 9009 +) +if not exist "REPLACE_WITH_STORE_PATH" ( + echo PREFLIGHT FAILED: store path does not exist: REPLACE_WITH_STORE_PATH + echo Did you replace REPLACE_WITH_STORE_PATH in this file with your real store path? + exit /b 3 +) +REM ---------------------------------------------------------------------------------- REM Optional: identify yourself to CFTC. Defaults to the repo URL if unset. REM set COTDATA_USER_AGENT=cotdata-vintage/0.1 (+contact you@example.com) diff --git a/src/cotdata/vintage_cli.py b/src/cotdata/vintage_cli.py index 4ee7263..c0874da 100644 --- a/src/cotdata/vintage_cli.py +++ b/src/cotdata/vintage_cli.py @@ -42,6 +42,17 @@ def _cmd_ingest(args) -> int: total_obs = total_rev = 0 for s in snaps: if s.get("report_type") != "legacy" or s.get("source_kind") != "annual_zip": + # No canonicaliser for this report type yet. Mark it SKIPPED rather than + # leaving it pending: a snapshot that never drains is re-selected by + # --pending on every future run, and if it ever carried restatement_suspect + # the alert below would then re-fire forever, which is how an alert gets + # ignored. Skipped snapshots surface once and then go quiet. + # Raw bytes are retained, so adding a canonicaliser later just means + # re-marking these pending and re-running ingest — nothing is lost. + vintage.update_snapshot( + s["snapshot_id"], parse_status="skipped", + parse_error=f"no canonicaliser for {s.get('report_type')}/" + f"{s.get('source_kind')} yet") continue path = config.store_root() / s["local_path"] try: diff --git a/tests/test_vintage_ingest.py b/tests/test_vintage_ingest.py index 6c27fb6..7b1151e 100644 --- a/tests/test_vintage_ingest.py +++ b/tests/test_vintage_ingest.py @@ -225,6 +225,41 @@ def test_cli_ingest_exits_nonzero_when_revisions_are_recorded(store_env, capsys) assert "RESTATEMENT SUSPECT" in capsys.readouterr().out +def test_unparseable_report_types_drain_out_of_pending(store_env): + """disagg/TFF/weekly-static have no canonicaliser yet. They must be marked SKIPPED, + not left pending: a snapshot that never drains is re-selected forever, and a suspect + one would then re-alert on every future run.""" + from cotdata import vintage, vintage_cli + vintage._write_manifest({"schema_version": 1, "snapshots": [ + {"snapshot_id": "d1", "report_type": "disaggregated", "source_kind": "annual_zip", + "local_path": "x", "parse_status": "pending", "retrieved_at": "2026-07-31T00:00:00Z"}, + {"snapshot_id": "w1", "report_type": "legacy", "source_kind": "weekly_static", + "local_path": "y", "parse_status": "pending", "retrieved_at": "2026-07-31T00:00:00Z"}, + ]}) + assert vintage_cli.main(["ingest", "--pending"]) == 0 + + after = {s["snapshot_id"]: s for s in vintage.read_snapshots()} + assert after["d1"]["parse_status"] == "skipped" + assert after["w1"]["parse_status"] == "skipped" + assert "no canonicaliser" in after["d1"]["parse_error"] + # and a second run selects nothing, so it stays quiet + assert vintage_cli.main(["ingest", "--pending"]) == 0 + + +def test_suspect_on_an_unparseable_type_alerts_once_then_drains(store_env): + """The alert-fatigue path this fix closes: a suspect that never drains re-fires + every run forever.""" + from cotdata import vintage, vintage_cli + vintage._write_manifest({"schema_version": 1, "snapshots": [ + {"snapshot_id": "d1", "report_type": "disaggregated", "source_kind": "annual_zip", + "local_path": "x", "parse_status": "pending", "restatement_suspect": True, + "report_year": 2025, "retrieved_at": "2026-07-31T00:00:00Z"}, + ]}) + with pytest.raises(SystemExit): # fires once + vintage_cli.main(["ingest", "--pending"]) + assert vintage_cli.main(["ingest", "--pending"]) == 0 # and then goes quiet + + def test_restatement_alert_does_not_fire_forever(store_env): """A suspect recorded in an EARLIER run must not keep failing every later run — an alert that never clears is one that gets switched off."""