diff --git a/docs/examples/windows/run-vintage.cmd b/docs/examples/windows/run-vintage.cmd index 9880348..f17a60a 100644 --- a/docs/examples/windows/run-vintage.cmd +++ b/docs/examples/windows/run-vintage.cmd @@ -49,8 +49,34 @@ REM is not safe in a filename; PowerShell gives a stable yyyy-MM-dd on any box. for /f %%i in ('powershell -NoProfile -Command "Get-Date -Format yyyy-MM-dd"') do set TODAY=%%i set VINTAGE_MARKER=REPLACE_WITH_STORE_PATH\vintage\REVISIONS_%TODAY%.txt -REM The store may not exist yet on a first run; the vintage dir must, to hold the log. +REM The vintage dir must exist BEFORE the first redirect below. cmd opens a >> target +REM before running the command, so without this the very first line fails with "The system +REM cannot find the path specified", the program never runs, and the task exits having +REM created nothing -- which looks identical to "the task never fired". if not exist "REPLACE_WITH_STORE_PATH\vintage" mkdir "REPLACE_WITH_STORE_PATH\vintage" + +REM ---- Preflight ------------------------------------------------------------------- +REM cotdata-vintage and cotdata-schedule are NEWER than the rest of the CLI, so a venv +REM installed before they existed will not have them and every step below would fail with +REM an unhelpful "cannot find path". Check once, up front, and say exactly what to do. +if not exist "REPLACE_WITH_VENV_PATH\Scripts\cotdata-vintage.exe" ( + echo PREFLIGHT FAILED: cotdata-vintage.exe not found. + echo looked in: REPLACE_WITH_VENV_PATH\Scripts\ + echo fix: cd to your cotdata checkout, then: git pull ^&^& .venv\Scripts\pip install -e . + exit /b 9009 +) +if not exist "REPLACE_WITH_VENV_PATH\Scripts\cotdata-schedule.exe" ( + echo PREFLIGHT FAILED: cotdata-schedule.exe not found. + echo looked in: REPLACE_WITH_VENV_PATH\Scripts\ + echo fix: cd to your cotdata checkout, then: git pull ^&^& .venv\Scripts\pip install -e . + exit /b 9009 +) +if not exist "REPLACE_WITH_STORE_PATH" ( + echo PREFLIGHT FAILED: store path does not exist: REPLACE_WITH_STORE_PATH + echo Did you replace REPLACE_WITH_STORE_PATH in this file with your real store path? + exit /b 3 +) +REM ---------------------------------------------------------------------------------- REM Optional: identify yourself to CFTC. Defaults to the repo URL if unset. REM set COTDATA_USER_AGENT=cotdata-vintage/0.1 (+contact you@example.com) diff --git a/src/cotdata/vintage_cli.py b/src/cotdata/vintage_cli.py index 4ee7263..c0874da 100644 --- a/src/cotdata/vintage_cli.py +++ b/src/cotdata/vintage_cli.py @@ -42,6 +42,17 @@ def _cmd_ingest(args) -> int: total_obs = total_rev = 0 for s in snaps: if s.get("report_type") != "legacy" or s.get("source_kind") != "annual_zip": + # No canonicaliser for this report type yet. Mark it SKIPPED rather than + # leaving it pending: a snapshot that never drains is re-selected by + # --pending on every future run, and if it ever carried restatement_suspect + # the alert below would then re-fire forever, which is how an alert gets + # ignored. Skipped snapshots surface once and then go quiet. + # Raw bytes are retained, so adding a canonicaliser later just means + # re-marking these pending and re-running ingest — nothing is lost. + vintage.update_snapshot( + s["snapshot_id"], parse_status="skipped", + parse_error=f"no canonicaliser for {s.get('report_type')}/" + f"{s.get('source_kind')} yet") continue path = config.store_root() / s["local_path"] try: diff --git a/tests/test_vintage_ingest.py b/tests/test_vintage_ingest.py index 6c27fb6..7b1151e 100644 --- a/tests/test_vintage_ingest.py +++ b/tests/test_vintage_ingest.py @@ -225,6 +225,41 @@ def test_cli_ingest_exits_nonzero_when_revisions_are_recorded(store_env, capsys) assert "RESTATEMENT SUSPECT" in capsys.readouterr().out +def test_unparseable_report_types_drain_out_of_pending(store_env): + """disagg/TFF/weekly-static have no canonicaliser yet. They must be marked SKIPPED, + not left pending: a snapshot that never drains is re-selected forever, and a suspect + one would then re-alert on every future run.""" + from cotdata import vintage, vintage_cli + vintage._write_manifest({"schema_version": 1, "snapshots": [ + {"snapshot_id": "d1", "report_type": "disaggregated", "source_kind": "annual_zip", + "local_path": "x", "parse_status": "pending", "retrieved_at": "2026-07-31T00:00:00Z"}, + {"snapshot_id": "w1", "report_type": "legacy", "source_kind": "weekly_static", + "local_path": "y", "parse_status": "pending", "retrieved_at": "2026-07-31T00:00:00Z"}, + ]}) + assert vintage_cli.main(["ingest", "--pending"]) == 0 + + after = {s["snapshot_id"]: s for s in vintage.read_snapshots()} + assert after["d1"]["parse_status"] == "skipped" + assert after["w1"]["parse_status"] == "skipped" + assert "no canonicaliser" in after["d1"]["parse_error"] + # and a second run selects nothing, so it stays quiet + assert vintage_cli.main(["ingest", "--pending"]) == 0 + + +def test_suspect_on_an_unparseable_type_alerts_once_then_drains(store_env): + """The alert-fatigue path this fix closes: a suspect that never drains re-fires + every run forever.""" + from cotdata import vintage, vintage_cli + vintage._write_manifest({"schema_version": 1, "snapshots": [ + {"snapshot_id": "d1", "report_type": "disaggregated", "source_kind": "annual_zip", + "local_path": "x", "parse_status": "pending", "restatement_suspect": True, + "report_year": 2025, "retrieved_at": "2026-07-31T00:00:00Z"}, + ]}) + with pytest.raises(SystemExit): # fires once + vintage_cli.main(["ingest", "--pending"]) + assert vintage_cli.main(["ingest", "--pending"]) == 0 # and then goes quiet + + def test_restatement_alert_does_not_fire_forever(store_env): """A suspect recorded in an EARLIER run must not keep failing every later run — an alert that never clears is one that gets switched off."""