forward-test: use SEC_USER_AGENT secret, retry throttled GDELT names, bound run time

.github/workflows/forward-test.yml

@@ -18,6 +18,8 @@ concurrency:
 jobs:
   run:
     runs-on: ubuntu-latest
+    # GDELT retries are bounded by a 40-minute budget in run_forward_test.py.
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
 
@@ -42,7 +44,8 @@ jobs:
 
       - name: Run forward test
         env:
-          SEC_USER_AGENT: ${{ vars.SEC_USER_AGENT || 'Aetherius Forward Test zarif@aetheriusresearch.tech' }}
+          # Prefer a secret: repo variables are printed in plain text in public Actions logs.
+          SEC_USER_AGENT: ${{ secrets.SEC_USER_AGENT || vars.SEC_USER_AGENT || 'Aetherius Forward Test zarif@aetheriusresearch.tech' }}
         run: |
           python simulations/forward_test/run_forward_test.py --log-dir _log/log --user-agent "$SEC_USER_AGENT"
           cp simulations/forward_test/watchlist.json _log/watchlist.json

aetherius/tests/test_forward_test.py

@@ -87,3 +87,78 @@ def test_report_flags_watchlist_changes() -> None:
     ]
     report = summ.build_report(runs, [], [], {}, {})
     assert "changed during the test" in report
+
+
+class _FakeClock:
+    def __init__(self) -> None:
+        self.t = 0.0
+
+    def __call__(self) -> float:
+        return self.t
+
+    def sleep(self, seconds: float) -> None:
+        self.t += seconds
+
+
+def _fake_fetch(fail_first: set[str], fail_always: set[str], clock: _FakeClock, cost: float = 1.0):
+    calls: list[str] = []
+
+    def fetch(company_name, **_kwargs):
+        clock.t += cost
+        calls.append(company_name)
+        if company_name in fail_always or (company_name in fail_first and calls.count(company_name) == 1):
+            raise RuntimeError("429 Too Many Requests")
+        return [{"url": f"https://example.com/{company_name}", "title": f"{company_name} news", "domain": "example.com",
+                 "seendate": "20261016T000000Z"}]
+
+    return fetch, calls
+
+
+GDELT_ITEMS = [{"ticker": t, "company_name": t, "aliases": []} for t in ("AAA", "BBB", "CCC")]
+NOW = datetime(2026, 10, 16, tzinfo=timezone.utc)
+
+
+def test_gdelt_retry_pass_recovers_throttled_names() -> None:
+    clock = _FakeClock()
+    fetch, calls = _fake_fetch({"BBB"}, set(), clock)
+    obs, errors = ft.fetch_gdelt(GDELT_ITEMS, NOW, NOW, None, fetch=fetch, sleep=clock.sleep, clock=clock)
+    assert errors == []
+    assert calls == ["AAA", "BBB", "CCC", "BBB"]
+    assert len(obs) == 3
+
+
+def test_gdelt_persistent_failure_is_reported_not_silent() -> None:
+    clock = _FakeClock()
+    fetch, calls = _fake_fetch(set(), {"CCC"}, clock)
+    obs, errors = ft.fetch_gdelt(GDELT_ITEMS, NOW, NOW, None, fetch=fetch, sleep=clock.sleep, clock=clock)
+    assert calls.count("CCC") == 2
+    assert errors == ["gdelt:CCC:429 Too Many Requests"]
+    assert len(obs) == 2
+
+
+def test_gdelt_budget_stops_new_requests_and_names_skipped() -> None:
+    clock = _FakeClock()
+    fetch, calls = _fake_fetch(set(), set(), clock, cost=100.0)
+    obs, errors = ft.fetch_gdelt(
+        GDELT_ITEMS, NOW, NOW, None, budget_seconds=150.0, fetch=fetch, sleep=clock.sleep, clock=clock
+    )
+    assert calls == ["AAA", "BBB"]
+    assert errors == ["gdelt:CCC:skipped: run time budget spent"]
+    assert clock.t < 150.0 + ft.GDELT_PAUSE_SECONDS + 100.0  # no pointless retry cooldown
+
+
+def test_report_counts_news_fetch_failures_per_name() -> None:
+    runs = [
+        {"recorded_at": "2026-10-16T00:00:00Z", "watchlist_sha256": "a", "errors": ["gdelt:NVDA:429 Too Many Requests"]},
+        {"recorded_at": "2026-10-17T00:00:00Z", "watchlist_sha256": "a", "errors": ["gdelt:NVDA:429", "edgar:X:no-cik"]},
+    ]
+    report = summ.build_report(runs, [], [], {}, {})
+    assert "| NVDA | 0 | 0 | 0.00 | 0 | 0 | 0 | 2 |" in report
+    assert "| X |" not in report
+
+
+def test_report_lists_every_preregistered_name() -> None:
+    runs = [{"recorded_at": "2026-10-16T00:00:00Z", "watchlist_sha256": "a", "errors": []}]
+    report = summ.build_report(runs, [], [], {}, {}, ["AAPL", "XOM"])
+    assert "| AAPL | 0 | 0 | 0.00 | 0 | 0 | 0 | 0 |" in report
+    assert "| XOM |" in report

simulations/forward_test/README.md

@@ -48,6 +48,12 @@ the outcome, and how many of its alerts are noise?**
 - **EDGAR metadata only.** Filings are scored on form type and date, with no
   adverse wording, so EDGAR rows alone never become flags.
 - **GDELT coverage** limits the news feed to the whitelisted domains in `gdelt_adapter.DOWNSIDE_DOMAINS`.
+- **GDELT throttling.** GDELT rate-limits GitHub's shared runners. On the first
+  run (2026-10-16), 6 of 20 names got "429 Too Many Requests" after every retry.
+  Each run now retries failed names once after a 2-minute cooldown, stops
+  starting requests after a 40-minute budget, and records every name it could
+  not fetch. The report's **News fetch failures** column shows these gaps: a
+  failed day is missing coverage, not a quiet day.
 
 ## Run locally
 

simulations/forward_test/run_forward_test.py

@@ -46,6 +46,9 @@
 
 WATCHLIST_PATH = HERE / "watchlist.json"
 GDELT_PAUSE_SECONDS = 6.0
+GDELT_RETRY_PAUSE_SECONDS = 15.0
+GDELT_COOLDOWN_SECONDS = 120.0
+GDELT_BUDGET_SECONDS = 40 * 60.0
 EDGAR_LOOKBACK_DAYS = 3
 
 
@@ -99,38 +102,78 @@ def _iso(dt: datetime) -> str:
     return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
-def fetch_gdelt(items: list[dict], start: datetime, end: datetime, client: httpx.Client) -> tuple[list[dict], list[str]]:
-    observations: dict[str, dict] = {}
-    errors: list[str] = []
-    for i, item in enumerate(items):
-        if i:
-            time.sleep(GDELT_PAUSE_SECONDS)
-        try:
-            articles = fetch_articles(
-                company_name=item["company_name"],
-                window_start=start,
-                window_end=end,
-                aliases=item.get("aliases"),
-                client=client,
-                domain_whitelist=DOWNSIDE_DOMAINS,
-            )
-        except Exception as exc:  # noqa: BLE001 - one failed name must not abort the run
-            errors.append(f"gdelt:{item['ticker']}:{exc}")
+def _gdelt_rows(item: dict, articles: list[dict], observations: dict[str, dict]) -> None:
+    for a in articles:
+        url = (a.get("url") or "").strip()
+        if not url or f"gdelt:{url}" in observations:
             continue
-        for a in articles:
-            url = (a.get("url") or "").strip()
-            if not url or f"gdelt:{url}" in observations:
+        observations[f"gdelt:{url}"] = {
+            "key": f"gdelt:{url}",
+            "source_type": "news",
+            "source_name": a.get("domain") or "GDELT news",
+            "source_url": url,
+            "title": a.get("title"),
+            "raw_text": _article_to_text(item["company_name"], a),
+            "observed_at": _iso(_parse_gdelt_seendate(a.get("seendate") or "")),
+            "source_confidence": 0.75,
+        }
+
+
+def fetch_gdelt(
+    items: list[dict],
+    start: datetime,
+    end: datetime,
+    client: httpx.Client,
+    budget_seconds: float = GDELT_BUDGET_SECONDS,
+    fetch=fetch_articles,
+    sleep=time.sleep,
+    clock=time.monotonic,
+) -> tuple[list[dict], list[str]]:
+    """Fetch GDELT news for every item, with a retry pass and a time budget.
+
+    GDELT throttles shared CI addresses hard. Names that fail the first pass
+    are retried once after a cooldown with a longer pause. Once the budget is
+    spent no new request starts; every name without news coverage is reported
+    in ``errors`` (``gdelt:<TICKER>:...``) so gaps are never silent.
+    """
+    observations: dict[str, dict] = {}
+    last_error: dict[str, str] = {}
+    deadline = clock() + budget_seconds
+    pending = list(items)
+    for attempt, pause in enumerate((GDELT_PAUSE_SECONDS, GDELT_RETRY_PAUSE_SECONDS)):
+        if attempt:
+            if not pending or clock() >= deadline:
+                break
+            print(f"gdelt: retrying {len(pending)} name(s) after {GDELT_COOLDOWN_SECONDS:.0f}s cooldown", flush=True)
+            sleep(GDELT_COOLDOWN_SECONDS)
+        failed = []
+        for i, item in enumerate(pending):
+            if clock() >= deadline:
+                failed += pending[i:]
+                for rest in pending[i:]:
+                    last_error[rest["ticker"]] = "skipped: run time budget spent"
+                break
+            if i:
+                sleep(pause)
+            try:
+                articles = fetch(
+                    company_name=item["company_name"],
+                    window_start=start,
+                    window_end=end,
+                    aliases=item.get("aliases"),
+                    client=client,
+                    domain_whitelist=DOWNSIDE_DOMAINS,
+                )
+            except Exception as exc:  # noqa: BLE001 - one failed name must not abort the run
+                last_error[item["ticker"]] = str(exc)
+                failed.append(item)
+                print(f"gdelt: {item['ticker']} failed (pass {attempt + 1}): {str(exc)[:80]}", flush=True)
                 continue
-            observations[f"gdelt:{url}"] = {
-                "key": f"gdelt:{url}",
-                "source_type": "news",
-                "source_name": a.get("domain") or "GDELT news",
-                "source_url": url,
-                "title": a.get("title"),
-                "raw_text": _article_to_text(item["company_name"], a),
-                "observed_at": _iso(_parse_gdelt_seendate(a.get("seendate") or "")),
-                "source_confidence": 0.75,
-            }
+            last_error.pop(item["ticker"], None)
+            _gdelt_rows(item, articles, observations)
+            print(f"gdelt: {item['ticker']} ok, {len(articles)} article(s) (pass {attempt + 1})", flush=True)
+        pending = failed
+    errors = [f"gdelt:{item['ticker']}:{last_error[item['ticker']]}" for item in pending]
     return list(observations.values()), errors
 
 

simulations/forward_test/summarize.py

@@ -73,13 +73,30 @@ def fetch_closes(ticker: str, start: datetime, end: datetime) -> list[tuple[date
     ]
 
 
-def build_report(runs: list[dict], observations: list[dict], flags: list[dict], labels: dict, prices: dict) -> str:
+def build_report(
+    runs: list[dict],
+    observations: list[dict],
+    flags: list[dict],
+    labels: dict,
+    prices: dict,
+    watchlist_tickers: list[str] | None = None,
+) -> str:
     L = ["# Aetherius Forward Test — Live Report", ""]
     if not runs:
         return "\n".join(L + ["No runs recorded yet."]) + "\n"
     first, last = runs[0]["recorded_at"], runs[-1]["recorded_at"]
     weeks = max((_parse(last) - _parse(first)).days / 7.0, 1 / 7.0)
-    tickers = sorted({o["ticker"] for o in observations} | {f["ticker"] for f in flags})
+    # A run whose news fetch failed for a name leaves a coverage gap; count them per name.
+    news_failures = Counter(
+        e.split(":", 2)[1] for r in runs for e in r.get("errors", []) if e.startswith("gdelt:") and e.count(":") >= 2
+    )
+    # Every pre-registered name gets a row, so a quiet name is visibly quiet rather than missing.
+    tickers = sorted(
+        set(watchlist_tickers or [])
+        | {o["ticker"] for o in observations}
+        | {f["ticker"] for f in flags}
+        | set(news_failures)
+    )
     L += [
         f"- Runs: {len(runs)} ({first} → {last}), ~{weeks:.1f} weeks",
         f"- Watchlist SHA-256: `{runs[-1]['watchlist_sha256']}`"
@@ -90,8 +107,11 @@ def build_report(runs: list[dict], observations: list[dict], flags: list[dict],
         "",
         "## Alert burden by name",
         "",
-        "| Ticker | Matched obs | Flags | Flags / week | Labeled | Material | Not material |",
-        "| --- | --- | --- | --- | --- | --- | --- |",
+        f"News fetch failures count runs (of {len(runs)}) in which GDELT returned nothing for that name; "
+        "those days are coverage gaps, not quiet days.",
+        "",
+        "| Ticker | Matched obs | Flags | Flags / week | Labeled | Material | Not material | News fetch failures |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
     ]
     obs_by = Counter(o["ticker"] for o in observations)
     flag_by = Counter(f["ticker"] for f in flags)
@@ -100,7 +120,8 @@ def build_report(runs: list[dict], observations: list[dict], flags: list[dict],
         lab = [labels[f["key"]] for f in own if f["key"] in labels]
         mat = sum(1 for x in lab if x.get("material"))
         L.append(
-            f"| {t} | {obs_by[t]} | {flag_by[t]} | {flag_by[t] / weeks:.2f} | {len(lab)} | {mat} | {len(lab) - mat} |"
+            f"| {t} | {obs_by[t]} | {flag_by[t]} | {flag_by[t] / weeks:.2f} | {len(lab)} | {mat} | {len(lab) - mat} "
+            f"| {news_failures[t]} |"
         )
     labeled = [labels[f["key"]] for f in flags if f["key"] in labels]
     L += ["", "## Precision (human-labeled)", ""]
@@ -169,7 +190,9 @@ def main() -> None:
 
     out = args.out or args.log_dir / "REPORT.md"
     out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(build_report(runs, observations, flags, labels, prices), encoding="utf-8")
+    watchlist = json.loads((HERE / "watchlist.json").read_text(encoding="utf-8"))
+    tickers = [i["ticker"] for i in watchlist["items"]]
+    out.write_text(build_report(runs, observations, flags, labels, prices, tickers), encoding="utf-8")
     print(f"wrote {out}")