From b22c0b8e4859b4e8c09c9b9259e80fe1c3f6ce4c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:13:45 +0000 Subject: [PATCH 1/2] forward-test: read SEC_USER_AGENT from a secret first Repository variables are printed unmasked in public Actions logs; secrets are masked. Falls back to the variable, then the default. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HTqzkhjyuhZZun8Az1tW6C --- .github/workflows/forward-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/forward-test.yml b/.github/workflows/forward-test.yml index 58f2d1b..1d7908e 100644 --- a/.github/workflows/forward-test.yml +++ b/.github/workflows/forward-test.yml @@ -42,7 +42,8 @@ jobs: - name: Run forward test env: - SEC_USER_AGENT: ${{ vars.SEC_USER_AGENT || 'Aetherius Forward Test zarif@aetheriusresearch.tech' }} + # Prefer a secret: repo variables are printed in plain text in public Actions logs. + SEC_USER_AGENT: ${{ secrets.SEC_USER_AGENT || vars.SEC_USER_AGENT || 'Aetherius Forward Test zarif@aetheriusresearch.tech' }} run: | python simulations/forward_test/run_forward_test.py --log-dir _log/log --user-agent "$SEC_USER_AGENT" cp simulations/forward_test/watchlist.json _log/watchlist.json From 0fe1c8e8dc47562f395bb899ea436e86c36658e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:53:15 +0000 Subject: [PATCH 2/2] forward-test: retry throttled GDELT names, bound run time, surface gaps The first live run took 22 minutes and 6 of 20 names (NVDA, AMZN, BAC, BA, GM, COIN) got GDELT 429s after every retry, leaving silent coverage gaps. Failed names now get one retry pass after a cooldown with a longer pause; a 40-minute budget stops new requests and records skipped names; progress prints per name; the job has a 60-minute timeout. The report lists all 20 pre-registered names and a per-name news-fetch-failure count. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HTqzkhjyuhZZun8Az1tW6C --- .github/workflows/forward-test.yml | 2 + aetherius/tests/test_forward_test.py | 75 ++++++++++++++ simulations/forward_test/README.md | 6 ++ simulations/forward_test/run_forward_test.py | 103 +++++++++++++------ simulations/forward_test/summarize.py | 35 +++++-- 5 files changed, 185 insertions(+), 36 deletions(-) diff --git a/.github/workflows/forward-test.yml b/.github/workflows/forward-test.yml index 1d7908e..270bdee 100644 --- a/.github/workflows/forward-test.yml +++ b/.github/workflows/forward-test.yml @@ -18,6 +18,8 @@ concurrency: jobs: run: runs-on: ubuntu-latest + # GDELT retries are bounded by a 40-minute budget in run_forward_test.py. + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/aetherius/tests/test_forward_test.py b/aetherius/tests/test_forward_test.py index 7db56a9..a27b05c 100644 --- a/aetherius/tests/test_forward_test.py +++ b/aetherius/tests/test_forward_test.py @@ -87,3 +87,78 @@ def test_report_flags_watchlist_changes() -> None: ] report = summ.build_report(runs, [], [], {}, {}) assert "changed during the test" in report + + +class _FakeClock: + def __init__(self) -> None: + self.t = 0.0 + + def __call__(self) -> float: + return self.t + + def sleep(self, seconds: float) -> None: + self.t += seconds + + +def _fake_fetch(fail_first: set[str], fail_always: set[str], clock: _FakeClock, cost: float = 1.0): + calls: list[str] = [] + + def fetch(company_name, **_kwargs): + clock.t += cost + calls.append(company_name) + if company_name in fail_always or (company_name in fail_first and calls.count(company_name) == 1): + raise RuntimeError("429 Too Many Requests") + return [{"url": f"https://example.com/{company_name}", "title": f"{company_name} news", "domain": "example.com", + "seendate": "20261016T000000Z"}] + + return fetch, calls + + +GDELT_ITEMS = [{"ticker": t, "company_name": t, "aliases": []} for t in ("AAA", "BBB", "CCC")] +NOW = datetime(2026, 10, 16, tzinfo=timezone.utc) + + +def test_gdelt_retry_pass_recovers_throttled_names() -> None: + clock = _FakeClock() + fetch, calls = _fake_fetch({"BBB"}, set(), clock) + obs, errors = ft.fetch_gdelt(GDELT_ITEMS, NOW, NOW, None, fetch=fetch, sleep=clock.sleep, clock=clock) + assert errors == [] + assert calls == ["AAA", "BBB", "CCC", "BBB"] + assert len(obs) == 3 + + +def test_gdelt_persistent_failure_is_reported_not_silent() -> None: + clock = _FakeClock() + fetch, calls = _fake_fetch(set(), {"CCC"}, clock) + obs, errors = ft.fetch_gdelt(GDELT_ITEMS, NOW, NOW, None, fetch=fetch, sleep=clock.sleep, clock=clock) + assert calls.count("CCC") == 2 + assert errors == ["gdelt:CCC:429 Too Many Requests"] + assert len(obs) == 2 + + +def test_gdelt_budget_stops_new_requests_and_names_skipped() -> None: + clock = _FakeClock() + fetch, calls = _fake_fetch(set(), set(), clock, cost=100.0) + obs, errors = ft.fetch_gdelt( + GDELT_ITEMS, NOW, NOW, None, budget_seconds=150.0, fetch=fetch, sleep=clock.sleep, clock=clock + ) + assert calls == ["AAA", "BBB"] + assert errors == ["gdelt:CCC:skipped: run time budget spent"] + assert clock.t < 150.0 + ft.GDELT_PAUSE_SECONDS + 100.0 # no pointless retry cooldown + + +def test_report_counts_news_fetch_failures_per_name() -> None: + runs = [ + {"recorded_at": "2026-10-16T00:00:00Z", "watchlist_sha256": "a", "errors": ["gdelt:NVDA:429 Too Many Requests"]}, + {"recorded_at": "2026-10-17T00:00:00Z", "watchlist_sha256": "a", "errors": ["gdelt:NVDA:429", "edgar:X:no-cik"]}, + ] + report = summ.build_report(runs, [], [], {}, {}) + assert "| NVDA | 0 | 0 | 0.00 | 0 | 0 | 0 | 2 |" in report + assert "| X |" not in report + + +def test_report_lists_every_preregistered_name() -> None: + runs = [{"recorded_at": "2026-10-16T00:00:00Z", "watchlist_sha256": "a", "errors": []}] + report = summ.build_report(runs, [], [], {}, {}, ["AAPL", "XOM"]) + assert "| AAPL | 0 | 0 | 0.00 | 0 | 0 | 0 | 0 |" in report + assert "| XOM |" in report diff --git a/simulations/forward_test/README.md b/simulations/forward_test/README.md index b9815f7..5c8f8a7 100644 --- a/simulations/forward_test/README.md +++ b/simulations/forward_test/README.md @@ -48,6 +48,12 @@ the outcome, and how many of its alerts are noise?** - **EDGAR metadata only.** Filings are scored on form type and date, with no adverse wording, so EDGAR rows alone never become flags. - **GDELT coverage** limits the news feed to the whitelisted domains in `gdelt_adapter.DOWNSIDE_DOMAINS`. +- **GDELT throttling.** GDELT rate-limits GitHub's shared runners. On the first + run (2026-10-16), 6 of 20 names got "429 Too Many Requests" after every retry. + Each run now retries failed names once after a 2-minute cooldown, stops + starting requests after a 40-minute budget, and records every name it could + not fetch. The report's **News fetch failures** column shows these gaps: a + failed day is missing coverage, not a quiet day. ## Run locally diff --git a/simulations/forward_test/run_forward_test.py b/simulations/forward_test/run_forward_test.py index 7164d38..941ff50 100644 --- a/simulations/forward_test/run_forward_test.py +++ b/simulations/forward_test/run_forward_test.py @@ -46,6 +46,9 @@ WATCHLIST_PATH = HERE / "watchlist.json" GDELT_PAUSE_SECONDS = 6.0 +GDELT_RETRY_PAUSE_SECONDS = 15.0 +GDELT_COOLDOWN_SECONDS = 120.0 +GDELT_BUDGET_SECONDS = 40 * 60.0 EDGAR_LOOKBACK_DAYS = 3 @@ -99,38 +102,78 @@ def _iso(dt: datetime) -> str: return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def fetch_gdelt(items: list[dict], start: datetime, end: datetime, client: httpx.Client) -> tuple[list[dict], list[str]]: - observations: dict[str, dict] = {} - errors: list[str] = [] - for i, item in enumerate(items): - if i: - time.sleep(GDELT_PAUSE_SECONDS) - try: - articles = fetch_articles( - company_name=item["company_name"], - window_start=start, - window_end=end, - aliases=item.get("aliases"), - client=client, - domain_whitelist=DOWNSIDE_DOMAINS, - ) - except Exception as exc: # noqa: BLE001 - one failed name must not abort the run - errors.append(f"gdelt:{item['ticker']}:{exc}") +def _gdelt_rows(item: dict, articles: list[dict], observations: dict[str, dict]) -> None: + for a in articles: + url = (a.get("url") or "").strip() + if not url or f"gdelt:{url}" in observations: continue - for a in articles: - url = (a.get("url") or "").strip() - if not url or f"gdelt:{url}" in observations: + observations[f"gdelt:{url}"] = { + "key": f"gdelt:{url}", + "source_type": "news", + "source_name": a.get("domain") or "GDELT news", + "source_url": url, + "title": a.get("title"), + "raw_text": _article_to_text(item["company_name"], a), + "observed_at": _iso(_parse_gdelt_seendate(a.get("seendate") or "")), + "source_confidence": 0.75, + } + + +def fetch_gdelt( + items: list[dict], + start: datetime, + end: datetime, + client: httpx.Client, + budget_seconds: float = GDELT_BUDGET_SECONDS, + fetch=fetch_articles, + sleep=time.sleep, + clock=time.monotonic, +) -> tuple[list[dict], list[str]]: + """Fetch GDELT news for every item, with a retry pass and a time budget. + + GDELT throttles shared CI addresses hard. Names that fail the first pass + are retried once after a cooldown with a longer pause. Once the budget is + spent no new request starts; every name without news coverage is reported + in ``errors`` (``gdelt::...``) so gaps are never silent. + """ + observations: dict[str, dict] = {} + last_error: dict[str, str] = {} + deadline = clock() + budget_seconds + pending = list(items) + for attempt, pause in enumerate((GDELT_PAUSE_SECONDS, GDELT_RETRY_PAUSE_SECONDS)): + if attempt: + if not pending or clock() >= deadline: + break + print(f"gdelt: retrying {len(pending)} name(s) after {GDELT_COOLDOWN_SECONDS:.0f}s cooldown", flush=True) + sleep(GDELT_COOLDOWN_SECONDS) + failed = [] + for i, item in enumerate(pending): + if clock() >= deadline: + failed += pending[i:] + for rest in pending[i:]: + last_error[rest["ticker"]] = "skipped: run time budget spent" + break + if i: + sleep(pause) + try: + articles = fetch( + company_name=item["company_name"], + window_start=start, + window_end=end, + aliases=item.get("aliases"), + client=client, + domain_whitelist=DOWNSIDE_DOMAINS, + ) + except Exception as exc: # noqa: BLE001 - one failed name must not abort the run + last_error[item["ticker"]] = str(exc) + failed.append(item) + print(f"gdelt: {item['ticker']} failed (pass {attempt + 1}): {str(exc)[:80]}", flush=True) continue - observations[f"gdelt:{url}"] = { - "key": f"gdelt:{url}", - "source_type": "news", - "source_name": a.get("domain") or "GDELT news", - "source_url": url, - "title": a.get("title"), - "raw_text": _article_to_text(item["company_name"], a), - "observed_at": _iso(_parse_gdelt_seendate(a.get("seendate") or "")), - "source_confidence": 0.75, - } + last_error.pop(item["ticker"], None) + _gdelt_rows(item, articles, observations) + print(f"gdelt: {item['ticker']} ok, {len(articles)} article(s) (pass {attempt + 1})", flush=True) + pending = failed + errors = [f"gdelt:{item['ticker']}:{last_error[item['ticker']]}" for item in pending] return list(observations.values()), errors diff --git a/simulations/forward_test/summarize.py b/simulations/forward_test/summarize.py index c72248b..4c8168c 100644 --- a/simulations/forward_test/summarize.py +++ b/simulations/forward_test/summarize.py @@ -73,13 +73,30 @@ def fetch_closes(ticker: str, start: datetime, end: datetime) -> list[tuple[date ] -def build_report(runs: list[dict], observations: list[dict], flags: list[dict], labels: dict, prices: dict) -> str: +def build_report( + runs: list[dict], + observations: list[dict], + flags: list[dict], + labels: dict, + prices: dict, + watchlist_tickers: list[str] | None = None, +) -> str: L = ["# Aetherius Forward Test — Live Report", ""] if not runs: return "\n".join(L + ["No runs recorded yet."]) + "\n" first, last = runs[0]["recorded_at"], runs[-1]["recorded_at"] weeks = max((_parse(last) - _parse(first)).days / 7.0, 1 / 7.0) - tickers = sorted({o["ticker"] for o in observations} | {f["ticker"] for f in flags}) + # A run whose news fetch failed for a name leaves a coverage gap; count them per name. + news_failures = Counter( + e.split(":", 2)[1] for r in runs for e in r.get("errors", []) if e.startswith("gdelt:") and e.count(":") >= 2 + ) + # Every pre-registered name gets a row, so a quiet name is visibly quiet rather than missing. + tickers = sorted( + set(watchlist_tickers or []) + | {o["ticker"] for o in observations} + | {f["ticker"] for f in flags} + | set(news_failures) + ) L += [ f"- Runs: {len(runs)} ({first} → {last}), ~{weeks:.1f} weeks", f"- Watchlist SHA-256: `{runs[-1]['watchlist_sha256']}`" @@ -90,8 +107,11 @@ def build_report(runs: list[dict], observations: list[dict], flags: list[dict], "", "## Alert burden by name", "", - "| Ticker | Matched obs | Flags | Flags / week | Labeled | Material | Not material |", - "| --- | --- | --- | --- | --- | --- | --- |", + f"News fetch failures count runs (of {len(runs)}) in which GDELT returned nothing for that name; " + "those days are coverage gaps, not quiet days.", + "", + "| Ticker | Matched obs | Flags | Flags / week | Labeled | Material | Not material | News fetch failures |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", ] obs_by = Counter(o["ticker"] for o in observations) flag_by = Counter(f["ticker"] for f in flags) @@ -100,7 +120,8 @@ def build_report(runs: list[dict], observations: list[dict], flags: list[dict], lab = [labels[f["key"]] for f in own if f["key"] in labels] mat = sum(1 for x in lab if x.get("material")) L.append( - f"| {t} | {obs_by[t]} | {flag_by[t]} | {flag_by[t] / weeks:.2f} | {len(lab)} | {mat} | {len(lab) - mat} |" + f"| {t} | {obs_by[t]} | {flag_by[t]} | {flag_by[t] / weeks:.2f} | {len(lab)} | {mat} | {len(lab) - mat} " + f"| {news_failures[t]} |" ) labeled = [labels[f["key"]] for f in flags if f["key"] in labels] L += ["", "## Precision (human-labeled)", ""] @@ -169,7 +190,9 @@ def main() -> None: out = args.out or args.log_dir / "REPORT.md" out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(build_report(runs, observations, flags, labels, prices), encoding="utf-8") + watchlist = json.loads((HERE / "watchlist.json").read_text(encoding="utf-8")) + tickers = [i["ticker"] for i in watchlist["items"]] + out.write_text(build_report(runs, observations, flags, labels, prices, tickers), encoding="utf-8") print(f"wrote {out}")