From 1bb908d728715e219d6407a5b1826b84b150f35a Mon Sep 17 00:00:00 2001 From: Mircea Lungu Date: Fri, 28 Aug 2026 23:12:03 +0200 Subject: [PATCH 1/2] Fail over between LLM providers, and notice when both are gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answering "if we run out of DeepSeek credit, are we back on Anthropic?" — we were not. _select_provider_and_key switched provider only when the API KEY was MISSING. A provider that has a key and refuses to serve produced a plain Exception, nothing caught it, and the article was left unassessed. Exactly the Anthropic-cap failure, re-armed on the other side the moment the crawl moved to DeepSeek. Three parts: 1. Failover. ProviderUnavailable marks "this provider cannot serve us" — out of credit, capped, rate limited, key rejected, down — and one call is retried on the other provider. Content failures (paywall, advertorial, wrong language) are facts about the ARTICLE and are deliberately NOT retried elsewhere; that would spend twice for the same answer. Status codes alone are not enough: Anthropic reports its spend cap as a 400 invalid_request_error, so the body is matched too. The exact production strings are in the tests. 2. Notification, once per process. A dead provider fails over on every article and the crawl handles ~2000/day, so per-event mail would be 2000 messages and would train us to filter the alert away. The crawler is a fresh process per hourly run, so a sustained outage sends about one an hour. Sent only AFTER the fallback call succeeds — a failover that rescued nothing is not a rescue — and wrapped so a broken mailer can never turn a successfully assessed article into a failure. 3. Assessment-coverage check in feed_delivery_health_check. Failover degrades two providers to one; this is what says so when both are gone. The delivery checks are a lagging indicator — the August outage ran a week before anyone noticed — so this watches the upstream step: of articles crawled in the last 24h, how many got a CEFR level? The 80% threshold is measured, not guessed: healthy Aug 15-19: 97, 97, 98, 98, 98 % capped Aug 12, 13, 20: 15, 38, 37 % Every healthy day and every broken day is replayed in the tests. Below 50 eligible articles it reports without alerting; "almost nothing was crawled" is crawler_liveness_check's job, and firing on a quiet window would teach us to ignore this one. Not wired into cron here — the crontab lives in the ops repo. The existing feed-delivery entry picks this up automatically, since the check runs inside it. --- tools/feed_delivery_health_check.py | 158 ++++++++++- tools/test_feed_delivery_health_check.py | 58 ++++ .../simplification_and_classification.py | 194 ++++++++++++-- zeeguu/core/test/test_provider_failover.py | 247 ++++++++++++++++++ 4 files changed, 631 insertions(+), 26 deletions(-) create mode 100644 zeeguu/core/test/test_provider_failover.py diff --git a/tools/feed_delivery_health_check.py b/tools/feed_delivery_health_check.py index 7030575a2..c36f7f914 100644 --- a/tools/feed_delivery_health_check.py +++ b/tools/feed_delivery_health_check.py @@ -113,6 +113,34 @@ def _env_level_set(name, default): # How many items to request from the recommender / inventory query. COUNT = _env_int("FEED_HEALTH_COUNT", 10) +# --- Assessment coverage ------------------------------------------------------ +# The segment checks above watch DELIVERY. This one watches the step that feeds +# them: of the articles crawled recently, how many actually got a CEFR assessment +# (and therefore a summary and per-level text)? +# +# It exists because delivery checks are a LAGGING indicator of an LLM outage. The +# Aug 2026 Anthropic spend cap starved assessment for a week before anyone +# noticed, and the switch to DeepSeek moved that risk rather than removing it — +# running out of DeepSeek credit looks identical from the outside. Provider +# failover now degrades two providers to one, but if BOTH are unavailable this is +# what says so, within hours instead of days. +# +# Thresholds from measured history (all languages, articles eligible to be +# assessed, per day): +# healthy Aug 15-19: 97, 97, 98, 98, 98 % +# capped Aug 12,13: 15, 38 % +# capped Aug 20,21: 37, 0 % +# 80% sits far below every healthy day and far above every broken one. +ASSESSMENT_WINDOW_HOURS = _env_int("FEED_HEALTH_ASSESSMENT_WINDOW_HOURS", 24) +ASSESSMENT_MIN_PCT = _env_int("FEED_HEALTH_ASSESSMENT_MIN_PCT", 80) +# Below this many eligible articles the ratio is noise (a quiet window, a crawl +# that has not run yet), so the check reports and does not alert. +ASSESSMENT_MIN_VOLUME = _env_int("FEED_HEALTH_ASSESSMENT_MIN_VOLUME", 50) +# Articles shorter than this are skipped by the pipeline itself +# (assess_summarize_and_classify), so counting them would depress the ratio for a +# reason that is not a fault. Keep in step with that check. +ASSESSMENT_MIN_WORDS = 100 + # --- Pure classification logic (unit-tested) ---------------------------------- # Kept free of ES/DB so the freshness + status logic can be tested with plain @@ -429,6 +457,96 @@ def evaluate_segment(segment, users, count, now, thresholds): ) +AssessmentCoverage = namedtuple( + "AssessmentCoverage", ["eligible", "assessed", "pct", "by_language", "status"] +) + + +def classify_assessment_coverage( + eligible, assessed, by_language, min_pct=ASSESSMENT_MIN_PCT, + min_volume=ASSESSMENT_MIN_VOLUME, +): + """ + Pure classification, so the thresholds can be tested without a database. + + LOW_VOLUME is reported and never alerts: a ratio over a handful of articles + says nothing, and firing on a quiet window would train us to ignore this. + """ + pct = round(100 * assessed / eligible) if eligible else 0 + if eligible < min_volume: + status = "LOW_VOLUME" + elif pct < min_pct: + status = "FAILING" + else: + status = "OK" + return AssessmentCoverage(eligible, assessed, pct, by_language, status) + + +def assessment_coverage(window_hours=ASSESSMENT_WINDOW_HOURS, language_codes=None): + """ + What fraction of recently crawled articles actually got assessed? + + Counts only articles the pipeline WOULD assess: originals (a simplified child + is never assessed separately), not broken, and long enough that + assess_summarize_and_classify does not skip them. + """ + from zeeguu.core.model.article import Article + from zeeguu.core.model import db + + since = datetime.now() - timedelta(hours=window_hours) + query = ( + db.session.query( + Language.code, + db.func.count(Article.id), + db.func.sum(db.case((Article.cefr_level.isnot(None), 1), else_=0)), + ) + .join(Language, Language.id == Article.language_id) + .filter(Article.parent_article_id.is_(None)) + .filter(Article.broken == 0) + .filter(Article.word_count >= ASSESSMENT_MIN_WORDS) + .filter(Article.published_time >= since) + .group_by(Language.code) + ) + if language_codes: + query = query.filter(Language.code.in_(language_codes)) + + by_language = {} + eligible = assessed = 0 + for code, n_eligible, n_assessed in query.all(): + n_assessed = int(n_assessed or 0) + by_language[code] = (n_eligible, n_assessed) + eligible += n_eligible + assessed += n_assessed + + return classify_assessment_coverage(eligible, assessed, by_language) + + +def assessment_coverage_lines(coverage, window_hours=ASSESSMENT_WINDOW_HOURS): + icon = {"OK": "✅", "FAILING": "❌", "LOW_VOLUME": "•"}[coverage.status] + lines = [ + "", + f"{icon} ASSESSMENT COVERAGE (last {window_hours}h): " + f"{coverage.assessed}/{coverage.eligible} = {coverage.pct}% " + f"(alert below {ASSESSMENT_MIN_PCT}%)", + ] + for code, (n_eligible, n_assessed) in sorted(coverage.by_language.items()): + pct = round(100 * n_assessed / n_eligible) if n_eligible else 0 + lines.append(f" {code}: {n_assessed}/{n_eligible} = {pct}%") + if coverage.status == "FAILING": + lines += [ + "", + " Articles are being crawled but not assessed: no CEFR level, no", + " summary, no per-level text. Feed cards fall back to the publisher's", + " blurb, identical at every CEFR level.", + " Most likely BOTH LLM providers are unavailable (provider failover", + " covers one being down). Check, in order:", + " - DeepSeek balance https://platform.deepseek.com/usage", + " - Anthropic spend cap (Console -> Billing/Limits)", + " - /var/log/zeeguu/crawler/ for the actual API error", + ] + return lines + + def build_report(results, alert_levels=ALERT_LEVELS): """Human-readable, structured per-segment summary lines.""" lines = [] @@ -491,6 +609,9 @@ def main(): parser.add_argument("--min-fresh", type=int, default=MIN_FRESH) parser.add_argument("--stale-days", type=int, default=STALE_DAYS) parser.add_argument("--count", type=int, default=COUNT) + parser.add_argument( + "--assessment-window-hours", type=int, default=ASSESSMENT_WINDOW_HOURS + ) parser.add_argument( "--alert-levels", type=str, @@ -527,10 +648,22 @@ def main(): thresholds = (args.min_articles, args.min_fresh, args.stale_days, args.fresh_days) now = datetime.now() + # Runs before the segment sweep and independently of it: if assessment has + # stopped, that is the cause and the segment results are the symptom. It also + # still reports when there are no active users to sweep, which is exactly the + # situation where nobody would notice the pipeline dying. + coverage = assessment_coverage(args.assessment_window_hours, language_codes) + coverage_report = assessment_coverage_lines(coverage, args.assessment_window_hours) + segments = active_users_by_segment(args.active_days, language_codes) if not segments: - print("No active users found in any segment — nothing to check.") - return 0 + print("\n".join(coverage_report)) + print("\nNo active users found in any segment — no delivery check to run.") + if coverage.status == "FAILING" and not args.dry_run: + ZeeguuMailer.send_mail( + "⚠️ Zeeguu: articles are being crawled but not assessed", coverage_report + ) + return 1 if coverage.status == "FAILING" else 0 results = [] for segment in sorted(segments, key=lambda s: (s.language_code, s.cefr_level)): @@ -544,7 +677,7 @@ def main(): alert_levels = { t.strip().upper() for t in args.alert_levels.split(",") if t.strip() } - report = build_report(results, alert_levels) + report = build_report(results, alert_levels) + coverage_report print("\n".join(report)) alerting_failures = [r for r in results if is_alerting(r, alert_levels)] @@ -553,7 +686,8 @@ def main(): for r in results if r.status in FAILING_STATUSES and not is_alerting(r, alert_levels) ] - if not alerting_failures: + coverage_failing = coverage.status == "FAILING" + if not alerting_failures and not coverage_failing: note = ( f" ({len(report_only_failures)} report-only issue(s) shown above)" if report_only_failures @@ -562,9 +696,19 @@ def main(): print(f"\nOK: no alerting segment failures.{note}") return 0 - subject = ( - f"⚠️ Zeeguu feed delivery: {len(alerting_failures)} segment(s) failing (empty/stale/error)" - ) + # Assessment coverage leads the subject when it is failing: it is upstream of + # the segment results, so it is the thing to go and fix. + if coverage_failing: + subject = ( + f"⚠️ Zeeguu: only {coverage.pct}% of crawled articles assessed " + f"(last {args.assessment_window_hours}h)" + ) + if alerting_failures: + subject += f" — {len(alerting_failures)} segment(s) also failing" + else: + subject = ( + f"⚠️ Zeeguu feed delivery: {len(alerting_failures)} segment(s) failing (empty/stale/error)" + ) if args.dry_run: print(f"\n[dry-run] would email: {subject}") else: diff --git a/tools/test_feed_delivery_health_check.py b/tools/test_feed_delivery_health_check.py index b59ad4d17..66cc2e656 100644 --- a/tools/test_feed_delivery_health_check.py +++ b/tools/test_feed_delivery_health_check.py @@ -19,6 +19,8 @@ from datetime import datetime, timedelta, timezone from tools.feed_delivery_health_check import ( + assessment_coverage_lines, + classify_assessment_coverage, freshness_summary, classify_segment, is_alerting, @@ -170,3 +172,59 @@ def test_hit_published_time_parses_iso_naive(): def test_hit_published_time_unparseable_returns_none(): assert _hit_published_time({"_source": {"published_time": ""}}) is None + + + +# --- Assessment coverage ------------------------------------------------------ +# The delivery checks above are a LAGGING indicator of an LLM outage: the Aug 2026 +# spend cap starved assessment for a week before anyone noticed. This one watches +# the upstream step directly. +# +# Thresholds are pinned to measured production history (all languages, articles +# eligible for assessment, per day): +# healthy Aug 15-19: 97, 97, 98, 98, 98 % +# capped Aug 12, 13: 15, 38 % +# capped Aug 20, 21: 37, 0 % + + +def _coverage(eligible, assessed, by_language=None): + return classify_assessment_coverage(eligible, assessed, by_language or {}) + + +def test_every_measured_healthy_day_passes(): + for pct in (97, 97, 98, 98, 98): + assert _coverage(1000, pct * 10).status == "OK", f"{pct}% should not alert" + + +def test_every_measured_capped_day_fails(): + for pct in (15, 38, 37, 0): + assert _coverage(1000, pct * 10).status == "FAILING", f"{pct}% should alert" + + +def test_the_partial_recovery_day_is_not_alarmed_about(): + """Aug 14 was 83%: the cap was raised mid-day, so the pipeline was already + working again. Alerting on a recovery would be noise.""" + assert _coverage(1232, 1028).status == "OK" + + +def test_a_quiet_window_reports_but_does_not_alert(): + """A ratio over a handful of articles says nothing, and firing here would + train us to ignore the alert.""" + assert _coverage(4, 0).status == "LOW_VOLUME" + + +def test_the_percentage_is_reported_for_the_email_subject(): + assert _coverage(1000, 370).pct == 37 + + +def test_zero_eligible_articles_does_not_divide_by_zero(): + result = _coverage(0, 0) + assert result.status == "LOW_VOLUME" + assert result.pct == 0 + + +def test_a_failing_report_says_where_to_look(): + coverage = _coverage(1000, 100, by_language={"da": (500, 50)}) + lines = "\n".join(assessment_coverage_lines(coverage)) + assert "deepseek.com" in lines and "Anthropic" in lines + assert "da: 50/500 = 10%" in lines diff --git a/zeeguu/core/llm_services/simplification_and_classification.py b/zeeguu/core/llm_services/simplification_and_classification.py index b36400c39..27451d685 100644 --- a/zeeguu/core/llm_services/simplification_and_classification.py +++ b/zeeguu/core/llm_services/simplification_and_classification.py @@ -46,11 +46,78 @@ def _get_next_simplification_provider() -> str: return "anthropic" +class ProviderUnavailable(Exception): + """ + The provider itself cannot serve us: out of credit, over a spend cap, rate + limited, key rejected, or down. + + Deliberately distinct from a bad RESPONSE (paywall, advertorial, wrong + language). Those are facts about the article and must NOT be retried on + another provider — only this one is worth failing over. + """ + + +def _other_provider(provider: str) -> str: + return "anthropic" if provider == "deepseek" else "deepseek" + + +def _api_key_for(provider: str): + if provider == "deepseek": + return os.environ.get("DEEPSEEK_API_SIMPLIFICATIONS") + return os.environ.get("ANTHROPIC_TEXT_SIMPLIFICATION_KEY") + + +# Statuses that mean "this provider cannot serve us right now" rather than "you +# sent a bad request": 402 is DeepSeek's Insufficient Balance, 429 is rate +# limiting, 401/403 a revoked or wrong key, 5xx/529 an outage. +_PROVIDER_DOWN_STATUSES = {401, 402, 403, 429, 500, 502, 503, 504, 529} + +# Anthropic reports its monthly spend cap as a 400 invalid_request_error — NOT a +# quota status — so the status code alone cannot catch the exact outage that +# starved the crawl through August. Matched on the body instead, which reads: +# 400 - {"type":"error","error":{"type":"invalid_request_error","message": +# "You have reached your specified API usage limits. You will regain +# access on 2026-09-01 at 00:00 UTC."}} +# DeepSeek's out-of-credit body says "Insufficient Balance" and already carries +# 402, so it is covered twice over. +_PROVIDER_DOWN_MARKERS = ("usage limits", "insufficient balance", "credit balance") + + +def _is_provider_unavailable(status_code, body: str) -> bool: + if status_code in _PROVIDER_DOWN_STATUSES: + return True + return any(marker in (body or "").lower() for marker in _PROVIDER_DOWN_MARKERS) + + +def _as_provider_unavailable_if_applicable(error: Exception) -> Exception: + """ + Re-label an Anthropic failure as ProviderUnavailable when the message says the + provider (not the request) is the problem. haiku_client raises a bare + Exception formatted as "Anthropic API error: {status} - {body}" — see + haiku_completion_or_raise — so the status is recovered from that text. A + network error (requests.Timeout etc.) has no status and is judged on the body + alone, i.e. left as-is: retrying a timeout on the other provider would just + double the wait on a slow article. + """ + import re + + message = str(error) + match = re.match(r"\w+ API error: (\d{3}) - ", message) + status_code = int(match.group(1)) if match else None + if _is_provider_unavailable(status_code, message): + return ProviderUnavailable(message) + return error + + def _select_provider_and_key(simplification_provider: str = None): """ Resolve the provider (round-robin unless one is given) and its API key, falling back to the other provider when the primary key is unset. Returns (provider, api_key). Shared by the assess-only and full-simplify paths. + + This handles ONLY a missing key. A provider that has a key but refuses to + serve — capped, out of credit, rate limited — is handled at call time by + _call_llm_with_provider_failover, because it is not knowable from here. """ if simplification_provider: provider = simplification_provider @@ -58,17 +125,11 @@ def _select_provider_and_key(simplification_provider: str = None): provider = _get_next_simplification_provider() log(f"Using {provider.upper()} provider for simplification") - if provider == "deepseek": - api_key = os.environ.get("DEEPSEEK_API_SIMPLIFICATIONS") - fallback_api_key = os.environ.get("ANTHROPIC_TEXT_SIMPLIFICATION_KEY") - else: - api_key = os.environ.get("ANTHROPIC_TEXT_SIMPLIFICATION_KEY") - fallback_api_key = os.environ.get("DEEPSEEK_API_SIMPLIFICATIONS") - + api_key = _api_key_for(provider) if not api_key: log(f"WARNING: {provider.upper()} API key not set, trying fallback") - provider = "anthropic" if provider == "deepseek" else "deepseek" - api_key = fallback_api_key + provider = _other_provider(provider) + api_key = _api_key_for(provider) if not api_key: raise Exception( "Neither DEEPSEEK_API_SIMPLIFICATIONS nor ANTHROPIC_TEXT_SIMPLIFICATION_KEY environment variable set" @@ -76,6 +137,92 @@ def _select_provider_and_key(simplification_provider: str = None): return provider, api_key +# Which (from -> to) failovers this PROCESS has already emailed about. +# +# Deliberately once per process, not once per failover: a dead provider fails +# over on EVERY article, and the crawl handles ~2000 a day, so per-event mail +# would be 2000 messages and would train us to filter the alert away. The crawler +# is a fresh process per hourly run, so a sustained outage sends about one mail an +# hour — enough to notice, little enough to read. The sustained case is also +# covered from the other side by the assessment-coverage check in +# tools/feed_delivery_health_check.py. +_failover_notified = set() + + +def _notify_failover_once(from_provider, to_provider, reason): + key = (from_provider, to_provider) + if key in _failover_notified: + return + _failover_notified.add(key) + + # Never let a mail problem turn a SUCCESSFUL failover into a failed article: + # the whole point of this path is that the work still got done. + try: + from zeeguu.core.emailer.zeeguu_mailer import ZeeguuMailer + + ZeeguuMailer.send_mail( + f"⚠️ Zeeguu LLM failover: {from_provider.upper()} → {to_provider.upper()}", + [ + f"{from_provider.upper()} could not serve the simplification " + f"pipeline, so it fell over to {to_provider.upper()}.", + "", + "Articles are still being assessed — this is the degraded-but-working", + f"state, on one provider instead of two. If {to_provider.upper()} also", + "goes, assessment stops entirely and feed cards fall back to the", + "publisher blurb at every CEFR level.", + "", + f"Reason given by {from_provider.upper()}:", + f" {reason}", + "", + "Check: DeepSeek balance https://platform.deepseek.com/usage", + " Anthropic spend cap (Console -> Billing/Limits)", + "", + "Sent once per crawl process, so expect roughly one an hour while", + "this lasts rather than one per article.", + ], + ) + except Exception as mail_error: + log(f" (could not send failover notification: {mail_error})") + + +def _call_llm_with_provider_failover(prompt, provider, api_key, max_tokens, timeout=180): + """ + Call the LLM, and if THIS provider cannot serve us, try the other one once. + Returns (result_text, model_name, provider_actually_used). + + Without this, a single provider's billing state stops all assessment in every + language: through August the Anthropic spend cap did exactly that, and after + switching the crawl to DeepSeek, running out of DeepSeek credit would have + reproduced it symptom-for-symptom (articles crawled, none assessed, nothing + in the logs but a per-article error). Two funded providers should degrade to + one, not to zero. + + Only ProviderUnavailable triggers the retry — a paywalled or wrong-language + response is the article's problem and retrying it elsewhere would just spend + twice for the same answer. + """ + try: + result, model_name = _call_simplification_llm( + prompt, provider, api_key, max_tokens, timeout + ) + return result, model_name, provider + except ProviderUnavailable as e: + fallback = _other_provider(provider) + fallback_key = _api_key_for(fallback) + if not fallback_key: + log(f" {provider.upper()} unavailable and no {fallback.upper()} key to fall back to") + raise + log(f" {provider.upper()} unavailable ({e}) — failing over to {fallback.upper()}") + result, model_name = _call_simplification_llm( + prompt, fallback, fallback_key, max_tokens, timeout + ) + # After the fallback call succeeds, so a failover that helps nobody + # (both providers down) raises instead of mailing about a rescue that + # did not happen. + _notify_failover_once(provider, fallback, str(e)) + return result, model_name, fallback + + def _call_simplification_llm(prompt, provider, api_key, max_tokens, timeout=180): """ Send `prompt` to the chosen provider and return (result_text, model_name). @@ -99,15 +246,22 @@ def _call_simplification_llm(prompt, provider, api_key, max_tokens, timeout=180) timeout=timeout, ) if response.status_code != 200: - raise Exception( - f"DEEPSEEK API error: {response.status_code} - {response.text}" - ) + message = f"DEEPSEEK API error: {response.status_code} - {response.text}" + if _is_provider_unavailable(response.status_code, response.text): + raise ProviderUnavailable(message) + raise Exception(message) result = response.json()["choices"][0]["message"]["content"].strip() else: # anthropic model_name = HAIKU_MODEL - result = haiku_completion_or_raise( - prompt, max_tokens=max_tokens, temperature=0.1, timeout=timeout - ).strip() + try: + result = haiku_completion_or_raise( + prompt, max_tokens=max_tokens, temperature=0.1, timeout=timeout + ).strip() + except Exception as e: + # haiku_client raises a bare Exception carrying the status and body + # (f"Anthropic API error: {status} - {text}"), so the status has to be + # read back out of the message to tell an outage from a bad request. + raise _as_provider_unavailable_if_applicable(e) log(f" {provider.upper()} responded in {time.time() - api_start_time:.2f}s ({len(result)} chars)") return result, model_name @@ -272,11 +426,13 @@ def generate(correction): # for a C2 article) plus the original summary. 2000 gives headroom over the # single-summary sizing so the last-emitted levels aren't truncated — still # a fraction of the 6000 the full multi-level bodies needed. - result, model_name = _call_simplification_llm( + result, model_name, used_provider = _call_llm_with_provider_failover( prompt + correction, provider, api_key, max_tokens=2000, timeout=120 ) _raise_if_paywall_or_advertorial(result) - assessment = _parse_assessment_and_summary(result, provider, model_name) + # used_provider, not provider: on a failover the row must record the model + # that actually wrote the text, or AIGenerator attributes it to the wrong one. + assessment = _parse_assessment_and_summary(result, used_provider, model_name) # Raised, not returned, so junk propagates out of the language-check retry # loop exactly as the bare-word rejection always has. _raise_if_flagged(assessment) @@ -785,11 +941,11 @@ def simplify_article_adaptive_levels( log(f" Prompt length: {len(prompt)} characters") def generate(correction): - result, model_name = _call_simplification_llm( + result, model_name, used_provider = _call_llm_with_provider_failover( prompt + correction, provider, api_key, max_tokens=6000, timeout=180 ) _raise_if_paywall_or_advertorial(result) - return _parse_adaptive_response(result, provider, model_name) + return _parse_adaptive_response(result, used_provider, model_name) simplification = generate_in_language( generate, diff --git a/zeeguu/core/test/test_provider_failover.py b/zeeguu/core/test/test_provider_failover.py new file mode 100644 index 000000000..865844853 --- /dev/null +++ b/zeeguu/core/test/test_provider_failover.py @@ -0,0 +1,247 @@ +""" +One provider's billing state must not stop all assessment. + +Through August the Anthropic monthly spend cap did exactly that: every article in +every language failed to assess, for a week at a time, with nothing in the logs +but a per-article error. Switching the crawl to DeepSeek moved the same risk +rather than removing it — running out of DeepSeek credit would look identical. + +Two funded providers should degrade to one, not to zero. +""" +from unittest import TestCase +from unittest.mock import patch + +from zeeguu.core.llm_services import simplification_and_classification as sac + +# The exact body Anthropic returns when the monthly spend cap is reached. Note +# the 400 and invalid_request_error: this is NOT a 429 or any other quota status, +# so a failover keyed on status codes alone would sail straight past it. +ANTHROPIC_CAP_BODY = ( + 'Anthropic API error: 400 - {"type":"error","error":{"type":' + '"invalid_request_error","message":"You have reached your specified API ' + 'usage limits. You will regain access on 2026-09-01 at 00:00 UTC."}}' +) + +# DeepSeek's out-of-credit response. Carries 402 as well, so it is caught twice. +DEEPSEEK_NO_CREDIT_BODY = '{"error":{"message":"Insufficient Balance"}}' + + +class ProviderUnavailableClassificationTest(TestCase): + def test_the_anthropic_spend_cap_counts_as_unavailable(self): + assert sac._is_provider_unavailable(400, ANTHROPIC_CAP_BODY) + + def test_deepseek_insufficient_balance_counts_as_unavailable(self): + assert sac._is_provider_unavailable(402, DEEPSEEK_NO_CREDIT_BODY) + # ...on the body alone too, in case the status ever changes. + assert sac._is_provider_unavailable(200, DEEPSEEK_NO_CREDIT_BODY) + + def test_rate_limits_outages_and_bad_keys_count(self): + for status in (401, 403, 429, 500, 503, 529): + assert sac._is_provider_unavailable(status, ""), status + + def test_an_ordinary_bad_request_does_not_count(self): + """A 400 that is genuinely our fault must NOT burn the other provider's + quota retrying the same broken request.""" + assert not sac._is_provider_unavailable( + 400, '{"error":{"message":"max_tokens must be positive"}}' + ) + + def test_the_anthropic_status_is_recovered_from_the_message(self): + """haiku_client raises a bare Exception carrying the status in its text, + so the classifier has to read it back out.""" + relabelled = sac._as_provider_unavailable_if_applicable( + Exception("Anthropic API error: 429 - rate limited") + ) + assert isinstance(relabelled, sac.ProviderUnavailable) + + def test_a_network_error_is_left_alone(self): + """No status and no marker: retrying a timeout on the other provider just + doubles the wait on a slow article.""" + original = Exception("Connection reset by peer") + assert sac._as_provider_unavailable_if_applicable(original) is original + + +class ProviderFailoverTest(TestCase): + def _call(self, side_effect, keys): + with patch.dict("os.environ", keys, clear=False), patch.object( + sac, "_call_simplification_llm", side_effect=side_effect + ) as llm: + return sac._call_llm_with_provider_failover( + "prompt", "deepseek", "dsk-key", max_tokens=100 + ), llm + + def test_a_capped_provider_fails_over_to_the_other(self): + (result, model, used), llm = self._call( + side_effect=[ + sac.ProviderUnavailable(DEEPSEEK_NO_CREDIT_BODY), + ("ORIGINAL_LEVEL: B1", "claude-haiku"), + ], + keys={ + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + ) + assert used == "anthropic", "should have switched providers" + assert model == "claude-haiku" + assert llm.call_count == 2 + + def test_the_provider_actually_used_is_reported_back(self): + """The caller records the model on the row it writes; reporting the + originally-selected provider after a failover would attribute the text to + a model that never wrote it.""" + (_, _, used), _ = self._call( + side_effect=[ + sac.ProviderUnavailable("capped"), + ("ORIGINAL_LEVEL: B1", "claude-haiku"), + ], + keys={ + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + ) + assert used != "deepseek" + + def test_a_content_failure_is_not_retried_elsewhere(self): + """A paywalled or malformed response is a fact about the ARTICLE. Retrying + it on the other provider spends twice for the same answer.""" + with patch.dict( + "os.environ", + { + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + clear=False, + ), patch.object( + sac, "_call_simplification_llm", side_effect=Exception("PAYWALL: nope") + ) as llm: + with self.assertRaises(Exception): + sac._call_llm_with_provider_failover( + "prompt", "deepseek", "dsk-key", max_tokens=100 + ) + assert llm.call_count == 1, "must not have tried the other provider" + + def test_with_only_one_key_the_original_error_surfaces(self): + """Nothing to fail over to: raise the real reason rather than a confusing + secondary error from an unconfigured provider.""" + with patch.dict( + "os.environ", {"ANTHROPIC_TEXT_SIMPLIFICATION_KEY": ""}, clear=False + ), patch.object( + sac, + "_call_simplification_llm", + side_effect=sac.ProviderUnavailable("out of credit"), + ) as llm: + with self.assertRaises(sac.ProviderUnavailable): + sac._call_llm_with_provider_failover( + "prompt", "deepseek", "dsk-key", max_tokens=100 + ) + assert llm.call_count == 1 + + def test_a_healthy_provider_is_not_second_guessed(self): + (result, model, used), llm = self._call( + side_effect=[("ORIGINAL_LEVEL: B1", "deepseek-chat")], + keys={ + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + ) + assert used == "deepseek" + assert llm.call_count == 1 + + +class FailoverNotificationTest(TestCase): + """ + A failover must be visible, but a dead provider fails over on EVERY article — + ~2000/day. Per-event mail would bury the signal it exists to raise, so this is + once per process (≈ one per hourly crawl run while degraded). + """ + + def setUp(self): + sac._failover_notified.clear() + + def _failover(self, times=1, llm_side_effect=None): + side_effect = llm_side_effect or ( + [sac.ProviderUnavailable("Insufficient Balance"), ("ok", "claude-haiku")] + * times + ) + with patch.dict( + "os.environ", + { + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + clear=False, + ), patch.object( + sac, "_call_simplification_llm", side_effect=side_effect + ), patch( + "zeeguu.core.emailer.zeeguu_mailer.ZeeguuMailer.send_mail" + ) as send_mail: + for _ in range(times): + sac._call_llm_with_provider_failover( + "prompt", "deepseek", "dsk-key", max_tokens=100 + ) + return send_mail + + def test_a_failover_sends_one_email(self): + send_mail = self._failover() + assert send_mail.call_count == 1 + subject = send_mail.call_args[0][0] + assert "DEEPSEEK" in subject and "ANTHROPIC" in subject + + def test_repeated_failovers_do_not_repeat_the_email(self): + send_mail = self._failover(times=25) + assert send_mail.call_count == 1, "one per process, not one per article" + + def test_the_reason_is_included_so_the_mail_is_actionable(self): + send_mail = self._failover() + body = "\n".join(send_mail.call_args[0][1]) + assert "Insufficient Balance" in body + + def test_a_broken_mailer_does_not_break_the_crawl(self): + """The article WAS assessed. Losing the notification must not turn that + into a failure.""" + with patch.dict( + "os.environ", + { + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + clear=False, + ), patch.object( + sac, + "_call_simplification_llm", + side_effect=[sac.ProviderUnavailable("capped"), ("ok", "claude-haiku")], + ), patch( + "zeeguu.core.emailer.zeeguu_mailer.ZeeguuMailer.send_mail", + side_effect=Exception("SMTP down"), + ): + result, _, used = sac._call_llm_with_provider_failover( + "prompt", "deepseek", "dsk-key", max_tokens=100 + ) + assert result == "ok" + assert used == "anthropic" + + def test_no_email_when_both_providers_are_down(self): + """Nothing was rescued, so there is nothing to report as a rescue — the + error propagates and the coverage check is what catches it.""" + with patch.dict( + "os.environ", + { + "DEEPSEEK_API_SIMPLIFICATIONS": "dsk-key", + "ANTHROPIC_TEXT_SIMPLIFICATION_KEY": "ant-key", + }, + clear=False, + ), patch.object( + sac, + "_call_simplification_llm", + side_effect=[ + sac.ProviderUnavailable("out of credit"), + sac.ProviderUnavailable("capped"), + ], + ), patch( + "zeeguu.core.emailer.zeeguu_mailer.ZeeguuMailer.send_mail" + ) as send_mail: + with self.assertRaises(sac.ProviderUnavailable): + sac._call_llm_with_provider_failover( + "prompt", "deepseek", "dsk-key", max_tokens=100 + ) + assert send_mail.call_count == 0 From 67ecda2b2127f59359e0aed7f0888da3b89d4896 Mon Sep 17 00:00:00 2001 From: Mircea Lungu Date: Sat, 29 Aug 2026 00:17:07 +0200 Subject: [PATCH 2/2] Retune the coverage threshold to the post-#717 baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 80% was derived from healthy days measured under Anthropic + the bare-word prompt, which under-enforced the prompt's own "fewer than 3 paragraphs = likely incomplete" rule: only 2-3% of articles were rejected as paywalled, so coverage sat at 97-98%. DeepSeek with the field-based prompt follows that rule literally. The 29 Aug Danish backfill rejected 118 of 629 (19%) — correctly: the rejected set averages 190 words and 1,043 characters against 598 words and 3,246 characters for the assessed set. That is what a paywall stub looks like, not a false positive. So healthy coverage is now ~81%, and an 80% threshold would have paged on a perfectly good day. 70% keeps a real margin under the new baseline while staying far above every capped day (15-38%), which is what the check exists to catch. An alarm that fires on normal operation is worse than no alarm. --- tools/feed_delivery_health_check.py | 12 ++++++++++-- tools/test_feed_delivery_health_check.py | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tools/feed_delivery_health_check.py b/tools/feed_delivery_health_check.py index c36f7f914..d87fd13fc 100644 --- a/tools/feed_delivery_health_check.py +++ b/tools/feed_delivery_health_check.py @@ -130,9 +130,17 @@ def _env_level_set(name, default): # healthy Aug 15-19: 97, 97, 98, 98, 98 % # capped Aug 12,13: 15, 38 % # capped Aug 20,21: 37, 0 % -# 80% sits far below every healthy day and far above every broken one. +# +# 70%, not 80%. Those healthy days were measured under Anthropic + the bare-word +# prompt, which under-enforced the prompt's own "fewer than 3 paragraphs = likely +# incomplete" rule: only 2-3% of articles were rejected as paywalled. DeepSeek +# with the field-based prompt (api#717) follows that rule literally, and the +# 29 Aug Danish backfill measured 19% rejected -- correctly, the rejected set +# averages 190 words against 598 for the assessed set. Healthy coverage is +# therefore ~81%, not ~98%, and an 80% threshold would fire on a good day. +# Still far above every capped day (15-38%), which is what this has to catch. ASSESSMENT_WINDOW_HOURS = _env_int("FEED_HEALTH_ASSESSMENT_WINDOW_HOURS", 24) -ASSESSMENT_MIN_PCT = _env_int("FEED_HEALTH_ASSESSMENT_MIN_PCT", 80) +ASSESSMENT_MIN_PCT = _env_int("FEED_HEALTH_ASSESSMENT_MIN_PCT", 70) # Below this many eligible articles the ratio is noise (a quiet window, a crawl # that has not run yet), so the check reports and does not alert. ASSESSMENT_MIN_VOLUME = _env_int("FEED_HEALTH_ASSESSMENT_MIN_VOLUME", 50) diff --git a/tools/test_feed_delivery_health_check.py b/tools/test_feed_delivery_health_check.py index 66cc2e656..1159b3c79 100644 --- a/tools/test_feed_delivery_health_check.py +++ b/tools/test_feed_delivery_health_check.py @@ -196,6 +196,14 @@ def test_every_measured_healthy_day_passes(): assert _coverage(1000, pct * 10).status == "OK", f"{pct}% should not alert" +def test_the_post_717_healthy_baseline_passes(): + """DeepSeek + the field-based prompt enforce the prompt's own "fewer than 3 + paragraphs" rule, which Anthropic under-enforced: the 29 Aug Danish backfill + rejected 19% as paywalled (correctly -- 190 words avg vs 598 for the assessed). + So healthy is ~81%, not ~98%, and a threshold above that fires on a good day.""" + assert _coverage(629, 511).status == "OK" + + def test_every_measured_capped_day_fails(): for pct in (15, 38, 37, 0): assert _coverage(1000, pct * 10).status == "FAILING", f"{pct}% should alert"