diff --git a/scripts/catalog/build.py b/scripts/catalog/build.py index e277264..aec5719 100644 --- a/scripts/catalog/build.py +++ b/scripts/catalog/build.py @@ -139,7 +139,7 @@ def build_catalog( # ── Probe ──────────────────────────────────────────────────────────────── print(f" → probe {cfg['base_url']}") - probe = probe_url(cfg["base_url"]) + probe = probe_url(cfg["base_url"], user_agent=cfg.get("user_agent")) radar_result = _probe_result_as_dict(probe) print(f" ← {radar_result['status']} (HTTP {radar_result['http_code']})") diff --git a/scripts/radar_check.py b/scripts/radar_check.py index 1c02f24..9aa88a3 100644 --- a/scripts/radar_check.py +++ b/scripts/radar_check.py @@ -139,9 +139,11 @@ def _build_probe_result( ) -def _probe_once(base_url: str, timeout: int = TIMEOUT_SECONDS) -> ProbeResult: +def _probe_once( + base_url: str, timeout: int = TIMEOUT_SECONDS, user_agent: str | None = None +) -> ProbeResult: """Single probe attempt (no retry). Uses lab_connectors HttpClient with SSL fallback.""" - client = HttpClient(timeout=timeout, user_agent=USER_AGENT) + client = HttpClient(timeout=timeout, user_agent=user_agent or USER_AGENT) result = client.get( base_url, allow_redirects=True, @@ -195,13 +197,13 @@ def _probe_once(base_url: str, timeout: int = TIMEOUT_SECONDS) -> ProbeResult: ) -def probe_url(base_url: str) -> ProbeResult: +def probe_url(base_url: str, user_agent: str | None = None) -> ProbeResult: """Probe URL with retry/backoff for SDMX endpoints known to be intermittent.""" if not is_sdmx_url(base_url): - result = _probe_once(base_url) + result = _probe_once(base_url, user_agent=user_agent) # Retry once with shorter timeout on transient failures (timeout / connection error) if result.http_code == "-" and result.status == "YELLOW": - result2 = _probe_once(base_url) + result2 = _probe_once(base_url, user_agent=user_agent) # Only upgrade if second attempt succeeds — otherwise keep original note if result2.status == "GREEN": return result2 @@ -221,7 +223,7 @@ def probe_url(base_url: str) -> ProbeResult: # Timeout dedicato + payload ridotto per evitare ReadTimeout su cataloghi # che serializzano migliaia di dataflow in una risposta unica. sdmx_fetch_url = catalog_fetch_url(base_url) - return _probe_once(sdmx_fetch_url, timeout=SDMX_TIMEOUT_SECONDS) + return _probe_once(sdmx_fetch_url, timeout=SDMX_TIMEOUT_SECONDS, user_agent=user_agent) def build_status_report( diff --git a/tests/test_catalog_build.py b/tests/test_catalog_build.py index 34bdab9..7e74254 100644 --- a/tests/test_catalog_build.py +++ b/tests/test_catalog_build.py @@ -111,7 +111,7 @@ def _make_repo(tmp_path: Path) -> Path: def _patch_engine(monkeypatch: pytest.MonkeyPatch) -> None: """Neutralizza HTTP: probe/dispatch/validate restituiscono fixture.""" - def fake_probe(base_url: str) -> ProbeResult: + def fake_probe(base_url: str, user_agent: str | None = None) -> ProbeResult: return ProbeResult(status="GREEN", http_code="200", note=None, final_url=base_url) def fake_dispatch(source_id, source_cfg, captured_at) -> CollectorResult: