Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/catalog/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def build_catalog(

# ── Probe ────────────────────────────────────────────────────────────────
print(f" → probe {cfg['base_url']}")
probe = probe_url(cfg["base_url"])
probe = probe_url(cfg["base_url"], user_agent=cfg.get("user_agent"))
radar_result = _probe_result_as_dict(probe)
print(f" ← {radar_result['status']} (HTTP {radar_result['http_code']})")

Expand Down
14 changes: 8 additions & 6 deletions scripts/radar_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,11 @@ def _build_probe_result(
)


def _probe_once(base_url: str, timeout: int = TIMEOUT_SECONDS) -> ProbeResult:
def _probe_once(
base_url: str, timeout: int = TIMEOUT_SECONDS, user_agent: str | None = None
) -> ProbeResult:
"""Single probe attempt (no retry). Uses lab_connectors HttpClient with SSL fallback."""
client = HttpClient(timeout=timeout, user_agent=USER_AGENT)
client = HttpClient(timeout=timeout, user_agent=user_agent or USER_AGENT)
result = client.get(
base_url,
allow_redirects=True,
Expand Down Expand Up @@ -195,13 +197,13 @@ def _probe_once(base_url: str, timeout: int = TIMEOUT_SECONDS) -> ProbeResult:
)


def probe_url(base_url: str) -> ProbeResult:
def probe_url(base_url: str, user_agent: str | None = None) -> ProbeResult:
"""Probe URL with retry/backoff for SDMX endpoints known to be intermittent."""
if not is_sdmx_url(base_url):
result = _probe_once(base_url)
result = _probe_once(base_url, user_agent=user_agent)
# Retry once with shorter timeout on transient failures (timeout / connection error)
if result.http_code == "-" and result.status == "YELLOW":
result2 = _probe_once(base_url)
result2 = _probe_once(base_url, user_agent=user_agent)
# Only upgrade if second attempt succeeds — otherwise keep original note
if result2.status == "GREEN":
return result2
Expand All @@ -221,7 +223,7 @@ def probe_url(base_url: str) -> ProbeResult:
# Timeout dedicato + payload ridotto per evitare ReadTimeout su cataloghi
# che serializzano migliaia di dataflow in una risposta unica.
sdmx_fetch_url = catalog_fetch_url(base_url)
return _probe_once(sdmx_fetch_url, timeout=SDMX_TIMEOUT_SECONDS)
return _probe_once(sdmx_fetch_url, timeout=SDMX_TIMEOUT_SECONDS, user_agent=user_agent)


def build_status_report(
Expand Down
2 changes: 1 addition & 1 deletion tests/test_catalog_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def _make_repo(tmp_path: Path) -> Path:
def _patch_engine(monkeypatch: pytest.MonkeyPatch) -> None:
"""Neutralizza HTTP: probe/dispatch/validate restituiscono fixture."""

def fake_probe(base_url: str) -> ProbeResult:
def fake_probe(base_url: str, user_agent: str | None = None) -> ProbeResult:
return ProbeResult(status="GREEN", http_code="200", note=None, final_url=base_url)

def fake_dispatch(source_id, source_cfg, captured_at) -> CollectorResult:
Expand Down
Loading