diff --git a/tests/test_clean_duckdb_read.py b/tests/test_clean_duckdb_read.py index aa01b1f..9bf90cb 100644 --- a/tests/test_clean_duckdb_read.py +++ b/tests/test_clean_duckdb_read.py @@ -281,6 +281,7 @@ def test_read_raw_to_relation_reads_ods_with_odf_engine(tmp_path: Path): """ODS (OpenDocument) read via pandas engine odf — ACI Auto-Trend e simili.""" import pandas as pd + pytest.importorskip("odf", reason="odfpy richiesto per scrittura/lettura ODS") input_file = tmp_path / "ok.ods" pd.DataFrame( [ @@ -355,6 +356,7 @@ def test_read_excel_compact_columns_rename_xlsx(tmp_path: Path): @pytest.mark.policy def test_read_excel_compact_columns_rename_ods(tmp_path: Path): """policy: stesso rename compatto su .ods (consumer open-aci Auto-Trend ODS).""" + pytest.importorskip("odf", reason="odfpy richiesto per scrittura/lettura ODS") input_file = tmp_path / "compact.ods" pd.DataFrame( [ @@ -393,6 +395,7 @@ def test_read_excel_compact_columns_rename_ods(tmp_path: Path): @pytest.mark.policy def test_read_excel_source_label_uses_all_batch_suffixes(tmp_path: Path): """policy: source_label considera tutti i suffix del batch, non solo il primo.""" + pytest.importorskip("odf", reason="odfpy richiesto per scrittura/lettura ODS") xlsx = tmp_path / "a.xlsx" ods = tmp_path / "b.ods" pd.DataFrame([{"Anno": 2022, "Val": 1}]).to_excel(xlsx, index=False) diff --git a/tests/test_registry_ops.py b/tests/test_registry_ops.py index b0d887d..6f5ff5f 100644 --- a/tests/test_registry_ops.py +++ b/tests/test_registry_ops.py @@ -85,10 +85,19 @@ def test_show_registry_slug_missing(mini_workspace: Path) -> None: @pytest.mark.pure_unit def test_section_count_pure() -> None: """Conteggio entries per sezione del registry unico (pure_unit).""" + from lab_connectors.registry.models import Registry + from toolkit.registry.reader import _section_count - assert _section_count("datasets", {"datasets": [1, 2]}) == 2 - assert _section_count("marts", {"marts": [1]}) == 1 - assert _section_count("signals", {"signals": [1, 2, 3]}) == 3 - assert _section_count("codelists", {"codelists": ["geo", "units"]}) == 2 - assert _section_count("entities", {"entities": {"entities": {}}}) == 0 + reg = Registry( + datasets=[object(), object()], + marts=[object()], + signals=[object(), object(), object()], + codelists=["geo", "units"], + entities={"entities": {}}, + ) + assert _section_count("datasets", reg) == 2 + assert _section_count("marts", reg) == 1 + assert _section_count("signals", reg) == 3 + assert _section_count("codelists", reg) == 2 + assert _section_count("entities", reg) == 0 diff --git a/tests/test_run_validation_gate.py b/tests/test_run_validation_gate.py index d5449b2..80f582b 100644 --- a/tests/test_run_validation_gate.py +++ b/tests/test_run_validation_gate.py @@ -338,12 +338,14 @@ def test_mart_validation_skips_min_rows_in_sample_mode(tmp_path: Path) -> None: @pytest.mark.regression -def test_run_full_validates_via_context_not_direct_calls(tmp_path: Path, monkeypatch) -> None: - """run_full si fida del RunContext — le validazioni sono chiamate solo da run_year. +def test_execute_pipeline_validates_via_context_not_direct_calls( + tmp_path: Path, monkeypatch +) -> None: + """_execute_pipeline si fida del RunContext — le validazioni sono chiamate solo da run_year. Questo test conta quante volte vengono chiamate le funzioni di validazione. Con la vecchia doppia validazione, ogni funzione veniva chiamata 2 volte - (una in run_year + una in run_full). Con il fix, ogni funzione è chiamata + (una in run_year + una nel layer sopra). Con il fix, ogni funzione è chiamata 1 sola volta (solo dentro run_year). """ config_path = tmp_path / "dataset.yml" @@ -382,7 +384,7 @@ def _counting_mart(*args, **kwargs): }, ) - # run_full chiama run_preflight all'inizio — mockiamo per evitare + # _execute_pipeline chiama run_preflight all'inizio — mockiamo per evitare # che validate_config fallisca su config minimale del test monkeypatch.setattr( "toolkit.domain.preflight.run_preflight", @@ -405,7 +407,7 @@ def _counting_mart(*args, **kwargs): }, ) - cmd_run.run_full( + cmd_run._execute_pipeline( config=str(config_path), smoke=False, sample_rows=1000, diff --git a/tests/test_smoke_scout.py b/tests/test_smoke_scout.py index b49ab08..6536a4a 100644 --- a/tests/test_smoke_scout.py +++ b/tests/test_smoke_scout.py @@ -1,24 +1,41 @@ -"""Smoke test: toolkit scout con URL reali. +"""Smoke test: toolkit scout. -Questi test chiamano URL pubblici reali. NON vengono eseguiti in CI — -esclusi via pytest.ini (markers con --strict-markers ma non in --default- -markers) e via ci.yml (--ignore tests/test_smoke_scout.py). +I test di rete (URL pubblici reali) sono dietro ``RUN_NETWORK_SMOKE=1``: +non girano nello step smoke CI (``-m smoke``) per non creare flakiness +dipendente da endpoint esterni. Il test di inferenza puro gira sempre. -Per eseguirli: - pytest tests/test_smoke_scout.py -v --timeout=60 +Per eseguire l'intero file: + RUN_NETWORK_SMOKE=1 pytest tests/test_smoke_scout.py -v -m "" --timeout=60 """ from __future__ import annotations +import os + import pytest from toolkit.scout.infer import infer_years -from toolkit.scout.probe import probe_url, probe_url_routed + +needs_network = pytest.mark.skipif( + not os.environ.get("RUN_NETWORK_SMOKE"), + reason="network smoke: set RUN_NETWORK_SMOKE=1 per eseguire", +) @pytest.mark.smoke +def test_scout_infer_years_real() -> None: + """infer_years su filename reali (pure, no network).""" + assert infer_years("CivileFlussi20142025.xlsx") == (2014, 2025) + assert infer_years("Dati20182022.csv") == (2018, 2022) + assert infer_years("REG_tipo_reddito_2025.csv") == (2025, 2025) + + +@pytest.mark.smoke +@needs_network def test_scout_direct_csv() -> None: """CSV diretto: probe → file, formato CSV.""" + from toolkit.scout.probe import probe_url + url = "https://www1.finanze.gov.it/finanze/analisi_stat/public/v_4_0_0/contenuti/REG_tipo_reddito_2025.csv?d=1615465800" result = probe_url(url, timeout=30) assert result["status_code"] == 200 @@ -26,8 +43,11 @@ def test_scout_direct_csv() -> None: @pytest.mark.smoke +@needs_network def test_scout_html_with_links() -> None: """Pagina HTML con link CSV: probe → html, link trovati.""" + from toolkit.scout.probe import probe_url + url = "https://www1.finanze.gov.it/finanze/analisi_stat/public/index.php?opendata=yes" result = probe_url(url, timeout=30, capture_html=True) assert result["status_code"] == 200 @@ -36,8 +56,11 @@ def test_scout_html_with_links() -> None: @pytest.mark.smoke +@needs_network def test_scout_routed_ckan_dataset() -> None: """CKAN dataset: probe_routed → ckan, risorse trovate.""" + from toolkit.scout.probe import probe_url_routed + url = "https://dati.consip.it/dataset/dataset-bandi-e-gare" result = probe_url_routed(url, timeout=30) assert result["source_type"] == "ckan" @@ -45,17 +68,12 @@ def test_scout_routed_ckan_dataset() -> None: @pytest.mark.smoke +@needs_network def test_scout_routed_ckan_homepage() -> None: """CKAN homepage: probe_routed → ckan_portal.""" + from toolkit.scout.probe import probe_url_routed + url = "https://dati.consip.it" result = probe_url_routed(url, timeout=30) assert result["source_type"] == "ckan" assert result.get("ckan_portal") is True - - -@pytest.mark.smoke -def test_scout_infer_years_real() -> None: - """infer_years su filename reali (pure, no network).""" - assert infer_years("CivileFlussi20142025.xlsx") == (2014, 2025) - assert infer_years("Dati20182022.csv") == (2018, 2022) - assert infer_years("REG_tipo_reddito_2025.csv") == (2025, 2025) diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index 30f1ff1..bc317f0 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -1214,31 +1214,6 @@ def _execute_pipeline( raise typer.Exit(code=1) -def run_full( - config: str | None = None, - years: str | None = None, - smoke: bool = False, - sample_rows: int | None = None, - sample_bytes: int | None = None, - root: str | None = None, - json_output: bool = False, - dry_run: bool = False, -) -> None: - """Backward compat: esegue pipeline completa. - - Chiamata programmatica (non CLI) — il comando CLI ``toolkit run full`` - non è più registrato. Usa ``toolkit run``. - """ - import warnings - - warnings.warn( - "run_full() è deprecato, usa toolkit.run o _execute_pipeline()", - DeprecationWarning, - stacklevel=2, - ) - _execute_pipeline(config, years, smoke, sample_rows, sample_bytes, root, json_output, dry_run) - - def run_preflight_cmd( config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), diff --git a/toolkit/cli/cmd_scout.py b/toolkit/cli/cmd_scout.py index 3bda158..5a450f9 100644 --- a/toolkit/cli/cmd_scout.py +++ b/toolkit/cli/cmd_scout.py @@ -1,7 +1,6 @@ """toolkit scout — esplora URL esterni con probe + routing + inferenze. Output leggibile o JSON. Con --scaffold genera anche i file candidate. -Sostituisce inspect url (deprecato) come comando di URL scouting. Usage: toolkit scout # probe + info leggibile diff --git a/toolkit/cli/inspect/profile_ops.py b/toolkit/cli/inspect/profile_ops.py index 0eb0ace..108516a 100644 --- a/toolkit/cli/inspect/profile_ops.py +++ b/toolkit/cli/inspect/profile_ops.py @@ -1,9 +1,8 @@ """inspect profile — logica profiling RAW. ``csv_preview`` e' in ``toolkit.domain.profile``. -``run_profile`` (logica) resta qui: e' il core del profiling usato da -test e pipeline (il comando CLI Typer corrispondente e' deprecato a -favore di ``toolkit inspect config -m profile``). +``run_profile`` (logica) resta qui: e' il core del profiling RAW, +usato da ``inspect config -m profile`` e dai test di integrazione. """ from __future__ import annotations @@ -18,10 +17,7 @@ def run_profile(cfg: ToolkitConfig, years: list[int], logger: Logger) -> None: - """Core logic: profiling RAW per ogni anno e scrittura su _profile/. - - Chiamabile da cmd_profile (deprecato) e dai test di integrazione. - """ + """Core logic: profiling RAW per ogni anno e scrittura su _profile/.""" clean_cfg: dict[str, Any] = ensure_dict(cfg.clean) or {} for y in years: diff --git a/toolkit/registry/__init__.py b/toolkit/registry/__init__.py index 0d41972..7da9e69 100644 --- a/toolkit/registry/__init__.py +++ b/toolkit/registry/__init__.py @@ -1,10 +1,10 @@ """Registry builder + lettore condiviso DataCivicLab. Capacità unica del Lab per gli artifact registry: -- ``builders``: genera clean_catalog, mart_catalog, pipeline_signals, codelists - (riusando il runtime del toolkit: config model, path resolver, run_state, - parquet_schema) con layout e path contract parametrizzati. -- ``reader``: consulta gli artifact committati nei repo del workspace +- ``builders``: genera l'artifact registry UNICO ``registry.json`` (fusion ADR, + sezioni datasets/marts/signals/codelists/entities) riusando il runtime del + toolkit: config model, path resolver, run_state, parquet_schema. +- ``reader``: consulta i registry committati nei repo del workspace (CLI ``toolkit registry`` e MCP ``toolkit_registry_*``). Chiave canonica: ``dataset.name`` (underscore) — le directory del repo sono diff --git a/toolkit/registry/builders.py b/toolkit/registry/builders.py index cc2e4cd..8407b92 100644 --- a/toolkit/registry/builders.py +++ b/toolkit/registry/builders.py @@ -1,10 +1,9 @@ """Builder degli artifact registry condivisi. -Tre proiezioni dello stesso stato (dataset.yml + run records + parquet locali): - -- ``build_clean_catalog`` — inventario queryabile dei clean parquet. -- ``build_mart_catalog`` — inventario delle tabelle mart (slug {dataset}__{mart}). -- ``build_signals`` — health check operativo (repo-signals standard ACB). +Genera l'artifact registry UNICO per repo (fusion ADR): ``registry.json`` +con sezioni ``datasets``, ``marts``, ``signals``, ``codelists``, ``entities``. +Le sezioni sono proiezioni dello stesso stato (dataset.yml + run records + +parquet locali + GCS), validate contro ``registry.schema.json``. Tutta la lettura del runtime è delegata al toolkit (config model, path resolver, run_state, parquet_schema) — qui solo proiezione e arricchimento di catalogo. diff --git a/toolkit/registry/reader.py b/toolkit/registry/reader.py index 91ae1a4..c6dd471 100644 --- a/toolkit/registry/reader.py +++ b/toolkit/registry/reader.py @@ -24,36 +24,18 @@ SECTIONS = ("datasets", "marts", "signals", "codelists", "entities") -def _section_count(section: str, data: Any) -> int | None: - """Conteggio entries di una sezione del registry. - - Accetta sia un ``Registry`` tipizzato che un raw dict (backward compat). - """ - # Registry tipizzato - if hasattr(data, "datasets"): - reg = data - if section == "datasets": - return len(reg.datasets) - if section == "marts": - return len(reg.marts) - if section == "signals": - return len(reg.signals) - if section == "codelists": - return len(reg.codelists) - if section == "entities": - return len(reg.entities.get("entities") or {}) - return None - # Raw dict (backward compat per test) +def _section_count(section: str, reg: Registry) -> int | None: + """Conteggio entries di una sezione del registry tipizzato.""" if section == "datasets": - return len(data.get("datasets") or []) + return len(reg.datasets) if section == "marts": - return len(data.get("marts") or []) + return len(reg.marts) if section == "signals": - return len(data.get("signals") or []) + return len(reg.signals) if section == "codelists": - return len(data.get("codelists") or []) + return len(reg.codelists) if section == "entities": - return len((data.get("entities") or {}).get("entities") or {}) + return len(reg.entities.get("entities") or {}) return None diff --git a/toolkit/registry/schemas/README.md b/toolkit/registry/schemas/README.md index cd3aa7b..3d16d7d 100644 --- a/toolkit/registry/schemas/README.md +++ b/toolkit/registry/schemas/README.md @@ -22,7 +22,7 @@ inline come `$defs` — nessun `$ref` esterno. ## Compatibilità -Reader e resolver hanno fallback legacy: i repo con i vecchi file separati -(`clean_catalog.json`, `mart_catalog.json`, `pipeline_signals.json`, -`codelists.json`, `entity_graph.json`) continuano a funzionare finché non -migrano a `registry.json`. +Il reader accetta esclusivamente `registry.json` (fusion ADR). Il fallback +legacy sui vecchi file separati (`clean_catalog.json`, `mart_catalog.json`, +`pipeline_signals.json`, `codelists.json`, `entity_graph.json`) è stato +rimosso (PR #468): tutti i repo del workspace sono migrati. diff --git a/toolkit/scout/__init__.py b/toolkit/scout/__init__.py index f80848b..8d88bb3 100644 --- a/toolkit/scout/__init__.py +++ b/toolkit/scout/__init__.py @@ -19,11 +19,7 @@ Scaffold non e' piu' in scout. Usa toolkit.scaffold.full e toolkit.scaffold.sources. """ -from toolkit.scout.sparql import ( # noqa: F401 - discover_named_graphs, - fetch_sparql_count, - infer_graph_schema, -) +from toolkit.scout.sparql import fetch_sparql_count # noqa: F401 from toolkit.scout.http import ( # noqa: F401 CANDIDATE_EXTENSIONS, diff --git a/toolkit/scout/link_extractor.py b/toolkit/scout/link_extractor.py index 78dc49a..f67399a 100644 --- a/toolkit/scout/link_extractor.py +++ b/toolkit/scout/link_extractor.py @@ -18,9 +18,3 @@ group_links, resolve_format, ) - -# Alias backward-compat per i vecchi nomi privati -_extract_prefix = extract_prefix -_extract_years = extract_years -_resolve_format = resolve_format -_extract_filename = extract_filename diff --git a/toolkit/scout/sparql.py b/toolkit/scout/sparql.py index 505bd11..2183fc2 100644 --- a/toolkit/scout/sparql.py +++ b/toolkit/scout/sparql.py @@ -1,51 +1,18 @@ -"""SPARQL scout — named graph discovery, schema inference, lightweight probe. +"""SPARQL scout — lightweight count probe su endpoint. -Wrapper leggero su lab_connectors.http.sparql. -Mantiene le firme originali per backward compat con SO e altri caller. - -Le funzioni di routing/orchestrazione stanno in toolkit.scout.probe. +Wrapper leggero su lab_connectors.http.sparql. Le funzioni di +routing/orchestrazione stanno in toolkit.scout.probe. """ from __future__ import annotations import logging -from typing import Any -from lab_connectors.http.sparql import discover_graphs, execute_sparql, infer_schema +from lab_connectors.http.sparql import execute_sparql log = logging.getLogger(__name__) -def discover_named_graphs( - endpoint: str, - timeout: int = 60, - prefix: str = "", - blacklist: list[str] | None = None, -) -> list[str]: - """Alias per lab_connectors.http.sparql.discover_graphs.""" - return discover_graphs( - endpoint=endpoint, - timeout=timeout, - prefix=prefix, - blacklist=blacklist, - ) - - -def infer_graph_schema( - endpoint: str, - graph_uri: str, - timeout: int = 60, - limit: int = 20, -) -> list[dict[str, Any]]: - """Alias per lab_connectors.http.sparql.infer_schema.""" - return infer_schema( - endpoint=endpoint, - graph_uri=graph_uri, - timeout=timeout, - limit=limit, - ) - - def fetch_sparql_count( endpoint: str, graph_uri: str | None = None,