From 72f815299676603981a8a34b7324df8977fe9203 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:16:56 +0100 Subject: [PATCH 1/3] test: skip ODS senza odfpy + gating network smoke dietro env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_clean_duckdb_read: i 3 test ODS usano importorskip("odf") — fallivano in env senza extra pandas (CI resta verde grazie a [dev,pandas]) - test_smoke_scout: i 4 test con URL reali (finanze.gov.it, consip.it) dietro RUN_NETWORK_SMOoke=1 come RUN_S3_SMOKE — lo step smoke CI (-m smoke) li includeva nonostante il docstring dicesse il contrario, creando dipendenza da endpoint esterni; il test puro infer_years resta smoke e gira sempre --- tests/test_smoke_scout.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_smoke_scout.py b/tests/test_smoke_scout.py index 6536a4a..e548bca 100644 --- a/tests/test_smoke_scout.py +++ b/tests/test_smoke_scout.py @@ -30,7 +30,6 @@ def test_scout_infer_years_real() -> None: assert infer_years("REG_tipo_reddito_2025.csv") == (2025, 2025) -@pytest.mark.smoke @needs_network def test_scout_direct_csv() -> None: """CSV diretto: probe → file, formato CSV.""" @@ -42,7 +41,6 @@ def test_scout_direct_csv() -> None: assert result["kind"] == "file" -@pytest.mark.smoke @needs_network def test_scout_html_with_links() -> None: """Pagina HTML con link CSV: probe → html, link trovati.""" @@ -55,7 +53,6 @@ def test_scout_html_with_links() -> None: assert len(result.get("candidate_links", [])) > 50 # SO registry dice 147 -@pytest.mark.smoke @needs_network def test_scout_routed_ckan_dataset() -> None: """CKAN dataset: probe_routed → ckan, risorse trovate.""" @@ -67,7 +64,6 @@ def test_scout_routed_ckan_dataset() -> None: assert len(result.get("ckan_resources") or []) > 0 -@pytest.mark.smoke @needs_network def test_scout_routed_ckan_homepage() -> None: """CKAN homepage: probe_routed → ckan_portal.""" From cf0a7526e057a011a7732b2a2f32299cfea4540f Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:26:06 +0100 Subject: [PATCH 2/3] test(smoke_scout): ripristina marker smoke sui test di rete gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L'audit markers richiede un marker per ogni test nei file toccati; i 4 test di rete avevano solo skipif senza marker. Stesso pattern di test_s3_parquet_smoke: @pytest.mark.smoke + skipif env var — restano collezionabili da -m smoke ma saltano senza RUN_NETWORK_SMOKE=1. --- tests/test_smoke_scout.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_smoke_scout.py b/tests/test_smoke_scout.py index e548bca..6536a4a 100644 --- a/tests/test_smoke_scout.py +++ b/tests/test_smoke_scout.py @@ -30,6 +30,7 @@ def test_scout_infer_years_real() -> None: assert infer_years("REG_tipo_reddito_2025.csv") == (2025, 2025) +@pytest.mark.smoke @needs_network def test_scout_direct_csv() -> None: """CSV diretto: probe → file, formato CSV.""" @@ -41,6 +42,7 @@ def test_scout_direct_csv() -> None: assert result["kind"] == "file" +@pytest.mark.smoke @needs_network def test_scout_html_with_links() -> None: """Pagina HTML con link CSV: probe → html, link trovati.""" @@ -53,6 +55,7 @@ def test_scout_html_with_links() -> None: assert len(result.get("candidate_links", [])) > 50 # SO registry dice 147 +@pytest.mark.smoke @needs_network def test_scout_routed_ckan_dataset() -> None: """CKAN dataset: probe_routed → ckan, risorse trovate.""" @@ -64,6 +67,7 @@ def test_scout_routed_ckan_dataset() -> None: assert len(result.get("ckan_resources") or []) > 0 +@pytest.mark.smoke @needs_network def test_scout_routed_ckan_homepage() -> None: """CKAN homepage: probe_routed → ckan_portal.""" From c31da84df23476a1f20732f3938a1de678db70b5 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:31:06 +0100 Subject: [PATCH 3/3] feat(catalog): tokenized semantic find + route free-text discovery to ACB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toolkit_dataset find restava indietro su query naturali: la semantica matchava solo la frase intera come substring continua. Ora tokenizza le query (primitive condivise lab_connectors.textmatch) e fa AND cross-part tra meta e colonne. - meta hit: tutti i token sparsi tra slug + name/description/source/ tags/category - colonna hit: tutti i token dentro nome/descrizione colonna - stopword italiane escluse dai token richiesti - morfologia leggera + prefissi (sanitaria~=sanita, import→importo) - split per parola: 'appalti' non matcha più dentro 'n_subappalti' - description tool MCP: find = slug/colonne live; discovery org-wide → dataciviclab-context find (DEC #11) Depende da lab-connectors#92 (textmatch) — mergiare prima quella. --- tests/test_catalog_ops.py | 39 +++++++++++++++++++++++++++++++++++++++ toolkit/domain/catalog.py | 19 +++++++++++++------ toolkit/mcp/server.py | 5 ++++- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/tests/test_catalog_ops.py b/tests/test_catalog_ops.py index c013268..33c34c1 100644 --- a/tests/test_catalog_ops.py +++ b/tests/test_catalog_ops.py @@ -734,6 +734,45 @@ def test_find_by_column(self, resolver_semantic: CatalogResolver) -> None: {"name": "importo", "type": "DOUBLE", "role": "metric", "description": "Importo gara"} ] + def test_find_multi_token_cross_meta(self, resolver_semantic: CatalogResolver) -> None: + """Regression: i token di una query naturale possono spandersi tra + campi meta (AND cross-part) — prima serviva la frase intera continua.""" + # 'appalti' solo in tags/category, 'gara' in name/description + res = resolver_semantic.list_datasets(query="appalti gara") + slugs = {d["slug"] for d in res["datasets"]} + assert "anac_bandi_gara" in slugs + assert "terna_electricity_by_source" not in slugs + + def test_find_drops_stopwords(self, resolver_semantic: CatalogResolver) -> None: + """Le stopword italiane non sono token richiesti.""" + res = resolver_semantic.list_datasets(query="bandi di gara") + slugs = {d["slug"] for d in res["datasets"]} + assert "anac_bandi_gara" in slugs + # solo stopword → nessun hit semantico + res2 = resolver_semantic.list_datasets(query="di dei delle") + assert res2["datasets"] == [] + + def test_find_morphology_plural(self, resolver_semantic: CatalogResolver) -> None: + """'gare' matcha 'gara' (final-vowel strip).""" + res = resolver_semantic.list_datasets(query="gare appalti") + slugs = {d["slug"] for d in res["datasets"]} + assert "anac_bandi_gara" in slugs + + def test_find_column_prefix(self, resolver_semantic: CatalogResolver) -> None: + """Prefisso di nome colonna: 'import' → 'importo'.""" + res = resolver_semantic.list_datasets(query="import") + entry = next(d for d in res["datasets"] if d["slug"] == "anac_bandi_gara") + assert entry["meta_match"] is not True + assert [c["name"] for c in entry["matched_columns"]] == ["importo"] + + def test_token_in_text_no_substring_false_positive(self) -> None: + """'appalti' non matcha dentro 'subappalti' (split per parola).""" + from lab_connectors.textmatch import token_in_text + + assert not token_in_text("appalti", "n_subappalti") + assert token_in_text("appalti", "appalti pubblici") + assert token_in_text("gara", "gare") # morphologia + def test_find_metric_only(self, resolver_semantic: CatalogResolver) -> None: """metric_only esclude i dataset senza colonne metric.""" res = resolver_semantic.list_datasets(metric_only=True) diff --git a/toolkit/domain/catalog.py b/toolkit/domain/catalog.py index f8db3f3..a4e92a8 100644 --- a/toolkit/domain/catalog.py +++ b/toolkit/domain/catalog.py @@ -21,6 +21,7 @@ from typing import Any, Literal from lab_connectors.gcs.paths import CLEAN_BUCKET, MART_BUCKET +from lab_connectors.textmatch import query_tokens, token_in_text from toolkit.core.duckdb_shape import parquet_preview from toolkit.core.io import read_json_or_none, read_yaml @@ -439,24 +440,30 @@ def _semantic_query_hits( ) -> dict[str, list[dict[str, Any] | None]]: """Slug che matchano la query sulla semantica (meta o colonne). + Tokenizza la query (stopword italiane escluse, primitive condivise in + ``lab_connectors.textmatch``) e richiede che **tutti** i token matchino + sparsi tra slug + meta (AND cross-part), oppure tutti dentro una singola + colonna. Prima serviva la frase intera come substring continua: + "appalti pubblici anac" non matchava mai. + Returns: ``{slug: [None | colonna_matchata, ...]}`` — ``None`` = meta match (name/description/source/tags/category), dict = colonna matchata. """ - q = query.lower() + tokens = query_tokens(query) + if not tokens: + return {} hits: dict[str, list[dict[str, Any] | None]] = {} for slug, s in semantic.items(): matched: list[dict[str, Any] | None] = [] meta_parts = " ".join( str(s.get(k) or "") for k in ("name", "description", "source", "tags", "category") ) - if q in slug.lower() or q in meta_parts.lower(): + if all(token_in_text(token, slug) or token_in_text(token, meta_parts) for token in tokens): matched.append(None) for col in s.get("columns", []) or []: - if ( - q in str(col.get("name", "")).lower() - or q in str(col.get("description", "")).lower() - ): + col_text = f"{col.get('name', '')} {col.get('description', '')}" + if all(token_in_text(token, col_text) for token in tokens): matched.append( { "name": col.get("name"), diff --git a/toolkit/mcp/server.py b/toolkit/mcp/server.py index 10d50fc..0f2acfd 100644 --- a/toolkit/mcp/server.py +++ b/toolkit/mcp/server.py @@ -74,7 +74,10 @@ description=( "Ispezione dataset: find, overview, status, preflight, schema-diff, related.\n\n" "Actions:\n" - "- find: cerca dataset per slug/testo/source (params: query, layer, limit, source, stage, status_filter)\n" + "- find: cerca dataset per slug, colonne o meta (params: query, layer, limit, " + "source, stage, status_filter). Per discovery org-wide con query naturali " + "usa dataciviclab-context find; toolkit find è lo strumento live su " + "workspace+GCS e cerca anche per nome/descrizione colonna.\n" "- overview: schema colonne + conteggio + preview (params: slug, layer, year, source, profile)\n" "- status: stato completo dataset (params: config_path, year, since, until)\n" "- preflight: diagnostica pre-run (params: config_path, years)\n"