From 4101f09cf8a1a9187af3a693991803268252dfe3 Mon Sep 17 00:00:00 2001 From: David Huser <4357648+davidhuser@users.noreply.github.com> Date: Wed, 15 Apr 2026 09:38:06 +0200 Subject: [PATCH 1/3] accuracy check cmd --- src/mail_municipalities/accuracy/check.py | 203 ++++++++++++++++++++++ src/mail_municipalities/cli.py | 57 ++++++ tests/test_accuracy.py | 118 +++++++++++++ 3 files changed, 378 insertions(+) create mode 100644 src/mail_municipalities/accuracy/check.py diff --git a/src/mail_municipalities/accuracy/check.py b/src/mail_municipalities/accuracy/check.py new file mode 100644 index 0000000..e9ea055 --- /dev/null +++ b/src/mail_municipalities/accuracy/check.py @@ -0,0 +1,203 @@ +"""Spot-check provider classification for specific domains. + +Creates probes in state.db so that ``accuracy send`` / ``accuracy collect`` +can verify the classification via NDR. Running ``accuracy check`` again +after collection shows the verified result. +""" + +from __future__ import annotations + +import json +import uuid +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger +from rich.console import Console +from rich.table import Table + +from mail_municipalities.accuracy.models import ( + CLASSIFIER_TO_EVAL, + NDR_TO_CLASSIFIER, + Probe, + ProbeStatus, +) +from mail_municipalities.accuracy.state import StateDB + +console = Console() + +_COUNTRIES = ("de", "at", "ch") + + +@dataclass(frozen=True) +class CheckResult: + domain: str + name: str | None + country: str | None + provider: str | None + confidence: float | None + status: str # "not_found" | probe status value | "new" + actual: str | None + match: bool | None + + +async def check_domains( + domains: list[str], + providers_dir: Path, + state: StateDB, +) -> list[CheckResult]: + """Look up domains, create probes for new ones, show status for existing.""" + # Build domain → (entry, cc) from all provider files. + catalog: dict[str, tuple[dict, str]] = {} + for cc in _COUNTRIES: + path = providers_dir / f"providers_{cc}.json" + if not path.exists(): + continue + data = json.loads(path.read_text()) + for entry in data["municipalities"]: + d = entry.get("domain") + if d: + catalog[d] = (entry, cc) + + # Load existing probes and NDRs. + all_probes = await state.get_all_probes() + all_ndrs = await state.get_all_ndrs() + probes_by_domain: dict[str, Probe] = {} + for p in all_probes: + probes_by_domain[p.domain] = p + ndr_by_probe = {ndr.probe_id: ndr for ndr in all_ndrs} + + results: list[CheckResult] = [] + new_probes: list[Probe] = [] + + for domain in domains: + if domain not in catalog: + results.append( + CheckResult( + domain=domain, + name=None, + country=None, + provider=None, + confidence=None, + status="not_found", + actual=None, + match=None, + ) + ) + continue + + entry, cc = catalog[domain] + provider = entry.get("provider") + confidence = entry.get("classification_confidence") + + existing = probes_by_domain.get(domain) + if existing is not None: + # Already probed — show current status and NDR result if available. + actual = None + match = None + ndr = ndr_by_probe.get(existing.probe_id) + if ndr is not None: + actual = NDR_TO_CLASSIFIER.get(ndr.ndr_provider.value, "unknown") + pred_eval = CLASSIFIER_TO_EVAL.get(provider or "", "unknown") + match = pred_eval == actual + results.append( + CheckResult( + domain=domain, + name=entry.get("name"), + country=cc.upper(), + provider=provider, + confidence=confidence, + status=existing.status.value, + actual=actual, + match=match, + ) + ) + else: + # New — create a probe. + probe = _make_probe(entry, cc) + new_probes.append(probe) + results.append( + CheckResult( + domain=domain, + name=entry.get("name"), + country=cc.upper(), + provider=provider, + confidence=confidence, + status="new", + actual=None, + match=None, + ) + ) + + if new_probes: + inserted = await state.insert_probes(new_probes) + logger.info("Created {} probe(s) for check domains", inserted) + + return results + + +def _make_probe(entry: dict, country: str) -> Probe: + """Create a Probe from a municipality entry.""" + short_uuid = uuid.uuid4().hex[:12] + domain = entry["domain"] + return Probe( + probe_id=uuid.uuid4().hex, + domain=domain, + municipality_code=str(entry["code"]), + municipality_name=entry["name"], + country=country, + recipient=f"validation-probe-{short_uuid}@{domain}", + predicted_provider=entry["provider"], + predicted_confidence=entry.get("classification_confidence", 0.0), + gateway=entry.get("gateway"), + status=ProbeStatus.PENDING, + ) + + +def print_check_table(results: list[CheckResult]) -> None: + """Print check results as a Rich table.""" + table = Table(title="Domain Classification Check", show_lines=True) + table.add_column("Domain", style="bold") + table.add_column("City") + table.add_column("Country") + table.add_column("Provider") + table.add_column("Confidence", justify="right") + table.add_column("Status") + table.add_column("Actual") + table.add_column("Match") + + for r in results: + if r.status == "not_found": + table.add_row(r.domain, "[dim]not found[/dim]", "", "", "", "", "", "") + continue + + match_str = "" + if r.match is True: + match_str = "[green]yes[/green]" + elif r.match is False: + match_str = "[red]no[/red]" + + status_str = r.status + if r.status == "new": + status_str = "[cyan]pending[/cyan]" + elif r.status == "ndr_received": + status_str = "[green]ndr_received[/green]" + + table.add_row( + r.domain, + r.name or "", + r.country or "", + r.provider or "", + f"{r.confidence:.0f}" if r.confidence is not None else "", + status_str, + r.actual or "", + match_str, + ) + + console.print(table) + + new_count = sum(1 for r in results if r.status == "new") + if new_count: + console.print( + f"\n[cyan]{new_count} new probe(s) created.[/cyan] Run [bold]accuracy send --no-dry-run[/bold] then [bold]accuracy collect[/bold] to verify." + ) diff --git a/src/mail_municipalities/cli.py b/src/mail_municipalities/cli.py index c044d49..89abcfb 100644 --- a/src/mail_municipalities/cli.py +++ b/src/mail_municipalities/cli.py @@ -672,6 +672,40 @@ async def _run() -> None: asyncio.run(_run()) +@_accuracy_app.command("check") +def accuracy_check_cmd( + domains: Annotated[ + list[str], + typer.Argument(help="One or more email domains to look up"), + ], + providers_dir: Annotated[ + Path, + typer.Option("--providers-dir", help="Directory with provider classification output"), + ] = Path("output/providers"), + verbose: Annotated[ + bool, + typer.Option("-v", "--verbose", help="Enable debug logging"), + ] = False, + output: Annotated[ + Optional[Path], + typer.Option("-o", "--output", help="Custom output directory"), + ] = None, +) -> None: + """Spot-check provider classification for specific domains.""" + cfg = _accuracy_config(output) + setup_logging(verbose, log_path=cfg.output_dir / "accuracy.log") + + from mail_municipalities.accuracy.check import check_domains, print_check_table + from mail_municipalities.accuracy.state import StateDB + + async def _run() -> None: + async with StateDB(cfg.state_db_path) as state: + results = await check_domains(domains, providers_dir, state) + print_check_table(results) + + asyncio.run(_run()) + + _accuracy_standalone_app = typer.Typer(add_completion=False) @@ -828,6 +862,29 @@ def _accuracy_status_main( accuracy_status_cmd(verbose=verbose, output=output) +@_accuracy_standalone_app.command("check") +def _accuracy_check_main( + domains: Annotated[ + list[str], + typer.Argument(help="One or more email domains to look up"), + ], + providers_dir: Annotated[ + Path, + typer.Option("--providers-dir", help="Directory with provider classification output"), + ] = Path("output/providers"), + verbose: Annotated[ + bool, + typer.Option("-v", "--verbose", help="Enable debug logging"), + ] = False, + output: Annotated[ + Optional[Path], + typer.Option("-o", "--output", help="Custom output directory"), + ] = None, +) -> None: + """Spot-check provider classification for specific domains.""" + accuracy_check_cmd(domains=domains, providers_dir=providers_dir, verbose=verbose, output=output) + + def resolve() -> None: """Entry point for 'resolve' script.""" _resolve_app() diff --git a/tests/test_accuracy.py b/tests/test_accuracy.py index aebfefe..bbdd466 100644 --- a/tests/test_accuracy.py +++ b/tests/test_accuracy.py @@ -2,9 +2,11 @@ from __future__ import annotations +import json import uuid from datetime import datetime, timezone from email.message import EmailMessage +from pathlib import Path import pytest @@ -700,3 +702,119 @@ def test_sender_address_explicit(self): _env_file=None, # type: ignore[call-arg] ) assert cfg.sender_address == "sender@custom.com" + + +# ── Check tests ────────────────────────────────────────────────── + + +class TestCheck: + @pytest.fixture + def providers_dir(self, tmp_path: Path) -> Path: + """Create temp providers JSON files.""" + for cc, entries in [ + ( + "de", + [ + { + "code": "1", + "name": "Berlin", + "domain": "berlin.de", + "provider": "domestic", + "classification_confidence": 80.0, + } + ], + ), + ( + "ch", + [ + { + "code": "2", + "name": "Zürich", + "domain": "zurich.ch", + "provider": "microsoft", + "classification_confidence": 100.0, + } + ], + ), + ( + "at", + [ + { + "code": "3", + "name": "Wien", + "domain": "wien.gv.at", + "provider": "domestic", + "classification_confidence": 80.0, + } + ], + ), + ]: + (tmp_path / f"providers_{cc}.json").write_text(json.dumps({"municipalities": entries})) + return tmp_path + + @pytest.mark.asyncio + async def test_check_creates_probes(self, providers_dir: Path, tmp_path: Path): + from mail_municipalities.accuracy.check import check_domains + + db_path = tmp_path / "state.db" + async with StateDB(db_path) as state: + results = await check_domains(["berlin.de", "zurich.ch"], providers_dir, state) + # Probes should have been created. + existing = await state.get_existing_domains() + + assert len(results) == 2 + assert results[0].name == "Berlin" + assert results[0].provider == "domestic" + assert results[0].status == "new" + assert results[1].provider == "microsoft" + assert "berlin.de" in existing + assert "zurich.ch" in existing + + @pytest.mark.asyncio + async def test_check_domains_not_found(self, providers_dir: Path, tmp_path: Path): + from mail_municipalities.accuracy.check import check_domains + + db_path = tmp_path / "state.db" + async with StateDB(db_path) as state: + results = await check_domains(["nonexistent.de"], providers_dir, state) + + assert len(results) == 1 + assert results[0].name is None + assert results[0].status == "not_found" + + @pytest.mark.asyncio + async def test_check_shows_ndr_result(self, providers_dir: Path, tmp_path: Path): + from mail_municipalities.accuracy.check import check_domains + + db_path = tmp_path / "state.db" + async with StateDB(db_path) as state: + probe = Probe( + probe_id="p1", + domain="berlin.de", + municipality_code="1", + municipality_name="Berlin", + country="de", + recipient="validation-probe-abc@berlin.de", + predicted_provider="domestic", + predicted_confidence=80.0, + ) + await state.insert_probes([probe]) + ndr = NdrResult( + probe_id="p1", + received_at=datetime.now(tz=timezone.utc), + ndr_from="mailer-daemon@berlin.de", + ndr_provider=NdrProvider.POSTFIX, + generating_mta="mail.berlin.de", + confidence=0.9, + evidence=[NdrEvidence(pattern="postfix_body", matched_value="Postfix")], + raw_headers="", + ) + await state.insert_ndr(ndr) + await state.update_probe_status("p1", ProbeStatus.NDR_RECEIVED) + + results = await check_domains(["berlin.de"], providers_dir, state) + + assert len(results) == 1 + assert results[0].status == "ndr_received" + assert results[0].actual == "self-hosted" + assert results[0].match is True From d2057e30512bb2c0be9794bf4c34aca3aab00420 Mon Sep 17 00:00:00 2001 From: David Huser <4357648+davidhuser@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:27:11 +0200 Subject: [PATCH 2/3] on-prem ndr parsing --- .../accuracy/ndr_parser.py | 20 +++++++++-- tests/test_accuracy.py | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/mail_municipalities/accuracy/ndr_parser.py b/src/mail_municipalities/accuracy/ndr_parser.py index dde7464..835eeac 100644 --- a/src/mail_municipalities/accuracy/ndr_parser.py +++ b/src/mail_municipalities/accuracy/ndr_parser.py @@ -83,12 +83,28 @@ def parse_ndr(msg: EmailMessage) -> tuple[NdrProvider, float, str, list[NdrEvide if ms_patterns: is_online = any( - "outlook.com" in p[1].lower() or "protection.outlook" in p[1].lower() + "outlook.com" in p[1].lower() or "protection.outlook" in p[1].lower() or "onmicrosoft.com" in p[1].lower() for p in ms_patterns if p[0] - in ("reporting-mta outlook", "received via outlook", "from postmaster@outlook", "dsn mentions outlook") + in ( + "reporting-mta outlook", + "received via outlook", + "from postmaster@outlook", + "from *.onmicrosoft.com", + "dsn mentions outlook", + ) ) provider = NdrProvider.MICROSOFT if is_online else NdrProvider.EXCHANGE_ONPREM + + # Definitive: Microsoft's own header distinguishes hosted vs hybrid. + entity_header = target_headers.get("x-ms-exchange-crosstenant-fromentityheader", "").lower() + if entity_header == "hosted": + provider = NdrProvider.MICROSOFT + evidence.append(NdrEvidence(pattern="crosstenant-fromentityheader", matched_value="Hosted")) + elif entity_header == "hybridonprem": + provider = NdrProvider.EXCHANGE_ONPREM + evidence.append(NdrEvidence(pattern="crosstenant-fromentityheader", matched_value="HybridOnPrem")) + conf = min(0.3 + 0.15 * len(ms_patterns), 1.0) for pat, val in ms_patterns: evidence.append(NdrEvidence(pattern=pat, matched_value=val)) diff --git a/tests/test_accuracy.py b/tests/test_accuracy.py index bbdd466..d06de63 100644 --- a/tests/test_accuracy.py +++ b/tests/test_accuracy.py @@ -361,6 +361,40 @@ def test_gmail_relay_headers_ignored(self): # Should detect Exchange, not Google. assert provider in (NdrProvider.MICROSOFT, NdrProvider.EXCHANGE_ONPREM) + def test_exchange_online_onmicrosoft_from(self): + """NDR from postmaster@*.onmicrosoft.com with Hosted entity header → MICROSOFT.""" + msg = _make_ndr_email( + from_addr="postmaster@stadtxyz.onmicrosoft.com", + extra_headers={ + "X-MS-Exchange-Message-Is-Ndr": "", + "X-Ms-Exchange-Crosstenant-Fromentityheader": "Hosted", + }, + received=[ + "from GVAP278CU002.outbound.protection.outlook.com by slusv0255.example.ch", + "from ZR3P278MB1195.CHEP278.PROD.OUTLOOK.COM by ZR3P278MB1195", + ], + body="Delivery has failed to these recipients or groups.", + ) + provider, confidence, mta, evidence = parse_ndr(msg) + assert provider == NdrProvider.MICROSOFT + + def test_exchange_hybrid_onprem(self): + """NDR from postmaster@domain.ch with HybridOnPrem entity header → EXCHANGE_ONPREM.""" + msg = _make_ndr_email( + from_addr="postmaster@bern.ch", + extra_headers={ + "X-MS-Exchange-Message-Is-Ndr": "", + "X-Ms-Exchange-Crosstenant-Fromentityheader": "HybridOnPrem", + }, + received=[ + "from GVAP278CU002.outbound.protection.outlook.com by mx.google.com", + "from AutoDiscover.bgov.ch by ZRH2EPF00000151.mail.protection.outlook.com", + ], + body="Delivery has failed to these recipients or groups.", + ) + provider, confidence, mta, evidence = parse_ndr(msg) + assert provider == NdrProvider.EXCHANGE_ONPREM + def test_unknown_ndr(self): msg = _make_ndr_email( from_addr="postmaster@somegateway.net", From dc13633eed44e3fa07aa06acdab2ef374beebb93 Mon Sep 17 00:00:00 2001 From: David Huser <4357648+davidhuser@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:13:30 +0200 Subject: [PATCH 3/3] add sqlite3 to vm --- scripts/cloud-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/cloud-config.yaml b/scripts/cloud-config.yaml index 4b42951..c1bdb16 100644 --- a/scripts/cloud-config.yaml +++ b/scripts/cloud-config.yaml @@ -4,6 +4,7 @@ packages: - htop - git - netcat-openbsd + - sqlite3 users: - name: scanner ssh-authorized-keys: