From 104073e8c510fb52504b87f6fc162862bf488db0 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:14:37 +0100 Subject: [PATCH 1/2] feat(reusable): add run-batch-command for selective PR runs New optional input 'run-batch-command' enables repos to use a different command (e.g. toolkit run --batch batch.txt) when a PR triggers the pipeline and batch.txt has content. Falls back to run-command otherwise. This lets repos with make run-all switch to batch mode on PRs without changing their schedule/dispatch behavior. --- .github/workflows/pipeline-reusable.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipeline-reusable.yml b/.github/workflows/pipeline-reusable.yml index c52f478..ad2d602 100644 --- a/.github/workflows/pipeline-reusable.yml +++ b/.github/workflows/pipeline-reusable.yml @@ -12,6 +12,7 @@ # - run-command: comando da eseguire (es. "make run-all" o "toolkit run --batch batch.txt") # - repo-slug: prefisso GCS e registry # - detect-paths: directory da monitorare (es. "datasets compose"). Vuoto = run all +# - run-batch-command: comando alternativo per batch selettivo su PR (opzionale) name: Pipeline (reusable) @@ -26,6 +27,10 @@ on: description: "Comando da eseguire per i dataset (es. make run-all)" required: true type: string + run-batch-command: + description: "Comando alternativo per batch selettivo su PR (es. make run-batch). Vuoto = usa run-command" + default: "" + type: string sync-pattern: description: "flat (rsync intera dir out/) o per-dataset (per slug/year)" default: "flat" @@ -139,10 +144,25 @@ jobs: echo "📦 $(wc -l < batch.txt | tr -d ' ') dataset (schedule/dispatch — tutti)" cat batch.txt + # ── Resolve run command (batch vs full) ────────────────────────── + - name: Resolve run command + id: run-cmd + if: inputs.detect-paths == '' || github.event_name != 'pull_request' || steps.detect.outputs.found == 'true' + run: | + if [ "${{ github.event_name }}" = "pull_request" ] && \ + [ -n "${{ inputs.run-batch-command }}" ] && \ + [ -s batch.txt ]; then + echo "cmd=${{ inputs.run-batch-command }}" >> "$GITHUB_OUTPUT" + echo "🔧 Batch mode: ${{ inputs.run-batch-command }}" + else + echo "cmd=${{ inputs.run-command }}" >> "$GITHUB_OUTPUT" + echo "🔧 Full mode: ${{ inputs.run-command }}" + fi + # ── Run datasets ──────────────────────────────────────────────── - name: Run datasets if: inputs.detect-paths == '' || github.event_name != 'pull_request' || steps.detect.outputs.found == 'true' - run: ${{ inputs.run-command }} + run: ${{ steps.run-cmd.outputs.cmd }} # ── Verify (opzionale) ────────────────────────────────────────── - name: Verify output From d32fa2cd961e24dd8f59cdca5db529a9cd184a38 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:06:29 +0100 Subject: [PATCH 2/2] fix(drift-check): fetch config from GitHub API instead of local file The drift-check script was looking for infra/agent-context-builder/ dataciviclab.config.yml locally, which doesn't exist in CI (separate repo). Now fetches from GitHub API by default, falls back to local file if --config is explicitly passed. --- scripts/drift_check.py | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/scripts/drift_check.py b/scripts/drift_check.py index a08fb47..3b3723c 100644 --- a/scripts/drift_check.py +++ b/scripts/drift_check.py @@ -22,6 +22,7 @@ La lista dei repo viene letta da `dataciviclab.config.yml` (agent-context-builder), che viene aggiornata automaticamente ogni giorno dal workflow discover-registries. +In CI fetcha il config da GitHub API; localmente usa il file se presente. Uso (locale): python scripts/drift_check.py [--token $GITHUB_TOKEN] [--config path/to/config.yml] @@ -63,6 +64,25 @@ def _load_repos_from_config(config_path: Path) -> list[str]: """Legge la lista repo da dataciviclab.config.yml (formato YAML semplice).""" text = config_path.read_text(encoding="utf-8") + return _parse_repos_yaml(text) + + +def _load_repos_from_github(token: str | None) -> list[str]: + """Fetch config da GitHub API (agent-context-builder/dataciviclab.config.yml).""" + url = ( + f"{API}/repos/dataciviclab/agent-context-builder" + "/contents/dataciviclab.config.yml?ref=main" + ) + req = urllib.request.Request( + url, headers={**api_headers(token), "Accept": RAW_ACCEPT} + ) + with urllib.request.urlopen(req, timeout=30) as resp: + text = resp.read().decode("utf-8", "replace") + return _parse_repos_yaml(text) + + +def _parse_repos_yaml(text: str) -> list[str]: + """Estrae la lista repo da un YAML semplice con chiave 'repos:'.""" in_repos = False repos: list[str] = [] for line in text.splitlines(): @@ -76,7 +96,6 @@ def _load_repos_from_config(config_path: Path) -> list[str]: if name and not name.startswith("#"): repos.append(name) elif stripped and not stripped.startswith("#"): - # fine della sezione repos break return repos @@ -243,12 +262,25 @@ def main() -> int: print("⚠️ Nessun token: API non autenticata (rate limit basso). " "Passa GITHUB_TOKEN/DRIFT_CHECK_TOKEN per risultati affidabili.") + # Fetch config: GitHub API di default, fallback a file locale se --config esplicito config_path = Path(args.config) - if not config_path.exists(): - print(f"❌ Config non trovato: {config_path}") + explicit_config = "--config" in sys.argv + if explicit_config and config_path.exists(): + repos = [r for r in _load_repos_from_config(config_path) if r not in _EXCLUDE] + print(f"📋 {len(repos)} repo da {config_path.name} (locale)") + elif token: + try: + repos = [r for r in _load_repos_from_github(token) if r not in _EXCLUDE] + print(f"📋 {len(repos)} repo da agent-context-builder (GitHub API)") + except urllib.error.HTTPError as exc: + print(f"❌ Impossibile fetchare config da GitHub: {exc.code}") + return 1 + elif config_path.exists(): + repos = [r for r in _load_repos_from_config(config_path) if r not in _EXCLUDE] + print(f"📋 {len(repos)} repo da {config_path.name} (locale, fallback)") + else: + print("❌ Nessun config disponibile (né GitHub API né file locale)") return 1 - repos = [r for r in _load_repos_from_config(config_path) if r not in _EXCLUDE] - print(f"📋 {len(repos)} repo da {config_path.name}") report = Report() for repo in repos: