Add PR-driven reference-data workflow-bundle pipeline (MetaPhlAn, mOTUs, SameStr)

.ci/import_reference_data.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Stage 3 of the IDC reference-data pipeline: import the reference-data bundles
+# built on Galaxy (Stage 2) onto CVMFS.
+#
+# For each request under data-managers/, this locates the build's workflow
+# invocation (via its history idc-<dm>-<version>), resolves the bundle
+# dataset(s) from that invocation's named outputs, and imports each with
+# galaxy-import-data-bundle - recording record/<dm>/<version> for idempotency
+# (already-imported builds are skipped).
+#
+# It must run INSIDE a CVMFS transaction, i.e. where ${CVMFS_ROOT} is writable
+# (the Jenkins job opens the transaction; see .ci/jenkins.sh). The Python
+# environment must have bioblend + our scripts, and galaxy-import-data-bundle
+# (from galaxy-maintenance-scripts) must be on PATH or given via IMPORT_CMD.
+#
+# Environment:
+#   GALAXY_URL          Galaxy the bundles were built on (default test.galaxyproject.org)
+#   EPHEMERIS_API_KEY   API key for GALAXY_URL (required)
+#   CVMFS_ROOT          CVMFS repo root (default /cvmfs/idc.galaxyproject.org)
+#   PYTHON              Python interpreter (default python3)
+#   IMPORT_CMD          galaxy-import-data-bundle executable (default: on PATH)
+set -euo pipefail
+
+: "${GALAXY_URL:=https://test.galaxyproject.org}"
+: "${CVMFS_ROOT:=/cvmfs/idc.galaxyproject.org}"
+: "${PYTHON:=python3}"
+: "${IMPORT_CMD:=galaxy-import-data-bundle}"
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+shopt -s nullglob
+found=false
+for req in "$REPO_DIR"/data-managers/*/*.yml "$REPO_DIR"/data-managers/*/*.yaml; do
+    found=true
+    dm="$(basename "$(dirname "$req")")"
+    version="$(basename "$req")"; version="${version%.*}"
+    echo "== reference data: ${dm}/${version} =="
+    # Skip data that already exists in the Galaxy data table (the build stage
+    # skips it too, so there is no history to import). --print-new emits the
+    # request only if its data is not present.
+    if [ -z "$("$PYTHON" "$REPO_DIR/scripts/check_data_exists.py" "$req" --print-new --reference-galaxy "$GALAXY_URL")" ]; then
+        echo "   already exists on $GALAXY_URL; skipping"
+        continue
+    fi
+    "$PYTHON" "$REPO_DIR/scripts/import_bundles.py" \
+        --galaxy-url "$GALAXY_URL" \
+        --history-name "idc-${dm}-${version}" \
+        --dm "$dm" --version "$version" \
+        --cvmfs-root "$CVMFS_ROOT" \
+        --import-cmd "$IMPORT_CMD"
+done
+$found || echo "No reference-data requests under data-managers/."

.ci/jenkins.sh

@@ -4,8 +4,23 @@ set -euo pipefail
 # Set this variable to 'true' to publish on successful installation
 : ${PUBLISH:=false}
 
+# Set to 'true' to import ONLY reference-data (data-managers/) bundles and skip
+# the genome (tool-data) build+import entirely. Also enabled by commenting
+# "@galaxybot deploy reference-data" on the PR (see check_bot_command).
+: ${REFERENCE_DATA_ONLY:=false}
+
 BUILD_GALAXY_URL="http://idc-build"
 PUBLISH_GALAXY_URL="https://usegalaxy.org"
+# Galaxy where the IDC reference-data (workflow-bundle) builds run - Stage 2
+# (.github/workflows/build.yml) builds bundles here for Stage 3 to import.
+REFERENCE_DATA_GALAXY_URL="https://test.galaxyproject.org"
+# API key for $REFERENCE_DATA_GALAXY_URL. Stage 3 uses it (via bioblend) to look
+# up the built bundles' dataset ids on that server; the bundle download itself is
+# unauthenticated. This is a DISTINCT server from $PUBLISH_GALAXY_URL, so it needs
+# its own key - do not reuse $EPHEMERIS_API_KEY, which authenticates the genome
+# import against $PUBLISH_GALAXY_URL. Falls back to $EPHEMERIS_API_KEY only for
+# backward compatibility when a reference-data-only key is not provided.
+: ${REFERENCE_DATA_API_KEY:=${EPHEMERIS_API_KEY:-}}
 SSH_MASTER_SOCKET_DIR="${HOME}/.cache/idc"
 MAIN_BRANCH='main'
 
@@ -34,7 +49,14 @@ USE_LOCAL_OVERLAYFS=false
 # Set to true to run the importer in a docker container
 USE_DOCKER="$USE_LOCAL_OVERLAYFS"
 
-REMOTE_PYTHON=/opt/rh/rh-python38/root/usr/bin/python3
+# Python for the remote ephemeris/maintenance venvs. The Stratum 0's system
+# python3 is only 3.9 (too old for galaxy-maintenance-scripts' deps, e.g.
+# yacman>=1.0 which needs 3.10+), and CVMFS-provided Pythons aren't reliably
+# present on every worker, so setup_remote_python() bootstraps a pinned
+# standalone CPython with uv. Set REMOTE_PYTHON in the environment to use a
+# specific interpreter instead and skip the uv bootstrap.
+REMOTE_PYTHON="${REMOTE_PYTHON:-}"
+: "${REMOTE_PYTHON_VERSION:=3.13}"
 REMOTE_WORKDIR_PARENT=/srv/idc
 
 # $EPHEMERIS_API_KEY and $IDC_VAULT_PASS should be set in the environment
@@ -154,10 +176,15 @@ function check_bot_command() {
     log 'Checking for Github PR Bot commands'
     log_debug "Value of \$ghprbCommentBody is: ${ghprbCommentBody:-UNSET}"
     case "${ghprbCommentBody:-UNSET}" in
+        "@galaxybot deploy reference-data"*)
+            PUBLISH=true
+            REFERENCE_DATA_ONLY=true
+            ;;
         "@galaxybot deploy"*)
             PUBLISH=true
             ;;
     esac
+    $REFERENCE_DATA_ONLY && log "Reference-data-only deploy: skipping genome build/import"
     if $PUBLISH; then
         log "Publish requested; running build and import"
     else
@@ -225,6 +252,27 @@ function setup_ephemeris() {
 }
 
 
+function setup_remote_python() {
+    # Sets global $REMOTE_PYTHON to a pinned standalone CPython bootstrapped with
+    # uv, so the remote venvs don't depend on the host OS Python (too old) or on
+    # a CVMFS-provided Python being mounted on this particular worker/Stratum 0.
+    # Honor an explicit REMOTE_PYTHON from the environment and skip the bootstrap.
+    if [ -n "$REMOTE_PYTHON" ]; then
+        log "Using preset REMOTE_PYTHON=${REMOTE_PYTHON}"
+        return
+    fi
+    log "Bootstrapping remote Python ${REMOTE_PYTHON_VERSION} with uv"
+    # uv is a single static binary; install it into the (ephemeral) workdir and
+    # let it fetch a managed CPython. UV_INSTALL_DIR sets the binary location;
+    # INSTALLER_NO_MODIFY_PATH keeps it from touching the idc user's profile.
+    exec_on "curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR='${REMOTE_WORKDIR}/uv' INSTALLER_NO_MODIFY_PATH=1 sh"
+    local uv="${REMOTE_WORKDIR}/uv/uv"
+    exec_on "$uv" python install "$REMOTE_PYTHON_VERSION"
+    REMOTE_PYTHON="$(exec_on "$uv" python find "$REMOTE_PYTHON_VERSION")"
+    log "Remote Python: ${REMOTE_PYTHON}"
+}
+
+
 function setup_remote_ephemeris() {
     # Sets global $EPHEMERIS_BIN
     EPHEMERIS_BIN="${REMOTE_WORKDIR}/ephemeris/bin"
@@ -424,6 +472,10 @@ function wait_for_build_galaxy() {
 
 
 function stop_build_galaxy() {
+    # The build Galaxy (and its ansible-venv) is only set up when there are
+    # genome data managers to run; an import-only run (e.g. reference-data-only,
+    # or "Nothing to build") never starts it, so there is nothing to tear down.
+    $BUILD_GALAXY_UP || return 0
     . ./ansible-venv/bin/activate
     log "Stopping Build Galaxy"
     pushd ansible
@@ -595,6 +647,45 @@ function import_tool_data_bundles() {
 }
 
 
+# ---------------------------------------------------------------------------
+# IDC reference-data (workflow-bundle) pipeline - Stage 3 import.
+# NOTE: UNTESTED against live Jenkins/Stratum 0 - review before enabling.
+# These import bundles built on $REFERENCE_DATA_GALAXY_URL by the Stage 2 build
+# (.github/workflows/build.yml) for requests under data-managers/. Only the
+# remote (non-docker) path is wired here; the USE_DOCKER/local-overlay path
+# would need the container invocation like import_tool_data_bundles.
+# ---------------------------------------------------------------------------
+
+function has_reference_data_requests() {
+    compgen -G "data-managers/*/*.yaml" >/dev/null 2>&1 || compgen -G "data-managers/*/*.yml" >/dev/null 2>&1
+}
+
+
+function import_reference_data_bundles() {
+    local req dm version
+    log "Importing IDC reference-data bundles"
+    copy_to scripts/get_bundle_urls.py
+    copy_to scripts/import_bundles.py
+    for req in data-managers/*/*.yaml data-managers/*/*.yml; do
+        [ -e "$req" ] || continue
+        dm="$(basename "$(dirname "$req")")"
+        version="$(basename "$req")"; version="${version%.*}"
+        # import_bundles.py skips gracefully when there is no build history
+        # idc-<dm>-<version> - which is the case when the build stage skipped this
+        # request because its data already exists.
+        log "Importing reference-data bundles for '${dm}/${version}'"
+        exec_on mkdir -p "/cvmfs/${REPO}/data" "/cvmfs/${REPO}/record/${dm}"
+        # import_bundles.py resolves the build's bundles from its workflow
+        # invocation (history idc-<dm>-<version>) and imports each, recording
+        # record/<dm>/<version> for idempotency. API key filtered by Jenkins.
+        exec_on "EPHEMERIS_API_KEY='$REFERENCE_DATA_API_KEY' TMPDIR='${REMOTE_WORKDIR}' ${EPHEMERIS_BIN}/python3 ${REMOTE_WORKDIR}/import_bundles.py \
+            --galaxy-url '$REFERENCE_DATA_GALAXY_URL' --history-name 'idc-${dm}-${version}' \
+            --dm '$dm' --version '$version' --cvmfs-root '/cvmfs/${REPO}' \
+            --import-cmd '${GALAXY_MAINTENANCE_SCRIPTS_BIN}/galaxy-import-data-bundle'"
+    done
+}
+
+
 function show_logs() {
     local lines=
     if [ -n "${1:-}" ]; then
@@ -682,13 +773,21 @@ function do_import_local() {
 function do_import_remote() {
     start_ssh_control
     create_remote_workdir
+    setup_remote_python
     setup_remote_ephemeris
     # from this point forward $EPHEMERIS_BIN refers to remote
-    if generate_import_tasks; then
+    local have_genome_tasks=false have_reference_data=false
+    if ! $REFERENCE_DATA_ONLY && generate_import_tasks; then
+        have_genome_tasks=true
+    fi
+    # UNTESTED: also open a transaction when only reference-data requests exist
+    has_reference_data_requests && have_reference_data=true
+    if $have_genome_tasks || $have_reference_data; then
         setup_galaxy_maintenance_scripts "$WORKDIR" "$REMOTE_PYTHON"
         begin_transaction
         update_tool_data_table_conf
-        import_tool_data_bundles
+        $have_genome_tasks && import_tool_data_bundles
+        $have_reference_data && import_reference_data_bundles
         check_for_repo_changes
         post_install
     else
@@ -706,7 +805,7 @@ function main() {
     detect_changes
     set_repo_vars
     setup_ephemeris
-    if generate_data_manager_tasks; then
+    if ! $REFERENCE_DATA_ONLY && generate_data_manager_tasks; then
         run_build_galaxy
         wait_for_build_galaxy
         #install_data_managers

.github/workflows/build.yml

@@ -0,0 +1,94 @@
+name: Build reference-data bundles
+
+# Stage 2 of the IDC reference-data pipeline: when a request is merged to main,
+# generate its data-manager-bundle workflow and run it on test.galaxyproject.org
+# (via planemo). The build runs asynchronously (--no_wait); the resulting bundle
+# datasets live in a history named idc-<dm>-<version>, which Stage 3 (Jenkins)
+# later imports onto CVMFS.
+#
+# Requires the repository secret TEST_API_KEY (a test.galaxyproject.org API key).
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "data-managers/**"
+  workflow_dispatch:
+    inputs:
+      request:
+        description: "Specific request file to build (default: all requests changed in the push)"
+        required: false
+        default: ""
+
+concurrency:
+  group: idc-build
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2 # need the previous commit to diff changed requests
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: pip install planemo "pydantic>=2" pyyaml gxformat2
+
+      - name: Select requests to build
+        run: |
+          set -euo pipefail
+          if [ -n "${{ github.event.inputs.request }}" ]; then
+            printf '%s\n' "${{ github.event.inputs.request }}" > candidates.txt
+          else
+            base="${{ github.event.before }}"
+            # branch-creation / unknown base reports an all-zero SHA; use previous commit
+            if [ -z "$base" ] || ! git cat-file -e "${base}^{commit}" 2>/dev/null; then
+              base="HEAD~1"
+            fi
+            git diff --name-only --diff-filter=AM "$base" "${{ github.sha }}" \
+              -- 'data-managers/*/*.yml' 'data-managers/*/*.yaml' > candidates.txt || true
+          fi
+          # drop requests whose version is already published (offline ledger) ...
+          python scripts/pending_requests.py --from-file candidates.txt > pending.txt
+          # ... and whose data already exists in a Galaxy data table (authoritative;
+          # e.g. already provided by the byhand CVMFS). --print-new emits only the
+          # requests that still need building.
+          python scripts/check_data_exists.py --from-file pending.txt --print-new \
+            --reference-galaxy https://test.galaxyproject.org > requests.txt
+          echo "Requests to build:"; cat requests.txt || true
+
+      - name: Build bundles on test.galaxyproject.org
+        env:
+          TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+        run: |
+          set -euo pipefail
+          if [ ! -s requests.txt ]; then echo "Nothing to build."; exit 0; fi
+          while read -r req; do
+            [ -n "$req" ] || continue
+            dm=$(basename "$(dirname "$req")")
+            version=$(basename "$req"); version="${version%.*}"
+            echo "::group::Building ${dm}/${version}"
+            # --reference-galaxy: if a chained request's upstream database already
+            # exists on the Galaxy, reference it instead of rebuilding it.
+            python scripts/generate_build.py "$req" --outdir build \
+              --reference-galaxy https://test.galaxyproject.org
+            b="build/${dm}/${version}"
+            planemo run "$b/workflow.gxwf.yml" "$b/job.yml" \
+              --galaxy_url https://test.galaxyproject.org \
+              --galaxy_user_key "$TEST_API_KEY" \
+              --history_name "idc-${dm}-${version}" \
+              --tags idc --no_wait \
+              --output_json "$b/invocation.json"
+            echo "::endgroup::"
+          done < requests.txt
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: idc-build
+          path: build/
+          if-no-files-found: ignore

.github/workflows/lint.yml

@@ -0,0 +1,48 @@
+name: Lint reference-data requests
+
+# Stage 1 of the IDC reference-data pipeline: validate contributed request
+# YAMLs and the build workflows they generate. Pure validation, no secrets, so
+# it is safe to run on pull requests from forks.
+
+on:
+  pull_request:
+    paths:
+      - "data-managers/**"
+      - "published.yml"
+      - "scripts/**"
+      - "tests/**"
+      - ".github/workflows/lint.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "data-managers/**"
+      - "published.yml"
+      - "scripts/**"
+      - "tests/**"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: pip install "pydantic>=2" pyyaml gxformat2 pytest
+      - name: Lint request files
+        run: python scripts/request_models.py
+      - name: Validate generated build workflows
+        # Generate a gxformat2 bundle workflow for every request and validate it
+        # with gxformat2 (strict schema, format2->native, semantic lint). Catches
+        # wiring errors - e.g. a chained request with no CHAIN_WIRING entry.
+        run: python scripts/generate_build.py --all --outdir "${RUNNER_TEMP}/idc-build-check"
+      - name: Run pipeline unit tests
+        run: python -m pytest tests/ -q
+      - name: Check whether requested data already exists
+        # Informational (non-blocking): query the tool data tables on
+        # test.galaxyproject.org (public, no key) and warn if a request's data is
+        # already available there from any source (e.g. the byhand CVMFS). The
+        # build stage skips these; here we just surface it to reviewers.
+        run: python scripts/check_data_exists.py --all --warn

.gitignore

@@ -106,3 +106,10 @@ working/
 # Ansible installed artifacts
 ansible/collections
 ansible/roles/*.*
+
+# macOS
+.DS_Store
+
+# ephemeris/shed-tools test output
+tool_test_output.json
+tool_test_output.html