molecule.yml

---
name: Molecule Test
on:
  workflow_call:
    inputs:
      scenarios:
        required: true
        type: string
        description: JSON array of scenario names
      distros:
        required: true
        type: string
        description: JSON array of distro identifiers
      releases:
        type: string
        default: '["8","9"]'
        description: JSON array of Elastic major versions
      timeout:
        type: number
        default: 60
      max-parallel:
        type: number
        default: 10
      skip-idempotence:
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  molecule:
    runs-on: self-hosted
    timeout-minutes: ${{ inputs.timeout }}

    env:
      COLLECTION_NAMESPACE: oddly
      COLLECTION_NAME: elasticstack
      # Ansible/molecule are Python; without PYTHONUNBUFFERED the
      # "FAILED - RETRYING (N retries left)" messages that Ansible emits
      # from the wait-for-capacity retry loop sit in Python's stdout
      # buffer and only flush when the buffer fills, making the log look
      # like a silent hang for tens of minutes.
      PYTHONUNBUFFERED: '1'
      ANSIBLE_PIPELINING: 'true'
      ANSIBLE_GATHERING: smart
      ANSIBLE_ANY_ERRORS_FATAL: 'true'
      ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false'
      ANSIBLE_TRANSFER_METHOD: sftp
      ANSIBLE_SSH_RETRIES: '5'
      ANSIBLE_HOST_KEY_CHECKING: 'false'
      ANSIBLE_INJECT_FACTS_AS_VARS: 'false'
      ANSIBLE_TIMEOUT: '60'
      ANSIBLE_FORKS: '10'
      ANSIBLE_SSH_ARGS: >-
        -o ControlMaster=auto
        -o ControlPersist=60s
        -o PreferredAuthentications=publickey

    strategy:
      fail-fast: false
      max-parallel: ${{ inputs.max-parallel }}
      matrix:
        distro: ${{ fromJSON(inputs.distros) }}
        scenario: ${{ fromJSON(inputs.scenarios) }}
        release: ${{ fromJSON(inputs.releases) }}

    steps:
      - name: Check out code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install dependencies
        run: |
          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
          source "$RUNNER_TEMP/venv/bin/activate"
          uv pip install -r requirements-test.txt
          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
        env:
          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt

      - name: Set up collections path
        run: |
          echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV"
          echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV"
          echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV"
          echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV"
          mkdir -p "$RUNNER_TEMP/ssh-cp"

      - name: Install collection
        env:
          CACHE_HOST: ${{ secrets.INCUS_HOST }}
        run: |
          mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE
          cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
          if [ -n "$CACHE_HOST" ]; then
            ansible-galaxy collection install \
              "http://${CACHE_HOST}:8082/collections/community-general-12.3.0.tar.gz" \
              "http://${CACHE_HOST}:8082/collections/community-crypto-3.1.1.tar.gz" \
              "http://${CACHE_HOST}:8082/collections/ansible-posix-2.1.0.tar.gz"
          else
            ansible-galaxy collection install \
              community.general:12.3.0 \
              community.crypto:3.1.1 \
              ansible.posix:2.1.0
          fi

      - name: Set up SSH key for molecule
        run: |
          echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519
          chmod 600 ${{ runner.temp }}/molecule_id_ed25519
          mkdir -p ~/.ssh
          ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true

      - name: Compute gate deadline
        # Keep the gate deadline 15 minutes under this job's
        # timeout-minutes so a starved job fails at the gate with a
        # queue/ledger verdict instead of at the workflow cancel with
        # nothing. Floor of 300s.
        id: gate
        run: |
          deadline=$(( (${{ inputs.timeout }} - 15) * 60 ))
          [ "$deadline" -ge 300 ] || deadline=300
          echo "deadline=$deadline" >> "$GITHUB_OUTPUT"

      - name: Acquire memory slot
        # Single-authority admission gate: MemTotal − reserve − committed
        # limits.memory − pending reservations, FIFO with bounded
        # overtakes. create.yml converts the reservation into committed
        # capacity right after the containers launch.
        uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7  # v1.0.1
        with:
          mode: acquire
          molecule-scenario: ${{ matrix.scenario }}
          deadline-seconds: ${{ steps.gate.outputs.deadline }}
          incus-host: ${{ secrets.INCUS_HOST }}
          ssh-key: ${{ runner.temp }}/molecule_id_ed25519

      - name: Converge
        run: molecule converge -s ${{ matrix.scenario }}
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
          PY_COLORS: '1'
          ANSIBLE_FORCE_COLOR: '1'
          ANSIBLE_DIFF_ALWAYS: 'true'
          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Verify
        run: molecule verify -s ${{ matrix.scenario }}
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Idempotence check
        if: ${{ !inputs.skip-idempotence }}
        run: molecule idempotence -s ${{ matrix.scenario }}
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
          PY_COLORS: '1'
          ANSIBLE_FORCE_COLOR: '1'
          ANSIBLE_DIFF_ALWAYS: 'true'
          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Release memory slot
        # Runs whether converge/verify/idempotence passed or failed, so
        # the reservation is freed promptly. Normally a no-op: create.yml
        # already converted the reservation at container launch.
        if: always()
        uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7  # v1.0.1
        with:
          mode: release

      - name: Collect and upload diagnostics
        if: failure()
        uses: ./.github/actions/collect-diagnostics
        with:
          artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }}
        env:
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519

      - name: Cleanup molecule
        if: always()
        run: molecule destroy -s ${{ matrix.scenario }}
        continue-on-error: true
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Clean up SSH state
        if: always()
        run: |
          # Kill orphaned SSH processes from this job
          pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true
          # Remove ControlMaster sockets and SSH key
          rm -rf "${{ runner.temp }}/ssh-cp"
          rm -f "${{ runner.temp }}/molecule_id_ed25519"