test_full_stack.yml

---
name: Test ElasticStack
on:
  workflow_dispatch:
    inputs:
      logLevel:
        description: 'Log level'
        required: true
        default: 'warning'
        type: choice
        options:
          - info
          - warning
          - debug
      distros:
        description: 'JSON array of distros (empty = all)'
        required: false
        type: string
  pull_request:
    types: [labeled]
  merge_group:
  schedule:
    - cron: "0 4 * * 2,4,6"  # Tue/Thu/Sat — 48 jobs, alternate with ES/logstash

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && github.event.label.name != 'ci:run' && github.run_id || 'ci' }}
  cancel-in-progress: true

jobs:
  changes:
    if: ${{ github.event_name != 'pull_request' || github.event.label.name == 'ci:run' }}
    runs-on: ubuntu-latest
    outputs:
      should_test: ${{ steps.filter.outputs.should_test }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
        id: filter
        with:
          filters: |
            should_test:
              - 'roles/elasticstack/**'
              - 'molecule/elasticstack_*/**'
              - 'molecule/es_kibana/**'
              - 'molecule/logstash_elasticsearch/**'
              - 'molecule/cert_renewal/**'
              - 'molecule/shared/**'
              - 'plugins/**'
              - '.github/workflows/test_full_stack.yml'

  lint_full:
    needs: changes
    if: needs.changes.outputs.should_test == 'true' || github.event_name != 'pull_request'
    uses: ./.github/workflows/test_linting.yml
    with:
      rolename: ''
    secrets: inherit

  molecule_full_stack_every_os:
    runs-on: self-hosted
    needs: [changes, lint_full]
    if: needs.changes.outputs.should_test == 'true' || github.event_name != 'pull_request'
    # 180 min: elasticstack_default is the heaviest scenario and observed
    # runs range 70-105 min in isolation, but the extra queue wait added
    # by the memory-capacity gate on a busy runner can push a single job
    # past the previous 120-min cap.  The memory-gate retry cap in
    # shared/create.yml is now 15 min so this ceiling is only reached
    # when the scenario genuinely runs long, not when the queue starves it.
    timeout-minutes: 180

    env:
      COLLECTION_NAMESPACE: oddly
      COLLECTION_NAME: elasticstack
      ANSIBLE_PIPELINING: 'true'
      ANSIBLE_GATHERING: smart
      ANSIBLE_ANY_ERRORS_FATAL: 'true'
      ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false'
      ANSIBLE_TRANSFER_METHOD: sftp
      ANSIBLE_SSH_RETRIES: '5'
      ANSIBLE_HOST_KEY_CHECKING: 'false'
      ANSIBLE_INJECT_FACTS_AS_VARS: 'false'
      ANSIBLE_TIMEOUT: '60'
      ANSIBLE_FORKS: '10'
      ANSIBLE_SSH_ARGS: >-
        -o ControlMaster=auto
        -o ControlPersist=60s
        -o PreferredAuthentications=publickey

    strategy:
      fail-fast: false
      # Each scenario/distro/release combo needs 15-25 GB of container RAM
      # on the shared incus-ci host. The admission gate (FIFO with bounded
      # overtakes, fail-fast) now guarantees heavy scenarios can't be
      # starved by smaller jobs, so the cap is back to the throughput
      # target it was originally sized for. The 3-slot value was a
      # workaround for gate starvation that no longer exists; if the
      # nightly storm shows starved jobs or OOMs, revert this commit
      # first and investigate the gate second.
      max-parallel: 6
      matrix:
        # Standardise on rockylinux10 (not 9) for PR runs to match the rest
        # of the workflows. Rocky 10 exercises the EL≥9 branch in
        # roles/repos/tasks/redhat.yml that Rocky 9 already covers, so no
        # coverage is lost.
        distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || (inputs.distros != '' && fromJSON(inputs.distros)) || fromJSON('["rockylinux9","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]') }}
        scenario:
          - elasticstack_default
          - es_kibana
          - logstash_elasticsearch
          - cert_renewal
        # PR/merge_group: 9 only — the full-stack matrix is 100+ min per job
        # and the biggest source of CI queue pressure. Cron and workflow_dispatch
        # keep both releases. ES 8.19 EOL 2027-07-15.
        release: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('[9]') || fromJSON('[8,9]') }}
        # Rocky 10 excluded: EPEL 10 does not ship Redis yet.
        # Rocky 10 is still tested via the ES modules workflow.

    steps:
      - name: Check out code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install dependencies
        run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt
        env:
          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt

      - name: Set up collections path
        run: |
          echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV"
          echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV"
          echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV"
          echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV"
          mkdir -p "$RUNNER_TEMP/ssh-cp"

      - name: Install collection
        env:
          CACHE_HOST: ${{ secrets.INCUS_HOST }}
        run: |
          mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE
          cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
          if [ -n "$CACHE_HOST" ]; then
            ansible-galaxy collection install \
              "http://${CACHE_HOST}:8082/collections/community-general-12.3.0.tar.gz" \
              "http://${CACHE_HOST}:8082/collections/community-crypto-3.1.1.tar.gz" \
              "http://${CACHE_HOST}:8082/collections/ansible-posix-2.1.0.tar.gz"
          else
            ansible-galaxy collection install \
              community.general:12.3.0 \
              community.crypto:3.1.1 \
              ansible.posix:2.1.0
          fi

      - name: Set up SSH key for molecule
        run: |
          echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519
          chmod 600 ${{ runner.temp }}/molecule_id_ed25519
          mkdir -p ~/.ssh
          ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true

      - name: Acquire memory slot
        # Single-authority admission gate: MemTotal − reserve − committed
        # limits.memory − pending reservations, FIFO with bounded
        # overtakes. Fails after its 45-min deadline instead of barging
        # in — a starved job is an explicit retryable failure, not an
        # OOM risk. create.yml converts the reservation into committed
        # capacity right after the containers launch.
        uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7  # v1.0.1
        with:
          mode: acquire
          molecule-scenario: ${{ matrix.scenario }}
          incus-host: ${{ secrets.INCUS_HOST }}
          ssh-key: ${{ runner.temp }}/molecule_id_ed25519

      - name: Converge
        run: molecule converge -s ${{ matrix.scenario }}
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
          PY_COLORS: '1'
          ANSIBLE_FORCE_COLOR: '1'
          ANSIBLE_DIFF_ALWAYS: 'true'
          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Verify
        run: molecule verify -s ${{ matrix.scenario }}
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Idempotence check
        if: matrix.scenario != 'cert_renewal'
        run: molecule idempotence -s ${{ matrix.scenario }}
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
          PY_COLORS: '1'
          ANSIBLE_FORCE_COLOR: '1'
          ANSIBLE_DIFF_ALWAYS: 'true'
          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Release memory slot
        # Runs whether converge/verify/idempotence passed or failed, so
        # the reservation is freed promptly. Normally a no-op: create.yml
        # already converted the reservation at container launch.
        if: always()
        uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7  # v1.0.1
        with:
          mode: release

      - name: Collect and upload diagnostics
        if: failure()
        uses: ./.github/actions/collect-diagnostics
        with:
          artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }}
        env:
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519

      - name: Cleanup molecule
        if: always()
        run: molecule destroy -s ${{ matrix.scenario }}
        continue-on-error: true
        env:
          MOLECULE_DISTRO: ${{ matrix.distro }}
          ELASTIC_RELEASE: ${{ matrix.release }}
          INCUS_HOST: ${{ secrets.INCUS_HOST }}
          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081

      - name: Clean up SSH state
        if: always()
        run: |
          # Kill orphaned SSH processes from this job
          pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true
          # Remove ControlMaster sockets and SSH key
          rm -rf "${{ runner.temp }}/ssh-cp"
          rm -f "${{ runner.temp }}/molecule_id_ed25519"

  full_stack_gate:
    runs-on: ubuntu-latest
    needs: [changes, lint_full, molecule_full_stack_every_os]
    if: ${{ always() && (github.event_name != 'pull_request' || github.event.label.name == 'ci:run') }}
    steps:
      - name: Check results
        run: |
          # If no relevant files changed, tests were skipped — that's OK
          if [[ "${{ needs.changes.outputs.should_test }}" != "true" && "${{ github.event_name }}" == "pull_request" ]]; then
            echo "No relevant changes — skipping full stack gate"
            exit 0
          fi
          if [[ "${{ needs.lint_full.result }}" != "success" ]] || \
             [[ "${{ needs.molecule_full_stack_every_os.result }}" != "success" ]]; then
            echo "Required jobs failed"
            exit 1
          fi