shared-test.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: Shared Local GPU and CPU Test

on:
  workflow_call:
    inputs:
      test_suite_name:
        description: 'Test suite name (vllm, sglang, trtllm)'
        required: true
        type: string
      framework:
        description: 'Backend the test image was built with (dynamo, sglang, triton, trtllm, vllm); labels test metadata and coverage/Allure artifacts'
        required: true
        type: string
      test_type:
        description: 'Test type (e.g. single-gpu, multi-gpu)'
        required: true
        type: string
      amd_runner:
        description: 'Runner to execute tests on (amd64 only)'
        required: true
        type: string
      target_tag_plain:
        description: 'Plain runtime image tag prefix from the build workflow'
        required: true
        type: string
      cuda_version:
        description: 'CUDA versions to test as a JSON array'
        required: true
        type: string
      platform:
        description: 'Target platforms to test as a JSON array'
        required: true
        type: string
      container_workspace:
        description: 'Workspace path inside the container where pytest should run'
        required: false
        type: string
        default: '/workspace'
      kvbm_mla_backend:
        description: 'KVBM MLA attention backend exported for KVBM tests'
        required: false
        type: string
        default: ''
      enable_coverage:
        description: 'Enable coverage collection (true/false)'
        required: false
        type: string
        default: 'false'
      run_sanity_check:
        description: 'Whether to run sanity check on the runtime image before executing tests'
        required: false
        type: boolean
        default: true
      run_cpu_only_tests:
        description: 'Whether to run CPU-only tests'
        required: false
        type: boolean
        default: false
      cpu_only_test_markers:
        description: 'CPU-only pytest markers'
        required: false
        type: string
      cpu_only_test_timeout_minutes:
        description: 'Timeout in minutes for CPU tests'
        required: false
        type: number
        default: 10
      cpu_parallel_mode:
        description: 'pytest-xdist mode for CPU-only tests: auto, none/0, or a worker count'
        required: false
        type: string
        default: 'auto'
      run_cpu_e2e_tests:
        description: 'Whether to run CPU end-to-end tests in a separate, non-retrying stage'
        required: false
        type: boolean
        default: false
      cpu_e2e_test_markers:
        description: 'CPU end-to-end pytest markers'
        required: false
        type: string
      cpu_e2e_test_timeout_minutes:
        description: 'Timeout in minutes for the isolated CPU end-to-end test stage'
        required: false
        type: number
        default: 10
      run_gpu_tests:
        description: 'Whether to run GPU tests'
        required: false
        type: boolean
        default: true
      gpu_test_markers:
        description: 'GPU pytest markers. Selects the full GPU pool. When run_gpu_parallel_tests=true, profiled tests run in the parallel stage and are excluded from the sequential stage by appending "and not profiled_vram_gib".'
        required: false
        type: string
      gpu_test_timeout_minutes:
        description: 'Timeout in minutes for each GPU test stage (parallel and sequential each get this budget)'
        required: false
        type: number
        default: 30
      run_gpu_parallel_tests:
        description: 'Whether to run the GPU-parallel stage (VRAM-aware concurrent subprocesses) before the sequential stage. Tests carrying @pytest.mark.profiled_vram_gib(N) within gpu_test_markers run here; the sequential stage runs the rest.'
        required: false
        type: boolean
        default: false
      gpu_parallel_max_vram_gib:
        description: 'Per-test VRAM budget in GiB for the parallel stage. Caps which profiled tests are admitted and feeds the orchestrator scheduler (tests/utils/pytest_parallel_gpu.py).'
        required: false
        type: string
        default: ''
      verify_gpu:
        description: 'Check the GPU through torch before the GPU stages. Set false for runtime images that ship without torch, leaving the suite to check the GPU it needs.'
        required: false
        type: boolean
        default: true
      source_ref:
        description: 'Git ref/SHA for image tags and checkout. Sidecar jobs always check out the caller SHA. Empty = caller SHA (github.sha).'
        required: false
        type: string
        default: ''
      image_tag_suffix:
        description: 'Suffix appended to the SHA-keyed image tag (e.g. "-nightly"). Empty = no suffix.'
        required: false
        type: string
        default: ''
      dd_env:
        description: 'Datadog env tag for test sessions (DD_ENV). Datadog Test Optimization settings (auto test retries, PR comments, etc.) are scoped per (service, env), so use a distinct value (e.g. "nightly", "post-merge") to configure those workflows independently of PR/pre-merge.'
        required: false
        type: string
        default: 'ci'
      dd_flaky_retry_enabled:
        description: 'Datadog Auto Test Retries (DD_CIVISIBILITY_FLAKY_RETRY_ENABLED). Set "false" to disable client-side flaky-test reruns regardless of the Datadog UI setting.'
        required: false
        type: string
        default: 'true'
      sidecar_backend:
        description: 'Backend name (vllm/sglang/trtllm) to install a dynamo-*-sidecar binary for before running tests. Empty = skip (non-sidecar callers).'
        required: false
        type: string
        default: ''
      sidecar_binary_artifact:
        description: 'Name of the workflow artifact (from shared-build-sidecar.yml) containing the amd64 dynamo-*-sidecar binaries. Required when sidecar_backend is set.'
        required: false
        type: string
        default: ''
    secrets:
      AWS_DEFAULT_REGION:
        required: true
      AWS_ACCOUNT_ID:
        required: true
      AZURE_ACR_HOSTNAME:
        required: true
      AZURE_ACR_USER:
        required: true
      AZURE_ACR_PASSWORD:
        required: true
      HF_TOKEN:
        required: false
      DD_API_KEY:
        required: true
      TRITON_ARTIFACTORY_USER:
        required: false # TODO: Make this required once credentials are available to CI.
      TRITON_ARTIFACTORY_TOKEN:
        required: false # TODO: Make this required once credentials are available to CI.

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        platform: ${{ fromJson(inputs.platform) }}
        cuda_version: ${{ fromJson(inputs.cuda_version) }}
    name: ${{ inputs.test_type }} ${{ matrix.cuda_version == '' && 'cpu' || format('cuda{0}', matrix.cuda_version) }}, ${{ matrix.platform }}
    runs-on: ${{ matrix.platform == 'amd64' && inputs.amd_runner || 'prod-tester-arm-v2' }}
    container:
      image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ inputs.source_ref != '' && inputs.source_ref || github.sha }}-${{ inputs.target_tag_plain }}${{ (matrix.cuda_version != '' && !startsWith(matrix.cuda_version, '13.')) && format('-cuda{0}', startsWith(matrix.cuda_version, '12.') && '12' || matrix.cuda_version) || '' }}${{ inputs.image_tag_suffix }}-test
      env:
        # The image creates user 'dynamo' at uid 1000, but the ARC k8s runner
        # executes the job pod as uid 1001 (runner's uid). Overriding HOME and
        # USER here sidesteps two symptoms of that mismatch:
        #   * actions/checkout probing $HOME/.gitconfig (owned by 1000, unreadable by 1001)
        #   * torch._inductor calling getpass.getuser() -> pwd.getpwuid(1001) -> KeyError
        HOME: /__w/dynamo/dynamo
        USER: dynamo
        # Datadog Test Optimization scoping (settings are per service+env).
        DD_SERVICE: ai-dynamo
        DD_ENV: ${{ inputs.dd_env }}
        DD_TAGS: 'team:dynamo'
        DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: ${{ inputs.dd_flaky_retry_enabled }}
        DD_CIVISIBILITY_FLAKY_RETRY_COUNT: '2'
        # dynamo tests run product code in subprocesses, so ddtrace's in-process
        # coverage.py collects nothing ("No data was collected" warnings) and TIA
        # gets no signal. Disable ITR (skips the settings API, coverage, skipping).
        DD_CIVISIBILITY_ITR_ENABLED: 'false'
        # Datadog Test Optimization is configured through ddtrace environment
        # variables (ddtrace is pinned in requirements.test.txt) to avoid extra
        # container-job hook steps.
        DD_API_KEY: ${{ secrets.DD_API_KEY }}
        DD_SITE: datadoghq.com
        DD_CIVISIBILITY_ENABLED: 'true'
        DD_CIVISIBILITY_AGENTLESS_ENABLED: 'true'
        DD_CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER: github
        PYTEST_ADDOPTS: >
          --ddtrace
          ${{ inputs.framework == 'triton' && '--ignore=tests/fault_tolerance/test_canary_rank_pause.py' || '' }}
          ${{ inputs.framework == 'triton' && '--ignore=tests/serve/test_vllm.py' || '' }}
          ${{ inputs.framework == 'triton' && '--ignore=tests/serve/test_sglang.py' || '' }}
          ${{ inputs.framework == 'triton' && '--ignore=tests/serve/test_trtllm.py' || '' }}
    permissions:
      contents: read
      actions: read
    services:
      minio:
        image: ${{ vars.ECR_REGISTRY }}/quay/minio/minio:latest
        env:
          MINIO_ROOT_USER: minioadmin
          MINIO_ROOT_PASSWORD: minioadmin
          MINIO_VOLUMES: /tmp/minio-data
          MINIO_CONSOLE_ADDRESS: ':9001'
    steps:
      - name: Checkout repository
        id: checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ inputs.sidecar_backend != '' && github.sha || inputs.source_ref || github.sha }}
      - name: Download sidecar binary
        if: ${{ inputs.sidecar_backend != '' }}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ inputs.sidecar_binary_artifact }}
          path: sidecar-artifact
      - name: Install sidecar binary
        if: ${{ inputs.sidecar_backend != '' }}
        shell: bash
        env:
          SIDECAR_BACKEND: ${{ inputs.sidecar_backend }}
          CONTAINER_WORKSPACE: ${{ inputs.container_workspace }}
        run: |
          set -euo pipefail
          # Use a workspace-local directory writable by the container user.
          bin_dir="${GITHUB_WORKSPACE}/.sidecar-bin"
          mkdir -p "${bin_dir}"
          install -m755 "sidecar-artifact/dynamo-${SIDECAR_BACKEND}-sidecar" \
            "${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar"
          echo "${bin_dir}" >> "${GITHUB_PATH}"
          if [ -z "${CONTAINER_WORKSPACE}" ]; then
            echo "::error::container_workspace is empty; refusing to replace test files"
            exit 1
          fi
          if [ ! "${GITHUB_WORKSPACE}" -ef "${CONTAINER_WORKSPACE}" ]; then
            # Replace the test inputs so files deleted by the PR cannot linger.
            for path in lib/sidecar tests examples/common conftest.py pyproject.toml Cargo.toml; do
              rm -rf "${CONTAINER_WORKSPACE}/${path}"
              mkdir -p "$(dirname "${CONTAINER_WORKSPACE}/${path}")"
              cp -r "${GITHUB_WORKSPACE}/${path}" "${CONTAINER_WORKSPACE}/${path}"
            done
          fi
      - name: Export KVBM MLA backend
        if: ${{ inputs.kvbm_mla_backend != '' }}
        shell: bash
        env:
          KVBM_MLA_BACKEND: ${{ inputs.kvbm_mla_backend }}
        run: |
          echo "KVBM_MLA_BACKEND=${KVBM_MLA_BACKEND}" >> "$GITHUB_ENV"
      - name: Run Sanity Check on Runtime Image
        if: ${{ inputs.run_sanity_check }}
        shell: bash
        # Run from the container's baked-in workspace, not the GHA checkout,
        # so the sanity check reflects what the image actually ships.
        working-directory: /workspace
        run: |
          set +e
          python ./dev/sanity_check.py --runtime-check --no-gpu-check
          SANITY_CHECK_EXIT_CODE=$?
          if [ ${SANITY_CHECK_EXIT_CODE} -ne 0 ]; then
            echo "ERROR: Sanity check failed - ai-dynamo packages not properly installed"
            exit ${SANITY_CHECK_EXIT_CODE}
          else
            echo "✅ Sanity check passed"
          fi

      - name: Run CPU-only tests (parallelized)
        id: cpu-only-tests
        if: ${{ inputs.run_cpu_only_tests }}
        timeout-minutes: ${{ inputs.cpu_only_test_timeout_minutes }}
        uses: $/.github/actions/pytest-local
        with:
          container_workspace: ${{ inputs.container_workspace }}
          pytest_marks: ${{ inputs.cpu_only_test_markers }}
          test_suite_name: ${{ inputs.test_suite_name }}
          framework: ${{ inputs.framework }}
          test_type: "pre_merge_cpu"
          platform_arch: ${{ matrix.platform }}
          hf_token: ${{ secrets.HF_TOKEN }}
          parallel_mode: ${{ inputs.cpu_parallel_mode }}
          enable_coverage: ${{ inputs.enable_coverage }}
          gpu_tests: 'false'
      - name: Run CPU end-to-end tests (isolated)
        id: cpu-e2e-tests
        # Long-lived deployment tests should not compete with the xdist unit
        # pool or retry until the parent step kills the runner container.
        if: ${{ !cancelled() && steps.checkout.outcome == 'success' && inputs.run_cpu_e2e_tests }}
        timeout-minutes: ${{ inputs.cpu_e2e_test_timeout_minutes }}
        uses: $/.github/actions/pytest-local
        with:
          container_workspace: ${{ inputs.container_workspace }}
          pytest_marks: ${{ inputs.cpu_e2e_test_markers }}
          test_suite_name: ${{ inputs.test_suite_name }}
          framework: ${{ inputs.framework }}
          test_type: "pre_merge_cpu_e2e"
          platform_arch: ${{ matrix.platform }}
          hf_token: ${{ secrets.HF_TOKEN }}
          parallel_mode: 'none'
          enable_coverage: ${{ inputs.enable_coverage }}
          gpu_tests: 'false'
          dd_flaky_retry_enabled: 'false'
      - name: Run GPU tests (parallel)
        id: gpu-parallel-tests
        timeout-minutes: ${{ inputs.gpu_test_timeout_minutes }}
        if: ${{ matrix.platform == 'amd64' && inputs.run_gpu_tests && inputs.run_gpu_parallel_tests }}
        uses: $/.github/actions/pytest-local
        with:
          container_workspace: ${{ inputs.container_workspace }}
          pytest_marks: ${{ inputs.gpu_test_markers }}
          test_suite_name: ${{ inputs.test_suite_name }}
          framework: ${{ inputs.framework }}
          test_type: "pre_merge_gpu_parallel"
          platform_arch: ${{ matrix.platform }}
          hf_token: ${{ secrets.HF_TOKEN }}
          # parallel_mode=auto + max_vram_gib hands off to the VRAM-aware
          # orchestrator (tests/utils/pytest_parallel_gpu.py via tests/conftest.py)
          parallel_mode: 'auto'
          max_vram_gib: ${{ inputs.gpu_parallel_max_vram_gib }}
          enable_coverage: ${{ inputs.enable_coverage }}
          gpu_tests: 'true'
          verify_gpu: ${{ inputs.verify_gpu }}
      - name: Run GPU tests (sequential)
        id: gpu-sequential-tests
        timeout-minutes: ${{ inputs.gpu_test_timeout_minutes }}
        # !cancelled() so a parallel-stage failure doesn't hide sequential results,
        # but require checkout to have succeeded so an infra/prereq failure (e.g. the
        # container/pod never coming online) skips this step cleanly instead of trying
        # to run the local pytest-local action against an un-checked-out workspace.
        if: ${{ !cancelled() && steps.checkout.outcome == 'success' && matrix.platform == 'amd64' && inputs.run_gpu_tests }}
        uses: $/.github/actions/pytest-local
        with:
          container_workspace: ${{ inputs.container_workspace }}
          # When the parallel stage runs, exclude profiled tests here so they don't run twice.
          pytest_marks: ${{ inputs.run_gpu_parallel_tests && format('({0}) and not profiled_vram_gib', inputs.gpu_test_markers) || inputs.gpu_test_markers }}
          test_suite_name: ${{ inputs.test_suite_name }}
          framework: ${{ inputs.framework }}
          test_type: "pre_merge_gpu"
          platform_arch: ${{ matrix.platform }}
          hf_token: ${{ secrets.HF_TOKEN }}
          triton_artifactory_user: ${{ secrets.TRITON_ARTIFACTORY_USER }}
          triton_artifactory_token: ${{ secrets.TRITON_ARTIFACTORY_TOKEN }}
          parallel_mode: 'none'
          enable_coverage: ${{ inputs.enable_coverage }}
          gpu_tests: 'true'
          verify_gpu: ${{ inputs.verify_gpu }}

      # One JUnit upload per job (one XML per stage inside). The name must keep
      # the test-results-* prefix consumed by the report workflows. Allure and
      # coverage are uploaded per stage by the pytest-local action.
      - name: Upload Test Results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        if: ${{ !cancelled() && steps.checkout.outcome == 'success' }}
        with:
          name: test-results-${{ inputs.test_suite_name }}-${{ matrix.platform }}-${{ github.run_id }}-${{ job.check_run_id }}
          path: test-results/pytest_test_report_*.xml
          retention-days: 7
          if-no-files-found: warn

      # A stage cut off mid-run is still reported as a successful step, so real
      # test failures go green. Take the verdict only from the status file
      # pytest-local writes last, never from the step outcome. Placed after the
      # upload so results survive a failure here, and once per job rather than
      # per stage to avoid three more container-hook round trips.
      - name: Verify test stages completed
        if: ${{ !cancelled() && steps.checkout.outcome == 'success' }}
        shell: bash
        env:
          STAGE_OUTCOMES: |
            pre_merge_cpu ${{ steps.cpu-only-tests.outcome }}
            pre_merge_cpu_e2e ${{ steps.cpu-e2e-tests.outcome }}
            pre_merge_gpu_parallel ${{ steps.gpu-parallel-tests.outcome }}
            pre_merge_gpu ${{ steps.gpu-sequential-tests.outcome }}
        run: |
          # Runs inside the job's test container. The planner image is
          # distroless and ships only the binaries listed in
          # container/Dockerfile.test, so this stays on bash builtins.
          set -uo pipefail
          status_dir="${GITHUB_WORKSPACE}/test-results/status"
          failed=0
          checked=""

          while read -r stage outcome; do
            [[ -z "${stage}" ]] && continue
            exitcode_file="${status_dir}/${stage}.exitcode"

            case "${outcome}" in
              ''|skipped)
                # An unresolvable id renders empty, which would otherwise read
                # as "skipped" and exempt a stage that really ran.
                if [[ -f "${exitcode_file}" ]]; then
                  echo "❌ ${stage}: wrote a status file but its outcome is"
                  echo "   '${outcome:-<empty>}'. Its id in STAGE_OUTCOMES does not"
                  echo "   resolve, so the stage ran unverified."
                  failed=1
                else
                  echo "⏭️  ${stage}: did not run"
                fi
                ;;
              success)
                if [[ ! -f "${exitcode_file}" ]]; then
                  echo "❌ ${stage}: reported success but wrote no status file."
                  echo "   Its script did not run to completion, so the result is"
                  echo "   unknown and cannot be treated as a pass."
                  failed=1
                else
                  # An empty file means the stage created it on entry and never
                  # reached the line that records an exit code. read is a
                  # builtin and returns non-zero on an empty file.
                  code=""
                  read -r code < "${exitcode_file}" || true
                  if [[ -z "${code}" ]]; then
                    echo "❌ ${stage}: reported success but recorded no exit code."
                    echo "   Its script was cut off mid-run, so the result is"
                    echo "   unknown and cannot be treated as a pass."
                    failed=1
                  elif [[ "${code}" != "0" ]]; then
                    echo "❌ ${stage}: pytest exited ${code}"
                    failed=1
                  else
                    echo "✅ ${stage}: pytest exited 0"
                  fi
                fi
                ;;
              *)
                echo "❌ ${stage}: step outcome '${outcome}'"
                failed=1
                ;;
            esac
            checked+=" ${stage}"
          done <<< "${STAGE_OUTCOMES}"

          # A stage added above but not listed here would escape this check.
          shopt -s nullglob
          for f in "${status_dir}"/*.exitcode; do
            stage="${f##*/}"
            stage="${stage%.exitcode}"
            if [[ " ${checked} " != *" ${stage} "* ]]; then
              echo "❌ ${stage}: ran but is not listed in STAGE_OUTCOMES, so its"
              echo "   result is unverified. Add it to this step."
              failed=1
            fi
          done

          # Without this the checks tab shows only "Process completed with exit
          # code 1" and the stage name stays buried in this step's log.
          if [[ "${failed}" -ne 0 ]]; then
            echo "::error title=Test stage verification failed::A pytest stage did not record a passing result. See this step's log for the stage."
          fi

          exit "${failed}"