Adds automatic test selection depending on the changes in the PRs

.github/workflows/build.yaml

@@ -87,13 +87,66 @@ jobs:
       if: always()
       run: docker image prune -f --filter "until=24h" || true
 
+  select-tests:
+    name: Select Tests
+    runs-on: ubuntu-latest
+    outputs:
+      run-all: ${{ steps.select.outputs.run-all }}
+      test-list-physx: ${{ steps.select.outputs.test-list-physx }}
+      test-list-newton: ${{ steps.select.outputs.test-list-newton }}
+      test-list-general: ${{ steps.select.outputs.test-list-general }}
+      test-list-isaaclab-tasks: ${{ steps.select.outputs.test-list-isaaclab-tasks }}
+      test-list-isaaclab-tasks-2: ${{ steps.select.outputs.test-list-isaaclab-tasks-2 }}
+      test-list-environments-training: ${{ steps.select.outputs.test-list-environments-training }}
+      test-list-flaky: ${{ steps.select.outputs.test-list-flaky }}
+      test-list-slightly-flaky: ${{ steps.select.outputs.test-list-slightly-flaky }}
+      test-list-curobo: ${{ steps.select.outputs.test-list-curobo }}
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Fetch coverage mapping
+      run: |
+        git fetch origin ci/coverage-map 2>/dev/null || true
+        git checkout origin/ci/coverage-map -- tools/test-dependency-map.json 2>/dev/null || echo '{}' > tools/test-dependency-map.json
+
+    - name: Select tests
+      id: select
+      run: |
+        set -euo pipefail
+        base_ref="${{ github.event.pull_request.base.ref }}"
+        if [ -z "$base_ref" ]; then
+          echo "No base branch (not a PR event). Running all tests."
+          echo "run-all=true" >> $GITHUB_OUTPUT
+          for job in physx newton general isaaclab-tasks isaaclab-tasks-2 environments-training flaky slightly-flaky curobo; do
+            echo "test-list-$job=" >> $GITHUB_OUTPUT
+          done
+          exit 0
+        fi
+        output=$(python tools/select_tests.py --base-branch "origin/$base_ref" --dry-run 2>> $GITHUB_STEP_SUMMARY)
+
+        run_all=$(echo "$output" | python -c "import sys,json; print(str(json.load(sys.stdin)['run_all']).lower())")
+        echo "run-all=$run_all" >> $GITHUB_OUTPUT
+
+        for job in physx newton general isaaclab-tasks isaaclab-tasks-2 environments-training flaky slightly-flaky curobo; do
+          list=$(echo "$output" | python -c "import sys,json; d=json.load(sys.stdin); print(d['jobs'].get('test-'+'$job',''))")
+          echo "test-list-$job=$list" >> $GITHUB_OUTPUT
+        done
+
+        # Log summary
+        echo "$output" | python -c "import sys,json; print(json.load(sys.stdin)['summary'])"
 
   test-isaaclab-tasks:
     name: IsaacLab Tasks Tests 1/2
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
     continue-on-error: true
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-isaaclab-tasks != ''
 
     steps:
     - name: Checkout Code
@@ -121,12 +174,9 @@ jobs:
         pytest-options: ""
         filter-pattern: "isaaclab_tasks"
         include-files: >-
-            test_multi_agent_environments.py,
-            test_pickplace_stack_environments.py,
-            test_environments.py,
-            test_factory_environments.py,
-            test_cartpole_showcase_environments.py,
-            test_teleop_environments.py
+          ${{ needs.select-tests.outputs.run-all != 'true'
+          && needs.select-tests.outputs.test-list-isaaclab-tasks
+          || 'test_multi_agent_environments.py,test_pickplace_stack_environments.py,test_environments.py,test_factory_environments.py,test_cartpole_showcase_environments.py,test_teleop_environments.py' }}
 
     - name: Upload IsaacLab Tasks Test Results
       uses: actions/upload-artifact@v4
@@ -161,7 +211,10 @@ jobs:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
     continue-on-error: true
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-isaaclab-tasks-2 != ''
 
     steps:
     - name: Checkout Code
@@ -189,13 +242,9 @@ jobs:
         pytest-options: ""
         filter-pattern: "isaaclab_tasks"
         include-files: >-
-          test_teleop_environments_with_stage_in_memory.py,
-          test_lift_teddy_bear.py,
-          test_environment_determinism.py,
-          test_hydra.py,
-          test_rl_device_separation.py,
-          test_cartpole_showcase_environments_with_stage_in_memory.py,
-          test_environments_with_stage_in_memory.py
+          ${{ needs.select-tests.outputs.run-all != 'true'
+          && needs.select-tests.outputs.test-list-isaaclab-tasks-2
+          || 'test_teleop_environments_with_stage_in_memory.py,test_lift_teddy_bear.py,test_environment_determinism.py,test_hydra.py,test_rl_device_separation.py,test_cartpole_showcase_environments_with_stage_in_memory.py,test_environments_with_stage_in_memory.py' }}
 
     - name: Upload IsaacLab Tasks 2 Test Results
       uses: actions/upload-artifact@v4
@@ -229,7 +278,10 @@ jobs:
     name: General Tests
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-general != ''
 
     steps:
     - name: Checkout Code
@@ -257,6 +309,7 @@ jobs:
         image-tag: ${{ env.DOCKER_IMAGE_TAG }}
         pytest-options: ""
         filter-pattern: "not isaaclab_tasks,isaaclab_newton,isaaclab_physx"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-general || '' }}
 
     - name: Upload General Test Results
       uses: actions/upload-artifact@v4
@@ -289,7 +342,10 @@ jobs:
   test-newton:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-newton != ''
 
     steps:
     - name: Checkout Code
@@ -316,6 +372,7 @@ jobs:
         image-tag: ${{ env.DOCKER_IMAGE_TAG }}
         pytest-options: ""
         filter-pattern: "isaaclab_newton"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-newton || '' }}
 
     - name: Upload Newton Test Results
       uses: actions/upload-artifact@v4
@@ -348,7 +405,10 @@ jobs:
   test-physx:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-physx != ''
 
     steps:
     - name: Checkout Code
@@ -375,6 +435,7 @@ jobs:
         image-tag: ${{ env.DOCKER_IMAGE_TAG }}
         pytest-options: ""
         filter-pattern: "isaaclab_physx"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-physx || '' }}
 
     - name: Upload PhysX Test Results
       uses: actions/upload-artifact@v4
@@ -409,7 +470,10 @@ jobs:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 120
     continue-on-error: true
-    needs: build-curobo
+    needs: [build-curobo, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-curobo != ''
 
     steps:
     - name: Checkout Code
@@ -437,6 +501,7 @@ jobs:
         pytest-options: ""
         filter-pattern: ""
         curobo-only: "true"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-curobo || '' }}
 
     - name: Upload cuRobo Test Results
       uses: actions/upload-artifact@v4
@@ -471,7 +536,10 @@ jobs:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
     continue-on-error: true
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-flaky != ''
 
     steps:
     - name: Checkout Code
@@ -499,6 +567,7 @@ jobs:
         pytest-options: ""
         filter-pattern: ""
         flaky-only: "true"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-flaky || '' }}
 
     - name: Upload Flaky Test Results
       uses: actions/upload-artifact@v4
@@ -532,7 +601,10 @@ jobs:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 60
     continue-on-error: true
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-slightly-flaky != ''
 
     steps:
     - name: Checkout Code
@@ -560,6 +632,7 @@ jobs:
         pytest-options: ""
         filter-pattern: ""
         slightly-flaky-only: "true"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-slightly-flaky || '' }}
 
     - name: Upload Slightly Flaky Test Results
       uses: actions/upload-artifact@v4
@@ -593,7 +666,10 @@ jobs:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 300
     continue-on-error: true
-    needs: build
+    needs: [build, select-tests]
+    if: >-
+      needs.select-tests.outputs.run-all == 'true' ||
+      needs.select-tests.outputs.test-list-environments-training != ''
 
     steps:
     - name: Checkout Code
@@ -620,7 +696,7 @@ jobs:
         image-tag: ${{ env.DOCKER_IMAGE_TAG }}
         pytest-options: ""
         filter-pattern: "isaaclab_tasks"
-        include-files: "test_environments_training.py"
+        include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-environments-training || 'test_environments_training.py' }}
 
     - name: Upload Environments Training Test Results
       uses: actions/upload-artifact@v4
@@ -650,7 +726,7 @@ jobs:
       run: docker image prune -f --filter "until=24h" || true
 
   combine-results:
-    needs: [build, build-curobo, test-isaaclab-tasks, test-isaaclab-tasks-2, test-general, test-newton, test-physx, test-curobo, test-flaky, test-slightly-flaky, test-environments-training]
+    needs: [build, build-curobo, select-tests, test-isaaclab-tasks, test-isaaclab-tasks-2, test-general, test-newton, test-physx, test-curobo, test-flaky, test-slightly-flaky, test-environments-training]
     runs-on: ubuntu-latest
     if: always()
     name: Combine Results

.github/workflows/coverage-map.yml

@@ -0,0 +1,97 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: Coverage Map
+
+on:
+  schedule:
+    # Run nightly at 4 AM UTC (8 PM PST)
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  issues: write
+
+env:
+  NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+  ISAACSIM_BASE_IMAGE: 'nvcr.io/nvidian/isaac-sim'
+  ISAACSIM_BASE_VERSION: 'latest-develop'
+  DOCKER_IMAGE_TAG: isaac-lab-coverage:${{ github.sha }}
+
+jobs:
+  collect-coverage:
+    name: Collect Test Coverage Map
+    runs-on: [self-hosted, gpu]
+    timeout-minutes: 720  # 12 hours max
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+        lfs: true
+
+    - name: Build Docker image
+      uses: ./.github/actions/ecr-build-push-pull
+      with:
+        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
+        isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }}
+        isaacsim-version: ${{ env.ISAACSIM_BASE_VERSION }}
+        dockerfile-path: docker/Dockerfile.base
+        cache-tag: cache-base
+
+    - name: Collect coverage
+      run: |
+        docker run --name coverage-collector \
+          --entrypoint bash --gpus all --network=host \
+          -e OMNI_KIT_ACCEPT_EULA=yes \
+          -e ACCEPT_EULA=Y \
+          -e ISAAC_SIM_HEADLESS=1 \
+          ${{ env.DOCKER_IMAGE_TAG }} \
+          -c "
+            cd /workspace/isaaclab
+            pip install coverage
+            python tools/collect_coverage_map.py --workers 4 --timeout 2000
+          "
+
+        docker cp coverage-collector:/workspace/isaaclab/tools/test-dependency-map.json tools/test-dependency-map.json
+        docker rm coverage-collector
+
+    - name: Commit mapping to ci/coverage-map branch
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+
+        # Create or switch to the ci/coverage-map branch
+        git fetch origin ci/coverage-map || true
+        git checkout ci/coverage-map 2>/dev/null || git checkout -b ci/coverage-map
+
+        # Copy the mapping file and commit
+        git add tools/test-dependency-map.json
+        git diff --cached --quiet || git commit -m "Update test dependency mapping $(date -u +%Y-%m-%d)"
+        git push origin ci/coverage-map
+
+    - name: Check for consecutive failures
+      if: failure()
+      run: |
+        # Count recent failures using gh CLI
+        recent_failures=$(gh run list --workflow=coverage-map.yml --limit=3 --json conclusion \
+          --jq '[.[] | select(.conclusion == "failure")] | length')
+
+        if [ "$recent_failures" -ge 3 ]; then
+          gh issue create \
+            --title "Coverage map nightly job failing" \
+            --body "The coverage-map workflow has failed 3+ consecutive times. PRs will fall back to running all tests until this is fixed." \
+            --label "ci"
+        fi
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Clean up
+      if: always()
+      run: |
+        docker rm -f coverage-collector 2>/dev/null || true
+        docker image prune -f --filter "until=24h" || true

docs/source/testing/index.rst

@@ -8,6 +8,7 @@ This section covers testing utilities and patterns for Isaac Lab development.
 .. toctree::
    :maxdepth: 2
 
+   test_selection
    mock_interfaces
    micro_benchmarks
    benchmarks