diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2a09001d9451..e848be5d9364 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -87,13 +87,66 @@ jobs: if: always() run: docker image prune -f --filter "until=24h" || true + select-tests: + name: Select Tests + runs-on: ubuntu-latest + outputs: + run-all: ${{ steps.select.outputs.run-all }} + test-list-physx: ${{ steps.select.outputs.test-list-physx }} + test-list-newton: ${{ steps.select.outputs.test-list-newton }} + test-list-general: ${{ steps.select.outputs.test-list-general }} + test-list-isaaclab-tasks: ${{ steps.select.outputs.test-list-isaaclab-tasks }} + test-list-isaaclab-tasks-2: ${{ steps.select.outputs.test-list-isaaclab-tasks-2 }} + test-list-environments-training: ${{ steps.select.outputs.test-list-environments-training }} + test-list-flaky: ${{ steps.select.outputs.test-list-flaky }} + test-list-slightly-flaky: ${{ steps.select.outputs.test-list-slightly-flaky }} + test-list-curobo: ${{ steps.select.outputs.test-list-curobo }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch coverage mapping + run: | + git fetch origin ci/coverage-map 2>/dev/null || true + git checkout origin/ci/coverage-map -- tools/test-dependency-map.json 2>/dev/null || echo '{}' > tools/test-dependency-map.json + + - name: Select tests + id: select + run: | + set -euo pipefail + base_ref="${{ github.event.pull_request.base.ref }}" + if [ -z "$base_ref" ]; then + echo "No base branch (not a PR event). Running all tests." + echo "run-all=true" >> $GITHUB_OUTPUT + for job in physx newton general isaaclab-tasks isaaclab-tasks-2 environments-training flaky slightly-flaky curobo; do + echo "test-list-$job=" >> $GITHUB_OUTPUT + done + exit 0 + fi + output=$(python tools/select_tests.py --base-branch "origin/$base_ref" --dry-run 2>> $GITHUB_STEP_SUMMARY) + + run_all=$(echo "$output" | python -c "import sys,json; print(str(json.load(sys.stdin)['run_all']).lower())") + echo "run-all=$run_all" >> $GITHUB_OUTPUT + + for job in physx newton general isaaclab-tasks isaaclab-tasks-2 environments-training flaky slightly-flaky curobo; do + list=$(echo "$output" | python -c "import sys,json; d=json.load(sys.stdin); print(d['jobs'].get('test-'+'$job',''))") + echo "test-list-$job=$list" >> $GITHUB_OUTPUT + done + + # Log summary + echo "$output" | python -c "import sys,json; print(json.load(sys.stdin)['summary'])" test-isaaclab-tasks: name: IsaacLab Tasks Tests 1/2 runs-on: [self-hosted, gpu] timeout-minutes: 180 continue-on-error: true - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-isaaclab-tasks != '' steps: - name: Checkout Code @@ -121,12 +174,9 @@ jobs: pytest-options: "" filter-pattern: "isaaclab_tasks" include-files: >- - test_multi_agent_environments.py, - test_pickplace_stack_environments.py, - test_environments.py, - test_factory_environments.py, - test_cartpole_showcase_environments.py, - test_teleop_environments.py + ${{ needs.select-tests.outputs.run-all != 'true' + && needs.select-tests.outputs.test-list-isaaclab-tasks + || 'test_multi_agent_environments.py,test_pickplace_stack_environments.py,test_environments.py,test_factory_environments.py,test_cartpole_showcase_environments.py,test_teleop_environments.py' }} - name: Upload IsaacLab Tasks Test Results uses: actions/upload-artifact@v4 @@ -161,7 +211,10 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 180 continue-on-error: true - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-isaaclab-tasks-2 != '' steps: - name: Checkout Code @@ -189,13 +242,9 @@ jobs: pytest-options: "" filter-pattern: "isaaclab_tasks" include-files: >- - test_teleop_environments_with_stage_in_memory.py, - test_lift_teddy_bear.py, - test_environment_determinism.py, - test_hydra.py, - test_rl_device_separation.py, - test_cartpole_showcase_environments_with_stage_in_memory.py, - test_environments_with_stage_in_memory.py + ${{ needs.select-tests.outputs.run-all != 'true' + && needs.select-tests.outputs.test-list-isaaclab-tasks-2 + || 'test_teleop_environments_with_stage_in_memory.py,test_lift_teddy_bear.py,test_environment_determinism.py,test_hydra.py,test_rl_device_separation.py,test_cartpole_showcase_environments_with_stage_in_memory.py,test_environments_with_stage_in_memory.py' }} - name: Upload IsaacLab Tasks 2 Test Results uses: actions/upload-artifact@v4 @@ -229,7 +278,10 @@ jobs: name: General Tests runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-general != '' steps: - name: Checkout Code @@ -257,6 +309,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" filter-pattern: "not isaaclab_tasks,isaaclab_newton,isaaclab_physx" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-general || '' }} - name: Upload General Test Results uses: actions/upload-artifact@v4 @@ -289,7 +342,10 @@ jobs: test-newton: runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-newton != '' steps: - name: Checkout Code @@ -316,6 +372,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" filter-pattern: "isaaclab_newton" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-newton || '' }} - name: Upload Newton Test Results uses: actions/upload-artifact@v4 @@ -348,7 +405,10 @@ jobs: test-physx: runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-physx != '' steps: - name: Checkout Code @@ -375,6 +435,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" filter-pattern: "isaaclab_physx" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-physx || '' }} - name: Upload PhysX Test Results uses: actions/upload-artifact@v4 @@ -409,7 +470,10 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 120 continue-on-error: true - needs: build-curobo + needs: [build-curobo, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-curobo != '' steps: - name: Checkout Code @@ -437,6 +501,7 @@ jobs: pytest-options: "" filter-pattern: "" curobo-only: "true" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-curobo || '' }} - name: Upload cuRobo Test Results uses: actions/upload-artifact@v4 @@ -471,7 +536,10 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 180 continue-on-error: true - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-flaky != '' steps: - name: Checkout Code @@ -499,6 +567,7 @@ jobs: pytest-options: "" filter-pattern: "" flaky-only: "true" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-flaky || '' }} - name: Upload Flaky Test Results uses: actions/upload-artifact@v4 @@ -532,7 +601,10 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 60 continue-on-error: true - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-slightly-flaky != '' steps: - name: Checkout Code @@ -560,6 +632,7 @@ jobs: pytest-options: "" filter-pattern: "" slightly-flaky-only: "true" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-slightly-flaky || '' }} - name: Upload Slightly Flaky Test Results uses: actions/upload-artifact@v4 @@ -593,7 +666,10 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 300 continue-on-error: true - needs: build + needs: [build, select-tests] + if: >- + needs.select-tests.outputs.run-all == 'true' || + needs.select-tests.outputs.test-list-environments-training != '' steps: - name: Checkout Code @@ -620,7 +696,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" filter-pattern: "isaaclab_tasks" - include-files: "test_environments_training.py" + include-files: ${{ needs.select-tests.outputs.run-all != 'true' && needs.select-tests.outputs.test-list-environments-training || 'test_environments_training.py' }} - name: Upload Environments Training Test Results uses: actions/upload-artifact@v4 @@ -650,7 +726,7 @@ jobs: run: docker image prune -f --filter "until=24h" || true combine-results: - needs: [build, build-curobo, test-isaaclab-tasks, test-isaaclab-tasks-2, test-general, test-newton, test-physx, test-curobo, test-flaky, test-slightly-flaky, test-environments-training] + needs: [build, build-curobo, select-tests, test-isaaclab-tasks, test-isaaclab-tasks-2, test-general, test-newton, test-physx, test-curobo, test-flaky, test-slightly-flaky, test-environments-training] runs-on: ubuntu-latest if: always() name: Combine Results diff --git a/.github/workflows/coverage-map.yml b/.github/workflows/coverage-map.yml new file mode 100644 index 000000000000..6a834b7b859c --- /dev/null +++ b/.github/workflows/coverage-map.yml @@ -0,0 +1,97 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: Coverage Map + +on: + schedule: + # Run nightly at 4 AM UTC (8 PM PST) + - cron: '0 4 * * *' + workflow_dispatch: + +permissions: + contents: write + issues: write + +env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + ISAACSIM_BASE_IMAGE: 'nvcr.io/nvidian/isaac-sim' + ISAACSIM_BASE_VERSION: 'latest-develop' + DOCKER_IMAGE_TAG: isaac-lab-coverage:${{ github.sha }} + +jobs: + collect-coverage: + name: Collect Test Coverage Map + runs-on: [self-hosted, gpu] + timeout-minutes: 720 # 12 hours max + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + lfs: true + + - name: Build Docker image + uses: ./.github/actions/ecr-build-push-pull + with: + image-tag: ${{ env.DOCKER_IMAGE_TAG }} + isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }} + isaacsim-version: ${{ env.ISAACSIM_BASE_VERSION }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Collect coverage + run: | + docker run --name coverage-collector \ + --entrypoint bash --gpus all --network=host \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e ISAAC_SIM_HEADLESS=1 \ + ${{ env.DOCKER_IMAGE_TAG }} \ + -c " + cd /workspace/isaaclab + pip install coverage + python tools/collect_coverage_map.py --workers 4 --timeout 2000 + " + + docker cp coverage-collector:/workspace/isaaclab/tools/test-dependency-map.json tools/test-dependency-map.json + docker rm coverage-collector + + - name: Commit mapping to ci/coverage-map branch + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Create or switch to the ci/coverage-map branch + git fetch origin ci/coverage-map || true + git checkout ci/coverage-map 2>/dev/null || git checkout -b ci/coverage-map + + # Copy the mapping file and commit + git add tools/test-dependency-map.json + git diff --cached --quiet || git commit -m "Update test dependency mapping $(date -u +%Y-%m-%d)" + git push origin ci/coverage-map + + - name: Check for consecutive failures + if: failure() + run: | + # Count recent failures using gh CLI + recent_failures=$(gh run list --workflow=coverage-map.yml --limit=3 --json conclusion \ + --jq '[.[] | select(.conclusion == "failure")] | length') + + if [ "$recent_failures" -ge 3 ]; then + gh issue create \ + --title "Coverage map nightly job failing" \ + --body "The coverage-map workflow has failed 3+ consecutive times. PRs will fall back to running all tests until this is fixed." \ + --label "ci" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Clean up + if: always() + run: | + docker rm -f coverage-collector 2>/dev/null || true + docker image prune -f --filter "until=24h" || true diff --git a/docs/source/testing/index.rst b/docs/source/testing/index.rst index ae8494a3ec89..e084f4febb2b 100644 --- a/docs/source/testing/index.rst +++ b/docs/source/testing/index.rst @@ -8,6 +8,7 @@ This section covers testing utilities and patterns for Isaac Lab development. .. toctree:: :maxdepth: 2 + test_selection mock_interfaces micro_benchmarks benchmarks diff --git a/docs/source/testing/test_selection.rst b/docs/source/testing/test_selection.rst new file mode 100644 index 000000000000..ca5b0163707f --- /dev/null +++ b/docs/source/testing/test_selection.rst @@ -0,0 +1,164 @@ +Coverage-Based Test Selection +============================= + +The CI uses a coverage-based test selection system to run only the tests affected by +your changes, rather than the full test suite on every PR. + + +How It Works +------------ + +The system has three components: + +1. **Coverage mapping** — A nightly workflow (``coverage-map.yml``) runs every test file + with ``coverage run`` and records which source files each test touches. The resulting + JSON mapping is committed to the ``ci/coverage-map`` branch as + ``tools/test-dependency-map.json``. + +2. **Test selection** — On every PR, a ``select-tests`` job runs + ``tools/select_tests.py`` to diff the PR against the base branch, look up which tests + cover the changed files, and assign them to the appropriate CI jobs (``test-physx``, + ``test-newton``, ``test-general``, etc.). + +3. **Job filtering** — Each CI test job is skipped entirely if it has no selected tests, + or runs with a filtered ``include-files`` list. When the mapping is missing or stale, + the system falls back to running all tests. + + +Fallback Behavior +----------------- + +The system falls back to running **all** tests when any of the following are true: + +* A changed file is not present in the mapping (e.g. a newly added file). +* The mapping is stale (older than 7 days). +* Non-Python files are changed (YAML, RST, Dockerfiles, etc.). +* CI infrastructure files are changed (``.github/``, ``docker/``, ``tools/conftest.py``). +* The mapping file is empty or missing. + +This ensures that the selective system never silently skips tests that should run. + + +Testing Locally +--------------- + +Unit tests +^^^^^^^^^^ + +The selection logic has its own test suite that runs without GPU or simulation: + +.. code-block:: bash + + ./isaaclab.sh -p -m pytest tools/test_select_tests.py -v \ + --override-ini="confcutdir=tools" --noconftest + +Dry-run against your branch +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create a dummy mapping and run the CLI to see what would be selected: + +.. code-block:: bash + + # Create a minimal mapping + cat > /tmp/test-map.json << 'EOF' + { + "metadata": { + "generated_at": "2026-03-16T00:00:00+00:00", + "commit": "abc123", + "test_file_count": 5, + "source_file_count": 10 + }, + "source_to_tests": { + "source/isaaclab/isaaclab/utils/math.py": [ + "source/isaaclab/test/utils/test_math.py" + ] + } + } + EOF + + # Dry-run against main (JSON to stdout, rationale to stderr) + python tools/select_tests.py \ + --base-branch origin/main \ + --mapping /tmp/test-map.json \ + --dry-run + +Fallback behavior +^^^^^^^^^^^^^^^^^ + +Verify that an empty mapping triggers a full test run: + +.. code-block:: bash + + echo '{}' > /tmp/empty-map.json + python tools/select_tests.py \ + --base-branch origin/main \ + --mapping /tmp/empty-map.json \ + --dry-run + +Full-path matching in conftest +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``conftest.py`` include-files filter supports full paths to disambiguate test files +with the same basename across packages (e.g. ``test_articulation.py`` in both +``isaaclab_physx`` and ``isaaclab_newton``): + +.. code-block:: bash + + ./isaaclab.sh -p -c " + from tools.conftest import _matches_include_files + + # Full path matches only the intended package + assert _matches_include_files( + 'source/isaaclab_physx/test/assets/test_articulation.py', + 'test_articulation.py', + {'source/isaaclab_physx/test/assets/test_articulation.py'}) + + # Does NOT match a different package + assert not _matches_include_files( + 'source/isaaclab_newton/test/assets/test_articulation.py', + 'test_articulation.py', + {'source/isaaclab_physx/test/assets/test_articulation.py'}) + + # Basename fallback still works + assert _matches_include_files( + 'source/isaaclab_physx/test/assets/test_articulation.py', + 'test_articulation.py', + {'test_articulation.py'}) + + print('All assertions passed') + " + + +Regenerating the Coverage Mapping +--------------------------------- + +The nightly workflow handles this automatically, but you can also build the mapping on a +local GPU workstation: + +.. code-block:: bash + + ./isaaclab.sh -p -m pip install coverage + ./isaaclab.sh -p tools/collect_coverage_map.py --workers 4 --timeout 2000 + +The mapping is written to ``tools/test-dependency-map.json``. Increase ``--workers`` if +your machine has sufficient GPU memory — each worker spawns a separate simulation instance. + +To push the mapping so CI can use it: + +.. code-block:: bash + + git checkout -b ci/coverage-map 2>/dev/null || git checkout ci/coverage-map + git add tools/test-dependency-map.json + git commit -m "Update test dependency mapping" + git push origin ci/coverage-map + + +CI Integration +-------------- + +The only way to test the full ``build.yaml`` wiring end-to-end is to push your branch and +open a PR. The ``select-tests`` job runs on ``ubuntu-latest`` and its outputs are visible +in the job logs. + +If the ``ci/coverage-map`` branch does not exist yet, the system falls back to +``run-all=true`` (the existing behavior), so it is always safe to merge. diff --git a/tools/collect_coverage_map.py b/tools/collect_coverage_map.py new file mode 100644 index 000000000000..b24cc4e94e29 --- /dev/null +++ b/tools/collect_coverage_map.py @@ -0,0 +1,237 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Collect per-test-file coverage and produce a source-to-test dependency mapping. + +This script discovers all test files, runs each one with ``coverage run``, and +then combines the per-test coverage data into a JSON mapping suitable for use +by ``select_tests.py``. + +Usage: + python tools/collect_coverage_map.py [--output PATH] [--workers N] [--timeout SECS] +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tempfile +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime, timezone +from pathlib import Path + + +def discover_test_files(source_dirs: list[str]) -> list[str]: + """Walk source directories and return all test_*.py file paths.""" + test_files = [] + for source_dir in source_dirs: + if not os.path.exists(source_dir): + continue + for root, _, files in os.walk(source_dir): + for f in files: + if f.startswith("test_") and f.endswith(".py"): + test_files.append(os.path.join(root, f)) + return sorted(test_files) + + +def run_test_with_coverage(test_file: str, coverage_dir: str, timeout: int) -> str | None: + """Run a single test file with coverage and return the coverage data file path. + + Args: + test_file: Absolute path to the test file. + coverage_dir: Directory to store .coverage. files. + timeout: Maximum seconds to allow the test to run. + + Returns: + Path to the coverage data file, or None if the test failed/timed out. + """ + # Use a deterministic hash so filenames are stable across runs + test_hash = hashlib.md5(test_file.encode()).hexdigest()[:16] # noqa: S324 + data_file = os.path.join(coverage_dir, f".coverage.{test_hash}") + + cmd = [ + sys.executable, + "-m", + "coverage", + "run", + f"--data-file={data_file}", + "--source=source/,scripts/", + "-m", + "pytest", + "--no-header", + "-x", # stop on first failure + test_file, + ] + + try: + subprocess.run( + cmd, + timeout=timeout, + capture_output=True, + text=True, + ) + # Even if the test failed, coverage data may have been written + if os.path.exists(data_file): + return data_file + return None + except subprocess.TimeoutExpired: + print(f"TIMEOUT: {test_file} (>{timeout}s)", file=sys.stderr) + return None + except Exception as e: + print(f"ERROR: {test_file}: {e}", file=sys.stderr) + return None + + +def extract_covered_files(data_file: str) -> set[str]: + """Extract the set of source files covered by a single test run. + + Args: + data_file: Path to a .coverage data file. + + Returns: + Set of source file paths (relative to repo root). + """ + # Use coverage's JSON export to get the file list + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + tmp_path = tmp.name + + try: + subprocess.run( + [sys.executable, "-m", "coverage", "json", f"--data-file={data_file}", "-o", tmp_path], + capture_output=True, + text=True, + check=True, + ) + with open(tmp_path) as f: + cov_data = json.load(f) + + repo_root = str(Path(__file__).resolve().parent.parent) + files = set() + for abs_path in cov_data.get("files", {}): + # Convert absolute paths to repo-relative + if abs_path.startswith(repo_root): + rel = os.path.relpath(abs_path, repo_root) + files.add(rel) + return files + except Exception: + return set() + finally: + os.unlink(tmp_path) + + +def build_mapping( + test_files: list[str], + coverage_dir: str, + timeout: int, + workers: int, + repo_root: str, +) -> dict: + """Run all tests with coverage and build the source-to-test mapping. + + Args: + test_files: List of test file absolute paths. + coverage_dir: Temp directory for coverage data files. + timeout: Per-test timeout in seconds. + workers: Number of parallel workers. + repo_root: Repository root directory. + + Returns: + The complete mapping dict ready for JSON serialization. + """ + source_to_tests = defaultdict(set) + completed = 0 + failed = 0 + + with ProcessPoolExecutor(max_workers=workers) as executor: + futures = {} + for test_file in test_files: + future = executor.submit(run_test_with_coverage, test_file, coverage_dir, timeout) + futures[future] = test_file + + for future in as_completed(futures): + test_file = futures[future] + data_file = future.result() + + if data_file is None: + failed += 1 + print(f"SKIP (no coverage): {test_file}", file=sys.stderr) + continue + + covered_files = extract_covered_files(data_file) + test_rel = os.path.relpath(test_file, repo_root) + + for src_file in covered_files: + source_to_tests[src_file].add(test_rel) + + completed += 1 + print(f"[{completed}/{len(test_files)}] {test_file} -> {len(covered_files)} files", file=sys.stderr) + + total = len(test_files) + print(f"\nCompleted: {completed}/{total}, Failed: {failed}/{total}", file=sys.stderr) + + # Check partial failure threshold + if total > 0 and completed / total < 0.5: + print("ERROR: Less than 50% of tests completed. Not writing mapping.", file=sys.stderr) + sys.exit(1) + + # Convert sets to sorted lists for JSON + mapping = { + "metadata": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "commit": _get_current_commit(), + "test_file_count": completed, + "source_file_count": len(source_to_tests), + }, + "source_to_tests": {k: sorted(v) for k, v in sorted(source_to_tests.items())}, + } + + return mapping + + +def _get_current_commit() -> str: + try: + result = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True) + return result.stdout.strip() + except Exception: + return "unknown" + + +def main(): + parser = argparse.ArgumentParser(description="Collect per-test coverage and produce dependency mapping.") + parser.add_argument("--output", default="tools/test-dependency-map.json", help="Output mapping JSON path.") + parser.add_argument("--workers", type=int, default=4, help="Number of parallel test workers.") + parser.add_argument("--timeout", type=int, default=2000, help="Per-test timeout in seconds.") + + args = parser.parse_args() + + repo_root = str(Path(__file__).resolve().parent.parent) + source_dirs = [ + os.path.join(repo_root, "scripts"), + os.path.join(repo_root, "source"), + ] + + print("Discovering test files...", file=sys.stderr) + test_files = discover_test_files(source_dirs) + print(f"Found {len(test_files)} test files", file=sys.stderr) + + with tempfile.TemporaryDirectory() as coverage_dir: + mapping = build_mapping(test_files, coverage_dir, args.timeout, args.workers, repo_root) + + output_path = os.path.join(repo_root, args.output) + with open(output_path, "w") as f: + json.dump(mapping, f, indent=2) + + print(f"Mapping written to {output_path}", file=sys.stderr) + print(f" {mapping['metadata']['test_file_count']} test files", file=sys.stderr) + print(f" {mapping['metadata']['source_file_count']} source files", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tools/conftest.py b/tools/conftest.py index 82ba353ff437..b073178fca4b 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -257,11 +257,29 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): return failed_tests, test_status +def _matches_include_files(full_path: str, basename: str, include_files: set[str]) -> bool: + """Check if a test file matches any entry in the include files set. + + Entries with '/' are matched as path suffixes. Entries without '/' are + matched as basenames. This allows precise full-path matching while + preserving backward compatibility with basename-only entries. + """ + for entry in include_files: + if "/" in entry: + if full_path.endswith(entry): + return True + else: + if basename == entry: + return True + return False + + def _collect_test_files( source_dirs, filter_pattern, exclude_pattern, include_files, + include_basenames, flaky_only, slightly_flaky_only, curobo_only, @@ -296,7 +314,7 @@ def _collect_test_files( # An explicit include_files entry overrides TESTS_TO_SKIP, allowing # dedicated jobs (e.g. test-environments-training) to run tests that # are otherwise excluded from general CI runs. - if file in test_settings.TESTS_TO_SKIP and file not in include_files: + if file in test_settings.TESTS_TO_SKIP and file not in include_basenames: print(f"Skipping {file} as it's in the skip list") continue @@ -308,7 +326,7 @@ def _collect_test_files( if exclude_pattern and exclude_pattern in full_path: print(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") continue - if include_files and file not in include_files: + if include_files and not _matches_include_files(full_path, file, include_files): print(f"Skipping {full_path} (not in include files list)") continue @@ -336,13 +354,18 @@ def pytest_sessionstart(session): isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" - # Parse include files list (comma-separated paths) + # Parse include files list (comma-separated). + # Entries with '/' are treated as path suffixes for precise matching. + # Entries without '/' are basenames (backward compatibility). include_files = set() if include_files_str: for f in include_files_str.split(","): f = f.strip() if f: - include_files.add(os.path.basename(f)) + include_files.add(f) + + # For TESTS_TO_SKIP override checking, extract basenames from all entries + include_basenames = {os.path.basename(f) for f in include_files} # Also try to get from pytest config if hasattr(session.config, "option") and hasattr(session.config.option, "filter_pattern"): @@ -356,6 +379,7 @@ def pytest_sessionstart(session): print(f"Filter pattern: '{filter_pattern}'") print(f"Exclude pattern: '{exclude_pattern}'") print(f"Include files: {include_files if include_files else 'none'}") + print(f"Include basenames: {include_basenames if include_basenames else 'none'}") print(f"Curobo-only mode: {curobo_only}") print(f"CUDA-issue-only mode: {cuda_issue_only}") print(f"Flaky-only mode: {flaky_only}") @@ -375,6 +399,7 @@ def pytest_sessionstart(session): filter_pattern, exclude_pattern, include_files, + include_basenames, flaky_only, slightly_flaky_only, curobo_only, diff --git a/tools/select_tests.py b/tools/select_tests.py new file mode 100644 index 000000000000..cfcfdf329b2b --- /dev/null +++ b/tools/select_tests.py @@ -0,0 +1,406 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Select tests to run based on changed files and a coverage-based dependency mapping. + +This script is used by CI to determine which test files need to run for a given PR. +It loads a pre-computed mapping of source files to test files (produced by +collect_coverage_map.py) and intersects it with the set of changed files. + +Usage: + python tools/select_tests.py [--mapping PATH] [--base-branch BRANCH] + [--max-age-days N] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +# Import test settings for category lists +sys.path.insert(0, str(Path(__file__).parent)) +import test_settings # noqa: E402 + +# Directories whose changes are ignored (no tests needed) +_IGNORE_PREFIXES = ("docs/",) + +# Directory prefixes that trigger full fallback (affect build/test infrastructure) +_INFRA_PREFIXES = (".github/", "docker/") + +# Infrastructure files that trigger full fallback +_INFRASTRUCTURE_FILES = { + "tools/conftest.py", + "tools/test_settings.py", + "tools/select_tests.py", + "tools/collect_coverage_map.py", + "pyproject.toml", +} + +# Patterns for infrastructure files matched by suffix +_INFRASTRUCTURE_SUFFIXES = ("setup.py", "config/extension.toml") + +# All CI job names (used for initializing the output dict) +_ALL_JOBS = [ + "test-physx", + "test-newton", + "test-general", + "test-isaaclab-tasks", + "test-isaaclab-tasks-2", + "test-environments-training", + "test-flaky", + "test-slightly-flaky", + "test-curobo", +] + +# Classifications that trigger full fallback +_FALLBACK_CLASSIFICATIONS = {"unmapped", "non_python_source", "infrastructure", "apps", "deleted_source", "renamed"} + +# Tasks 1/2 file list (mirrors build.yaml test-isaaclab-tasks include-files). +# Keep in sync with the include-files lists in .github/workflows/build.yaml. +_TASKS_1_FILES = { + "test_multi_agent_environments.py", + "test_pickplace_stack_environments.py", + "test_environments.py", + "test_factory_environments.py", + "test_cartpole_showcase_environments.py", + "test_teleop_environments.py", +} + +# Tasks 2/2 file list (mirrors build.yaml test-isaaclab-tasks-2 include-files). +# Keep in sync with the include-files lists in .github/workflows/build.yaml. +_TASKS_2_FILES = { + "test_teleop_environments_with_stage_in_memory.py", + "test_lift_teddy_bear.py", + "test_environment_determinism.py", + "test_hydra.py", + "test_rl_device_separation.py", + "test_cartpole_showcase_environments_with_stage_in_memory.py", + "test_environments_with_stage_in_memory.py", +} + + +def load_mapping(path: str) -> dict | None: + """Load the test dependency mapping from a JSON file. + + Args: + path: Path to the mapping JSON file. + + Returns: + The parsed mapping dict, or None if the file is missing, empty, or invalid. + """ + try: + with open(path) as f: + data = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return None + + if not data or "source_to_tests" not in data: + return None + + return data + + +def is_mapping_stale(metadata: dict, max_age_days: int) -> bool: + """Check if the mapping is too old to trust. + + Args: + metadata: The "metadata" dict from the mapping file. + max_age_days: Maximum age in days before considering the mapping stale. + + Returns: + True if the mapping is stale or has no timestamp. + """ + generated_at = metadata.get("generated_at") + if not generated_at: + return True + + try: + generated_dt = datetime.fromisoformat(generated_at) + if generated_dt.tzinfo is None: + generated_dt = generated_dt.replace(tzinfo=timezone.utc) + age = datetime.now(timezone.utc) - generated_dt + return age.days > max_age_days + except (ValueError, TypeError): + return True + + +def _is_test_file(path: str) -> bool: + """Check if a path looks like a test file. + + Matches test files directly in test/ or in any subdirectory of test/. + """ + return bool(re.match(r"(source|scripts)/[^/]+/test(/.*)?/test_[^/]+\.py$", path)) + + +def _is_under_source_or_scripts(path: str) -> bool: + return path.startswith("source/") or path.startswith("scripts/") + + +def classify_file( + path: str, + status: str, + mapping_keys: set[str], +) -> str: + """Classify a changed file for test selection purposes. + + Args: + path: File path relative to repo root. + status: Git diff status character (M, A, D, R, etc.). + mapping_keys: Set of source file paths present in the coverage mapping. + + Returns: + One of: 'test', 'mapped', 'unmapped', 'non_python_source', 'infrastructure', + 'apps', 'deleted_source', 'renamed', 'ignore'. + """ + # Ignored directories + if any(path.startswith(p) for p in _IGNORE_PREFIXES): + return "ignore" + + # CI/build infrastructure directories + if any(path.startswith(p) for p in _INFRA_PREFIXES): + return "infrastructure" + + # Renamed files + if status == "R": + if _is_under_source_or_scripts(path): + return "renamed" + return "ignore" + + # Deleted files + if status == "D": + if _is_under_source_or_scripts(path) and path.endswith(".py"): + return "deleted_source" + return "ignore" + + # Apps directory + if path.startswith("apps/"): + return "apps" + + # Infrastructure files (exact match) + if path in _INFRASTRUCTURE_FILES: + return "infrastructure" + + # Infrastructure files (suffix match) + if any(path.endswith(s) for s in _INFRASTRUCTURE_SUFFIXES): + return "infrastructure" + + # Test files + if _is_test_file(path): + return "test" + + # Source/scripts files + if _is_under_source_or_scripts(path): + if not path.endswith(".py"): + return "non_python_source" + if path in mapping_keys: + return "mapped" + return "unmapped" + + # Everything else (root files, etc.) — ignore + return "ignore" + + +def assign_test_to_job(test_path: str) -> str: + """Assign a test file to the correct CI job. + + Uses the file's full path and basename to determine which job should run it. + Priority: special category lists > package path > default. + + Args: + test_path: Full path to the test file, relative to repo root. + + Returns: + Job name string (e.g. 'test-physx', 'test-general'). + """ + basename = test_path.rsplit("/", 1)[-1] if "/" in test_path else test_path + + # Special category lists take priority (these tests are excluded from normal jobs) + if basename == "test_environments_training.py": + return "test-environments-training" + if basename in set(test_settings.FLAKY_TESTS): + return "test-flaky" + if basename in set(test_settings.SLIGHTLY_FLAKY_TESTS): + return "test-slightly-flaky" + if basename in set(test_settings.CUROBO_TESTS): + return "test-curobo" + + # Package-based assignment using full path + if "isaaclab_physx" in test_path: + return "test-physx" + if "isaaclab_newton" in test_path: + return "test-newton" + if "isaaclab_tasks" in test_path: + if basename in _TASKS_1_FILES: + return "test-isaaclab-tasks" + if basename in _TASKS_2_FILES: + return "test-isaaclab-tasks-2" + # Tasks tests not in either split go to tasks-1 by default + return "test-isaaclab-tasks" + + return "test-general" + + +def select_tests( + mapping_path: str, + changed_files: list[tuple[str, str]], + max_age_days: int, +) -> dict: + """Determine which tests to run based on changed files and the coverage mapping. + + Args: + mapping_path: Path to the test-dependency-map.json file. + changed_files: List of (status, path) tuples from git diff --name-status. + max_age_days: Maximum mapping age in days before triggering full fallback. + + Returns: + Dict with keys: + - "run_all": bool indicating whether to run all tests. + - "jobs": dict mapping job name to comma-separated test file paths. + - "summary": human-readable summary string. + """ + jobs = {job: set() for job in _ALL_JOBS} + result = {"run_all": False, "jobs": {}, "summary": ""} + + # Load mapping + mapping_data = load_mapping(mapping_path) + if mapping_data is None: + result["run_all"] = True + result["jobs"] = {job: "" for job in _ALL_JOBS} + result["summary"] = "Mapping not available. Falling back to all tests." + return result + + # Check staleness + if is_mapping_stale(mapping_data.get("metadata", {}), max_age_days): + result["run_all"] = True + result["jobs"] = {job: "" for job in _ALL_JOBS} + result["summary"] = f"Mapping is older than {max_age_days} days. Falling back to all tests." + return result + + source_to_tests = mapping_data["source_to_tests"] + mapping_keys = set(source_to_tests.keys()) + + # Classify each changed file and collect tests + classifications = {} + for status, path in changed_files: + classification = classify_file(path, status, mapping_keys) + classifications[path] = classification + + if classification in _FALLBACK_CLASSIFICATIONS: + result["run_all"] = True + result["jobs"] = {job: "" for job in _ALL_JOBS} + result["summary"] = f"Full fallback triggered by {classification} file: {path}" + return result + + if classification == "test": + job = assign_test_to_job(path) + jobs[job].add(path) + elif classification == "mapped": + for test_path in source_to_tests[path]: + job = assign_test_to_job(test_path) + jobs[job].add(test_path) + # 'ignore' requires no action + + # Build output + total_selected = sum(len(v) for v in jobs.values()) + total_tests = mapping_data.get("metadata", {}).get("test_file_count", "?") + result["jobs"] = {job: ",".join(sorted(tests)) for job, tests in jobs.items()} + + n_changed = len(changed_files) + n_mapped = sum(1 for c in classifications.values() if c == "mapped") + n_ignored = sum(1 for c in classifications.values() if c == "ignore") + n_test = sum(1 for c in classifications.values() if c == "test") + + if total_tests != "?" and total_tests > 0: + pct = (1 - total_selected / total_tests) * 100 + result["summary"] = ( + f"Selected {total_selected}/{total_tests} tests ({pct:.0f}% reduction). " + f"{n_changed} files changed: {n_mapped} mapped, {n_test} test files, {n_ignored} ignored." + ) + else: + result["summary"] = f"Selected {total_selected} tests. {n_changed} files changed." + + return result + + +def parse_git_diff_output(lines: list[str]) -> list[tuple[str, str]]: + """Parse the output of ``git diff --name-status`` into (status, path) tuples. + + Handles rename lines (Rxxoldnew) by extracting only the new path. + + Args: + lines: Lines of git diff --name-status output. + + Returns: + List of (status, path) tuples. Status is a single character (M, A, D, R). + """ + results = [] + for line in lines: + line = line.strip() + if not line: + continue + parts = line.split("\t") + status = parts[0] + + if status.startswith("R"): + # Rename: status is like R100, and there are two paths (old, new) + new_path = parts[2] if len(parts) >= 3 else parts[1] + results.append(("R", new_path)) + else: + # M, A, D, C, etc. + path = parts[1] if len(parts) >= 2 else "" + results.append((status[0], path)) + + return results + + +def get_changed_files(base_branch: str) -> list[tuple[str, str]]: + """Get the list of changed files between HEAD and the base branch. + + Args: + base_branch: The base branch to diff against (e.g. 'origin/main'). + + Returns: + List of (status, path) tuples. + """ + cmd = ["git", "diff", "--name-status", f"{base_branch}...HEAD"] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return parse_git_diff_output(result.stdout.strip().split("\n")) + + +def main(): + parser = argparse.ArgumentParser(description="Select tests based on changed files and coverage mapping.") + parser.add_argument("--mapping", default="tools/test-dependency-map.json", help="Path to the mapping JSON file.") + parser.add_argument("--base-branch", default="origin/main", help="Base branch to diff against.") + parser.add_argument("--max-age-days", type=int, default=7, help="Maximum mapping age in days.") + parser.add_argument("--dry-run", action="store_true", help="Print selection rationale to stderr.") + + args = parser.parse_args() + + # Get changed files + changed_files = get_changed_files(args.base_branch) + + # Run selection + result = select_tests(args.mapping, changed_files, args.max_age_days) + + # Dry-run output to stderr + if args.dry_run: + print(result["summary"], file=sys.stderr) + mapping_data = load_mapping(args.mapping) + mapping_keys = set(mapping_data.get("source_to_tests", {}).keys()) if mapping_data else set() + for status, path in changed_files: + classification = classify_file(path, status, mapping_keys) + print(f" {status} {path} -> {classification}", file=sys.stderr) + + # Output JSON to stdout + json.dump(result, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/tools/test_select_tests.py b/tools/test_select_tests.py new file mode 100644 index 000000000000..46473fb3fb76 --- /dev/null +++ b/tools/test_select_tests.py @@ -0,0 +1,553 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the test selection script.""" + +import ast +import json +import textwrap +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import select_tests + + +def _load_matches_include_files(): + """Extract _matches_include_files from conftest.py without importing the module. + + conftest.py has top-level imports (junitparser) that are unavailable in + lightweight test environments, so we parse the function source with AST + and compile it in isolation. + """ + source = Path(__file__).parent.joinpath("conftest.py").read_text() + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "_matches_include_files": + func_source = textwrap.dedent(ast.get_source_segment(source, node)) + namespace: dict = {} + exec(compile(func_source, "conftest.py", "exec"), namespace) # noqa: S102 + return namespace["_matches_include_files"] + raise RuntimeError("_matches_include_files not found in conftest.py") + + +_matches_include_files = _load_matches_include_files() + + +class TestLoadMapping: + """Tests for loading the dependency mapping file.""" + + def test_load_valid_mapping(self, tmp_path): + """A valid mapping file should be loaded and its contents returned.""" + mapping = { + "metadata": { + "generated_at": "2026-03-16T04:00:00Z", + "commit": "abc123", + "test_file_count": 2, + "source_file_count": 1, + }, + "source_to_tests": { + "source/isaaclab/isaaclab/utils/math.py": [ + "source/isaaclab/test/utils/test_math.py", + ], + }, + } + mapping_path = tmp_path / "test-dependency-map.json" + mapping_path.write_text(json.dumps(mapping)) + + result = select_tests.load_mapping(str(mapping_path)) + assert result["source_to_tests"]["source/isaaclab/isaaclab/utils/math.py"] == [ + "source/isaaclab/test/utils/test_math.py" + ] + + def test_load_missing_mapping(self, tmp_path): + """A missing mapping file should return None.""" + result = select_tests.load_mapping(str(tmp_path / "nonexistent.json")) + assert result is None + + def test_load_empty_mapping(self, tmp_path): + """An empty mapping file should return None.""" + mapping_path = tmp_path / "test-dependency-map.json" + mapping_path.write_text("{}") + + result = select_tests.load_mapping(str(mapping_path)) + assert result is None + + def test_load_invalid_json(self, tmp_path): + """An invalid JSON file should return None.""" + mapping_path = tmp_path / "test-dependency-map.json" + mapping_path.write_text("not json") + + result = select_tests.load_mapping(str(mapping_path)) + assert result is None + + +class TestClassifyFile: + """Tests for classifying changed files.""" + + def test_test_file_in_subdirectory(self): + """A test file in a test subdirectory should be classified as 'test'.""" + result = select_tests.classify_file("source/isaaclab/test/utils/test_math.py", "M", mapping_keys=set()) + assert result == "test" + + def test_test_file_directly_in_test_dir(self): + """A test file directly in test/ (no subdirectory) should be classified as 'test'.""" + result = select_tests.classify_file("source/isaaclab_tasks/test/test_environments.py", "M", mapping_keys=set()) + assert result == "test" + + def test_test_file_under_scripts(self): + """A test file under scripts/ should be classified as 'test'.""" + result = select_tests.classify_file("scripts/tools/test/test_something.py", "M", mapping_keys=set()) + assert result == "test" + + def test_mapped_source_file(self): + """A Python source file that exists in the mapping should be classified as 'mapped'.""" + keys = {"source/isaaclab/isaaclab/utils/math.py"} + result = select_tests.classify_file("source/isaaclab/isaaclab/utils/math.py", "M", mapping_keys=keys) + assert result == "mapped" + + def test_unmapped_source_file(self): + """A Python source file NOT in the mapping should be classified as 'unmapped'.""" + result = select_tests.classify_file("source/isaaclab/isaaclab/new_module.py", "M", mapping_keys=set()) + assert result == "unmapped" + + def test_non_python_file_under_source(self): + """A non-Python file under source/ should be classified as 'non_python_source'.""" + result = select_tests.classify_file("source/isaaclab/test/utils/test_config.yaml", "M", mapping_keys=set()) + assert result == "non_python_source" + + def test_infrastructure_conftest(self): + """tools/conftest.py should be classified as 'infrastructure'.""" + result = select_tests.classify_file("tools/conftest.py", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_infrastructure_test_settings(self): + """tools/test_settings.py should be classified as 'infrastructure'.""" + result = select_tests.classify_file("tools/test_settings.py", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_infrastructure_root_pyproject(self): + """Root pyproject.toml should be classified as 'infrastructure'.""" + result = select_tests.classify_file("pyproject.toml", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_infrastructure_setup_py(self): + """A setup.py should be classified as 'infrastructure'.""" + result = select_tests.classify_file("source/isaaclab/setup.py", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_infrastructure_extension_toml(self): + """An extension.toml in config/ should be classified as 'infrastructure'.""" + result = select_tests.classify_file("source/isaaclab/config/extension.toml", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_ci_tooling_select_tests(self): + """tools/select_tests.py should be classified as 'infrastructure'.""" + result = select_tests.classify_file("tools/select_tests.py", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_ci_tooling_collect_coverage(self): + """tools/collect_coverage_map.py should be classified as 'infrastructure'.""" + result = select_tests.classify_file("tools/collect_coverage_map.py", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_apps_file(self): + """A file under apps/ should be classified as 'apps'.""" + result = select_tests.classify_file("apps/something.py", "M", mapping_keys=set()) + assert result == "apps" + + def test_docs_file_ignored(self): + """A docs file should be classified as 'ignore'.""" + result = select_tests.classify_file("docs/README.md", "M", mapping_keys=set()) + assert result == "ignore" + + def test_github_file_is_infrastructure(self): + """A .github/ file should be classified as 'infrastructure'.""" + result = select_tests.classify_file(".github/workflows/build.yaml", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_docker_file_is_infrastructure(self): + """A docker/ file should be classified as 'infrastructure'.""" + result = select_tests.classify_file("docker/Dockerfile.base", "M", mapping_keys=set()) + assert result == "infrastructure" + + def test_deleted_source_file(self): + """A deleted Python source file should be classified as 'deleted_source'.""" + result = select_tests.classify_file("source/isaaclab/isaaclab/old.py", "D", mapping_keys=set()) + assert result == "deleted_source" + + def test_deleted_non_source_file_ignored(self): + """A deleted docs file should be classified as 'ignore'.""" + result = select_tests.classify_file("docs/old.md", "D", mapping_keys=set()) + assert result == "ignore" + + def test_renamed_file(self): + """A renamed file should be classified as 'renamed'.""" + result = select_tests.classify_file("source/isaaclab/isaaclab/new_name.py", "R", mapping_keys=set()) + assert result == "renamed" + + +class TestStalenessCheck: + """Tests for mapping staleness detection.""" + + def test_fresh_mapping_is_not_stale(self): + """A mapping generated just now should not be stale.""" + now = datetime.now(timezone.utc).isoformat() + metadata = {"generated_at": now} + assert select_tests.is_mapping_stale(metadata, max_age_days=7) is False + + def test_old_mapping_is_stale(self): + """A mapping older than max_age_days should be stale.""" + old = (datetime.now(timezone.utc) - timedelta(days=10)).isoformat() + metadata = {"generated_at": old} + assert select_tests.is_mapping_stale(metadata, max_age_days=7) is True + + def test_missing_timestamp_is_stale(self): + """A mapping without generated_at should be considered stale.""" + assert select_tests.is_mapping_stale({}, max_age_days=7) is True + + +class TestAssignTestToJob: + """Tests for assigning test files to CI jobs.""" + + def test_physx_test(self): + """A test under isaaclab_physx should go to test-physx.""" + assert select_tests.assign_test_to_job("source/isaaclab_physx/test/assets/test_articulation.py") == "test-physx" + + def test_newton_test(self): + """A test under isaaclab_newton should go to test-newton.""" + assert ( + select_tests.assign_test_to_job("source/isaaclab_newton/test/assets/test_articulation.py") == "test-newton" + ) + + def test_general_test(self): + """A test under isaaclab (core) should go to test-general.""" + assert select_tests.assign_test_to_job("source/isaaclab/test/utils/test_math.py") == "test-general" + + def test_tasks_1_test(self): + """A tasks test in the first split should go to test-isaaclab-tasks.""" + assert ( + select_tests.assign_test_to_job("source/isaaclab_tasks/test/test_environments.py") == "test-isaaclab-tasks" + ) + + def test_tasks_2_test(self): + """A tasks test in the second split should go to test-isaaclab-tasks-2.""" + assert ( + select_tests.assign_test_to_job("source/isaaclab_tasks/test/test_environment_determinism.py") + == "test-isaaclab-tasks-2" + ) + + def test_environments_training(self): + """test_environments_training.py should go to test-environments-training.""" + assert ( + select_tests.assign_test_to_job("source/isaaclab_tasks/test/test_environments_training.py") + == "test-environments-training" + ) + + def test_flaky_test(self): + """A test in FLAKY_TESTS should go to test-flaky.""" + assert select_tests.assign_test_to_job("source/isaaclab/test/test_logger.py") == "test-flaky" + + def test_slightly_flaky_test(self): + """A test in SLIGHTLY_FLAKY_TESTS should go to test-slightly-flaky.""" + assert ( + select_tests.assign_test_to_job("source/isaaclab_physx/test/assets/test_surface_gripper.py") + == "test-slightly-flaky" + ) + + def test_curobo_test(self): + """A test in CUROBO_TESTS should go to test-curobo.""" + assert ( + select_tests.assign_test_to_job("source/isaaclab_tasks/test/test_environments_skillgen.py") == "test-curobo" + ) + + def test_duplicate_basename_different_packages(self): + """Duplicate basenames in different packages should go to different jobs.""" + physx_job = select_tests.assign_test_to_job("source/isaaclab_physx/test/assets/test_articulation.py") + newton_job = select_tests.assign_test_to_job("source/isaaclab_newton/test/assets/test_articulation.py") + assert physx_job == "test-physx" + assert newton_job == "test-newton" + + +class TestSelectTests: + """End-to-end tests for the select_tests function.""" + + def _make_mapping(self, tmp_path, source_to_tests, age_days=0): + """Helper to create a mapping file with given content and age.""" + generated = (datetime.now(timezone.utc) - timedelta(days=age_days)).isoformat() + mapping = { + "metadata": { + "generated_at": generated, + "commit": "abc123", + "test_file_count": sum(len(v) for v in source_to_tests.values()), + "source_file_count": len(source_to_tests), + }, + "source_to_tests": source_to_tests, + } + path = tmp_path / "map.json" + path.write_text(json.dumps(mapping)) + return str(path) + + def test_mapped_source_selects_tests(self, tmp_path): + """Changing a mapped source file should select its tests.""" + mapping_path = self._make_mapping( + tmp_path, + { + "source/isaaclab/isaaclab/utils/math.py": [ + "source/isaaclab/test/utils/test_math.py", + "source/isaaclab_physx/test/assets/test_articulation.py", + ], + }, + ) + changed = [("M", "source/isaaclab/isaaclab/utils/math.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is False + assert "source/isaaclab/test/utils/test_math.py" in result["jobs"]["test-general"] + assert "source/isaaclab_physx/test/assets/test_articulation.py" in result["jobs"]["test-physx"] + + def test_unmapped_source_triggers_fallback(self, tmp_path): + """Changing an unmapped source file should trigger run_all.""" + mapping_path = self._make_mapping( + tmp_path, + { + "source/isaaclab/isaaclab/utils/math.py": ["source/isaaclab/test/utils/test_math.py"], + }, + ) + changed = [("M", "source/isaaclab/isaaclab/new_module.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_modified_test_file_is_always_included(self, tmp_path): + """Modifying a test file directly should include it.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "source/isaaclab/test/utils/test_math.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is False + assert "source/isaaclab/test/utils/test_math.py" in result["jobs"]["test-general"] + + def test_non_python_file_triggers_fallback(self, tmp_path): + """Changing a non-Python file under source/ should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "source/isaaclab/test/utils/test_config.yaml")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_docs_only_change(self, tmp_path): + """Changing only docs files should select no tests and not trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "docs/README.md")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is False + assert all(v == "" for v in result["jobs"].values()) + + def test_infrastructure_file_triggers_fallback(self, tmp_path): + """Changing tools/conftest.py should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "tools/conftest.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_deleted_source_triggers_fallback(self, tmp_path): + """Deleting a source file should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("D", "source/isaaclab/isaaclab/old_module.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_renamed_file_triggers_fallback(self, tmp_path): + """Renaming a source file should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("R", "source/isaaclab/isaaclab/renamed.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_stale_mapping_triggers_fallback(self, tmp_path): + """A mapping older than max_age_days should trigger run_all.""" + mapping_path = self._make_mapping( + tmp_path, + {"source/isaaclab/isaaclab/utils/math.py": ["source/isaaclab/test/utils/test_math.py"]}, + age_days=10, + ) + changed = [("M", "source/isaaclab/isaaclab/utils/math.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_missing_mapping_triggers_fallback(self, tmp_path): + """A missing mapping file should trigger run_all.""" + result = select_tests.select_tests(str(tmp_path / "nope.json"), [("M", "source/x.py")], max_age_days=7) + assert result["run_all"] is True + + def test_apps_file_triggers_fallback(self, tmp_path): + """Changing a file under apps/ should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "apps/something.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_mixed_mapped_and_ignored(self, tmp_path): + """Mapped source + ignored docs should select only the mapped tests, no fallback.""" + mapping_path = self._make_mapping( + tmp_path, + { + "source/isaaclab/isaaclab/utils/math.py": ["source/isaaclab/test/utils/test_math.py"], + }, + ) + changed = [ + ("M", "source/isaaclab/isaaclab/utils/math.py"), + ("M", "docs/README.md"), + ] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is False + assert "source/isaaclab/test/utils/test_math.py" in result["jobs"]["test-general"] + + def test_ci_tooling_change_triggers_fallback(self, tmp_path): + """Changing tools/select_tests.py should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "tools/select_tests.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_job_assignment_in_output(self, tmp_path): + """Tests from different packages should appear under their respective jobs.""" + mapping_path = self._make_mapping( + tmp_path, + { + "source/isaaclab/isaaclab/utils/math.py": [ + "source/isaaclab/test/utils/test_math.py", + "source/isaaclab_physx/test/assets/test_rigid_object.py", + "source/isaaclab_newton/test/assets/test_rigid_object.py", + ], + }, + ) + changed = [("M", "source/isaaclab/isaaclab/utils/math.py")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is False + assert "source/isaaclab/test/utils/test_math.py" in result["jobs"]["test-general"] + assert "source/isaaclab_physx/test/assets/test_rigid_object.py" in result["jobs"]["test-physx"] + assert "source/isaaclab_newton/test/assets/test_rigid_object.py" in result["jobs"]["test-newton"] + + def test_github_change_triggers_fallback(self, tmp_path): + """Changing a .github/ file should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", ".github/workflows/build.yaml")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + def test_docker_change_triggers_fallback(self, tmp_path): + """Changing a docker/ file should trigger run_all.""" + mapping_path = self._make_mapping(tmp_path, {}) + changed = [("M", "docker/Dockerfile.base")] + + result = select_tests.select_tests(mapping_path, changed, max_age_days=7) + + assert result["run_all"] is True + + +class TestMatchesIncludeFiles: + """Tests for _matches_include_files in conftest.py.""" + + def test_full_path_match(self): + """A full-path entry should match only that exact path suffix.""" + include = {"source/isaaclab_physx/test/assets/test_articulation.py"} + assert _matches_include_files( + "source/isaaclab_physx/test/assets/test_articulation.py", "test_articulation.py", include + ) + + def test_full_path_no_cross_package_match(self): + """A full-path entry for physx should NOT match newton.""" + include = {"source/isaaclab_physx/test/assets/test_articulation.py"} + assert not _matches_include_files( + "source/isaaclab_newton/test/assets/test_articulation.py", "test_articulation.py", include + ) + + def test_basename_match(self): + """A basename-only entry should match any file with that basename.""" + include = {"test_articulation.py"} + assert _matches_include_files( + "source/isaaclab_physx/test/assets/test_articulation.py", "test_articulation.py", include + ) + assert _matches_include_files( + "source/isaaclab_newton/test/assets/test_articulation.py", "test_articulation.py", include + ) + + def test_no_match(self): + """A file not in the include set should not match.""" + include = {"source/isaaclab_physx/test/assets/test_articulation.py"} + assert not _matches_include_files("source/isaaclab/test/utils/test_math.py", "test_math.py", include) + + def test_empty_include_files(self): + """An empty include set should match nothing.""" + assert not _matches_include_files("source/isaaclab/test/utils/test_math.py", "test_math.py", set()) + + +class TestParseGitDiff: + """Tests for parsing git diff --name-status output.""" + + def test_parse_modified_file(self): + """Standard modified file line should be parsed.""" + lines = ["M\tsource/isaaclab/isaaclab/utils/math.py"] + result = select_tests.parse_git_diff_output(lines) + assert result == [("M", "source/isaaclab/isaaclab/utils/math.py")] + + def test_parse_added_file(self): + """Added file line should be parsed.""" + lines = ["A\tsource/isaaclab/isaaclab/new.py"] + result = select_tests.parse_git_diff_output(lines) + assert result == [("A", "source/isaaclab/isaaclab/new.py")] + + def test_parse_deleted_file(self): + """Deleted file line should be parsed.""" + lines = ["D\tsource/isaaclab/isaaclab/old.py"] + result = select_tests.parse_git_diff_output(lines) + assert result == [("D", "source/isaaclab/isaaclab/old.py")] + + def test_parse_renamed_file(self): + """Renamed file line (Rxx) should be parsed, using the new path.""" + lines = ["R100\tsource/old.py\tsource/new.py"] + result = select_tests.parse_git_diff_output(lines) + assert result == [("R", "source/new.py")] + + def test_parse_multiple_lines(self): + """Multiple lines should all be parsed.""" + lines = [ + "M\tsource/a.py", + "A\tsource/b.py", + "D\tdocs/c.md", + ] + result = select_tests.parse_git_diff_output(lines) + assert len(result) == 3 + + def test_parse_empty_lines_ignored(self): + """Empty lines should be skipped.""" + lines = ["M\tsource/a.py", "", "A\tsource/b.py"] + result = select_tests.parse_git_diff_output(lines) + assert len(result) == 2