ci: extract unit and integration test steps into composite actions

.github/actions/integration-tests/action.yml

@@ -0,0 +1,166 @@
+name: "Integration Tests"
+description: |
+  Runs the infra integration suite on one configuration: starts the
+  databases, builds the packages, initializes the host, builds the sandbox
+  template, starts the services, and runs make test-integration.
+
+  Matrix definitions, sharding strategy, and result/log uploads stay in the
+  calling workflows (.github/workflows/integration_tests.yml). The workspace
+  root must have this repository's layout. The logic lives in a composite
+  action (instead of inline in the workflow) so that other workflows can run
+  the same steps.
+
+inputs:
+  compress_enabled:
+    description: "Enable compression (true/false)"
+    required: false
+    default: "false"
+  compress_type:
+    description: "Compression type (zstd, lz4)"
+    required: false
+    default: ""
+  compress_level:
+    description: "Compression level (zstd: 1=fastest, 2=default; lz4: 0)"
+    required: false
+    default: ""
+  compress_workers:
+    description: "Number of frame encode workers"
+    required: false
+    default: ""
+  dedup_mode:
+    description: "Memfile dedup mode"
+    required: false
+    default: "default"
+  disable_memfd:
+    description: "Disable memfd-backed guest memory"
+    required: false
+    default: "false"
+  test-shard:
+    description: |
+      Test shard to run, resolved by TEST_SHARD in tests/integration/Makefile.
+      "all" runs the suite unsharded.
+    required: false
+    default: "all"
+  tests-only:
+    description: |
+      Path to a test allow-list file (for example
+      scripts/compression-tests.tsv). Empty runs the whole suite.
+    required: false
+    default: ""
+  test-parallelism:
+    description: |
+      go test -parallel value. Keep at 4: the host is the bottleneck, not
+      client-side concurrency. Measured on the zstd1 config: at 8 the
+      per-test times inflated 70-140% and template builds flaked on
+      timeouts; at 12 concurrent FC builds starved envd inits. Wall time is
+      cut by sharding across runners instead.
+    required: false
+    default: "4"
+  orchestrator-make-path:
+    description: |
+      Directory whose Makefile starts the orchestrator (run-debug target).
+      Lets callers run the suite against a compatible orchestrator build
+      from a different directory. Requires the same input on the
+      start-services action.
+    required: false
+    default: "packages/orchestrator"
+
+runs:
+  using: "composite"
+  steps:
+    # Kick off the database containers first: their image pulls and boots
+    # run in the background and overlap with the package builds below.
+    - name: Start Databases
+      uses: ./.github/actions/start-databases
+
+    # The compressed configs only run what the allow-list names, so verify
+    # its stale-entry guard still bites before depending on it.
+    - name: Check Test Allow-list
+      shell: bash
+      run: make -C tests/integration check-tests-allowlist
+
+    - name: Build Packages
+      uses: ./.github/actions/build-packages
+
+    # Warm the alternate orchestrator build so the run-debug rebuild inside
+    # start-services finishes within the health-check timeout.
+    - name: Build alternate orchestrator
+      if: inputs.orchestrator-make-path != 'packages/orchestrator'
+      shell: bash
+      run: make -C "${{ inputs.orchestrator-make-path }}" build-debug
+
+    - name: Initialize Host
+      uses: ./.github/actions/host-init
+
+    # Must run before Start Services: create-build manages its own sandbox
+    # network state (slot pools, namespaces) and conflicts with a running
+    # orchestrator doing the same - overlapping them wedges the
+    # provisioning VM until its deadline.
+    - name: Build Template
+      uses: ./.github/actions/build-sandbox-template
+      with:
+        compress_enabled: ${{ inputs.compress_enabled }}
+        compress_type: ${{ inputs.compress_type }}
+        compress_level: ${{ inputs.compress_level }}
+        compress_workers: ${{ inputs.compress_workers }}
+
+    - name: Start Services
+      uses: ./.github/actions/start-services
+      with:
+        compress_enabled: ${{ inputs.compress_enabled }}
+        compress_type: ${{ inputs.compress_type }}
+        compress_level: ${{ inputs.compress_level }}
+        compress_workers: ${{ inputs.compress_workers }}
+        dedup_mode: ${{ inputs.dedup_mode }}
+        disable_memfd: ${{ inputs.disable_memfd }}
+        orchestrator-make-path: ${{ inputs.orchestrator-make-path }}
+
+    # Unknown action inputs only produce a warning. If start-services does
+    # not support orchestrator-make-path yet, it silently starts the default
+    # orchestrator - catch that here instead of green-lighting wrong results.
+    - name: Verify the alternate orchestrator is running
+      if: inputs.orchestrator-make-path != 'packages/orchestrator'
+      shell: bash
+      run: |
+        if ! sudo pgrep -af "${{ inputs.orchestrator-make-path }}/bin/"; then
+          echo "::error::No orchestrator process from ${{ inputs.orchestrator-make-path }} is running."
+          echo "::error::The start-services action probably ignored orchestrator-make-path (version without this input)."
+          exit 1
+        fi
+
+    - name: Run Integration Tests
+      shell: bash
+      env:
+        TESTS_API_SERVER_URL: "http://localhost:3000"
+        TESTS_API_INTERNAL_GRPC_ADDRESS: "localhost:5009"
+        TESTS_ORCHESTRATOR_HOST: "localhost:5008"
+        TESTS_ENVD_PROXY: "http://localhost:3002"
+        TESTS_CLIENT_PROXY: "http://localhost:3002"
+        TESTS_ONLY: ${{ inputs.tests-only }}
+        TEST_PARALLELISM: ${{ inputs.test-parallelism }}
+        TEST_SHARD: ${{ inputs.test-shard }}
+      run: |
+        make test-integration
+
+    - name: Check for Data Races in Service Logs
+      if: always()
+      shell: bash
+      run: |
+        echo "Checking service logs for data races..."
+        RACE_FOUND=0
+
+        for log_file in ~/logs/*.log; do
+          if [ -f "$log_file" ]; then
+            if grep -q "WARNING: DATA RACE" "$log_file"; then
+              echo "::error::Data race detected in $(basename "$log_file")"
+              grep -A 30 "WARNING: DATA RACE" "$log_file" | head -200
+              RACE_FOUND=1
+            fi
+          fi
+        done
+
+        if [ "$RACE_FOUND" -eq 1 ]; then
+          echo "::error::Data races were detected in service logs. See above for details."
+          exit 1
+        fi
+        echo "No data races detected in service logs."

.github/actions/unit-tests/action.yml

@@ -0,0 +1,108 @@
+name: "Unit Tests"
+description: |
+  Runs the unit test shard for one infra package, including the host setup
+  that the package needs (bindfs for envd, NBD/hugepages/uffd for the
+  orchestrator). Always writes coverage.txt and junit.xml into the package
+  directory; callers decide whether to upload them.
+
+  The workspace root must have this repository's layout. The logic lives in
+  a composite action (instead of inline in pr-tests.yml) so that other
+  workflows can run the same steps.
+
+inputs:
+  package:
+    description: "Package directory to test, for example packages/api"
+    required: true
+  test-path:
+    description: "Go test path pattern inside the package"
+    required: false
+    default: "./..."
+  sudo:
+    description: "Run tests under sudo (needed by envd and orchestrator)"
+    required: false
+    default: "false"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Go
+      uses: ./.github/actions/go-setup-cache
+      with:
+        cache-dependency-paths: |
+          go.work
+          ${{ inputs.package }}/go.mod
+          ${{ inputs.package }}/go.sum
+
+    - name: Reap orphaned Testcontainers from previous runs
+      shell: bash
+      # Persistent runner + Ryuk disabled: prune leftover containers
+      # from SIGKILL/timeout/OOM in prior jobs.
+      # Two `docker ps` calls because different filter keys AND inside
+      # one call. Don't anchor `name=^reaper_` — Docker silently drops
+      # anchored patterns (docker/cli#1201).
+      run: |
+        set -euo pipefail
+        ids=$(
+          {
+            docker ps -aq --filter "label=org.testcontainers=true"
+            docker ps -aq --filter "name=reaper_"
+          } | sort -u
+        )
+        if [ -n "${ids}" ]; then
+          # shellcheck disable=SC2086
+          docker rm -f ${ids}
+        fi
+
+    - name: Setup envd tests
+      if: inputs.package == 'packages/envd'
+      shell: bash
+      run: |
+        # Install bindfs for FUSE mount tests
+        sudo apt-get update && sudo apt-get install -y bindfs
+
+    - name: Setup orchestrator tests
+      if: inputs.package == 'packages/orchestrator'
+      shell: bash
+      run: |
+        make -C packages/orchestrator fetch-busybox
+        echo "HOST_BUSYBOX_DIR=$(pwd)/packages/orchestrator/.busybox" >> "$GITHUB_ENV"
+
+        # Enable unprivileged uffd mode
+        echo 1 | sudo tee /proc/sys/vm/unprivileged_userfaultfd
+
+        # Enable hugepages
+        sudo mkdir -p /mnt/hugepages
+        sudo mount -t hugetlbfs none /mnt/hugepages
+        echo 2000 | sudo tee /proc/sys/vm/nr_hugepages
+
+        # Enable NBD
+        sudo modprobe nbd nbds_max=256
+
+        # Disable inotify watching of change events for NBD devices
+        echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules
+        sudo udevadm control --reload-rules
+        sudo udevadm trigger
+
+    - name: Install gotestsum
+      shell: bash
+      run: go install gotest.tools/gotestsum@v1.13.0
+
+    # sudo strips PATH; pass it through so gotestsum/go resolve to the
+    # runner's toolchain instead of /usr/bin/go. Trap chowns reports back to
+    # the runner so uploads can read them even when tests fail.
+    - name: Run tests that require sudo
+      if: inputs.sudo == 'true'
+      shell: bash
+      working-directory: ${{ inputs.package }}
+      run: |
+        trap 'sudo chown "$(id -u):$(id -g)" coverage.txt junit.xml 2>/dev/null || true' EXIT
+        sudo -E env "PATH=$PATH" gotestsum --junitfile=junit.xml --format=standard-verbose \
+          -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ inputs.test-path }}
+
+    - name: Run tests
+      if: inputs.sudo != 'true'
+      shell: bash
+      working-directory: ${{ inputs.package }}
+      run: |
+        gotestsum --junitfile=junit.xml --format=standard-verbose \
+          -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ inputs.test-path }}

.github/workflows/integration_tests.yml

@@ -1,3 +1,9 @@
+# Runs the integration test suite.
+#
+# The suite flow lives in the composite action
+# .github/actions/integration-tests; only the matrix/sharding strategy and
+# the result uploads live here.
+
 name: Integration Tests
 
 on:
@@ -81,98 +87,21 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v5
 
-      # Kick off the database containers first: their image pulls and boots
-      # run in the background and overlap with the package builds below.
-      - name: Start Databases
-        uses: ./.github/actions/start-databases
-
-      # The compressed configs only run what the allow-list names, so verify
-      # its stale-entry guard still bites before depending on it.
-      - name: Check Test Allow-list
-        run: make -C tests/integration check-tests-allowlist
-        shell: bash
-
-      - name: Build Packages
-        uses: ./.github/actions/build-packages
-
-      - name: Initialize Host
-        uses: ./.github/actions/host-init
-
-      # Must run before Start Services: create-build manages its own sandbox
-      # network state (slot pools, namespaces) and conflicts with a running
-      # orchestrator doing the same - overlapping them wedges the
-      # provisioning VM until its deadline.
-      - name: Build Template
-        uses: ./.github/actions/build-sandbox-template
-        with:
-          compress_enabled: ${{ matrix.compress_enabled }}
-          compress_type: ${{ matrix.compress_type }}
-          compress_level: ${{ matrix.compress_level }}
-          compress_workers: ${{ matrix.compress_workers }}
-
-      - name: Start Services
-        uses: ./.github/actions/start-services
+      - name: Run integration tests
+        uses: ./.github/actions/integration-tests
         with:
           compress_enabled: ${{ matrix.compress_enabled }}
           compress_type: ${{ matrix.compress_type }}
           compress_level: ${{ matrix.compress_level }}
           compress_workers: ${{ matrix.compress_workers }}
           dedup_mode: ${{ matrix.dedup_mode }}
           disable_memfd: ${{ matrix.disable_memfd }}
-
-      - name: Run Integration Tests
-        env:
-          TESTS_API_SERVER_URL: "http://localhost:3000"
-          TESTS_API_INTERNAL_GRPC_ADDRESS: "localhost:5009"
-          TESTS_ORCHESTRATOR_HOST: "localhost:5008"
-          TESTS_ENVD_PROXY: "http://localhost:3002"
-          TESTS_CLIENT_PROXY: "http://localhost:3002"
-          # The uncompressed config runs the whole suite. The compressed configs
-          # only re-run the tests whose subject is writing or reading back a
-          # snapshot, which is all the compression knobs can affect — the
-          # allow-list documents the criteria and the code paths involved.
-          TESTS_ONLY: ${{ matrix.compress_enabled == 'true' && 'scripts/compression-tests.tsv' || '' }}
-          # Keep go test -parallel at 4: the host is the bottleneck, not
-          # client-side concurrency. Measured on the zstd1 config: at 8 the
-          # per-test times inflated 70-140% (net wall win of only ~1 min) and
-          # 10 template builds flaked on timeouts; at 12 concurrent FC builds
-          # (x8 zstd workers each) starved envd inits into "syncing took too
-          # long" failures across 87 tests. Wall time is cut by sharding
-          # across runners instead (see the matrix above).
-          TEST_PARALLELISM: "4"
-          TEST_SHARD: ${{ matrix.shard }}
-        run: |
-          # Run the integration tests
-          make test-integration
-
-      - name: Check for Data Races in Service Logs
-        if: always()
-        run: |
-          echo "Checking service logs for data races..."
-          RACE_FOUND=0
-
-          for log_file in ~/logs/*.log; do
-            if [ -f "$log_file" ]; then
-              if grep -q "WARNING: DATA RACE" "$log_file"; then
-                echo "::error::Data race detected in $(basename "$log_file")"
-                echo "=========================================="
-                echo "DATA RACES IN: $log_file"
-                echo "=========================================="
-                # Extract and print all data race blocks
-                grep -A 30 "WARNING: DATA RACE" "$log_file" | head -200
-                echo ""
-                RACE_FOUND=1
-              fi
-            fi
-          done
-
-          if [ "$RACE_FOUND" -eq 1 ]; then
-            echo "::error::Data races were detected in service logs. See above for details."
-            exit 1
-          else
-            echo "No data races detected in service logs."
-          fi
-        shell: bash
+          test-shard: ${{ matrix.shard }}
+          # The uncompressed config runs the whole suite. The compressed
+          # configs only re-run the tests whose subject is writing or reading
+          # back a snapshot, which is all the compression knobs can affect —
+          # the allow-list documents the criteria and the code paths involved.
+          tests-only: ${{ matrix.compress_enabled == 'true' && 'scripts/compression-tests.tsv' || '' }}
 
       - name: Upload Test Results
         if: ${{ always() && inputs.publish == true }}