From a92fbdf8ed768b21fc9baf2b607155f2e5a1a756 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 3 Aug 2026 09:10:43 +0000 Subject: [PATCH] ci: extract unit and integration test steps into composite actions Moves the test step logic from pr-tests.yml and integration_tests.yml into two composite actions (.github/actions/unit-tests and .github/actions/integration-tests) so that other workflows can run the same steps. The workflows keep the matrices, sharding strategy, and Codecov/artifact uploads. Behavior changes: - The integration-tests action starts the databases itself (first step, so the pulls still overlap with the package builds). - The integration-tests action takes test-shard, tests-only, and test-parallelism as inputs instead of workflow-level env. --- .github/actions/integration-tests/action.yml | 166 +++++++++++++++++++ .github/actions/unit-tests/action.yml | 108 ++++++++++++ .github/workflows/integration_tests.yml | 99 ++--------- .github/workflows/pr-tests.yml | 88 ++-------- 4 files changed, 300 insertions(+), 161 deletions(-) create mode 100644 .github/actions/integration-tests/action.yml create mode 100644 .github/actions/unit-tests/action.yml diff --git a/.github/actions/integration-tests/action.yml b/.github/actions/integration-tests/action.yml new file mode 100644 index 0000000000..cde0982032 --- /dev/null +++ b/.github/actions/integration-tests/action.yml @@ -0,0 +1,166 @@ +name: "Integration Tests" +description: | + Runs the infra integration suite on one configuration: starts the + databases, builds the packages, initializes the host, builds the sandbox + template, starts the services, and runs make test-integration. + + Matrix definitions, sharding strategy, and result/log uploads stay in the + calling workflows (.github/workflows/integration_tests.yml). The workspace + root must have this repository's layout. The logic lives in a composite + action (instead of inline in the workflow) so that other workflows can run + the same steps. + +inputs: + compress_enabled: + description: "Enable compression (true/false)" + required: false + default: "false" + compress_type: + description: "Compression type (zstd, lz4)" + required: false + default: "" + compress_level: + description: "Compression level (zstd: 1=fastest, 2=default; lz4: 0)" + required: false + default: "" + compress_workers: + description: "Number of frame encode workers" + required: false + default: "" + dedup_mode: + description: "Memfile dedup mode" + required: false + default: "default" + disable_memfd: + description: "Disable memfd-backed guest memory" + required: false + default: "false" + test-shard: + description: | + Test shard to run, resolved by TEST_SHARD in tests/integration/Makefile. + "all" runs the suite unsharded. + required: false + default: "all" + tests-only: + description: | + Path to a test allow-list file (for example + scripts/compression-tests.tsv). Empty runs the whole suite. + required: false + default: "" + test-parallelism: + description: | + go test -parallel value. Keep at 4: the host is the bottleneck, not + client-side concurrency. Measured on the zstd1 config: at 8 the + per-test times inflated 70-140% and template builds flaked on + timeouts; at 12 concurrent FC builds starved envd inits. Wall time is + cut by sharding across runners instead. + required: false + default: "4" + orchestrator-make-path: + description: | + Directory whose Makefile starts the orchestrator (run-debug target). + Lets callers run the suite against a compatible orchestrator build + from a different directory. Requires the same input on the + start-services action. + required: false + default: "packages/orchestrator" + +runs: + using: "composite" + steps: + # Kick off the database containers first: their image pulls and boots + # run in the background and overlap with the package builds below. + - name: Start Databases + uses: ./.github/actions/start-databases + + # The compressed configs only run what the allow-list names, so verify + # its stale-entry guard still bites before depending on it. + - name: Check Test Allow-list + shell: bash + run: make -C tests/integration check-tests-allowlist + + - name: Build Packages + uses: ./.github/actions/build-packages + + # Warm the alternate orchestrator build so the run-debug rebuild inside + # start-services finishes within the health-check timeout. + - name: Build alternate orchestrator + if: inputs.orchestrator-make-path != 'packages/orchestrator' + shell: bash + run: make -C "${{ inputs.orchestrator-make-path }}" build-debug + + - name: Initialize Host + uses: ./.github/actions/host-init + + # Must run before Start Services: create-build manages its own sandbox + # network state (slot pools, namespaces) and conflicts with a running + # orchestrator doing the same - overlapping them wedges the + # provisioning VM until its deadline. + - name: Build Template + uses: ./.github/actions/build-sandbox-template + with: + compress_enabled: ${{ inputs.compress_enabled }} + compress_type: ${{ inputs.compress_type }} + compress_level: ${{ inputs.compress_level }} + compress_workers: ${{ inputs.compress_workers }} + + - name: Start Services + uses: ./.github/actions/start-services + with: + compress_enabled: ${{ inputs.compress_enabled }} + compress_type: ${{ inputs.compress_type }} + compress_level: ${{ inputs.compress_level }} + compress_workers: ${{ inputs.compress_workers }} + dedup_mode: ${{ inputs.dedup_mode }} + disable_memfd: ${{ inputs.disable_memfd }} + orchestrator-make-path: ${{ inputs.orchestrator-make-path }} + + # Unknown action inputs only produce a warning. If start-services does + # not support orchestrator-make-path yet, it silently starts the default + # orchestrator - catch that here instead of green-lighting wrong results. + - name: Verify the alternate orchestrator is running + if: inputs.orchestrator-make-path != 'packages/orchestrator' + shell: bash + run: | + if ! sudo pgrep -af "${{ inputs.orchestrator-make-path }}/bin/"; then + echo "::error::No orchestrator process from ${{ inputs.orchestrator-make-path }} is running." + echo "::error::The start-services action probably ignored orchestrator-make-path (version without this input)." + exit 1 + fi + + - name: Run Integration Tests + shell: bash + env: + TESTS_API_SERVER_URL: "http://localhost:3000" + TESTS_API_INTERNAL_GRPC_ADDRESS: "localhost:5009" + TESTS_ORCHESTRATOR_HOST: "localhost:5008" + TESTS_ENVD_PROXY: "http://localhost:3002" + TESTS_CLIENT_PROXY: "http://localhost:3002" + TESTS_ONLY: ${{ inputs.tests-only }} + TEST_PARALLELISM: ${{ inputs.test-parallelism }} + TEST_SHARD: ${{ inputs.test-shard }} + run: | + make test-integration + + - name: Check for Data Races in Service Logs + if: always() + shell: bash + run: | + echo "Checking service logs for data races..." + RACE_FOUND=0 + + for log_file in ~/logs/*.log; do + if [ -f "$log_file" ]; then + if grep -q "WARNING: DATA RACE" "$log_file"; then + echo "::error::Data race detected in $(basename "$log_file")" + grep -A 30 "WARNING: DATA RACE" "$log_file" | head -200 + RACE_FOUND=1 + fi + fi + done + + if [ "$RACE_FOUND" -eq 1 ]; then + echo "::error::Data races were detected in service logs. See above for details." + exit 1 + fi + echo "No data races detected in service logs." diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml new file mode 100644 index 0000000000..72ba73f0af --- /dev/null +++ b/.github/actions/unit-tests/action.yml @@ -0,0 +1,108 @@ +name: "Unit Tests" +description: | + Runs the unit test shard for one infra package, including the host setup + that the package needs (bindfs for envd, NBD/hugepages/uffd for the + orchestrator). Always writes coverage.txt and junit.xml into the package + directory; callers decide whether to upload them. + + The workspace root must have this repository's layout. The logic lives in + a composite action (instead of inline in pr-tests.yml) so that other + workflows can run the same steps. + +inputs: + package: + description: "Package directory to test, for example packages/api" + required: true + test-path: + description: "Go test path pattern inside the package" + required: false + default: "./..." + sudo: + description: "Run tests under sudo (needed by envd and orchestrator)" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Setup Go + uses: ./.github/actions/go-setup-cache + with: + cache-dependency-paths: | + go.work + ${{ inputs.package }}/go.mod + ${{ inputs.package }}/go.sum + + - name: Reap orphaned Testcontainers from previous runs + shell: bash + # Persistent runner + Ryuk disabled: prune leftover containers + # from SIGKILL/timeout/OOM in prior jobs. + # Two `docker ps` calls because different filter keys AND inside + # one call. Don't anchor `name=^reaper_` — Docker silently drops + # anchored patterns (docker/cli#1201). + run: | + set -euo pipefail + ids=$( + { + docker ps -aq --filter "label=org.testcontainers=true" + docker ps -aq --filter "name=reaper_" + } | sort -u + ) + if [ -n "${ids}" ]; then + # shellcheck disable=SC2086 + docker rm -f ${ids} + fi + + - name: Setup envd tests + if: inputs.package == 'packages/envd' + shell: bash + run: | + # Install bindfs for FUSE mount tests + sudo apt-get update && sudo apt-get install -y bindfs + + - name: Setup orchestrator tests + if: inputs.package == 'packages/orchestrator' + shell: bash + run: | + make -C packages/orchestrator fetch-busybox + echo "HOST_BUSYBOX_DIR=$(pwd)/packages/orchestrator/.busybox" >> "$GITHUB_ENV" + + # Enable unprivileged uffd mode + echo 1 | sudo tee /proc/sys/vm/unprivileged_userfaultfd + + # Enable hugepages + sudo mkdir -p /mnt/hugepages + sudo mount -t hugetlbfs none /mnt/hugepages + echo 2000 | sudo tee /proc/sys/vm/nr_hugepages + + # Enable NBD + sudo modprobe nbd nbds_max=256 + + # Disable inotify watching of change events for NBD devices + echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules + sudo udevadm control --reload-rules + sudo udevadm trigger + + - name: Install gotestsum + shell: bash + run: go install gotest.tools/gotestsum@v1.13.0 + + # sudo strips PATH; pass it through so gotestsum/go resolve to the + # runner's toolchain instead of /usr/bin/go. Trap chowns reports back to + # the runner so uploads can read them even when tests fail. + - name: Run tests that require sudo + if: inputs.sudo == 'true' + shell: bash + working-directory: ${{ inputs.package }} + run: | + trap 'sudo chown "$(id -u):$(id -g)" coverage.txt junit.xml 2>/dev/null || true' EXIT + sudo -E env "PATH=$PATH" gotestsum --junitfile=junit.xml --format=standard-verbose \ + -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ inputs.test-path }} + + - name: Run tests + if: inputs.sudo != 'true' + shell: bash + working-directory: ${{ inputs.package }} + run: | + gotestsum --junitfile=junit.xml --format=standard-verbose \ + -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ inputs.test-path }} diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 14198a2e45..6b081e7659 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,3 +1,9 @@ +# Runs the integration test suite. +# +# The suite flow lives in the composite action +# .github/actions/integration-tests; only the matrix/sharding strategy and +# the result uploads live here. + name: Integration Tests on: @@ -81,37 +87,8 @@ jobs: - name: Checkout Code uses: actions/checkout@v5 - # Kick off the database containers first: their image pulls and boots - # run in the background and overlap with the package builds below. - - name: Start Databases - uses: ./.github/actions/start-databases - - # The compressed configs only run what the allow-list names, so verify - # its stale-entry guard still bites before depending on it. - - name: Check Test Allow-list - run: make -C tests/integration check-tests-allowlist - shell: bash - - - name: Build Packages - uses: ./.github/actions/build-packages - - - name: Initialize Host - uses: ./.github/actions/host-init - - # Must run before Start Services: create-build manages its own sandbox - # network state (slot pools, namespaces) and conflicts with a running - # orchestrator doing the same - overlapping them wedges the - # provisioning VM until its deadline. - - name: Build Template - uses: ./.github/actions/build-sandbox-template - with: - compress_enabled: ${{ matrix.compress_enabled }} - compress_type: ${{ matrix.compress_type }} - compress_level: ${{ matrix.compress_level }} - compress_workers: ${{ matrix.compress_workers }} - - - name: Start Services - uses: ./.github/actions/start-services + - name: Run integration tests + uses: ./.github/actions/integration-tests with: compress_enabled: ${{ matrix.compress_enabled }} compress_type: ${{ matrix.compress_type }} @@ -119,60 +96,12 @@ jobs: compress_workers: ${{ matrix.compress_workers }} dedup_mode: ${{ matrix.dedup_mode }} disable_memfd: ${{ matrix.disable_memfd }} - - - name: Run Integration Tests - env: - TESTS_API_SERVER_URL: "http://localhost:3000" - TESTS_API_INTERNAL_GRPC_ADDRESS: "localhost:5009" - TESTS_ORCHESTRATOR_HOST: "localhost:5008" - TESTS_ENVD_PROXY: "http://localhost:3002" - TESTS_CLIENT_PROXY: "http://localhost:3002" - # The uncompressed config runs the whole suite. The compressed configs - # only re-run the tests whose subject is writing or reading back a - # snapshot, which is all the compression knobs can affect — the - # allow-list documents the criteria and the code paths involved. - TESTS_ONLY: ${{ matrix.compress_enabled == 'true' && 'scripts/compression-tests.tsv' || '' }} - # Keep go test -parallel at 4: the host is the bottleneck, not - # client-side concurrency. Measured on the zstd1 config: at 8 the - # per-test times inflated 70-140% (net wall win of only ~1 min) and - # 10 template builds flaked on timeouts; at 12 concurrent FC builds - # (x8 zstd workers each) starved envd inits into "syncing took too - # long" failures across 87 tests. Wall time is cut by sharding - # across runners instead (see the matrix above). - TEST_PARALLELISM: "4" - TEST_SHARD: ${{ matrix.shard }} - run: | - # Run the integration tests - make test-integration - - - name: Check for Data Races in Service Logs - if: always() - run: | - echo "Checking service logs for data races..." - RACE_FOUND=0 - - for log_file in ~/logs/*.log; do - if [ -f "$log_file" ]; then - if grep -q "WARNING: DATA RACE" "$log_file"; then - echo "::error::Data race detected in $(basename "$log_file")" - echo "==========================================" - echo "DATA RACES IN: $log_file" - echo "==========================================" - # Extract and print all data race blocks - grep -A 30 "WARNING: DATA RACE" "$log_file" | head -200 - echo "" - RACE_FOUND=1 - fi - fi - done - - if [ "$RACE_FOUND" -eq 1 ]; then - echo "::error::Data races were detected in service logs. See above for details." - exit 1 - else - echo "No data races detected in service logs." - fi - shell: bash + test-shard: ${{ matrix.shard }} + # The uncompressed config runs the whole suite. The compressed + # configs only re-run the tests whose subject is writing or reading + # back a snapshot, which is all the compression knobs can affect — + # the allow-list documents the criteria and the code paths involved. + tests-only: ${{ matrix.compress_enabled == 'true' && 'scripts/compression-tests.tsv' || '' }} - name: Upload Test Results if: ${{ always() && inputs.publish == true }} diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 395e7a44c7..ff6139cba0 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -1,3 +1,8 @@ +# Runs the unit test shards on PRs. +# +# The test logic lives in the composite action .github/actions/unit-tests; +# only the matrix and the Codecov uploads live here. + name: Run tests on PRs on: @@ -25,7 +30,8 @@ jobs: GIN_MODE: test # Parallel `go test ./...` binaries derive the same Ryuk session id # and race on `reaper_` creation. Disable Ryuk; t.Cleanup - # handles normal exit, the prune step below handles abnormal exit. + # handles normal exit, the reap step in the unit-tests action handles + # abnormal exit. TESTCONTAINERS_RYUK_DISABLED: "true" # Surfaced as env so upload steps can gate on presence (skipped on fork PRs). CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -75,82 +81,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 - - name: Setup Go - uses: ./.github/actions/go-setup-cache + - name: Run unit tests + uses: ./.github/actions/unit-tests with: - cache-dependency-paths: | - go.work - ${{ matrix.package }}/go.mod - ${{ matrix.package }}/go.sum - - - name: Reap orphaned Testcontainers from previous runs - # Persistent runner + Ryuk disabled: prune leftover containers - # from SIGKILL/timeout/OOM in prior jobs. - # Two `docker ps` calls because different filter keys AND inside - # one call. Don't anchor `name=^reaper_` — Docker silently drops - # anchored patterns (docker/cli#1201). - run: | - set -euo pipefail - ids=$( - { - docker ps -aq --filter "label=org.testcontainers=true" - docker ps -aq --filter "name=reaper_" - } | sort -u - ) - if [ -n "${ids}" ]; then - # shellcheck disable=SC2086 - docker rm -f ${ids} - fi - - - name: Setup envd tests - run: | - # Install bindfs for FUSE mount tests - sudo apt-get update && sudo apt-get install -y bindfs - if: matrix.package == 'packages/envd' - - - name: Setup orchestrator tests - run: | - make -C packages/orchestrator fetch-busybox - echo "HOST_BUSYBOX_DIR=$(pwd)/packages/orchestrator/.busybox" >> "$GITHUB_ENV" - - # Enable unprivileged uffd mode - echo 1 | sudo tee /proc/sys/vm/unprivileged_userfaultfd - - # Enable hugepages - sudo mkdir -p /mnt/hugepages - sudo mount -t hugetlbfs none /mnt/hugepages - echo 2000 | sudo tee /proc/sys/vm/nr_hugepages - - # Enable NBD - sudo modprobe nbd nbds_max=256 - - # Disable inotify watching of change events for NBD devices - echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules - sudo udevadm control --reload-rules - sudo udevadm trigger - - if: matrix.package == 'packages/orchestrator' - - - name: Install gotestsum - run: go install gotest.tools/gotestsum@v1.13.0 - - # sudo strips PATH; pass it through so gotestsum/go resolve to the runner's - # toolchain (1.26.5) instead of /usr/bin/go. Trap chowns reports back to - # the runner so uploads can read them even when tests fail. - - name: Run tests that require sudo - working-directory: ${{ matrix.package }} - run: | - trap 'sudo chown "$(id -u):$(id -g)" coverage.txt junit.xml 2>/dev/null || true' EXIT - sudo -E env "PATH=$PATH" gotestsum --junitfile=junit.xml --format=standard-verbose \ - -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ matrix.test_path }} - if: matrix.sudo == true - - - name: Run tests - working-directory: ${{ matrix.package }} - run: | - gotestsum --junitfile=junit.xml --format=standard-verbose \ - -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ matrix.test_path }} - if: matrix.sudo == false + package: ${{ matrix.package }} + test-path: ${{ matrix.test_path }} + sudo: ${{ matrix.sudo }} - name: Upload coverage to Codecov if: ${{ !cancelled() && env.CODECOV_TOKEN != '' &&