conformance.yml

name: Conformance

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    - cron: '0 6 * * 0' # weekly Sunday 6am UTC
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_TERM_COLOR: always

jobs:
  changes:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      code: ${{ steps.detect.outputs.code || 'true' }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - if: github.event_name == 'pull_request'
        id: detect
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          predicate-quantifier: 'every'
          filters: |
            code:
              - '**'
              - '!**/*.md'
              - '!**/*.txt'
              - '!**/*.html'
              - '!docs/**'
              - '!website/**'
              - '!LICENSE'
              - '!.gitignore'

  # Compile the (large) conformance integration-test crate ONCE and ship the
  # prebuilt binaries as a nextest archive. Previously every conformance run
  # built and executed all tests serially in a single 2-core job (~69 min of
  # `nextest run`). Now the build happens here and the run fans out across the
  # `conformance` matrix below, each partition skipping compilation entirely.
  conformance-build:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 50
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      # The conformance harness boots this binary per test via its
      # workspace-relative path (crates/fakecloud-testkit/src/lib.rs), which
      # resolves identically on every GitHub-hosted runner, so a downloaded
      # binary works regardless of where the archived test binaries extract.
      - name: Build fakecloud
        run: cargo build --bin fakecloud

      - name: Upload fakecloud binary
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: fakecloud-binary-conformance
          path: target/debug/fakecloud
          if-no-files-found: error
          retention-days: 1

      # CARGO_PROFILE_TEST_DEBUG=0 strips debuginfo from the archived binaries,
      # shrinking this compile, the archive upload, and every partition's archive
      # download (all on the workflow's critical path). Assertion-failure
      # locations are preserved via std's #[track_caller], so CI failure output
      # still points at the test line.
      - name: Archive conformance test binaries
        env:
          CARGO_PROFILE_TEST_DEBUG: "0"
        run: cargo nextest archive -P ci -p fakecloud-conformance --archive-file conformance-archive.tar.zst

      - name: Upload conformance test archive
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: fakecloud-conformance-archive
          path: conformance-archive.tar.zst
          if-no-files-found: error
          retention-days: 1

  # Run the prebuilt integration tests across N hash partitions. hash:i/N is a
  # complete, disjoint partition of the test set, so coverage is exact by
  # construction (no separate coverage guard needed, unlike the heterogeneous
  # E2E matrix). Conformance tests are API-level and tolerate a missing Docker
  # daemon (see lambda_invoke), so no container setup is required here.
  conformance:
    name: Conformance ${{ matrix.partition }}
    needs: [changes, conformance-build]
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        partition: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6']
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      # No free-disk and no rust-cache here: this job runs prebuilt test binaries
      # from the archive and never compiles. It only needs the ~hundred-MB archive
      # plus a short-lived fakecloud per test, which fit easily in the runner's
      # ~117G free root, so the free-disk reclaim (a variable 2-4 min) is pure
      # overhead on the workflow's critical path.
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest

      - name: Download fakecloud binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fakecloud-binary-conformance
          path: target/debug

      - name: Mark fakecloud binary executable
        run: chmod +x target/debug/fakecloud

      - name: Download conformance test archive
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fakecloud-conformance-archive

      - name: Disk diagnostics before nextest
        run: df -h

      # Run prebuilt binaries straight from the archive -- no compilation.
      # --workspace-remap . points runtime workspace lookups at this checkout
      # (path matches the build runner on GitHub-hosted runners). nextest extracts
      # to its own temp dir under /tmp (ample free space on the root fs here).
      - name: Run conformance partition
        run: |
          cargo nextest run -P ci \
            --archive-file conformance-archive.tar.zst \
            --workspace-remap . \
            --partition "hash:${{ matrix.partition }}"

      - name: Disk diagnostics after nextest
        if: always()
        run: df -h

  # The coverage gate: boots fakecloud and probes every operation to confirm no
  # conformance regression, then audits handwritten-test coverage. It spawns its
  # own short-lived server and does NOT need the integration-test archive, so it
  # runs in parallel with the matrix as a small standalone job.
  conformance-check:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Build server
        run: cargo build --bin fakecloud

      - name: Build conformance tool
        run: cargo build -p fakecloud-conformance

      - name: Check conformance (fails if coverage drops)
        id: check
        run: |
          set -o pipefail
          cargo run -p fakecloud-conformance -- check \
            --json-out conformance-report.json \
            --markdown-summary-out conformance-summary.md \
            2>&1 | tee conformance-report.txt

      - name: Audit handwritten tests (fails if implemented action lacks a test)
        if: always() && steps.check.outcome != 'skipped'
        run: cargo run -p fakecloud-conformance -- audit

      - name: Check vendored Smithy models match aws-models
        if: always() && steps.check.outcome != 'skipped'
        run: cargo run -p fakecloud-conformance -- vendored-models

      - name: Post summary
        if: always()
        run: |
          if [ -f conformance-summary.md ]; then
            cat conformance-summary.md >> $GITHUB_STEP_SUMMARY
          fi

      - name: Upload report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: conformance-report
          path: |
            conformance-report.txt
            conformance-report.json
            conformance-summary.md
          retention-days: 90