scaling.yml

name: Netcode Scaling Benchmark

# Builds every selected netcode once (in parallel), then runs all requested sessions sequentially
# on one prepared fleet. Every case launches fresh players; no measured cases overlap.
#
# Runner budget: the largest session's server + measured + loadgen runners (19 by default).
#
# Fusion's client count is capped by fusion_max_clients (Photon CCU plan). If Photon refuses a
# connection the server proceeds with the clients it has after -connectTimeout, and the summary
# shows the actual count in its "connected at start" table.

on:
  workflow_dispatch:
    inputs:
      netcodes:
        description: "Comma-separated netcodes to run"
        required: true
        default: "purrnet,fishnet,mirror,ngo,fusion"
      sessions:
        description: "Comma-separated sessions as connections@tickHz (a bare number means 20 Hz)"
        required: true
        default: "10@20,100@20,100@60"
      bench_seconds:
        description: "Steady-state measurement window per test, seconds (Idle and Static always use 5)"
        required: true
        default: "10"
      bench_objects:
        description: "Objects spawned per test"
        required: true
        default: "100"
      profiling:
        description: "Development builds (adds a CPU-by-marker table; release builds give cleaner absolute CPU numbers)"
        type: boolean
        required: false
        default: false
      runner:
        description: "runs-on label for build + client jobs (ubuntu-latest = GitHub-hosted, free for public repos; blacksmith-4vcpu-ubuntu-2404 = fixed hardware)"
        required: false
        default: "ubuntu-latest"
      region:
        description: "Photon Cloud region (Fusion only; eu is closest to bench-server)"
        required: false
        default: "eu"
      measured_clients:
        description: "Single-process measured client runners per run (the rest of the connections come from loadgen runners)"
        required: false
        default: "10"
      loadgen_procs:
        description: "Client processes per loadgen runner (3 per vCPU is the tested ratio; ubuntu-latest has 4)"
        required: false
        default: "12"
      loadgen_runner:
        description: "runs-on label for loadgen runners (empty = same as runner)"
        required: false
        default: ""
      server_runner:
        description: "Server runner label (bench-server = dedicated Hetzner box; empty = same as runner)"
        required: false
        default: "bench-server"
      fusion_max_clients:
        description: "Client cap for Fusion (Photon CCU plan limit; if the dedicated server counts as one CCU, set this to plan - 1)"
        required: false
        default: "100"
      max_parallel:
        description: "Compatibility input: must be 1. The reused fleet runs all measurements sequentially."
        required: false
        default: "1"

# The dedicated bench-server takes one session at a time: a second dispatch waits for the running
# one to finish instead of starting alongside it. GitHub keeps one waiting run per group, so a third
# dispatch replaces the waiting one.
concurrency:
  group: netcode-scaling-bench
  cancel-in-progress: false

jobs:
  prep:
    name: Plan runs
    runs-on: ubuntu-latest
    outputs:
      netcode_matrix: ${{ steps.gen.outputs.netcode_matrix }}
      expected: ${{ steps.gen.outputs.expected }}
    steps:
      - uses: actions/checkout@v7
      - id: gen
        env:
          NETCODES: ${{ inputs.netcodes }}
          SESSIONS: ${{ inputs.sessions }}
          MEASURED_CLIENTS: ${{ inputs.measured_clients }}
          LOADGEN_PROCS: ${{ inputs.loadgen_procs }}
          FUSION_MAX_CLIENTS: ${{ inputs.fusion_max_clients }}
          BENCH_SECONDS: ${{ inputs.bench_seconds }}
          BENCH_OBJECTS: ${{ inputs.bench_objects }}
          MAX_PARALLEL: ${{ inputs.max_parallel }}
          RUNNER: ${{ inputs.runner }}
          LOADGEN_RUNNER: ${{ inputs.loadgen_runner }}
          SERVER_RUNNER: ${{ inputs.server_runner }}
        run: python3 .github/scripts/bench-fleet.py plan

  build:
    name: Build
    needs: prep
    strategy:
      fail-fast: false
      matrix:
        netcode: ${{ fromJSON(needs.prep.outputs.netcode_matrix) }}
    uses: ./.github/workflows/build.yml
    secrets: inherit
    with:
      netcode: ${{ matrix.netcode }}
      profiling: ${{ inputs.profiling }}
      runner: ${{ inputs.runner }}

  bench:
    name: Sequential benchmark fleet
    needs: [prep, build]
    if: ${{ !cancelled() && needs.prep.result == 'success' && needs.build.result == 'success' }}
    uses: ./.github/workflows/benchmark.yml
    secrets: inherit
    with:
      netcodes: ${{ inputs.netcodes }}
      sessions: ${{ inputs.sessions }}
      measured_clients: ${{ inputs.measured_clients }}
      loadgen_procs: ${{ inputs.loadgen_procs }}
      fusion_max_clients: ${{ inputs.fusion_max_clients }}
      bench_seconds: ${{ inputs.bench_seconds }}
      bench_objects: ${{ inputs.bench_objects }}
      build_players: false
      profiling: ${{ inputs.profiling }}
      runner: ${{ inputs.runner }}
      loadgen_runner: ${{ inputs.loadgen_runner }}
      server_runner: ${{ inputs.server_runner }}
      region: ${{ inputs.region }}

  combine:
    name: Render scaling tables
    needs: [prep, bench]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout (for aggregation scripts)
        uses: actions/checkout@v7
        with:
          ref: ${{ github.ref }}

      - name: Download scaling datapoints
        uses: actions/download-artifact@v8
        with:
          pattern: bench-scaling-*
          path: all
          merge-multiple: true

      - name: Merge datapoints
        run: |
          # No matching artifacts (e.g. a failed build) means download-artifact may not create all/.
          mkdir -p all
          bash .github/scripts/versions.sh > all/versions.json
          cat all/versions.json
          bash .github/scripts/bench-scaling.sh all results-out

      - name: Render interactive report
        id: report
        env:
          BENCH_RESULT: ${{ needs.bench.result }}
        run: |
          set -eu
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          PAGES_URL="https://$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]').github.io/${{ github.event.repository.name }}/"
          COUNT=$(jq 'length' results-out/scaling.json 2>/dev/null || echo 0)
          EXPECTED=${{ needs.prep.outputs.expected || '0' }}
          COMPLETE=false
          [ "$COUNT" -eq "$EXPECTED" ] && [ "$COUNT" -gt 0 ] && [ "$BENCH_RESULT" = "success" ] && COMPLETE=true
          echo "Datapoints: $COUNT of $EXPECTED planned (complete=$COMPLETE)"
          echo "count=$COUNT" >> "$GITHUB_OUTPUT"
          echo "complete=$COMPLETE" >> "$GITHUB_OUTPUT"
          echo "run_url=$RUN_URL" >> "$GITHUB_OUTPUT"
          echo "pages_url=$PAGES_URL" >> "$GITHUB_OUTPUT"
          [ "$COUNT" -gt 0 ] || { echo "No datapoints; skipping report"; exit 0; }
          python3 .github/scripts/render-report.py results-out/scaling.json results-out/report.html \
            --versions all/versions.json --run-url "$RUN_URL" --title "Netcode Scaling Report"
          python3 .github/scripts/render-summary.py results-out/scaling.json \
            --versions all/versions.json --run-url "$RUN_URL" --report-url "$PAGES_URL" \
            --svg-out results-out/svg > results-out/latest.md
          # The Markdown lives in docs/ next to the SVGs it references.
          sed -i 's#results-out/svg/##g' results-out/latest.md
          cat results-out/latest.md
          # Same block as the job summary, minus the picture (its SVGs are not reachable from here).
          sed '/<picture>/,/<\/picture>/d' results-out/latest.md >> "$GITHUB_STEP_SUMMARY"

      - name: Set up Python for promotional charts
        if: ${{ steps.report.outputs.complete == 'true' }}
        uses: actions/setup-python@v7
        with:
          python-version: '3.12'
          cache: pip
          cache-dependency-path: .github/scripts/requirements-promo.txt

      - name: Install chart dependencies
        if: ${{ steps.report.outputs.complete == 'true' }}
        run: python -m pip install -r .github/scripts/requirements-promo.txt

      - name: Check promotional chart calculations
        if: ${{ steps.report.outputs.complete == 'true' }}
        run: python .github/scripts/test-promo.py

      - name: Render promotional charts
        id: promo
        if: ${{ steps.report.outputs.complete == 'true' }}
        run: |
          set -eu
          python .github/scripts/render-promo.py \
            --data results-out/scaling.json --summary results-out/latest.md \
            --output results-out/promo --source-revision "$(git rev-parse HEAD)" --ci

      - name: Upload combined results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-results
          path: |
            all/
            results-out/
          if-no-files-found: warn
          retention-days: 90

      # Commit the latest report (docs/) on the default branch only, and only when every planned
      # datapoint exists and fleet coordination succeeded. Partial results remain downloadable;
      # a missing worker or failed completion barrier must not replace the published report.
      - name: Commit latest results to the repository
        if: ${{ success() && steps.report.outputs.complete == 'true' && steps.promo.outcome == 'success' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
        run: |
          set -eu
          mkdir -p docs
          cp results-out/report.html docs/index.html
          cp results-out/scaling.json docs/latest.json
          cp results-out/latest.md docs/latest.md
          cp results-out/svg/latest-light.svg results-out/svg/latest-dark.svg docs/
          # Replace the bundle so a custom suite cannot retain charts from an older run.
          # Dated archives and shared logo assets live outside this directory.
          rm -rf -- docs/promo/latest
          mkdir -p docs/promo/latest
          cp -a results-out/promo/. docs/promo/latest/
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add docs/
          if git diff --cached --quiet; then
            echo "Nothing to commit"
            exit 0
          fi
          git commit -m "bench: latest results from run ${{ github.run_id }} [skip ci]"
          for i in 1 2 3; do
            git pull --rebase origin "${{ github.event.repository.default_branch }}" && git push origin HEAD:"${{ github.event.repository.default_branch }}" && exit 0
            sleep 5
          done
          echo "::error::Could not push results commit"
          exit 1