Netcode Scaling Benchmark #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Netcode Scaling Benchmark | |
| # Builds every selected netcode once (in parallel), then runs one benchmark session per entry in | |
| # `sessions` (connections@tickHz, e.g. 100@60; up to max_parallel side by side). Inside a session | |
| # every netcode runs back to back on the same server machine and the same client machines, so | |
| # numbers within a session are comparable across netcodes. Finally renders the report. | |
| # | |
| # Runner budget per session: 1 server + measured_clients + ceil((size - measured) / loadgen_procs) | |
| # loadgen runners, e.g. size 100 = 1 + 10 + 8 = 19 runners. Every runner joins the tailnet as an | |
| # ephemeral device, so max_parallel x 19 must stay under the Tailscale device limit. | |
| # | |
| # Fusion's client count is capped by fusion_max_clients (Photon CCU plan). If Photon refuses a | |
| # connection the server proceeds with the clients it has after -connectTimeout, and the summary | |
| # shows the actual count in its "connected at start" table. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| netcodes: | |
| description: "Comma-separated netcodes to run" | |
| required: true | |
| default: "purrnet,fishnet,mirror,ngo,fusion" | |
| sessions: | |
| description: "Comma-separated sessions as connections@tickHz (a bare number means 20 Hz)" | |
| required: true | |
| default: "10@20,100@20,100@60" | |
| bench_seconds: | |
| description: "Steady-state measurement window per test, seconds (Idle and Static always use 5)" | |
| required: true | |
| default: "10" | |
| bench_objects: | |
| description: "Objects spawned per test" | |
| required: true | |
| default: "100" | |
| profiling: | |
| description: "Development builds (adds a CPU-by-marker table; release builds give cleaner absolute CPU numbers)" | |
| type: boolean | |
| required: false | |
| default: false | |
| runner: | |
| description: "runs-on label for build + client jobs (ubuntu-latest = GitHub-hosted, free for public repos; blacksmith-4vcpu-ubuntu-2404 = fixed hardware)" | |
| required: false | |
| default: "ubuntu-latest" | |
| region: | |
| description: "Photon Cloud region (Fusion only; eu is closest to bench-server)" | |
| required: false | |
| default: "eu" | |
| measured_clients: | |
| description: "Single-process measured client runners per run (the rest of the connections come from loadgen runners)" | |
| required: false | |
| default: "10" | |
| loadgen_procs: | |
| description: "Client processes per loadgen runner (3 per vCPU is the tested ratio; ubuntu-latest has 4)" | |
| required: false | |
| default: "12" | |
| loadgen_runner: | |
| description: "runs-on label for loadgen runners (empty = same as runner)" | |
| required: false | |
| default: "" | |
| server_runner: | |
| description: "runs-on label for the server job (bench-server = the dedicated Hetzner box, which runs one session at a time, so keep max_parallel at 1; empty = same as runner)" | |
| required: false | |
| default: "bench-server" | |
| fusion_max_clients: | |
| description: "Client cap for Fusion (Photon CCU plan limit; if the dedicated server counts as one CCU, set this to plan - 1)" | |
| required: false | |
| default: "100" | |
| max_parallel: | |
| description: "Benchmark sessions (connection counts) in flight at once. Every runner is a tailnet device: keep max_parallel x runners-per-session under your Tailscale device limit (100 on the Personal plan). bench-server runs one session at a time, so leave at 1 unless server_runner points elsewhere" | |
| required: false | |
| default: "1" | |
| # The dedicated bench-server takes one session at a time: a second dispatch waits for the running | |
| # one to finish instead of starting alongside it. GitHub keeps one waiting run per group, so a third | |
| # dispatch replaces the waiting one. | |
| concurrency: | |
| group: netcode-scaling-bench | |
| cancel-in-progress: false | |
| jobs: | |
| prep: | |
| name: Plan runs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| netcode_matrix: ${{ steps.gen.outputs.netcode_matrix }} | |
| size_matrix: ${{ steps.gen.outputs.size_matrix }} | |
| expected: ${{ steps.gen.outputs.expected }} | |
| max_parallel: ${{ steps.gen.outputs.max_parallel }} | |
| steps: | |
| - id: gen | |
| run: | | |
| set -eu | |
| NETCODES=$(echo "${{ inputs.netcodes }}" | tr ',' '\n' | sed 's/ //g' | grep -v '^$') | |
| SESSIONS=$(echo "${{ inputs.sessions }}" | tr ',' '\n' | sed 's/ //g' | grep -v '^$') | |
| for n in $NETCODES; do | |
| case "$n" in purrnet|fishnet|mirror|ngo|fusion) ;; *) echo "::error::Unknown netcode '$n'"; exit 1;; esac | |
| done | |
| echo "netcode_matrix=$(printf '%s\n' $NETCODES | jq -R . | jq -s -c .)" >> "$GITHUB_OUTPUT" | |
| echo "max_parallel=${{ inputs.max_parallel }}" >> "$GITHUB_OUTPUT" | |
| # One session per connections@tick entry; the session itself runs the netcodes back to | |
| # back and works out measured / loadgen runner counts per netcode (see benchmark.yml). | |
| # The tag (c100t60) names artifacts and tailnet hosts and tells the aggregator which | |
| # row the datapoint belongs to. Fusion sessions share one Photon CCU plan, so each | |
| # session waits for the earlier ones to finish Fusion before starting its own (fusion_after). | |
| ENTRIES="" | |
| AFTER="" | |
| COUNT=0 | |
| for s in $SESSIONS; do | |
| SIZE=${s%@*} | |
| TICK=20 | |
| [ "$s" != "$SIZE" ] && TICK=${s#*@} | |
| case "$SIZE$TICK" in *[!0-9]*) echo "::error::Bad session '$s' (want connections@tickHz)"; exit 1;; esac | |
| TAG="c${SIZE}t${TICK}" | |
| ENTRIES="$ENTRIES{\"size\":\"$SIZE\",\"tick\":\"$TICK\",\"tag\":\"$TAG\",\"fusion_after\":\"${AFTER#,}\"}," | |
| AFTER="$AFTER,$TAG" | |
| COUNT=$((COUNT + 1)) | |
| done | |
| MATRIX="{\"include\":[${ENTRIES%,}]}" | |
| echo "$MATRIX" | jq . | |
| echo "size_matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT" | |
| echo "expected=$(( $(printf '%s\n' $NETCODES | wc -l) * COUNT ))" >> "$GITHUB_OUTPUT" | |
| build: | |
| name: Build | |
| needs: prep | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| netcode: ${{ fromJSON(needs.prep.outputs.netcode_matrix) }} | |
| uses: ./.github/workflows/build.yml | |
| secrets: inherit | |
| with: | |
| netcode: ${{ matrix.netcode }} | |
| profiling: ${{ inputs.profiling }} | |
| runner: ${{ inputs.runner }} | |
| bench: | |
| name: ${{ matrix.size }} connections @ ${{ matrix.tick }} Hz | |
| needs: [prep, build] | |
| if: ${{ !cancelled() && needs.prep.result == 'success' }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ fromJSON(needs.prep.outputs.max_parallel) }} | |
| matrix: ${{ fromJSON(needs.prep.outputs.size_matrix) }} | |
| uses: ./.github/workflows/benchmark.yml | |
| secrets: inherit | |
| with: | |
| netcodes: ${{ inputs.netcodes }} | |
| size: ${{ matrix.size }} | |
| measured_clients: ${{ inputs.measured_clients }} | |
| loadgen_procs: ${{ inputs.loadgen_procs }} | |
| fusion_max_clients: ${{ inputs.fusion_max_clients }} | |
| bench_seconds: ${{ inputs.bench_seconds }} | |
| bench_objects: ${{ inputs.bench_objects }} | |
| tick_rate: ${{ matrix.tick }} | |
| tag: ${{ matrix.tag }} | |
| fusion_after: ${{ matrix.fusion_after }} | |
| build_players: false | |
| profiling: ${{ inputs.profiling }} | |
| runner: ${{ inputs.runner }} | |
| loadgen_runner: ${{ inputs.loadgen_runner }} | |
| server_runner: ${{ inputs.server_runner }} | |
| region: ${{ inputs.region }} | |
| combine: | |
| name: Render scaling tables | |
| needs: [prep, bench] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout (for aggregation scripts) | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Download scaling datapoints | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: bench-scaling-* | |
| path: all | |
| merge-multiple: true | |
| - name: Merge datapoints | |
| run: | | |
| bash .github/scripts/versions.sh > all/versions.json | |
| cat all/versions.json | |
| bash .github/scripts/bench-scaling.sh all results-out | |
| - name: Render interactive report | |
| id: report | |
| run: | | |
| set -eu | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| PAGES_URL="https://$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]').github.io/${{ github.event.repository.name }}/" | |
| COUNT=$(jq 'length' results-out/scaling.json 2>/dev/null || echo 0) | |
| EXPECTED=${{ needs.prep.outputs.expected }} | |
| COMPLETE=false | |
| [ "$COUNT" -ge "$EXPECTED" ] && [ "$COUNT" -gt 0 ] && COMPLETE=true | |
| echo "Datapoints: $COUNT of $EXPECTED planned (complete=$COMPLETE)" | |
| echo "count=$COUNT" >> "$GITHUB_OUTPUT" | |
| echo "complete=$COMPLETE" >> "$GITHUB_OUTPUT" | |
| echo "run_url=$RUN_URL" >> "$GITHUB_OUTPUT" | |
| echo "pages_url=$PAGES_URL" >> "$GITHUB_OUTPUT" | |
| [ "$COUNT" -gt 0 ] || { echo "No datapoints; skipping report"; exit 0; } | |
| python3 .github/scripts/render-report.py results-out/scaling.json results-out/report.html \ | |
| --versions all/versions.json --run-url "$RUN_URL" --title "Netcode Scaling Report" | |
| python3 .github/scripts/render-summary.py results-out/scaling.json \ | |
| --versions all/versions.json --run-url "$RUN_URL" --report-url "$PAGES_URL" \ | |
| --svg-out results-out/svg > results-out/latest.md | |
| # The Markdown lives in docs/ next to the SVGs it references. | |
| sed -i 's#results-out/svg/##g' results-out/latest.md | |
| cat results-out/latest.md | |
| # Same block as the job summary, minus the picture (its SVGs are not reachable from here). | |
| sed '/<picture>/,/<\/picture>/d' results-out/latest.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload combined results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| all/ | |
| results-out/ | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| # Commit the latest report (docs/) on the default branch only, and only when every planned | |
| # datapoint exists (a partial run must not overwrite a complete one). A datapoint still | |
| # exists when a few client jobs failed, so flaky clients do not block the update. | |
| - name: Commit latest results to the repository | |
| if: ${{ steps.report.outputs.complete == 'true' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} | |
| run: | | |
| set -eu | |
| mkdir -p docs | |
| cp results-out/report.html docs/index.html | |
| cp results-out/scaling.json docs/latest.json | |
| cp results-out/latest.md docs/latest.md | |
| cp results-out/svg/latest-light.svg results-out/svg/latest-dark.svg docs/ | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add docs/ | |
| if git diff --cached --quiet; then | |
| echo "Nothing to commit" | |
| exit 0 | |
| fi | |
| git commit -m "bench: latest results from run ${{ github.run_id }} [skip ci]" | |
| for i in 1 2 3; do | |
| git pull --rebase origin "${{ github.event.repository.default_branch }}" && git push origin HEAD:"${{ github.event.repository.default_branch }}" && exit 0 | |
| sleep 5 | |
| done | |
| echo "::error::Could not push results commit" | |
| exit 1 |