Netcode Scaling Benchmark #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Netcode Scaling Benchmark | |
| # Builds every selected netcode once (in parallel), then runs the distributed benchmark for each | |
| # netcode at each connection count, netcodes side by side (max_parallel) one size at a time, and | |
| # finally renders a cross-netcode scaling table with charts into the job summary. | |
| # | |
| # Runner budget per run: 1 server + measured_clients + ceil((size - measured) / loadgen_procs) | |
| # loadgen runners, e.g. size 100 = 1 + 10 + 4 = 15 runners. Every runner joins the tailnet as an | |
| # ephemeral device, so max_parallel x 15 must stay under the Tailscale device limit. | |
| # | |
| # Fusion's client count is capped by fusion_max_clients (Photon CCU plan). If Photon refuses a | |
| # connection the server proceeds with the clients it has after -connectTimeout, and the summary | |
| # shows the actual count in its "connected at start" table. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| netcodes: | |
| description: "Comma-separated netcodes to run" | |
| required: true | |
| default: "purrnet,fishnet,mirror,ngo,fusion" | |
| sizes: | |
| description: "Comma-separated connection counts" | |
| required: true | |
| default: "10,50,100" | |
| bench_seconds: | |
| description: "Steady-state measurement window per test, seconds" | |
| required: true | |
| default: "20" | |
| bench_objects: | |
| description: "Objects spawned per test" | |
| required: true | |
| default: "100" | |
| profiling: | |
| description: "Development builds (adds a CPU-by-marker table; release builds give cleaner absolute CPU numbers)" | |
| type: boolean | |
| required: false | |
| default: false | |
| runner: | |
| description: "runs-on label for build + benchmark jobs (blacksmith-4vcpu-ubuntu-2404 = fixed hardware; ubuntu-latest = GitHub-hosted)" | |
| required: false | |
| default: "blacksmith-4vcpu-ubuntu-2404" | |
| region: | |
| description: "Photon Cloud region (Fusion only)" | |
| required: false | |
| default: "us" | |
| measured_clients: | |
| description: "Single-process measured client runners per run (the rest of the connections come from loadgen runners)" | |
| required: false | |
| default: "10" | |
| loadgen_procs: | |
| description: "Client processes per loadgen runner (3 per vCPU is the tested ratio)" | |
| required: false | |
| default: "24" | |
| loadgen_runner: | |
| description: "runs-on label for loadgen runners" | |
| required: false | |
| default: "blacksmith-8vcpu-ubuntu-2404" | |
| fusion_max_clients: | |
| description: "Client cap for Fusion (Photon CCU plan limit; if the dedicated server counts as one CCU, set this to plan - 1)" | |
| required: false | |
| default: "100" | |
| max_parallel: | |
| description: "Benchmark runs in flight at once (netcodes run side by side, one size at a time). Every runner is a tailnet device: keep max_parallel x runners-per-run under your Tailscale device limit (100 on the Personal plan)" | |
| required: false | |
| default: "5" | |
| jobs: | |
| prep: | |
| name: Plan runs | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| outputs: | |
| netcode_matrix: ${{ steps.gen.outputs.netcode_matrix }} | |
| bench_matrix: ${{ steps.gen.outputs.bench_matrix }} | |
| max_parallel: ${{ steps.gen.outputs.max_parallel }} | |
| steps: | |
| - id: gen | |
| run: | | |
| set -eu | |
| NETCODES=$(echo "${{ inputs.netcodes }}" | tr ',' '\n' | sed 's/ //g' | grep -v '^$') | |
| SIZES=$(echo "${{ inputs.sizes }}" | tr ',' '\n' | sed 's/ //g' | grep -v '^$') | |
| MEASURED_MAX=${{ inputs.measured_clients }} | |
| PROCS=${{ inputs.loadgen_procs }} | |
| for n in $NETCODES; do | |
| case "$n" in purrnet|fishnet|mirror|ngo|fusion) ;; *) echo "::error::Unknown netcode '$n'"; exit 1;; esac | |
| done | |
| echo "netcode_matrix=$(printf '%s\n' $NETCODES | jq -R . | jq -s -c .)" >> "$GITHUB_OUTPUT" | |
| echo "max_parallel=${{ inputs.max_parallel }}" >> "$GITHUB_OUTPUT" | |
| # Size-major order: with max_parallel slots the netcodes run side by side at one size, | |
| # then move on to the next size together. Each run = 1 server + measured clients + | |
| # ceil(rest / procs) loadgen runners; the last loadgen runner takes the remainder. | |
| ENTRIES="" | |
| for s in $SIZES; do | |
| for n in $NETCODES; do | |
| CAP=200 | |
| [ "$n" = "fusion" ] && CAP=${{ inputs.fusion_max_clients }} | |
| TOTAL=$s | |
| [ "$TOTAL" -gt "$CAP" ] && TOTAL=$CAP | |
| MEASURED=$TOTAL | |
| [ "$MEASURED" -gt "$MEASURED_MAX" ] && MEASURED=$MEASURED_MAX | |
| REST=$((TOTAL - MEASURED)) | |
| if [ "$REST" -le 0 ]; then | |
| LG=0; LAST=0 | |
| else | |
| LG=$(( (REST + PROCS - 1) / PROCS )) | |
| LAST=$(( REST - (LG - 1) * PROCS )) | |
| fi | |
| ENTRIES="$ENTRIES{\"netcode\":\"$n\",\"size\":\"$s\",\"measured\":\"$MEASURED\",\"loadgen\":\"$LG\",\"procs\":\"$PROCS\",\"procs_last\":\"$LAST\",\"tag\":\"c$s\"}," | |
| done | |
| done | |
| MATRIX="{\"include\":[${ENTRIES%,}]}" | |
| echo "$MATRIX" | jq . | |
| echo "bench_matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT" | |
| build: | |
| name: Build | |
| needs: prep | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| netcode: ${{ fromJSON(needs.prep.outputs.netcode_matrix) }} | |
| uses: ./.github/workflows/build.yml | |
| secrets: inherit | |
| with: | |
| netcode: ${{ matrix.netcode }} | |
| profiling: ${{ inputs.profiling }} | |
| runner: ${{ inputs.runner }} | |
| bench: | |
| name: ${{ matrix.netcode }} @ ${{ matrix.size }} | |
| needs: [prep, build] | |
| if: ${{ !cancelled() && needs.prep.result == 'success' }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ fromJSON(needs.prep.outputs.max_parallel) }} | |
| matrix: ${{ fromJSON(needs.prep.outputs.bench_matrix) }} | |
| uses: ./.github/workflows/benchmark.yml | |
| secrets: inherit | |
| with: | |
| netcode: ${{ matrix.netcode }} | |
| measured_clients: ${{ matrix.measured }} | |
| loadgen_jobs: ${{ matrix.loadgen }} | |
| procs_per_loadgen: ${{ matrix.procs }} | |
| procs_last_loadgen: ${{ matrix.procs_last }} | |
| bench_seconds: ${{ inputs.bench_seconds }} | |
| bench_objects: ${{ inputs.bench_objects }} | |
| tag: ${{ matrix.tag }} | |
| player_artifact: player-${{ matrix.netcode }} | |
| profiling: ${{ inputs.profiling }} | |
| runner: ${{ inputs.runner }} | |
| loadgen_runner: ${{ inputs.loadgen_runner }} | |
| region: ${{ inputs.region }} | |
| combine: | |
| name: Render scaling tables | |
| needs: [prep, bench] | |
| if: always() | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout (for aggregation scripts) | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Download scaling datapoints | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: bench-scaling-* | |
| path: all | |
| merge-multiple: true | |
| - name: Render scaling curve | |
| run: | | |
| bash .github/scripts/versions.sh > all/versions.json | |
| cat all/versions.json | |
| bash .github/scripts/bench-scaling.sh \ | |
| all \ | |
| "${{ inputs.bench_seconds }}" \ | |
| "${{ inputs.bench_objects }}" \ | |
| all/versions.json \ | |
| results-out | |
| - name: Render interactive report | |
| id: report | |
| run: | | |
| set -eu | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| PAGES_URL="https://$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]').github.io/${{ github.event.repository.name }}/" | |
| COUNT=$(jq 'length' results-out/scaling.json 2>/dev/null || echo 0) | |
| EXPECTED=$(echo '${{ needs.prep.outputs.bench_matrix }}' | jq '.include | length') | |
| COMPLETE=false | |
| [ "$COUNT" -ge "$EXPECTED" ] && [ "$COUNT" -gt 0 ] && COMPLETE=true | |
| echo "Datapoints: $COUNT of $EXPECTED planned (complete=$COMPLETE)" | |
| echo "count=$COUNT" >> "$GITHUB_OUTPUT" | |
| echo "complete=$COMPLETE" >> "$GITHUB_OUTPUT" | |
| echo "run_url=$RUN_URL" >> "$GITHUB_OUTPUT" | |
| echo "pages_url=$PAGES_URL" >> "$GITHUB_OUTPUT" | |
| [ "$COUNT" -gt 0 ] || { echo "No datapoints; skipping report"; exit 0; } | |
| python3 .github/scripts/render-report.py results-out/scaling.json results-out/report.html \ | |
| --versions all/versions.json --run-url "$RUN_URL" --title "Netcode Scaling Report" | |
| python3 .github/scripts/render-summary.py results-out/scaling.json \ | |
| --versions all/versions.json --run-url "$RUN_URL" --report-url "$PAGES_URL" > results-out/latest.md | |
| cat results-out/latest.md | |
| - name: Upload combined results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| all/ | |
| results-out/ | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| # Commit the latest report + README block on the default branch only, and only when every | |
| # planned datapoint exists (a partial run must not overwrite a complete one). A datapoint | |
| # still exists when a few client jobs failed, so flaky clients do not block the update. | |
| - name: Commit latest results to the repository | |
| if: ${{ steps.report.outputs.complete == 'true' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} | |
| run: | | |
| set -eu | |
| mkdir -p docs | |
| cp results-out/report.html docs/index.html | |
| cp results-out/scaling.json docs/latest.json | |
| cp results-out/latest.md docs/latest.md | |
| python3 - <<'EOF' | |
| import re, pathlib | |
| readme = pathlib.Path("README.md") | |
| text = readme.read_text(encoding="utf-8") | |
| block = pathlib.Path("docs/latest.md").read_text(encoding="utf-8").strip() | |
| new = re.sub(r"<!-- BENCH:START -->.*?<!-- BENCH:END -->", | |
| "<!-- BENCH:START -->\n" + block + "\n<!-- BENCH:END -->", text, flags=re.S) | |
| if new == text and "<!-- BENCH:START -->" not in text: | |
| new = text.rstrip() + "\n\n## Latest results\n\n<!-- BENCH:START -->\n" + block + "\n<!-- BENCH:END -->\n" | |
| readme.write_text(new, encoding="utf-8") | |
| EOF | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add README.md docs/ | |
| if git diff --cached --quiet; then | |
| echo "Nothing to commit" | |
| exit 0 | |
| fi | |
| git commit -m "bench: latest results from run ${{ github.run_id }} [skip ci]" | |
| for i in 1 2 3; do | |
| git pull --rebase origin "${{ github.event.repository.default_branch }}" && git push origin HEAD:"${{ github.event.repository.default_branch }}" && exit 0 | |
| sleep 5 | |
| done | |
| echo "::error::Could not push results commit" | |
| exit 1 |