TeraSky-OSS · vrabbi · Sep 16, 2026 · Sep 16, 2026 · Sep 16, 2026 · Sep 16, 2026
@@ -36,6 +36,11 @@ jobs:
             script: hack/e2e-test-legacy-claims.sh
           - name: kind + Crossplane + cert-manager, package-managed XRD conversion guard
             script: hack/e2e-test-package-managed.sh
+          # Moving a target between webhook servers under load. This is the
+          # claim automatic sharding rests on, and the only place in CI that
+          # would notice the window reopening.
+          - name: kind + cert-manager, reassignment under load
+            script: hack/e2e-reassign.sh
     steps:
       - name: Checkout
         uses: actions/checkout@v7

@@ -0,0 +1,166 @@
+name: Scale
+
+# A scale target nobody runs is a scale target nobody trusts.
+# `make test-e2e-scale` reaches 100 CRDs x 100 objects and was explicitly
+# outside the CI matrix, so the first time it ran in anger would have been
+# the first time anyone discovered it had bit-rotted.
+#
+# This runs it nightly at a raised envelope, publishes the numbers as an
+# artifact and as a job summary, and fails on a large relative regression
+# against the previous run — never on an absolute timing, because absolute
+# timings on a hosted runner vary too much between runs to gate on.
+on:
+  schedule:
+    # 03:30 UTC daily: an hour after the soak, so the two never contend for
+    # the same runner pool or for the reviewer's attention.
+    - cron: "30 3 * * *"
+  workflow_dispatch:
+    inputs:
+      targets:
+        description: "CRDs to generate (each with 3 versions)"
+        required: false
+        default: "300"
+      instances:
+        description: "Objects per CRD"
+        required: false
+        default: "20"
+      parallel:
+        description: "Concurrent Get/List workers"
+        required: false
+        default: "16"
+      threshold:
+        description: "Fail when a measurement exceeds this multiple of the previous run"
+        required: false
+        default: "1.5"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: scale-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  scale:
+    name: Cluster-scale Get/List
+    runs-on: ubuntu-latest
+    # The envelope below takes roughly 25 minutes end to end on a standard
+    # runner: ~6 for the images and kind, ~12 to apply 300 CRDs and create
+    # 6000 objects, and the rest driving traffic. The timeout is generous
+    # against that, because a run killed at the boundary produces no
+    # artifact and therefore no trend.
+    timeout-minutes: 75
+    permissions:
+      # Reading the previous run's artifact through the API, for the
+      # regression comparison. Nothing here writes.
+      actions: read
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          # This job runs downloaded code (kind, helm, the chart). checkout
+          # persists the job token in .git/config by default, where that
+          # code can read it.
+          persist-credentials: false
+
+      - name: Install kind
+        run: |
+          sudo curl -Lo /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64
+          sudo chmod +x /usr/local/bin/kind
+
+      - name: Install kubectl
+        run: |
+          KUBECTL_VERSION="$(curl -sSL https://dl.k8s.io/release/stable.txt)"
+          sudo curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
+          sudo chmod +x /usr/local/bin/kubectl
+
+      - name: Install Helm
+        uses: azure/setup-helm@v5
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      # Validated as decimals before they reach a shell. workflow_dispatch
+      # needs write access, so this is not a privilege boundary — it keeps a
+      # typo from becoming a confusing kubectl error twenty minutes in.
+      - name: Validate the envelope
+        env:
+          TARGETS: ${{ github.event.inputs.targets || '300' }}
+          INSTANCES: ${{ github.event.inputs.instances || '20' }}
+          PARALLEL: ${{ github.event.inputs.parallel || '16' }}
+          THRESHOLD: ${{ github.event.inputs.threshold || '1.5' }}
+        run: |
+          for pair in "targets:$TARGETS" "instances:$INSTANCES" "parallel:$PARALLEL"; do
+            name="${pair%%:*}"; value="${pair#*:}"
+            case "$value" in ''|*[!0-9]*) echo "$name must be a positive integer, got '$value'" >&2; exit 1 ;; esac
+            [ "$value" -gt 0 ] || { echo "$name must be greater than zero" >&2; exit 1; }
+          done
+          case "$THRESHOLD" in ''|*[!0-9.]*) echo "threshold must be a number, got '$THRESHOLD'" >&2; exit 1 ;; esac
+          python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 1.0 else 1)" "$THRESHOLD" \
+            || { echo "threshold must be greater than 1.0, got '$THRESHOLD'" >&2; exit 1; }
+
+      # The comparison baseline. `gh run list` finds the most recent
+      # successful run of this workflow on this branch and pulls its
+      # artifact; a first run, or one after a schema change, simply has no
+      # baseline and says so in the summary rather than failing.
+      - name: Fetch the previous run's result
+        id: previous
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          mkdir -p /tmp/previous
+          run_id="$(gh run list --workflow scale.yml --branch "${GITHUB_REF_NAME}" \
+            --status success --limit 1 --json databaseId --jq '.[0].databaseId' 2>/dev/null || true)"
+          if [ -z "${run_id}" ] || [ "${run_id}" = "null" ]; then
+            echo "no previous successful run to compare against"
+            exit 0
+          fi
+          if gh run download "${run_id}" --name scale-result --dir /tmp/previous 2>/dev/null; then
+            echo "previous=/tmp/previous/scale-result.json" >> "$GITHUB_OUTPUT"
+            echo "comparing against run ${run_id}"
+          else
+            echo "run ${run_id} published no scale-result artifact"
+          fi
+
+      - name: Run the scale envelope
+        env:
+          TARGETS: ${{ github.event.inputs.targets || '300' }}
+          INSTANCES: ${{ github.event.inputs.instances || '20' }}
+          PARALLEL: ${{ github.event.inputs.parallel || '16' }}
+          RESULT_JSON: /tmp/scale-result.json
+        run: ./hack/e2e-scale.sh
+
+      # Published whether the run passed or failed, and before the
+      # regression check reads it: a failed run's numbers are the ones
+      # somebody needs, and they must not be lost to a non-zero exit.
+      - name: Upload the result
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: scale-result
+          path: /tmp/scale-result.json
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Render the summary and check for a regression
+        if: always()
+        env:
+          PREVIOUS: ${{ steps.previous.outputs.previous }}
+          THRESHOLD: ${{ github.event.inputs.threshold || '1.5' }}
+        run: |
+          if [ ! -f /tmp/scale-result.json ]; then
+            echo "### :x: Scale run produced no result file" >> "$GITHUB_STEP_SUMMARY"
+            echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "The run failed before scalegen wrote its measurements; see the job log." >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+          args=(--current /tmp/scale-result.json --threshold "$THRESHOLD" --summary "$GITHUB_STEP_SUMMARY")
+          if [ -n "${PREVIOUS}" ]; then
+            args+=(--previous "${PREVIOUS}")
+          fi
+          ./hack/scale-report.py "${args[@]}"
@@ -62,6 +62,15 @@ lint-fix: golangci-lint ## Run golangci-lint with --fix. Not every linter can au
 bench: ## Run microbenchmarks (times are not asserted; see docs/operations/capacity.md).
 	go test -run=^$$ -bench=. -benchmem -count=1 -benchtime=200ms ./pkg/engine/ ./internal/webhookserver/
 
+.PHONY: bench-mem
+bench-mem: ## Run the memory benchmarks behind the sizing table in docs/operations/capacity.md.
+	@echo "== bytes retained per compiled plan, and allocation churn per compile =="
+	go test -run=^$$ -bench 'BenchmarkCompiledPlanRetained|BenchmarkCompilePeakAlloc' -benchtime 200x -count=1 ./pkg/engine/
+	@echo
+	@echo "== registry footprint per target, and the transient peak during initial sync =="
+	@echo "   (one fleet is built per iteration, so -benchtime 1x; read B/target, B/peak, B/steady)"
+	go test -run=^$$ -bench 'BenchmarkRegistryRetained|BenchmarkInitialSyncPeak' -benchtime 1x -count=1 ./internal/webhookserver/
+
 .PHONY: test-e2e
 test-e2e: ## Run the real end-to-end test: kind + cert-manager + Crossplane + this operator, both features enabled, proving the conversion webhook works against a live apiserver. Requires docker, kind, kubectl, and helm on PATH. Set KEEP_CLUSTER=1 to skip teardown for debugging.
 	./hack/e2e-test.sh
@@ -86,12 +95,16 @@ test-e2e-package-managed: ## Run the e2e test for the XRD conversion guard: repl
 test-e2e-soak: ## Roll the webhook-server repeatedly under sustained reads/writes and assert zero failed and zero WRONG conversions. Slow (~15 min); also runs nightly in CI.
 	./hack/e2e-soak.sh
 
+.PHONY: test-e2e-reassign
+test-e2e-reassign: ## Move a target between ConversionWebhookServers under sustained load and assert zero failed and zero WRONG conversions. Proves a rebalance never leaves a target unserved.
+	./hack/e2e-reassign.sh
+
 .PHONY: test-e2e-load
 test-e2e-load: ## Synthetic ConversionReview load against a kind cluster (native CRD). Prints latency/throughput for docs/operations/capacity.md.
 	./hack/e2e-load.sh
 
 .PHONY: test-e2e-scale
-test-e2e-scale: ## Cluster-scale Get/List through the live conversion webhook (native CRDs). Configurable via TARGETS/INSTANCES/PARALLEL. See docs/operations/capacity.md.
+test-e2e-scale: ## Cluster-scale Get/List through the live conversion webhook (native CRDs). Configurable via TARGETS/INSTANCES/PARALLEL; set RESULT_JSON to write the measurements as JSON. Also runs nightly in CI. See docs/operations/capacity.md.
 	./hack/e2e-scale.sh
 
 ##@ Build

@@ -172,15 +172,16 @@ a [kind](https://kind.sigs.k8s.io/) cluster, builds this repo's
 - `make test-e2e-legacy-claims` (`hack/e2e-test-legacy-claims.sh`) — `scope: LegacyCluster` with `claimNames`, the shape every cluster upgraded from Crossplane 1.x still runs and the only one that generates a **claim CRD**: proves a claim created at `v1` reads back correctly converted at `v2` and `v3`, that the bare `spec.*` machinery layout (`compositionRef`, `claimRef`, `resourceRef`, `compositeDeletePolicy`, `writeConnectionSecretToRef`) survives conversion on both object classes, that a condition the test itself writes survives alongside Crossplane's, that **both** generated CRDs carry `spec.conversion` and `ConversionPropagated` reaches True, and that `convctl test --live` and `migrate-storage --prune-stored-versions` cover both.
 - `make test-e2e-package-managed` (`hack/e2e-test-package-managed.sh`) — the **XRD conversion guard**: replays the Crossplane package establisher's full non-SSA replace of an XRD in a loop and asserts that not one read at a non-storage version ever comes back unconverted. Then repeats with the guard disabled and asserts the loop **does** catch bad reads — a guard test that cannot fail is not a test. The failure mode is an HTTP 200 with wrong data, so the loop checks converted field values rather than exit codes. Also needs `python3`.
 - `make test-e2e-load` (`hack/e2e-load.sh`) — native-CRD kind cluster, then synthetic `ConversionReview` batches of varying object count/size against the live webhook-server; prints latency/throughput for [Capacity planning](docs/operations/capacity.md).
-- `make test-e2e-scale` (`hack/e2e-scale.sh`) — native-CRD kind cluster, then a generated fleet of CRDs (3 versions each, 3–10 strategies per spoke, all 29 strategies used) plus parallel Get/List of live CRs through the apiserver conversion path. Override `TARGETS`, `INSTANCES`, and `PARALLEL` (for example `TARGETS=100 INSTANCES=100 PARALLEL=32`). Not in the CI matrix.
+- `make test-e2e-reassign` (`hack/e2e-reassign.sh`) — moves a target between two `ConversionWebhookServer` instances, three times (an explicit `webhookServerRef` pin, an unpin, and a sharding-driven move), while sustained reads and writes flow through it, and asserts **zero failed requests and zero wrong values**. Also asserts each move was *verified* against the destination's published served targets rather than taking the unverified fallback, so it cannot pass with the handover mechanism removed. Also needs `python3`.
+- `make test-e2e-scale` (`hack/e2e-scale.sh`) — native-CRD kind cluster, then a generated fleet of CRDs (3 versions each, 3–10 strategies per spoke, all 29 strategies used) plus parallel Get/List of live CRs through the apiserver conversion path. Override `TARGETS`, `INSTANCES`, and `PARALLEL` (for example `TARGETS=100 INSTANCES=100 PARALLEL=32`); set `RESULT_JSON` to write the measurements as JSON. Not in the PR matrix — it runs nightly (`.github/workflows/scale.yml`) at 300 × 20, publishing an artifact and failing on a relative regression.
 
 Requires `docker`, `kind`, `kubectl`, and `helm` on `PATH` (plus `go` for
-`test-e2e-legacy-claims` and `python3` for `test-e2e-package-managed`). The
-five correctness scripts run identically in CI (`.github/workflows/e2e.yml`,
-as a matrix) and locally. `make test-e2e-load` and `make test-e2e-scale` are
-local/capacity targets (`test-e2e-load` also needs `python3` and `curl`) and
-are not in that matrix. Set `KEEP_CLUSTER=1`
-to skip teardown for local debugging.
+`test-e2e-legacy-claims` and `python3` for `test-e2e-package-managed` and
+`test-e2e-reassign`). The six correctness scripts run identically in CI
+(`.github/workflows/e2e.yml`, as a matrix) and locally. `make test-e2e-load`
+and `make test-e2e-scale` are capacity targets (`test-e2e-load` also needs
+`python3` and `curl`) and are not in that matrix; the scale one has its own
+nightly workflow. Set `KEEP_CLUSTER=1` to skip teardown for local debugging.
 
 ## License