diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 77a75de..459545c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,6 +36,11 @@ jobs: script: hack/e2e-test-legacy-claims.sh - name: kind + Crossplane + cert-manager, package-managed XRD conversion guard script: hack/e2e-test-package-managed.sh + # Moving a target between webhook servers under load. This is the + # claim automatic sharding rests on, and the only place in CI that + # would notice the window reopening. + - name: kind + cert-manager, reassignment under load + script: hack/e2e-reassign.sh steps: - name: Checkout uses: actions/checkout@v7 diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml new file mode 100644 index 0000000..535e019 --- /dev/null +++ b/.github/workflows/scale.yml @@ -0,0 +1,166 @@ +name: Scale + +# A scale target nobody runs is a scale target nobody trusts. +# `make test-e2e-scale` reaches 100 CRDs x 100 objects and was explicitly +# outside the CI matrix, so the first time it ran in anger would have been +# the first time anyone discovered it had bit-rotted. +# +# This runs it nightly at a raised envelope, publishes the numbers as an +# artifact and as a job summary, and fails on a large relative regression +# against the previous run — never on an absolute timing, because absolute +# timings on a hosted runner vary too much between runs to gate on. +on: + schedule: + # 03:30 UTC daily: an hour after the soak, so the two never contend for + # the same runner pool or for the reviewer's attention. + - cron: "30 3 * * *" + workflow_dispatch: + inputs: + targets: + description: "CRDs to generate (each with 3 versions)" + required: false + default: "300" + instances: + description: "Objects per CRD" + required: false + default: "20" + parallel: + description: "Concurrent Get/List workers" + required: false + default: "16" + threshold: + description: "Fail when a measurement exceeds this multiple of the previous run" + required: false + default: "1.5" + +permissions: + contents: read + +concurrency: + group: scale-${{ github.ref }} + cancel-in-progress: true + +jobs: + scale: + name: Cluster-scale Get/List + runs-on: ubuntu-latest + # The envelope below takes roughly 25 minutes end to end on a standard + # runner: ~6 for the images and kind, ~12 to apply 300 CRDs and create + # 6000 objects, and the rest driving traffic. The timeout is generous + # against that, because a run killed at the boundary produces no + # artifact and therefore no trend. + timeout-minutes: 75 + permissions: + # Reading the previous run's artifact through the API, for the + # regression comparison. Nothing here writes. + actions: read + contents: read + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + # This job runs downloaded code (kind, helm, the chart). checkout + # persists the job token in .git/config by default, where that + # code can read it. + persist-credentials: false + + - name: Install kind + run: | + sudo curl -Lo /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 + sudo chmod +x /usr/local/bin/kind + + - name: Install kubectl + run: | + KUBECTL_VERSION="$(curl -sSL https://dl.k8s.io/release/stable.txt)" + sudo curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + sudo chmod +x /usr/local/bin/kubectl + + - name: Install Helm + uses: azure/setup-helm@v5 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + # Validated as decimals before they reach a shell. workflow_dispatch + # needs write access, so this is not a privilege boundary — it keeps a + # typo from becoming a confusing kubectl error twenty minutes in. + - name: Validate the envelope + env: + TARGETS: ${{ github.event.inputs.targets || '300' }} + INSTANCES: ${{ github.event.inputs.instances || '20' }} + PARALLEL: ${{ github.event.inputs.parallel || '16' }} + THRESHOLD: ${{ github.event.inputs.threshold || '1.5' }} + run: | + for pair in "targets:$TARGETS" "instances:$INSTANCES" "parallel:$PARALLEL"; do + name="${pair%%:*}"; value="${pair#*:}" + case "$value" in ''|*[!0-9]*) echo "$name must be a positive integer, got '$value'" >&2; exit 1 ;; esac + [ "$value" -gt 0 ] || { echo "$name must be greater than zero" >&2; exit 1; } + done + case "$THRESHOLD" in ''|*[!0-9.]*) echo "threshold must be a number, got '$THRESHOLD'" >&2; exit 1 ;; esac + python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 1.0 else 1)" "$THRESHOLD" \ + || { echo "threshold must be greater than 1.0, got '$THRESHOLD'" >&2; exit 1; } + + # The comparison baseline. `gh run list` finds the most recent + # successful run of this workflow on this branch and pulls its + # artifact; a first run, or one after a schema change, simply has no + # baseline and says so in the summary rather than failing. + - name: Fetch the previous run's result + id: previous + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + mkdir -p /tmp/previous + run_id="$(gh run list --workflow scale.yml --branch "${GITHUB_REF_NAME}" \ + --status success --limit 1 --json databaseId --jq '.[0].databaseId' 2>/dev/null || true)" + if [ -z "${run_id}" ] || [ "${run_id}" = "null" ]; then + echo "no previous successful run to compare against" + exit 0 + fi + if gh run download "${run_id}" --name scale-result --dir /tmp/previous 2>/dev/null; then + echo "previous=/tmp/previous/scale-result.json" >> "$GITHUB_OUTPUT" + echo "comparing against run ${run_id}" + else + echo "run ${run_id} published no scale-result artifact" + fi + + - name: Run the scale envelope + env: + TARGETS: ${{ github.event.inputs.targets || '300' }} + INSTANCES: ${{ github.event.inputs.instances || '20' }} + PARALLEL: ${{ github.event.inputs.parallel || '16' }} + RESULT_JSON: /tmp/scale-result.json + run: ./hack/e2e-scale.sh + + # Published whether the run passed or failed, and before the + # regression check reads it: a failed run's numbers are the ones + # somebody needs, and they must not be lost to a non-zero exit. + - name: Upload the result + if: always() + uses: actions/upload-artifact@v7 + with: + name: scale-result + path: /tmp/scale-result.json + if-no-files-found: warn + retention-days: 90 + + - name: Render the summary and check for a regression + if: always() + env: + PREVIOUS: ${{ steps.previous.outputs.previous }} + THRESHOLD: ${{ github.event.inputs.threshold || '1.5' }} + run: | + if [ ! -f /tmp/scale-result.json ]; then + echo "### :x: Scale run produced no result file" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "The run failed before scalegen wrote its measurements; see the job log." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + args=(--current /tmp/scale-result.json --threshold "$THRESHOLD" --summary "$GITHUB_STEP_SUMMARY") + if [ -n "${PREVIOUS}" ]; then + args+=(--previous "${PREVIOUS}") + fi + ./hack/scale-report.py "${args[@]}" diff --git a/Makefile b/Makefile index 8066d4f..fc7801a 100755 --- a/Makefile +++ b/Makefile @@ -62,6 +62,15 @@ lint-fix: golangci-lint ## Run golangci-lint with --fix. Not every linter can au bench: ## Run microbenchmarks (times are not asserted; see docs/operations/capacity.md). go test -run=^$$ -bench=. -benchmem -count=1 -benchtime=200ms ./pkg/engine/ ./internal/webhookserver/ +.PHONY: bench-mem +bench-mem: ## Run the memory benchmarks behind the sizing table in docs/operations/capacity.md. + @echo "== bytes retained per compiled plan, and allocation churn per compile ==" + go test -run=^$$ -bench 'BenchmarkCompiledPlanRetained|BenchmarkCompilePeakAlloc' -benchtime 200x -count=1 ./pkg/engine/ + @echo + @echo "== registry footprint per target, and the transient peak during initial sync ==" + @echo " (one fleet is built per iteration, so -benchtime 1x; read B/target, B/peak, B/steady)" + go test -run=^$$ -bench 'BenchmarkRegistryRetained|BenchmarkInitialSyncPeak' -benchtime 1x -count=1 ./internal/webhookserver/ + .PHONY: test-e2e test-e2e: ## Run the real end-to-end test: kind + cert-manager + Crossplane + this operator, both features enabled, proving the conversion webhook works against a live apiserver. Requires docker, kind, kubectl, and helm on PATH. Set KEEP_CLUSTER=1 to skip teardown for debugging. ./hack/e2e-test.sh @@ -86,12 +95,16 @@ test-e2e-package-managed: ## Run the e2e test for the XRD conversion guard: repl test-e2e-soak: ## Roll the webhook-server repeatedly under sustained reads/writes and assert zero failed and zero WRONG conversions. Slow (~15 min); also runs nightly in CI. ./hack/e2e-soak.sh +.PHONY: test-e2e-reassign +test-e2e-reassign: ## Move a target between ConversionWebhookServers under sustained load and assert zero failed and zero WRONG conversions. Proves a rebalance never leaves a target unserved. + ./hack/e2e-reassign.sh + .PHONY: test-e2e-load test-e2e-load: ## Synthetic ConversionReview load against a kind cluster (native CRD). Prints latency/throughput for docs/operations/capacity.md. ./hack/e2e-load.sh .PHONY: test-e2e-scale -test-e2e-scale: ## Cluster-scale Get/List through the live conversion webhook (native CRDs). Configurable via TARGETS/INSTANCES/PARALLEL. See docs/operations/capacity.md. +test-e2e-scale: ## Cluster-scale Get/List through the live conversion webhook (native CRDs). Configurable via TARGETS/INSTANCES/PARALLEL; set RESULT_JSON to write the measurements as JSON. Also runs nightly in CI. See docs/operations/capacity.md. ./hack/e2e-scale.sh ##@ Build diff --git a/README.md b/README.md index 0dd70a9..16ac468 100644 --- a/README.md +++ b/README.md @@ -172,15 +172,16 @@ a [kind](https://kind.sigs.k8s.io/) cluster, builds this repo's - `make test-e2e-legacy-claims` (`hack/e2e-test-legacy-claims.sh`) — `scope: LegacyCluster` with `claimNames`, the shape every cluster upgraded from Crossplane 1.x still runs and the only one that generates a **claim CRD**: proves a claim created at `v1` reads back correctly converted at `v2` and `v3`, that the bare `spec.*` machinery layout (`compositionRef`, `claimRef`, `resourceRef`, `compositeDeletePolicy`, `writeConnectionSecretToRef`) survives conversion on both object classes, that a condition the test itself writes survives alongside Crossplane's, that **both** generated CRDs carry `spec.conversion` and `ConversionPropagated` reaches True, and that `convctl test --live` and `migrate-storage --prune-stored-versions` cover both. - `make test-e2e-package-managed` (`hack/e2e-test-package-managed.sh`) — the **XRD conversion guard**: replays the Crossplane package establisher's full non-SSA replace of an XRD in a loop and asserts that not one read at a non-storage version ever comes back unconverted. Then repeats with the guard disabled and asserts the loop **does** catch bad reads — a guard test that cannot fail is not a test. The failure mode is an HTTP 200 with wrong data, so the loop checks converted field values rather than exit codes. Also needs `python3`. - `make test-e2e-load` (`hack/e2e-load.sh`) — native-CRD kind cluster, then synthetic `ConversionReview` batches of varying object count/size against the live webhook-server; prints latency/throughput for [Capacity planning](docs/operations/capacity.md). -- `make test-e2e-scale` (`hack/e2e-scale.sh`) — native-CRD kind cluster, then a generated fleet of CRDs (3 versions each, 3–10 strategies per spoke, all 29 strategies used) plus parallel Get/List of live CRs through the apiserver conversion path. Override `TARGETS`, `INSTANCES`, and `PARALLEL` (for example `TARGETS=100 INSTANCES=100 PARALLEL=32`). Not in the CI matrix. +- `make test-e2e-reassign` (`hack/e2e-reassign.sh`) — moves a target between two `ConversionWebhookServer` instances, three times (an explicit `webhookServerRef` pin, an unpin, and a sharding-driven move), while sustained reads and writes flow through it, and asserts **zero failed requests and zero wrong values**. Also asserts each move was *verified* against the destination's published served targets rather than taking the unverified fallback, so it cannot pass with the handover mechanism removed. Also needs `python3`. +- `make test-e2e-scale` (`hack/e2e-scale.sh`) — native-CRD kind cluster, then a generated fleet of CRDs (3 versions each, 3–10 strategies per spoke, all 29 strategies used) plus parallel Get/List of live CRs through the apiserver conversion path. Override `TARGETS`, `INSTANCES`, and `PARALLEL` (for example `TARGETS=100 INSTANCES=100 PARALLEL=32`); set `RESULT_JSON` to write the measurements as JSON. Not in the PR matrix — it runs nightly (`.github/workflows/scale.yml`) at 300 × 20, publishing an artifact and failing on a relative regression. Requires `docker`, `kind`, `kubectl`, and `helm` on `PATH` (plus `go` for -`test-e2e-legacy-claims` and `python3` for `test-e2e-package-managed`). The -five correctness scripts run identically in CI (`.github/workflows/e2e.yml`, -as a matrix) and locally. `make test-e2e-load` and `make test-e2e-scale` are -local/capacity targets (`test-e2e-load` also needs `python3` and `curl`) and -are not in that matrix. Set `KEEP_CLUSTER=1` -to skip teardown for local debugging. +`test-e2e-legacy-claims` and `python3` for `test-e2e-package-managed` and +`test-e2e-reassign`). The six correctness scripts run identically in CI +(`.github/workflows/e2e.yml`, as a matrix) and locally. `make test-e2e-load` +and `make test-e2e-scale` are capacity targets (`test-e2e-load` also needs +`python3` and `curl`) and are not in that matrix; the scale one has its own +nightly workflow. Set `KEEP_CLUSTER=1` to skip teardown for local debugging. ## License diff --git a/api/v1alpha1/conversionwebhookserver_types.go b/api/v1alpha1/conversionwebhookserver_types.go index c5f312e..4b9c1f7 100644 --- a/api/v1alpha1/conversionwebhookserver_types.go +++ b/api/v1alpha1/conversionwebhookserver_types.go @@ -177,6 +177,17 @@ type ConversionWebhookServerSpec struct { // +optional CacheSelector *metav1.LabelSelector `json:"cacheSelector,omitempty"` + // Sharding opts this instance into the pool that unpinned conversion + // configs are distributed across. See ShardingSpec. + // +optional + Sharding *ShardingSpec `json:"sharding,omitempty"` + + // StartupProbe bounds how long a replica may take to compile every + // assigned plan before the kubelet restarts it. See StartupProbeSpec: + // it polls /readyz, so its budget is a deadline on the sync itself. + // +optional + StartupProbe *StartupProbeSpec `json:"startupProbe,omitempty"` + // Rollout controls how a replica leaves service during a rolling // update or a node drain. The defaults are chosen so that a rollout // causes zero failed conversions; see RolloutSpec. @@ -261,6 +272,154 @@ type RolloutSpec struct { DefaultTopologySpread *bool `json:"defaultTopologySpread,omitempty"` } +// ShardingSpec opts an instance into automatic assignment: conversion +// configs that express no preference are distributed across every +// instance that opts in, instead of all landing on the one marked +// default. +// +// It changes nothing about explicit assignment. A config with +// spec.webhookServerRef set goes exactly where it says, sharded pool or +// not — deliberate pinning stays the strongest statement in the system, +// and tenant isolation is built on it. +type ShardingSpec struct { + // Enabled adds this instance to the pool unpinned configs are + // distributed across. + // + // While the pool is non-empty it, not spec.default, is what serves + // unpinned configs — so the instance marked default must be a member + // of it. Admission enforces that, because otherwise enabling sharding + // on a second instance would silently drain every unpinned config off + // the default one. Enable it on the default instance first. + // +optional + // +kubebuilder:default=true + Enabled *bool `json:"enabled,omitempty"` + + // Weight biases this instance's share of the pool, for a fleet whose + // instances are not the same size. An instance with weight 2 receives + // approximately twice the share of one with weight 1. Weights are + // relative, so scaling all of them changes nothing. + // +optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=1 + Weight *int32 `json:"weight,omitempty"` +} + +// StartupProbeSpec configures the webhook-server's startupProbe: the +// budget a replica gets to finish its cold start before the kubelet gives +// up on it. +// +// The probe polls /readyz, not /healthz. The plain endpoint carrying +// /healthz comes up before the registry sync, so a startupProbe pointed at +// it would succeed within milliseconds and bound nothing; /readyz stays +// false until the initial sync completes, which is what makes this a +// deadline on the sync. While the probe is in flight the kubelet runs +// neither of the other two, so a slow sync is not also fighting the +// liveness probe's own 3 × 10 s. +// +// The deadline matters because the initial sync retries infrastructure +// failures without a limit. Without it, a replica wedged mid-sync stays +// liveness-healthy and never ready: out of the Service, never restarted, +// and visible only as a gap in readyReplicas. +// +// PeriodSeconds × FailureThreshold is the budget. The defaults give five +// minutes, against a measured cold start of well under a second for a +// thousand 50-leaf targets (see docs/operations/capacity.md) — the margin +// is for informer cache sync on a large cluster, which dominates and is +// not this operator's to control. Erring long is deliberate: an +// over-tight threshold turns a slow start into a crash loop, while an +// over-long one only delays the restart of a pod that is not taking +// traffic anyway. +type StartupProbeSpec struct { + // Enabled renders the startupProbe, and defaults to true. Setting it + // to false removes the deadline on the cold start entirely: the + // liveness probe reads /healthz, which answers before the sync begins, + // so nothing then restarts a replica that never finishes syncing. + // +optional + // +kubebuilder:default=true + Enabled *bool `json:"enabled,omitempty"` + + // +optional + // +kubebuilder:default=5 + // +kubebuilder:validation:Minimum=1 + PeriodSeconds *int32 `json:"periodSeconds,omitempty"` + + // +optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=1 + FailureThreshold *int32 `json:"failureThreshold,omitempty"` +} + +// ShardingEnabled reports whether this instance is a member of the +// automatic-assignment pool. Absent means no — sharding is opt-in per +// instance, so an existing fleet behaves exactly as it did before the +// field existed. +func (s *ConversionWebhookServerSpec) ShardingEnabled() bool { + if s.Sharding == nil { + return false + } + if s.Sharding.Enabled == nil { + // `sharding: {}` means enabled: writing the block at all is the + // opt-in, and the CRD's own default agrees. + return true + } + return *s.Sharding.Enabled +} + +// ShardWeight is this instance's relative share of the pool, defaulting +// to 1. A non-positive stored value (only reachable by bypassing +// admission) is treated as 1 rather than as "never selected", because a +// weight of zero would silently make a pool member invisible. +func (s *ConversionWebhookServerSpec) ShardWeight() uint32 { + if s.Sharding == nil || s.Sharding.Weight == nil || *s.Sharding.Weight <= 0 { + return 1 + } + return uint32(*s.Sharding.Weight) +} + +// WebhookServerServiceName is the Service that fronts one instance's +// pods. It lives here, in the leaf package, because two independent +// binaries have to agree on it: the operator names the Service and writes +// it into the target's spec.conversion, and each webhook-server replica +// reads that same field back to tell whether a target still points at it +// during a handover. +func WebhookServerServiceName(serverName string) string { + return serverName + "-webhook-server" +} + +// Startup-probe defaults, mirrored from the kubebuilder markers on +// StartupProbeSpec so the controller can reason about the values an unset +// field will actually produce rather than about the literal nil. +const ( + DefaultStartupProbePeriodSeconds int32 = 5 + DefaultStartupProbeFailureThreshold int32 = 60 +) + +// StartupProbeEnabled reports whether the webhook-server's startupProbe +// should be rendered, with the same default the CRD carries. +func (s *ConversionWebhookServerSpec) StartupProbeEnabled() bool { + if s.StartupProbe == nil || s.StartupProbe.Enabled == nil { + return true + } + return *s.StartupProbe.Enabled +} + +// StartupProbeTiming returns the period and failure threshold the +// startupProbe should be rendered with. Their product is the cold-start +// budget. +func (s *ConversionWebhookServerSpec) StartupProbeTiming() (periodSeconds, failureThreshold int32) { + periodSeconds, failureThreshold = DefaultStartupProbePeriodSeconds, DefaultStartupProbeFailureThreshold + if s.StartupProbe == nil { + return periodSeconds, failureThreshold + } + if s.StartupProbe.PeriodSeconds != nil { + periodSeconds = *s.StartupProbe.PeriodSeconds + } + if s.StartupProbe.FailureThreshold != nil { + failureThreshold = *s.StartupProbe.FailureThreshold + } + return periodSeconds, failureThreshold +} + // AssignedConfigRef is one XRDConversionConfig the resolver currently // assigns to this instance. This reflects DESIRED assignment as computed // by the shared resolver, not proof that every replica has actually loaded @@ -292,6 +451,28 @@ type ConversionWebhookServerStatus struct { Endpoint string `json:"endpoint,omitempty"` // +optional AssignedConfigs []AssignedConfigRef `json:"assignedConfigs,omitempty"` + + // ServedTargets is the set of target resources that EVERY live + // replica of this instance reports a compiled, servable plan for — + // an intersection, not a union, because a target one replica out of + // three can serve is a target that fails one request in three. + // + // Unlike AssignedConfigs, which is desired state computed by the + // shared resolver, this is reported by the replicas themselves: each + // publishes its own registry contents into a Lease, and this field + // is the aggregate. It is what makes a safe handover possible — the + // operator will not repoint a target at this instance until the + // instance says it can already serve it. + // +optional + // +listType=atomic + ServedTargets []string `json:"servedTargets,omitempty"` + + // ReportingReplicas is how many live replica Leases fed + // ServedTargets. A value below ReadyReplicas means at least one + // ready replica has not published yet, and the intersection above is + // not yet a statement about the whole instance. + // +optional + ReportingReplicas int32 `json:"reportingReplicas,omitempty"` } // Condition type constants for ConversionWebhookServer. diff --git a/api/v1alpha1/crdconversionconfig_types.go b/api/v1alpha1/crdconversionconfig_types.go index 27c5725..0bd79c5 100644 --- a/api/v1alpha1/crdconversionconfig_types.go +++ b/api/v1alpha1/crdconversionconfig_types.go @@ -110,6 +110,18 @@ func (c *CRDConversionConfig) WebhookServerRefField() *WebhookServerRef { return c.Spec.WebhookServerRef } +// ShardKey implements internal/assign's generic ConfigLike constraint — +// see XRDConversionConfig.ShardKey. +func (c *CRDConversionConfig) ShardKey() string { + return c.Spec.TargetCRD.Name +} + +// AppliedWebhookURL implements internal/assign's ServingConfigLike +// constraint — see XRDConversionConfig.AppliedWebhookURL. +func (c *CRDConversionConfig) AppliedWebhookURL() string { + return c.Status.WebhookURL +} + // ConditionCRDHealthy mirrors ConditionXRDHealthy for the native-CRD // target: True once the target CustomResourceDefinition exists and its // own Established condition is True. diff --git a/api/v1alpha1/sharding_defaulting_test.go b/api/v1alpha1/sharding_defaulting_test.go new file mode 100644 index 0000000..17a921e --- /dev/null +++ b/api/v1alpha1/sharding_defaulting_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "os" + "path/filepath" + "testing" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" + "k8s.io/apimachinery/pkg/runtime" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + sigsyaml "sigs.k8s.io/yaml" +) + +// cwsStructural loads the generated ConversionWebhookServer CRD and +// returns the structural schema the apiserver would default against — +// the real manifest, not a hand-written approximation, so a change to the +// kubebuilder markers shows up here. +func cwsStructural(t *testing.T) *structuralschema.Structural { + t.Helper() + scheme := runtime.NewScheme() + install.Install(scheme) + + path := filepath.Join("..", "..", "config", "crd", "bases", "terasky.com_conversionwebhookservers.yaml") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + jsonData, err := sigsyaml.YAMLToJSON(data) + if err != nil { + t.Fatalf("converting %s to JSON: %v", path, err) + } + var crd extv1.CustomResourceDefinition + if err := k8syaml.Unmarshal(jsonData, &crd); err != nil { + t.Fatalf("unmarshaling %s: %v", path, err) + } + for _, v := range crd.Spec.Versions { + if v.Schema == nil || v.Schema.OpenAPIV3Schema == nil { + continue + } + var internal apiextensions.JSONSchemaProps + if err := scheme.Convert(v.Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatalf("converting schema for %s: %v", v.Name, err) + } + s, err := structuralschema.NewStructural(&internal) + if err != nil { + t.Fatalf("NewStructural for %s: %v", v.Name, err) + } + return s + } + t.Fatal("the ConversionWebhookServer CRD has no versioned schema") + return nil +} + +func defaultCWS(t *testing.T, spec map[string]any) map[string]any { + t.Helper() + obj := map[string]any{ + "apiVersion": GroupVersion.String(), + "kind": "ConversionWebhookServer", + "metadata": map[string]any{"name": "default"}, + "spec": spec, + } + structuraldefaulting.Default(obj, cwsStructural(t)) + out, _ := obj["spec"].(map[string]any) + return out +} + +// Sharding is off unless the block is written, and the `default: true` on +// spec.sharding.enabled does not change that — structural defaulting only +// descends into an object that is present, and spec.sharding itself +// carries no default. +// +// This is asserted against the apiserver's own defaulting algorithm rather +// than reasoned about, because reading `default: true` off the leaf and +// concluding "sharding is on by default" is an easy and consequential +// mistake: it would mean every install silently moved unpinned configs off +// spec.default and into a shard pool. The chart's template filters the +// block out entirely for the same reason, and ShardingEnabled() agrees +// with both. +func TestSharding_IsOffWhenTheBlockIsAbsent(t *testing.T) { + spec := defaultCWS(t, map[string]any{"default": true}) + if _, present := spec["sharding"]; present { + t.Fatalf("defaulting invented a sharding block: %#v", spec["sharding"]) + } + + var cws ConversionWebhookServerSpec + if cws.ShardingEnabled() { + t.Error("ShardingEnabled() must be false when spec.sharding is unset") + } +} + +// The other half: writing the block at all IS the opt-in, and the leaf +// default is what makes `sharding: {}` mean enabled. Both halves have to +// hold together, or the documented behaviour is only half true. +func TestSharding_EmptyBlockMeansEnabled(t *testing.T) { + spec := defaultCWS(t, map[string]any{"sharding": map[string]any{}}) + sharding, ok := spec["sharding"].(map[string]any) + if !ok { + t.Fatalf("sharding block disappeared: %#v", spec) + } + if enabled, _ := sharding["enabled"].(bool); !enabled { + t.Errorf("sharding.enabled defaulted to %#v, want true", sharding["enabled"]) + } + if weight, _ := sharding["weight"].(int64); weight != 1 { + t.Errorf("sharding.weight defaulted to %#v, want 1", sharding["weight"]) + } + + enabledTrue := true + cws := ConversionWebhookServerSpec{Sharding: &ShardingSpec{}} + if !cws.ShardingEnabled() { + t.Error("ShardingEnabled() must agree with the CRD: an empty block is enabled") + } + cws.Sharding.Enabled = &enabledTrue + if !cws.ShardingEnabled() { + t.Error("ShardingEnabled() must be true for an explicit enabled: true") + } +} + +// And explicit false stays false, so the block is not a one-way door. +func TestSharding_ExplicitlyDisabled(t *testing.T) { + spec := defaultCWS(t, map[string]any{"sharding": map[string]any{"enabled": false}}) + sharding, _ := spec["sharding"].(map[string]any) + if enabled, _ := sharding["enabled"].(bool); enabled { + t.Error("defaulting overwrote an explicit sharding.enabled: false") + } + + disabled := false + cws := ConversionWebhookServerSpec{Sharding: &ShardingSpec{Enabled: &disabled}} + if cws.ShardingEnabled() { + t.Error("ShardingEnabled() must be false for an explicit enabled: false") + } +} diff --git a/api/v1alpha1/xrdconversionconfig_types.go b/api/v1alpha1/xrdconversionconfig_types.go index 18fb6eb..f285a3f 100644 --- a/api/v1alpha1/xrdconversionconfig_types.go +++ b/api/v1alpha1/xrdconversionconfig_types.go @@ -541,6 +541,28 @@ func (c *XRDConversionConfig) WebhookServerRefField() *WebhookServerRef { return c.Spec.WebhookServerRef } +// ShardKey is what automatic assignment hashes: the target resource's +// name, not the config's own. +// +// The target is the thing being served — it is the registry key and the +// /convert/{name} path — so hashing it means renaming a config does not +// move the resource it converts, and two configs can never disagree about +// where one target belongs. Since one target may carry at most one config +// (enforced by a unique field index and by admission), the two keys +// partition identically; only their stability under a rename differs. +func (c *XRDConversionConfig) ShardKey() string { + return c.Spec.TargetXRD.Name +} + +// AppliedWebhookURL is the URL the operator last wrote into the target's +// spec.conversion, or empty if it has never applied one. It implements +// internal/assign's ServingConfigLike constraint: an instance a target +// still points at is an instance that is still serving it, even after the +// resolver has reassigned the config elsewhere. +func (c *XRDConversionConfig) AppliedWebhookURL() string { + return c.Status.WebhookURL +} + // SpokeVersionRules is every rule declared for one spoke version. type SpokeVersionRules struct { Version string `json:"version"` @@ -714,6 +736,19 @@ const ( // reconcile speed — but until it is True, nothing is actually // converting. ConditionConversionPropagated = "ConversionPropagated" + // ConditionHandoverReady reports whether the ConversionWebhookServer a + // target is being moved TO can already serve it. It appears only once + // a move has happened — a target being applied for the first time has + // no previous server still covering it, so there is nothing to hand + // over. False means the target is deliberately still pointed at its + // current server, which is still serving it. + // + // It is not cleared afterwards: it is the verdict on the last + // handover, and "the instance now serving this target was verified + // able to serve it before it was pointed here" stays true. That is + // also what makes a HandoverUnverified verdict stick around long + // enough to be noticed. + ConditionHandoverReady = "HandoverReady" // ConditionApplied reasons used by FailClosed drift handling. ReasonReverted = "Reverted" diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6a9496d..4f3874a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -595,6 +595,16 @@ func (in *ConversionWebhookServerSpec) DeepCopyInto(out *ConversionWebhookServer *out = new(v1.LabelSelector) (*in).DeepCopyInto(*out) } + if in.Sharding != nil { + in, out := &in.Sharding, &out.Sharding + *out = new(ShardingSpec) + (*in).DeepCopyInto(*out) + } + if in.StartupProbe != nil { + in, out := &in.StartupProbe, &out.StartupProbe + *out = new(StartupProbeSpec) + (*in).DeepCopyInto(*out) + } if in.Rollout != nil { in, out := &in.Rollout, &out.Rollout *out = new(RolloutSpec) @@ -634,6 +644,11 @@ func (in *ConversionWebhookServerStatus) DeepCopyInto(out *ConversionWebhookServ *out = make([]AssignedConfigRef, len(*in)) copy(*out, *in) } + if in.ServedTargets != nil { + in, out := &in.ServedTargets, &out.ServedTargets + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConversionWebhookServerStatus. @@ -1310,6 +1325,31 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardingSpec) DeepCopyInto(out *ShardingSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardingSpec. +func (in *ShardingSpec) DeepCopy() *ShardingSpec { + if in == nil { + return nil + } + out := new(ShardingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SingletonArrayToObjectParams) DeepCopyInto(out *SingletonArrayToObjectParams) { *out = *in @@ -1390,6 +1430,36 @@ func (in *SpokeVersionRules) DeepCopy() *SpokeVersionRules { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StartupProbeSpec) DeepCopyInto(out *StartupProbeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.PeriodSeconds != nil { + in, out := &in.PeriodSeconds, &out.PeriodSeconds + *out = new(int32) + **out = **in + } + if in.FailureThreshold != nil { + in, out := &in.FailureThreshold, &out.FailureThreshold + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StartupProbeSpec. +func (in *StartupProbeSpec) DeepCopy() *StartupProbeSpec { + if in == nil { + return nil + } + out := new(StartupProbeSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetCRDRef) DeepCopyInto(out *TargetCRDRef) { *out = *in diff --git a/charts/declarative-conversion-operator/README.md b/charts/declarative-conversion-operator/README.md index 586ee38..edbb93b 100644 --- a/charts/declarative-conversion-operator/README.md +++ b/charts/declarative-conversion-operator/README.md @@ -52,6 +52,10 @@ helm upgrade declarative-conversion-operator charts/declarative-conversion-opera | `certManager.issuerRef` | Issuer/ClusterIssuer for `ConversionWebhookServer` certificates | bootstrap self-signed `ClusterIssuer` | | `admissionWebhook.certificate.issuerRef` | Issuer/ClusterIssuer for this operator's own admission-webhook certificate (a separate trust surface) | bootstrap self-signed `ClusterIssuer` | | `conversionWebhookServer.autoscaling.enabled` | Use an HPA instead of a fixed replica count for the default instance | `false` | +| `conversionWebhookServer.sharding.enabled` / `.weight` | Distribute configs with no explicit `webhookServerRef` across every instance that opts in. Enable it on this (default) instance **before** any other, or admission rejects the pool | `null` (off) / `null` (1) | +| `conversionWebhookServer.startupProbe.periodSeconds` / `.failureThreshold` | The cold-start budget before the kubelet restarts a replica. Their product; the CRD default is five minutes | `null` (5) / `null` (60) | +| `manager.maxConcurrentReconciles` | Objects each controller reconciles at once. Raise when the dashboard's workqueue depth stays non-zero | `1` | +| `metrics.prometheusRule.workqueueDepthThreshold` / `.workqueueBacklogFor` / `.reconcileErrorRateThreshold` | Thresholds for the two controller-health alerts | `10` / `15m` / `0.1` | | `metrics.serviceMonitor.enabled` | Create Prometheus Operator `ServiceMonitor`s (opt-in; not auto-detected) | `false` | | `metrics.prometheusRule.enabled` | Create a `PrometheusRule` with built-in conversion/manager alerts | `false` | | `dashboards.enabled` | Create Grafana sidecar dashboard ConfigMaps (`grafana_dashboard: "1"`): Conversion Overview, Conversion Target Detail, and Conversion Platform Stability | `false` | diff --git a/charts/declarative-conversion-operator/crds/terasky.com_conversionwebhookservers.yaml b/charts/declarative-conversion-operator/crds/terasky.com_conversionwebhookservers.yaml index 4675085..b699812 100644 --- a/charts/declarative-conversion-operator/crds/terasky.com_conversionwebhookservers.yaml +++ b/charts/declarative-conversion-operator/crds/terasky.com_conversionwebhookservers.yaml @@ -1126,7 +1126,9 @@ spec: description: Selects a key of a ConfigMap. properties: key: - description: The key to select. + description: |- + The key to select from the ConfigMap's Data field. + Keys in the BinaryData field are not currently propagated to container env vars. type: string name: default: "" @@ -1259,10 +1261,21 @@ spec: description: VolumeMount describes a mounting of a Volume within a container. properties: - mountPath: + bindMountOptions: description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. + bindMountOptions is the list of additional bind mount options to apply when + mounting this volume into the container. Allowed values are noexec, + nodev, and nosuid. These are Linux mount options and have no effect on + Windows nodes. + This field is not supported with image volumes. + This is an alpha field and requires enabling the VolumeBindMountOptions feature gate. + items: + type: string + type: array + x-kubernetes-list-type: set + mountPath: + description: Path within the container at which the volume should + be mounted. type: string mountPropagation: description: |- @@ -1534,6 +1547,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer items: description: |- items if unspecified, each key-value pair in the Data field of the referenced @@ -1566,6 +1586,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -1652,6 +1679,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer items: description: Items is a list of downward API volume file items: @@ -1716,6 +1750,13 @@ spec: - resource type: object x-kubernetes-map-type: atomic + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -1734,6 +1775,18 @@ spec: Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer sizeLimit: anyOf: - type: integer @@ -1827,8 +1880,8 @@ spec: * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be + copied to dataSource when dataSourceRef.namespace is not specified. If the namespace is specified, then dataSourceRef will not be copied to dataSource. properties: apiGroup: @@ -1873,7 +1926,6 @@ spec: specified. * While dataSource only allows local objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. properties: apiGroup: @@ -2440,6 +2492,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer sources: description: |- sources is the list of volume projections. Each entry in this list @@ -2539,6 +2598,13 @@ spec: Mutually-exclusive with name. The contents of all selected ClusterTrustBundles will be unified and deduplicated. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -2579,6 +2645,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -2674,6 +2747,13 @@ spec: - resource type: object x-kubernetes-map-type: atomic + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -2781,6 +2861,13 @@ spec: description: Kubelet's generated CSRs will be addressed to this signer. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer userAnnotations: additionalProperties: type: string @@ -2840,6 +2927,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -2887,6 +2981,13 @@ spec: path is the path relative to the mount point of the file to project the token into. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -3093,6 +3194,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer items: description: |- items If unspecified, each key-value pair in the Data field of the referenced @@ -3125,6 +3233,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -3426,6 +3541,59 @@ spec: type: object serviceAccountName: type: string + sharding: + description: |- + Sharding opts this instance into the pool that unpinned conversion + configs are distributed across. See ShardingSpec. + properties: + enabled: + default: true + description: |- + Enabled adds this instance to the pool unpinned configs are + distributed across. + + While the pool is non-empty it, not spec.default, is what serves + unpinned configs — so the instance marked default must be a member + of it. Admission enforces that, because otherwise enabling sharding + on a second instance would silently drain every unpinned config off + the default one. Enable it on the default instance first. + type: boolean + weight: + default: 1 + description: |- + Weight biases this instance's share of the pool, for a fleet whose + instances are not the same size. An instance with weight 2 receives + approximately twice the share of one with weight 1. Weights are + relative, so scaling all of them changes nothing. + format: int32 + minimum: 1 + type: integer + type: object + startupProbe: + description: |- + StartupProbe bounds how long a replica may take to compile every + assigned plan before the kubelet restarts it. See StartupProbeSpec: + it polls /readyz, so its budget is a deadline on the sync itself. + properties: + enabled: + default: true + description: |- + Enabled renders the startupProbe, and defaults to true. Setting it + to false removes the deadline on the cold start entirely: the + liveness probe reads /healthz, which answers before the sync begins, + so nothing then restarts a replica that never finishes syncing. + type: boolean + failureThreshold: + default: 60 + format: int32 + minimum: 1 + type: integer + periodSeconds: + default: 5 + format: int32 + minimum: 1 + type: integer + type: object tolerations: items: description: |- @@ -3738,6 +3906,31 @@ spec: replicas: format: int32 type: integer + reportingReplicas: + description: |- + ReportingReplicas is how many live replica Leases fed + ServedTargets. A value below ReadyReplicas means at least one + ready replica has not published yet, and the intersection above is + not yet a statement about the whole instance. + format: int32 + type: integer + servedTargets: + description: |- + ServedTargets is the set of target resources that EVERY live + replica of this instance reports a compiled, servable plan for — + an intersection, not a union, because a target one replica out of + three can serve is a target that fails one request in three. + + Unlike AssignedConfigs, which is desired state computed by the + shared resolver, this is reported by the replicas themselves: each + publishes its own registry contents into a Lease, and this field + is the aggregate. It is what makes a safe handover possible — the + operator will not repoint a target at this instance until the + instance says it can already serve it. + items: + type: string + type: array + x-kubernetes-list-type: atomic type: object type: object served: true diff --git a/charts/declarative-conversion-operator/files/dashboards/conversion-overview.json b/charts/declarative-conversion-operator/files/dashboards/conversion-overview.json index d6b87a0..b8b7b97 100644 --- a/charts/declarative-conversion-operator/files/dashboards/conversion-overview.json +++ b/charts/declarative-conversion-operator/files/dashboards/conversion-overview.json @@ -176,6 +176,83 @@ "refId": "A" } ] + }, + { + "id": 12, + "title": "Controller health", + "type": "row", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, + "panels": [] + }, + { + "id": 13, + "title": "Workqueue depth", + "type": "timeseries", + "description": "Items waiting to be reconciled, per controller. This is the leading indicator: depth rises before phases go stale and before ConversionPropagated lags. Depth that returns to zero between bursts is healthy; depth that stays above zero means the controller cannot keep up, and --max-concurrent-reconciles is the lever.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [ + { + "expr": "sum by (job, controller) (workqueue_depth{app_kubernetes_io_name=~\"declarative-conversion-operator|declarative-conversion-webhook-server\"})", + "legendFormat": "{{job}} / {{controller}}", + "refId": "A" + } + ] + }, + { + "id": 14, + "title": "Workqueue add rate", + "type": "timeseries", + "description": "Enqueues per second. Read it next to depth: a high add rate with flat depth is a busy controller keeping up, while a low add rate with rising depth means individual reconciles are slow.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { + "expr": "sum by (job, controller) (rate(workqueue_adds_total{app_kubernetes_io_name=~\"declarative-conversion-operator|declarative-conversion-webhook-server\"}[5m]))", + "legendFormat": "{{job}} / {{controller}}", + "refId": "A" + } + ] + }, + { + "id": 15, + "title": "Workqueue work duration", + "type": "timeseries", + "description": "How long one reconcile takes. p99 pulling away from p50 points at a single slow target rather than at general load.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le, job, controller) (rate(workqueue_work_duration_seconds_bucket{app_kubernetes_io_name=~\"declarative-conversion-operator|declarative-conversion-webhook-server\"}[5m])))", + "legendFormat": "p50 {{job}} / {{controller}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, controller) (rate(workqueue_work_duration_seconds_bucket{app_kubernetes_io_name=~\"declarative-conversion-operator|declarative-conversion-webhook-server\"}[5m])))", + "legendFormat": "p99 {{job}} / {{controller}}", + "refId": "B" + } + ] + }, + { + "id": 16, + "title": "Reconcile error rate", + "type": "timeseries", + "description": "Reconciles returning an error, per second. These are retried with backoff, so a sustained rate both delays convergence and feeds the queue depth above.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { + "expr": "sum by (job, controller) (rate(controller_runtime_reconcile_errors_total{app_kubernetes_io_name=~\"declarative-conversion-operator|declarative-conversion-webhook-server\"}[5m]))", + "legendFormat": "{{job}} / {{controller}}", + "refId": "A" + } + ] } ], "schemaVersion": 39, diff --git a/charts/declarative-conversion-operator/templates/conversion-webhook-server/conversionwebhookserver.yaml b/charts/declarative-conversion-operator/templates/conversion-webhook-server/conversionwebhookserver.yaml index 5274a43..4741cf4 100755 --- a/charts/declarative-conversion-operator/templates/conversion-webhook-server/conversionwebhookserver.yaml +++ b/charts/declarative-conversion-operator/templates/conversion-webhook-server/conversionwebhookserver.yaml @@ -64,6 +64,26 @@ spec: rollout: {{- toYaml . | nindent 4 }} {{- end }} + {{- /* + sharding, filtered the same way. Emitting an empty block would be + worse than emitting nothing: `sharding: {}` is the opt-in, since the + CRD defaults enabled to true inside the block. + */}} + {{- $sh := .Values.conversionWebhookServer.sharding | default dict }} + {{- $shSet := dict }} + {{- range $k, $v := $sh }}{{- if not (kindIs "invalid" $v) }}{{- $_ := set $shSet $k $v }}{{- end }}{{- end }} + {{- with $shSet }} + sharding: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- /* startupProbe is filtered the same way and for the same reason. */}} + {{- $sp := .Values.conversionWebhookServer.startupProbe | default dict }} + {{- $spSet := dict }} + {{- range $k, $v := $sp }}{{- if not (kindIs "invalid" $v) }}{{- $_ := set $spSet $k $v }}{{- end }}{{- end }} + {{- with $spSet }} + startupProbe: + {{- toYaml . | nindent 4 }} + {{- end }} {{- if .Values.conversionWebhookServer.podDisruptionBudget.enabled }} podDisruptionBudget: minAvailable: {{ .Values.conversionWebhookServer.podDisruptionBudget.minAvailable }} diff --git a/charts/declarative-conversion-operator/templates/manager/deployment.yaml b/charts/declarative-conversion-operator/templates/manager/deployment.yaml index a00fda0..1c6e670 100644 --- a/charts/declarative-conversion-operator/templates/manager/deployment.yaml +++ b/charts/declarative-conversion-operator/templates/manager/deployment.yaml @@ -48,6 +48,7 @@ spec: - --enable-xrd-support={{ .Values.features.crossplane.enabled }} - --enable-crd-support={{ .Values.features.nativeCRD.enabled }} - --enable-xrd-conversion-guard={{ and .Values.features.crossplane.enabled .Values.features.crossplane.conversionGuard.enabled }} + - --max-concurrent-reconciles={{ int .Values.manager.maxConcurrentReconciles }} {{- with .Values.manager.extraArgs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/declarative-conversion-operator/templates/monitoring/prometheusrule.yaml b/charts/declarative-conversion-operator/templates/monitoring/prometheusrule.yaml index 71bed4a..2320bb8 100644 --- a/charts/declarative-conversion-operator/templates/monitoring/prometheusrule.yaml +++ b/charts/declarative-conversion-operator/templates/monitoring/prometheusrule.yaml @@ -113,6 +113,27 @@ spec: annotations: summary: "Config {{`{{ $labels.target }}`}} phase transitioned to {{`{{ $labels.to_phase }}`}}" description: "config_kind={{`{{ $labels.config_kind }}`}} from={{`{{ $labels.from_phase }}`}} reason={{`{{ $labels.reason }}`}}. Stale means KeepServingStale after drift; Failed usually means FailClosed revert." + - alert: ControllerWorkqueueBacklog + # Workqueue depth is the leading indicator: it rises before + # phases go stale, before ConversionPropagated lags, and before + # any of this operator's own metrics notice anything. A healthy + # controller drains its queue between bursts, so what matters is + # a depth that stays up, not one that spikes. + expr: sum by (job, controller) (workqueue_depth{app_kubernetes_io_name=~"declarative-conversion-operator|declarative-conversion-webhook-server"}) > {{ .Values.metrics.prometheusRule.workqueueDepthThreshold | default 10 }} + for: {{ .Values.metrics.prometheusRule.workqueueBacklogFor | default "15m" }} + labels: + severity: warning + annotations: + summary: "Reconcile backlog on controller {{`{{ $labels.controller }}`}}" + description: "job={{`{{ $labels.job }}`}}. This controller's workqueue has stayed above the backlog threshold, so config changes are converging slowly or not at all — expect stale phases and a lagging ConversionPropagated to follow. Check reconcile latency and error rate on the same dashboard row, then raise --max-concurrent-reconciles if the apiserver has the headroom." + - alert: ControllerReconcileErrors + expr: sum by (job, controller) (rate(controller_runtime_reconcile_errors_total{app_kubernetes_io_name=~"declarative-conversion-operator|declarative-conversion-webhook-server"}[5m])) > {{ .Values.metrics.prometheusRule.reconcileErrorRateThreshold | default 0.1 }} + for: 15m + labels: + severity: warning + annotations: + summary: "Sustained reconcile errors on controller {{`{{ $labels.controller }}`}}" + description: "job={{`{{ $labels.job }}`}}. Reconciles are returning errors faster than the threshold and being retried with backoff, which both delays convergence and feeds the workqueue depth. Unlike dco_manager_analyze_failures_total this counts infrastructure failures too — a lost API connection, a rejected write — so check the controller's logs rather than the config's status first." - alert: ConversionWebhookReplicaNotReady expr: dco_webhook_ready == 0 for: 5m diff --git a/charts/declarative-conversion-operator/templates/monitoring/servicemonitor.yaml b/charts/declarative-conversion-operator/templates/monitoring/servicemonitor.yaml index 3f83dab..6d7482a 100644 --- a/charts/declarative-conversion-operator/templates/monitoring/servicemonitor.yaml +++ b/charts/declarative-conversion-operator/templates/monitoring/servicemonitor.yaml @@ -10,6 +10,15 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + # Copied onto every series this scrape produces, so the shipped + # controller-health panels and alerts can tell this operator's + # controller-runtime metrics — workqueue_*, controller_runtime_* — from + # any other operator's in the same Prometheus. Those metric names are + # library-wide, not ours, and without a label to select on an alert + # about somebody else's reconcile backlog would arrive attributed to + # this chart. + targetLabels: + - app.kubernetes.io/name selector: matchLabels: {{- include "declarative-conversion-operator.managerSelectorLabels" . | nindent 6 }} @@ -33,6 +42,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + # See the manager ServiceMonitor above. + targetLabels: + - app.kubernetes.io/name namespaceSelector: any: true selector: diff --git a/charts/declarative-conversion-operator/templates/rbac/clusterrole.yaml b/charts/declarative-conversion-operator/templates/rbac/clusterrole.yaml index 18b257f..278366e 100644 --- a/charts/declarative-conversion-operator/templates/rbac/clusterrole.yaml +++ b/charts/declarative-conversion-operator/templates/rbac/clusterrole.yaml @@ -78,11 +78,18 @@ rules: resources: ["conversionwebhookservers/status", "crdconversionconfigs/status", "xrdconversionconfigs/status"] verbs: ["get", "patch", "update"] --- -# RBAC for cmd/webhook-server pods. Deliberately minimal and read-only: -# each replica runs its own controller-runtime informers directly against -# the API server so it can compile its own in-memory registry without -# depending on the operator manager at request time. Nothing here grants -# write access — the webhook-server binary never mutates any object. +# RBAC for cmd/webhook-server pods. Deliberately minimal: each replica runs +# its own controller-runtime informers directly against the API server so +# it can compile its own in-memory registry without depending on the +# operator manager at request time. +# +# Cluster-wide, this is read-only — the webhook-server never mutates an +# XRD, a CRD, or a conversion config. The one thing it does write is its +# own served-target Lease, and that grant is a namespaced Role (below) +# rather than part of this ClusterRole, so a webhook-server pod cannot +# touch a Lease anywhere else in the cluster. Leases are how leader +# election is implemented across the ecosystem; cluster-wide write on them +# is not a grant to hand out for a bookkeeping annotation. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -103,4 +110,57 @@ rules: resources: ["customresourcedefinitions"] verbs: ["get", "list", "watch"] {{- end }} +--- +# The one write a webhook-server replica makes: its own served-target +# Lease, which is what lets the operator verify an instance can already +# serve a target before moving that target onto it. +# +# Namespaced, and in the namespace the chart's own ConversionWebhookServer +# instance actually runs in — spec.namespace, which defaults to the release +# namespace but does not have to equal it. A ConversionWebhookServer +# created outside this chart needs the same Role and RoleBinding in its own +# namespace. Without it the replicas still serve conversions perfectly +# well; what is lost is the verified handover, and the config reports +# HandoverReady with reason HandoverUnverified rather than failing. +# +# No list or watch: a replica reads back exactly one Lease, by name, and +# enumerating the namespace's Leases would hand it every leader-election +# holder identity for nothing. +# +# What this is NOT: an own-Lease restriction. RBAC cannot express "only the +# Lease named after your own pod" — resourceNames needs names known when +# the Role is written, and these are derived from generated pod names. A +# compromised webhook-server pod can therefore get/update/patch any Lease +# in this namespace, including the operator's own leader-election Lease. +# The bound that does hold is the namespace: give an instance its own +# spec.namespace if that residual matters to you, and this grant reaches +# nothing else. See docs/security/rbac.md. +{{- $cwsNamespace := default .Release.Namespace .Values.conversionWebhookServer.namespace }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "declarative-conversion-operator.fullname" . }}-webhook-server-leases + namespace: {{ $cwsNamespace }} + labels: + {{- include "declarative-conversion-operator.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "declarative-conversion-operator.fullname" . }}-webhook-server-leases + namespace: {{ $cwsNamespace }} + labels: + {{- include "declarative-conversion-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "declarative-conversion-operator.fullname" . }}-webhook-server-leases +subjects: + - kind: ServiceAccount + name: {{ include "declarative-conversion-operator.webhookServerServiceAccountName" . }} + namespace: {{ .Release.Namespace }} {{- end }} diff --git a/charts/declarative-conversion-operator/tests/conversionwebhookserver_test.yaml b/charts/declarative-conversion-operator/tests/conversionwebhookserver_test.yaml index 7087137..42f8add 100644 --- a/charts/declarative-conversion-operator/tests/conversionwebhookserver_test.yaml +++ b/charts/declarative-conversion-operator/tests/conversionwebhookserver_test.yaml @@ -32,6 +32,43 @@ tests: - isNull: path: spec.rollout.defaultTopologySpread + - it: emits no sharding block when nothing is set + asserts: + # An empty `sharding: {}` would be the opt-in, since the CRD defaults + # enabled to true inside the block — so "nothing set" has to render + # nothing at all, not an empty map. + - isNull: + path: spec.sharding + + - it: emits only the sharding keys that were set + set: + conversionWebhookServer.sharding.weight: 3 + asserts: + - equal: + path: spec.sharding.weight + value: 3 + - isNull: + path: spec.sharding.enabled + + - it: emits no startupProbe block when nothing is set + asserts: + - isNull: + path: spec.startupProbe + + - it: emits only the startupProbe keys that were set + set: + conversionWebhookServer.startupProbe.failureThreshold: 120 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 120 + # Same reasoning as rollout above: an explicit null here would + # override the CRD's default period with nothing. + - isNull: + path: spec.startupProbe.periodSeconds + - isNull: + path: spec.startupProbe.enabled + - it: emits no extraArgs when nothing is set asserts: - isNull: diff --git a/charts/declarative-conversion-operator/tests/rbac_test.yaml b/charts/declarative-conversion-operator/tests/rbac_test.yaml index 42c38ed..fbb7a56 100644 --- a/charts/declarative-conversion-operator/tests/rbac_test.yaml +++ b/charts/declarative-conversion-operator/tests/rbac_test.yaml @@ -66,3 +66,57 @@ tests: asserts: - hasDocuments: count: 0 + + # The one write a webhook-server pod makes has to stay namespaced. A + # cluster-wide leases grant would let a conversion pod overwrite any + # leader-election Lease in the cluster, which is not a trade worth making + # for a bookkeeping annotation. + - it: grants webhook-server Lease access through a namespaced Role only + asserts: + - hasDocuments: + count: 4 + - isKind: + of: Role + documentIndex: 2 + - equal: + path: metadata.namespace + value: NAMESPACE + documentIndex: 2 + - equal: + path: rules + value: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "patch"] + documentIndex: 2 + # The webhook-server ClusterRole must stay free of it. + - notMatchRegex: + path: rules[*].apiGroups[*] + pattern: coordination.k8s.io + documentIndex: 1 + - isKind: + of: RoleBinding + documentIndex: 3 + + # The Role has to land where the webhook-server pods actually run. If it + # followed the release namespace instead, an instance with its own + # spec.namespace would have replicas that cannot publish their Leases, + # and every move onto it would silently take the unverified path. + - it: creates the Lease Role in the webhook-server namespace, not the release namespace + set: + conversionWebhookServer.namespace: tenant-a + asserts: + - equal: + path: metadata.namespace + value: tenant-a + documentIndex: 2 + - equal: + path: metadata.namespace + value: tenant-a + documentIndex: 3 + # The ServiceAccount still lives in the release namespace; only the + # Role and its binding move. + - equal: + path: subjects[0].namespace + value: NAMESPACE + documentIndex: 3 diff --git a/charts/declarative-conversion-operator/tests/toggles_test.yaml b/charts/declarative-conversion-operator/tests/toggles_test.yaml index 1671362..cd977c7 100644 --- a/charts/declarative-conversion-operator/tests/toggles_test.yaml +++ b/charts/declarative-conversion-operator/tests/toggles_test.yaml @@ -107,3 +107,13 @@ tests: asserts: - isKind: of: NetworkPolicy + + - it: passes the reconcile concurrency through to the manager + templates: + - manager/deployment.yaml + set: + manager.maxConcurrentReconciles: 4 + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --max-concurrent-reconciles=4 diff --git a/charts/declarative-conversion-operator/values.schema.json b/charts/declarative-conversion-operator/values.schema.json index ba06b73..515a13f 100644 --- a/charts/declarative-conversion-operator/values.schema.json +++ b/charts/declarative-conversion-operator/values.schema.json @@ -68,6 +68,7 @@ "additionalProperties": false, "properties": { "replicaCount": { "type": "integer", "minimum": 0 }, + "maxConcurrentReconciles": { "type": "integer", "minimum": 1 }, "leaderElection": { "type": "object", "additionalProperties": false, @@ -209,6 +210,25 @@ "defaultTopologySpread": { "type": ["boolean", "null"] } } }, + "sharding": { + "type": "object", + "additionalProperties": false, + "description": "Automatic assignment across instances. Null leaves it off. Enable it on the default instance before enabling it anywhere else — admission rejects a pool the default instance is not in.", + "properties": { + "enabled": { "type": ["boolean", "null"] }, + "weight": { "type": ["integer", "null"], "minimum": 1 } + } + }, + "startupProbe": { + "type": "object", + "additionalProperties": false, + "description": "How long a replica may take to compile every assigned plan before the kubelet restarts it. periodSeconds x failureThreshold is the cold-start budget; null leaves the CRD's own default of 5 x 60, i.e. five minutes.", + "properties": { + "enabled": { "type": ["boolean", "null"] }, + "periodSeconds": { "type": ["integer", "null"], "minimum": 1 }, + "failureThreshold": { "type": ["integer", "null"], "minimum": 1 } + } + }, "resources": { "$ref": "#/definitions/resources" }, "certificate": { "$ref": "#/definitions/certificate" }, "service": { @@ -246,7 +266,10 @@ "properties": { "enabled": { "type": "boolean" }, "labels": { "type": "object", "additionalProperties": { "type": "string" } }, - "propagationLagFor": { "$ref": "#/definitions/positiveDuration" } + "propagationLagFor": { "$ref": "#/definitions/positiveDuration" }, + "workqueueDepthThreshold": { "type": "integer", "minimum": 0 }, + "workqueueBacklogFor": { "$ref": "#/definitions/positiveDuration" }, + "reconcileErrorRateThreshold": { "type": "number", "minimum": 0 } } } } diff --git a/charts/declarative-conversion-operator/values.yaml b/charts/declarative-conversion-operator/values.yaml index 044e5c4..dcdc9f2 100755 --- a/charts/declarative-conversion-operator/values.yaml +++ b/charts/declarative-conversion-operator/values.yaml @@ -85,6 +85,12 @@ manager: replicaCount: 1 leaderElection: enabled: true + # How many objects each controller reconciles at once. The shipped + # dashboard's "Controller health" row plots workqueue depth, which is the + # leading indicator of a backlog; raise this when depth is persistently + # non-zero and the apiserver has the QPS headroom. A given object is + # never reconciled by two workers at once regardless of this value. + maxConcurrentReconciles: 1 image: {} resources: requests: @@ -199,6 +205,28 @@ conversionWebhookServer: maxUnavailable: null maxSurge: null defaultTopologySpread: null + # How long a replica may take to compile every assigned plan before the + # kubelet gives up on it. The probe polls /readyz, which stays false + # until the initial sync completes, so periodSeconds x failureThreshold + # is a deadline on the sync itself; the CRD's defaults give five minutes. + # Without it a replica wedged mid-sync would stay liveness-healthy and + # never ready, out of the Service and never restarted. See + # docs/operations/capacity.md for the measured cold start this is sized + # against. Leave a key null to take the CRD's default. + startupProbe: + enabled: null + periodSeconds: null + failureThreshold: null + # Automatic assignment. Off unless you set it. When any instance opts + # in, the pool of opted-in instances — not spec.default — is what serves + # configs with no explicit webhookServerRef, distributed by rendezvous + # hashing on the target name. The instance marked default must be a pool + # member while a pool exists, which admission enforces, so enable this + # on the chart's own default instance FIRST and on additional instances + # afterwards. See docs/configuration/conversionwebhookserver.md. + sharding: + enabled: null + weight: null resources: requests: cpu: 50m @@ -237,6 +265,18 @@ metrics: # comfortably longer than a healthy re-render and shorter than anyone's # patience with silently-unconverted reads. propagationLagFor: 5m + # Workqueue depth is the leading indicator of a reconcile backlog: it + # rises before phases go stale and before ConversionPropagated lags. A + # healthy controller drains its queue between bursts, so the alert is + # about a depth that STAYS up — tune the threshold to your fleet size + # (a bulk apply of two hundred configs legitimately spikes it) and the + # duration to how long a backlog may last before it is a problem. + workqueueDepthThreshold: 10 + workqueueBacklogFor: 15m + # Reconciles returning an error, per second, sustained for 15m. Counts + # infrastructure failures as well as config ones, so it is noisier than + # dco_manager_analyze_failures_total and set above zero on purpose. + reconcileErrorRateThreshold: 0.1 # Grafana sidecar ConfigMaps (label grafana_dashboard: "1") for every JSON # file under files/dashboards/ (overview, per-target detail, platform diff --git a/cmd/manager/main.go b/cmd/manager/main.go index d498e7b..df21862 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -66,6 +66,7 @@ func main() { enableXRDSupport bool enableCRDSupport bool enableXRDGuard bool + maxConcurrent int ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metrics endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -78,6 +79,10 @@ func main() { flag.BoolVar(&enableXRDGuard, "enable-xrd-conversion-guard", true, "Register a mutating admission webhook on compositeresourcedefinitions that re-injects spec.conversion into writes that would drop it. "+ "Exists because Crossplane's package establisher writes established objects with a full client.Update rather than a Server-Side Apply, stripping the field on every revision reconcile. "+ "Both upstream write paths carry a TODO to move to SSA; turn this off once a Crossplane version lands that no longer needs it. Requires --enable-xrd-support.") + flag.IntVar(&maxConcurrent, "max-concurrent-reconciles", controller.DefaultMaxConcurrentReconciles, + "How many objects each controller reconciles at once. The shipped dashboard plots workqueue depth per controller; "+ + "raise this when depth is persistently non-zero and the apiserver has the headroom, since the cost is QPS. "+ + "A given object is never reconciled by two workers at once regardless of this value.") zapOpts := zap.Options{Development: false} zapOpts.BindFlags(flag.CommandLine) flag.Parse() @@ -85,6 +90,11 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts))) logger := ctrl.Log.WithName("manager") + if maxConcurrent < 1 { + fmt.Fprintf(os.Stderr, "--max-concurrent-reconciles must be at least 1, got %d\n", maxConcurrent) + os.Exit(1) + } + namespace := currentNamespace() restConfig := ctrl.GetConfigOrDie() @@ -136,9 +146,10 @@ func main() { // active. if enableXRDSupport { if err := (&controller.XRDConversionConfigReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - DefaultServerNamespace: namespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + DefaultServerNamespace: namespace, + MaxConcurrentReconciles: maxConcurrent, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "XRDConversionConfig") os.Exit(1) @@ -148,9 +159,10 @@ func main() { } if enableCRDSupport { if err := (&controller.CRDConversionConfigReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - DefaultServerNamespace: namespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + DefaultServerNamespace: namespace, + MaxConcurrentReconciles: maxConcurrent, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "CRDConversionConfig") os.Exit(1) @@ -159,12 +171,13 @@ func main() { logger.Info("native CRD support disabled (--enable-crd-support=false)") } if err := (&controller.ConversionWebhookServerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - DefaultNamespace: namespace, - DefaultImage: defaultImage, - EnableXRDSupport: enableXRDSupport, - EnableCRDSupport: enableCRDSupport, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + DefaultNamespace: namespace, + DefaultImage: defaultImage, + EnableXRDSupport: enableXRDSupport, + EnableCRDSupport: enableCRDSupport, + MaxConcurrentReconciles: maxConcurrent, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "ConversionWebhookServer") os.Exit(1) diff --git a/cmd/scalegen/main.go b/cmd/scalegen/main.go index 7603ecf..5e6ba59 100644 --- a/cmd/scalegen/main.go +++ b/cmd/scalegen/main.go @@ -32,6 +32,7 @@ import ( func main() { opts := scalegen.Options{Out: os.Stdout} var qps float64 + var resultJSON string flag.StringVar(&opts.Kubeconfig, "kubeconfig", "", "kubeconfig path (default: KUBECONFIG / in-cluster)") flag.StringVar(&opts.Namespace, "namespace", "dco-scale", "namespace for generated CRs") flag.IntVar(&opts.Targets, "targets", 4, "number of CRDs to generate (each has 3 versions)") @@ -47,11 +48,22 @@ func main() { flag.IntVar(&opts.Burst, "burst", 200, "client-go burst") flag.BoolVar(&opts.Reset, "reset", false, "delete previously generated CRDs in this group before applying") flag.BoolVar(&opts.DryRun, "dry-run", false, "print strategy coverage without talking to a cluster") + flag.StringVar(&resultJSON, "result-json", "", "write the run's measurements to this path as JSON, for publishing as an artifact and diffing against a previous run") flag.Parse() opts.QPS = float32(qps) - if _, err := scalegen.Run(context.Background(), opts); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "scalegen: %v\n", err) + res, runErr := scalegen.Run(context.Background(), opts) + // Written even when the run reported errors: a scale run that failed + // is exactly when the numbers are worth keeping, and a scheduled job + // with no artifact to look at is a job nobody can act on. + if resultJSON != "" && res != nil { + if err := scalegen.WriteReport(resultJSON, res.ToReport(time.Now())); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "scalegen: writing %s: %v\n", resultJSON, err) + os.Exit(1) + } + } + if runErr != nil { + _, _ = fmt.Fprintf(os.Stderr, "scalegen: %v\n", runErr) os.Exit(1) } } diff --git a/cmd/webhook-server/main.go b/cmd/webhook-server/main.go index e61d0f7..b4ad6c5 100755 --- a/cmd/webhook-server/main.go +++ b/cmd/webhook-server/main.go @@ -33,10 +33,11 @@ import ( "syscall" "time" + "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -49,6 +50,54 @@ import ( var scheme = runtime.NewScheme() +// initialSyncRetryInterval is how long to wait before re-attempting an +// initial sync that hit an infrastructure error. Short, because the +// startupProbe budget is what it is spending. +const initialSyncRetryInterval = 2 * time.Second + +// initialSyncWithRetry runs the startup pass until it completes without an +// infrastructure error, or the context ends. +// +// There is no attempt cap on purpose, and it is the same decision as the +// one against --registry-ready-timeout: a replica that never becomes ready +// degrades throughput, while one that reports ready with a hole in its +// registry answers ConversionReviews for the missing target with a failure +// the apiserver turns into a failed write. The startupProbe bounds how +// long this may go on, and the log line says what it is waiting for. +func initialSyncWithRetry(ctx context.Context, logger logr.Logger, reconciler *webhookserver.Reconciler) (webhookserver.InitialSyncStats, error) { + for attempt := 1; ; attempt++ { + stats, err := reconciler.InitialSync(ctx) + if err == nil { + return stats, nil + } + if ctx.Err() != nil { + return stats, ctx.Err() + } + logger.Error(err, "initial registry sync hit an infrastructure error; retrying before reporting ready", + "attempt", attempt, "retryIn", initialSyncRetryInterval.String()) + select { + case <-ctx.Done(): + return stats, ctx.Err() + case <-time.After(initialSyncRetryInterval): + } + } +} + +// effectiveInitialSyncWorkers reports the pool size InitialSync actually +// used, so the cold-start log line states the parallelism that produced +// the elapsed time next to it rather than the flag value, which is 0 by +// default and says nothing. +func effectiveInitialSyncWorkers(flagValue, targets int) int { + workers := flagValue + if workers <= 0 { + workers = webhookserver.DefaultInitialSyncWorkers() + } + if targets > 0 && workers > targets { + workers = targets + } + return workers +} + func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(teraskyv1alpha1.AddToScheme(scheme)) @@ -71,6 +120,7 @@ func main() { maxRequestBytes int64 requestTimeout time.Duration shutdownTimeout time.Duration + initialSyncPar int ) flag.StringVar(&serverName, "webhook-server-name", "", "Name of the ConversionWebhookServer instance this replica belongs to (required).") flag.StringVar(&tlsCertDir, "tls-cert-dir", "/tls", "Directory containing tls.crt and tls.key for the conversion endpoint.") @@ -85,6 +135,7 @@ func main() { flag.StringVar(&cacheSelector, "cache-label-selector", "", "JSON metav1.LabelSelector scoping this replica's informers. It covers the XRDConversionConfig and CRDConversionConfig objects AND the CustomResourceDefinition/CompositeResourceDefinition objects holding their schemas, so targets must carry the label too. Empty watches everything.") flag.Int64Var(&maxRequestBytes, "max-request-bytes", webhookserver.DefaultMaxRequestBytes, "Maximum ConversionReview request body size. A larger body is answered with a ConversionReview failure rather than being read. Raise it if legitimate batches are being rejected.") flag.DurationVar(&requestTimeout, "request-timeout", webhookserver.DefaultRequestTimeout, "Maximum time one ConversionReview may occupy a worker. Must stay below the apiserver's own fixed 30s conversion timeout plus this server's write timeout.") + flag.IntVar(&initialSyncPar, "initial-sync-workers", 0, "How many conversion plans this replica compiles concurrently during its cold start. 0 uses GOMAXPROCS. Compilation is CPU-bound and independent per target; the bound exists because each worker holds a decoded schema and a half-built plan. Lower it if the cold start is competing with something else for CPU; raising it above GOMAXPROCS buys nothing.") flag.DurationVar(&shutdownTimeout, "shutdown-timeout", webhookserver.DefaultShutdownTimeout, "How long to let in-flight ConversionReviews finish after a termination signal. This value plus the pod's preStop sleep must stay below terminationGracePeriodSeconds, or the kubelet SIGKILLs mid-review and the apiserver reports a failed write.") opts := ctrl.Options{Scheme: scheme} zapOpts := zap.Options{Development: false} @@ -114,6 +165,10 @@ func main() { fmt.Fprintf(os.Stderr, "--shutdown-timeout must be positive, got %s\n", shutdownTimeout) os.Exit(1) } + if initialSyncPar < 0 { + fmt.Fprintf(os.Stderr, "--initial-sync-workers must not be negative, got %d\n", initialSyncPar) + os.Exit(1) + } rootCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -145,6 +200,7 @@ func main() { os.Exit(1) } opts.Cache = cacheOpts + opts.Client = webhookserver.ClientOptions() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), opts) if err != nil { logger.Error(err, "unable to start manager") @@ -154,21 +210,52 @@ func main() { registry := webhookserver.NewRegistry() metricsReg := prometheus.NewRegistry() // A dedicated registry starts empty — unlike the process-wide default - // one, it has no Go runtime or process collectors. Without these, - // /metrics exposes this operator's own counters and nothing about the - // process serving them: no resident memory, no goroutine count, no GC - // behaviour. That makes the replica's memory footprint — the thing + // one, it has no Go runtime or process collectors. Without those, + // /metrics would expose this operator's own counters and nothing about + // the process serving them: no resident memory, no goroutine count, no + // GC behaviour, which makes the replica's memory footprint — the thing // --cache-label-selector exists to control — unmeasurable from outside // the pod. - metricsReg.MustRegister( - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - ) - metrics := webhookserver.NewMetrics(metricsReg, metricsReg) + // + // They are not registered here. controller-runtime's own registry, + // which CombinedGatherer pairs with this one, already carries both — + // and its Go collector is configured with the full runtime/metrics + // set, a superset of the plain one. Registering a second copy here + // would be a duplicate metric name, and prometheus.Gatherers fails the + // whole scrape on one of those, taking every dco_webhook_* series with + // it. Asserted by TestCombinedGatherer_NoDuplicateSeries. + // The gatherer is a pair, not just metricsReg: controller-runtime + // registers its own workqueue depth/latency and reconcile counters on + // its package-global registry, and this binary serves /metrics from a + // dedicated one — so without this, a replica's registry reconcile loop + // was the one controller in the system with no queue-depth signal at + // all. Gathering both means the shipped "Controller health" dashboard + // row covers the webhook-server as well as the manager. + metrics := webhookserver.NewMetrics(metricsReg, webhookserver.CombinedGatherer(metricsReg)) + + // The publisher writes this replica's servable target set into its own + // Lease, which is how the operator knows it is safe to move a target + // onto this instance. Its identity comes from the downward API; a + // Deployment that predates those env vars leaves it disabled, which + // costs nothing but the ability to receive moved work safely. + publisher := &webhookserver.TargetPublisher{ + Client: mgr.GetClient(), + Registry: registry, + ServerName: serverName, + Namespace: os.Getenv("POD_NAMESPACE"), + PodName: os.Getenv("POD_NAME"), + PodUID: types.UID(os.Getenv("POD_UID")), + } + publisher.Init() + if !publisher.Enabled() { + logger.Info("served-target publishing is disabled: POD_NAME/POD_NAMESPACE/POD_UID are not all set. " + + "Conversions are unaffected, but the operator cannot verify this instance is ready before moving a target onto it") + } reconciler := &webhookserver.Reconciler{ Client: mgr.GetClient(), ServerName: serverName, Registry: registry, Metrics: metrics, EnableXRDSupport: enableXRDSupport, EnableCRDSupport: enableCRDSupport, + InitialSyncWorkers: initialSyncPar, Publisher: publisher, } if err := reconciler.SetupWithManager(mgr); err != nil { logger.Error(err, "unable to set up registry reconciler") @@ -188,20 +275,6 @@ func main() { ctx := rootCtx - mgrErrCh := make(chan error, 1) - go func() { mgrErrCh <- mgr.Start(ctx) }() - go certReloader.Run(ctx) - - if !mgr.GetCache().WaitForCacheSync(ctx) { - logger.Error(errors.New("cache sync failed"), "unable to sync cache before initial registry population") - os.Exit(1) - } - if err := reconciler.InitialSync(ctx); err != nil { - logger.Error(err, "initial registry sync encountered errors; continuing, affected XRDs will retry via watch events") - } - server.SetReady(true) - logger.Info("registry synced, marking replica ready", "serverName", serverName) - // Both servers carry the same timeouts. The conversion endpoint needs // them because it is in the apiserver's write path; the plain endpoint // needs them because it also serves /debug/registry, and an endpoint @@ -228,17 +301,95 @@ func main() { MaxHeaderBytes: webhookserver.DefaultMaxHeaderBytes, } + mgrErrCh := make(chan error, 1) + go func() { mgrErrCh <- mgr.Start(ctx) }() + go certReloader.Run(ctx) + + // The plain endpoint comes up before the cache sync, not after it. It + // carries /healthz, /readyz and /metrics, and a cold start is the one + // time those are worth having: previously nothing listened until the + // registry was fully compiled, so a slow start was indistinguishable + // from a hung process — every probe got connection-refused and the + // only evidence was the pod log. /readyz stays false throughout (it + // reads the same gate SetReady flips below), so nothing joins the + // Service early; what changes is that the startupProbe now measures a + // live process rather than an absent listener. go func() { - logger.Info("serving conversion requests", "address", conversionAddr) - if err := conversionSrv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { - logger.Error(err, "conversion server exited unexpectedly") + logger.Info("serving health/metrics/debug", "address", plainAddr) + if err := plainSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error(err, "plain HTTP server exited unexpectedly") os.Exit(1) } }() + + // Racing the cache sync against the manager's own exit. WaitForCacheSync + // returns false only when ctx is done, so on its own it blocks forever + // if mgr.Start fails early — a missing RBAC verb on a watched kind, say. + // That used to be survivable by accident: nothing listened yet, so the + // probes failed and the kubelet restarted the pod. Now the plain + // endpoint is up and answering /healthz, so liveness passes and the pod + // would sit not-ready forever with the manager's error never logged. + syncedCh := make(chan bool, 1) + go func() { syncedCh <- mgr.GetCache().WaitForCacheSync(ctx) }() + select { + case synced := <-syncedCh: + if !synced { + logger.Error(errors.New("cache sync failed"), "unable to sync cache before initial registry population") + os.Exit(1) + } + case err := <-mgrErrCh: + logger.Error(err, "manager exited before the cache finished syncing; the replica cannot serve conversions") + os.Exit(1) + } + // Readiness stays strictly behind a completed InitialSync. A + // "--registry-ready-timeout" that reported ready anyway after N + // seconds was considered and deliberately not added: a replica that + // joins the Service with a partially-populated registry answers + // ConversionReviews for the targets it has not compiled yet with a + // failure, and the apiserver turns that into a failed write on an + // unrelated resource. An unavailable replica degrades throughput; a + // half-loaded one corrupts the answer. The startupProbe is the + // supported lever for a slow cold start, sized from the budget this + // metric and log line publish. See docs/operations/capacity.md. + // + // An infrastructure failure during the sync — a failed read of a target, + // a failed server list — is retried here rather than shrugged off. The + // watch-driven reconciler only retries what a later watch event + // re-delivers, and a transient Get failure at startup may never produce + // one: the config would simply be missing from this replica's registry + // until somebody edited it. Retrying until it succeeds keeps readiness + // honest; the startupProbe is what bounds how long that may take. + syncStats, err := initialSyncWithRetry(ctx, logger, reconciler) + if err != nil { + // Only reachable when the context is done, i.e. the process is + // shutting down. Falling through to SetReady would advertise a + // registry that was never completed. + logger.Error(err, "shutting down before the initial registry sync completed") + return + } + // Published synchronously, before readiness rather than after: a + // replica that is about to start taking traffic should already be + // eligible to receive moved work, not eligible one heartbeat later. + if err := publisher.Publish(ctx); err != nil { + logger.Error(err, "unable to publish served targets after the initial sync; retrying on the heartbeat") + } + go publisher.Run(ctx) + + server.SetReady(true) + logger.Info("registry synced, marking replica ready", + "serverName", serverName, + "targets", syncStats.Targets, + "workers", effectiveInitialSyncWorkers(initialSyncPar, syncStats.Targets), + "elapsed", syncStats.Duration.String()) + + // The conversion endpoint, unlike the plain one, only starts once the + // registry is populated: it is on the apiserver's write path, and a + // listener that accepts before it can answer correctly is worse than + // no listener at all. go func() { - logger.Info("serving health/metrics/debug", "address", plainAddr) - if err := plainSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - logger.Error(err, "plain HTTP server exited unexpectedly") + logger.Info("serving conversion requests", "address", conversionAddr) + if err := conversionSrv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { + logger.Error(err, "conversion server exited unexpectedly") os.Exit(1) } }() diff --git a/config/crd/bases/terasky.com_conversionwebhookservers.yaml b/config/crd/bases/terasky.com_conversionwebhookservers.yaml index 4675085..b699812 100644 --- a/config/crd/bases/terasky.com_conversionwebhookservers.yaml +++ b/config/crd/bases/terasky.com_conversionwebhookservers.yaml @@ -1126,7 +1126,9 @@ spec: description: Selects a key of a ConfigMap. properties: key: - description: The key to select. + description: |- + The key to select from the ConfigMap's Data field. + Keys in the BinaryData field are not currently propagated to container env vars. type: string name: default: "" @@ -1259,10 +1261,21 @@ spec: description: VolumeMount describes a mounting of a Volume within a container. properties: - mountPath: + bindMountOptions: description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. + bindMountOptions is the list of additional bind mount options to apply when + mounting this volume into the container. Allowed values are noexec, + nodev, and nosuid. These are Linux mount options and have no effect on + Windows nodes. + This field is not supported with image volumes. + This is an alpha field and requires enabling the VolumeBindMountOptions feature gate. + items: + type: string + type: array + x-kubernetes-list-type: set + mountPath: + description: Path within the container at which the volume should + be mounted. type: string mountPropagation: description: |- @@ -1534,6 +1547,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer items: description: |- items if unspecified, each key-value pair in the Data field of the referenced @@ -1566,6 +1586,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -1652,6 +1679,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer items: description: Items is a list of downward API volume file items: @@ -1716,6 +1750,13 @@ spec: - resource type: object x-kubernetes-map-type: atomic + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -1734,6 +1775,18 @@ spec: Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer sizeLimit: anyOf: - type: integer @@ -1827,8 +1880,8 @@ spec: * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be + copied to dataSource when dataSourceRef.namespace is not specified. If the namespace is specified, then dataSourceRef will not be copied to dataSource. properties: apiGroup: @@ -1873,7 +1926,6 @@ spec: specified. * While dataSource only allows local objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. properties: apiGroup: @@ -2440,6 +2492,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer sources: description: |- sources is the list of volume projections. Each entry in this list @@ -2539,6 +2598,13 @@ spec: Mutually-exclusive with name. The contents of all selected ClusterTrustBundles will be unified and deduplicated. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -2579,6 +2645,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -2674,6 +2747,13 @@ spec: - resource type: object x-kubernetes-map-type: atomic + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -2781,6 +2861,13 @@ spec: description: Kubelet's generated CSRs will be addressed to this signer. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer userAnnotations: additionalProperties: type: string @@ -2840,6 +2927,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -2887,6 +2981,13 @@ spec: path is the path relative to the mount point of the file to project the token into. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - path type: object @@ -3093,6 +3194,13 @@ spec: mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer + defaultUser: + description: |- + defaultUser is Optional: The owner UID of the created files by default. + The defaultUser field is only used as a fallback when the item-level user field is unset. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer items: description: |- items If unspecified, each key-value pair in the Data field of the referenced @@ -3125,6 +3233,13 @@ spec: May not contain the path element '..'. May not start with the string '..'. type: string + user: + description: |- + user is Optional: The owner UID of the created file. + If specified, the item-level user field takes precedence over defaultUser. + (Alpha) This field requires the AtomicWriteVolumeUserFields feature gate to be enabled. + format: int64 + type: integer required: - key - path @@ -3426,6 +3541,59 @@ spec: type: object serviceAccountName: type: string + sharding: + description: |- + Sharding opts this instance into the pool that unpinned conversion + configs are distributed across. See ShardingSpec. + properties: + enabled: + default: true + description: |- + Enabled adds this instance to the pool unpinned configs are + distributed across. + + While the pool is non-empty it, not spec.default, is what serves + unpinned configs — so the instance marked default must be a member + of it. Admission enforces that, because otherwise enabling sharding + on a second instance would silently drain every unpinned config off + the default one. Enable it on the default instance first. + type: boolean + weight: + default: 1 + description: |- + Weight biases this instance's share of the pool, for a fleet whose + instances are not the same size. An instance with weight 2 receives + approximately twice the share of one with weight 1. Weights are + relative, so scaling all of them changes nothing. + format: int32 + minimum: 1 + type: integer + type: object + startupProbe: + description: |- + StartupProbe bounds how long a replica may take to compile every + assigned plan before the kubelet restarts it. See StartupProbeSpec: + it polls /readyz, so its budget is a deadline on the sync itself. + properties: + enabled: + default: true + description: |- + Enabled renders the startupProbe, and defaults to true. Setting it + to false removes the deadline on the cold start entirely: the + liveness probe reads /healthz, which answers before the sync begins, + so nothing then restarts a replica that never finishes syncing. + type: boolean + failureThreshold: + default: 60 + format: int32 + minimum: 1 + type: integer + periodSeconds: + default: 5 + format: int32 + minimum: 1 + type: integer + type: object tolerations: items: description: |- @@ -3738,6 +3906,31 @@ spec: replicas: format: int32 type: integer + reportingReplicas: + description: |- + ReportingReplicas is how many live replica Leases fed + ServedTargets. A value below ReadyReplicas means at least one + ready replica has not published yet, and the intersection above is + not yet a statement about the whole instance. + format: int32 + type: integer + servedTargets: + description: |- + ServedTargets is the set of target resources that EVERY live + replica of this instance reports a compiled, servable plan for — + an intersection, not a union, because a target one replica out of + three can serve is a target that fails one request in three. + + Unlike AssignedConfigs, which is desired state computed by the + shared resolver, this is reported by the replicas themselves: each + publishes its own registry contents into a Lease, and this field + is the aggregate. It is what makes a safe handover possible — the + operator will not repoint a target at this instance until the + instance says it can already serve it. + items: + type: string + type: array + x-kubernetes-list-type: atomic type: object type: object served: true diff --git a/docs/architecture.md b/docs/architecture.md index 6938b9e..31531ee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,7 +21,7 @@ flowchart LR E -->|"pkg/engine.Convert()"| E ``` -The controller never patches the XRD until *all* of validation, XRD health, and webhook-server health pass — see [XRDConversionConfig: ordering](configuration/xrdconversionconfig.md#ordering-nothing-touches-the-xrd-until-every-gate-passes) for the exact gate sequence. +The controller never patches the XRD until *all* of validation, XRD health, and webhook-server health pass — and, when the patch would move the target to a different instance, until that instance reports it can already serve it. See [XRDConversionConfig: ordering](configuration/xrdconversionconfig.md#ordering-nothing-touches-the-xrd-until-every-gate-passes) for the exact gate sequence and [Moving a target between instances](#moving-a-target-between-instances) for the last one. ## The XRD conversion guard @@ -77,6 +77,73 @@ Each `ConversionWebhookServer` replica is symmetric and self-sufficient — ther - A single config's compile failure is **non-fatal**: the pod keeps serving whatever was last good for that XRD, recording the failure only in metrics and `/debug/registry` — it never crash-loops or de-readies the whole pod over one bad config. - **Readiness** gates on both informer cache sync *and* a completed first reconcile pass over every currently-existing config, closing the classic "added to Service endpoints before the registry is populated" gap. - A registry miss (a `ConversionReview` for an XRD this replica has no compiled plan for) fails closed with a clear `503`, rather than guessing. +- A replica holds a compiled plan while **either** the shared resolver assigns the target to its instance **or** the live target's `spec.conversion` still names its Service. The second clause is what makes a handover safe from the losing side — see below. + +## Moving a target between instances + +Assignment is not static. `spec.webhookServerRef` can be edited, and +[automatic sharding](configuration/conversionwebhookserver.md#automatic-sharding) +rebalances unpinned configs when an instance is added or removed. Each of +those means repointing a target's `spec.conversion` at a different Service, +and that is the one operation in this design with a genuine race: the +apiserver starts calling the new Service the moment the write lands, and a +replica that has not compiled the plan yet answers `503`. Every read and +write of that resource fails until it has — an outage produced by a +scaling decision, on resources that had nothing to do with it. + +The window is closed from both ends, and neither end requires the operator +to call a pod: + +``` +assignment changes ──▶ destination replicas compile the plan + │ (they watch the same objects the operator does) + ▼ + each publishes its servable target set into its own Lease + │ + ▼ + operator reads those, waits until EVERY live replica reports the target + │ + ▼ + operator patches spec.conversion → destination + │ + ▼ + source replicas see the target stop naming them, and start a 30s drain + │ + ▼ + only then do they drop the plan +``` + +Until that patch lands the target still names the **source**, and the +source is still serving it — because a replica keeps a plan for as long as +the live target points at it, not merely for as long as it is assigned. +That is what makes "wait" safe rather than merely slower: at no point is +the target unserved. + +**The drain closes the other end of the same window.** The apiserver +refreshes a CRD's conversion configuration *asynchronously* after the write +that changed it, so for a moment after the repoint it is still calling the +source's Service. A replica that dropped its plan the instant the object +changed would answer those calls with a 503 — reported as a failed read or +write on a resource that was only being rebalanced. It is the same shape of +race as the `preStop` sleep one layer down, and it gets the same treatment: +wait out the propagation rather than try to observe it. + +That was not theory. Before the drain existed, `hack/e2e-reassign.sh` +caught exactly one failed write in 9,456 across three reassignments, with +the registry-miss message. One in ten thousand is small, and it is not +zero. + +The config's `HandoverReady` condition reports where in that sequence a +move is. `hack/e2e-reassign.sh` drives three reassignments under sustained +reads and writes and asserts zero failures and zero wrong values. + +**Why a Lease.** The operator's reconcile loop deliberately makes no +network calls to webhook-server pods, so per-replica state has to be +published rather than queried. A Lease is owned by its Pod, so it is +collected with it; `renewTime` is a first-class staleness signal for a pod +that is alive but wedged; and it is a small dedicated object, so a +thirty-second heartbeat is not rewriting something other controllers watch. +The aggregate lands on `ConversionWebhookServer.status.servedTargets`. ## One cluster, one install diff --git a/docs/configuration/conversionwebhookserver.md b/docs/configuration/conversionwebhookserver.md index d0196a6..799b501 100755 --- a/docs/configuration/conversionwebhookserver.md +++ b/docs/configuration/conversionwebhookserver.md @@ -39,12 +39,14 @@ spec: | `replicas` | Fixed replica count. Mutually exclusive with `autoscaling` — once autoscaling is set, the HPA owns the replica count and this controller stops driving it directly. | | `autoscaling.{minReplicas,maxReplicas,targetCPUUtilizationPercentage}` | Creates a `HorizontalPodAutoscaler` for this instance instead of a fixed count. | | `image.{repository,tag,digest,pullPolicy}` | Overrides the webhook-server image for this instance. Omit to use the operator's own default (set via Helm `image.webhookServer.*` / a manager flag). When `digest` is set it takes precedence over `tag` (`repository@digest`). | -| `resources`, `nodeSelector`, `tolerations`, `affinity`, `priorityClassName`, `topologySpreadConstraints`, `serviceAccountName` | Standard Kubernetes pod-scheduling knobs, applied to this instance's Deployment. | +| `resources`, `nodeSelector`, `tolerations`, `affinity`, `priorityClassName`, `topologySpreadConstraints`, `serviceAccountName` | Standard Kubernetes pod-scheduling knobs, applied to this instance's Deployment. Setting `resources.limits.memory` also makes the operator set `GOMEMLIMIT` on the container at 90% of it, so the Go garbage collector is at least aware of the ceiling the kernel enforces. It is a soft target, not a cap — it cannot free live memory — so it bounds the cold-start transient, not the working set; size the limit for the live set regardless. See [Capacity planning](../operations/capacity.md#peak-versus-steady-state). Set `GOMEMLIMIT` in `extraEnv` to override. | | `podLabels` | Merged onto the pod template. Keys the controller uses for the Deployment selector (`app.kubernetes.io/name`, `instance`, `managed-by`) are ignored so a mis-set label cannot break rolling updates. | | `podAnnotations` | Set on the webhook-server pod template. | | `extraArgs` | Additional container arguments appended after operator-managed flags (`--webhook-server-name`, `--tls-cert-dir`, bind addresses, feature toggles, `--cache-label-selector`). For optional webhook-server flags (e.g. `--cert-reload-interval`, `--max-request-bytes`, `--request-timeout`, `--shutdown-timeout`, zap options). Admission and reconcile reject ExtraArgs that name those managed flags. | -| `extraEnv`, `extraVolumes`, `extraVolumeMounts` | Appended after the operator-managed environment / `tls`+`tmp` volumes. Use for custom CA bundles, proxies, or tenant env. | +| `extraEnv`, `extraVolumes`, `extraVolumeMounts` | Appended after the operator-managed environment / `tls`+`tmp` volumes. Use for custom CA bundles, proxies, or tenant env. An explicit `GOMEMLIMIT` here replaces the one derived from `resources.limits.memory` rather than being appended alongside it. | | `cacheSelector` | Optional `metav1.LabelSelector`. When set, webhook-server replicas watch only matching `XRDConversionConfig` / `CRDConversionConfig` objects **and** only matching `CustomResourceDefinition` / `CompositeResourceDefinition` objects — so the targets have to carry the label too. Unset (the default) watches everything. See [Capacity planning](../operations/capacity.md#memory-the-webhook-server). | +| `sharding.{enabled,weight}` | Opts this instance into the pool that unpinned configs are distributed across, by weighted rendezvous hashing on the target name. **Off unless the `sharding` block is written**: the CRD's `default: true` sits on `sharding.enabled`, and structural defaulting only descends into an object that is present, so omitting the block leaves the instance out of the pool. Writing the block *is* the opt-in — `sharding: {}` means enabled. While a pool exists it, not `spec.default`, serves unpinned configs — so the default instance must be a member, which admission enforces. See [Automatic sharding](#automatic-sharding). | +| `startupProbe.{enabled,periodSeconds,failureThreshold}` | The cold-start budget: `periodSeconds × failureThreshold`, defaulting to `5 × 60` (five minutes). The probe polls `/readyz`, which stays false until the registry has compiled every assigned plan, so the budget is a deadline on the sync itself; while it is in flight the kubelet runs neither of the other two probes. See [Capacity planning](../operations/capacity.md#cold-start-how-long-before-a-replica-can-serve). | | `rollout.{preStopSleepSeconds,terminationGracePeriodSeconds,maxUnavailable,maxSurge,defaultTopologySpread}` | How a replica leaves service. Defaults (`5`, `45`, `0`, `1`, `true`) make a rolling update cause zero failed conversions; they are one set, and admission rejects a combination where preStop + `--shutdown-timeout` exceeds the grace period. See the [HA checklist](../operations/ha-checklist.md#rolling-updates). | | `certificate.issuerRef` | The cert-manager `Issuer`/`ClusterIssuer` for this instance's webhook TLS certificate. `certificate.dnsNames`, `.duration`, `.renewBefore` are also available. | | `service.{type,port,annotations}` | The `Service` fronting this instance's pods. | @@ -71,9 +73,18 @@ status: - name: xwidgets-conversion xrdName: xwidgets.example.org phase: Applied + reportingReplicas: 2 + servedTargets: + - xwidgets.example.org ``` -`status.assignedConfigs` reflects the **desired** assignment as computed by the shared resolver every reconcile — not proof that every replica has actually loaded that config. Per-pod actual state (what's really compiled and serving right now) is deliberately kept out of this status field, to avoid the operator's own reconcile loop depending on a network call to the webhook-server pods; check each pod's own `/debug/registry` endpoint or its metrics for that. +`status.assignedConfigs` reflects the **desired** assignment as computed by the shared resolver every reconcile — not proof that every replica has actually loaded that config. + +`status.servedTargets` is the other half: what every live replica reports it can *actually* serve, as an **intersection** — a target two replicas out of three hold is a target that fails one request in three, so it does not appear here. `reportingReplicas` says how many replicas fed that intersection; a value below `readyReplicas` means at least one ready replica has not published yet, and the list is not yet a statement about the whole instance. + +The gap between the two lists is exactly the window in which a target has been *given* to this instance but the instance cannot serve it yet, and it is what the [handover gate](#moving-a-target-is-gated-not-immediate) waits on. + +This is still not a network call: the replicas publish it themselves into a Lease each, and the operator reads those through an informer like anything else. For a single pod's exact registry contents, its own `/debug/registry` endpoint and its `dco_webhook_registry_entry_loaded` metric remain the finer-grained answer. ### Conditions @@ -85,6 +96,8 @@ status: | `DefaultConflict` | More than one instance is marked `default` — shouldn't happen if the admission webhook works, but direct edits/restores can still produce it. The reconciler flags this loudly and does **not** auto-fix it. | | `DeletionBlocked` | Deletion is being held by the finalizer — see [Deletion safety](#deletion-safety). | +The `XRDConversionConfig` / `CRDConversionConfig` side of a move carries its own `HandoverReady` condition — see [Moving a target is gated, not immediate](#moving-a-target-is-gated-not-immediate). + ## Multiple instances Create additional instances for scale-out or tenancy, then point specific configs at them: @@ -116,9 +129,112 @@ spec: Every replica of every instance is symmetric and self-sufficient: each runs its own lightweight controller-runtime manager watching `XRDConversionConfig`, `ConversionWebhookServer`, and the relevant XRDs directly — there's no push mechanism from the main operator, and no leader election, since there's no shared state to coordinate. A single config's compile failure only affects that config: the pod keeps serving whatever was last good for every other XRD, and never crash-loops or de-readies over one bad config. +## Automatic sharding + +Hand-assigning every config with `webhookServerRef` is right for tenant +isolation and wrong as a scaling story. `spec.sharding` distributes the +configs that express no preference across every instance that opts in: + +```yaml +apiVersion: terasky.com/v1alpha1 +kind: ConversionWebhookServer +metadata: + name: default +spec: + default: true + sharding: + enabled: true +--- +apiVersion: terasky.com/v1alpha1 +kind: ConversionWebhookServer +metadata: + name: shard-b +spec: + sharding: + enabled: true + weight: 2 # twice the share of an instance at the default weight of 1 + # ... +``` + +Assignment is resolved in strict precedence order, by the same shared +resolver the operator and every webhook-server replica run independently: + +1. **`spec.webhookServerRef`** — deliberate pinning wins over everything. + Sharding can never move a pinned config, which is what keeps tenant + isolation intact. +2. **The sharding pool**, if any instance opts in, picked by weighted + rendezvous hashing on the *target resource's* name. +3. **`spec.default`**, exactly as before, when no instance opts in. + +### Enable it on the default instance first + +While a pool exists it, not `spec.default`, answers for unpinned configs. +The instance marked default must therefore be a pool member, and admission +rejects a state where it is not — otherwise enabling sharding on one +non-default instance would move every unpinned config onto it in a single +write. + +There is always a valid ordering. Enabling sharding on the default +instance alone makes the pool that one instance, so nothing moves. Each +instance added afterwards takes a bounded share. + +### Why rendezvous hashing + +Adding an instance moves only the targets that instance now wins — in +expectation `1/(N+1)` of them — and moves **nothing** between the instances +that were already there. Removing one moves only its own. A hash ring +merely approximates that, with a quality that depends on a virtual-node +count somebody has to tune; rendezvous has no such knob and no shared +state, so every party computes the same answer from the same objects +without coordinating. + +### Moving a target is gated, not immediate + +A move — a changed `webhookServerRef`, or a rebalance after an instance is +added — means repointing the target's `spec.conversion` at a different +Service. Doing that the moment the assignment changes would open a window +in which the apiserver calls replicas that have not compiled the plan yet, +and every read and write of that resource fails until they have. + +So the operator waits for the destination to confirm it can already serve +the target: + +- Each replica publishes the targets it holds a compiled plan for into a + Lease of its own. `status.servedTargets` is the **intersection** across + live replicas — a target two replicas out of three can serve is not a + target the instance serves — and `status.reportingReplicas` says how many + fed it. +- The config's `HandoverReady` condition reports the verdict. `False` with + reason `HandoverPending` means the move is deliberately being held. It is + not cleared once the move completes: it is the verdict on the *last* + handover, which stays true — and which is how an unverified one stays + visible long enough to be noticed. +- Meanwhile the **source** keeps serving: a replica holds a compiled plan + for as long as either the assignment *or* the live target points at it. + That is what makes waiting safe rather than merely slower. +- And it keeps serving for **30 seconds after** the target stops naming it. + The apiserver refreshes a CRD's conversion configuration asynchronously + after the write, so for a moment it is still calling the old Service; a + replica that dropped its plan immediately would answer those calls with a + 503. Same shape as the `preStop` sleep, same treatment. + +An instance whose replicas publish nothing at all — a fleet mid-upgrade, or +one in a namespace with no Lease `Role` — cannot be verified. That case is +indistinguishable from a destination whose replicas simply have not +reported *yet*, and the second is much the more common, so the move waits +**30 seconds** for a first report (`HandoverReady=False`, reason +`HandoverAwaitingReports`) before concluding that none is coming. Only then +does it proceed as every earlier release did, and say so: +`HandoverReady=True` with reason `HandoverUnverified`. A healthy replica +publishes in well under a second, so the wait is only ever paid by a fleet +that genuinely cannot report. See [RBAC](../security/rbac.md) for the Role +the replicas need. + ## Deletion safety -Deleting a `ConversionWebhookServer` runs the same finalizer-gated safety check: the operator lists every `XRDConversionConfig`, resolves its assignment, and blocks deletion (`DeletionBlocked` condition, listing the dependent configs by name) if **any** of them resolve to this instance — explicitly via `webhookServerRef`, or implicitly as the fallback `default`. The break-glass override is the same pattern as `XRDConversionConfig`: +Deleting a `ConversionWebhookServer` runs the same finalizer-gated safety check: the operator lists every `XRDConversionConfig`, resolves its assignment, and blocks deletion (`DeletionBlocked` condition, listing the dependent configs by name) if **any** of them resolve to this instance — explicitly via `webhookServerRef`, implicitly as the fallback `default`, or by landing here through [sharding](#automatic-sharding). + +It also blocks on a config whose *target* still points its conversion webhook here, even after the resolver has moved the config elsewhere. That is the mid-handover state, and during it this instance is the one answering every `ConversionReview` for that target — so judging by assignment alone would approve deleting the instance a live target depends on. The break-glass override is the same pattern as `XRDConversionConfig`: ```console kubectl annotate conversionwebhookserver default \ diff --git a/docs/configuration/crdconversionconfig.md b/docs/configuration/crdconversionconfig.md index 3f297b5..2c7f812 100644 --- a/docs/configuration/crdconversionconfig.md +++ b/docs/configuration/crdconversionconfig.md @@ -40,11 +40,11 @@ Structurally identical to `XRDConversionConfig`'s status, with two native-CRD-sp - `status.observedCRDGeneration` instead of `status.observedXRDGeneration`. - The health condition is named `CRDHealthy` instead of `XRDHealthy` — `True` once the target `CustomResourceDefinition` exists and its own `Established` condition is `True`. -Every other condition (`Validated`, `WebhookServerReady`, `Applied`, `Stale`, `DeletionBlocked`), phase, and `spokeStatuses` field behaves identically — see [XRDConversionConfig: Status](xrdconversionconfig.md#status) for the full reference. +Every other condition (`Validated`, `WebhookServerReady`, `Applied`, `Stale`, `DeletionBlocked`, `HandoverReady`), phase, and `spokeStatuses` field behaves identically — see [XRDConversionConfig: Status](xrdconversionconfig.md#status) for the full reference. ## Ordering and safety -The gate sequence, drift handling, and deletion safety are byte-for-byte the same algorithm as `XRDConversionConfig`'s (see [there](xrdconversionconfig.md#ordering-nothing-touches-the-xrd-until-every-gate-passes) for the full walkthrough) — validate, resolve the assigned `ConversionWebhookServer`, confirm the CRD is `Established`, confirm the server is ready, and only then server-side-apply `spec.conversion` onto the CRD. The same `conversion.terasky.com/allow-unsafe-delete` break-glass annotation applies for deleting a config while the CRD still serves more than one version. +The gate sequence, drift handling, and deletion safety are byte-for-byte the same algorithm as `XRDConversionConfig`'s (see [there](xrdconversionconfig.md#ordering-nothing-touches-the-xrd-until-every-gate-passes) for the full walkthrough) — validate, resolve the assigned `ConversionWebhookServer`, confirm the CRD is `Established`, confirm the server is ready, confirm the destination can already serve the target if this is a [move between instances](conversionwebhookserver.md#moving-a-target-is-gated-not-immediate), and only then server-side-apply `spec.conversion` onto the CRD. The same `conversion.terasky.com/allow-unsafe-delete` break-glass annotation applies for deleting a config while the CRD still serves more than one version. Promoting a different version to be the hub works the same way too — including why it's safe under the default drift policy — see [XRDConversionConfig: Changing the hub version](xrdconversionconfig.md#changing-the-hub-version), reading `storage: true`/`storage: false` wherever it says `referenceable`. diff --git a/docs/configuration/xrdconversionconfig.md b/docs/configuration/xrdconversionconfig.md index da92e51..46306b0 100644 --- a/docs/configuration/xrdconversionconfig.md +++ b/docs/configuration/xrdconversionconfig.md @@ -198,6 +198,7 @@ status: | `Applied` | `spec.conversion` has been patched onto the XRD. | | `Stale` | The live XRD's schema no longer matches what was last validated (see below). | | `DeletionBlocked` | Deletion is being held by the finalizer — see [Deletion safety](#deletion-safety). | +| `HandoverReady` | The `ConversionWebhookServer` this target was last moved to could already serve it at the moment it was repointed. Only present once a move has happened — a first apply has no previous server to hand over from. `False` (reason `HandoverPending`) means the move is deliberately being held and the target is still served by its current instance; `False` with reason `HandoverAwaitingReports` means no replica of the destination has published anything yet and the move is waiting 30 seconds for a first report; reason `HandoverUnverified` means that wait expired with the destination publishing no served-target Leases at all, so the move went ahead as earlier releases did without being verified. It is not cleared once the move settles: it is the verdict on the last handover. See [Moving a target between instances](conversionwebhookserver.md#moving-a-target-is-gated-not-immediate). | | `ConversionPropagated` | Every CRD Crossplane generates from the target XRD carries the conversion webhook this operator applied. **`Applied` is not the same thing:** `Applied` means the operator patched the XRD; nothing converts anything until Crossplane re-renders `{plural}.{group}` with that webhook block. Reasons: `Propagated`, `NotPropagated`, `GeneratedCRDNotFound`, `CABundleStale`. Per-CRD detail is in `status.generatedCRDs`. | | `PackageManaged` | The target XRD is owned by a Crossplane `ConfigurationRevision`, i.e. it ships inside a `Configuration` package. The message names the revision. This is **informational, not a failure** — but it means the package establisher re-writes the XRD with a full `client.Update` on every revision reconcile, stripping `spec.conversion` and this operator's annotations each time. See [Limitations](../limitations.md) and the `dco_manager_conversion_reverts_total` metric. `False` (reason `NotPackageManaged`) means nothing re-establishes the XRD out of band. | @@ -213,7 +214,8 @@ On every reconcile, in order: 4. Resolve the target `ConversionWebhookServer` (explicit `webhookServerRef`, or whichever instance is `default`). 5. Confirm the XRD is `Established`. 6. Confirm the assigned `ConversionWebhookServer`'s Deployment is `Available`, its Service has ready endpoints, and its certificate is ready. -7. **Only now**: server-side-apply `spec.conversion` onto the XRD, scoped to just that field (plus a couple of tracking annotations) — never a full-object apply, so this never fights any other owner of the XRD. +7. If this reconcile would **move** the target to a different instance — a changed `webhookServerRef`, or a sharding rebalance — confirm the destination reports it can already serve the target. Until it does the move is held, the XRD keeps naming the current instance, and that instance keeps serving it. See [Moving a target between instances](conversionwebhookserver.md#moving-a-target-is-gated-not-immediate). +8. **Only now**: server-side-apply `spec.conversion` onto the XRD, scoped to just that field (plus a couple of tracking annotations) — never a full-object apply, so this never fights any other owner of the XRD. ## Drift handling diff --git a/docs/limitations.md b/docs/limitations.md index 085b75f..7f809b3 100755 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -31,7 +31,10 @@ This page is deliberately blunt about what the operator does *not* do today, so The [XRD conversion guard](architecture.md#the-xrd-conversion-guard) closes the window by re-injecting the stanza inside the same admission request. It is on by default, `failurePolicy: Ignore` (so a package install never depends on this operator being up), add-only, and a no-op for an XRD wired to somebody else's webhook. **What it does not restore is the `caBundle`** — the controller reconciles that on its next pass, so a certificate rotation is still repaired asynchronously rather than in-request. Whether or not the guard is on, the `PackageManaged` condition and `dco_manager_conversion_reverts_total` make the hazard visible. Turn the guard off with `features.crossplane.conversionGuard.enabled=false` once a Crossplane version lands whose establisher uses Server-Side Apply. - **CRD schema changes require a manual step on Helm upgrade.** CRDs are installed once at `helm install` and never touched by `helm upgrade`/`helm uninstall` (Helm's own recommended convention for CRD-heavy charts) — see [Upgrading](installation.md#upgrading). -- **Per-pod webhook-server state isn't surfaced back into `ConversionWebhookServer.status`.** `status.assignedConfigs` reflects *desired* assignment computed by the shared resolver, not confirmation that every replica has actually finished compiling and loading a given config — that's a deliberate trade-off to avoid the operator's reconcile loop depending on network calls to webhook-server pods. Check each pod's `/debug/registry` endpoint or metrics for real per-pod state. +- **Per-replica state is published, not queried, and it is a set rather than a detail.** `status.assignedConfigs` is *desired* assignment computed by the shared resolver; `status.servedTargets` is what every live replica reports it can actually serve, as an intersection, with `status.reportingReplicas` saying how many fed it. Replicas publish that set into a Lease each and the operator reads those through an informer — the reconcile loop still makes no network call to a pod, which is the constraint that shaped the design. What the aggregate cannot tell you is *which* replica is missing a target, or why: for that, a pod's own `/debug/registry` endpoint and its `dco_webhook_registry_entry_loaded` metric remain the finer-grained answer. A replica that cannot write its Lease — an instance in a namespace with no Lease `Role`, or a fleet mid-upgrade — publishes nothing. A move onto such an instance waits 30 seconds for a first report and then proceeds unverified (`HandoverReady` with reason `HandoverUnverified`) rather than being blocked forever. The wait is what distinguishes "cannot report" from "has not reported yet", which look identical from the operator and of which the second is far more common; it is the one case where the operator repoints a target without positive confirmation. +- **A handover is verified against Lease reports, not against pod identity.** The operator counts the live Leases an instance's replicas publish and requires at least as many as the Deployment reports ready. It does not check that each Lease belongs to a *currently* ready pod — that would need a Pod informer in the operator, and the reconcile loop deliberately depends on no pod-level state. The window this leaves is small and bounded from two sides: a Lease is owned by its pod, so it is garbage-collected when the pod goes, and one whose `renewTime` stops advancing is discounted after 90 seconds. Inside that window a just-replaced replica's Lease could stand in for its successor, and a move could be approved a few seconds before the new replica has compiled the target. +- **A target stays on its old webhook server for 30 seconds after a move.** The apiserver refreshes a CRD's conversion configuration asynchronously after the write that changed it, so a replica that dropped its plan the instant the target stopped naming it would answer the in-flight calls with a 503. The drain is fixed, not configurable: it is short enough not to matter and long enough to cover the propagation, and the only cost of it being longer than necessary is one compiled plan (~18 KiB) held on a replica that no longer needs it. See [Moving a target between instances](architecture.md#moving-a-target-between-instances). +- **Automatic sharding balances by count, not by cost.** Rendezvous hashing distributes *targets* evenly across the pool (weighted, if you set weights); it knows nothing about how large each target's schema is or how much traffic it takes. A fleet with one enormous XRD and a hundred small ones can land the enormous one anywhere. Use `spec.webhookServerRef` to pin the outliers and let sharding spread the rest, or set `spec.sharding.weight` to bias the split. - **Spoke-to-spoke conversions always route through the hub** — two `Convert` calls, never a direct spoke-to-spoke path. This keeps compilation cost linear in the number of spoke versions. A 1000-element `forEach` spoke-to-spoke convert is ~2.3× a single hop and still under 1 ms ([Capacity planning](operations/capacity.md)); shortcut plans are not implemented. - **`--request-timeout` bounds a batch, not a single object.** The conversion loop checks the deadline before each object and again before reporting success, so a large batch cannot run indefinitely. It cannot interrupt one object mid-conversion: `engine.Convert`'s operations take no context, so a `forEach` over a very large array runs to completion first. `--max-request-bytes` is what bounds how large that array can be. Plumbing a context through every operation is the real fix and is not done. - **The webhook-server's cache is built from the feature flags, not from what is on the cluster.** `--enable-xrd-support=false` is what keeps `CompositeResourceDefinition` out of the informer cache entirely. controller-runtime resolves every cached kind through the RESTMapper when the manager is constructed, so on a cluster without Crossplane a replica told to support XRDs does not degrade — it exits at startup with `no matches for kind "CompositeResourceDefinition"`. That is deliberate and matches the manager's own behaviour, but it means the flags must match the cluster rather than describing a preference. @@ -56,9 +59,30 @@ live in [`pkg/engine/*_bench_test.go`](https://github.com/TeraSky-OSS/declarativ and are summarized in [Capacity planning](operations/capacity.md). Extremely large or deeply nested schemas beyond the published 1000-leaf / 1000-element points are still unvalidated against a live apiserver. Re-run `make bench` -locally; `make test-e2e-load` posts synthetic ConversionReview batches at a -live webhook-server; `make test-e2e-scale` drives real Get/List through the -apiserver conversion path against a generated CRD fleet (up to 100×100). +locally; `make bench-mem` reproduces the memory-per-target numbers (bytes +retained per compiled plan, registry footprint, and the transient peak +during initial sync); `make test-e2e-load` posts synthetic ConversionReview +batches at a live webhook-server; `make test-e2e-scale` drives real Get/List +through the apiserver conversion path against a generated CRD fleet. + +**The envelope CI exercises unattended is 300 CRDs × 20 objects**, nightly, +with the results published as an artifact and diffed against the previous +run ([the nightly scale run](operations/capacity.md#the-nightly-scale-run)). +That is below the 1000-CRD figure the phase proposal named, and +deliberately so: a standard hosted runner is four shared vCPUs hosting a +whole single-node control plane, and an aspirational number that always +fails is worth less than a smaller one that always runs. The envelope is a +workflow input so the ceiling can be raised on evidence. Anything above it +— and the 100×1000 local figures on that page — is a workstation +measurement, not a continuously-verified one. + +The published memory figures come from Go benchmarks, not from a loaded +cluster: they measure the compiled registry and the cold-start transient +accurately, and say nothing directly about the informer cache, which is the +dominant term and is measured separately by +`hack/measure-cache-memory.sh`. The worked sizing example in +[Capacity planning](operations/capacity.md#worked-example) extrapolates the +informer term from one 300-CRD cluster; treat it as an order of magnitude. What the beta label promises about the surfaces above — and how a Helm value or a CLI flag is removed once it exists — is in the diff --git a/docs/observability.md b/docs/observability.md index 12b8c7c..9205281 100755 --- a/docs/observability.md +++ b/docs/observability.md @@ -47,6 +47,8 @@ Emitted by each ConversionWebhookServer replica (dedicated registry in | `dco_webhook_registry_reload_total` | Counter | `target`, `result` | Attempted (re)compiles | | `dco_webhook_registry_compile_errors_total` | Counter | `target`, `reason` | Compile failures that left a stale-or-absent plan in place | | `dco_webhook_ready` | Gauge | — | `1` after this replica's registry completed initial sync | +| `dco_webhook_initial_sync_duration_seconds` | Gauge | — | Seconds this replica spent compiling every assigned plan before reporting ready. Written once; `0` on a replica still cold, which `dco_webhook_ready` disambiguates | +| `dco_webhook_initial_sync_targets` | Gauge | — | Configs walked during that cold start. Divide the duration by it for a per-target cost | ### `direction` on the two latency histograms @@ -148,6 +150,82 @@ covers it; enabling the XRD conversion guard closes the window entirely. --- +## Controller health: the leading indicator + +controller-runtime exports workqueue and reconcile metrics for **every** +controller in both processes. They are not this operator's own metrics, and +they are the ones that move first. + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `workqueue_depth` | Gauge | `name`, `controller`, `priority` | Items waiting to be reconciled | +| `workqueue_adds_total` | Counter | `name`, `controller` | Enqueues | +| `workqueue_queue_duration_seconds` | Histogram | `name`, `controller` | How long an item waited before a worker picked it up | +| `workqueue_work_duration_seconds` | Histogram | `name`, `controller` | How long one reconcile took | +| `workqueue_retries_total` | Counter | `name`, `controller` | Requeues after a failed reconcile | +| `controller_runtime_reconcile_total` | Counter | `controller`, `result` | Reconciles, by outcome | +| `controller_runtime_reconcile_errors_total` | Counter | `controller` | Reconciles that returned an error | +| `controller_runtime_reconcile_time_seconds` | Histogram | `controller` | Reconcile latency | +| `controller_runtime_active_workers` | Gauge | `controller` | Workers currently busy | +| `controller_runtime_max_concurrent_reconciles` | Gauge | `controller` | The ceiling `--max-concurrent-reconciles` sets | + +Both processes export them. The manager serves controller-runtime's +registry directly; the webhook-server serves its own dedicated registry +*and* controller-runtime's from one handler, so a replica's registry +reconcile loop is visible on the same panels as the manager's controllers. + +### Why depth is the metric to watch + +The causal chain runs in one direction, and every link is slower to notice +than the one before it: + +``` +workqueue_depth rises + → reconciles are queued longer than they take to run + → a config's status.phase goes Stale + → the XRD keeps an old spec.conversion, or never gets one + → ConversionPropagated lags, and reads come back unconverted +``` + +By the time `dco_manager_conversion_propagated` drops to 0 the backlog has +already been there for a while. Depth is the only signal in that chain that +moves *before* anything is wrong for a user, which is what makes the panels +worth having rather than decorative. + +What matters is a depth that **stays** up. A bulk apply of two hundred +configs legitimately spikes the queue and then drains it; that is the +shape the alert's `for:` exists to tolerate. + +```promql +# Backlog, per controller, in both processes +sum by (job, controller) (workqueue_depth) + +# Is the backlog "lots of work" or "slow work"? High adds + flat depth is the +# first; low adds + rising depth is the second. +sum by (job, controller) (rate(workqueue_adds_total[5m])) + +# Reconcile latency +histogram_quantile(0.99, sum by (le, job, controller) (rate(workqueue_work_duration_seconds_bucket[5m]))) + +# Saturation: workers busy against the configured ceiling +sum by (controller) (controller_runtime_active_workers) + / sum by (controller) (controller_runtime_max_concurrent_reconciles) +``` + +### The lever + +`--max-concurrent-reconciles` (Helm: `manager.maxConcurrentReconciles`) +sets how many objects each controller reconciles at once. It defaults to +1 — controller-runtime's own default — and is worth raising when depth is +persistently non-zero *and* work duration is not the problem. The cost is +apiserver QPS, which is why it is not raised by default. + +Correctness does not depend on it: controller-runtime guarantees a given +object key is never reconciled by two workers simultaneously, and nothing +in these reconcile paths shares mutable state across keys. + +--- + ## Watch-map metric When a secondary watch map function fails to `List` related configs (API @@ -188,8 +266,31 @@ count by (pod) (dco_webhook_registry_entry_loaded == 1) ``` `ConversionWebhookServer.status.assignedConfigs` remains the cluster-level -**desired** set computed by the shared resolver. Use it together with the -per-pod gauges above — not as a substitute for them. +**desired** set computed by the shared resolver. `status.servedTargets` is +the reported counterpart — the intersection of what every live replica +publishes it can serve, with `status.reportingReplicas` saying how many fed +it. Between them they answer "is this instance ready for this target?" +without a scrape; the per-pod gauges above remain the finer-grained answer +to *which* replica is missing one. + +### Cold start + +The plain endpoint (`/healthz`, `/readyz`, `/metrics`) listens *before* the +informer cache syncs, so a replica that is still compiling is visibly alive +rather than indistinguishable from a hung process. Its `/readyz` stays +`503` and the conversion endpoint does not listen at all until the registry +is populated. + +```promql +# Slowest cold start in the fleet — size startupProbe.failureThreshold from this +max(dco_webhook_initial_sync_duration_seconds) + +# Per-target cold-start cost for your schemas +dco_webhook_initial_sync_duration_seconds / dco_webhook_initial_sync_targets +``` + +See [Capacity planning](operations/capacity.md#cold-start-how-long-before-a-replica-can-serve) +for the measured curve and the reasoning behind the default budget. --- @@ -199,12 +300,16 @@ The chart ships: - **PrometheusRule** (`metrics.prometheusRule.enabled`) — compile errors, fleet/replica not-ready, high latency, lossy rate, error ratio, manager - analyze failures, and Stale/Failed phase transitions. Expressions are - unit-tested under `hack/prometheus/` (`make test-prometheus`). + analyze failures, Stale/Failed phase transitions, and the two + controller-health alerts (`ControllerWorkqueueBacklog`, + `ControllerReconcileErrors`) whose thresholds are + `metrics.prometheusRule.workqueueDepthThreshold`, + `.workqueueBacklogFor` and `.reconcileErrorRateThreshold`. Expressions + are unit-tested under `hack/prometheus/` (`make test-prometheus`). - **Grafana dashboard ConfigMaps** (`dashboards.enabled`) — labeled `grafana_dashboard: "1"` for the Grafana sidecar; JSON under `charts/declarative-conversion-operator/files/dashboards/`: - - [`conversion-overview.json`](https://github.com/terasky-oss/declarative-conversion-operator/blob/main/charts/declarative-conversion-operator/files/dashboards/conversion-overview.json) — fleet-wide overview + - [`conversion-overview.json`](https://github.com/terasky-oss/declarative-conversion-operator/blob/main/charts/declarative-conversion-operator/files/dashboards/conversion-overview.json) — fleet-wide overview, plus a **Controller health** row (workqueue depth, add rate, work duration p50/p99, reconcile error rate) for both processes - [`conversion-target-detail.json`](https://github.com/terasky-oss/declarative-conversion-operator/blob/main/charts/declarative-conversion-operator/files/dashboards/conversion-target-detail.json) — one XRD/CRD via the `target` dropdown (`target` label) - [`conversion-stability.json`](https://github.com/terasky-oss/declarative-conversion-operator/blob/main/charts/declarative-conversion-operator/files/dashboards/conversion-stability.json) — platform stability deep dive: conversions/s, failure rate, latency, registry, manager control plane, and webhook/manager pod CPU/memory/restarts (kubelet cAdvisor + kube-state-metrics) @@ -212,9 +317,11 @@ The chart ships: between them. Resource panels on the stability dashboard need kubelet cAdvisor and kube-state-metrics (kube-prometheus-stack provides both); conversion/manager panels only need this chart's `ServiceMonitor`s. - Webhook process Go collectors are **not** on `/metrics` (dedicated - registry) — use cAdvisor for webhook CPU/memory. The manager scrape - still exposes `go_goroutines` / `process_*`. + Both scrapes expose `go_goroutines` / `process_*`: the webhook-server's + `/metrics` gathers controller-runtime's registry alongside its own, and + that registry carries the Go and process collectors. cAdvisor is still + the better source for container-level memory, because it measures the + same working set the kernel enforces a limit against. --- diff --git a/docs/operations/capacity.md b/docs/operations/capacity.md index b1c7f4d..d0e4ed6 100755 --- a/docs/operations/capacity.md +++ b/docs/operations/capacity.md @@ -128,6 +128,123 @@ is 256 MiB, which the cluster above fits with room to spare after this change and did not before. A cluster with substantially more or larger CRDs should raise it or set a `cacheSelector`. +There are three terms, and they are not the same size: + +| Term | What it scales with | Measured | +|---|---|---| +| **Informer cache** | every CRD and XRD the replica watches, schemas included | the 121 MiB above, for 300 two-version 200-property CRDs — **the dominant term** | +| **Compiled registry** | number of targets × their schema size | ~18 KiB per target (see below) | +| **Cold-start transient** | allocation churn while compiling, not anything retained | 8–13× the steady registry, depending on fleet size — **what an OOM kill is decided against** | + +#### Bytes per compiled plan + +`make bench-mem`, `BenchmarkCompiledPlanRetained` in `pkg/engine`. Live heap +either side of building N plans and holding them all — not `-benchmem`'s +`B/op`, which counts the garbage a compile produces as well as what survives +it: + +| Leaves (per version) | Retained per plan | Churned per compile | Ratio | +|---|---:|---:|---:| +| 10 | 2.5 KiB | 47 KiB | 19× | +| 100 | 21 KiB | 467 KiB | 22× | +| 1000 | 234 KiB | 4.7 MiB | 20× | + +Retained cost is linear in leaf count, about **240 bytes per leaf**. The +number that matters operationally is the third column: **a compile churns +roughly twenty times what it keeps.** + +#### Registry footprint + +`BenchmarkRegistryRetained` in `internal/webhookserver`, over a fleet of +two-version targets of 50 leaves each with one `FieldRename` rule per leaf — +so each target carries one compiled plan: + +| Targets | Retained per target | Registry total | +|---|---:|---:| +| 10 | 83 KiB | 0.8 MiB | +| 100 | 18.6 KiB | 1.8 MiB | +| 1000 | 18.4 KiB | 18 MiB | + +The 10-target row is fixed per-replica overhead divided by ten, not a real +per-target cost; from a hundred targets up the figure is flat at ~18 KiB. +**A thousand targets is 18 MiB of registry** — small enough that the registry +is never the reason a replica needs a bigger limit. + +#### Peak versus steady state + +`BenchmarkInitialSyncPeak`, sampling live heap every 2 ms through the cold +start: + +| Targets | Steady registry | Peak during sync | Ratio | +|---|---:|---:|---:| +| 100 | 1.8 MiB | 23–26 MiB | ~13× | +| 1000 | 18 MiB | 140 MiB | ~8× | + +The ratio falls as the fleet grows because the fixed per-replica overhead +stops dominating, not because the transient gets cheaper in absolute terms. + +This is the finding worth acting on. The peak is not memory the replica +needs; it is memory the garbage collector has not reclaimed yet, because +with the default `GOGC` the heap is allowed to double the live set before a +collection — and a cold start allocates twenty times what it keeps, as fast +as it can, across every core. + +The kernel enforcing a container memory limit does not wait for the GC. +**`GOMEMLIMIT` is what makes the GC aware of the same number**, so the +operator sets it on every webhook-server container at 90% of +`spec.resources.limits.memory` whenever a limit is set. With it, the same +thousand-target run peaks at 61 MiB instead of 140 MiB, taking 1.6 s instead +of 0.45 s — which is the trade a memory limit is asking for. Set +`GOMEMLIMIT` yourself in `spec.extraEnv` to override the derived value; the +operator leaves an explicit one alone. + +!!! warning "`GOMEMLIMIT` is a soft target, not a cap" + It makes the collector work harder as the heap approaches the number — + it cannot free memory that is still live, and it does not cover + allocations outside the Go runtime. Against a **transient** peak, which + is what the table above measures, that is exactly the right lever. Against + a **live** working set larger than the limit it does nothing except + collect continuously, and the container is OOM-killed anyway, now with a + CPU burn in front of it. + + So `GOMEMLIMIT` is not a substitute for sizing the limit. Size it for the + live set — registry plus informer cache, the two terms below — and leave + headroom on top. What `GOMEMLIMIT` changes is how much headroom the + cold-start transient needs: measured at a thousand targets it took the + peak from 140 MiB to 61 MiB. Much smaller, not zero, and not a + guaranteed ceiling — the collector can be outrun. + +#### Worked example + +A cluster with **1000 targets averaging 200 leaves per version**, on the +chart's default 256 MiB limit: + +- Registry: 200 leaves × 240 B ≈ 48 KiB per plan, ×1000 ≈ **48 MiB**. +- Informer cache: the schemas those targets live in. Extrapolating the + measured 121 MiB for 300 two-version 200-property CRDs gives roughly + **400 MiB** — this term alone blows the default limit. +- Cold-start transient: several hundred MiB on top without `GOMEMLIMIT`, + substantially less with it — see the note below. + +So: **set `cacheSelector`, or raise the limit to ~1 GiB.** The registry is +not the problem at any plausible scale; the informer cache is, and it is the +one term this operator can only narrow, never shrink. + +Note which term `GOMEMLIMIT` can and cannot help with here, because this +example is the case that makes the distinction concrete. The ~400 MiB +informer cache is **live**, so no GC setting brings it under a 256 MiB +limit — only `cacheSelector` or a bigger limit will. The cold-start +transient is the one term it does move, and it shrinks it by roughly half +rather than removing it: budget for a reduced transient on top of whatever +limit the live set demands, not for none. + +The chart's 256 MiB default was reviewed against these numbers and left +alone. It is right for the cluster it is a default for — a few dozen +targets, a few hundred CRDs — and raising it would silently raise the +scheduling floor for every install to serve the minority that need it. What +was wrong was that the Go runtime had no idea the limit existed, so the +cold-start transient was sized by `GOGC` alone. `GOMEMLIMIT` tells it. + ### How these numbers were taken `hack/measure-cache-memory.sh` builds two real images from two real commits, @@ -195,6 +312,108 @@ That is a 99% reduction in cached objects. Memory scales with that store, so the same ratio applies to RAM. Use a selector per tenant (or per team) when one cluster holds many configs but each webhook instance only serves a slice. +## Cold start: how long before a replica can serve + +A replica does not answer conversions until its registry holds a compiled +plan for every target assigned to it. That startup pass — `InitialSync` — is +the whole of the cold start, and it is what a `startupProbe` has to be sized +against. + +`BenchmarkInitialSync` in `internal/webhookserver/initialsync_bench_test.go` +walks N targets, each a two-version XRD of 50 leaves per version with one +`FieldRename` rule per leaf, and compiles them all: + +| Targets | Serial | Parallel (GOMAXPROCS=24) | Speed-up | +|---|---:|---:|---:| +| 10 | 12 ms | 8 ms | 1.5× | +| 100 | 73 ms | 50 ms | 1.5× | +| 1000 | 825 ms | 391 ms | 2.1× | + +Compilation is CPU-bound and independent per target, so `InitialSync` runs a +bounded worker pool — `GOMAXPROCS` by default, `--initial-sync-workers` to +override. The benchmark's client is a fake backed by one mutex, so its +API-read term is more serialised than a real informer cache and the speed-up +above is a floor, not a ceiling. + +**A thousand targets is under a second of compile.** The cold-start budget is +therefore dominated not by this operator but by the informer cache sync in +front of it: a replica watching every CRD and XRD on a large cluster spends +most of its startup waiting for those LISTs. That is what +`spec.cacheSelector` reduces, and it is why the default budget is minutes +rather than seconds. + +### The startup probe + +`ConversionWebhookServer.spec.startupProbe` renders a `startupProbe` on the +webhook-server container; `periodSeconds × failureThreshold` is the budget, +defaulting to 5 × 60, i.e. five minutes. + +It polls **`/readyz`**, which is what makes the budget real. The plain +endpoint carrying `/healthz`, `/readyz` and `/metrics` comes up *before* the +registry sync, so `/healthz` answers within milliseconds of process start; a +`startupProbe` pointed at it would succeed immediately and bound nothing. +`/readyz` stays false until the initial sync completes, so +`periodSeconds × failureThreshold` is a deadline on the sync itself. + +The kubelet runs neither of the other two probes while a `startupProbe` is +in flight, so a slow sync is not simultaneously fighting the liveness +probe's own 3 × 10 s. Once the probe succeeds, liveness (`/healthz`) and +readiness (`/readyz`) take over as usual. + +Two things the deadline buys, beyond not crash-looping a slow replica: + +- **The initial sync retries infrastructure failures without a limit** — a + failed read of a target, a failed server list — because a watch-driven + reconciler will not necessarily re-deliver an event for what failed. That + is the right behaviour for a transient failure and the wrong one for a + permanent one, and the `startupProbe` is what distinguishes them. +- **Without it a wedged replica is invisible.** It would stay + liveness-healthy and never ready: out of the Service, never restarted, + showing up only as a gap in `readyReplicas`. + +Erring long is deliberate. An over-tight threshold turns a slow start into a +crash loop; an over-long one only delays the restart of a pod that is not +taking traffic anyway. + +### Measuring your own + +Two metrics and a log line, published once per replica at the moment it +reports ready: + +```promql +# Cold start, per replica +dco_webhook_initial_sync_duration_seconds + +# Targets that cold start compiled +dco_webhook_initial_sync_targets + +# Per-target cost for your schemas +dco_webhook_initial_sync_duration_seconds / dco_webhook_initial_sync_targets +``` + +``` +registry synced, marking replica ready serverName=default targets=812 workers=8 elapsed=1.412s +``` + +Set `failureThreshold` from the slowest cold start you observe, with room to +spare — it is a deadline on exactly the interval this metric measures. The +plain HTTP endpoint (`/healthz`, `/readyz`, `/metrics`) comes up *before* +the cache sync, so a replica that is still cold is visibly alive and +scrapeable rather than indistinguishable from a hung process, and the +`startupProbe` is polling a live listener rather than collecting +connection-refused. + +### Reporting ready anyway after a timeout + +Considered and deliberately not implemented. A `--registry-ready-timeout` +that let a replica join the Service with a partially-populated registry +would have it answer ConversionReviews for targets it has not compiled yet +with a failure — which the apiserver turns into a failed write on a +resource that has nothing to do with the slow config. An unavailable replica +degrades throughput; a half-loaded one corrupts the answer. The +`startupProbe` is the supported lever, and the current fail-closed ordering +stands. + ## Registry copy-on-write at 100+ entries `Registry.Set` copies the whole map of pointers and atomically swaps it so @@ -284,8 +503,10 @@ apiserver Get/List (which invoke the conversion webhook) in parallel: lists the whole fleet. Re-apply with `--reset` if older CRDs lack the category. -Defaults are a smoke size (4 CRDs × 5 CRs). Override with env vars — this is -**not** in the CI e2e matrix; 100×100 and 100×1000 are local capacity runs. +Defaults are a smoke size (4 CRDs × 5 CRs). Override with env vars. The +100×100 and 100×1000 figures below are **local** capacity runs on a +workstation; the envelope CI exercises unattended is the nightly one +described under [The nightly scale run](#the-nightly-scale-run). ```console # smoke (default) @@ -349,6 +570,84 @@ is the bottleneck, not conversion. Re-run with `TARGETS=100 INSTANCES=1000 PARALLEL=60 make test-e2e-scale` after changing the serving path. +### The nightly scale run + +`.github/workflows/scale.yml` runs `hack/e2e-scale.sh` on a schedule at +**300 CRDs × 20 objects** (6,000 objects, 900 served versions), publishes +`scale-result.json` as a 90-day artifact, renders it into the job summary, +and compares it against the previous successful run. + +That is where the numbers in this section come from from now on. A local +run on a workstation is still the right tool for investigating a change; +the scheduled run is what notices one nobody was looking for. + +**Regression detection is relative, never absolute.** Absolute timings on a +hosted runner vary by a factor of two between runs for reasons that have +nothing to do with this code, so a threshold tight enough to catch a real +regression would fire constantly. The check fails when a measurement +exceeds a configurable multiple — 1.5× by default — of the *same +measurement in the previous run at the same envelope, under the same report +schema*. Below a noise floor (20 ms, 1 s, 32 MiB depending on the unit) a +ratio is not treated as a signal: a p50 that moved from 2 ms to 4 ms is a +2× regression by arithmetic and scheduler noise by every other reading. + +Two things are checked absolutely rather than as a trend, because for them +zero is the only acceptable value: any Get/List error, and a run that +issued no requests at all — which would otherwise report zero of everything +and look like a pass. + +A failure names the measurement. The summary's comparison table marks the +offending row **REGRESSED** and the job log repeats it as +`FAIL: listV1 p50: 90.0 ms -> 190.0 ms (2.11x, threshold 1.50x)`, so the +first question ("what got slower?") is answered without downloading +anything. + +The artifact carries more than latency: `hack/scale-observe.py` merges in +the webhook-server's **cold-start time** +(`dco_webhook_initial_sync_duration_seconds`) and its **loaded working +set** (from the kubelet Summary API), so the two numbers this page's memory +and cold-start sections are about are trended by the same job. Both are +gated against the previous run alongside the latency figures. + +The working set is a single sample taken once the replicas are Ready again +after the restart, so it is the **steady state with the fleet loaded, not +the transient peak** — the peak happens before readiness, where nothing is +sampling. The peak-versus-steady table above is what measures that, from +`make bench-mem`. + +#### Why 300 CRDs, and not the 1000 the proposal asked for + +The target in [the phase proposal](../proposals/next-phases.md) is 1000 +CRDs, on the reasoning that it is roughly the CRD count of a mature +Crossplane cluster. The scheduled run does not reach it yet, and +configuring an aspirational number that always fails would be worse than +publishing a smaller one that always runs. + +A standard GitHub-hosted runner is 4 vCPU and 16 GiB, hosting a +single-node kind cluster whose apiserver, etcd, the operator and the +webhook-server replicas all share those four cores. Two terms make CRD +count, rather than object count, the binding constraint there: + +- **Applying CRDs is apiserver-CPU-bound, not IO-bound.** Each `CustomResourceDefinition` + write makes the apiserver rebuild parts of its aggregated OpenAPI + document and re-establish the resource's handler. On the workstation runs + below that cost is invisible next to object creation; on four shared + cores it is not. +- **Every CRD is watched three times over** — by the apiserver, by the + operator, and by each webhook-server replica — and each replica also + holds its schemas resident. At 1000 CRDs that is the informer footprint + the [sizing section](#worked-example) puts at several hundred MiB per + replica, against a 16 GiB box already running a control plane. + +300 × 20 completes in roughly 25 minutes end to end and has headroom +against the job's 75-minute timeout, which is what "reliable" has to mean +for something that runs unattended. The envelope is a `workflow_dispatch` +input precisely so the ceiling can be probed upward with evidence rather +than moved by assertion: run it at 500, then 750, and raise the default +when a higher number has run clean several times. A run at a different +envelope publishes its numbers and skips the comparison, so probing cannot +produce a false regression. + | Flag / env | Default | Meaning | |---|---|---| | `--targets` / `TARGETS` | 4 | Number of CRDs (each with 3 versions) | @@ -363,6 +662,7 @@ the serving path. | `--list-repeats` / `LIST_REPEATS` | 3 | List calls per CRD per spoke version | | `--get-repeats` / `GET_REPEATS` | 1 | Get calls per instance per spoke version | | `--dry-run` | false | Print strategy coverage only (no cluster) | +| `--result-json` / `RESULT_JSON` | unset | Write the run's measurements as JSON, and merge in the cluster-side observations | Native CRDs are used on purpose: they exercise the same `pkg/engine` + webhook-server path as XRDs without requiring Crossplane. Times above include diff --git a/docs/operations/ha-checklist.md b/docs/operations/ha-checklist.md index 6bda042..2aad427 100644 --- a/docs/operations/ha-checklist.md +++ b/docs/operations/ha-checklist.md @@ -49,6 +49,30 @@ them leads to over-provisioning the wrong one. ``` An empty result means every ready replica can serve that target. +- [ ] **Know your cold-start budget.** A replica compiles every assigned plan + before it can serve. The health endpoint (`/healthz`, `/readyz`, + `/metrics`) listens from the start, so the replica is visibly alive + throughout; `/readyz` stays `503` and the *conversion* endpoint does + not listen at all until the registry is populated. The `startupProbe` + (`conversionWebhookServer.startupProbe`, five minutes by default) + polls `/readyz`, so its `periodSeconds × failureThreshold` is a + deadline on the sync — and while it is in flight the kubelet runs + neither of the other two probes, so a slow sync is not also fighting + the liveness probe's 3 × 10 s. Size it from + `dco_webhook_initial_sync_duration_seconds`: too tight turns a slow + start into a crash loop, too loose only delays the restart of a pod + that is not taking traffic anyway. + Check the measured figure after a scale-out and raise + `failureThreshold` if it is close: + + ```promql + max(dco_webhook_initial_sync_duration_seconds) + ``` + + The compile itself is under a second for a thousand targets; what + makes a cold start slow is the informer cache sync in front of it, + which `spec.cacheSelector` is the lever for. See + [Capacity planning](capacity.md#cold-start-how-long-before-a-replica-can-serve). ## Rolling updates diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index d06fa66..1856c89 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -35,6 +35,8 @@ uncovered or which rule is lossy while the config is still `Invalid`. | `Invalid` | `Validated=False`, message names the hub version | `spec.hubVersion` isn't the target's storage version (`referenceable: true` on an XRD, `storage: true` on a CRD). | Point `hubVersion` at the real storage version — it can't be an arbitrary spoke. | | `Invalid` | `XRDHealthy=False` / `CRDHealthy=False` | The target XRD/CRD doesn't exist, or its `Established` condition isn't `True`. | Check the name in `spec.targetXRD.name`/`spec.targetCRD.name`; `kubectl get crd ` and wait for `Established`. | | `Validated` | `WebhookServerReady=False` | The assigned `ConversionWebhookServer`'s Deployment isn't `Available`, its Service has no ready endpoints, or its certificate isn't ready. | Debug the `ConversionWebhookServer` first — see [below](#the-conversionwebhookserver-never-becomes-available). Nothing is patched onto the target until it is ready. | +| `Validated` / `Stale` | `HandoverReady=False`, reason `HandoverPending` | The config has been **moved** to another `ConversionWebhookServer` (a changed `webhookServerRef`, or a sharding rebalance) and the destination has not yet reported that it can serve the target. This is the gate working: the target is still pointed at, and still served by, its current instance. | Usually resolves in seconds. If it does not, check `kubectl get conversionwebhookserver -o jsonpath='{.status.reportingReplicas}/{.status.servedTargets}'` — a `reportingReplicas` below `readyReplicas` means a ready replica has not published its Lease yet. | +| `Validated` / `Stale` | `HandoverReady=False`, reason `HandoverUnknown` | A replica of the destination could not state what it serves (its target set exceeded the annotation cap, or the annotation would not decode). | The instance is holding an implausible number of targets, or something else is writing its Leases. Check `kubectl -n get leases -l conversion.terasky.com/webhook-server=`. | | `Failed` | — | An unexpected error, distinct from a validation failure. | Manager logs. This is the phase that should never be normal; it's worth an issue if the cause isn't obviously environmental. | **The target resource is never patched in any of these phases.** A config that @@ -110,6 +112,35 @@ fixtures: convctl test --xrd xrd.yaml --config config.yaml --live ``` +## `Applied`, but the last move was never verified + +`HandoverReady=True` with reason `HandoverUnverified` is not a failure: the +move completed and the target is patched. What it says is that the +destination published no served-target Leases *within the 30-second grace +period*, so the operator could not confirm the instance was ready before +repointing — it proceeded the way every release before this feature did. + +Seeing `HandoverReady=False` with reason `HandoverAwaitingReports` instead +means that wait is still running. It resolves on its own within thirty +seconds, either into a verified move or into the unverified one below. + +| Cause | Fix | +|---|---| +| The instance's replicas cannot write Leases in their namespace. | Almost always missing RBAC. An instance whose `spec.namespace` is not the release namespace needs the webhook-server Lease `Role` and `RoleBinding` created there — see [RBAC](../security/rbac.md). | +| A fleet mid-upgrade, where the replicas predate the feature. | Nothing to do; it resolves once every replica is on a version that publishes. | +| The replicas have no downward-API identity (`POD_NAME`/`POD_NAMESPACE`/`POD_UID`). | The operator sets these on every Deployment it reconciles. A Deployment that predates that is re-applied on the next reconcile of its `ConversionWebhookServer`. | + +```console +kubectl -n get leases -l conversion.terasky.com/webhook-server= +kubectl get conversionwebhookserver -o jsonpath='{.status.reportingReplicas}' +``` + +Until it is fixed, a move onto that instance is unverified — which is the +pre-existing behaviour, not a new hazard, but it is the one window in which +a rebalance could briefly fail conversions. The grace period narrows that +window to fleets that genuinely cannot publish, rather than every fleet +whose replicas had not got round to it. + ## `Applied` but not `Propagated` **Symptom.** The config reports `Phase: Applied` and `Applied=True`, but reads at a non-storage version come back **relabelled and unconverted** — right `apiVersion`, old field layout, HTTP 200, no error. diff --git a/docs/proposals/next-phases.md b/docs/proposals/next-phases.md index b95da6c..f54f74f 100644 --- a/docs/proposals/next-phases.md +++ b/docs/proposals/next-phases.md @@ -973,6 +973,76 @@ apiserver's write path". ## Phase 15 — Scale +> **Shipped.** Every deliverable below landed. Seven things are worth +> recording, four of them deviations and three of them defects the work +> turned up rather than confirmed: +> +> - **The cold-start work found a defect, not just a missing metric.** A +> webhook-server replica *used to* listen on no port at all until its +> registry was populated, so both probes got connection-refused until +> then — which made the liveness probe's own 3 × 10 s the *entire* +> cold-start budget. A replica holding enough targets to exceed thirty +> seconds would have been killed and restarted forever, reading as a +> crash loop rather than as a slow start. Two changes close it: +> `spec.startupProbe` suspends the other two probes while the sync runs, +> and the health endpoint now comes up *before* the cache sync, so a cold +> replica answers `/healthz`, reports `/readyz` 503, and is visibly alive. +> The conversion endpoint still waits for a populated registry. +> - **The memory work found a second one.** The steady registry is small — +> about 18 KiB per target — but compiling churns roughly twenty times +> what it retains, and with the default `GOGC` a thousand-target cold +> start peaks around 140 MiB against 18 MiB of steady state. The kernel +> enforcing a container limit does not wait for the collector, so the +> operator now sets `GOMEMLIMIT` from `resources.limits.memory` — which +> roughly halves that transient (140 MiB to 61 MiB at a thousand targets) +> and, being a soft target rather than a ceiling, does nothing for a live +> working set that exceeds the limit. The +> chart's 256 MiB default was reviewed and left alone: it was not wrong, +> it was unenforceable. +> - **`--registry-ready-timeout` was considered and deliberately not +> added** (15.2 raises it as an open question). An unavailable replica +> degrades throughput; a half-loaded one corrupts the answer. Recorded in +> [Capacity planning](../operations/capacity.md) so it does not have to be +> re-argued. +> - **The reassignment e2e paid for itself on its first clean run.** Three +> moves under load produced exactly one failed write in 9,456. The +> apiserver refreshes a CRD's conversion configuration asynchronously +> after the write that changed it, so for a moment after a repoint it is +> still calling the source — and a replica that dropped its plan the +> instant the object changed answered that call with a 503. Replicas now +> drain for thirty seconds after a target stops naming them, which is the +> same race and the same treatment as the pod's `preStop` sleep one layer +> down. +> - **Sharding needed a prerequisite the issue predicted, and it changed +> the webhook-server too.** Per-target readiness is published rather than +> queried — each replica writes its servable set into a Lease, and +> `status.servedTargets` is the intersection — because the operator's +> reconcile loop must not call pods. The half that is not obvious is on +> the *losing* side: a replica now holds a plan while either the resolver +> assigns the target to it **or** the live target still names its Service. +> Without that, waiting for the destination would itself be the outage. +> It also fixes a race that predates sharding: editing `webhookServerRef` +> by hand always had this window. +> - **Rendezvous hashing, not the "consistent hashing" the issue names.** +> Same intent, better disruption property and no virtual-node count to +> tune. See the design note on +> [#157](https://github.com/terasky-oss/declarative-conversion-operator/issues/157). +> - **The nightly scale run is 300 CRDs, not the 1000 named here.** A +> standard hosted runner is four shared vCPUs hosting an entire +> single-node control plane, and applying CRDs is apiserver-CPU-bound. +> 300 × 20 completes in ~25 minutes with real headroom; an aspirational +> number that always fails would be worth less than a smaller one that +> always runs. The envelope is a workflow input so the ceiling can be +> raised on evidence, and a run at a different envelope skips the +> comparison rather than reporting a false regression. +> +> One item was widened. 15.5 says "no new metrics need registering"; that +> is true of the manager, which serves controller-runtime's registry +> directly, and false of the webhook-server, which deliberately serves a +> dedicated one — so its registry reconciler was the only controller in the +> system with no queue-depth signal anywhere. Its `/metrics` now gathers +> both registries. + - **Automatic sharding.** `assign.ResolveAssignment` supports explicit and default assignment; add a policy that balances N targets across M `ConversionWebhookServer` instances, with a `spec.shardCount` and rebalance diff --git a/docs/roadmap.md b/docs/roadmap.md index 4da8401..dd05ad6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -5,7 +5,7 @@ timeline. Phases already shipped stay listed so the arc is visible; later phases are invitations to [open an issue or PR](https://github.com/terasky-oss/declarative-conversion-operator/issues) if one of them matters to you sooner. -## Shipped (phases 0–14) +## Shipped (phases 0–15) | Phase | Epic | Intent | |---|---|---| @@ -24,10 +24,11 @@ if one of them matters to you sooner. | **12 — XRD/CRD API evolution lifecycle** | [#125](https://github.com/terasky-oss/declarative-conversion-operator/issues/125) | The migration *sequence* made checkable rather than left in prose: `convctl plan` prints the ordered, gated path from where a target is to the version you name, offers exactly one step at a time, and marks the three steps that genuinely need a cluster as `UNKNOWN` with the command that answers them instead of guessing. Golden-corpus testing (`--record` / `--golden`) so a config edit shows up in review as *"this changes the output for these three objects, in these fields"*. `--validate-output` checking every converted object against the destination schema through the apiserver's own validator, and required-field analysis catching at compile time what would otherwise fail at admission. Property-based round-trip fuzzing whose generated values take their lexical shape from the rules, not just the schema. `convctl compat --base/--head` classifying the delta between two git revisions into eight breaking-change classes, designed as a required status check. `convctl versions` answering *"is it safe to unserve this version yet?"* from `managedFields`, live object counts and `storedVersions`. | | **13 — CI/CD: official GitHub Actions** | [#133](https://github.com/terasky-oss/declarative-conversion-operator/issues/133) | A supported way to get `convctl` into a pipeline, where before the reference workflow's install step was `exit 1`. First-party composite Actions — `setup-convctl`, which verifies the cosign signature by default and still verifies on a cache hit; `convctl-test`, with a JUnit artifact, a job summary and annotations on the diff; `convctl-diff`, as a sticky PR comment that says "no deltas" rather than vanishing; `convctl-fleet`, one aggregated report across clusters — each exercised by a test workflow whose most important job proves verification *fails* on a tampered archive. CI-native output formats (`github`, `sarif`, `markdown`) built on real source locations, so a finding lands on the line of the config that produced it. `convctl lint` over a whole tree, with unpaired and duplicate configs reported rather than skipped. A published `convctl` image. Bounded `--live` sampling that says plainly when it sampled. `--package`, so a Configuration's XRDs can be tested from a local `.xpkg` before publishing. Homebrew, Scoop, deb and rpm, and a `version` command that reports something a bug report can use. | | **14 — Production readiness** | [#145](https://github.com/terasky-oss/declarative-conversion-operator/issues/145) | Informer caches that scale with the number of conversion configs rather than with the cluster: no Secret informer at all in the manager, label-scoped owned workloads, and a webhook-server cache transform that strips what the engine never reads. Timeouts and a body limit on both HTTP servers, with an oversized body answered as a well-formed failing `ConversionReview`. The panic path preserving the request UID, so its message actually arrives. Per-object conversion metrics, so a mixed-direction batch stops being attributed to whichever object was last. Rollout safety — `preStop`, grace period, spread, `maxUnavailable: 0` — proved by a nightly soak that rolls the webhook-server under load and asserts zero failed **and zero wrong** conversions. A curated `.golangci.yml` with every finding fixed, `govulncheck`/CodeQL/Trivy/Scorecard/Dependabot, a chart `values.schema.json`, `helm-unittest` in place of the CI `grep` block, and a [deprecation policy](deprecation-policy.md). | +| **15 — Performance and scale** | [#156](https://github.com/terasky-oss/declarative-conversion-operator/issues/156) | Scaling out stops being a manual assignment exercise: `spec.sharding` distributes unpinned configs across every instance that opts in, by rendezvous hashing, with explicit `webhookServerRef` still winning over everything. The move is the hard part, and it is gated — each replica publishes what it can actually serve into a Lease, `status.servedTargets` is the intersection across them, and the operator does not repoint a target until the destination confirms it — the one exception being an instance that publishes nothing at all for a 30-second grace period, where it proceeds unverified rather than blocking a fleet that structurally cannot report, and says so on the condition. The source keeps serving a target that still names it. Proved by an e2e that reassigns three times under sustained load and asserts zero failed **and zero wrong** conversions. A cold start that is measured (`dco_webhook_initial_sync_duration_seconds`), parallel, and bounded: the health endpoint now comes up *before* the registry sync, so a cold replica is visibly alive while `/readyz` and the conversion endpoint still wait, and a `startupProbe` on `/readyz` turns `periodSeconds × failureThreshold` into a real deadline on the sync — without which a replica wedged mid-sync would stay liveness-healthy and never ready, out of the Service and never restarted. Memory per target published for the first time, which found that compiling churns twenty times what it retains while the Go runtime had no idea a container limit existed: `GOMEMLIMIT` tells it, which roughly halves the cold-start transient — a soft target rather than a ceiling, so the live working set is still the thing to size the limit against. A nightly scale run at a raised envelope, publishing an artifact and failing on a relative regression that it names. And controller-runtime's workqueue depth — the leading indicator of a reconcile backlog — on the shipped dashboard and in two alerts, with `--max-concurrent-reconciles` as the lever it prompts. | ## Proposed next phases -Phases 0–14 are complete — 14 was taken out of order because four of its +Phases 0–15 are complete — 14 was taken out of order because four of its items were concrete defects on the apiserver's write path and did not warrant waiting for a phase, and 13 landed after 12 rather than before it despite the epic's own note to build the road first. Each phase below has an epic with per-deliverable @@ -37,7 +38,6 @@ entry — is in [Review and proposed next phases](proposals/next-phases.md). | Phase | Epic | Intent | |---|---|---| -| **15 — Performance and scale** | [#156](https://github.com/terasky-oss/declarative-conversion-operator/issues/156) | Automatic sharding across `ConversionWebhookServer` instances, a measured cold-start budget, per-target memory numbers, a nightly scale run at a raised envelope, and workqueue observability. | | **16 — Engine and strategy expansion** | [#162](https://github.com/terasky-oss/declarative-conversion-operator/issues/162) | `oneOf`/`anyOf` branch mapping, `$ref`/`allOf` flattening, and further strategies driven by real migrations. | ## Design seams worth knowing @@ -46,6 +46,7 @@ entry — is in [Review and proposed next phases](proposals/next-phases.md). - **Observability is chart-optional.** ServiceMonitor / PrometheusRule / Grafana dashboards ship with the chart and stay off unless enabled. - **Crossplane 2.x is the target; 1.x control planes are a non-goal.** The v1 compatibility layer inside 2.x — `scope: LegacyCluster`, claims, connection secrets — is in scope, and phase 11 made it first-class. - **"Applied" and "converting" are different claims.** The operator patching the XRD is not the same as Crossplane having re-rendered the generated CRD with that webhook; `ConversionPropagated` is the condition that distinguishes them ([Limitations](limitations.md), [XRDConversionConfig](configuration/xrdconversionconfig.md#applied-is-not-the-same-as-conversion-works)). +- **Who serves a target is a pure function; moving it is not.** Assignment is computed identically by the operator and by every webhook-server replica from the same objects, with no coordination. Repointing a target at a different instance is the one operation that races, and it is gated on the destination reporting it can already serve it — see [Moving a target between instances](architecture.md#moving-a-target-between-instances). - **No cross-cluster coordination.** Every operator and webhook-server replica assumes a single cluster ([Architecture: One cluster, one install](architecture.md#one-cluster-one-install), [Limitations](limitations.md)). --- diff --git a/docs/security/rbac.md b/docs/security/rbac.md index dd9768a..e948796 100755 --- a/docs/security/rbac.md +++ b/docs/security/rbac.md @@ -78,18 +78,52 @@ practice: ## Webhook-server ServiceAccount -Used by every `ConversionWebhookServer` pod. **Read/watch only** — the -webhook-server binary never mutates cluster state. Each replica runs its -own informers so it can compile conversion plans without depending on the +Used by every `ConversionWebhookServer` pod. Each replica runs its own +informers so it can compile conversion plans without depending on the manager at request time. +Cluster-wide it is **read/watch only** — the webhook-server binary never +mutates an XRD, a CRD, or a conversion config: + | API group | Resource | Verbs | Why | |---|---|---|---| | `terasky.com` | `xrdconversionconfigs`, `crdconversionconfigs`, `conversionwebhookservers` | `get`, `list`, `watch` | Discover assigned configs and the owning server. | | `apiextensions.crossplane.io` | `compositeresourcedefinitions` | `get`, `list`, `watch` | Read live XRD schemas to (re)compile plans. | | `apiextensions.k8s.io` | `customresourcedefinitions` | `get`, `list`, `watch` | Same for native CRDs. | -No access to Secrets, no write verbs, no ability to patch XRDs/CRDs. +It writes exactly one thing, and that grant is a **namespaced `Role`**, not +part of the ClusterRole: + +| API group | Resource | Verbs | Scope | Why | +|---|---|---|---|---| +| `coordination.k8s.io` | `leases` | `get`, `create`, `update`, `patch` | the release namespace only | Each replica publishes the targets it can serve into a Lease of its own, so the operator can verify an instance is ready before moving a target onto it — see [Moving a target between instances](../architecture.md#moving-a-target-between-instances). | + +Two deliberate narrowings there. **Namespaced**, because Leases are how +leader election is implemented across the ecosystem and cluster-wide write +on them is not a grant to hand out for a bookkeeping annotation. And **no +`list` or `watch`**: a replica reads back exactly one Lease, by name, so +the ability to enumerate the namespace's Leases — and with it every +leader-election holder identity — buys nothing. + +**What this is not: an own-Lease restriction.** RBAC cannot express "only +the Lease named after your own pod" — `resourceNames` needs names known +when the `Role` is written, and these are derived from generated pod names. +So a compromised webhook-server pod can `get`, `update` or `patch` *any* +Lease in its namespace, which by default includes this operator's own +leader-election Lease; disrupting that would stall reconciles until the +lease expired. The bound that does hold is the namespace. + +If that residual matters to you, give the instance its own +`spec.namespace`. The chart creates the `Role` in whatever namespace the +default instance runs in, so an instance isolated in its own namespace +leaves this grant reaching nothing else. A `ConversionWebhookServer` +created outside the chart needs the same `Role` and `RoleBinding` in its +own namespace; without them the replicas still serve conversions, and what +is lost is the verified handover — a move onto that instance waits 30 +seconds for a report that cannot come, then proceeds with `HandoverReady` +reason `HandoverUnverified` rather than failing. + +No access to Secrets, and no ability to patch XRDs or CRDs. ## Related docs diff --git a/hack/e2e-reassign.sh b/hack/e2e-reassign.sh new file mode 100755 index 0000000..151f5f1 --- /dev/null +++ b/hack/e2e-reassign.sh @@ -0,0 +1,412 @@ +#!/usr/bin/env bash +# +# Reassignment e2e: move a target from one ConversionWebhookServer to +# another while sustained reads and writes flow through it, and assert that +# not one of them failed or came back wrong. +# +# This is the claim automatic sharding rests on. Rebalancing a fleet means +# repointing a target's spec.conversion at a different Service, and the +# apiserver starts calling the new one the moment that write lands. If the +# new instance has not compiled the plan yet, every read and write of the +# resource fails until it has — an outage caused by a scaling decision, +# affecting resources that had nothing to do with it. +# +# Two moves are exercised, because they fail differently: +# +# 1. An explicit webhookServerRef change — the manual case, which has +# always been possible and has always had this window. +# 2. Enabling sharding, which moves a share of the unpinned configs +# without anyone naming a target at all. +# +# The assertion is on correctness as well as on errors, for the same reason +# the soak's is: a read that returns HTTP 200 with the wrong value is worse +# than one that fails, because nothing reports it. +# +# hack/e2e-reassign.sh [--duration SECONDS] [--objects N] +# +# Prerequisites: docker, kind, kubectl, helm, python3. +# Set KEEP_CLUSTER=1 to skip teardown. +set -euo pipefail + +# shellcheck source=hack/e2e-common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/e2e-common.sh" + +CLUSTER_NAME="${CLUSTER_NAME:-declarative-conversion-e2e-reassign}" +NAMESPACE="${NAMESPACE:-declarative-conversion-system}" +IMG_TAG="e2e-reassign-$(date +%s 2>/dev/null || echo local)" +MANAGER_IMG="ghcr.io/terasky-oss/declarative-conversion-operator:${IMG_TAG}" +WEBHOOK_IMG="ghcr.io/terasky-oss/declarative-conversion-webhook-server:${IMG_TAG}" +CERT_MANAGER_VERSION="v1.21.1" +RELEASE_NAME="declarative-conversion-operator" + +PROXY_PORT="${PROXY_PORT:-18003}" +DURATION=240 +OBJECTS=8 +MOVE_TIMEOUT="${MOVE_TIMEOUT:-180}" +TARGET_CRD="gadgets.nativecrd.example.org" +CONFIG_NAME="gadgets-e2e-conversion" +RESULT_JSON="${RESULT_JSON:-/tmp/e2e-reassign-result.json}" +PROXY_PID="" +CLIENT_PID="" +STOP_FILE="$(mktemp "${TMPDIR:-/tmp}/e2e-reassign-stop.XXXXXX")" +rm -f "${STOP_FILE}" + +while [ $# -gt 0 ]; do + case "$1" in + --duration) DURATION="$2"; shift 2 ;; + --objects) OBJECTS="$2"; shift 2 ;; + --move-timeout) MOVE_TIMEOUT="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +require_positive_int() { + local name="$1" value="$2" + case "${value}" in + ''|*[!0-9]*) echo "FAIL: ${name} must be a positive integer, got '${value}'" >&2; exit 2 ;; + esac + if [ "${value}" -le 0 ]; then + echo "FAIL: ${name} must be greater than zero, got '${value}'" >&2 + exit 2 + fi +} +require_positive_int --duration "${DURATION}" +require_positive_int --objects "${OBJECTS}" +require_positive_int --move-timeout "${MOVE_TIMEOUT}" + +reassign_cleanup() { + local code=$? + # set +e first: e2e_cleanup has to run even when the test failed, and + # under `set -e` a non-zero status inside the trap can end it early — + # leaving the kind cluster behind on exactly the runs where nobody wants + # a stray cluster. + set +e + [ -n "${CLIENT_PID}" ] && kill "${CLIENT_PID}" >/dev/null 2>&1 + [ -n "${PROXY_PID}" ] && kill "${PROXY_PID}" >/dev/null 2>&1 + rm -f "${STOP_FILE}" + (exit "${code}") + e2e_cleanup +} +trap reassign_cleanup EXIT + +require_cmd docker +require_cmd kind +require_cmd kubectl +require_cmd helm +require_cmd python3 + +# wait_for_assignment blocks until the config reports the named server AND +# the webhook URL the operator wrote actually points at it. The status field +# alone is not enough: it is set before the gate, so it names the intended +# server while the target still points at the old one, which is exactly the +# state this test is about. +wait_for_assignment() { + local want="$1" deadline=$(( $(date +%s) + MOVE_TIMEOUT )) + while [ "$(date +%s)" -lt "${deadline}" ]; do + local assigned url + assigned="$(kubectl get crdconversionconfig "${CONFIG_NAME}" -o jsonpath='{.status.assignedWebhookServer}' 2>/dev/null || true)" + url="$(kubectl get crdconversionconfig "${CONFIG_NAME}" -o jsonpath='{.status.webhookURL}' 2>/dev/null || true)" + if [ "${assigned}" = "${want}" ] && case "${url}" in *"${want}-webhook-server"*) true ;; *) false ;; esac; then + echo "OK: ${CONFIG_NAME} is served by ${want} (${url})" + return 0 + fi + sleep 2 + done + echo "FAIL: ${CONFIG_NAME} did not move to ${want} within ${MOVE_TIMEOUT}s" + kubectl get crdconversionconfig "${CONFIG_NAME}" -o yaml || true + kubectl get conversionwebhookserver -o yaml || true + exit 1 +} + +# assert_handover_verified proves the Lease-backed readiness signal is what +# allowed the move, rather than the compatibility fallback for a fleet that +# publishes nothing. Without this the test would pass just as happily with +# the whole mechanism removed. +assert_handover_verified() { + local reason + reason="$(kubectl get crdconversionconfig "${CONFIG_NAME}" \ + -o jsonpath='{.status.conditions[?(@.type=="HandoverReady")].reason}' 2>/dev/null || true)" + if [ "${reason}" != "HandoverReady" ]; then + echo "FAIL: HandoverReady reason is '${reason}', want 'HandoverReady'." + echo " The move went through without the destination confirming it could serve the target," + echo " which means the readiness signal is not working and the window is still open." + kubectl get crdconversionconfig "${CONFIG_NAME}" -o yaml || true + kubectl -n "${NAMESPACE}" get leases -l conversion.terasky.com/webhook-server -o yaml || true + exit 1 + fi + echo "OK: the handover was verified against the destination's published served targets" +} + +create_kind_cluster +build_and_load_images +install_cert_manager +# Native CRD only: the reassignment window is a property of the webhook +# Service the target names, identical whether that target is a CRD or an +# XRD, and skipping Crossplane takes several minutes off the setup. +install_operator \ + --set features.crossplane.enabled=false \ + --set features.nativeCRD.enabled=true + +log "Waiting for the default ConversionWebhookServer to become Available" +kubectl wait --for=condition=Available --timeout=180s conversionwebhookserver/default + +log "Creating a second ConversionWebhookServer" +cat </dev/null 2>&1 || true +kubectl patch conversionwebhookserver default --type=merge -p '{"spec":{"sharding":null}}' >/dev/null 2>&1 || true +kubectl patch crdconversionconfig "${CONFIG_NAME}" --type=json \ + -p '[{"op":"remove","path":"/spec/webhookServerRef"}]' >/dev/null 2>&1 || true + +log "Applying the native Gadget CRD and CRDConversionConfig" +kubectl apply -f "${REPO_ROOT}/test/e2e/testdata/crd.yaml" +kubectl wait --for=condition=Established --timeout=60s "crd/${TARGET_CRD}" +kubectl apply -f "${REPO_ROOT}/test/e2e/testdata/crdconversionconfig.yaml" +kubectl wait --for=condition=Applied --timeout=120s "crdconversionconfig/${CONFIG_NAME}" +wait_for_assignment default + +# Both instances must be publishing before the first move, or the move would +# take the unverified fallback path and prove nothing. +log "Waiting for both instances to publish their served targets" +deadline=$(( $(date +%s) + MOVE_TIMEOUT )) +while [ "$(date +%s)" -lt "${deadline}" ]; do + a="$(kubectl get conversionwebhookserver default -o jsonpath='{.status.reportingReplicas}' 2>/dev/null || echo 0)" + b="$(kubectl get conversionwebhookserver shard-b -o jsonpath='{.status.reportingReplicas}' 2>/dev/null || echo 0)" + if [ "${a:-0}" -ge 1 ] && [ "${b:-0}" -ge 1 ]; then + echo "OK: default reports ${a} replicas, shard-b reports ${b}" + break + fi + sleep 2 +done +if [ "${a:-0}" -lt 1 ] || [ "${b:-0}" -lt 1 ]; then + echo "FAIL: replicas never published their served targets (default=${a:-0}, shard-b=${b:-0});" + echo " check the webhook-server Lease RBAC in namespace ${NAMESPACE}" + kubectl -n "${NAMESPACE}" get leases || true + exit 1 +fi + +log "Seeding ${OBJECTS} Gadgets at the storage version" +names="" +for i in $(seq 1 "${OBJECTS}"); do + name="move-${i}" + cat </dev/null +apiVersion: nativecrd.example.org/v2 +kind: Gadget +metadata: + name: ${name} + namespace: default +spec: + storageGB: "${i}" + replicas: 1 +EOF + names="${names}${names:+,}${name}" +done + +log "Starting kubectl proxy on 127.0.0.1:${PROXY_PORT}" +kubectl proxy --port="${PROXY_PORT}" >/dev/null 2>&1 & +PROXY_PID=$! +for _ in $(seq 1 30); do + if curl -sf "http://127.0.0.1:${PROXY_PORT}/healthz" >/dev/null 2>&1; then break; fi + sleep 1 +done + +# Prove the checker can tell right from wrong before trusting it to say +# "zero mismatches". A checker that silently passed everything would make +# this whole test a no-op. +log "Self-check: one clean pass before any move" +python3 "${REPO_ROOT}/hack/e2e-soak-client.py" \ + --base "http://127.0.0.1:${PROXY_PORT}" --duration 5 \ + --names "${names}" --out "${RESULT_JSON}.pre" +pre_bad="$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(d['read_failures']+d['write_failures']+d['mismatches'])" "${RESULT_JSON}.pre")" +if [ "${pre_bad}" != "0" ]; then + echo "FAIL: the cluster was already unhealthy before the first move:" + cat "${RESULT_JSON}.pre" + exit 1 +fi +log "Self-check passed" + +log "Driving reads and writes across three reassignments" +python3 "${REPO_ROOT}/hack/e2e-soak-client.py" \ + --base "http://127.0.0.1:${PROXY_PORT}" --duration "$(( DURATION + 4 * MOVE_TIMEOUT ))" \ + --stop-file "${STOP_FILE}" \ + --names "${names}" --out "${RESULT_JSON}" & +CLIENT_PID=$! + +driver_alive() { + if ! kill -0 "${CLIENT_PID}" 2>/dev/null; then + echo "FAIL: the traffic driver exited before $1; the move would have been unobserved" + exit 1 + fi +} + +gap=$(( DURATION / 4 )) +sleep "${gap}" +driver_alive "the explicit move to shard-b" + +log "Move 1/3: pinning the config to shard-b with an explicit webhookServerRef" +kubectl patch crdconversionconfig "${CONFIG_NAME}" --type=merge \ + -p '{"spec":{"webhookServerRef":{"name":"shard-b"}}}' +wait_for_assignment shard-b +assert_handover_verified + +sleep "${gap}" +driver_alive "the move back to default" + +log "Move 2/3: unpinning, which returns the config to the default instance" +kubectl patch crdconversionconfig "${CONFIG_NAME}" --type=json \ + -p '[{"op":"remove","path":"/spec/webhookServerRef"}]' +wait_for_assignment default +assert_handover_verified + +sleep "${gap}" +driver_alive "the sharding move" + +# Enabling sharding on a non-default instance while the default sits outside +# the pool would move every unpinned config at once. Admission rejects it, +# and the rejection is worth asserting: it is the invariant that makes the +# pool safe to prefer over spec.default. +log "Admission must reject a pool the default instance is not in" +if kubectl patch conversionwebhookserver shard-b --type=merge \ + -p '{"spec":{"sharding":{"enabled":true}}}' 2>/dev/null; then + echo "FAIL: enabling sharding on shard-b alone was accepted; that moves every unpinned config off the default instance in one write" + exit 1 +fi +echo "OK: rejected, as it must be" + +log "Move 3/3: enabling sharding on the default instance, then on shard-b" +kubectl patch conversionwebhookserver default --type=merge \ + -p '{"spec":{"sharding":{"enabled":true}}}' +# A pool of one is still the whole pool, so nothing may move here. +wait_for_assignment default +kubectl patch conversionwebhookserver shard-b --type=merge \ + -p '{"spec":{"sharding":{"enabled":true}}}' + +# Where the target lands is decided by rendezvous hashing, so the test must +# not assume an answer — it asserts only that it lands somewhere, that the +# choice is stable, and that the traffic never broke. +log "Waiting for the sharded assignment to settle" +deadline=$(( $(date +%s) + MOVE_TIMEOUT )) +settled="" +while [ "$(date +%s)" -lt "${deadline}" ]; do + assigned="$(kubectl get crdconversionconfig "${CONFIG_NAME}" -o jsonpath='{.status.assignedWebhookServer}' 2>/dev/null || true)" + url="$(kubectl get crdconversionconfig "${CONFIG_NAME}" -o jsonpath='{.status.webhookURL}' 2>/dev/null || true)" + case "${url}" in *"${assigned}-webhook-server"*) settled="${assigned}"; break ;; esac + sleep 2 +done +if [ -z "${settled}" ]; then + echo "FAIL: the sharded assignment never settled" + kubectl get crdconversionconfig "${CONFIG_NAME}" -o yaml || true + exit 1 +fi +echo "OK: sharding placed ${TARGET_CRD} on ${settled}" + +# Rendezvous hashing is deterministic in the target name and the pool, both +# of which are fixed here, so this fixture lands on shard-b. Asserting that +# rather than accepting whatever came out is the point: if it ever resolved +# to `default`, no move would have happened and the run would pass without +# having exercised a sharded handover at all — the one thing this step is +# for. A hash change should fail here and prompt a new fixture, loudly. +assert_eq "${settled}" "shard-b" "sharding moved the target off the default instance" +assert_handover_verified + +# Force a reconcile and re-read, rather than re-reading the status the poll +# above already saw. An assignment that is unstable across reconciles — +# flapping between pool members — would otherwise pass this. +# +# Clearing status.assignedWebhookServer first is what makes the re-read mean +# anything. The field already holds the right answer, so polling it straight +# after the annotation would return on the first read, before the reconcile +# the annotation triggers had run. Blanked, it can only come back if a +# reconcile actually completed — and the resolver recomputes the assignment +# from the pool rather than reading the old value, so what comes back is a +# fresh answer, not a remembered one. status.webhookURL is deliberately left +# alone: the handover gate reads it, and clearing it would open the gate. +kubectl patch crdconversionconfig "${CONFIG_NAME}" --subresource=status --type=merge \ + -p '{"status":{"assignedWebhookServer":null}}' >/dev/null +kubectl annotate crdconversionconfig "${CONFIG_NAME}" \ + "e2e.terasky.com/poke=$(date +%s)" --overwrite >/dev/null +again="" +for _ in $(seq 1 60); do + again="$(kubectl get crdconversionconfig "${CONFIG_NAME}" -o jsonpath='{.status.assignedWebhookServer}')" + [ -n "${again}" ] && break + sleep 1 +done +if [ -z "${again}" ]; then + echo "FAIL: the assignment was never recomputed after a forced reconcile" + kubectl get crdconversionconfig "${CONFIG_NAME}" -o yaml || true + exit 1 +fi +assert_eq "${again}" "${settled}" "the sharded assignment is stable across a fresh reconcile" + +sleep "${gap}" +driver_alive "the run finishing" + +log "Reassignments complete; asking the traffic driver to stop" +touch "${STOP_FILE}" +wait "${CLIENT_PID}" +CLIENT_PID="" + +log "Result:" +cat "${RESULT_JSON}" +echo + +python3 - "${RESULT_JSON}" <<'PYCHK' +import json, sys +d = json.load(open(sys.argv[1])) +ok = True + +if d.get("stopped_by") == "duration": + print("FAIL: the traffic driver hit its maximum duration instead of being stopped after the moves; a move did not complete") + ok = False + +if d["reads"] < 100 or d["writes"] < 100: + print(f"FAIL: too little traffic to prove anything: {d['reads']} reads, {d['writes']} writes") + ok = False + +if d["read_failures"] or d["write_failures"]: + print(f"FAIL: {d['read_failures']} failed reads and {d['write_failures']} failed writes while the target was being " + "moved between webhook servers; no target may be unserved during a move") + ok = False + +if d["mismatches"]: + print(f"FAIL: {d['mismatches']} requests returned successfully with the WRONG converted value") + ok = False + +if not ok: + for s in d.get("samples", []): + print(f" [{s['kind']}] {s['detail']}") + sys.exit(1) + +print(f"OK: {d['reads']} reads and {d['writes']} writes across three reassignments, " + f"0 failures, 0 wrong values ({d.get('conflicts', 0)} benign write conflicts)") +PYCHK + +log "All reassignment assertions passed" diff --git a/hack/e2e-scale.sh b/hack/e2e-scale.sh index aea54c8..ad20f0c 100755 --- a/hack/e2e-scale.sh +++ b/hack/e2e-scale.sh @@ -9,6 +9,12 @@ # # TARGETS=100 INSTANCES=100 PARALLEL=32 ./hack/e2e-scale.sh # +# Set RESULT_JSON to write the run's measurements as JSON — latency +# percentiles, throughput, plus the webhook-server's cold-start time and +# its loaded working set read off the cluster. That is what the nightly +# Scale workflow publishes as an artifact and diffs against the previous +# run. +# # Prerequisites: docker, kind, kubectl, helm. # Set KEEP_CLUSTER=1 to skip teardown. set -euo pipefail @@ -36,6 +42,7 @@ BURST="${BURST:-200}" RESET="${RESET:-1}" LIST_REPEATS="${LIST_REPEATS:-3}" GET_REPEATS="${GET_REPEATS:-1}" +RESULT_JSON="${RESULT_JSON:-}" trap e2e_cleanup EXIT @@ -43,6 +50,7 @@ require_cmd docker require_cmd kind require_cmd kubectl require_cmd helm +require_cmd python3 create_kind_cluster build_and_load_images @@ -71,6 +79,56 @@ SCALE_ARGS=( if [ "${RESET}" != "0" ]; then SCALE_ARGS+=(--reset) fi -go run "${REPO_ROOT}/cmd/scalegen" "${SCALE_ARGS[@]}" +if [ -n "${RESULT_JSON}" ]; then + SCALE_ARGS+=(--result-json "${RESULT_JSON}") +fi + +# The run's exit status is kept rather than propagated immediately: a run +# that ended with get/list errors is exactly the run whose numbers and +# cluster-side observations are worth keeping, and `set -e` would throw +# them away. +scale_rc=0 +go run "${REPO_ROOT}/cmd/scalegen" "${SCALE_ARGS[@]}" || scale_rc=$? + +if [ -n "${RESULT_JSON}" ] && [ -f "${RESULT_JSON}" ]; then + # Roll the webhook-server before measuring, and only after the traffic is + # finished so it cannot perturb the latency numbers. + # + # Without this the cold-start figure is meaningless: the replicas started + # before any of this fleet existed, so they synced zero targets in + # microseconds, and the artifact would trend that forever. Restarting them + # now makes them compile the whole generated fleet, which is the number + # this run is supposed to publish — and it makes the working set that + # follows a loaded steady state rather than an empty one. + dep="$(kubectl -n "${NAMESPACE}" get deploy \ + -l app.kubernetes.io/name=declarative-conversion-webhook-server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [ -n "${dep}" ]; then + log "Restarting ${dep} so the cold start is measured against the generated fleet" + kubectl -n "${NAMESPACE}" rollout restart "deployment/${dep}" + kubectl -n "${NAMESPACE}" rollout status "deployment/${dep}" --timeout=600s + else + echo "WARN: no webhook-server deployment found; cold start will not be measured" >&2 + fi + + log "Collecting cluster-side observations (cold start, loaded working set)" + # Not `|| true`. The script tolerates partial collection internally — one + # unscrapeable pod does not discard the run — but if it collected nothing + # at all, the artifact is missing half of what a nightly run exists to + # publish, and reporting that green would hide it. The result file is + # still written and still uploaded, so failing here loses no data. + observe_rc=0 + python3 "${REPO_ROOT}/hack/scale-observe.py" \ + --result "${RESULT_JSON}" --namespace "${NAMESPACE}" || observe_rc=$? +fi + +if [ "${scale_rc}" -ne 0 ]; then + echo "FAIL: the scale run reported errors (exit ${scale_rc})" + exit "${scale_rc}" +fi +if [ "${observe_rc:-0}" -ne 0 ]; then + echo "FAIL: cluster-side observations could not be collected (exit ${observe_rc})" + exit "${observe_rc}" +fi log "Scale e2e finished" diff --git a/hack/prometheus/rules.test.yml b/hack/prometheus/rules.test.yml index 4fca2f1..bb69611 100644 --- a/hack/prometheus/rules.test.yml +++ b/hack/prometheus/rules.test.yml @@ -470,3 +470,124 @@ tests: exp_annotations: summary: "The conversion webhook panicked while converting xfoos.example.org" description: "A recovered panic is always a bug in declarative-conversion-operator, and every request that hits it fails a write on the apiserver's admission path. The pod log carries the stack trace. No `for:` delay: one panic is already worth waking up for." + + # --- ControllerWorkqueueBacklog --- + - name: a sustained queue depth fires + interval: 1m + input_series: + # Depth climbs and stays there: the controller is not draining. + - series: 'workqueue_depth{app_kubernetes_io_name="declarative-conversion-operator",job="dco-manager",controller="xrdconversionconfig",priority=""}' + values: "0 2 6 12 18 24 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30" + alert_rule_test: + - eval_time: 5m + alertname: ControllerWorkqueueBacklog + exp_alerts: [] + - eval_time: 20m + alertname: ControllerWorkqueueBacklog + exp_alerts: + - exp_labels: + severity: warning + job: dco-manager + controller: xrdconversionconfig + exp_annotations: + summary: "Reconcile backlog on controller xrdconversionconfig" + description: "job=dco-manager. This controller's workqueue has stayed above the backlog threshold, so config changes are converging slowly or not at all — expect stale phases and a lagging ConversionPropagated to follow. Check reconcile latency and error rate on the same dashboard row, then raise --max-concurrent-reconciles if the apiserver has the headroom." + + - name: a burst that drains does not fire + interval: 1m + input_series: + # A bulk apply spikes the queue well past the threshold and then it + # empties. This is the shape the `for:` exists to tolerate — alerting + # on it would make the panel useless on any cluster where configs are + # applied in batches. + - series: 'workqueue_depth{app_kubernetes_io_name="declarative-conversion-operator",job="dco-manager",controller="xrdconversionconfig",priority=""}' + values: "0 40 35 20 8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0" + alert_rule_test: + - eval_time: 20m + alertname: ControllerWorkqueueBacklog + exp_alerts: [] + + - name: a depth at the threshold does not fire + interval: 1m + input_series: + # Exactly 10 is not "> 10". Asserted because an off-by-one here would + # page on a queue that is merely busy. + - series: 'workqueue_depth{app_kubernetes_io_name="declarative-conversion-operator",job="dco-manager",controller="conversionwebhookserver",priority=""}' + values: "10+0x30" + alert_rule_test: + - eval_time: 20m + alertname: ControllerWorkqueueBacklog + exp_alerts: [] + + # --- ControllerReconcileErrors --- + - name: a sustained reconcile error rate fires + interval: 1m + input_series: + # 30 errors per minute = 0.5/s, well above the 0.1/s threshold. + - series: 'controller_runtime_reconcile_errors_total{app_kubernetes_io_name="declarative-conversion-operator",job="dco-manager",controller="crdconversionconfig"}' + values: "0+30x40" + alert_rule_test: + - eval_time: 5m + alertname: ControllerReconcileErrors + exp_alerts: [] + - eval_time: 25m + alertname: ControllerReconcileErrors + exp_alerts: + - exp_labels: + severity: warning + job: dco-manager + controller: crdconversionconfig + exp_annotations: + summary: "Sustained reconcile errors on controller crdconversionconfig" + description: "job=dco-manager. Reconciles are returning errors faster than the threshold and being retried with backoff, which both delays convergence and feeds the workqueue depth. Unlike dco_manager_analyze_failures_total this counts infrastructure failures too — a lost API connection, a rejected write — so check the controller's logs rather than the config's status first." + + - name: the occasional retried error does not fire + interval: 1m + input_series: + # One error every ten minutes: a config in permanent error being + # retried under backoff. Real, already reported by the config's own + # status, and not worth a second alert. + - series: 'controller_runtime_reconcile_errors_total{app_kubernetes_io_name="declarative-conversion-operator",job="dco-manager",controller="crdconversionconfig"}' + values: "0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 4" + alert_rule_test: + - eval_time: 35m + alertname: ControllerReconcileErrors + exp_alerts: [] + + # workqueue_* and controller_runtime_* are controller-runtime's names, not + # this operator's: every controller-runtime workload in the cluster emits + # them. Without a selector, somebody else's reconcile backlog would page + # whoever owns this chart's alerts. + - name: another operator's backlog does not fire this operator's alert + interval: 1m + input_series: + - series: 'workqueue_depth{app_kubernetes_io_name="some-other-operator",job="other",controller="widgets",priority=""}' + values: "50+0x30" + - series: 'controller_runtime_reconcile_errors_total{app_kubernetes_io_name="some-other-operator",job="other",controller="widgets"}' + values: "0+60x40" + alert_rule_test: + - eval_time: 25m + alertname: ControllerWorkqueueBacklog + exp_alerts: [] + - eval_time: 25m + alertname: ControllerReconcileErrors + exp_alerts: [] + + # The webhook-server's own registry reconciler is in scope: it is this + # operator's controller, it just lives in the other binary. + - name: the webhook-server's own reconcile backlog does fire + interval: 1m + input_series: + - series: 'workqueue_depth{app_kubernetes_io_name="declarative-conversion-webhook-server",job="dco-webhook",controller="webhookserver-registry-xrd",priority=""}' + values: "30+0x30" + alert_rule_test: + - eval_time: 20m + alertname: ControllerWorkqueueBacklog + exp_alerts: + - exp_labels: + severity: warning + job: dco-webhook + controller: webhookserver-registry-xrd + exp_annotations: + summary: "Reconcile backlog on controller webhookserver-registry-xrd" + description: "job=dco-webhook. This controller's workqueue has stayed above the backlog threshold, so config changes are converging slowly or not at all — expect stale phases and a lagging ConversionPropagated to follow. Check reconcile latency and error rate on the same dashboard row, then raise --max-concurrent-reconciles if the apiserver has the headroom." diff --git a/hack/prometheus/rules.yaml b/hack/prometheus/rules.yaml index 2853226..92f4d6d 100644 --- a/hack/prometheus/rules.yaml +++ b/hack/prometheus/rules.yaml @@ -97,6 +97,22 @@ groups: annotations: summary: "Config {{ $labels.target }} phase transitioned to {{ $labels.to_phase }}" description: "config_kind={{ $labels.config_kind }} from={{ $labels.from_phase }} reason={{ $labels.reason }}. Stale means KeepServingStale after drift; Failed usually means FailClosed revert." + - alert: ControllerWorkqueueBacklog + expr: sum by (job, controller) (workqueue_depth{app_kubernetes_io_name=~"declarative-conversion-operator|declarative-conversion-webhook-server"}) > 10 + for: 15m + labels: + severity: warning + annotations: + summary: "Reconcile backlog on controller {{ $labels.controller }}" + description: "job={{ $labels.job }}. This controller's workqueue has stayed above the backlog threshold, so config changes are converging slowly or not at all — expect stale phases and a lagging ConversionPropagated to follow. Check reconcile latency and error rate on the same dashboard row, then raise --max-concurrent-reconciles if the apiserver has the headroom." + - alert: ControllerReconcileErrors + expr: sum by (job, controller) (rate(controller_runtime_reconcile_errors_total{app_kubernetes_io_name=~"declarative-conversion-operator|declarative-conversion-webhook-server"}[5m])) > 0.1 + for: 15m + labels: + severity: warning + annotations: + summary: "Sustained reconcile errors on controller {{ $labels.controller }}" + description: "job={{ $labels.job }}. Reconciles are returning errors faster than the threshold and being retried with backoff, which both delays convergence and feeds the workqueue depth. Unlike dco_manager_analyze_failures_total this counts infrastructure failures too — a lost API connection, a rejected write — so check the controller's logs rather than the config's status first." - alert: ConversionWebhookReplicaNotReady expr: dco_webhook_ready == 0 for: 5m diff --git a/hack/scale-observe.py b/hack/scale-observe.py new file mode 100755 index 0000000..33cdd0c --- /dev/null +++ b/hack/scale-observe.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Merge cluster-side observations into a scalegen result file. + +scalegen measures the run from the client's side: latency and throughput +through the apiserver's conversion path. Two of the numbers a scale run is +supposed to publish are not visible from there — + + * cold start, i.e. how long a webhook-server replica spent compiling + every assigned plan before it could serve, which is the term that grows + with the fleet and the one a startupProbe has to be sized against; and + * the loaded working set, which is what decides whether the envelope + fits in a container limit at all. Note that this is the steady state + after the cold start, not the transient peak during it — the peak + happens before the replicas are Ready, where nothing is sampling, and + is measured by the -benchmem benchmarks instead. + +Both are read off the cluster here and merged into the same JSON, so the +artifact a scheduled run publishes is one file rather than three. + +Individual failures are tolerated — one unscrapeable pod should not +discard a twenty-five-minute run — but collecting NOTHING is an error, so +a nightly that lost both measurements cannot report itself green. + +Usage: + hack/scale-observe.py --result FILE --namespace NS [--label SELECTOR] +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys + +WEBHOOK_SELECTOR = "app.kubernetes.io/name=declarative-conversion-webhook-server" +METRICS_PORT = 8443 + + +def kubectl(*args: str) -> str: + return subprocess.run( + ["kubectl", *args], check=True, capture_output=True, text=True + ).stdout + + +def warn(message: str) -> None: + print(f"scale-observe: {message}", file=sys.stderr) + + +def running_pods(namespace: str, selector: str) -> list[tuple[str, str]]: + """Ready, not-terminating pods. + + Running is not enough. Immediately after a rolling restart — which is + when this is called, so that the cold start is measured against the + real fleet — the outgoing replicas are still Running and still + reporting the metrics from *before* the fleet existed. Counting them + would inflate the replica count and mix two generations' numbers. + """ + raw = kubectl( + "-n", namespace, "get", "pod", "-l", selector, + "--field-selector=status.phase=Running", "-o", "json", + ) + out = [] + for p in json.loads(raw).get("items", []): + if p["metadata"].get("deletionTimestamp"): + continue + ready = any( + c.get("type") == "Ready" and c.get("status") == "True" + for c in p.get("status", {}).get("conditions", []) + ) + if ready: + out.append((p["metadata"]["name"], p["spec"]["nodeName"])) + return out + + +def gauge(metrics: str, name: str) -> float | None: + """Read an unlabelled gauge out of a Prometheus exposition body.""" + for line in metrics.splitlines(): + if line.startswith("#"): + continue + parts = line.split(None, 1) + if len(parts) == 2 and parts[0] == name: + try: + return float(parts[1]) + except ValueError: + return None + return None + + +def working_set_bytes(node: str, namespace: str, pod: str) -> float | None: + """Working set from the kubelet Summary API. + + The same number hack/measure-cache-memory.sh uses, and the same one a + container memory limit is enforced against — unlike RSS, it excludes + reclaimable page cache. + + This is a single sample taken once the replicas are Ready again, so it + is the loaded STEADY state, not the transient peak during the cold + start: the peak happens before readiness, where nothing is sampling. + The peak is measured by the -benchmem benchmarks instead, and published + in docs/operations/capacity.md. + + Matched on namespace as well as name, because a pod name is unique only + within its namespace and the Summary API reports the whole node. + """ + summary = json.loads(kubectl("get", "--raw", f"/api/v1/nodes/{node}/proxy/stats/summary")) + for entry in summary.get("pods", []): + ref = entry.get("podRef", {}) + if ref.get("name") == pod and ref.get("namespace") == namespace: + value = entry.get("memory", {}).get("workingSetBytes") + return float(value) if value is not None else None + return None + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--result", required=True, help="scalegen --result-json file to merge into") + ap.add_argument("--namespace", required=True) + ap.add_argument("--label", default=WEBHOOK_SELECTOR) + args = ap.parse_args() + + with open(args.result, encoding="utf-8") as fh: + report = json.load(fh) + + observed: dict[str, float] = {} + try: + pods = running_pods(args.namespace, args.label) + except subprocess.CalledProcessError as err: + warn(f"could not list webhook-server pods: {err.stderr.strip()}") + pods = [] + + if not pods: + warn("no running webhook-server pods; skipping cluster-side observations") + + sync_seconds: list[float] = [] + sync_targets: list[float] = [] + peak_bytes: list[float] = [] + for pod, node in pods: + try: + metrics = kubectl( + "get", "--raw", + f"/api/v1/namespaces/{args.namespace}/pods/{pod}:{METRICS_PORT}/proxy/metrics", + ) + except subprocess.CalledProcessError as err: + warn(f"could not scrape {pod}: {err.stderr.strip()}") + else: + value = gauge(metrics, "dco_webhook_initial_sync_duration_seconds") + if value is not None: + sync_seconds.append(value) + value = gauge(metrics, "dco_webhook_initial_sync_targets") + if value is not None: + sync_targets.append(value) + + try: + value = working_set_bytes(node, args.namespace, pod) + except (subprocess.CalledProcessError, json.JSONDecodeError) as err: + warn(f"could not read the kubelet summary for {pod}: {err}") + else: + if value is not None: + peak_bytes.append(value) + + # The slowest replica is the one that decides the rollout, and the + # largest is the one that decides the limit, so both are maxima rather + # than averages. + if sync_seconds: + observed["webhookInitialSyncSeconds"] = max(sync_seconds) + if sync_targets: + observed["webhookInitialSyncTargets"] = max(sync_targets) + if peak_bytes: + observed["webhookWorkingSetBytes"] = max(peak_bytes) + if pods: + observed["webhookReplicas"] = float(len(pods)) + + # Judged on the two measurement classes, not on `observed` being + # non-empty: webhookReplicas is there whenever a Ready pod exists, so a + # run where every metrics scrape and every kubelet read failed would + # still look successful and publish an artifact with nothing in it for + # the nightly diff to compare. + if not sync_seconds and not peak_bytes: + warn("collected no cluster observations at all; the run's cold-start and working-set numbers are missing") + return 1 + + report["observed"] = {**report.get("observed", {}), **observed} + with open(args.result, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=2) + fh.write("\n") + print(f"scale-observe: merged {len(observed)} cluster observations into {args.result}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hack/scale-report.py b/hack/scale-report.py new file mode 100755 index 0000000..e2754b2 --- /dev/null +++ b/hack/scale-report.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Render a scale run as markdown, and fail it on a regression. + +A scale target nobody runs is a scale target nobody trusts — and a +scheduled run whose output nobody can read is the same thing with extra +steps. This turns scalegen's JSON into a job summary, and compares it +against the previous run's artifact. + +The comparison gates on a RELATIVE change, never on an absolute number. +Absolute timings on a hosted runner vary by a factor of two between runs +for reasons that have nothing to do with this code, so a threshold tight +enough to catch a real regression would fire constantly, and one loose +enough not to would catch nothing. A run that is 1.5x the previous run on +the same measurement is a signal; a run that took 40 ms instead of 30 ms is +not. + +Two measurements are exempt from ratio comparison and checked absolutely, +because for them zero is the only acceptable value: error counts, and the +requirement that the run did any work at all. + +Usage: + hack/scale-report.py --current cur.json [--previous prev.json] + [--threshold 1.5] [--summary out.md] +""" +from __future__ import annotations + +import argparse +import json +import sys + +# Lower is better for everything here except throughput, which the report +# carries but which is derived from p50 — so comparing both would double- +# count the same movement. Throughput is rendered, not gated. +LATENCY_FIELDS = ("p50Ms", "p99Ms") + +# Cluster-side observations worth gating on, and what they mean. +# +# webhookWorkingSetBytes is a single sample taken once the replicas are +# Ready again after the restart — so it is the loaded steady state, NOT the +# transient peak during the cold start. The peak happens before readiness, +# where nothing is sampling; pkg/engine's and internal/webhookserver's +# -benchmem benchmarks are what measure that, and they are in +# docs/operations/capacity.md. +OBSERVED_GATED = { + "webhookInitialSyncSeconds": "webhook-server cold start", + "webhookWorkingSetBytes": "webhook-server working set after the cold start", +} + +# Below this, the measurement is too small for a ratio to mean anything: a +# p50 that moved from 2 ms to 4 ms is a 2x regression by arithmetic and +# scheduler noise by every other reading. +NOISE_FLOOR_MS = 20.0 +NOISE_FLOOR_SECONDS = 1.0 +NOISE_FLOOR_BYTES = 32 * 1024 * 1024 + + +def human_bytes(value: float) -> str: + for unit in ("B", "KiB", "MiB", "GiB"): + if abs(value) < 1024 or unit == "GiB": + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} GiB" + + +def noise_floor(key: str) -> float: + if key.endswith("Bytes"): + return NOISE_FLOOR_BYTES + if key.endswith("Seconds"): + return NOISE_FLOOR_SECONDS + return NOISE_FLOOR_MS + + +def envelope_delta(previous: dict, current: dict) -> list[str]: + """Every input the two runs disagree on. + + Targets and instances alone are not enough: the same fleet driven at 16 + workers and at 60, or at a different QPS or strategy mix, is two + different measurements wearing the same field names. Comparing them + produces a confident answer to a question nobody asked. + """ + prev_env = previous.get("envelope") or { + "targets": str(previous.get("targets")), + "instances": str(previous.get("instances")), + } + cur_env = current.get("envelope") or { + "targets": str(current.get("targets")), + "instances": str(current.get("instances")), + } + out = [] + for key in sorted(set(prev_env) | set(cur_env)): + was, now = prev_env.get(key), cur_env.get(key) + if was != now: + out.append(f"{key} {was} -> {now}") + return out + + +def render(report: dict, previous: dict | None) -> list[str]: + lines: list[str] = [] + lines.append("## Scale run") + lines.append("") + lines.append( + f"**{report['targets']} CRDs x 3 versions**, " + f"{report['instances']} objects each " + f"({report['totalObjects']} objects total), " + f"recorded {report.get('recordedAt', 'unknown')}." + ) + lines.append("") + lines.append(f"Fleet creation took {report['createMs'] / 1000:.1f}s.") + lines.append("") + + lines.append("| Operation | n | errors | p50 | p99 | max | per-worker /s |") + lines.append("|---|---:|---:|---:|---:|---:|---:|") + for name in sorted(report.get("measurements", {})): + m = report["measurements"][name] + lines.append( + f"| `{name}` | {m['n']} | {m['errors']} | " + f"{m['p50Ms']:.1f} ms | {m['p99Ms']:.1f} ms | {m['maxMs']:.1f} ms | " + f"{m.get('throughputPerSecond', 0):.1f} |" + ) + lines.append("") + + observed = report.get("observed") or {} + if not observed: + lines.append( + "> :warning: **No cluster-side observations were collected.** Cold start and " + "working set are missing from this run, so neither is trended against the " + "previous one. See the job log for what `scale-observe.py` reported." + ) + lines.append("") + if observed: + lines.append("| Observed on the cluster | Value |") + lines.append("|---|---:|") + for key in sorted(observed): + value = observed[key] + shown = human_bytes(value) if key.endswith("Bytes") else f"{value:g}" + lines.append(f"| `{key}` | {shown} |") + lines.append("") + + if previous is None: + lines.append("_No previous run to compare against; this run becomes the baseline._") + lines.append("") + return lines + + +def compare(current: dict, previous: dict, threshold: float) -> tuple[list[str], list[str]]: + """Return (rendered rows, regression messages).""" + lines = ["| Measurement | previous | current | change |", "|---|---:|---:|---:|"] + regressions: list[str] = [] + + def row(label: str, key: str, was: float, now: float) -> None: + if key.endswith("Bytes"): + shown_was, shown_now = human_bytes(was), human_bytes(now) + elif key.endswith("Seconds"): + shown_was, shown_now = f"{was:.2f}s", f"{now:.2f}s" + elif key == "createMs": + # Fleet creation is minutes at any interesting envelope, and + # "120000.0 ms" is not a number anyone reads at a glance. + shown_was, shown_now = f"{was / 1000:.1f}s", f"{now / 1000:.1f}s" + else: + shown_was, shown_now = f"{was:.1f} ms", f"{now:.1f} ms" + + if was <= 0: + lines.append(f"| {label} | — | {shown_now} | new |") + return + ratio = now / was + marker = "" + if ratio > threshold and now > noise_floor(key): + marker = " **REGRESSED**" + regressions.append( + f"{label}: {shown_was} -> {shown_now} ({ratio:.2f}x, threshold {threshold:.2f}x)" + ) + lines.append(f"| {label} | {shown_was} | {shown_now} | {ratio:.2f}x{marker} |") + + for name in sorted(current.get("measurements", {})): + prev_m = previous.get("measurements", {}).get(name) + if not prev_m: + continue + cur_m = current["measurements"][name] + for field in LATENCY_FIELDS: + row(f"`{name}` {field[:-2]}", field, prev_m.get(field, 0.0), cur_m.get(field, 0.0)) + + row("fleet creation", "createMs", previous.get("createMs", 0.0), current.get("createMs", 0.0)) + + cur_obs, prev_obs = current.get("observed") or {}, previous.get("observed") or {} + for key, label in OBSERVED_GATED.items(): + if key in cur_obs and key in prev_obs: + row(label, key, prev_obs[key], cur_obs[key]) + + return lines, regressions + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--current", required=True) + ap.add_argument("--previous", default="") + ap.add_argument("--threshold", type=float, default=1.5, + help="fail when a measurement exceeds this multiple of the previous run") + ap.add_argument("--summary", default="", help="write the rendered markdown here as well as to stdout") + args = ap.parse_args() + + if args.threshold <= 1.0: + print("scale-report: --threshold must be greater than 1.0", file=sys.stderr) + return 2 + + with open(args.current, encoding="utf-8") as fh: + current = json.load(fh) + + previous = None + lines = [] + if args.previous: + try: + with open(args.previous, encoding="utf-8") as fh: + previous = json.load(fh) + except (OSError, json.JSONDecodeError) as err: + lines.append(f"_Previous run could not be read ({err}); treating this run as the baseline._") + lines.append("") + + failures: list[str] = [] + + # Errors are not a trend. Any is a failure, whatever the last run did. + for name, m in sorted(current.get("measurements", {}).items()): + if m.get("errors"): + failures.append(f"{name}: {m['errors']} of {m['n']} requests failed") + total_n = sum(m.get("n", 0) for m in current.get("measurements", {}).values()) + if total_n == 0: + failures.append("the run issued no requests at all, so it measured nothing") + + body = render(current, previous) + lines = body + lines + + if previous is not None: + if previous.get("schemaVersion") != current.get("schemaVersion"): + lines.append( + f"_Previous run used report schema v{previous.get('schemaVersion')} and this one " + f"v{current.get('schemaVersion')}; the fields may not mean the same thing, so no " + "comparison was made._" + ) + lines.append("") + elif envelope_delta(previous, current): + lines.append( + "_Previous run used a different envelope (" + + ", ".join(envelope_delta(previous, current)) + + "); no comparison was made. A run at a different parallelism, QPS or " + "strategy mix measures a different thing._" + ) + lines.append("") + else: + lines.append(f"### Against the previous run (threshold {args.threshold:.2f}x)") + lines.append("") + rows, regressions = compare(current, previous, args.threshold) + lines.extend(rows) + lines.append("") + failures.extend(regressions) + + if failures: + lines.append("### :x: This run failed") + lines.append("") + for f in failures: + lines.append(f"- {f}") + lines.append("") + else: + lines.append("### :white_check_mark: No regression") + lines.append("") + + text = "\n".join(lines) + "\n" + sys.stdout.write(text) + if args.summary: + with open(args.summary, "a", encoding="utf-8") as fh: + fh.write(text) + + if failures: + for f in failures: + print(f"FAIL: {f}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/internal/assign/assign.go b/internal/assign/assign.go index 752ea7d..b1aa7c6 100644 --- a/internal/assign/assign.go +++ b/internal/assign/assign.go @@ -23,14 +23,16 @@ limitations under the License. // // The resolver is generic over ConfigLike so it works identically for // XRDConversionConfig and CRDConversionConfig — assignment only ever -// depends on spec.webhookServerRef and each ConversionWebhookServer's -// spec.default, neither of which differs between the two config kinds. +// depends on spec.webhookServerRef, the target resource's name, and each +// ConversionWebhookServer's spec.default and spec.sharding, none of which +// differ between the two config kinds. package assign import ( "errors" "fmt" "sort" + "strings" teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" ) @@ -38,19 +40,33 @@ import ( // ConfigLike is the minimal surface ResolveAssignment needs from a // conversion config object. Both XRDConversionConfig and // CRDConversionConfig satisfy it: GetName comes from the embedded -// metav1.ObjectMeta, and WebhookServerRefField is a small accessor each -// type defines itself (see api/v1alpha1). +// metav1.ObjectMeta, and WebhookServerRefField and ShardKey are small +// accessors each type defines itself (see api/v1alpha1). type ConfigLike interface { GetName() string WebhookServerRefField() *teraskyv1alpha1.WebhookServerRef + // ShardKey is what automatic assignment hashes — the target + // resource's name. See XRDConversionConfig.ShardKey. + ShardKey() string } // ResolveAssignment returns the name of the ConversionWebhookServer that -// should serve cfg's conversions: the explicitly referenced instance if -// cfg's webhookServerRef is set, otherwise whichever instance in -// allServers is marked default. Exactly one default is required — zero or -// more than one is a misconfiguration reported as an error, never silently -// resolved by picking one. +// should serve cfg's conversions, in strict precedence order: +// +// 1. The explicitly referenced instance, if cfg's webhookServerRef is +// set. Deliberate pinning is the strongest statement in the system and +// nothing below can override it — tenant isolation depends on that. +// 2. The shard pool, if any instance opts into sharding, picked by +// weighted rendezvous hashing on the target resource's name. +// 3. The instance marked default. Exactly one default is required — zero +// or more than one is a misconfiguration reported as an error, never +// silently resolved by picking one. +// +// The pool takes precedence over the default rather than the other way +// round because a pool that the default instance sits outside of would +// serve nothing: admission requires the default to be a pool member +// whenever the pool is non-empty, precisely so that this ordering never +// moves work off the default by surprise. func ResolveAssignment[T ConfigLike](cfg T, allServers []teraskyv1alpha1.ConversionWebhookServer) (string, error) { if ref := cfg.WebhookServerRefField(); ref != nil && ref.Name != "" { name := ref.Name @@ -62,6 +78,10 @@ func ResolveAssignment[T ConfigLike](cfg T, allServers []teraskyv1alpha1.Convers return "", fmt.Errorf("webhookServerRef %q does not match any existing ConversionWebhookServer", name) } + if pool := ShardPool(allServers); len(pool) > 0 { + return PickShard(pool, cfg.ShardKey()), nil + } + var defaults []string for _, s := range allServers { if s.Spec.Default { @@ -118,3 +138,67 @@ func ConfigsAssignedTo[V any, PV interface { } return out } + +// ServingConfigLike adds what "is this instance still serving the target?" +// needs on top of ConfigLike: the webhook URL the operator last wrote into +// the target's spec.conversion. +type ServingConfigLike interface { + ConfigLike + AppliedWebhookURL() string +} + +// ServedBy reports whether serverName is currently on the hook for cfg's +// target — either because the resolver assigns it there, or because the +// target's live conversion webhook still points at that instance's +// Service. +// +// The second clause exists because assignment and reality diverge during a +// handover. When a target moves from A to B, the resolver stops assigning +// it to A immediately, but the target keeps naming A until the operator +// patches it — and A keeps serving it for exactly that reason (see +// webhookserver.Reconciler). Anything deciding "is it safe to delete this +// instance?" has to use this rather than assignment alone, or it will +// approve deleting the instance that is answering every ConversionReview +// for that target right now. +func ServedBy[T ServingConfigLike](cfg T, allServers []teraskyv1alpha1.ConversionWebhookServer, serverName string) bool { + if IsAssignedTo(cfg, allServers, serverName) { + return true + } + return TargetPointsAt(cfg.AppliedWebhookURL(), serverName) +} + +// TargetPointsAt reports whether an applied conversion webhook URL names +// serverName's Service — that is, whether that instance is the endpoint the +// apiserver is calling for this target right now. +// +// This, not the recorded assignment, is what tells a move from a settled +// state. status.assignedWebhookServer is written as soon as the resolver +// answers, which is *before* the target is repointed; reading it back on +// the next reconcile would say the move had already happened. The URL is +// only written after a successful apply, so it is the one field that +// reflects the target rather than the intention. +func TargetPointsAt(appliedURL, serverName string) bool { + if appliedURL == "" { + return false + } + // The URL the operator writes is + // https://..svc, so the service name is + // delimited on both sides and cannot be matched by accident. + return strings.HasPrefix(appliedURL, "https://"+teraskyv1alpha1.WebhookServerServiceName(serverName)+".") +} + +// ConfigsServedBy is ConfigsAssignedTo widened to ServedBy: every config +// this instance is on the hook for, assigned or merely still pointed at. +func ConfigsServedBy[V any, PV interface { + *V + ServingConfigLike +}](allConfigs []V, allServers []teraskyv1alpha1.ConversionWebhookServer, serverName string) []PV { + var out []PV + for i := range allConfigs { + cfg := PV(&allConfigs[i]) + if ServedBy(cfg, allServers, serverName) { + out = append(out, cfg) + } + } + return out +} diff --git a/internal/assign/shard.go b/internal/assign/shard.go new file mode 100644 index 0000000..c06369b --- /dev/null +++ b/internal/assign/shard.go @@ -0,0 +1,102 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package assign + +import ( + "hash/fnv" + "math" + "sort" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +// ShardPool is the ordered set of ConversionWebhookServer instances that +// unpinned conversion configs are distributed across — every instance with +// spec.sharding.enabled. Sorted by name so every party computing an +// assignment walks the same list in the same order. +// +// Empty means sharding is not in use anywhere, and assignment falls back +// to the single default instance exactly as it did before. +func ShardPool(allServers []teraskyv1alpha1.ConversionWebhookServer) []teraskyv1alpha1.ConversionWebhookServer { + var pool []teraskyv1alpha1.ConversionWebhookServer + for _, s := range allServers { + if s.Spec.ShardingEnabled() { + pool = append(pool, s) + } + } + sort.Slice(pool, func(i, j int) bool { return pool[i].Name < pool[j].Name }) + return pool +} + +// PickShard returns the pool member that owns key, by weighted rendezvous +// (highest-random-weight) hashing. An empty pool yields "". +// +// Rendezvous rather than a hash ring, for three reasons that matter here: +// +// - **Minimal, bounded movement.** Adding an instance moves only the keys +// that instance now wins — in expectation 1/(N+1) of them — and moves +// nothing between the instances that were already there. Removing one +// moves only its own keys. That is the optimal disruption property, and +// a ring only approximates it, with a quality that depends on how many +// virtual nodes you remembered to configure. +// - **Even distribution with no tuning.** A ring with too few virtual +// nodes distributes badly; there is no equivalent knob to get wrong +// here. +// - **No shared state.** The answer is a pure function of (key, pool), so +// the operator and every webhook-server replica compute it +// independently and always agree — which is the property the whole +// assign package exists to preserve. +// +// The cost is O(N) per lookup instead of O(log N). N is the number of +// webhook-server instances, which is single digits. +func PickShard(pool []teraskyv1alpha1.ConversionWebhookServer, key string) string { + best, bestScore := "", math.Inf(-1) + for _, s := range pool { + score := shardScore(s.Name, key, s.Spec.ShardWeight()) + // Ties broken by name so the result never depends on pool order. + // Reachable only on a hash collision, but "never depends on the + // order" has to hold unconditionally for every party to agree. + if score > bestScore || (score == bestScore && s.Name < best) { + best, bestScore = s.Name, score + } + } + return best +} + +// shardScore is the weighted rendezvous score for one (server, key) pair. +// +// The weighting is the standard one: with h uniform in (0, 1), +// -weight / ln(h) is distributed such that the probability of a given +// server holding the maximum is exactly its share of the total weight. +// Using ln(h) directly rather than a linear multiplier is what makes +// weights behave proportionally instead of merely monotonically. +func shardScore(serverName, key string, weight uint32) float64 { + h := fnv.New64a() + // Length-delimited rather than concatenated, so that ("ab", "c") and + // ("a", "bc") cannot hash to the same value and quietly correlate two + // unrelated assignments. + _, _ = h.Write([]byte(serverName)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(key)) + sum := h.Sum64() + + // Map to (0, 1): 2^53 divisor keeps the result exactly representable, + // and the +1 keeps it strictly positive so the logarithm is finite. + const mantissa = 1 << 53 + u := float64(sum%mantissa+1) / float64(mantissa+1) + return -float64(weight) / math.Log(u) +} diff --git a/internal/assign/shard_test.go b/internal/assign/shard_test.go new file mode 100644 index 0000000..61bd925 --- /dev/null +++ b/internal/assign/shard_test.go @@ -0,0 +1,279 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package assign + +import ( + "fmt" + "math" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +func shardedServer(name string, weight int32, isDefault bool) teraskyv1alpha1.ConversionWebhookServer { + s := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: name}} + s.Spec.Default = isDefault + s.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{} + if weight > 0 { + s.Spec.Sharding.Weight = &weight + } + return s +} + +func targets(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("xthings%d.example.org", i) + } + return out +} + +func assignAll(pool []teraskyv1alpha1.ConversionWebhookServer, keys []string) map[string]string { + out := make(map[string]string, len(keys)) + for _, k := range keys { + out[k] = PickShard(pool, k) + } + return out +} + +func TestPickShard_EmptyPool(t *testing.T) { + if got := PickShard(nil, "xfoos.example.org"); got != "" { + t.Fatalf("PickShard on an empty pool = %q, want the empty string", got) + } +} + +func TestPickShard_IsDeterministicAndOrderIndependent(t *testing.T) { + forward := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("a", 0, false), shardedServer("b", 0, false), shardedServer("c", 0, false)} + reversed := []teraskyv1alpha1.ConversionWebhookServer{forward[2], forward[1], forward[0]} + + // Every party — the operator and every webhook-server replica — + // computes this independently. If the answer depended on list order, + // two of them could disagree about who serves a target, which is the + // one thing the assign package exists to prevent. + for _, key := range targets(200) { + if a, b := PickShard(forward, key), PickShard(reversed, key); a != b { + t.Fatalf("PickShard(%q) = %q forwards but %q reversed", key, a, b) + } + } +} + +func TestPickShard_DistributesEvenly(t *testing.T) { + pool := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("a", 0, false), shardedServer("b", 0, false), shardedServer("c", 0, false), shardedServer("d", 0, false)} + keys := targets(4000) + counts := map[string]int{} + for _, k := range keys { + counts[PickShard(pool, k)]++ + } + want := len(keys) / len(pool) + for _, s := range pool { + got := counts[s.Name] + // ±20% of the fair share. Wide enough not to be flaky on a + // different hash, tight enough to catch a scoring bug that + // collapses the distribution onto one or two members. + if math.Abs(float64(got-want)) > float64(want)*0.2 { + t.Errorf("server %q got %d of %d keys, want roughly %d (±20%%)", s.Name, got, len(keys), want) + } + } +} + +// The whole reason for rendezvous hashing: adding an instance must move +// only the keys that instance wins, and must not reshuffle the rest. +func TestPickShard_AddingAServerMovesABoundedFraction(t *testing.T) { + before := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("a", 0, false), shardedServer("b", 0, false), shardedServer("c", 0, false)} + after := append(append([]teraskyv1alpha1.ConversionWebhookServer{}, before...), shardedServer("d", 0, false)) + + keys := targets(4000) + was, now := assignAll(before, keys), assignAll(after, keys) + + moved, movedToNew := 0, 0 + for _, k := range keys { + if was[k] == now[k] { + continue + } + moved++ + if now[k] == "d" { + movedToNew++ + } + } + // Every move must be onto the new server. A key moving from a to b + // when neither changed is the failure mode a hash ring with too few + // virtual nodes exhibits, and it is invisible in the aggregate count. + if moved != movedToNew { + t.Errorf("%d keys moved but only %d moved onto the new server; the rest were reshuffled between servers that did not change", moved, movedToNew) + } + // Expected share is 1/4 = 25%. + fraction := float64(moved) / float64(len(keys)) + if fraction < 0.2 || fraction > 0.3 { + t.Errorf("adding a fourth server moved %.1f%% of keys, want roughly 25%%", fraction*100) + } +} + +func TestPickShard_RemovingAServerMovesOnlyItsOwn(t *testing.T) { + before := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("a", 0, false), shardedServer("b", 0, false), shardedServer("c", 0, false)} + after := before[:2] + + keys := targets(3000) + was, now := assignAll(before, keys), assignAll(after, keys) + for _, k := range keys { + if was[k] != "c" && was[k] != now[k] { + t.Fatalf("key %q moved from %q to %q although neither server was removed", k, was[k], now[k]) + } + } +} + +func TestPickShard_WeightBiasesTheShare(t *testing.T) { + pool := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("heavy", 3, false), shardedServer("light", 1, false)} + keys := targets(4000) + counts := map[string]int{} + for _, k := range keys { + counts[PickShard(pool, k)]++ + } + ratio := float64(counts["heavy"]) / float64(counts["light"]) + if ratio < 2.5 || ratio > 3.5 { + t.Errorf("weight 3 against weight 1 produced a %.2f:1 split (%d vs %d), want roughly 3:1", + ratio, counts["heavy"], counts["light"]) + } +} + +func TestShardPool_OnlyOptedInMembers(t *testing.T) { + plain := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "plain"}} + off := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "off"}} + disabled := false + off.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{Enabled: &disabled} + + pool := ShardPool([]teraskyv1alpha1.ConversionWebhookServer{plain, off, shardedServer("in", 0, true)}) + if len(pool) != 1 || pool[0].Name != "in" { + t.Fatalf("pool = %v, want only the instance that opted in", names(pool)) + } +} + +func names(servers []teraskyv1alpha1.ConversionWebhookServer) []string { + out := make([]string, len(servers)) + for i, s := range servers { + out[i] = s.Name + } + return out +} + +// Precedence: an explicit ref beats the pool, the pool beats the default, +// and the default still answers when no pool exists. +func TestResolveAssignment_ShardPrecedence(t *testing.T) { + cfg := &teraskyv1alpha1.XRDConversionConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg"}} + cfg.Spec.TargetXRD.Name = "xfoos.example.org" + + plainDefault := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "default"}} + plainDefault.Spec.Default = true + pinned := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}} + + // No pool: the default answers, exactly as before sharding existed. + got, err := ResolveAssignment(cfg, []teraskyv1alpha1.ConversionWebhookServer{plainDefault, pinned}) + if err != nil || got != "default" { + t.Fatalf("without a pool: got %q, err %v; want %q", got, err, "default") + } + + // A pool exists: it answers instead of the default. + pool := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("default", 0, true), shardedServer("shard-b", 0, false)} + got, err = ResolveAssignment(cfg, pool) + if err != nil { + t.Fatalf("with a pool: %v", err) + } + if got != PickShard(ShardPool(pool), cfg.Spec.TargetXRD.Name) { + t.Fatalf("with a pool: got %q, want the rendezvous winner", got) + } + + // An explicit ref wins over everything. This is what tenant isolation + // is built on; sharding must not be able to move a pinned config. + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "tenant-a"} + withPinned := append(append([]teraskyv1alpha1.ConversionWebhookServer{}, pool...), pinned) + got, err = ResolveAssignment(cfg, withPinned) + if err != nil || got != "tenant-a" { + t.Fatalf("with an explicit ref: got %q, err %v; want %q", got, err, "tenant-a") + } +} + +// Sharding removes the need for a default instance entirely: a pool is a +// complete answer for an unpinned config, where before an absent default +// was a hard error. +func TestResolveAssignment_PoolWithoutADefault(t *testing.T) { + cfg := &teraskyv1alpha1.CRDConversionConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg"}} + cfg.Spec.TargetCRD.Name = "foos.example.org" + + pool := []teraskyv1alpha1.ConversionWebhookServer{shardedServer("a", 0, false), shardedServer("b", 0, false)} + got, err := ResolveAssignment(cfg, pool) + if err != nil { + t.Fatalf("unexpected error with a pool and no default: %v", err) + } + if got != "a" && got != "b" { + t.Fatalf("got %q, want a pool member", got) + } +} + +// Deletion safety cannot be judged by assignment alone. Mid-handover a +// config resolves to its destination while its target still points at the +// source — and the source is still answering every ConversionReview for +// it, because that is what makes waiting for the handover safe. An +// instance in that state must not be deletable. +func TestServedBy_CoversTheMidHandoverSource(t *testing.T) { + srvA := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv-a"}} + srvB := teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv-b"}} + servers := []teraskyv1alpha1.ConversionWebhookServer{srvA, srvB} + + cfg := &teraskyv1alpha1.XRDConversionConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg"}} + cfg.Spec.TargetXRD.Name = "xfoos.example.org" + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + cfg.Status.WebhookURL = "https://srv-a-webhook-server.dco-system.svc/convert/xfoos.example.org" + + if !ServedBy(cfg, servers, "srv-a") { + t.Error("srv-a is the instance the target still names, so it is still serving it") + } + if !ServedBy(cfg, servers, "srv-b") { + t.Error("srv-b is the assigned destination, so it is on the hook too") + } + if IsAssignedTo(cfg, servers, "srv-a") { + t.Error("fixture is wrong: srv-a should no longer be the assigned server") + } +} + +// The prefix match must not be satisfied by a name that merely starts the +// same way, or deleting "srv" would be blocked by a config pointed at +// "srv-a". +func TestServedBy_DoesNotMatchAPrefixOfAnotherService(t *testing.T) { + servers := []teraskyv1alpha1.ConversionWebhookServer{ + {ObjectMeta: metav1.ObjectMeta{Name: "srv"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "srv-a"}}, + } + cfg := &teraskyv1alpha1.CRDConversionConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg"}} + cfg.Spec.TargetCRD.Name = "foos.example.org" + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-a"} + cfg.Status.WebhookURL = "https://srv-a-webhook-server.dco-system.svc/convert/foos.example.org" + + if ServedBy(cfg, servers, "srv") { + t.Error(`a target pointed at "srv-a" must not read as pointing at "srv"`) + } +} + +func TestServedBy_NeverAppliedIsNotServed(t *testing.T) { + servers := []teraskyv1alpha1.ConversionWebhookServer{{ObjectMeta: metav1.ObjectMeta{Name: "srv-a"}}} + cfg := &teraskyv1alpha1.XRDConversionConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg"}} + cfg.Spec.TargetXRD.Name = "xfoos.example.org" + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "other"} + if ServedBy(cfg, servers, "srv-a") { + t.Error("a config that has never been applied points at nobody") + } +} diff --git a/internal/controller/cacheopts.go b/internal/controller/cacheopts.go index e35a6ae..65375bc 100644 --- a/internal/controller/cacheopts.go +++ b/internal/controller/cacheopts.go @@ -19,6 +19,7 @@ package controller import ( appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/labels" @@ -70,6 +71,12 @@ func ManagerCacheOptions() cache.Options { &corev1.Service{}: owned, &autoscalingv2.HorizontalPodAutoscaler{}: owned, &policyv1.PodDisruptionBudget{}: owned, + // The served-target Leases webhook-server replicas publish. + // Scoping these is not optional: an unscoped Lease informer + // holds one object per node from kube-node-lease, plus every + // leader election in the cluster, to read a handful this + // operator's own pods wrote. + &coordinationv1.Lease{}: owned, }, } } @@ -101,7 +108,7 @@ func ManagerClientOptions() client.Options { // memory without reading the source. func CacheScopeDescription() string { return "Secrets: uncached (read-through to the API server); " + - "Deployments/Services/HorizontalPodAutoscalers/PodDisruptionBudgets: " + + "Deployments/Services/HorizontalPodAutoscalers/PodDisruptionBudgets/Leases: " + ManagedByLabel + "=" + ManagedByValue + "; " + "XRDConversionConfigs/CRDConversionConfigs/ConversionWebhookServers/CRDs/XRDs: cluster-wide" } diff --git a/internal/controller/cacheopts_test.go b/internal/controller/cacheopts_test.go index 5319d6a..9efeb84 100644 --- a/internal/controller/cacheopts_test.go +++ b/internal/controller/cacheopts_test.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/labels" @@ -52,13 +53,18 @@ func TestManagerCacheOptions_ScopesEveryOwnedType(t *testing.T) { opts := ManagerCacheOptions() // Exactly the types ConversionWebhookServerReconciler.SetupWithManager - // passes to Owns(). Adding an Owns() without adding it here silently - // reintroduces a cluster-wide informer. + // passes to Owns() or Watches(), other than the operator's own CRDs. + // Adding one without adding it here silently reintroduces a + // cluster-wide informer. want := []string{ fmt.Sprintf("%T", &appsv1.Deployment{}), fmt.Sprintf("%T", &corev1.Service{}), fmt.Sprintf("%T", &autoscalingv2.HorizontalPodAutoscaler{}), fmt.Sprintf("%T", &policyv1.PodDisruptionBudget{}), + // Unscoped, this one is worse than the others: it would hold a + // Lease per node from kube-node-lease plus every leader election + // in the cluster. + fmt.Sprintf("%T", &coordinationv1.Lease{}), } got := map[string]bool{} for obj, by := range opts.ByObject { diff --git a/internal/controller/concurrency.go b/internal/controller/concurrency.go new file mode 100644 index 0000000..8ac3d80 --- /dev/null +++ b/internal/controller/concurrency.go @@ -0,0 +1,43 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ctrlcontroller "sigs.k8s.io/controller-runtime/pkg/controller" + +// DefaultMaxConcurrentReconciles matches controller-runtime's own default. +// It is named here rather than left implicit because the shipped dashboard +// now plots workqueue depth, which invites the question "can I turn this +// up?" — and a lever whose default nobody can state is not much of a +// lever. +const DefaultMaxConcurrentReconciles = 1 + +// controllerOptions builds the per-controller options every controller in +// this package is wired with. A zero or negative value means "leave +// controller-runtime's default alone", so an unset field on a reconciler +// struct behaves exactly as it did before the field existed. +// +// Raising it is safe for these controllers specifically: each reconcile is +// keyed by one config or one server, controller-runtime guarantees a given +// key is never reconciled by two workers at once, and nothing in the +// reconcile path shares mutable state across keys. What it costs is +// apiserver QPS, which is the reason it is not raised by default. +func controllerOptions(maxConcurrent int) ctrlcontroller.Options { + if maxConcurrent <= 0 { + return ctrlcontroller.Options{} + } + return ctrlcontroller.Options{MaxConcurrentReconciles: maxConcurrent} +} diff --git a/internal/controller/conversionwebhookserver_controller.go b/internal/controller/conversionwebhookserver_controller.go index 3551492..9751c90 100755 --- a/internal/controller/conversionwebhookserver_controller.go +++ b/internal/controller/conversionwebhookserver_controller.go @@ -21,10 +21,12 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "time" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -40,13 +42,17 @@ import ( applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1" applypolicyv1 "k8s.io/client-go/applyconfigurations/policy/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" "github.com/terasky-oss/declarative-conversion-operator/internal/assign" + "github.com/terasky-oss/declarative-conversion-operator/internal/servedtargets" "github.com/terasky-oss/declarative-conversion-operator/internal/watchmap" ) @@ -86,6 +92,11 @@ type ConversionWebhookServerReconciler struct { // for a webhook-server pod exactly the same way it is for the manager. EnableXRDSupport bool EnableCRDSupport bool + + // MaxConcurrentReconciles bounds how many objects this controller + // reconciles at once. Zero leaves controller-runtime's own default + // (1) in place. See internal/controller/concurrency.go. + MaxConcurrentReconciles int } // +kubebuilder:rbac:groups=terasky.com,resources=conversionwebhookservers,verbs=get;list;watch;create;update;patch;delete @@ -432,9 +443,73 @@ func (r *ConversionWebhookServerReconciler) reconcileDeployment(ctx context.Cont WithResources(applycorev1.ResourceRequirements(). WithRequests(server.Spec.Resources.Requests). WithLimits(server.Spec.Resources.Limits)) + + // The startupProbe is the cold-start budget: period x failureThreshold + // is how long a replica may take to compile every assigned plan before + // the kubelet restarts it. + // + // It polls /readyz, not /healthz, and the distinction is the whole + // point. The plain endpoint — /healthz, /readyz, /metrics — comes up + // before the registry sync (see cmd/webhook-server/main.go), so + // /healthz answers within milliseconds of process start and a + // startupProbe pointed at it would succeed immediately and bound + // nothing. /readyz is false until InitialSync completes, so probing it + // is what turns the threshold into a real deadline on the sync. + // + // That deadline matters because the sync retries infrastructure + // failures without a limit. Without it, a replica wedged mid-sync + // stays liveness-healthy and not-ready forever: out of the Service, + // never restarted, and visible only as a gap in readyReplicas. + // + // While the startupProbe is in flight the kubelet runs neither of the + // other two probes, so a slow sync is not also fighting the liveness + // probe's own 3 x 10 s. + if server.Spec.StartupProbeEnabled() { + period, threshold := server.Spec.StartupProbeTiming() + container = container.WithStartupProbe(applycorev1.Probe(). + WithHTTPGet(applycorev1.HTTPGetAction().WithPath("/readyz").WithPort(intstr.FromInt32(webhookServerMetricsPort)).WithScheme(corev1.URISchemeHTTP)). + WithPeriodSeconds(period).WithFailureThreshold(threshold)) + } if pullPolicy != "" { container = container.WithImagePullPolicy(pullPolicy) } + // The replica's own identity, via the downward API. It uses this to + // publish the set of targets it can serve into a Lease of its own — + // the signal that lets a target be moved onto this instance without a + // window in which nothing serves it. POD_UID is what makes that Lease + // a child of the pod, so it is collected with it. + container = container.WithEnv( + applycorev1.EnvVar().WithName("POD_NAME").WithValueFrom( + applycorev1.EnvVarSource().WithFieldRef( + applycorev1.ObjectFieldSelector().WithFieldPath("metadata.name"))), + applycorev1.EnvVar().WithName("POD_NAMESPACE").WithValueFrom( + applycorev1.EnvVarSource().WithFieldRef( + applycorev1.ObjectFieldSelector().WithFieldPath("metadata.namespace"))), + applycorev1.EnvVar().WithName("POD_UID").WithValueFrom( + applycorev1.EnvVarSource().WithFieldRef( + applycorev1.ObjectFieldSelector().WithFieldPath("metadata.uid"))), + ) + + // GOMEMLIMIT, derived from the container's own memory limit. + // + // The steady registry footprint is small — about 18 KiB per target for + // a 50-leaf two-version schema — but compiling those plans churns + // roughly twenty times what it retains, and with the default GOGC the + // heap is allowed to grow to twice the live set before a collection. + // Measured, a thousand-target cold start peaks around 142 MiB of heap + // against 18 MiB of steady registry. A memory *limit* is enforced by + // the kernel, which does not wait for the GC; GOMEMLIMIT is what makes + // the GC aware of the same number. With it at 64 MiB the same run + // peaks at 61 MiB instead, taking longer to do it — which is the + // trade a limit is asking for. + // + // Only set when a memory limit exists (there is nothing to derive it + // from otherwise) and only when the operator has not set it itself. + if memLimitBytes, ok := webhookServerMemoryLimitBytes(server); ok && !hasEnvVar(server.Spec.ExtraEnv, goMemLimitEnv) { + container = container.WithEnv(applycorev1.EnvVar(). + WithName(goMemLimitEnv). + WithValue(strconv.FormatInt(memLimitBytes, 10))) + } for _, e := range server.Spec.ExtraEnv { ec, err := viaJSON[applycorev1.EnvVarApplyConfiguration](e) if err != nil { @@ -645,6 +720,21 @@ func (r *ConversionWebhookServerReconciler) updateStatus(ctx context.Context, se sort.Slice(refs, func(i, j int) bool { return refs[i].Name < refs[j].Name }) server.Status.AssignedConfigs = refs + // AssignedConfigs is desired state; ServedTargets is reported state. + // Publishing both is the point — the gap between them is exactly the + // window in which a target has been given to this instance but the + // instance cannot serve it yet, which used to be invisible. + served, reporting, truncated, err := readServedTargets(ctx, r.Client, server, r.DefaultNamespace) + if err != nil { + return err + } + if truncated { + // Publishing a partial intersection would read as a complete one. + served = nil + } + server.Status.ServedTargets = served + server.Status.ReportingReplicas = reporting + return nil } @@ -714,8 +804,13 @@ func (r *ConversionWebhookServerReconciler) reconcileDelete(ctx context.Context, if err := r.List(ctx, &allServers); err != nil { return ctrl.Result{}, err } - dependentXRD := assign.ConfigsAssignedTo(xrdConfigs.Items, allServers.Items, server.Name) - dependentCRD := assign.ConfigsAssignedTo(crdConfigs.Items, allServers.Items, server.Name) + // ServedBy, not IsAssignedTo. During a handover the resolver has + // already moved a config to another instance while this one is + // still the endpoint the target names and still answering every + // ConversionReview for it. Judging by assignment alone would let + // that instance be deleted out from under a live target. + dependentXRD := assign.ConfigsServedBy(xrdConfigs.Items, allServers.Items, server.Name) + dependentCRD := assign.ConfigsServedBy(crdConfigs.Items, allServers.Items, server.Name) if len(dependentXRD)+len(dependentCRD) > 0 { names := make([]string, 0, len(dependentXRD)+len(dependentCRD)) for _, c := range dependentXRD { @@ -732,7 +827,7 @@ func (r *ConversionWebhookServerReconciler) reconcileDelete(ctx context.Context, } meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{ Type: teraskyv1alpha1.CWSConditionDeletionBlocked, Status: metav1.ConditionTrue, Reason: "ConfigsStillAssigned", - Message: fmt.Sprintf("%d config(s) still resolve to this instance%s: %v. Reassign them or add annotation %q=\"true\" to force.", len(dependentXRD)+len(dependentCRD), suffix, names, teraskyv1alpha1.AllowForceDeleteAnnotation), + Message: fmt.Sprintf("%d config(s) still resolve to this instance, or still have their target pointed at it%s: %v. Reassign them or add annotation %q=\"true\" to force.", len(dependentXRD)+len(dependentCRD), suffix, names, teraskyv1alpha1.AllowForceDeleteAnnotation), }) if err := r.Status().Patch(ctx, server, client.MergeFrom(orig)); err != nil { return ctrl.Result{}, err @@ -759,10 +854,49 @@ func (r *ConversionWebhookServerReconciler) SetupWithManager(mgr ctrl.Manager) e Owns(&autoscalingv2.HorizontalPodAutoscaler{}). Owns(&policyv1.PodDisruptionBudget{}). Watches(&teraskyv1alpha1.XRDConversionConfig{}, handler.EnqueueRequestsFromMapFunc(enqueueAllServers(r.Client))). + // Replica Leases feed status.servedTargets. The predicate is + // load-bearing: every replica renews its Lease on a 30-second + // heartbeat, and reconciling a ConversionWebhookServer — which + // server-side-applies a Deployment, Service, HPA and PDB — that + // often, per replica, for a renewTime that changes nothing this + // controller reads, would be pure churn. Only a change to the + // reported target set is worth a reconcile. + Watches(&coordinationv1.Lease{}, + handler.EnqueueRequestsFromMapFunc(enqueueServerForLease), + builder.WithPredicates(servedTargetsChanged())). + WithOptions(controllerOptions(r.MaxConcurrentReconciles)). Named("conversionwebhookserver"). Complete(r) } +// enqueueServerForLease maps a replica's served-target Lease back to the +// instance it belongs to, which the Lease's own label names. +func enqueueServerForLease(_ context.Context, obj client.Object) []reconcile.Request { + name := obj.GetLabels()[servedtargets.WebhookServerLabel] + if name == "" { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: name}}} +} + +// servedTargetsChanged passes a Lease event through only when it could +// change this controller's answer: any create or delete, and an update +// that alters the reported target set. A renewTime-only update is the +// heartbeat and is deliberately dropped — staleness is evaluated when the +// aggregate is next read, not on a timer here. +func servedTargetsChanged() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return true + } + old, updated := e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations() + return old[servedtargets.TargetsAnnotation] != updated[servedtargets.TargetsAnnotation] || + old[servedtargets.TruncatedAnnotation] != updated[servedtargets.TruncatedAnnotation] + }, + } +} + func enqueueAllServers(c client.Client) func(ctx context.Context, obj client.Object) []reconcile.Request { return func(ctx context.Context, _ client.Object) []reconcile.Request { var list teraskyv1alpha1.ConversionWebhookServerList @@ -785,6 +919,47 @@ func enqueueAllServers(c client.Client) func(ctx context.Context, obj client.Obj // existed, or one that simply omits it, has to get the same safe behaviour // as one that spells it out. +// goMemLimitEnv and goMemLimitFraction: the Go runtime's soft memory +// limit, set to a fraction of the container's hard one. The headroom is +// for everything the Go heap is not — goroutine stacks, the runtime's own +// bookkeeping, and whatever the allocator has not returned to the OS yet. +// 90% is the conventional figure and leaves ~25 MiB at the chart's default +// 256 MiB limit. +const ( + goMemLimitEnv = "GOMEMLIMIT" + goMemLimitFraction = 90 +) + +// webhookServerMemoryLimitBytes returns the GOMEMLIMIT value to set from +// the instance's own memory limit, and whether there is one to derive it +// from. A limit too small to leave any headroom yields no value rather +// than a nonsensically tiny one — the pod has bigger problems than its GC +// pacing at that point. +func webhookServerMemoryLimitBytes(server *teraskyv1alpha1.ConversionWebhookServer) (int64, bool) { + limit, ok := server.Spec.Resources.Limits[corev1.ResourceMemory] + if !ok { + return 0, false + } + bytes := limit.Value() + if bytes <= 0 { + return 0, false + } + derived := bytes / 100 * goMemLimitFraction + if derived <= 0 { + return 0, false + } + return derived, true +} + +func hasEnvVar(env []corev1.EnvVar, name string) bool { + for _, e := range env { + if e.Name == name { + return true + } + } + return false +} + func rolloutPreStopSeconds(server *teraskyv1alpha1.ConversionWebhookServer) int32 { if server.Spec.Rollout == nil || server.Spec.Rollout.PreStopSleepSeconds == nil { return teraskyv1alpha1.DefaultPreStopSleepSeconds diff --git a/internal/controller/crdconversionconfig_controller.go b/internal/controller/crdconversionconfig_controller.go index a1b4c7e..6b0485a 100644 --- a/internal/controller/crdconversionconfig_controller.go +++ b/internal/controller/crdconversionconfig_controller.go @@ -64,6 +64,11 @@ type CRDConversionConfigReconciler struct { // DefaultServerNamespace is used for instances that don't set // spec.namespace — normally the operator's own install namespace. DefaultServerNamespace string + + // MaxConcurrentReconciles bounds how many objects this controller + // reconciles at once. Zero leaves controller-runtime's own default + // (1) in place. See internal/controller/concurrency.go. + MaxConcurrentReconciles int } // +kubebuilder:rbac:groups=terasky.com,resources=crdconversionconfigs,verbs=get;list;watch;create;update;patch;delete @@ -209,6 +214,13 @@ func (r *CRDConversionConfigReconciler) reconcileNormal(ctx context.Context, cfg r.setInvalid(cfg, wasApplied, fmt.Sprintf("could not resolve a ConversionWebhookServer: %v", err)) return ctrl.Result{}, r.patchStatus(ctx, orig, cfg) } + // Is this reconcile a move? Judged from the URL last applied to the + // target, not from status.assignedWebhookServer — that field is + // written as soon as the resolver answers, which is before the target + // is repointed, so reading it back on the next reconcile would say the + // move had already happened and the gate would open after one pass. + movingServers := wasApplied && orig.Status.WebhookURL != "" && + !assign.TargetPointsAt(orig.Status.WebhookURL, serverName) cfg.Status.AssignedWebhookServer = serverName // Step 5: CRD health gate. @@ -241,6 +253,33 @@ func (r *CRDConversionConfigReconciler) reconcileNormal(ctx context.Context, cfg Type: teraskyv1alpha1.ConditionWebhookServerReady, Status: metav1.ConditionTrue, Reason: "ServerReady", Message: fmt.Sprintf("ConversionWebhookServer %q is Available", serverName), }) + // Step 6b: the handover gate. See the XRD controller's copy of this + // comment for the full reasoning; in short, repointing a target at a + // new instance before that instance can serve it fails every read and + // write of the resource for the duration, and waiting is safe because + // the old instance keeps serving a target that still names it. + if movingServers { + verdict, err := checkHandover(ctx, r.Client, &server, r.DefaultServerNamespace, cfg.Spec.TargetCRD.Name, orig.Status.Conditions, time.Now()) + if err != nil { + return ctrl.Result{}, err + } + meta.SetStatusCondition(&cfg.Status.Conditions, metav1.Condition{ + Type: teraskyv1alpha1.ConditionHandoverReady, Status: boolStatus(verdict.OK), + Reason: verdict.Reason, Message: verdict.Message, + }) + if !verdict.OK { + setPhasePendingOrStale(&cfg.Status.Conditions, &cfg.Status.Phase, wasApplied, verdict.Reason, verdict.Message) + cfg.Status.Message = verdict.Message + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.patchStatus(ctx, orig, cfg) + } + } + // Deliberately not removed when this reconcile is not a move. The + // condition is the verdict on the last handover, and it stays true + // afterwards — "the instance now serving this target was verified able + // to serve it before it was pointed here" does not stop being true. + // Leaving it is what makes a HandoverUnverified stick around long + // enough for somebody to notice that their replicas cannot publish. + // Step 7: only now, patch the CRD. caBundle, err := r.readCABundle(ctx, &server) if err != nil { @@ -459,6 +498,7 @@ func (r *CRDConversionConfigReconciler) SetupWithManager(mgr ctrl.Manager) error For(&teraskyv1alpha1.CRDConversionConfig{}). Watches(&extv1.CustomResourceDefinition{}, handler.EnqueueRequestsFromMapFunc(r.mapCRDToConfigs)). Watches(&teraskyv1alpha1.ConversionWebhookServer{}, enqueue.PacedMapFuncs(r.mapServerToAssignedConfigs, r.mapServerTransitionToAssignedConfigs, enqueue.CWSConfigEnqueueQPS)). + WithOptions(controllerOptions(r.MaxConcurrentReconciles)). Named("crdconversionconfig"). Complete(r) } diff --git a/internal/controller/cws_fanout_test.go b/internal/controller/cws_fanout_test.go index 0fe8471..1593060 100644 --- a/internal/controller/cws_fanout_test.go +++ b/internal/controller/cws_fanout_test.go @@ -136,3 +136,83 @@ func TestMapServerToAssignedCRDConfigs_FiltersAssignment(t *testing.T) { t.Fatalf("expected only cfg-a for srv-a, got %#v", reqs) } } + +// Adding a sharded instance moves a share of the unpinned configs onto it. +// The fan-out on that create must enqueue exactly those, and reach them +// through the same paced handler as every other CWS-driven fan-out — +// rebalancing a fleet is the largest burst this watch ever produces, so it +// is the one that most needs the pacing. +func TestMapServerToAssignedXRDConfigs_ShardedFanoutIsBoundedAndPaced(t *testing.T) { + poolA := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv-a"}} + poolA.Spec.Default = true + poolA.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{} + poolB := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv-b"}} + poolB.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{} + + const total = 400 + objs := []runtime.Object{poolA, poolB} + for i := 0; i < total; i++ { + objs = append(objs, renameRuleXRDConfig(fmt.Sprintf("cfg-%03d", i), fmt.Sprintf("x%d.example.org", i))) + } + // Pinned configs are never rebalanced, whatever the pool does. + pinned := renameRuleXRDConfig("pinned", "pinned.example.org") + pinned.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-a"} + objs = append(objs, pinned) + + c := newFakeClient(objs...).Build() + reqs, err := mapServerToAssignedXRDConfigs(context.Background(), c, poolB) + if err != nil { + t.Fatalf("mapServerToAssignedXRDConfigs: %v", err) + } + + // Roughly half of the unpinned configs, and none of the pinned one. + if len(reqs) == 0 || len(reqs) == total+1 { + t.Fatalf("srv-b was enqueued %d of %d configs; a shard should take a share, not none or all", len(reqs), total+1) + } + if len(reqs) < total/4 || len(reqs) > (3*total)/4 { + t.Errorf("srv-b was enqueued %d of %d unpinned configs, want roughly half", len(reqs), total) + } + for _, r := range reqs { + if r.Name == "pinned" { + t.Fatal("a config pinned to srv-a was enqueued as belonging to srv-b") + } + } + + if spread := enqueue.FanoutSpread(len(reqs), enqueue.CWSConfigEnqueueQPS); spread == 0 { + t.Fatalf("a rebalance of %d configs must be paced, not dumped into the workqueue at once", len(reqs)) + } +} + +// Removing a sharded instance has to re-reconcile the configs it used to +// hold, computed from the pre-delete view — after the object is gone there +// is nothing left to derive them from. +func TestMapServerTransition_ShardRemoval_EnqueuesItsFormerConfigs(t *testing.T) { + poolA := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv-a"}} + poolA.Spec.Default = true + poolA.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{} + poolB := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv-b"}} + poolB.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{} + + const total = 200 + objs := []runtime.Object{poolA} + var names []string + for i := 0; i < total; i++ { + name := fmt.Sprintf("cfg-%03d", i) + names = append(names, name) + objs = append(objs, renameRuleXRDConfig(name, fmt.Sprintf("x%d.example.org", i))) + } + + // The live list no longer contains srv-b; only the deleted object's own + // view can say what it used to serve. + c := newFakeClient(objs...).Build() + reqs, err := mapXRDConfigsForServerViews(context.Background(), c, "srv-b", poolB) + if err != nil { + t.Fatalf("mapXRDConfigsForServerViews: %v", err) + } + if len(reqs) == 0 { + t.Fatal("deleting a pool member enqueued nothing; the configs it held would keep pointing at a Service that no longer exists") + } + if len(reqs) >= len(names) { + t.Fatalf("enqueued %d of %d configs for a removed shard, want only its own share", len(reqs), len(names)) + } +} diff --git a/internal/controller/gomemlimit_test.go b/internal/controller/gomemlimit_test.go new file mode 100644 index 0000000..79f98a1 --- /dev/null +++ b/internal/controller/gomemlimit_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +func serverWithResources(limits corev1.ResourceList, extraEnv ...corev1.EnvVar) *teraskyv1alpha1.ConversionWebhookServer { + return &teraskyv1alpha1.ConversionWebhookServer{ + ObjectMeta: metav1.ObjectMeta{Name: "srv"}, + Spec: teraskyv1alpha1.ConversionWebhookServerSpec{ + Namespace: "operator-ns", + Certificate: teraskyv1alpha1.CertificateSpec{IssuerRef: teraskyv1alpha1.CertificateIssuerRef{Name: "ca-issuer"}}, + Resources: corev1.ResourceRequirements{Limits: limits}, + ExtraEnv: extraEnv, + }, + } +} + +func envValue(env []corev1.EnvVar, name string) (string, bool) { + for _, e := range env { + if e.Name == name { + return e.Value, true + } + } + return "", false +} + +// The chart's default limit is 256Mi. Without GOMEMLIMIT the Go heap will +// happily grow past it during a cold start — measured at ~142 MiB of +// transient heap for a thousand targets — and the kernel, not the GC, is +// what notices. +func TestGOMEMLIMIT_DerivedFromTheMemoryLimit(t *testing.T) { + dep := reconcileToDeployment(t, serverWithResources(corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("256Mi"), + })) + got, ok := envValue(dep.Spec.Template.Spec.Containers[0].Env, "GOMEMLIMIT") + if !ok { + t.Fatal("no GOMEMLIMIT: the GC then has no idea the container has a limit at all") + } + // 256Mi = 268435456; 90% of it, computed without overflowing. + if want := "241591860"; got != want { + t.Errorf("GOMEMLIMIT = %s, want %s (90%% of 256Mi)", got, want) + } +} + +func TestGOMEMLIMIT_AbsentWithoutAMemoryLimit(t *testing.T) { + dep := reconcileToDeployment(t, serverWithResources(nil)) + if got, ok := envValue(dep.Spec.Template.Spec.Containers[0].Env, "GOMEMLIMIT"); ok { + t.Fatalf("GOMEMLIMIT = %s, want none: there is no limit to derive it from", got) + } +} + +// An operator who sets it deliberately has a reason; deriving a second +// value would leave two entries with the same name in the container spec. +func TestGOMEMLIMIT_ExplicitValueWins(t *testing.T) { + dep := reconcileToDeployment(t, serverWithResources( + corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi")}, + corev1.EnvVar{Name: "GOMEMLIMIT", Value: "100MiB"}, + )) + env := dep.Spec.Template.Spec.Containers[0].Env + count := 0 + for _, e := range env { + if e.Name == "GOMEMLIMIT" { + count++ + } + } + if count != 1 { + t.Fatalf("GOMEMLIMIT appears %d times in %+v, want exactly once", count, env) + } + if got, _ := envValue(env, "GOMEMLIMIT"); got != "100MiB" { + t.Errorf("GOMEMLIMIT = %s, want the operator's own 100MiB", got) + } +} diff --git a/internal/controller/handover_test.go b/internal/controller/handover_test.go new file mode 100644 index 0000000..74f656c --- /dev/null +++ b/internal/controller/handover_test.go @@ -0,0 +1,267 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +// applyOnce drives the config to Applied against its current assignment. +func applyOnce(t *testing.T, r *XRDConversionConfigReconciler) *teraskyv1alpha1.XRDConversionConfig { + t.Helper() + for i := 0; i < 3; i++ { + if _, err := reconcileXRD(t, r, "cfg"); err != nil { + t.Fatalf("reconcile %d: %v", i, err) + } + } + got := getXRDConfig(t, r, "cfg") + if got.Status.Phase != teraskyv1alpha1.PhaseApplied { + t.Fatalf("fixture did not reach Applied: phase %q (%s)", got.Status.Phase, got.Status.Message) + } + return got +} + +// moveTo repoints the config at another server and reconciles once. +func moveTo(t *testing.T, r *XRDConversionConfigReconciler, server string) *teraskyv1alpha1.XRDConversionConfig { + t.Helper() + cfg := getXRDConfig(t, r, "cfg") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: server} + if err := r.Update(context.Background(), cfg); err != nil { + t.Fatalf("repointing the config: %v", err) + } + if _, err := reconcileXRD(t, r, "cfg"); err != nil { + t.Fatalf("reconcile after the move: %v", err) + } + return getXRDConfig(t, r, "cfg") +} + +// The move is held until the destination reports it can already serve the +// target. Until then the XRD still names the source, and the source keeps +// serving it — so nothing is unserved while the gate is closed. +func TestXRDHandover_WaitsUntilTheDestinationCanServe(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + srcServer, srcSecret := readyServer("srv-a") + dstServer, dstSecret := readyServer("srv-b") + dstServer.Spec.Default = false + + c := newFakeClient(xrd, cfg, srcServer, srcSecret, dstServer, dstSecret, + // srv-a's replicas serve it; srv-b's have published nothing about + // it yet, which is the state immediately after a reassignment. + replicaLease("srv-a", "a-1", "operator-ns", []string{"xfoos.example.org"}), + replicaLease("srv-a", "a-2", "operator-ns", []string{"xfoos.example.org"}), + replicaLease("srv-b", "b-1", "operator-ns", nil), + replicaLease("srv-b", "b-2", "operator-ns", nil), + ).Build() + r := &XRDConversionConfigReconciler{Client: c, DefaultServerNamespace: "operator-ns"} + + applyOnce(t, r) + got := moveTo(t, r, "srv-b") + + if meta.IsStatusConditionTrue(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) { + t.Fatal("HandoverReady is True although srv-b reports it cannot serve the target yet") + } + cond := meta.FindStatusCondition(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) + if cond == nil || cond.Reason != "HandoverPending" { + t.Fatalf("HandoverReady condition = %+v, want reason HandoverPending", cond) + } + if !strings.Contains(got.Status.Message, "srv-b") { + t.Errorf("status message does not name the destination: %q", got.Status.Message) + } + // And critically: the XRD has NOT been repointed. + if got.Status.WebhookURL == "" || !strings.Contains(got.Status.WebhookURL, "srv-a") { + t.Fatalf("webhook URL is %q; while the gate is closed the target must still point at srv-a", got.Status.WebhookURL) + } +} + +// The gate has to stay closed across repeated reconciles, which is where +// the obvious implementation gets it wrong: status.assignedWebhookServer is +// written as soon as the resolver answers, so reading it back on the next +// pass says the move already happened and the target is repointed +// unverified on reconcile two. +func TestXRDHandover_StaysClosedAcrossRepeatedReconciles(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + srcServer, srcSecret := readyServer("srv-a") + dstServer, dstSecret := readyServer("srv-b") + dstServer.Spec.Default = false + + c := newFakeClient(xrd, cfg, srcServer, srcSecret, dstServer, dstSecret, + replicaLease("srv-b", "b-1", "operator-ns", nil), + replicaLease("srv-b", "b-2", "operator-ns", nil), + ).Build() + r := &XRDConversionConfigReconciler{Client: c, DefaultServerNamespace: "operator-ns"} + + applyOnce(t, r) + moveTo(t, r, "srv-b") + + for i := 0; i < 5; i++ { + if _, err := reconcileXRD(t, r, "cfg"); err != nil { + t.Fatalf("reconcile %d: %v", i, err) + } + got := getXRDConfig(t, r, "cfg") + if !strings.Contains(got.Status.WebhookURL, "srv-a") { + t.Fatalf("after %d further reconciles the target was repointed at %q although srv-b still reports it cannot serve it", + i+1, got.Status.WebhookURL) + } + if meta.IsStatusConditionTrue(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) { + t.Fatalf("HandoverReady went True on reconcile %d with nothing having changed", i+1) + } + } +} + +// Once the destination publishes the target, the move goes through. +func TestXRDHandover_ProceedsOnceTheDestinationReports(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + srcServer, srcSecret := readyServer("srv-a") + dstServer, dstSecret := readyServer("srv-b") + dstServer.Spec.Default = false + + c := newFakeClient(xrd, cfg, srcServer, srcSecret, dstServer, dstSecret, + replicaLease("srv-b", "b-1", "operator-ns", []string{"xfoos.example.org"}), + replicaLease("srv-b", "b-2", "operator-ns", []string{"xfoos.example.org"}), + ).Build() + r := &XRDConversionConfigReconciler{Client: c, DefaultServerNamespace: "operator-ns"} + + applyOnce(t, r) + got := moveTo(t, r, "srv-b") + + if !meta.IsStatusConditionTrue(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) { + cond := meta.FindStatusCondition(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) + t.Fatalf("HandoverReady = %+v, want True once both destination replicas report the target", cond) + } + if got.Status.Phase != teraskyv1alpha1.PhaseApplied { + t.Fatalf("phase = %q, want Applied (%s)", got.Status.Phase, got.Status.Message) + } + if !strings.Contains(got.Status.WebhookURL, "srv-b") { + t.Fatalf("webhook URL is %q, want it repointed at srv-b", got.Status.WebhookURL) + } +} + +// The condition survives the reconciles that follow a completed move. It +// is the verdict on the last handover, and an operator whose replicas +// cannot publish their Leases only ever finds out through a +// HandoverUnverified that is still there when they look. +func TestXRDHandover_VerdictPersistsAfterTheMove(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + srcServer, srcSecret := readyServer("srv-a") + dstServer, dstSecret := readyServer("srv-b") + dstServer.Spec.Default = false + + c := newFakeClient(xrd, cfg, srcServer, srcSecret, dstServer, dstSecret, + replicaLease("srv-b", "b-1", "operator-ns", []string{"xfoos.example.org"}), + replicaLease("srv-b", "b-2", "operator-ns", []string{"xfoos.example.org"}), + ).Build() + r := &XRDConversionConfigReconciler{Client: c, DefaultServerNamespace: "operator-ns"} + + applyOnce(t, r) + moveTo(t, r, "srv-b") + for i := 0; i < 3; i++ { + if _, err := reconcileXRD(t, r, "cfg"); err != nil { + t.Fatalf("reconcile %d: %v", i, err) + } + } + cond := meta.FindStatusCondition(getXRDConfig(t, r, "cfg").Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) + if cond == nil { + t.Fatal("the handover verdict was cleared once the move settled; nobody would ever see an unverified one") + } + if cond.Status != metav1.ConditionTrue || cond.Reason != "HandoverReady" { + t.Fatalf("HandoverReady = %+v after the move settled", cond) + } +} + +// A fleet whose replicas publish nothing — mid-upgrade, or a namespace +// with no Lease Role — must keep working exactly as it did before the gate +// existed, and say that the handover was not verified. +func TestXRDHandover_ProceedsUnverifiedWhenNobodyPublishes(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + srcServer, srcSecret := readyServer("srv-a") + dstServer, dstSecret := readyServer("srv-b") + dstServer.Spec.Default = false + + c := newFakeClient(xrd, cfg, srcServer, srcSecret, dstServer, dstSecret).Build() + r := &XRDConversionConfigReconciler{Client: c, DefaultServerNamespace: "operator-ns"} + + applyOnce(t, r) + got := moveTo(t, r, "srv-b") + + // First it waits. "Nobody has published" is indistinguishable from + // "nobody has published *yet*", and the second is the common case, so + // the move does not go through on the strength of silence alone. + cond := meta.FindStatusCondition(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "HandoverAwaitingReports" { + t.Fatalf("HandoverReady = %+v, want False/HandoverAwaitingReports", cond) + } + if strings.Contains(got.Status.WebhookURL, "srv-b") { + t.Fatalf("webhook URL is %q; the move must not happen while the grace period is still running", got.Status.WebhookURL) + } + + // Backdate the refusal past the grace period — the fleet structurally + // cannot report, so no amount of further waiting will change it. + cond.LastTransitionTime = metav1.NewTime(time.Now().Add(-UnreportedGracePeriod - time.Second)) + // Assigned rather than passed through meta.SetStatusCondition, which + // deliberately will not restamp LastTransitionTime while the status is + // unchanged — the very property blockedFor relies on. + got.Status.Conditions = []metav1.Condition{*cond} + if err := r.Status().Update(context.Background(), got); err != nil { + t.Fatalf("backdating the handover condition: %v", err) + } + if _, err := reconcileXRD(t, r, "cfg"); err != nil { + t.Fatalf("reconcile after the grace period: %v", err) + } + got = getXRDConfig(t, r, "cfg") + + cond = meta.FindStatusCondition(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) + if cond == nil || cond.Status != metav1.ConditionTrue || cond.Reason != "HandoverUnverified" { + t.Fatalf("HandoverReady = %+v, want True/HandoverUnverified once the grace period has passed", cond) + } + if !strings.Contains(got.Status.WebhookURL, "srv-b") { + t.Fatalf("webhook URL is %q; an unverifiable handover must still proceed eventually, as it did before", got.Status.WebhookURL) + } +} + +// A first apply has no previous server still covering the target, so +// gating it would delay every new config for nothing. +func TestXRDHandover_FirstApplyIsNotGated(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + server, secret := readyServer("srv") + + c := newFakeClient(xrd, cfg, server, secret, + // Replicas are up and publishing, but know nothing about this + // target yet — exactly the state a brand-new config starts in. + replicaLease("srv", "p-1", "operator-ns", nil), + replicaLease("srv", "p-2", "operator-ns", nil), + ).Build() + r := &XRDConversionConfigReconciler{Client: c, DefaultServerNamespace: "operator-ns"} + + got := applyOnce(t, r) + if meta.FindStatusCondition(got.Status.Conditions, teraskyv1alpha1.ConditionHandoverReady) != nil { + t.Fatal("a first apply set a HandoverReady condition; there is nothing to hand over from") + } +} diff --git a/internal/controller/naming.go b/internal/controller/naming.go index aa283db..0f87b0b 100644 --- a/internal/controller/naming.go +++ b/internal/controller/naming.go @@ -16,13 +16,15 @@ limitations under the License. package controller +import teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" + // Naming conventions for the child resources a ConversionWebhookServer // reconciles, shared with the XRDConversionConfig reconciler so it can // locate a server's Service/Certificate Secret without a second CRD // round-trip. func cwsDeploymentName(server string) string { return server + "-webhook-server" } -func cwsServiceName(server string) string { return server + "-webhook-server" } +func cwsServiceName(server string) string { return teraskyv1alpha1.WebhookServerServiceName(server) } func cwsCertificateName(server string) string { return server + "-webhook-server-cert" } func cwsCertificateSecretName(server string) string { return server + "-webhook-server-tls" } func cwsPDBName(server string) string { return server + "-webhook-server" } diff --git a/internal/controller/servedtargets.go b/internal/controller/servedtargets.go new file mode 100644 index 0000000..bca6c97 --- /dev/null +++ b/internal/controller/servedtargets.go @@ -0,0 +1,189 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" + "github.com/terasky-oss/declarative-conversion-operator/internal/servedtargets" +) + +// UnreportedGracePeriod is how long a move waits for the destination to +// publish anything at all before giving up on verifying it and proceeding +// unverified. +// +// Sized to be long against how quickly a working replica reports — it +// publishes on its own watch of the same objects the operator just +// changed, so a healthy fleet answers in well under a second — and short +// against how long an operator would tolerate a move stalling on a fleet +// that structurally cannot report. Thirty seconds is the same order as the +// drain period on the other side of the same move. +const UnreportedGracePeriod = 30 * time.Second + +// readServedTargets aggregates one instance's replica Leases into "what +// can every live replica of this instance actually serve right now". +// +// See internal/servedtargets for why this is published rather than +// queried, and why the aggregate is an intersection. +func readServedTargets(ctx context.Context, c client.Client, server *teraskyv1alpha1.ConversionWebhookServer, defaultNamespace string) (served []string, reporting int32, truncated bool, err error) { + ns := server.Spec.Namespace + if ns == "" { + ns = defaultNamespace + } + var leases coordinationv1.LeaseList + if err := c.List(ctx, &leases, + client.InNamespace(ns), + client.MatchingLabels{servedtargets.WebhookServerLabel: server.Name}, + ); err != nil { + return nil, 0, false, fmt.Errorf("listing served-target leases for %q: %w", server.Name, err) + } + served, reporting, truncated = servedtargets.Aggregate(leases.Items, time.Now()) + return served, reporting, truncated, nil +} + +// HandoverVerdict answers the one question a move has to ask: may the +// operator repoint this target's conversion webhook at this instance yet? +type HandoverVerdict struct { + // OK is true when the move may proceed. + OK bool + // Reason is a condition Reason; Message explains it. + Reason, Message string +} + +// canServeTarget decides whether it is safe to hand target over to +// server. +// +// The safe sequence is: the new instance compiles the plan (which it does +// on its own, from the same watch events the operator sees) → it publishes +// that it can serve the target → only then does the operator patch the +// target's spec.conversion to point at it. Until that patch lands, the +// target still names the *old* instance, and the old instance keeps +// serving it precisely because it is still named — see +// webhookserver.Reconciler, which holds a plan for as long as either the +// assignment or the live target points at it. +// +// The fallback when an instance publishes nothing at all is to proceed, +// but only after waiting UnreportedGracePeriod for a report that never +// comes. That ordering is the whole of it: +// +// - Waiting first covers the case that looks identical from here and is +// by far the more common one — the destination's replicas simply have +// not processed the config update yet. Proceeding immediately on "no +// reports" would repoint the target at replicas that have not compiled +// it, which is the registry-miss outage this sequence exists to +// prevent. +// - Proceeding eventually covers a fleet that structurally cannot +// report: mid-upgrade replicas running an image that predates Lease +// publication, or an instance in a namespace whose Role was never +// created. Blocking those forever would be a regression against every +// previous release, which moved targets with no verification at all. +// +// The grace period is what tells the two apart without needing a +// capability flag that something would have to set correctly. A fleet that +// can report does so in well under it; a fleet that cannot never will, and +// says so in the condition. +func canServeTarget(served []string, reporting, readyReplicas int32, truncated bool, target, serverName string, blockedFor time.Duration) HandoverVerdict { + switch { + case readyReplicas == 0: + // Reachable only by bypassing the ConversionWebhookServer health + // gate that runs before this, but "no ready replica" must never + // read as "nothing objects": repointing a target at an instance + // with nothing running is the outage this whole sequence exists + // to avoid. + return HandoverVerdict{ + Reason: "HandoverPending", + Message: fmt.Sprintf("ConversionWebhookServer %q has no ready replicas, so it cannot serve %q; the target stays on its current server", serverName, target), + } + case reporting == 0 && blockedFor < UnreportedGracePeriod: + return HandoverVerdict{ + Reason: "HandoverAwaitingReports", + Message: fmt.Sprintf("no replica of ConversionWebhookServer %q has published its served targets yet, so whether it can serve %q is unknown; "+ + "waiting up to %s for a report before moving the target", serverName, target, UnreportedGracePeriod), + } + case reporting == 0: + return HandoverVerdict{ + OK: true, + Reason: "HandoverUnverified", + Message: fmt.Sprintf("no replica of ConversionWebhookServer %q published its served targets within %s, so the handover of %q could not be verified; "+ + "proceeding as earlier releases did. Check that the webhook-server replicas are up to date and permitted to write Leases in their namespace", + serverName, UnreportedGracePeriod, target), + } + case truncated: + return HandoverVerdict{ + Reason: "HandoverUnknown", + Message: fmt.Sprintf("a replica of ConversionWebhookServer %q could not report its served targets, so whether it can serve %q is unknown; "+ + "the target stays on its current server", serverName, target), + } + case reporting < readyReplicas: + return HandoverVerdict{ + Reason: "HandoverPending", + Message: fmt.Sprintf("%d of %d ready replicas of ConversionWebhookServer %q have reported their served targets; waiting before repointing %q", + reporting, readyReplicas, serverName, target), + } + case !servedtargets.Contains(served, target): + return HandoverVerdict{ + Reason: "HandoverPending", + Message: fmt.Sprintf("ConversionWebhookServer %q is not yet serving %q on all %d reporting replicas; the target stays on its current server until it is", + serverName, target, reporting), + } + } + return HandoverVerdict{ + OK: true, + Reason: "HandoverReady", + Message: fmt.Sprintf("all %d reporting replicas of ConversionWebhookServer %q already serve %q", reporting, serverName, target), + } +} + +// checkHandover is the call site's convenience wrapper: read the Leases, +// then judge. A List failure is returned as an error so the reconcile +// retries, rather than being silently read as "not ready" — which would +// stall a move on a transient API blip. +func checkHandover(ctx context.Context, c client.Client, server *teraskyv1alpha1.ConversionWebhookServer, defaultNamespace, target string, conditions []metav1.Condition, now time.Time) (HandoverVerdict, error) { + served, reporting, truncated, err := readServedTargets(ctx, c, server, defaultNamespace) + if err != nil { + return HandoverVerdict{}, err + } + return canServeTarget(served, reporting, server.Status.ReadyReplicas, truncated, target, server.Name, blockedFor(conditions, now)), nil +} + +// blockedFor is how long this handover has already been refused. +// +// Read from the HandoverReady condition rather than from a status field of +// its own: the condition is already False for exactly as long as the move +// is blocked, and meta.SetStatusCondition only restamps LastTransitionTime +// when the status changes — so the timestamp survives the reason moving +// between waiting states, which is what "how long has this been blocked" +// should measure. A True or absent condition means this is the first +// refusal, and the clock starts now. +func blockedFor(conditions []metav1.Condition, now time.Time) time.Duration { + c := meta.FindStatusCondition(conditions, teraskyv1alpha1.ConditionHandoverReady) + if c == nil || c.Status != metav1.ConditionFalse || c.LastTransitionTime.IsZero() { + return 0 + } + if d := now.Sub(c.LastTransitionTime.Time); d > 0 { + return d + } + return 0 +} diff --git a/internal/controller/servedtargets_test.go b/internal/controller/servedtargets_test.go new file mode 100644 index 0000000..82a0f9e --- /dev/null +++ b/internal/controller/servedtargets_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "reflect" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" + "github.com/terasky-oss/declarative-conversion-operator/internal/servedtargets" +) + +func replicaLease(server, pod, namespace string, targets []string) *coordinationv1.Lease { + value, _ := servedtargets.Encode(targets) + now := metav1.NewMicroTime(time.Now()) + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: servedtargets.LeaseName(pod), + Namespace: namespace, + Labels: map[string]string{ + servedtargets.WebhookServerLabel: server, + ManagedByLabel: ManagedByValue, + }, + Annotations: map[string]string{servedtargets.TargetsAnnotation: value}, + }, + Spec: coordinationv1.LeaseSpec{RenewTime: &now}, + } +} + +func TestReadServedTargets_IntersectsAcrossReplicas(t *testing.T) { + server := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv"}} + server.Spec.Namespace = "dco-system" + + c := newFakeClient( + replicaLease("srv", "pod-1", "dco-system", []string{"a", "b"}), + replicaLease("srv", "pod-2", "dco-system", []string{"a"}), + // Another instance's replica, and one in another namespace. + replicaLease("other", "pod-3", "dco-system", []string{"zz"}), + replicaLease("srv", "pod-4", "elsewhere", []string{"zz"}), + ).Build() + + served, reporting, truncated, err := readServedTargets(context.Background(), c, server, "operator-ns") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if truncated { + t.Fatal("nothing was truncated") + } + if reporting != 2 { + t.Fatalf("reporting = %d, want 2 — leases for other instances or namespaces must not be counted", reporting) + } + if !reflect.DeepEqual(served, []string{"a"}) { + t.Fatalf("served = %v, want the intersection [a]", served) + } +} + +// spec.namespace is optional; an instance that omits it lives in the +// operator's own namespace, and looking in the wrong one would report +// every instance as publishing nothing. +func TestReadServedTargets_FallsBackToTheOperatorNamespace(t *testing.T) { + server := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "srv"}} + c := newFakeClient(replicaLease("srv", "pod-1", "operator-ns", []string{"a"})).Build() + + served, reporting, _, err := readServedTargets(context.Background(), c, server, "operator-ns") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if reporting != 1 || !reflect.DeepEqual(served, []string{"a"}) { + t.Fatalf("served = %v, reporting = %d", served, reporting) + } +} + +func TestCanServeTarget(t *testing.T) { + const target = "xfoos.example.org" + cases := []struct { + name string + served []string + reporting int32 + readyReplicas int32 + truncated bool + blockedFor time.Duration + wantOK bool + wantReason string + }{{ + // Nothing published *yet*. This looks identical to a fleet that + // cannot publish, and is far more often a destination whose + // replicas have not processed the config update. Waiting first is + // what tells the two apart. + name: "nothing published yet waits", + reporting: 0, readyReplicas: 2, + wantOK: false, wantReason: "HandoverAwaitingReports", + }, { + // Still nothing after the grace period: a fleet mid-upgrade, or an + // instance in a namespace whose Lease Role was never created. + // Blocking those forever would be a regression against every + // previous release, so the move proceeds and says it was not + // verified. + name: "nothing published after the grace period proceeds unverified", + reporting: 0, readyReplicas: 2, blockedFor: UnreportedGracePeriod, + wantOK: true, wantReason: "HandoverUnverified", + }, { + // The grace period must not become a blanket timeout that + // approves anything. A replica that HAS reported and does not + // serve the target is a definite no, however long we have waited. + name: "waiting does not expire into approving a definite no", + served: []string{"afoos.example.org"}, reporting: 2, readyReplicas: 2, + blockedFor: 10 * UnreportedGracePeriod, + wantOK: false, wantReason: "HandoverPending", + }, { + name: "all reporting replicas serve it", + served: []string{"afoos.example.org", target}, reporting: 2, readyReplicas: 2, + wantOK: true, wantReason: "HandoverReady", + }, { + name: "not yet compiled anywhere", + served: []string{"afoos.example.org"}, reporting: 2, readyReplicas: 2, + wantOK: false, wantReason: "HandoverPending", + }, { + // The intersection says yes, but a ready replica has not spoken. + // Its silence is not agreement. + name: "a ready replica has not reported", + served: []string{target}, reporting: 1, readyReplicas: 3, + wantOK: false, wantReason: "HandoverPending", + }, { + name: "a replica could not state its set", + truncated: true, reporting: 2, readyReplicas: 2, + wantOK: false, wantReason: "HandoverUnknown", + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + v := canServeTarget(tc.served, tc.reporting, tc.readyReplicas, tc.truncated, target, "srv-b", tc.blockedFor) + if v.OK != tc.wantOK || v.Reason != tc.wantReason { + t.Fatalf("got OK=%v reason=%q, want OK=%v reason=%q (message: %s)", v.OK, v.Reason, tc.wantOK, tc.wantReason, v.Message) + } + if v.Message == "" { + t.Error("every verdict needs a message: it is what lands on the config's status condition") + } + }) + } +} + +// The ConversionWebhookServer health gate runs before this, so it should +// not be reachable — but "no ready replica" must never read as "nothing +// objects". Repointing a target at an instance with nothing running is the +// outage the whole sequence exists to avoid. +func TestCanServeTarget_NoReadyReplicasNeverApproves(t *testing.T) { + for _, blocked := range []time.Duration{0, 10 * UnreportedGracePeriod} { + v := canServeTarget(nil, 0, 0, false, "xfoos.example.org", "srv-b", blocked) + if v.OK { + t.Fatalf("approved a handover to an instance with no ready replicas after %s: %+v", blocked, v) + } + if v.Reason != "HandoverPending" { + t.Fatalf("reason = %q, want HandoverPending", v.Reason) + } + } +} + +// blockedFor reads the clock off the HandoverReady condition, so the +// grace period has to start when the move was first refused and keep +// running across the reason changing between waiting states — which is +// exactly when meta.SetStatusCondition does NOT restamp the timestamp. +func TestBlockedFor(t *testing.T) { + now := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + cond := func(status metav1.ConditionStatus, age time.Duration) []metav1.Condition { + return []metav1.Condition{{ + Type: teraskyv1alpha1.ConditionHandoverReady, + Status: status, + LastTransitionTime: metav1.NewTime(now.Add(-age)), + }} + } + for _, tc := range []struct { + name string + conditions []metav1.Condition + want time.Duration + }{ + {"no condition is a fresh refusal", nil, 0}, + {"a satisfied handover is not blocked", cond(metav1.ConditionTrue, time.Hour), 0}, + {"a refused handover counts from its transition", cond(metav1.ConditionFalse, 20*time.Second), 20 * time.Second}, + {"a clock that went backwards does not go negative", cond(metav1.ConditionFalse, -time.Minute), 0}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := blockedFor(tc.conditions, now); got != tc.want { + t.Fatalf("blockedFor = %s, want %s", got, tc.want) + } + }) + } +} diff --git a/internal/controller/startupprobe_test.go b/internal/controller/startupprobe_test.go new file mode 100644 index 0000000..84eab69 --- /dev/null +++ b/internal/controller/startupprobe_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +func serverWithStartupProbe(sp *teraskyv1alpha1.StartupProbeSpec) *teraskyv1alpha1.ConversionWebhookServer { + return &teraskyv1alpha1.ConversionWebhookServer{ + ObjectMeta: metav1.ObjectMeta{Name: "srv"}, + Spec: teraskyv1alpha1.ConversionWebhookServerSpec{ + Namespace: "operator-ns", + Certificate: teraskyv1alpha1.CertificateSpec{IssuerRef: teraskyv1alpha1.CertificateIssuerRef{Name: "ca-issuer"}}, + StartupProbe: sp, + }, + } +} + +// An instance created before spec.startupProbe existed must still get one, +// which is why the controller defaults rather than relying on the CRD +// markers alone — the markers only default a field that is present. +func TestStartupProbe_DefaultsWithoutASpec(t *testing.T) { + dep := reconcileToDeployment(t, serverWithStartupProbe(nil)) + c := dep.Spec.Template.Spec.Containers[0] + + if c.StartupProbe == nil { + t.Fatal("no startupProbe: the liveness probe's 30s then bounds the whole cold start, and a replica slower than that never finishes one") + } + if c.StartupProbe.PeriodSeconds != 5 || c.StartupProbe.FailureThreshold != 60 { + t.Errorf("startupProbe period/threshold = %d/%d, want 5/60 (a five-minute budget)", + c.StartupProbe.PeriodSeconds, c.StartupProbe.FailureThreshold) + } + // /readyz, not /healthz. The plain endpoint carrying /healthz comes up + // before the registry sync, so a startupProbe pointed at it succeeds + // within milliseconds and bounds nothing — the budget every doc + // describes would not exist. /readyz is false until InitialSync + // completes, which is what makes period x failureThreshold a real + // deadline on the sync and gets a wedged replica restarted instead of + // left not-ready forever. + if c.StartupProbe.HTTPGet == nil || c.StartupProbe.HTTPGet.Path != "/readyz" { + t.Fatalf("startupProbe must poll /readyz, got %+v", c.StartupProbe.HTTPGet) + } + if c.LivenessProbe == nil || c.LivenessProbe.HTTPGet == nil || c.LivenessProbe.HTTPGet.Path != "/healthz" { + t.Fatalf("the liveness probe must stay on /healthz, got %+v", c.LivenessProbe) + } +} + +func TestStartupProbe_HonoursAnExplicitBudget(t *testing.T) { + period, threshold := int32(10), int32(90) + dep := reconcileToDeployment(t, serverWithStartupProbe(&teraskyv1alpha1.StartupProbeSpec{ + PeriodSeconds: &period, FailureThreshold: &threshold, + })) + c := dep.Spec.Template.Spec.Containers[0] + if c.StartupProbe == nil || c.StartupProbe.PeriodSeconds != 10 || c.StartupProbe.FailureThreshold != 90 { + t.Fatalf("startupProbe = %+v, want period 10 / threshold 90", c.StartupProbe) + } +} + +func TestStartupProbe_CanBeTurnedOff(t *testing.T) { + off := false + dep := reconcileToDeployment(t, serverWithStartupProbe(&teraskyv1alpha1.StartupProbeSpec{Enabled: &off})) + c := dep.Spec.Template.Spec.Containers[0] + if c.StartupProbe != nil { + t.Fatalf("startupProbe = %+v, want none when explicitly disabled", c.StartupProbe) + } + // Disabling the startupProbe must not take the other two with it. + if c.ReadinessProbe == nil || c.LivenessProbe == nil { + t.Fatal("readiness and liveness probes must survive disabling the startupProbe") + } +} diff --git a/internal/controller/xrdconversionconfig_controller.go b/internal/controller/xrdconversionconfig_controller.go index 3f9b04a..a055e0a 100644 --- a/internal/controller/xrdconversionconfig_controller.go +++ b/internal/controller/xrdconversionconfig_controller.go @@ -62,6 +62,11 @@ type XRDConversionConfigReconciler struct { // resources live when spec.namespace is unset — normally the // operator's own install namespace. DefaultServerNamespace string + + // MaxConcurrentReconciles bounds how many objects this controller + // reconciles at once. Zero leaves controller-runtime's own default + // (1) in place. See internal/controller/concurrency.go. + MaxConcurrentReconciles int } // +kubebuilder:rbac:groups=terasky.com,resources=xrdconversionconfigs,verbs=get;list;watch;create;update;patch;delete @@ -218,6 +223,13 @@ func (r *XRDConversionConfigReconciler) reconcileNormal(ctx context.Context, cfg r.setInvalid(cfg, wasApplied, fmt.Sprintf("could not resolve a ConversionWebhookServer: %v", err)) return ctrl.Result{}, r.patchStatus(ctx, orig, cfg) } + // Is this reconcile a move? Judged from the URL last applied to the + // target, not from status.assignedWebhookServer — that field is + // written as soon as the resolver answers, which is before the target + // is repointed, so reading it back on the next reconcile would say the + // move had already happened and the gate would open after one pass. + movingServers := wasApplied && orig.Status.WebhookURL != "" && + !assign.TargetPointsAt(orig.Status.WebhookURL, serverName) cfg.Status.AssignedWebhookServer = serverName // Step 5: XRD health gate. @@ -250,6 +262,46 @@ func (r *XRDConversionConfigReconciler) reconcileNormal(ctx context.Context, cfg Type: teraskyv1alpha1.ConditionWebhookServerReady, Status: metav1.ConditionTrue, Reason: "ServerReady", Message: fmt.Sprintf("ConversionWebhookServer %q is Available", serverName), }) + // Step 6b: the handover gate. + // + // Assignment can move a target from one instance to another — an + // operator editing spec.webhookServerRef, or automatic sharding + // rebalancing after an instance is added or removed. Repointing the + // target's spec.conversion the moment the assignment changes opens a + // window in which the apiserver sends ConversionReviews to replicas + // that have not compiled the plan yet, and every read and write of + // that resource fails for the duration. + // + // So the move waits. While it waits the target still names the old + // instance, and the old instance keeps serving it — a webhook-server + // replica holds a compiled plan for as long as EITHER the assignment + // or the live target points at it, which is what makes "wait" safe + // rather than merely slower. Nothing is ever unserved. + // + // Only on a move: a first apply has no previous server to hand over + // from, so gating it would just delay every new config for no gain. + if movingServers { + verdict, err := checkHandover(ctx, r.Client, &server, r.DefaultServerNamespace, cfg.Spec.TargetXRD.Name, orig.Status.Conditions, time.Now()) + if err != nil { + return ctrl.Result{}, err + } + meta.SetStatusCondition(&cfg.Status.Conditions, metav1.Condition{ + Type: teraskyv1alpha1.ConditionHandoverReady, Status: boolStatus(verdict.OK), + Reason: verdict.Reason, Message: verdict.Message, + }) + if !verdict.OK { + setPhasePendingOrStale(&cfg.Status.Conditions, &cfg.Status.Phase, wasApplied, verdict.Reason, verdict.Message) + cfg.Status.Message = verdict.Message + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.patchStatus(ctx, orig, cfg) + } + } + // Deliberately not removed when this reconcile is not a move. The + // condition is the verdict on the last handover, and it stays true + // afterwards — "the instance now serving this target was verified able + // to serve it before it was pointed here" does not stop being true. + // Leaving it is what makes a HandoverUnverified stick around long + // enough for somebody to notice that their replicas cannot publish. + // Step 7: only now, patch the XRD. caBundle, err := r.readCABundle(ctx, &server) if err != nil { @@ -618,6 +670,7 @@ func (r *XRDConversionConfigReconciler) SetupWithManager(mgr ctrl.Manager) error // already exists for CRDConversionConfig, so it costs nothing new. Watches(&extv1.CustomResourceDefinition{}, handler.EnqueueRequestsFromMapFunc(r.mapGeneratedCRDToConfigs)). Watches(&teraskyv1alpha1.ConversionWebhookServer{}, enqueue.PacedMapFuncs(r.mapServerToAssignedConfigs, r.mapServerTransitionToAssignedConfigs, enqueue.CWSConfigEnqueueQPS)). + WithOptions(controllerOptions(r.MaxConcurrentReconciles)). Named("xrdconversionconfig"). Complete(r) } diff --git a/internal/scalegen/report.go b/internal/scalegen/report.go new file mode 100644 index 0000000..245a23b --- /dev/null +++ b/internal/scalegen/report.go @@ -0,0 +1,159 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scalegen + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "time" +) + +// ReportSchemaVersion is bumped when a field changes meaning. The nightly +// workflow compares a run against the previous run's artifact, and +// comparing two reports written to different schemas would produce a +// confident, wrong regression verdict — so the comparison refuses when the +// versions differ. +// Bumped to 2 when ThroughputPerSecond changed from a per-worker figure +// derived from p50 to the real N/elapsed rate, and when Envelope was +// added. A v1 report compared against a v2 one would read the same field +// name as two different quantities. +const ReportSchemaVersion = 2 + +// Report is the machine-readable form of a Result: the scale run's +// numbers, shaped for a scheduled job to publish as an artifact, render in +// a job summary, and diff against the previous run. +// +// Durations are milliseconds as floats rather than Go duration strings, +// because everything downstream — the regression check, a spreadsheet, +// anything plotting a trend — wants a number. +type Report struct { + SchemaVersion int `json:"schemaVersion"` + RecordedAt string `json:"recordedAt"` + + Targets int `json:"targets"` + Instances int `json:"instances"` + TotalObjects int `json:"totalObjects"` + + // Envelope is every input that changes what the run measures. The + // regression check requires two reports to agree on all of it before + // comparing them: a manual probe at a different parallelism or QPS + // measures a different thing, and diffing it against the nightly + // would produce a confident answer to a question nobody asked. + Envelope map[string]string `json:"envelope"` + + CreateMs float64 `json:"createMs"` + + // Measurements is keyed by operation so the regression check can walk + // it without knowing the operation names, and so adding one later does + // not break a comparison against an older artifact. + Measurements map[string]Measurement `json:"measurements"` + + // StrategyCoverage records how many of the fleet's conversions used + // each strategy. A run whose coverage collapsed is measuring something + // other than what the previous run measured. + StrategyCoverage map[string]int `json:"strategyCoverage,omitempty"` + + // Observed is filled in by the harness around this package — peak + // memory and cold-start time come from the cluster, not from the + // client driving it. Absent when the harness did not collect them. + Observed map[string]float64 `json:"observed,omitempty"` +} + +// Measurement is one operation class's latency and error count. +type Measurement struct { + N int `json:"n"` + Errors int `json:"errors"` + P50Ms float64 `json:"p50Ms"` + P99Ms float64 `json:"p99Ms"` + MaxMs float64 `json:"maxMs"` + // ThroughputPerSecond is n divided by the wall-clock the operation + // class took, across all workers — the rate the fleet actually + // achieved, so a run that got slower shows up here even when the + // percentiles are noisy. + ThroughputPerSecond float64 `json:"throughputPerSecond"` + // ElapsedMs is that wall-clock, kept so the rate can be re-derived + // and so a comparison can tell "fewer requests" from "slower ones". + ElapsedMs float64 `json:"elapsedMs"` +} + +func ms(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) } + +// ToReport converts a Result into its published form. +func (r *Result) ToReport(now time.Time) *Report { + rep := &Report{ + SchemaVersion: ReportSchemaVersion, + RecordedAt: now.UTC().Format(time.RFC3339), + Targets: r.Targets, + Instances: r.Instances, + TotalObjects: r.Targets * r.Instances, + Envelope: r.Envelope, + CreateMs: ms(r.Create), + Measurements: map[string]Measurement{ + "listV1": measurement(r.ListV1), + "listV2": measurement(r.ListV2), + "getV1": measurement(r.GetV1), + "getV2": measurement(r.GetV2), + }, + } + if len(r.Coverage) > 0 { + rep.StrategyCoverage = make(map[string]int, len(r.Coverage)) + for strategy, n := range r.Coverage { + rep.StrategyCoverage[string(strategy)] = n + } + } + return rep +} + +func measurement(s Stats) Measurement { + m := Measurement{ + N: s.N, Errors: s.Errors, + P50Ms: ms(s.P50), P99Ms: ms(s.P99), MaxMs: ms(s.Max), + ElapsedMs: ms(s.Elapsed), + } + if s.Elapsed > 0 { + m.ThroughputPerSecond = float64(s.N) / s.Elapsed.Seconds() + } + return m +} + +// WriteReport writes the report to path, creating parent directories. +// Deliberately indented: these files are read by humans during an incident +// at least as often as by the comparison script. +func WriteReport(path string, rep *Report) error { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating the result directory: %w", err) + } + data, err := json.MarshalIndent(rep, "", " ") + if err != nil { + return fmt.Errorf("encoding the scale report: %w", err) + } + return os.WriteFile(path, append(data, '\n'), 0o600) +} + +// MeasurementNames returns the report's operation keys in a stable order, +// so a rendered summary does not reshuffle its rows between runs. +func (r *Report) MeasurementNames() []string { + out := make([]string, 0, len(r.Measurements)) + for name := range r.Measurements { + out = append(out, name) + } + sort.Strings(out) + return out +} diff --git a/internal/scalegen/run.go b/internal/scalegen/run.go index 611d4cf..f69654f 100644 --- a/internal/scalegen/run.go +++ b/internal/scalegen/run.go @@ -23,6 +23,7 @@ import ( "io" "math" "sort" + "strconv" "strings" "sync" "sync/atomic" @@ -66,11 +67,16 @@ type Options struct { // Stats is latency for one operation class (get or list). type Stats struct { - N int - Errors int - P50 time.Duration - P99 time.Duration - Max time.Duration + N int + Errors int + P50 time.Duration + P99 time.Duration + Max time.Duration + // Elapsed is the wall-clock the whole class took, across all workers. + // Throughput is N/Elapsed and nothing else: dividing by a percentile + // would report per-worker latency dressed up as a rate, and would not + // move when the parallelism did. + Elapsed time.Duration Samples []string } @@ -84,6 +90,9 @@ type Result struct { ListV2 Stats GetV1 Stats GetV2 Stats + // Envelope records every input that changes what the run measures, so + // two reports can be checked for comparability before being diffed. + Envelope map[string]string } func (o Options) withDefaults() Options { @@ -126,6 +135,30 @@ func (o Options) withDefaults() Options { return o } +// envelope is every knob that changes what a run measures. Targets and +// instances are in the report on their own; these are the rest, and they +// matter just as much — the same fleet driven at 16 workers and at 60 is +// two different measurements wearing the same field names. +func (o Options) envelope() map[string]string { + return map[string]string{ + "targets": strconv.Itoa(o.Targets), + "instances": strconv.Itoa(o.Instances), + "parallel": strconv.Itoa(o.Parallel), + "listRepeats": strconv.Itoa(o.ListRepeats), + "getRepeats": strconv.Itoa(o.GetRepeats), + "strategiesMin": strconv.Itoa(o.StrategiesMin), + "strategiesMax": strconv.Itoa(o.StrategiesMax), + "seed": strconv.FormatInt(o.Seed, 10), + "qps": strconv.FormatFloat(float64(o.QPS), 'f', -1, 32), + "burst": strconv.Itoa(o.Burst), + // Reset decides whether the fleet is created or merely confirmed: + // without it, Create short-circuits on AlreadyExists and createMs + // measures a no-op. Two runs that disagree on it are not + // comparable even when every other knob matches. + "reset": strconv.FormatBool(o.Reset), + } +} + func (o Options) logf(format string, args ...any) { _, _ = fmt.Fprintf(o.Out, format+"\n", args...) } @@ -145,7 +178,7 @@ func Run(ctx context.Context, opts Options) (*Result, error) { opts.logf(" %-24s %d", s.Name, cov[s.Name]) } if opts.DryRun { - return &Result{Targets: len(targets), Instances: opts.Instances, Coverage: cov}, nil + return &Result{Targets: len(targets), Instances: opts.Instances, Coverage: cov, Envelope: opts.envelope()}, nil } cfg, err := restConfig(opts.Kubeconfig, opts.QPS, opts.Burst) @@ -257,6 +290,7 @@ func Run(ctx context.Context, opts Options) (*Result, error) { res := &Result{ Targets: len(targets), Instances: opts.Instances, Coverage: cov, Create: createDur, ListV1: listV1, ListV2: listV2, GetV1: getV1, GetV2: getV2, + Envelope: opts.envelope(), } printResult(opts.Out, res) if listV1.Errors+listV2.Errors+getV1.Errors+getV2.Errors > 0 { @@ -447,6 +481,7 @@ func benchGets(ctx context.Context, dyn dynamic.Interface, targets []Target, ns, } func collectTimed(ctx context.Context, parallel int, jobs []func() timed) Stats { + start := time.Now() out := make([]timed, len(jobs)) sem := make(chan struct{}, parallel) var wg sync.WaitGroup @@ -465,7 +500,9 @@ func collectTimed(ctx context.Context, parallel int, jobs []func() timed) Stats }() } wg.Wait() - return summarize(out) + stats := summarize(out) + stats.Elapsed = time.Since(start) + return stats } func runErrPool(ctx context.Context, parallel int, jobs []func() error) error { diff --git a/internal/servedtargets/servedtargets.go b/internal/servedtargets/servedtargets.go new file mode 100644 index 0000000..7124d10 --- /dev/null +++ b/internal/servedtargets/servedtargets.go @@ -0,0 +1,254 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package servedtargets is the wire format for the one piece of state +// that flows from webhook-server replicas back to the operator: which +// targets each replica can actually serve right now. +// +// The operator's reconcile loop deliberately makes no network calls to +// webhook-server pods (see docs/limitations.md), so this cannot be a +// query. Instead each replica publishes its own registry contents into a +// coordination.k8s.io Lease, and the operator reads those the same way it +// reads anything else — through an informer. +// +// A Lease rather than a ConfigMap or the Pod's own annotations: +// +// - It is owned by the Pod, so it is garbage-collected with it and +// needs no reaping of its own. +// - renewTime is a first-class staleness signal, which is exactly the +// backstop needed for a pod that is alive but wedged — the case +// ownership cannot cover. +// - It is a small, dedicated object, so a 30-second heartbeat is not +// rewriting something other controllers watch. +// +// Both the publisher (internal/webhookserver) and the readers +// (internal/controller) live on this package so the encoding is defined +// once. +package servedtargets + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "io" + "sort" + "strings" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" +) + +const ( + // WebhookServerLabel carries the ConversionWebhookServer name a + // published Lease belongs to. It is what scopes the operator's Lease + // informer — without a label there is nothing to select on, and the + // cache would hold every Lease in the cluster, which on any cluster + // means one per node. + WebhookServerLabel = "conversion.terasky.com/webhook-server" + + // TargetsAnnotation holds the encoded set of targets this replica has + // a compiled, servable plan for. + TargetsAnnotation = "conversion.terasky.com/served-targets" + + // TruncatedAnnotation is "true" when the replica's target set did not + // fit in MaxEncodedBytes. Readers must treat a truncated Lease as "I + // cannot tell you what I serve" rather than as a partial answer — + // silently reading a truncated list as complete would report a target + // as unserved and, worse, could report one as served that is not in + // the part that survived. + TruncatedAnnotation = "conversion.terasky.com/served-targets-truncated" + + // LeaseNamePrefix distinguishes these Leases from leader-election + // ones sharing the namespace. + LeaseNamePrefix = "dco-served-" + + // MaxEncodedBytes caps the annotation. Kubernetes limits an object's + // total annotations to 256 KiB; this leaves room for the rest. + // Encoded, a thousand typical target names come to a few kilobytes, + // so the cap is reached somewhere north of fifty thousand targets on + // one replica — far outside any envelope this project claims, which + // is why exceeding it degrades to "unknown" rather than to a more + // elaborate chunking scheme. + MaxEncodedBytes = 192 * 1024 + + // MaxDecodedBytes caps the UNCOMPRESSED set, which is the size a + // reader has to be prepared to expand. Target names are long and + // highly similar, so they compress by roughly four to one — the + // compressed cap alone would let a very repetitive set through and + // then surprise the reader. + MaxDecodedBytes = 1024 * 1024 + + // Heartbeat is how often a replica renews its Lease, and LeaseDuration + // is what it advertises as the validity window. A reader treats a + // Lease whose renewTime is older than StaleAfter as gone. + Heartbeat = 30 * time.Second + LeaseDuration = 90 * time.Second + StaleAfter = 3 * Heartbeat + + // MaxClockSkew bounds how far in the FUTURE a renewTime may be before + // the Lease is treated as stale rather than fresh. + // + // Without it, a replica whose clock jumped forward and then wedged + // keeps looking live for as long as the jump lasts — and a stale + // report is exactly what could authorise a handover onto an instance + // that has stopped serving the target. Publishers stamp renewTime from + // their own clock, so some skew is expected; an hour of it is not. + MaxClockSkew = 5 * time.Minute +) + +// LeaseName is the Lease a given replica publishes to. Keyed by pod name, +// which is unique within a namespace, so two instances sharing a +// namespace cannot collide. +func LeaseName(podName string) string { return LeaseNamePrefix + podName } + +// Encode packs a target set into an annotation value: sorted, joined, +// gzipped and base64'd. Sorted so an unchanged set encodes to an +// unchanged string and the publisher can skip the write; gzipped because +// target names are long, highly similar strings and compress by roughly +// four to one. +// +// The second return is true when the result exceeded MaxEncodedBytes, in +// which case the value is empty and the caller must set TruncatedAnnotation +// instead of publishing a partial set. +func Encode(targets []string) (string, bool) { + sorted := append([]string(nil), targets...) + sort.Strings(sorted) + + joined := strings.Join(sorted, "\n") + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + // gzip.Writer only fails if the underlying writer does, and + // bytes.Buffer does not. + _, _ = zw.Write([]byte(joined)) + _ = zw.Close() + + // Both sizes are checked. The compressed one is what has to fit in the + // annotation; the uncompressed one is what a reader has to be willing + // to expand, and target names compress well enough that a set could + // pass the first check and blow past the reader's limit — which would + // come back as a silently truncated prefix that looks like a complete + // answer. + encoded := base64.StdEncoding.EncodeToString(buf.Bytes()) + if len(encoded) > MaxEncodedBytes || len(joined) > MaxDecodedBytes { + return "", true + } + return encoded, false +} + +// Decode reverses Encode. An empty value decodes to an empty set, which +// is what a replica serving nothing publishes. +func Decode(value string) ([]string, error) { + if value == "" { + return nil, nil + } + raw, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, fmt.Errorf("decoding served-targets annotation: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("decompressing served-targets annotation: %w", err) + } + defer func() { _ = zr.Close() }() + // Bounded by the same cap the writer honours: an annotation is + // attacker-influenced only by whoever can already write Leases in the + // namespace, but an unbounded decompress is not something to leave in + // a reconcile loop regardless. + // One byte past the limit, so an oversized payload is an ERROR rather + // than a silently truncated prefix. A prefix would decode into a + // shorter list that looks complete, and the targets missing from it + // would read as "this replica cannot serve them" — holding a handover + // open indefinitely for a reason nothing reports. + out, err := io.ReadAll(io.LimitReader(zr, MaxDecodedBytes+1)) + if err != nil { + return nil, fmt.Errorf("reading served-targets annotation: %w", err) + } + if len(out) > MaxDecodedBytes { + return nil, fmt.Errorf("served-targets annotation decodes to more than %d bytes", MaxDecodedBytes) + } + if len(out) == 0 { + return nil, nil + } + return strings.Split(string(out), "\n"), nil +} + +// Aggregate reduces one instance's replica Leases to what the instance as +// a whole can serve. +// +// Served is the INTERSECTION across live replicas, not the union: a +// target that two replicas out of three can serve is a target that fails +// one request in three, so it is not something the instance serves. +// Reporting is how many live Leases contributed. Truncated is true if any +// live replica could not state its set, in which case Served is not a +// complete answer and callers must not treat an absence as a negative. +// +// A Lease is live if its renewTime is within StaleAfter of now. Leases are +// owned by their Pod and normally vanish with it; the staleness check is +// the backstop for a pod that is running but no longer publishing. +func Aggregate(leases []coordinationv1.Lease, now time.Time) (served []string, reporting int32, truncated bool) { + // Counts are int32 to match reporting: a count can never exceed the + // number of live leases, which is a replica count. + var counts map[string]int32 + for i := range leases { + l := &leases[i] + if l.Spec.RenewTime == nil { + continue + } + // Negative age is a renewTime in the future: the publisher's clock + // is ahead of ours. A little is normal; more than MaxClockSkew is + // not something to accept as proof of liveness. + if age := now.Sub(l.Spec.RenewTime.Time); age > StaleAfter || age < -MaxClockSkew { + continue + } + reporting++ + if l.Annotations[TruncatedAnnotation] == "true" { + truncated = true + continue + } + names, err := Decode(l.Annotations[TargetsAnnotation]) + if err != nil { + // An undecodable Lease is not evidence of anything. Treating + // it as truncated makes callers fall back to "cannot tell", + // which is the fail-closed reading. + truncated = true + continue + } + if counts == nil { + counts = map[string]int32{} + } + for _, n := range names { + counts[n]++ + } + } + if truncated || reporting == 0 { + return nil, reporting, truncated + } + for name, n := range counts { + if n == reporting { + served = append(served, name) + } + } + sort.Strings(served) + return served, reporting, false +} + +// Contains reports whether name is in a sorted set, as returned by +// Aggregate or read back from status.servedTargets. +func Contains(sortedSet []string, name string) bool { + i := sort.SearchStrings(sortedSet, name) + return i < len(sortedSet) && sortedSet[i] == name +} diff --git a/internal/servedtargets/servedtargets_test.go b/internal/servedtargets/servedtargets_test.go new file mode 100644 index 0000000..ebc9afa --- /dev/null +++ b/internal/servedtargets/servedtargets_test.go @@ -0,0 +1,262 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package servedtargets + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "reflect" + "strings" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestEncodeDecode_RoundTrips(t *testing.T) { + in := []string{"zfoos.example.org", "afoos.example.org", "mfoos.example.org"} + value, truncated := Encode(in) + if truncated { + t.Fatal("three names should not truncate") + } + got, err := Decode(value) + if err != nil { + t.Fatalf("decode: %v", err) + } + want := []string{"afoos.example.org", "mfoos.example.org", "zfoos.example.org"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip = %v, want %v (sorted)", got, want) + } +} + +// The publisher skips the write when the encoded value is unchanged, which +// only holds if the encoding does not depend on map iteration order. +func TestEncode_IsStableAcrossInputOrder(t *testing.T) { + a, _ := Encode([]string{"a", "b", "c"}) + b, _ := Encode([]string{"c", "a", "b"}) + if a != b { + t.Fatal("the same set in a different order encoded differently; every heartbeat would look like a change") + } +} + +func TestEncodeDecode_EmptySet(t *testing.T) { + value, truncated := Encode(nil) + if truncated { + t.Fatal("the empty set should not truncate") + } + got, err := Decode(value) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(got) != 0 { + t.Fatalf("decoded %v, want nothing — a replica serving nothing is a legitimate state", got) + } +} + +func TestDecode_RejectsGarbage(t *testing.T) { + if _, err := Decode("not base64 at all !!!"); err == nil { + t.Fatal("expected an error on a value that is not base64") + } + if _, err := Decode("aGVsbG8="); err == nil { + t.Fatal("expected an error on base64 that is not gzip") + } +} + +// lease builds a Lease renewed `age` ago. A negative age puts renewTime in +// the future, which is how the clock-skew cases are expressed. +func lease(name string, age time.Duration, targets []string) coordinationv1.Lease { + value, truncated := Encode(targets) + renew := metav1.NewMicroTime(time.Now().Add(-age)) + l := coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: map[string]string{}}, + Spec: coordinationv1.LeaseSpec{RenewTime: &renew}, + } + if truncated { + l.Annotations[TruncatedAnnotation] = "true" + } else { + l.Annotations[TargetsAnnotation] = value + } + return l +} + +// The aggregate is an intersection, not a union. A target two replicas out +// of three can serve is a target that fails one request in three, and +// reporting it as served would make the handover gate wave through exactly +// the move it exists to hold back. +func TestAggregate_IsAnIntersection(t *testing.T) { + leases := []coordinationv1.Lease{ + lease("a", 0, []string{"x", "y", "z"}), + lease("b", 0, []string{"x", "y"}), + lease("c", 0, []string{"x", "z"}), + } + served, reporting, truncated := Aggregate(leases, time.Now()) + if truncated { + t.Fatal("no lease was truncated") + } + if reporting != 3 { + t.Fatalf("reporting = %d, want 3", reporting) + } + if !reflect.DeepEqual(served, []string{"x"}) { + t.Fatalf("served = %v, want only [x]", served) + } +} + +func TestAggregate_IgnoresStaleLeases(t *testing.T) { + leases := []coordinationv1.Lease{ + lease("live", 0, []string{"x"}), + lease("wedged", StaleAfter+time.Minute, []string{"x", "y"}), + } + served, reporting, _ := Aggregate(leases, time.Now()) + if reporting != 1 { + t.Fatalf("reporting = %d, want 1: a replica that stopped renewing is not reporting", reporting) + } + if !reflect.DeepEqual(served, []string{"x"}) { + t.Fatalf("served = %v, want [x] from the live replica alone", served) + } +} + +func TestAggregate_NoLeases(t *testing.T) { + served, reporting, truncated := Aggregate(nil, time.Now()) + if served != nil || reporting != 0 || truncated { + t.Fatalf("Aggregate(nil) = %v/%d/%v, want nothing reported", served, reporting, truncated) + } +} + +// A truncated or undecodable Lease means "I cannot tell you what I serve". +// Reading the remaining replicas' intersection as the answer would report +// a target as unserved — or, worse, let a partial list look complete. +func TestAggregate_TruncationPoisonsTheAnswer(t *testing.T) { + broken := lease("broken", 0, nil) + delete(broken.Annotations, TargetsAnnotation) + broken.Annotations[TruncatedAnnotation] = "true" + + served, reporting, truncated := Aggregate([]coordinationv1.Lease{lease("ok", 0, []string{"x"}), broken}, time.Now()) + if !truncated { + t.Fatal("expected the aggregate to report truncation") + } + if served != nil { + t.Fatalf("served = %v, want nothing when the answer is incomplete", served) + } + if reporting != 2 { + t.Fatalf("reporting = %d, want 2: the replica is alive, it just cannot say what it serves", reporting) + } +} + +func TestAggregate_UndecodableLeaseIsTreatedAsUnknown(t *testing.T) { + bad := lease("bad", 0, []string{"x"}) + bad.Annotations[TargetsAnnotation] = "!!!not base64!!!" + _, _, truncated := Aggregate([]coordinationv1.Lease{bad}, time.Now()) + if !truncated { + t.Fatal("an undecodable annotation is not evidence of anything; it must read as unknown") + } +} + +func TestEncode_TruncatesBeyondTheCap(t *testing.T) { + // Deliberately incompressible names, so the cap is reached at a + // realistic-ish count rather than never. + many := make([]string, 400000) + for i := range many { + many[i] = fmt.Sprintf("%d-a1b2c3d4e5f6a7b8c9d0.very-long-group-name-%d.example.org", i, i*7919) + } + value, truncated := Encode(many) + if !truncated { + t.Fatalf("expected truncation past the %d-byte cap, got %d bytes", MaxEncodedBytes, len(value)) + } + if value != "" { + t.Fatal("a truncated encode must return nothing: a partial list read as complete is worse than no list") + } +} + +func TestContains(t *testing.T) { + set := []string{"a", "m", "z"} + for _, in := range set { + if !Contains(set, in) { + t.Errorf("Contains(%v, %q) = false", set, in) + } + } + for _, out := range []string{"", "b", "zz"} { + if Contains(set, out) { + t.Errorf("Contains(%v, %q) = true", set, out) + } + } +} + +// A renewTime in the future means the publisher's clock is ahead. A little +// is normal; a lot would keep a wedged replica looking live for as long as +// the jump lasts, and a stale report is exactly what could authorise a +// handover onto an instance that has stopped serving the target. +func TestAggregate_RejectsRenewalsTooFarInTheFuture(t *testing.T) { + skewed := lease("skewed", -(MaxClockSkew + time.Minute), []string{"x"}) + _, reporting, _ := Aggregate([]coordinationv1.Lease{skewed}, time.Now()) + if reporting != 0 { + t.Fatalf("reporting = %d, want 0: a renewTime %s in the future is not proof of liveness", reporting, MaxClockSkew+time.Minute) + } + + // A small skew is normal and must still count. + fine := lease("fine", -(MaxClockSkew / 2), []string{"x"}) + served, reporting, _ := Aggregate([]coordinationv1.Lease{fine}, time.Now()) + if reporting != 1 || !reflect.DeepEqual(served, []string{"x"}) { + t.Fatalf("a small clock skew must still be accepted; got served=%v reporting=%d", served, reporting) + } +} + +// Encode checks the compressed size, which a highly compressible set can +// slip past. Without the uncompressed check too, Decode would hand back a +// truncated prefix that reads as a complete answer — and every target +// missing from it would look unservable, holding a handover open forever. +func TestEncodeDecode_BoundsTheDecodedSizeToo(t *testing.T) { + // Maximally compressible: one literal name over and over. A formatted + // index would make every name distinct, and the payload could then + // trip MaxEncodedBytes instead — passing this test without ever + // reaching the bound it is about. + const name = "targets.example.org" + many := make([]string, 0, MaxDecodedBytes/len(name)+16) + for i := 0; i < cap(many); i++ { + many = append(many, name) + } + + // Stated rather than assumed: this payload is over the decoded bound + // and comfortably under the encoded one, so truncation can only be the + // decoded check firing. + joined := strings.Join(many, "\n") + if len(joined) <= MaxDecodedBytes { + t.Fatalf("fixture is %d bytes decoded, which does not exceed the %d-byte bound it is meant to test", len(joined), MaxDecodedBytes) + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write([]byte(joined)); err != nil { + t.Fatalf("compressing the fixture: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("closing the compressor: %v", err) + } + if compressed := len(base64.StdEncoding.EncodeToString(buf.Bytes())); compressed > MaxEncodedBytes { + t.Fatalf("fixture compresses to %d bytes, over the %d-byte encoded bound — it would trip the wrong check", compressed, MaxEncodedBytes) + } + + value, truncated := Encode(many) + if !truncated { + t.Fatalf("a set of %d names decoding to %d bytes, over the %d-byte bound, must be reported as truncated", + len(many), len(joined), MaxDecodedBytes) + } + if value != "" { + t.Fatal("a truncated encode must return nothing") + } +} diff --git a/internal/webhook/conversionwebhookserver_webhook.go b/internal/webhook/conversionwebhookserver_webhook.go index 8b2d9eb..0859a02 100644 --- a/internal/webhook/conversionwebhookserver_webhook.go +++ b/internal/webhook/conversionwebhookserver_webhook.go @@ -49,7 +49,10 @@ func (v *ConversionWebhookServerValidator) ValidateCreate(ctx context.Context, s if err := teraskyv1alpha1.ValidateWebhookServerRollout(server.Spec.Rollout, server.Spec.ExtraArgs); err != nil { return nil, err } - return nil, v.checkDefault(ctx, server) + if err := v.checkDefault(ctx, server); err != nil { + return nil, err + } + return nil, v.checkShardPool(ctx, server) } func (v *ConversionWebhookServerValidator) ValidateUpdate(ctx context.Context, _, newServer *teraskyv1alpha1.ConversionWebhookServer) (admission.Warnings, error) { @@ -59,7 +62,10 @@ func (v *ConversionWebhookServerValidator) ValidateUpdate(ctx context.Context, _ if err := teraskyv1alpha1.ValidateWebhookServerRollout(newServer.Spec.Rollout, newServer.Spec.ExtraArgs); err != nil { return nil, err } - return nil, v.checkDefault(ctx, newServer) + if err := v.checkDefault(ctx, newServer); err != nil { + return nil, err + } + return nil, v.checkShardPool(ctx, newServer) } func (v *ConversionWebhookServerValidator) checkDefault(ctx context.Context, server *teraskyv1alpha1.ConversionWebhookServer) error { @@ -81,6 +87,73 @@ func (v *ConversionWebhookServerValidator) checkDefault(ctx context.Context, ser return nil } +// checkShardPool enforces the one invariant automatic assignment needs: +// while any instance opts into sharding, the instance marked default must +// be one of them. +// +// Without it, enabling sharding on a single non-default instance would +// move every unpinned config onto that one instance in a single admission +// — a fleet-wide reassignment triggered by what looks like a local change. +// The pool takes precedence over spec.default precisely so that a +// half-configured pool cannot leave unpinned configs unserved, and this +// check is what makes that precedence safe to have. +// +// There is always a valid ordering: enable sharding on the default +// instance first, then on the others. Doing it that way moves nothing on +// the first step, and a bounded share on each one after. +func (v *ConversionWebhookServerValidator) checkShardPool(ctx context.Context, server *teraskyv1alpha1.ConversionWebhookServer) error { + var list teraskyv1alpha1.ConversionWebhookServerList + if err := v.Client.List(ctx, &list); err != nil { + return fmt.Errorf("listing existing ConversionWebhookServers: %w", err) + } + + // The incoming object replaces its stored copy, so the check is made + // against the fleet as it will be, not as it is. + fleet := make([]teraskyv1alpha1.ConversionWebhookServer, 0, len(list.Items)+1) + found := false + for _, other := range list.Items { + if other.Name == server.Name { + fleet = append(fleet, *server) + found = true + continue + } + fleet = append(fleet, other) + } + if !found { + fleet = append(fleet, *server) + } + + pool := assign.ShardPool(fleet) + if len(pool) == 0 { + return nil + } + var defaultName string + for _, s := range fleet { + if s.Spec.Default { + defaultName = s.Name + break + } + } + // No default at all is legal once a pool exists: the pool is what + // answers for unpinned configs, so nothing is left unresolved. + if defaultName == "" { + return nil + } + for _, s := range pool { + if s.Name == defaultName { + return nil + } + } + names := make([]string, 0, len(pool)) + for _, s := range pool { + names = append(names, s.Name) + } + return fmt.Errorf("ConversionWebhookServer %q is marked default but is not in the sharding pool %v; "+ + "while a pool exists it, not spec.default, serves configs with no explicit webhookServerRef, so this would move every unpinned config off %q at once. "+ + "Set spec.sharding.enabled on %q as well (do that first when building a pool), or unset spec.default", + defaultName, names, defaultName, defaultName) +} + func (v *ConversionWebhookServerValidator) ValidateDelete(ctx context.Context, server *teraskyv1alpha1.ConversionWebhookServer) (admission.Warnings, error) { if server.Annotations[teraskyv1alpha1.AllowForceDeleteAnnotation] == "true" { return nil, nil @@ -97,8 +170,13 @@ func (v *ConversionWebhookServerValidator) ValidateDelete(ctx context.Context, s if err := v.Client.List(ctx, &allServers); err != nil { return nil, fmt.Errorf("listing ConversionWebhookServers: %w", err) } - dependentXRD := assign.ConfigsAssignedTo(xrdConfigs.Items, allServers.Items, server.Name) - dependentCRD := assign.ConfigsAssignedTo(crdConfigs.Items, allServers.Items, server.Name) + // ServedBy rather than IsAssignedTo: mid-handover a config resolves to + // its destination while its target still points here, and this + // instance is still the one answering for it. The controller's + // finalizer check uses the same rule, so admission and reconcile + // cannot disagree about whether a delete is safe. + dependentXRD := assign.ConfigsServedBy(xrdConfigs.Items, allServers.Items, server.Name) + dependentCRD := assign.ConfigsServedBy(crdConfigs.Items, allServers.Items, server.Name) total := len(dependentXRD) + len(dependentCRD) if total == 0 { return nil, nil @@ -115,5 +193,5 @@ func (v *ConversionWebhookServerValidator) ValidateDelete(ctx context.Context, s if server.Spec.Default { suffix = " (this is the DEFAULT instance — configs with no explicit webhookServerRef depend on it too)" } - return nil, fmt.Errorf("%d config(s) still resolve to this instance%s: %v; reassign them first, or add annotation %q=\"true\" to force", total, suffix, names, teraskyv1alpha1.AllowForceDeleteAnnotation) + return nil, fmt.Errorf("%d config(s) still resolve to this instance, or still have their target pointed at it%s: %v; reassign them first, or add annotation %q=\"true\" to force", total, suffix, names, teraskyv1alpha1.AllowForceDeleteAnnotation) } diff --git a/internal/webhook/conversionwebhookserver_webhook_test.go b/internal/webhook/conversionwebhookserver_webhook_test.go index f8d60d2..b8a07f2 100644 --- a/internal/webhook/conversionwebhookserver_webhook_test.go +++ b/internal/webhook/conversionwebhookserver_webhook_test.go @@ -18,6 +18,7 @@ package webhook import ( "context" + "strings" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -165,3 +166,87 @@ func TestConversionWebhookServerValidator_ValidateDelete_ForceAnnotationBypasses t.Fatalf("expected the force-delete annotation to bypass the block, got: %v", err) } } + +func sharded(name string, isDefault bool) *teraskyv1alpha1.ConversionWebhookServer { + s := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: name}} + s.Spec.Default = isDefault + s.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{} + return s +} + +// Enabling sharding on a non-default instance while the default stays out +// of the pool would move every unpinned config onto the pool in one +// admission — a fleet-wide reassignment produced by what reads as a local +// change to one object. +func TestShardPool_RejectsADefaultOutsideThePool(t *testing.T) { + existingDefault := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "default"}} + existingDefault.Spec.Default = true + + c := newFakeClient(existingDefault).Build() + v := &ConversionWebhookServerValidator{Client: c} + + _, err := v.ValidateCreate(context.Background(), sharded("shard-b", false)) + if err == nil { + t.Fatal("expected a pool that excludes the default instance to be rejected") + } + for _, want := range []string{"default", "sharding pool", "spec.sharding.enabled"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error does not mention %q: %v", want, err) + } + } +} + +// The valid ordering: enable it on the default first, which moves nothing +// because the pool is then that one instance, then add the others. +func TestShardPool_AcceptsTheDefaultAsAPoolMember(t *testing.T) { + c := newFakeClient().Build() + v := &ConversionWebhookServerValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), sharded("default", true)); err != nil { + t.Fatalf("enabling sharding on the default instance must be accepted: %v", err) + } + + c = newFakeClient(sharded("default", true)).Build() + v = &ConversionWebhookServerValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), sharded("shard-b", false)); err != nil { + t.Fatalf("adding a second pool member must be accepted: %v", err) + } +} + +// A fleet with no default at all is legal once a pool exists — the pool is +// a complete answer for an unpinned config. +func TestShardPool_AcceptsAPoolWithNoDefault(t *testing.T) { + c := newFakeClient(sharded("shard-a", false)).Build() + v := &ConversionWebhookServerValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), sharded("shard-b", false)); err != nil { + t.Fatalf("a pool with no default instance must be accepted: %v", err) + } +} + +// The check runs against the fleet as it WILL be: turning sharding off on +// the last pool member, or on the default, has to be allowed even though +// the stored copy still says otherwise. +func TestShardPool_JudgesThePostUpdateFleet(t *testing.T) { + stored := sharded("default", true) + c := newFakeClient(stored).Build() + v := &ConversionWebhookServerValidator{Client: c} + + off := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "default"}} + off.Spec.Default = true + disabled := false + off.Spec.Sharding = &teraskyv1alpha1.ShardingSpec{Enabled: &disabled} + + if _, err := v.ValidateUpdate(context.Background(), stored, off); err != nil { + t.Fatalf("turning sharding off on the last pool member must be accepted: %v", err) + } +} + +func TestShardPool_NoPoolIsAlwaysFine(t *testing.T) { + existingDefault := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "default"}} + existingDefault.Spec.Default = true + c := newFakeClient(existingDefault).Build() + v := &ConversionWebhookServerValidator{Client: c} + plain := &teraskyv1alpha1.ConversionWebhookServer{ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}} + if _, err := v.ValidateCreate(context.Background(), plain); err != nil { + t.Fatalf("a fleet with no sharding at all must be unaffected: %v", err) + } +} diff --git a/internal/webhookserver/cache.go b/internal/webhookserver/cache.go index 3ea4ddc..e494869 100644 --- a/internal/webhookserver/cache.go +++ b/internal/webhookserver/cache.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" + coordinationv1 "k8s.io/api/coordination/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -203,3 +204,19 @@ func CountMatchingLabels(all []labels.Set, sel labels.Selector) int { } return n } + +// ClientOptions keeps the served-target Lease out of the cache. +// +// A replica writes exactly one Lease, its own, and reads it back only to +// build the next update. Caching that would cost a Lease informer, and a +// Lease informer is not a small thing to add: on any cluster there is +// already one per node in kube-node-lease, plus every leader election in +// every namespace. Read-through is a single GET every thirty seconds +// against an object this process wrote itself. +func ClientOptions() client.Options { + return client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{&coordinationv1.Lease{}}, + }, + } +} diff --git a/internal/webhookserver/handover.go b/internal/webhookserver/handover.go new file mode 100644 index 0000000..f69f328 --- /dev/null +++ b/internal/webhookserver/handover.go @@ -0,0 +1,96 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "time" + + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +// DefaultTargetDrainPeriod is how long a replica keeps serving a target +// after that target has stopped naming it. +// +// It exists for the same reason the pod's preStop sleep does, one layer +// up. The apiserver refreshes a CRD's conversion configuration +// asynchronously after the write that changed it, so for a short window +// after the operator repoints a target the apiserver is still calling the +// old Service. Dropping the plan the instant the object changes answers +// those calls with a 503 — and the apiserver reports that as a failed read +// or write on a resource that was merely being rebalanced. +// +// This is not hypothetical: hack/e2e-reassign.sh caught exactly one failed +// write in 9,456 during three reassignments before this existed, with the +// registry-miss message. One in ten thousand is small and it is not zero, +// and a rebalance is not supposed to cost anything. +// +// Thirty seconds is chosen the way the preStop sleep is: comfortably +// longer than the propagation it waits out, and cheap to be wrong about in +// the safe direction. The only cost of an over-long drain is that a +// replica holds one compiled plan — about 18 KiB — a little longer than it +// needs to. +const DefaultTargetDrainPeriod = 30 * time.Second + +// A replica serves a target when EITHER of two things is true: the shared +// resolver assigns the target to this instance, or the target's live +// spec.conversion still points its webhook at this instance's Service. +// +// The second clause is what makes a handover safe from the losing end. +// When a target moves from instance A to instance B — an edited +// webhookServerRef, or sharding rebalancing after an instance is added — +// A's assignment changes the instant the object does, but the target's +// spec.conversion still names A until the operator gets round to patching +// it. Dropping the plan on the assignment change alone leaves that window +// with a webhook configured to call A and an A that answers "unknown +// target". Holding it until the target stops naming this instance closes +// the window from A's side, exactly as the operator's handover gate closes +// it from B's. +// +// Matching on the Service name alone, not on namespace: instance names are +// cluster-unique and the Service name is derived from them, so the name is +// already unambiguous — and a replica does not otherwise need to know +// which namespace its own Service lives in. + +// xrdPointsAtServer reports whether an XRD's spec.conversion webhook still +// names this instance's Service. Unstructured because Crossplane's XRD +// type is not vendored here; a missing or differently-shaped +// spec.conversion simply reads as "no". +func xrdPointsAtServer(xrd *unstructured.Unstructured, serverName string) bool { + if xrd == nil { + return false + } + name, found, err := unstructured.NestedString(xrd.Object, "spec", "conversion", "webhook", "clientConfig", "service", "name") + if err != nil || !found { + return false + } + return name == teraskyv1alpha1.WebhookServerServiceName(serverName) +} + +// crdPointsAtServer is xrdPointsAtServer for a native CRD. +func crdPointsAtServer(crd *extv1.CustomResourceDefinition, serverName string) bool { + if crd == nil || crd.Spec.Conversion == nil || crd.Spec.Conversion.Webhook == nil { + return false + } + svc := crd.Spec.Conversion.Webhook.ClientConfig + if svc == nil || svc.Service == nil { + return false + } + return svc.Service.Name == teraskyv1alpha1.WebhookServerServiceName(serverName) +} diff --git a/internal/webhookserver/handover_test.go b/internal/webhookserver/handover_test.go new file mode 100644 index 0000000..db64efa --- /dev/null +++ b/internal/webhookserver/handover_test.go @@ -0,0 +1,392 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "context" + "testing" + "time" + + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" +) + +// pointXRDAt writes the spec.conversion an applied XRD would carry. +func pointXRDAt(xrd *unstructured.Unstructured, serverName string) { + _ = unstructured.SetNestedMap(xrd.Object, map[string]any{ + "strategy": "Webhook", + "webhook": map[string]any{ + "clientConfig": map[string]any{ + "service": map[string]any{ + "name": teraskyv1alpha1.WebhookServerServiceName(serverName), + "namespace": "dco-system", + "path": "/convert/" + xrd.GetName(), + }, + }, + }, + }, "spec", "conversion") +} + +func pointCRDAt(crd *extv1.CustomResourceDefinition, serverName string) { + crd.Spec.Conversion = &extv1.CustomResourceConversion{ + Strategy: extv1.WebhookConverter, + Webhook: &extv1.WebhookConversion{ + ClientConfig: &extv1.WebhookClientConfig{ + Service: &extv1.ServiceReference{ + Name: teraskyv1alpha1.WebhookServerServiceName(serverName), + Namespace: "dco-system", + }, + }, + }, + } +} + +func twoServers() []*teraskyv1alpha1.ConversionWebhookServer { + a := &teraskyv1alpha1.ConversionWebhookServer{} + a.Name = "srv-a" + b := &teraskyv1alpha1.ConversionWebhookServer{} + b.Name = "srv-b" + return []*teraskyv1alpha1.ConversionWebhookServer{a, b} +} + +// The losing half of a safe handover. The config has been reassigned to +// srv-b, but the XRD's conversion webhook still names srv-a's Service — +// so srv-a is still the endpoint the apiserver calls, and dropping the +// plan now would fail every read and write of the resource until the +// operator gets round to repointing it. +func TestReconcileOneXRD_KeepsServingWhileTheTargetStillPointsHere(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + pointXRDAt(xrd, "srv-a") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(xrd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true} + + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + entry, ok := r.Registry.Get("xfoos.example.org") + if !ok || entry.Router == nil { + t.Fatal("srv-a dropped the plan while the XRD still pointed its conversion webhook at srv-a; every read of the resource fails until the operator repoints it") + } +} + +// Once the operator has repointed the XRD, the handover is over — but not +// instantly. The apiserver refreshes a CRD's conversion configuration +// asynchronously after the write, so for a short window it is still +// calling srv-a; dropping the plan the moment the object changes answers +// those calls with a 503. The plan goes only after the drain. +// +// This is not theoretical. Before the drain existed, hack/e2e-reassign.sh +// caught exactly one failed write in 9,456 across three reassignments, +// with the registry-miss message. +func TestReconcileOneXRD_DrainsBeforeDroppingAHandedOverTarget(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + pointXRDAt(xrd, "srv-b") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + clock := newFakeClock() + c := newFakeClient(xrd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true, now: clock.Now} + r.Registry.Set("xfoos.example.org", &CompiledEntry{}) + + requeue, err := r.reconcileOneXRD(context.Background(), "cfg") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue != DefaultTargetDrainPeriod { + t.Fatalf("requeue = %s, want the drain period %s — without a requeue the plan would be held until the next watch event", + requeue, DefaultTargetDrainPeriod) + } + if _, ok := r.Registry.Get("xfoos.example.org"); !ok { + t.Fatal("srv-a dropped the plan immediately; the apiserver is still routing here for a moment after the repoint") + } + + // Part way through: still held. + clock.Advance(DefaultTargetDrainPeriod / 2) + requeue, err = r.reconcileOneXRD(context.Background(), "cfg") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue <= 0 { + t.Fatalf("requeue = %s half way through the drain, want the remainder", requeue) + } + if _, ok := r.Registry.Get("xfoos.example.org"); !ok { + t.Fatal("srv-a dropped the plan half way through the drain") + } + + // Past it: dropped. + clock.Advance(DefaultTargetDrainPeriod) + requeue, err = r.reconcileOneXRD(context.Background(), "cfg") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue != 0 { + t.Fatalf("requeue = %s after the drain elapsed, want none", requeue) + } + if _, ok := r.Registry.Get("xfoos.example.org"); ok { + t.Fatal("srv-a kept the plan after the drain elapsed; it would hold a plan for everything it has ever served") + } +} + +// A move that reverts mid-drain must not leave the target scheduled for +// removal — otherwise the next reconcile after the revert would drop a +// plan this replica has just been given back. +func TestReconcileOneXRD_DrainIsCancelledIfTheTargetComesBack(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + pointXRDAt(xrd, "srv-b") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + clock := newFakeClock() + c := newFakeClient(xrd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true, now: clock.Now} + r.Registry.Set("xfoos.example.org", &CompiledEntry{}) + + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("starting the drain: %v", err) + } + + // The move is abandoned: the config points back at srv-a. + live := getXRDConfigFromClient(t, r, "cfg") + live.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-a"} + if err := r.Update(context.Background(), live); err != nil { + t.Fatalf("reverting the move: %v", err) + } + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("reconcile after the revert: %v", err) + } + + // Long past the original deadline, and it must still be here. + clock.Advance(DefaultTargetDrainPeriod * 10) + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("reconcile after the drain would have elapsed: %v", err) + } + entry, ok := r.Registry.Get("xfoos.example.org") + if !ok || entry.Router == nil { + t.Fatal("a drain from an abandoned move fired anyway and dropped a target this replica owns") + } +} + +// An XRD that names nobody — never applied, or reverted — must not keep a +// plan alive on a server the resolver no longer assigns it to. +func TestReconcileOneXRD_DropsWhenTheTargetNamesNobody(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(xrd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true, TargetDrainPeriod: -1} + r.Registry.Set("xfoos.example.org", &CompiledEntry{}) + + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := r.Registry.Get("xfoos.example.org"); ok { + t.Fatal("kept a plan for a target that neither assigns to this server nor points at it") + } +} + +func TestReconcileOneCRD_KeepsServingWhileTheTargetStillPointsHere(t *testing.T) { + crd := establishedCRD("foos.example.org") + pointCRDAt(crd, "srv-a") + cfg := renameRuleCRDConfig("cfg", "foos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(crd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableCRDSupport: true} + + if _, err := r.reconcileOneCRD(context.Background(), "cfg"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + entry, ok := r.Registry.Get("foos.example.org") + if !ok || entry.Router == nil { + t.Fatal("srv-a dropped the plan while the CRD still pointed its conversion webhook at srv-a") + } +} + +func TestReconcileOneCRD_DrainsThenDropsAHandedOverTarget(t *testing.T) { + crd := establishedCRD("foos.example.org") + pointCRDAt(crd, "srv-b") + cfg := renameRuleCRDConfig("cfg", "foos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + clock := newFakeClock() + c := newFakeClient(crd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableCRDSupport: true, now: clock.Now} + r.Registry.Set("foos.example.org", &CompiledEntry{}) + + requeue, err := r.reconcileOneCRD(context.Background(), "cfg") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue != DefaultTargetDrainPeriod { + t.Fatalf("requeue = %s, want the drain period %s", requeue, DefaultTargetDrainPeriod) + } + if _, ok := r.Registry.Get("foos.example.org"); !ok { + t.Fatal("srv-a dropped the plan immediately after the CRD was repointed") + } + + clock.Advance(DefaultTargetDrainPeriod + time.Second) + if _, err := r.reconcileOneCRD(context.Background(), "cfg"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := r.Registry.Get("foos.example.org"); ok { + t.Fatal("srv-a kept the plan after the drain elapsed") + } +} + +// A deleted target cannot be pointing at anybody, so the grace this adds +// must not extend to one. +func TestReconcileOneXRD_DropsWhenTheTargetIsGone(t *testing.T) { + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true} + r.Registry.Set("xfoos.example.org", &CompiledEntry{}) + + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := r.Registry.Get("xfoos.example.org"); ok { + t.Fatal("kept a plan for a target that no longer exists and is not assigned here") + } +} + +// Not-assigned-and-target-missing must drop rather than record a failure: +// "the XRD is gone" is not this replica's problem to report when the +// config does not belong to it. +func TestReconcileOneXRD_MissingTargetNotOursRecordsNoFailure(t *testing.T) { + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true} + + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry, ok := r.Registry.Get("xfoos.example.org"); ok { + t.Fatalf("recorded %+v for a config this replica does not serve", entry) + } +} + +func TestPointsAtServer_Shapes(t *testing.T) { + if xrdPointsAtServer(nil, "srv-a") { + t.Error("a nil XRD points at nobody") + } + bare := establishedXRD("xfoos.example.org") + if xrdPointsAtServer(bare, "srv-a") { + t.Error("an XRD with no spec.conversion points at nobody") + } + if crdPointsAtServer(nil, "srv-a") || crdPointsAtServer(establishedCRD("foos.example.org"), "srv-a") { + t.Error("a CRD with no spec.conversion points at nobody") + } + pointed := establishedXRD("xfoos.example.org") + pointXRDAt(pointed, "srv-a") + if !xrdPointsAtServer(pointed, "srv-a") { + t.Error("an XRD pointed at srv-a should read as pointing at srv-a") + } + if xrdPointsAtServer(pointed, "srv-b") { + t.Error("an XRD pointed at srv-a must not read as pointing at srv-b") + } +} + +// getXRDConfigFromClient reads the live config back so a test can mutate +// and re-apply it through the same fake client the reconciler reads from. +func getXRDConfigFromClient(t *testing.T, r *Reconciler, name string) *teraskyv1alpha1.XRDConversionConfig { + t.Helper() + var cfg teraskyv1alpha1.XRDConversionConfig + if err := r.Get(context.Background(), types.NamespacedName{Name: name}, &cfg); err != nil { + t.Fatalf("getting XRDConversionConfig %q: %v", name, err) + } + return &cfg +} + +// A replica that holds no plan for the target has nothing to protect, and +// must not requeue itself every thirty seconds to remove something that is +// not there. On a replica serving none of a large fleet that would be one +// pointless timer per config. +func TestReconcileOneXRD_NoDrainWhenNothingIsHeld(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + pointXRDAt(xrd, "srv-b") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(xrd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true} + + requeue, err := r.reconcileOneXRD(context.Background(), "cfg") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue != 0 { + t.Fatalf("requeue = %s for a target this replica never held, want none", requeue) + } +} + +// A config deleted mid-drain must not leave its deadline behind, or the +// map grows by one entry for every config that ever churned. +func TestForgetConfig_ClearsAPendingDrain(t *testing.T) { + xrd := establishedXRD("xfoos.example.org") + pointXRDAt(xrd, "srv-b") + cfg := renameRuleXRDConfig("cfg", "xfoos.example.org") + cfg.Spec.WebhookServerRef = &teraskyv1alpha1.WebhookServerRef{Name: "srv-b"} + + servers := twoServers() + c := newFakeClient(xrd, cfg, servers[0], servers[1]).Build() + r := &Reconciler{Client: c, ServerName: "srv-a", Registry: NewRegistry(), EnableXRDSupport: true} + r.Registry.Set("xfoos.example.org", &CompiledEntry{}) + + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("starting the drain: %v", err) + } + r.mu.Lock() + pending := len(r.drainUntil) + r.mu.Unlock() + if pending != 1 { + t.Fatalf("expected one pending drain, got %d", pending) + } + + if err := r.Delete(context.Background(), cfg); err != nil { + t.Fatalf("deleting the config: %v", err) + } + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + t.Fatalf("reconcile after delete: %v", err) + } + r.mu.Lock() + pending = len(r.drainUntil) + r.mu.Unlock() + if pending != 0 { + t.Fatalf("a deleted config left %d drain deadline(s) behind", pending) + } +} diff --git a/internal/webhookserver/initialsync_bench_test.go b/internal/webhookserver/initialsync_bench_test.go new file mode 100644 index 0000000..d3a13de --- /dev/null +++ b/internal/webhookserver/initialsync_bench_test.go @@ -0,0 +1,140 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "context" + "fmt" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" + "github.com/terasky-oss/declarative-conversion-operator/pkg/xrdadapter" +) + +// leafySchema is an openAPIV3Schema with leaves spec.N string +// leaves, shaped the way pkg/engine's own compile benchmark shapes its +// fixture so the two curves are comparable. +func leafySchema(prefix string, leaves int) map[string]any { + props := make(map[string]any, leaves) + for i := 0; i < leaves; i++ { + props[fmt.Sprintf("%s%d", prefix, i)] = map[string]any{"type": "string"} + } + return map[string]any{"openAPIV3Schema": map[string]any{ + "type": "object", + "properties": map[string]any{ + "spec": map[string]any{"type": "object", "properties": props}, + }, + }} +} + +// leafyXRD is establishedXRD at an arbitrary schema size: hub v2 declares +// spec.fN, spoke v1 declares spec.gN, and the config below renames each +// pair. One rule per leaf is the common case the Phase 9 compile benchmark +// measures, so the per-target cost here is that benchmark's cost plus the +// reconcile around it. +func leafyXRD(name string, leaves int) *unstructured.Unstructured { + xrd := &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"name": name, "generation": int64(1)}, + "spec": map[string]any{ + "scope": "Namespaced", + "versions": []any{ + map[string]any{"name": "v2", "served": true, "referenceable": true, "schema": leafySchema("f", leaves)}, + map[string]any{"name": "v1", "served": true, "referenceable": false, "schema": leafySchema("g", leaves)}, + }, + }, + "status": map[string]any{ + "conditions": []any{map[string]any{"type": "Established", "status": "True"}}, + }, + }} + xrd.SetGroupVersionKind(xrdadapter.GroupVersionKind) + return xrd +} + +func leafyXRDConfig(name, targetXRD string, leaves int) *teraskyv1alpha1.XRDConversionConfig { + rules := make([]teraskyv1alpha1.ConversionRule, leaves) + for i := 0; i < leaves; i++ { + rules[i] = teraskyv1alpha1.ConversionRule{ + Strategy: teraskyv1alpha1.StrategyFieldRename, + FieldRename: &teraskyv1alpha1.FieldRenameParams{ + HubPath: fmt.Sprintf("spec.f%d", i), + SpokePath: fmt.Sprintf("spec.g%d", i), + }, + } + } + cfg := renameRuleXRDConfig(name, targetXRD) + cfg.Spec.Spokes[0].Rules = rules + return cfg +} + +// benchFleet is coldStartFleet without the deliberately-missing XRD: a +// cold-start benchmark should measure the work a healthy replica does, not +// the cost of a config nobody would leave broken. +func benchFleet(n, leaves int) []runtime.Object { + server := &teraskyv1alpha1.ConversionWebhookServer{} + server.Name = "srv" + server.Spec.Default = true + + objs := []runtime.Object{server} + for i := 0; i < n; i++ { + target := fmt.Sprintf("xfoos%d.example.org", i) + objs = append(objs, leafyXRD(target, leaves), leafyXRDConfig(fmt.Sprintf("cfg-%d", i), target, leaves)) + } + return objs +} + +// BenchmarkInitialSync is the cold-start curve: how long a replica spends +// compiling every assigned plan before it can serve, against the number of +// targets assigned to it and the size of the worker pool. +// +// The fake client stands in for an informer-backed cache, so the API-read +// term is understated relative to a real cluster and the compile term is +// the honest one. That is the right bias for this measurement: compile is +// what scales with the schemas, and it is what parallelising addresses. +// See docs/operations/capacity.md for the published numbers. +func BenchmarkInitialSync(b *testing.B) { + // 50 leaves per version is a modest real XRD — the Phase 9 ladder runs + // 10/100/1000 leaves for a single compile, and multiplying the top of + // that by a thousand targets would measure the benchmark harness's + // patience rather than anything operational. + const leaves = 50 + for _, targets := range []int{10, 100, 1000} { + objs := benchFleet(targets, leaves) + for _, workers := range []int{1, 0} { + name := fmt.Sprintf("targets=%d/workers=%d", targets, workers) + if workers == 0 { + name = fmt.Sprintf("targets=%d/workers=GOMAXPROCS", targets) + } + b.Run(name, func(b *testing.B) { + c := newFakeClient(objs...).Build() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r := &Reconciler{ + Client: c, ServerName: "srv", Registry: NewRegistry(), + EnableXRDSupport: true, InitialSyncWorkers: workers, + } + if _, err := r.InitialSync(context.Background()); err != nil { + b.Fatal(err) + } + } + }) + } + } +} diff --git a/internal/webhookserver/initialsync_test.go b/internal/webhookserver/initialsync_test.go new file mode 100644 index 0000000..6ee168b --- /dev/null +++ b/internal/webhookserver/initialsync_test.go @@ -0,0 +1,199 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" + "github.com/terasky-oss/declarative-conversion-operator/pkg/xrdadapter" +) + +// coldStartFleet builds n XRD targets and n matching configs, plus the one +// default ConversionWebhookServer they all resolve to. One of them names a +// target that does not exist, so the fixture covers the error path as well +// as the happy one — a parallel walk that lost a recorded failure would +// look identical to a serial one that never had it otherwise. +func coldStartFleet(n int) []runtime.Object { + server := &teraskyv1alpha1.ConversionWebhookServer{} + server.Name = "srv" + server.Spec.Default = true + + objs := []runtime.Object{server} + for i := 0; i < n; i++ { + target := fmt.Sprintf("xfoos%d.example.org", i) + objs = append(objs, renameRuleXRDConfig(fmt.Sprintf("cfg-%d", i), target)) + if i%10 == 3 { + continue // no XRD for this one: recordFailure path. + } + objs = append(objs, establishedXRD(target)) + } + return objs +} + +// registryFingerprint reduces a registry to the facts a caller can observe: +// which targets are present, whether each has a servable plan, and what +// error (if any) is recorded against it. +func registryFingerprint(r *Registry) []string { + snap := r.Snapshot() + out := make([]string, 0, len(snap)) + for name, entry := range snap { + servable := entry != nil && entry.Router != nil + lastErr := "" + if entry != nil { + lastErr = entry.LastError + } + out = append(out, fmt.Sprintf("%s servable=%t err=%q", name, servable, lastErr)) + } + sort.Strings(out) + return out +} + +func TestInitialSync_ParallelMatchesSerial(t *testing.T) { + const targets = 40 + objs := coldStartFleet(targets) + + serial := &Reconciler{ + Client: newFakeClient(objs...).Build(), ServerName: "srv", + Registry: NewRegistry(), EnableXRDSupport: true, InitialSyncWorkers: 1, + } + parallel := &Reconciler{ + Client: newFakeClient(objs...).Build(), ServerName: "srv", + Registry: NewRegistry(), EnableXRDSupport: true, InitialSyncWorkers: 8, + } + + serialStats, err := serial.InitialSync(context.Background()) + if err != nil { + t.Fatalf("serial sync: %v", err) + } + parallelStats, err := parallel.InitialSync(context.Background()) + if err != nil { + t.Fatalf("parallel sync: %v", err) + } + + if serialStats.Targets != targets || parallelStats.Targets != targets { + t.Fatalf("expected both walks to report %d targets, got serial=%d parallel=%d", + targets, serialStats.Targets, parallelStats.Targets) + } + + want, got := registryFingerprint(serial.Registry), registryFingerprint(parallel.Registry) + if len(want) != len(got) { + t.Fatalf("registry size differs: serial=%d parallel=%d", len(want), len(got)) + } + for i := range want { + if want[i] != got[i] { + t.Fatalf("registry entry %d differs:\n serial: %s\n parallel: %s", i, want[i], got[i]) + } + } + // Guard against the fixture quietly becoming all-happy-path: if no + // entry ever records a failure, the comparison above proves less than + // it looks like it does. + failures := 0 + for _, line := range want { + if !strings.HasSuffix(line, `err=""`) { + failures++ + } + } + if failures == 0 { + t.Fatal("expected the fleet fixture to include at least one failing target") + } +} + +func TestInitialSync_PublishesColdStartMetrics(t *testing.T) { + objs := coldStartFleet(5) + m := newTestMetrics() + r := &Reconciler{ + Client: newFakeClient(objs...).Build(), ServerName: "srv", + Registry: NewRegistry(), Metrics: m, EnableXRDSupport: true, + } + + stats, err := r.InitialSync(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := testutil.ToFloat64(m.InitialSyncTargets); got != 5 { + t.Fatalf("dco_webhook_initial_sync_targets = %v, want 5", got) + } + if got := testutil.ToFloat64(m.InitialSyncDuration); got <= 0 { + t.Fatalf("dco_webhook_initial_sync_duration_seconds = %v, want a positive elapsed time", got) + } + if stats.Duration <= 0 { + t.Fatalf("stats.Duration = %v, want a positive elapsed time", stats.Duration) + } + // The per-target gauge refresh is suppressed during the walk; the one + // sync at the end is what has to leave the gauges correct. + if got := testutil.ToFloat64(m.RegistrySize); int(got) != r.Registry.Len() { + t.Fatalf("dco_webhook_registry_size = %v after the bulk pass, want %d", got, r.Registry.Len()) + } +} + +// An infrastructure failure during the startup pass has to come back as an +// error, because readiness is gated on this returning cleanly. A bad +// config does not: reconcileOne* records that into the registry and +// returns nil, since retrying cannot fix it. +func TestInitialSync_ReportsInfrastructureErrors(t *testing.T) { + objs := coldStartFleet(6) + failing := newFakeClient(objs...).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if u, ok := obj.(*unstructured.Unstructured); ok && u.GroupVersionKind() == xrdadapter.GroupVersionKind && key.Name == "xfoos1.example.org" { + return errors.New("the apiserver said no") + } + return c.Get(ctx, key, obj, opts...) + }, + }).Build() + + r := &Reconciler{Client: failing, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: true} + _, err := r.InitialSync(context.Background()) + if err == nil { + t.Fatal("expected a failed target read to be reported; readiness is gated on this returning cleanly") + } + if !strings.Contains(err.Error(), "the apiserver said no") { + t.Fatalf("the error should carry the cause: %v", err) + } + + // And the other targets still loaded: one bad read must not abandon + // the pass. + if got := r.Registry.Len(); got < 4 { + t.Fatalf("registry holds %d entries; the other targets should still have compiled", got) + } +} + +// A config whose target does not exist is a configuration problem, not an +// infrastructure one. It is recorded against the registry and must not +// keep the replica from reporting ready — otherwise one broken config +// would hold the whole replica out of service. +func TestInitialSync_BadConfigIsNotAnError(t *testing.T) { + // coldStartFleet deliberately leaves one config with no XRD. + r := &Reconciler{ + Client: newFakeClient(coldStartFleet(12)...).Build(), ServerName: "srv", + Registry: NewRegistry(), EnableXRDSupport: true, + } + if _, err := r.InitialSync(context.Background()); err != nil { + t.Fatalf("a config with a missing target must not fail the sync: %v", err) + } +} diff --git a/internal/webhookserver/memory_bench_test.go b/internal/webhookserver/memory_bench_test.go new file mode 100644 index 0000000..2d7d71b --- /dev/null +++ b/internal/webhookserver/memory_bench_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "context" + "fmt" + "runtime" + "sync/atomic" + "testing" + "time" +) + +// retainedBytes reports live heap bytes after a settling GC. Two cycles, +// because the first can leave objects that only became unreachable during +// it still counted. +func retainedBytes() uint64 { + runtime.GC() + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + return ms.HeapAlloc +} + +// BenchmarkRegistryRetained is the registry's own footprint at scale: what +// a replica holds once every assigned plan is compiled and nothing else is +// happening. This is the term that decides how many targets fit in a +// memory limit — together with the informer cache, which is measured +// end-to-end by hack/measure-cache-memory.sh rather than here, because it +// is a property of the cluster's CRDs and not of this code. +// +// The whole fleet is built inside one b.N iteration, so run it with +// -benchtime=1x; the reported B/target is what to read. +func BenchmarkRegistryRetained(b *testing.B) { + const leaves = 50 + for _, targets := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("targets=%d", targets), func(b *testing.B) { + objs := benchFleet(targets, leaves) + c := newFakeClient(objs...).Build() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + before := retainedBytes() + b.StartTimer() + + r := &Reconciler{ + Client: c, ServerName: "srv", Registry: NewRegistry(), + EnableXRDSupport: true, + } + if _, err := r.InitialSync(context.Background()); err != nil { + b.Fatal(err) + } + + b.StopTimer() + after := retainedBytes() + if got := r.Registry.Len(); got != targets { + b.Fatalf("registry holds %d entries, want %d", got, targets) + } + b.ReportMetric(float64(after-before)/float64(targets), "B/target") + runtime.KeepAlive(r) + b.StartTimer() + } + }) + } +} + +// BenchmarkInitialSyncPeak is the transient side: how far above the steady +// registry footprint a replica goes while compiling everything at once. It +// is the number a memory limit has to clear, because the OOM killer does +// not wait for the GC. +// +// Sampling ReadMemStats stops the world, so the sampled peak is an +// approximation that also slows the run down slightly. That bias is +// conservative in the right direction — a slower run gives the GC more +// chances to run, so a sampled peak understates rather than overstates. +func BenchmarkInitialSyncPeak(b *testing.B) { + const leaves = 50 + for _, targets := range []int{100, 1000} { + b.Run(fmt.Sprintf("targets=%d", targets), func(b *testing.B) { + objs := benchFleet(targets, leaves) + c := newFakeClient(objs...).Build() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + base := retainedBytes() + var peak atomic.Uint64 + done := make(chan struct{}) + go func() { + var ms runtime.MemStats + ticker := time.NewTicker(2 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + runtime.ReadMemStats(&ms) + for { + cur := peak.Load() + if ms.HeapAlloc <= cur || peak.CompareAndSwap(cur, ms.HeapAlloc) { + break + } + } + } + } + }() + b.StartTimer() + + r := &Reconciler{ + Client: c, ServerName: "srv", Registry: NewRegistry(), + EnableXRDSupport: true, + } + if _, err := r.InitialSync(context.Background()); err != nil { + b.Fatal(err) + } + + b.StopTimer() + close(done) + steady := retainedBytes() + b.ReportMetric(float64(peak.Load()-base), "B/peak") + b.ReportMetric(float64(steady-base), "B/steady") + runtime.KeepAlive(r) + b.StartTimer() + } + }) + } +} diff --git a/internal/webhookserver/metrics.go b/internal/webhookserver/metrics.go index 1ae41c4..d69ff17 100755 --- a/internal/webhookserver/metrics.go +++ b/internal/webhookserver/metrics.go @@ -21,8 +21,30 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) +// CombinedGatherer pairs this binary's dedicated metric registry with +// controller-runtime's package-global one. +// +// cmd/webhook-server deliberately does not use controller-runtime's own +// metrics server — the conversion path must not share a listener with +// anything else — which used to mean the registry reconciler was the one +// controller in the system with no workqueue depth, no queue latency and +// no reconcile error counter exposed anywhere. Gathering both registries +// from the one handler fixes that without giving controller-runtime a +// listener of its own. +// +// The two registries do not overlap: the dedicated one carries the +// dco_webhook_* series plus the Go and process collectors, and +// controller-runtime's carries workqueue_*, controller_runtime_* and +// rest_client_*. prometheus.Gatherers fails the whole scrape on a +// duplicate metric name, so that separation is asserted by a test rather +// than assumed. +func CombinedGatherer(own prometheus.Gatherer) prometheus.Gatherer { + return prometheus.Gatherers{own, ctrlmetrics.Registry} +} + // Metrics is the webhook server's Prometheus metric set, registered on a // dedicated registry (not the global default) so cmd/webhook-server has // full control over exactly what /metrics exposes. @@ -41,6 +63,14 @@ type Metrics struct { RegistryCompileErr *prometheus.CounterVec Ready prometheus.Gauge + // InitialSyncDuration and InitialSyncTargets describe the cold start: + // how long this replica spent compiling every assigned plan before it + // reported ready, and how many targets that was. Both are written + // exactly once, immediately before SetReady(true) — they are the + // startup budget an operator sizes a startupProbe against. + InitialSyncDuration prometheus.Gauge + InitialSyncTargets prometheus.Gauge + // gatherer is the registry metrics were registered on, used by // PlainMux's /metrics handler. Must be the underlying Gatherer when // reg is a WrapRegistererWith* wrapper (those implement Registerer only). @@ -111,9 +141,17 @@ func NewMetrics(reg prometheus.Registerer, gatherer prometheus.Gatherer) *Metric Name: "dco_webhook_ready", Help: "1 if this replica's registry has completed its initial sync and is serving traffic.", }), + InitialSyncDuration: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "dco_webhook_initial_sync_duration_seconds", + Help: "Wall-clock seconds this replica spent in the initial registry sync — every assigned plan compiled — before it reported ready. Written once, immediately before readiness; it reads 0 on a replica that is still cold, which dco_webhook_ready disambiguates.", + }), + InitialSyncTargets: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "dco_webhook_initial_sync_targets", + Help: "Number of conversion configs this replica walked during its initial sync. Divide the duration by this to get the per-target cold-start cost for your schemas.", + }), gatherer: gatherer, } - reg.MustRegister(m.ReviewDuration, m.ReviewRequestsTotal, m.ObjectsTotal, m.ObjectDuration, m.BatchSize, m.LossyTotal, m.PanicsTotal, m.RegistrySize, m.RegistryEntryLoaded, m.RegistryLastReload, m.RegistryReloadTotal, m.RegistryCompileErr, m.Ready) + reg.MustRegister(m.ReviewDuration, m.ReviewRequestsTotal, m.ObjectsTotal, m.ObjectDuration, m.BatchSize, m.LossyTotal, m.PanicsTotal, m.RegistrySize, m.RegistryEntryLoaded, m.RegistryLastReload, m.RegistryReloadTotal, m.RegistryCompileErr, m.Ready, m.InitialSyncDuration, m.InitialSyncTargets) return m } diff --git a/internal/webhookserver/metrics_test.go b/internal/webhookserver/metrics_test.go index 41f06d0..6a5eb9b 100644 --- a/internal/webhookserver/metrics_test.go +++ b/internal/webhookserver/metrics_test.go @@ -20,7 +20,9 @@ import ( "context" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + "k8s.io/client-go/util/workqueue" teraskyv1alpha1 "github.com/terasky-oss/declarative-conversion-operator/api/v1alpha1" "github.com/terasky-oss/declarative-conversion-operator/pkg/engine" @@ -67,7 +69,7 @@ func TestReconcileOneXRD_UpdatesRegistryReadinessMetrics(t *testing.T) { EnableXRDSupport: true, Metrics: metrics, } - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } if got := testutil.ToFloat64(metrics.RegistrySize); got != 1 { @@ -77,3 +79,60 @@ func TestReconcileOneXRD_UpdatesRegistryReadinessMetrics(t *testing.T) { t.Fatalf("expected entry_loaded=1 after compile, got %v", got) } } + +// The dedicated registry and controller-runtime's package-global one are +// served from the same handler. prometheus.Gatherers fails the entire +// scrape if the two ever register the same metric name, which would take +// out every dco_webhook_* series along with the controller ones — so the +// separation is asserted rather than assumed. +func TestCombinedGatherer_NoDuplicateSeries(t *testing.T) { + reg := prometheus.NewRegistry() + NewMetrics(reg, reg) + + // A metric family with no series is not gathered at all, and + // controller-runtime's workqueue vectors have no series until a queue + // exists. Creating one materialises them through the global metrics + // provider controller-runtime installs — the same path a real + // controller takes. + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{Name: "combined-gatherer-test"}, + ) + defer q.ShutDown() + q.Add("x") + + families, err := CombinedGatherer(reg).Gather() + if err != nil { + t.Fatalf("gathering both registries: %v", err) + } + + names := map[string]int{} + for _, f := range families { + names[f.GetName()]++ + } + for name, n := range names { + if n > 1 { + t.Errorf("metric family %q appears %d times across the two registries", name, n) + } + } + if _, ok := names["dco_webhook_registry_size"]; !ok { + t.Error("the dedicated registry's own series are missing from the combined gather") + } + // controller-runtime registers these from an init(), so their absence + // would mean the combined gatherer is not reaching that registry at + // all — which is the whole point of it. + if _, ok := names["workqueue_depth"]; !ok { + t.Error("workqueue_depth is missing: the webhook-server's reconcile loop still has no queue-depth signal") + } + // cmd/webhook-server deliberately does not register its own Go and + // process collectors, because controller-runtime's registry already + // has them and a second copy would be the duplicate checked above. + // That makes their presence here load-bearing rather than incidental: + // without them a replica's memory footprint is unmeasurable from + // outside the pod. + for _, name := range []string{"go_goroutines", "process_resident_memory_bytes"} { + if _, ok := names[name]; !ok { + t.Errorf("%s is missing: the process serving /metrics is invisible on it", name) + } + } +} diff --git a/internal/webhookserver/publisher.go b/internal/webhookserver/publisher.go new file mode 100644 index 0000000..bdab793 --- /dev/null +++ b/internal/webhookserver/publisher.go @@ -0,0 +1,247 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "context" + "fmt" + "sort" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/terasky-oss/declarative-conversion-operator/internal/servedtargets" +) + +// TargetPublisher writes this replica's servable target set into its own +// Lease, so the operator can answer "is every replica of this instance +// already serving target X?" without ever calling a pod. +// +// That question is what makes a safe handover possible. When a config's +// assignment moves from one ConversionWebhookServer to another, the +// operator must not repoint the target's conversion webhook at the new +// instance until the new instance can already serve it — otherwise there +// is a window in which the apiserver sends ConversionReviews to a replica +// that has not compiled the plan yet, and every read and write of that +// resource fails. +// +// Publishing is best-effort by construction. A replica that cannot write +// its Lease still serves conversions perfectly well; what it loses is the +// ability to have work moved *onto* it safely, which the operator reports +// rather than works around. +type TargetPublisher struct { + Client client.Client + Registry *Registry + ServerName string + // Namespace, PodName and PodUID come from the downward API. PodUID + // is what makes the Lease a child of this pod, so it is collected + // with it — without an owner reference, every replaced replica would + // leave a Lease behind claiming to serve things. + Namespace string + PodName string + PodUID types.UID + // Heartbeat defaults to servedtargets.Heartbeat. + Heartbeat time.Duration + + notify chan struct{} +} + +// Enabled reports whether this publisher has everything it needs, +// including PodUID. The downward-API values are absent on an older +// webhook-server Deployment the operator has not yet re-applied, and +// publishing nothing at all is better than publishing a Lease that no +// reader can attribute to a pod. +// +// PodUID is required rather than optional because it is the +// ownerReference: a Lease without one outlives the pod that wrote it, +// stops renewing, and sits in the namespace as a report from a replica +// that no longer exists. Readers do discount it after StaleAfter, but a +// Lease nothing ever collects is a leak, and during the window before it +// goes stale it is a report attributable to nobody. +func (p *TargetPublisher) Enabled() bool { + return p != nil && p.Client != nil && p.Registry != nil && + p.ServerName != "" && p.Namespace != "" && p.PodName != "" && p.PodUID != "" +} + +// Notify asks for an out-of-band publish, called after the registry +// changes. Non-blocking and coalescing: the channel holds one token, so a +// burst of registry writes during the initial sync produces one extra +// publish rather than hundreds. +func (p *TargetPublisher) Notify() { + if p == nil || p.notify == nil { + return + } + select { + case p.notify <- struct{}{}: + default: + } +} + +// Init prepares the notification channel. Called before the Reconciler +// starts so a Notify during the initial sync is not dropped on the floor. +func (p *TargetPublisher) Init() { + if p != nil && p.notify == nil { + p.notify = make(chan struct{}, 1) + } +} + +// Run publishes on every notification and at least once per heartbeat, +// until ctx is done. It never returns an error: a failed publish is +// logged and retried on the next tick, because the alternative — taking +// the process down over a Lease — would turn a bookkeeping problem into a +// conversion outage. +func (p *TargetPublisher) Run(ctx context.Context) { + if !p.Enabled() { + return + } + p.Init() + logger := ctrl.LoggerFrom(ctx).WithName("served-targets") + + beat := p.Heartbeat + if beat <= 0 { + beat = servedtargets.Heartbeat + } + ticker := time.NewTicker(beat) + defer ticker.Stop() + + publish := func() { + if err := p.Publish(ctx); err != nil && ctx.Err() == nil { + logger.Error(err, "unable to publish served targets; work cannot be moved onto this instance until it succeeds", + "lease", servedtargets.LeaseName(p.PodName), "namespace", p.Namespace) + } + } + publish() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + publish() + case <-p.notify: + publish() + } + } +} + +// Publish upserts this replica's Lease. Exported for tests and for the +// one synchronous call cmd/webhook-server makes at startup, so a replica +// that has just finished its initial sync is immediately eligible to +// receive moved work rather than waiting out a heartbeat. +func (p *TargetPublisher) Publish(ctx context.Context) error { + if !p.Enabled() { + return nil + } + encoded, truncated := servedtargets.Encode(p.servableTargets()) + + name := servedtargets.LeaseName(p.PodName) + var lease coordinationv1.Lease + err := p.Client.Get(ctx, types.NamespacedName{Name: name, Namespace: p.Namespace}, &lease) + switch { + case apierrors.IsNotFound(err): + lease = coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: p.Namespace}} + p.stamp(&lease, encoded, truncated) + if err := p.Client.Create(ctx, &lease); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("creating served-targets lease: %w", err) + } + return nil + case err != nil: + return fmt.Errorf("getting served-targets lease: %w", err) + } + + p.stamp(&lease, encoded, truncated) + if err := p.Client.Update(ctx, &lease); err != nil { + return fmt.Errorf("updating served-targets lease: %w", err) + } + return nil +} + +// stamp fills in everything the readers depend on. renewTime is refreshed +// on every publish including the ones where the target set has not +// changed — that is the whole point of the heartbeat, since a stale +// renewTime is how a reader tells a wedged replica from a working one. +func (p *TargetPublisher) stamp(lease *coordinationv1.Lease, encoded string, truncated bool) { + if lease.Labels == nil { + lease.Labels = map[string]string{} + } + lease.Labels[servedtargets.WebhookServerLabel] = p.ServerName + lease.Labels[managedByLabel] = managedByValue + + if lease.Annotations == nil { + lease.Annotations = map[string]string{} + } + if truncated { + // Publishing a partial set would be worse than publishing none: + // a reader cannot tell a missing name from an omitted one, and + // would conclude a served target is unserved. + delete(lease.Annotations, servedtargets.TargetsAnnotation) + lease.Annotations[servedtargets.TruncatedAnnotation] = "true" + } else { + lease.Annotations[servedtargets.TargetsAnnotation] = encoded + delete(lease.Annotations, servedtargets.TruncatedAnnotation) + } + + // Enabled() guarantees PodUID, so every Lease this writes is owned by + // the pod that wrote it and is collected with it. + if len(lease.OwnerReferences) == 0 { + lease.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "v1", + Kind: "Pod", + Name: p.PodName, + UID: p.PodUID, + }} + } + + holder := p.PodName + seconds := int32(servedtargets.LeaseDuration / time.Second) + now := metav1.NewMicroTime(time.Now()) + lease.Spec.HolderIdentity = &holder + lease.Spec.LeaseDurationSeconds = &seconds + lease.Spec.RenewTime = &now + if lease.Spec.AcquireTime == nil { + lease.Spec.AcquireTime = &now + } +} + +// servableTargets is the registry filtered to entries that can actually +// answer a ConversionReview. An error-only placeholder is in the registry +// but has no Router, so it is exactly the kind of entry that must not +// count as "this instance can serve it". +func (p *TargetPublisher) servableTargets() []string { + snap := p.Registry.Snapshot() + out := make([]string, 0, len(snap)) + for name, entry := range snap { + if entry != nil && entry.Router != nil { + out = append(out, name) + } + } + sort.Strings(out) + return out +} + +// managedByLabel/managedByValue mirror internal/controller's constants of +// the same name. Duplicated rather than imported for the same reason the +// target indexes are: cmd/webhook-server is a separate binary that should +// not link the operator's controller package. +const ( + managedByLabel = "app.kubernetes.io/managed-by" + managedByValue = "declarative-conversion-operator" +) diff --git a/internal/webhookserver/publisher_test.go b/internal/webhookserver/publisher_test.go new file mode 100644 index 0000000..cc93f30 --- /dev/null +++ b/internal/webhookserver/publisher_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookserver + +import ( + "context" + "reflect" + "testing" + + coordinationv1 "k8s.io/api/coordination/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/terasky-oss/declarative-conversion-operator/internal/servedtargets" + "github.com/terasky-oss/declarative-conversion-operator/pkg/engine" +) + +func newTestPublisher(reg *Registry) *TargetPublisher { + return &TargetPublisher{ + Client: newFakeClient().Build(), + Registry: reg, + ServerName: "srv", + Namespace: "dco-system", + PodName: "srv-webhook-server-abc123", + PodUID: types.UID("uid-1"), + } +} + +func readLease(t *testing.T, p *TargetPublisher) *coordinationv1.Lease { + t.Helper() + var l coordinationv1.Lease + key := types.NamespacedName{Name: servedtargets.LeaseName(p.PodName), Namespace: p.Namespace} + if err := p.Client.Get(context.Background(), key, &l); err != nil { + t.Fatalf("getting the published lease: %v", err) + } + return &l +} + +func TestPublish_CreatesThenUpdates(t *testing.T) { + reg := NewRegistry() + reg.Set("xfoos.example.org", &CompiledEntry{Router: &engine.Router{Hub: "v2"}}) + p := newTestPublisher(reg) + + if err := p.Publish(context.Background()); err != nil { + t.Fatalf("first publish: %v", err) + } + l := readLease(t, p) + if l.Labels[servedtargets.WebhookServerLabel] != "srv" { + t.Fatalf("lease labels = %v; without the server label the operator's informer never sees it", l.Labels) + } + if l.Labels[managedByLabel] != managedByValue { + t.Fatalf("lease labels = %v; without the managed-by label the manager's scoped Lease cache never sees it", l.Labels) + } + if len(l.OwnerReferences) != 1 || l.OwnerReferences[0].UID != "uid-1" { + t.Fatalf("owner references = %v; without one, a replaced replica leaves a Lease behind claiming to serve things", l.OwnerReferences) + } + if l.Spec.RenewTime == nil { + t.Fatal("renewTime is unset, so readers cannot tell a wedged replica from a working one") + } + got, err := servedtargets.Decode(l.Annotations[servedtargets.TargetsAnnotation]) + if err != nil { + t.Fatalf("decode: %v", err) + } + if !reflect.DeepEqual(got, []string{"xfoos.example.org"}) { + t.Fatalf("published %v, want the one servable target", got) + } + + // Second publish must update in place rather than fail on an + // already-existing object. + reg.Set("bars.example.org", &CompiledEntry{Router: &engine.Router{Hub: "v2"}}) + if err := p.Publish(context.Background()); err != nil { + t.Fatalf("second publish: %v", err) + } + got, _ = servedtargets.Decode(readLease(t, p).Annotations[servedtargets.TargetsAnnotation]) + if !reflect.DeepEqual(got, []string{"bars.example.org", "xfoos.example.org"}) { + t.Fatalf("published %v after the second target compiled", got) + } +} + +// An error-only registry entry has no Router: the replica knows about the +// target and cannot serve it. Publishing it would tell the operator the +// handover is safe when it is not. +func TestPublish_ExcludesEntriesWithNoCompiledPlan(t *testing.T) { + reg := NewRegistry() + reg.Set("good.example.org", &CompiledEntry{Router: &engine.Router{Hub: "v2"}}) + reg.RecordError("broken.example.org", "analysis failed") + p := newTestPublisher(reg) + + if err := p.Publish(context.Background()); err != nil { + t.Fatalf("publish: %v", err) + } + got, _ := servedtargets.Decode(readLease(t, p).Annotations[servedtargets.TargetsAnnotation]) + if !reflect.DeepEqual(got, []string{"good.example.org"}) { + t.Fatalf("published %v, want only the target with a compiled plan", got) + } +} + +func TestPublisher_DisabledWithoutAnIdentity(t *testing.T) { + // PodUID counts as identity too: without it the Lease has no owner + // reference, so it outlives the pod that wrote it. + withoutUID := newTestPublisher(NewRegistry()) + withoutUID.PodUID = "" + if withoutUID.Enabled() { + t.Fatal("a publisher with no pod UID must be disabled: its Lease would never be garbage-collected") + } + + p := newTestPublisher(NewRegistry()) + p.PodName = "" + if p.Enabled() { + t.Fatal("a publisher with no pod name must be disabled: its Lease could not be attributed to a replica") + } + // And must be inert rather than panicking or writing something wrong. + if err := p.Publish(context.Background()); err != nil { + t.Fatalf("a disabled publisher must be a no-op, got %v", err) + } + var l coordinationv1.Lease + err := p.Client.Get(context.Background(), types.NamespacedName{Name: servedtargets.LeaseName(""), Namespace: p.Namespace}, &l) + if err == nil { + t.Fatal("a disabled publisher wrote a Lease") + } +} + +func TestPublisher_NotifyIsNonBlockingAndCoalescing(t *testing.T) { + p := newTestPublisher(NewRegistry()) + p.Init() + // More notifications than the channel can hold. A blocking send here + // would deadlock the reconcile loop that calls it. + for i := 0; i < 100; i++ { + p.Notify() + } + if got := len(p.notify); got != 1 { + t.Fatalf("notify channel holds %d tokens, want 1 — a burst must coalesce", got) + } +} + +func TestPublisher_NilIsInert(t *testing.T) { + var p *TargetPublisher + p.Notify() // must not panic: the Reconciler calls this unconditionally. + if p.Enabled() { + t.Fatal("a nil publisher must not report itself enabled") + } +} diff --git a/internal/webhookserver/reconciler.go b/internal/webhookserver/reconciler.go index 059f21e..714b476 100644 --- a/internal/webhookserver/reconciler.go +++ b/internal/webhookserver/reconciler.go @@ -18,8 +18,11 @@ package webhookserver import ( "context" + "errors" "fmt" + "runtime" "sync" + "sync/atomic" "time" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -86,9 +89,33 @@ type Reconciler struct { // fatal at startup on a cluster without Crossplane installed. EnableXRDSupport bool EnableCRDSupport bool + // Publisher, when set, is told after every registry change so this + // replica's served-target Lease keeps up with what it can actually + // serve. Nil in tests and on a replica with no downward-API + // identity; a nil Publisher's Notify is a no-op. + Publisher *TargetPublisher + // InitialSyncWorkers bounds the parallelism of InitialSync's + // startup pass. Zero means DefaultInitialSyncWorkers(). It has no + // effect on the watch-driven path, whose concurrency is + // controller-runtime's to decide. + InitialSyncWorkers int + // TargetDrainPeriod is how long this replica keeps serving a target + // after the target has stopped naming it. Zero means + // DefaultTargetDrainPeriod; see handover.go for why it is not zero. + TargetDrainPeriod time.Duration + // now is time.Now, overridden in tests that need to advance the drain + // clock without sleeping through it. + now func() time.Time + + // bulkSync suppresses the per-target registry gauge refresh while a + // bulk pass (InitialSync) is running; see syncRegistryMetrics. + bulkSync atomic.Bool mu sync.Mutex configToTarget map[string]string + // drainUntil holds, per config key, the moment this replica may stop + // serving a target it no longer owns. See handover.go. + drainUntil map[string]time.Time } // reconcileXRD/reconcileCRD are the two watch-driven entry points, @@ -96,17 +123,19 @@ type Reconciler struct { // controller-runtime has no notion of "one Reconciler, two watched // types" — each controller needs its own entry point). func (r *Reconciler) reconcileXRD(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { - if err := r.reconcileOneXRD(ctx, req.Name); err != nil { + requeue, err := r.reconcileOneXRD(ctx, req.Name) + if err != nil { return ctrl.Result{}, err } - return ctrl.Result{}, nil + return ctrl.Result{RequeueAfter: requeue}, nil } func (r *Reconciler) reconcileCRD(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { - if err := r.reconcileOneCRD(ctx, req.Name); err != nil { + requeue, err := r.reconcileOneCRD(ctx, req.Name) + if err != nil { return ctrl.Result{}, err } - return ctrl.Result{}, nil + return ctrl.Result{RequeueAfter: requeue}, nil } func (r *Reconciler) ensureConfigToTarget() { @@ -114,6 +143,71 @@ func (r *Reconciler) ensureConfigToTarget() { if r.configToTarget == nil { r.configToTarget = map[string]string{} } + if r.drainUntil == nil { + r.drainUntil = map[string]time.Time{} + } + r.mu.Unlock() +} + +// timeNow is time.Now unless a test has overridden it. +func (r *Reconciler) timeNow() time.Time { + if r.now != nil { + return r.now() + } + return time.Now() +} + +// drainRemaining decides whether this replica may drop its plan for a +// target that no longer names it, and how long to wait if not. +// +// See handover.go: the apiserver refreshes a CRD's conversion +// configuration asynchronously after the write that changed it, so for a +// short window after the operator repoints a target the apiserver is still +// calling this replica. Dropping the plan the instant the object changes +// answers those calls with a 503, which the apiserver reports as a failed +// read or write on a resource that was merely being rebalanced. +// +// Returns 0 when the drain has elapsed (or there is nothing to drain), and +// the remaining time otherwise, which the caller turns into a requeue. +func (r *Reconciler) drainRemaining(key, targetName string) time.Duration { + period := r.TargetDrainPeriod + if period == 0 { + period = DefaultTargetDrainPeriod + } + if period < 0 { + return 0 + } + // Nothing to protect: this replica has no plan for the target, so + // there is no window in which it could answer wrongly. Requeueing for + // thirty seconds to remove something that is not there would be pure + // churn — and on a replica that serves none of a large fleet, it + // would be that churn once per config. + if _, held := r.Registry.Get(targetName); !held { + r.cancelDrain(key) + return 0 + } + now := r.timeNow() + + r.mu.Lock() + defer r.mu.Unlock() + deadline, started := r.drainUntil[key] + if !started { + r.drainUntil[key] = now.Add(period) + return period + } + if remaining := deadline.Sub(now); remaining > 0 { + return remaining + } + delete(r.drainUntil, key) + return 0 +} + +// cancelDrain forgets a pending removal, called whenever the target turns +// out to still be this replica's after all — a rebalance that reverted, or +// a move that was abandoned. +func (r *Reconciler) cancelDrain(key string) { + r.mu.Lock() + delete(r.drainUntil, key) r.mu.Unlock() } @@ -128,7 +222,7 @@ func configKey(kind, name string) string { return kind + "/" + name } // pass. It returns an error only for transient infrastructure failures // worth an automatic retry; business-logic failures are recorded into the // Registry instead. -func (r *Reconciler) reconcileOneXRD(ctx context.Context, name string) error { +func (r *Reconciler) reconcileOneXRD(ctx context.Context, name string) (time.Duration, error) { r.ensureConfigToTarget() key := configKey("xrd", name) @@ -136,61 +230,78 @@ func (r *Reconciler) reconcileOneXRD(ctx context.Context, name string) error { err := r.Get(ctx, types.NamespacedName{Name: name}, &cfg) if apierrors.IsNotFound(err) { r.forgetConfig(key) - return nil + return 0, nil } if err != nil { - return fmt.Errorf("getting XRDConversionConfig %q: %w", name, err) + return 0, fmt.Errorf("getting XRDConversionConfig %q: %w", name, err) } if !cfg.DeletionTimestamp.IsZero() { r.forgetConfig(key) - return nil + return 0, nil } r.rememberConfig(key, cfg.Spec.TargetXRD.Name) var servers teraskyv1alpha1.ConversionWebhookServerList if err := r.List(ctx, &servers); err != nil { - return fmt.Errorf("listing ConversionWebhookServers: %w", err) - } - assigned, err := assign.ResolveAssignment(&cfg, servers.Items) - if err != nil || assigned != r.ServerName { - r.Registry.Remove(cfg.Spec.TargetXRD.Name) - if r.Metrics != nil { - r.Metrics.SyncRegistryMetrics(r.Registry) - } - return nil + return 0, fmt.Errorf("listing ConversionWebhookServers: %w", err) } + assigned, assignErr := assign.ResolveAssignment(&cfg, servers.Items) + // An unresolvable assignment is a bad config, not a transient failure: + // retrying cannot fix it, and this replica does not serve the target + // on that basis either way. + mine := assignErr == nil && assigned == r.ServerName xrd := &unstructured.Unstructured{} xrd.SetGroupVersionKind(xrdadapter.GroupVersionKind) if err := r.Get(ctx, types.NamespacedName{Name: cfg.Spec.TargetXRD.Name}, xrd); err != nil { if apierrors.IsNotFound(err) { + if !mine { + // A deleted target cannot be pointing at anybody, so + // there is nothing for a drain to protect. + r.dropTarget(key, cfg.Spec.TargetXRD.Name) + return 0, nil + } r.recordFailure(cfg.Spec.TargetXRD.Name, "XRDNotFound", fmt.Sprintf("target XRD %q not found", cfg.Spec.TargetXRD.Name)) - return nil + return 0, nil } - return fmt.Errorf("getting target XRD %q: %w", cfg.Spec.TargetXRD.Name, err) + return 0, fmt.Errorf("getting target XRD %q: %w", cfg.Spec.TargetXRD.Name, err) } + // Not ours by assignment, and the XRD no longer points its conversion + // webhook here either: the handover is over, bar the drain. See + // handover.go for why both clauses exist and why the drain does. + if !mine && !xrdPointsAtServer(xrd, r.ServerName) { + if wait := r.drainRemaining(key, cfg.Spec.TargetXRD.Name); wait > 0 { + return wait, nil + } + r.dropTarget(key, cfg.Spec.TargetXRD.Name) + return 0, nil + } + // Still ours, so any drain in progress was for a move that did not + // happen, or reverted. + r.cancelDrain(key) + ruleSets, err := cfg.ToRuleSets() if err != nil { r.recordFailure(cfg.Spec.TargetXRD.Name, "InvalidRules", fmt.Sprintf("invalid rule configuration: %v", err)) - return nil + return 0, nil } report, err := engine.Analyze(engine.AnalyzeInput{Source: xrdadapter.New(xrd), HubVersion: cfg.Spec.HubVersion, Spokes: ruleSets}) if err != nil { r.recordFailure(cfg.Spec.TargetXRD.Name, "AnalyzeFailed", fmt.Sprintf("analysis failed: %v", err)) - return nil + return 0, nil } if report.HasErrors() { r.recordFailure(cfg.Spec.TargetXRD.Name, "ValidationErrors", "analysis produced validation errors; keeping any previously compiled plan in place") - return nil + return 0, nil } r.compileAndRegister(cfg.Spec.TargetXRD.Name, cfg.Spec.HubVersion, cfg.Spec.ConversionReviewVersions, report, fmt.Sprintf("gen=%d/%d", xrd.GetGeneration(), cfg.Generation)) - return nil + return 0, nil } // reconcileOneCRD is reconcileOneXRD's counterpart for CRDConversionConfig. -func (r *Reconciler) reconcileOneCRD(ctx context.Context, name string) error { +func (r *Reconciler) reconcileOneCRD(ctx context.Context, name string) (time.Duration, error) { r.ensureConfigToTarget() key := configKey("crd", name) @@ -198,56 +309,64 @@ func (r *Reconciler) reconcileOneCRD(ctx context.Context, name string) error { err := r.Get(ctx, types.NamespacedName{Name: name}, &cfg) if apierrors.IsNotFound(err) { r.forgetConfig(key) - return nil + return 0, nil } if err != nil { - return fmt.Errorf("getting CRDConversionConfig %q: %w", name, err) + return 0, fmt.Errorf("getting CRDConversionConfig %q: %w", name, err) } if !cfg.DeletionTimestamp.IsZero() { r.forgetConfig(key) - return nil + return 0, nil } r.rememberConfig(key, cfg.Spec.TargetCRD.Name) var servers teraskyv1alpha1.ConversionWebhookServerList if err := r.List(ctx, &servers); err != nil { - return fmt.Errorf("listing ConversionWebhookServers: %w", err) - } - assigned, err := assign.ResolveAssignment(&cfg, servers.Items) - if err != nil || assigned != r.ServerName { - r.Registry.Remove(cfg.Spec.TargetCRD.Name) - if r.Metrics != nil { - r.Metrics.SyncRegistryMetrics(r.Registry) - } - return nil + return 0, fmt.Errorf("listing ConversionWebhookServers: %w", err) } + assigned, assignErr := assign.ResolveAssignment(&cfg, servers.Items) + mine := assignErr == nil && assigned == r.ServerName var crd extv1.CustomResourceDefinition if err := r.Get(ctx, types.NamespacedName{Name: cfg.Spec.TargetCRD.Name}, &crd); err != nil { if apierrors.IsNotFound(err) { + if !mine { + r.dropTarget(key, cfg.Spec.TargetCRD.Name) + return 0, nil + } r.recordFailure(cfg.Spec.TargetCRD.Name, "CRDNotFound", fmt.Sprintf("target CRD %q not found", cfg.Spec.TargetCRD.Name)) - return nil + return 0, nil + } + return 0, fmt.Errorf("getting target CRD %q: %w", cfg.Spec.TargetCRD.Name, err) + } + + // See the XRD path, and handover.go. + if !mine && !crdPointsAtServer(&crd, r.ServerName) { + if wait := r.drainRemaining(key, cfg.Spec.TargetCRD.Name); wait > 0 { + return wait, nil } - return fmt.Errorf("getting target CRD %q: %w", cfg.Spec.TargetCRD.Name, err) + r.dropTarget(key, cfg.Spec.TargetCRD.Name) + return 0, nil } + r.cancelDrain(key) ruleSets, err := cfg.ToRuleSets() if err != nil { r.recordFailure(cfg.Spec.TargetCRD.Name, "InvalidRules", fmt.Sprintf("invalid rule configuration: %v", err)) - return nil + return 0, nil } report, err := engine.Analyze(engine.AnalyzeInput{Source: crdadapter.New(&crd), HubVersion: cfg.Spec.HubVersion, Spokes: ruleSets}) if err != nil { r.recordFailure(cfg.Spec.TargetCRD.Name, "AnalyzeFailed", fmt.Sprintf("analysis failed: %v", err)) - return nil + return 0, nil } if report.HasErrors() { r.recordFailure(cfg.Spec.TargetCRD.Name, "ValidationErrors", "analysis produced validation errors; keeping any previously compiled plan in place") - return nil + return 0, nil } r.compileAndRegister(cfg.Spec.TargetCRD.Name, cfg.Spec.HubVersion, cfg.Spec.ConversionReviewVersions, report, fmt.Sprintf("gen=%d/%d", crd.Generation, cfg.Generation)) - return nil + return 0, nil } // compileAndRegister builds the CompiledEntry from an analysis report and @@ -275,8 +394,8 @@ func (r *Reconciler) compileAndRegister(targetName, hubVersion string, reviewVer if r.Metrics != nil { r.Metrics.RegistryReloadTotal.WithLabelValues(targetName, "success").Inc() r.Metrics.RegistryLastReload.WithLabelValues(targetName).Set(float64(time.Now().Unix())) - r.Metrics.SyncRegistryMetrics(r.Registry) } + r.registryChanged() } func (r *Reconciler) rememberConfig(key, targetName string) { @@ -285,54 +404,214 @@ func (r *Reconciler) rememberConfig(key, targetName string) { r.configToTarget[key] = targetName } +// dropTarget removes a target this replica no longer serves and clears +// any drain bookkeeping for it. +func (r *Reconciler) dropTarget(key, targetName string) { + r.cancelDrain(key) + r.Registry.Remove(targetName) + r.registryChanged() +} + func (r *Reconciler) forgetConfig(key string) { r.mu.Lock() targetName, ok := r.configToTarget[key] delete(r.configToTarget, key) + // A config deleted mid-drain would otherwise leave its deadline + // behind forever — one map entry per config that ever churned. + delete(r.drainUntil, key) r.mu.Unlock() if ok { r.Registry.Remove(targetName) - if r.Metrics != nil { - r.Metrics.SyncRegistryMetrics(r.Registry) - } + r.registryChanged() } } +// registryChanged republishes everything derived from the registry: the +// per-target gauges, and this replica's served-target Lease. Suppressed +// while a bulk pass (InitialSync) is running. +// +// SyncRegistryMetrics rebuilds every series from a full snapshot, so it is +// O(targets) per call. On the watch-driven path that is one call per +// change and unnoticeable. During InitialSync it would be one call per +// target — quadratic in the target count, on the cold start this phase +// exists to shorten, and every intermediate state it publishes is +// immediately superseded anyway. InitialSync therefore suppresses it and +// syncs once at the end, which is the only state a scrape can observe: the +// replica is not in the Service's endpoints until it reports ready. The +// same reasoning applies with more force to the Lease, which is an API +// write. +func (r *Reconciler) registryChanged() { + if r.bulkSync.Load() { + return + } + if r.Metrics != nil { + r.Metrics.SyncRegistryMetrics(r.Registry) + } + r.Publisher.Notify() +} + func (r *Reconciler) recordFailure(targetName, reason, msg string) { r.Registry.RecordError(targetName, msg) if r.Metrics != nil { r.Metrics.RegistryReloadTotal.WithLabelValues(targetName, "error").Inc() r.Metrics.RegistryCompileErr.WithLabelValues(targetName, reason).Inc() - r.Metrics.SyncRegistryMetrics(r.Registry) } + r.registryChanged() +} + +// InitialSyncStats is what one InitialSync did: how many configs it walked +// and how long that took. Returned rather than logged from inside so the +// caller owns the log line and the metric — cmd/webhook-server is the only +// place that knows whether this replica is about to become ready. +type InitialSyncStats struct { + Targets int + Duration time.Duration } -// InitialSync runs reconcileOneXRD/reconcileOneCRD synchronously for every +// DefaultInitialSyncWorkers is the bound on InitialSync's worker pool when +// InitialSyncWorkers is left at zero. Compilation is CPU-bound and +// independent per target, so the useful parallelism is the number of cores +// the container is actually allowed to use — GOMAXPROCS, which respects a +// CPU limit when the runtime is configured for it. The pool is bounded +// rather than unbounded because every worker holds a decoded schema and a +// half-built plan; an unbounded fan-out across a thousand targets would +// turn a CPU problem into a memory one at exactly the moment the replica +// has the least headroom. +func DefaultInitialSyncWorkers() int { + if n := runtime.GOMAXPROCS(0); n > 0 { + return n + } + return 1 +} + +// InitialSync runs reconcileOneXRD/reconcileOneCRD for every // currently-existing config of whichever kinds are enabled, so the caller // can gate readiness on "cache synced AND every config has been through at // least one reconcile attempt" rather than cache-sync alone — closing the // classic gap where a pod is added to a Service's endpoints before its // registry reflects reality. -func (r *Reconciler) InitialSync(ctx context.Context) error { +// +// The walk is parallel across a bounded pool (see InitialSyncWorkers), +// because it is the whole of a replica's cold start: with hundreds of +// targets, compiling them one at a time is the difference between a pod +// that is ready in a second and one a startupProbe has to be told to wait +// for. Parallelism is safe by construction — each call reads one config +// and its target from the shared informer cache and writes one entry into +// the copy-on-write Registry under its own lock, and the two calls never +// share intermediate state. It is also *observationally* identical to the +// serial walk: distinct configs write distinct registry keys, so no +// ordering between them is visible in the result. +func (r *Reconciler) InitialSync(ctx context.Context) (InitialSyncStats, error) { + start := time.Now() + var stats InitialSyncStats + + // Both lists are read before any compiling starts, so a slow compile + // cannot make the target count a moving target. + var work []func(context.Context) error if r.EnableXRDSupport { var list teraskyv1alpha1.XRDConversionConfigList if err := r.List(ctx, &list); err != nil { - return fmt.Errorf("listing XRDConversionConfigs for initial sync: %w", err) + return stats, fmt.Errorf("listing XRDConversionConfigs for initial sync: %w", err) } for _, cfg := range list.Items { - _ = r.reconcileOneXRD(ctx, cfg.Name) // best-effort; the watch-driven reconciler retries transient failures. + work = append(work, func(ctx context.Context) error { + _, err := r.reconcileOneXRD(ctx, cfg.Name) + return err + }) } } if r.EnableCRDSupport { var list teraskyv1alpha1.CRDConversionConfigList if err := r.List(ctx, &list); err != nil { - return fmt.Errorf("listing CRDConversionConfigs for initial sync: %w", err) + return stats, fmt.Errorf("listing CRDConversionConfigs for initial sync: %w", err) } for _, cfg := range list.Items { - _ = r.reconcileOneCRD(ctx, cfg.Name) // best-effort; the watch-driven reconciler retries transient failures. + work = append(work, func(ctx context.Context) error { + _, err := r.reconcileOneCRD(ctx, cfg.Name) + return err + }) } } - return nil + + stats.Targets = len(work) + r.bulkSync.Store(true) + errs := r.runInitialSyncWork(ctx, work) + r.bulkSync.Store(false) + r.registryChanged() + + stats.Duration = time.Since(start) + if r.Metrics != nil { + r.Metrics.InitialSyncTargets.Set(float64(stats.Targets)) + r.Metrics.InitialSyncDuration.Set(stats.Duration.Seconds()) + } + // An infrastructure failure is returned, not swallowed. reconcileOne* + // already distinguishes the two: a bad config records itself into the + // registry and returns nil, because retrying cannot fix it. What comes + // back here is a failed API read — and the watch-driven reconciler + // only retries what a later watch event re-delivers, which a transient + // Get failure at startup may never produce. Reporting ready on top of + // that would leave a hole in the registry that answers every + // ConversionReview for the missing target with a failure. + if len(errs) > 0 { + return stats, fmt.Errorf("initial sync: %d of %d targets failed to load: %w", len(errs), stats.Targets, errors.Join(errs...)) + } + return stats, nil +} + +// runInitialSyncWork drains work across at most InitialSyncWorkers +// goroutines and returns every error the units reported. +// +// It deliberately does not stop early on the first failure or on a +// cancelled context: each unit is bounded, and the caller needs the whole +// picture — "three targets failed" is a different situation from "one +// did". Cancellation reaches the individual API reads through ctx, which +// is what actually makes a cancelled sync fast. +func (r *Reconciler) runInitialSyncWork(ctx context.Context, work []func(context.Context) error) []error { + if len(work) == 0 { + return nil + } + workers := r.InitialSyncWorkers + if workers <= 0 { + workers = DefaultInitialSyncWorkers() + } + if workers > len(work) { + workers = len(work) + } + if workers <= 1 { + var errs []error + for _, fn := range work { + if err := fn(ctx); err != nil { + errs = append(errs, err) + } + } + return errs + } + + next := make(chan func(context.Context) error) + var ( + mu sync.Mutex + errs []error + wg sync.WaitGroup + ) + wg.Add(workers) + for i := 0; i < workers; i++ { + go func() { + defer wg.Done() + for fn := range next { + if err := fn(ctx); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + } + } + }() + } + for _, fn := range work { + next <- fn + } + close(next) + wg.Wait() + return errs } // SetupWithManager wires up one controller per enabled config kind, each diff --git a/internal/webhookserver/reconciler_test.go b/internal/webhookserver/reconciler_test.go index fc73300..892b71a 100644 --- a/internal/webhookserver/reconciler_test.go +++ b/internal/webhookserver/reconciler_test.go @@ -35,7 +35,7 @@ func TestReconcileOneXRD_HappyPath_Compiles(t *testing.T) { c := newFakeClient(xrd, cfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: true} - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } entry, ok := r.Registry.Get("xfoos.example.org") @@ -54,7 +54,7 @@ func TestReconcileOneXRD_ConfigNotFound_Forgets(t *testing.T) { r.configToTarget[configKey("xrd", "cfg")] = "xfoos.example.org" r.Registry.Set("xfoos.example.org", &CompiledEntry{}) - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } if _, ok := r.Registry.Get("xfoos.example.org"); ok { @@ -75,7 +75,7 @@ func TestReconcileOneXRD_DeletionTimestamp_Forgets(t *testing.T) { r.configToTarget[configKey("xrd", "cfg")] = "xfoos.example.org" r.Registry.Set("xfoos.example.org", &CompiledEntry{}) - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } if _, ok := r.Registry.Get("xfoos.example.org"); ok { @@ -93,9 +93,11 @@ func TestReconcileOneXRD_NotAssignedToThisReplica_Removed(t *testing.T) { c := newFakeClient(xrd, cfg, otherServer).Build() registry := NewRegistry() registry.Set("xfoos.example.org", &CompiledEntry{Router: nil}) - r := &Reconciler{Client: c, ServerName: "this-srv", Registry: registry, EnableXRDSupport: true} + // Drain disabled: this test is about the assignment decision, and the + // drain has its own tests. + r := &Reconciler{Client: c, ServerName: "this-srv", Registry: registry, EnableXRDSupport: true, TargetDrainPeriod: -1} - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } if _, ok := r.Registry.Get("xfoos.example.org"); ok { @@ -112,7 +114,7 @@ func TestReconcileOneXRD_TargetXRDMissing_RecordsFailure(t *testing.T) { c := newFakeClient(cfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: true} - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } entry, ok := r.Registry.Get("missing.example.org") @@ -132,7 +134,7 @@ func TestReconcileOneXRD_InvalidRules_RecordsFailure(t *testing.T) { c := newFakeClient(xrd, cfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: true} - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } entry, ok := r.Registry.Get("xfoos.example.org") @@ -152,7 +154,7 @@ func TestReconcileOneXRD_AnalysisErrors_RecordsFailure(t *testing.T) { c := newFakeClient(xrd, cfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: true} - if err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneXRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } entry, ok := r.Registry.Get("xfoos.example.org") @@ -171,7 +173,7 @@ func TestReconcileOneCRD_HappyPath_Compiles(t *testing.T) { c := newFakeClient(crd, cfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableCRDSupport: true} - if err := r.reconcileOneCRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneCRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } entry, ok := r.Registry.Get("foos.example.org") @@ -189,7 +191,7 @@ func TestReconcileOneCRD_TargetCRDMissing_RecordsFailure(t *testing.T) { c := newFakeClient(cfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableCRDSupport: true} - if err := r.reconcileOneCRD(context.Background(), "cfg"); err != nil { + if _, err := r.reconcileOneCRD(context.Background(), "cfg"); err != nil { t.Fatalf("unexpected error: %v", err) } entry, ok := r.Registry.Get("missing.example.org") @@ -210,9 +212,13 @@ func TestInitialSync_PopulatesRegistryForEnabledKinds(t *testing.T) { c := newFakeClient(xrd, xrdCfg, crd, crdCfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: true, EnableCRDSupport: true} - if err := r.InitialSync(context.Background()); err != nil { + stats, err := r.InitialSync(context.Background()) + if err != nil { t.Fatalf("unexpected error: %v", err) } + if stats.Targets != 2 { + t.Fatalf("expected the stats to report both configs, got %d", stats.Targets) + } if _, ok := r.Registry.Get("xfoos.example.org"); !ok { t.Fatalf("expected the XRD's config to have been synced") } @@ -231,9 +237,13 @@ func TestInitialSync_DisabledKindsAreSkipped(t *testing.T) { c := newFakeClient(xrd, xrdCfg, server).Build() r := &Reconciler{Client: c, ServerName: "srv", Registry: NewRegistry(), EnableXRDSupport: false, EnableCRDSupport: false} - if err := r.InitialSync(context.Background()); err != nil { + stats, err := r.InitialSync(context.Background()) + if err != nil { t.Fatalf("unexpected error: %v", err) } + if stats.Targets != 0 { + t.Fatalf("expected no targets to be walked, got %d", stats.Targets) + } if r.Registry.Len() != 0 { t.Fatalf("expected nothing to sync when both kinds are disabled, got len=%d", r.Registry.Len()) } diff --git a/internal/webhookserver/testutil_test.go b/internal/webhookserver/testutil_test.go index 4863c1a..f72cc81 100644 --- a/internal/webhookserver/testutil_test.go +++ b/internal/webhookserver/testutil_test.go @@ -17,6 +17,8 @@ limitations under the License. package webhookserver import ( + "time" + "github.com/prometheus/client_golang/prometheus" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -174,3 +176,14 @@ func renameRuleCRDConfig(name, targetCRD string) *teraskyv1alpha1.CRDConversionC }, } } + +// fakeClock lets a test walk the drain period without sleeping through +// thirty seconds of it. +type fakeClock struct{ t time.Time } + +func newFakeClock() *fakeClock { + return &fakeClock{t: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} +} + +func (c *fakeClock) Now() time.Time { return c.t } +func (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) } diff --git a/pkg/engine/memory_bench_test.go b/pkg/engine/memory_bench_test.go new file mode 100644 index 0000000..bd68ca0 --- /dev/null +++ b/pkg/engine/memory_bench_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2026 The declarative-conversion-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "fmt" + "runtime" + "testing" +) + +// retainedBytes reports live heap bytes after a settling GC. Two cycles, +// because the first can leave objects that only became unreachable during +// it still counted. +func retainedBytes() uint64 { + runtime.GC() + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + return ms.HeapAlloc +} + +// BenchmarkCompiledPlanRetained answers "how much memory does one compiled +// plan cost?", which -benchmem cannot: B/op is bytes *allocated* per +// operation, most of which is garbage from the compile itself. What sizes a +// replica is what survives — the Ops, their pre-split paths and pre-decoded +// tables — so this holds every plan it builds and reads the live heap +// either side. +// +// Reported as B/plan rather than B/op. Run it with -benchtime=x so the +// denominator is a number you chose; the default time-based mode makes b.N +// depend on machine speed, which is fine for the average but makes a single +// run harder to reason about. The published numbers are in +// docs/operations/capacity.md. +func BenchmarkCompiledPlanRetained(b *testing.B) { + for _, n := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("leaves=%d", n), func(b *testing.B) { + hub := nLeafSchema(n) + spoke := nLeafSchema(n) + rs := RuleSet{HubVersion: "v2", SpokeVersion: "v1", Rules: nRenameRules(n)} + + // Allocated before the baseline so the slice's own backing + // array is not counted as plan memory. + plans := make([]*Plan, 0, b.N) + before := retainedBytes() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + plan, _, err := Compile(rs, &hub, &spoke) + if err != nil { + b.Fatal(err) + } + plans = append(plans, plan) + } + b.StopTimer() + + after := retainedBytes() + runtime.KeepAlive(plans) + b.ReportMetric(float64(after-before)/float64(b.N), "B/plan") + }) + } +} + +// BenchmarkCompilePeakAlloc is the transient side of the same question. +// Compile allocates heavily and most of it is immediately garbage, so a +// replica compiling many plans at once can peak well above the steady +// footprint the benchmark above measures — and the peak is what an OOM kill +// is decided against, not the steady state. +func BenchmarkCompilePeakAlloc(b *testing.B) { + for _, n := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("leaves=%d", n), func(b *testing.B) { + hub := nLeafSchema(n) + spoke := nLeafSchema(n) + rs := RuleSet{HubVersion: "v2", SpokeVersion: "v1", Rules: nRenameRules(n)} + + var ms runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&ms) + startTotal := ms.TotalAlloc + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, _, err := Compile(rs, &hub, &spoke); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + + runtime.ReadMemStats(&ms) + // Churn per compile: everything allocated, live or not. The + // gap between this and B/plan above is what the GC has to keep + // up with during a cold start. + b.ReportMetric(float64(ms.TotalAlloc-startTotal)/float64(b.N), "B/churn") + }) + } +}