diff --git a/.custom-gcl.yml b/.custom-gcl.yml index d9ae33b..e2ac6c7 100644 --- a/.custom-gcl.yml +++ b/.custom-gcl.yml @@ -8,4 +8,4 @@ plugins: # logcheck validates structured logging calls and parameters (e.g., balanced key-value pairs) - module: "sigs.k8s.io/logtools" import: "sigs.k8s.io/logtools/logcheck/gclplugin" - version: latest + version: v0.10.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a1c303b..c610d80 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,13 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone the code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: registry: ghcr.io username: ${{ github.actor }} @@ -29,7 +29,7 @@ jobs: - name: Compute image tags id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 with: images: ghcr.io/${{ github.repository }} tags: | @@ -38,7 +38,7 @@ jobs: type=ref,event=tag - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: . push: true @@ -55,7 +55,7 @@ jobs: needs: image steps: - name: Clone the code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - name: Log in to GHCR run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin @@ -86,13 +86,13 @@ jobs: pull-requests: write steps: - name: Clone the code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 - name: Run semantic-release id: semantic - uses: cycjimmy/semantic-release-action@v4 + uses: cycjimmy/semantic-release-action@16ca923e6ccbb50770c415a0ccd43709a8c5f7a4 with: # semantic-release 24 bundles conventional-changelog-writer 8. # Preset 9 requires writer 9 and fails while generating notes. diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..f342b6f --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,25 @@ +name: Security +on: + push: + pull_request: +permissions: + contents: read +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 + with: + go-version-file: go.mod + - run: (cd cmd && go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan=module) + - run: make docker-build IMG=celld-operator:scan + - uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 + with: + image-ref: celld-operator:scan + vuln-type: os,library + severity: HIGH,CRITICAL + exit-code: '1' + ignore-unfixed: false diff --git a/.github/workflows/test-chart.yml b/.github/workflows/test-chart.yml index 9572916..a099562 100644 --- a/.github/workflows/test-chart.yml +++ b/.github/workflows/test-chart.yml @@ -12,6 +12,8 @@ jobs: contents: read name: Run on Ubuntu runs-on: ubuntu-latest + env: + IMG: controller:latest steps: - name: Clone the code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -23,9 +25,9 @@ jobs: with: go-version-file: go.mod - - name: Install the latest version of kind + - name: Install kind run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH) + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$(go env GOARCH) chmod +x ./kind sudo mv ./kind /usr/local/bin/kind @@ -38,59 +40,26 @@ jobs: - name: Prepare celld-operator run: | go mod tidy - make docker-build IMG=celld-operator:v0.1.0 - kind load docker-image celld-operator:v0.1.0 + make docker-build + kind load docker-image $IMG - name: Install Helm - run: | - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - - name: Verify Helm installation - run: helm version + run: make install-helm - name: Lint Helm Chart run: | helm lint ./dist/chart + go test ./test/chart -# TODO: Uncomment if cert-manager is enabled -# - name: Install cert-manager via Helm -# run: | -# helm repo add jetstack https://charts.jetstack.io -# helm repo update -# helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --set crds.enabled=true -# -# - name: Wait for cert-manager to be ready -# run: | -# kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager -# kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-cainjector -# kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-webhook - -# TODO: Uncomment if Prometheus is enabled -# - name: Install Prometheus Operator CRDs -# run: | -# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -# helm repo update -# helm install prometheus-crds prometheus-community/prometheus-operator-crds -# -# - name: Install Prometheus via Helm -# run: | -# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -# helm repo update -# helm install prometheus prometheus-community/prometheus --namespace monitoring --create-namespace -# -# - name: Wait for Prometheus to be ready -# run: | -# kubectl wait --namespace monitoring --for=condition=available --timeout=300s deployment/prometheus-server - - - name: Install Helm chart for project + - name: Deploy manager via Helm run: | - helm install my-release ./dist/chart --create-namespace --namespace celld-operator-system + make helm-deploy HELM_EXTRA_ARGS="--set controllerManager.container.imagePullPolicy=Never" + test "$(kubectl get deployment celld-operator-controller-manager -n celld-operator-system -o jsonpath='{.spec.template.spec.containers[0].image}')" = "$IMG" - name: Check Helm release status run: | - helm status my-release --namespace celld-operator-system + make helm-status -# TODO: Uncomment if prometheus.enabled is set to true to confirm that the ServiceMonitor gets created -# - name: Check Presence of ServiceMonitor -# run: | -# kubectl wait --namespace celld-operator-system --for=jsonpath='{.kind}'=ServiceMonitor servicemonitor/celld-operator-controller-manager-metrics-monitor + - name: Delete isolated cluster + if: always() + run: kind delete cluster diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 83e295e..5a22e9e 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -23,9 +23,9 @@ jobs: with: go-version-file: go.mod - - name: Install the latest version of kind + - name: Install kind v0.32.0 run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH) + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$(go env GOARCH) chmod +x ./kind sudo mv ./kind /usr/local/bin/kind diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27e82c8..4c7e264 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,3 +27,6 @@ jobs: run: | go mod tidy make test + make chart-sync + git diff --exit-code -- config/crd config/rbac dist/chart + (cd cmd && go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan=module) diff --git a/Dockerfile b/Dockerfile index 5b59f51..0fa86f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.26 AS builder +FROM golang:1.26.8@sha256:3c3e25a4da13fd0478eed2df1eb35a0e667094a7124d3993a6a1d30f71c17e79 AS builder ARG TARGETOS ARG TARGETARCH @@ -23,7 +23,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o ma # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM gcr.io/distroless/static:nonroot@sha256:1c2c046bc09ed40fad370b599a0b1ae7987f55b01e247cf27a7c27cd97e5bbc7 WORKDIR / COPY --from=builder /workspace/manager . USER 65532:65532 diff --git a/Makefile b/Makefile index 3ded957..b9835ef 100644 --- a/Makefile +++ b/Makefile @@ -86,9 +86,13 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist esac .PHONY: test-e2e -test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v - $(MAKE) cleanup-test-e2e +test-e2e: manifests generate fmt vet ## Run e2e in a newly created, isolated Kind cluster. + @case "$(KIND_CLUSTER)" in celld-operator-test-*|celld-operator-audit-*) ;; *) echo "Use a dedicated celld-operator-test-* cluster name" >&2; exit 1;; esac + @if $(KIND) get clusters | grep -Fxq "$(KIND_CLUSTER)"; then echo "Refusing to reuse an existing cluster" >&2; exit 1; fi + @set -e; task_kubeconfig=$$(mktemp); \ + trap 'rm -f "$$task_kubeconfig"; $(KIND) delete cluster --name "$(KIND_CLUSTER)"' EXIT; \ + $(KIND) create cluster --name "$(KIND_CLUSTER)" --kubeconfig "$$task_kubeconfig"; \ + KUBECONFIG="$$task_kubeconfig" KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v .PHONY: cleanup-test-e2e cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests @@ -257,3 +261,58 @@ endef define gomodver $(shell go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $(1) 2>/dev/null) endef + +##@ Helm Deployment + +## Helm binary to use for deploying the chart +HELM ?= helm +## Namespace to deploy the Helm release +HELM_NAMESPACE ?= celld-operator-system +## Name of the Helm release +HELM_RELEASE ?= celld-operator +## Path to the Helm chart directory +HELM_CHART_DIR ?= dist/chart +## Additional arguments to pass to helm commands +HELM_EXTRA_ARGS ?= + +.PHONY: install-helm +install-helm: ## Require Helm v3.19.0, installing the pinned release if needed. + @set -e; \ + if [ "$$($(HELM) version --template '{{.Version}}' 2>/dev/null || true)" != "v3.19.0" ]; then \ + task_installer=$$(mktemp); trap 'rm -f "$$task_installer"' EXIT; \ + curl -fsSL https://raw.githubusercontent.com/helm/helm/v3.19.0/scripts/get-helm-3 -o "$$task_installer"; \ + printf '%s %s\n' '4a01413bf2a767ae744b8bbe4485cd83654d9a0a769c92377afc36328d5a007a' "$$task_installer" | sha256sum -c -; \ + DESIRED_VERSION=v3.19.0 bash "$$task_installer"; \ + fi; \ + test "$$($(HELM) version --template '{{.Version}}')" = "v3.19.0" + +.PHONY: helm-deploy +helm-deploy: install-helm ## Deploy manager to the K8s cluster via Helm. Specify an image with IMG. + task_image="$(IMG)"; $(HELM) upgrade --install $(HELM_RELEASE) $(HELM_CHART_DIR) \ + --namespace $(HELM_NAMESPACE) \ + --create-namespace \ + --set controllerManager.container.image.repository="$${task_image%:*}" \ + --set controllerManager.container.image.tag="$${task_image##*:}" \ + --wait \ + --timeout 5m \ + $(HELM_EXTRA_ARGS) + +.PHONY: helm-uninstall +helm-uninstall: ## Uninstall the Helm release from the K8s cluster. + $(HELM) uninstall $(HELM_RELEASE) --namespace $(HELM_NAMESPACE) + +.PHONY: helm-status +helm-status: ## Show Helm release status. + $(HELM) status $(HELM_RELEASE) --namespace $(HELM_NAMESPACE) + +.PHONY: helm-history +helm-history: ## Show Helm release history. + $(HELM) history $(HELM_RELEASE) --namespace $(HELM_NAMESPACE) + +.PHONY: helm-rollback +helm-rollback: ## Rollback to previous Helm release. + $(HELM) rollback $(HELM_RELEASE) --namespace $(HELM_NAMESPACE) + +.PHONY: chart-sync +chart-sync: manifests generate ## Refresh the customized Helm CRD and RBAC from controller-gen. + python3 hack/sync-chart.py diff --git a/PROJECT b/PROJECT index ae34692..86a15fd 100644 --- a/PROJECT +++ b/PROJECT @@ -7,7 +7,9 @@ domain: celld-operator.io layout: - go.kubebuilder.io/v4 plugins: - helm.kubebuilder.io/v1-alpha: {} + helm.kubebuilder.io/v2-alpha: + manifests: dist/install.yaml + output: dist projectName: celld-operator repo: github.com/ezgamehost/celld-operator resources: @@ -16,8 +18,6 @@ resources: namespaced: true controller: true domain: celld-operator.io - group: "" - # Empty group: the CRD group is the domain itself (celld-operator.io). kind: WorkerApp path: github.com/ezgamehost/celld-operator/api/v1alpha1 version: v1alpha1 diff --git a/README.md b/README.md index 7d4b915..4937c70 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,10 @@ flowchart LR P0 & P1 & P2 --> B[("bucket\ns3://cells/apps/tenant")] ``` +See [production operation and upgrade requirements](docs/production.md) before deploying or upgrading. + Each application is its own fleet with its own bucket prefix and its own -credentials — tenancy lives at the Kubernetes layer. Even a full runtime -compromise inside one fleet reaches only that app's pods and that app's -prefix. This is a platform for *your* applications (or your customers' apps -under your operation), not for anonymous hostile code. +credentials — tenancy lives at the Kubernetes layer. WorkerApp authors are trusted workload deployers who can reference namespace Secrets. Isolate tenants with namespace RBAC and scoped bucket credentials; verify network policy and encrypted peer transport on your platform. ## Prerequisites @@ -69,7 +68,7 @@ under your operation), not for anonymous hostile code. - **Istio** for internal-listener AuthorizationPolicies (ambient mode recommended: it encrypts the peer network without putting a sidecar in celld's drain path). - - **KEDA** + **Prometheus** (scraping the operator) for autoscaling. + - **KEDA v2.12.0+** + **Prometheus** (scraping the operator) for autoscaling. ## Install @@ -114,7 +113,7 @@ celld deploy . --bucket s3://platform-cells/apps/chat \ --endpoint https://ACCOUNT.r2.cloudflarestorage.com --region auto ``` -**2. Create the WorkerApp:** +**2. Configure the operator endpoint/identity allowlists** for the values below (see [production configuration](docs/production.md)), then create the WorkerApp: ```yaml apiVersion: celld-operator.io/v1alpha1 @@ -149,7 +148,7 @@ spec: # type: ClusterIP # hostnames at all — consumers use -celld..svc:8080 # annotations: {} # or LoadBalancer + annotations for a private LB websockets: true # long idle timeouts, sticky-friendly, slow scale-down - # durability: fleet # celld's default since 0.3; "bucket" waits for the upload before acking + durability: bucket # operator default; fleet requires PVCs and >=3 replicas # trustForwardedHeaders: true # request.url from X-Forwarded-*; only behind a proxy that replaces both autoscaling: enabled: true @@ -192,16 +191,14 @@ retains a conservative gated restart when this field changes. v0.4 does not need that restart to adopt code, but it gives the CR an explicit convergence point and keeps `status.rolledOutAppVersion` tied to the declared deployment. -Prefer `celld deploy` to be the whole story? Set `appVersion: auto`. The -operator follows the bucket's `deploy/current.json`; a new publish updates the -expected version within one `--deploy-poll-interval` (default 60s), while celld -itself normally adopts within 30s. In pinned mode the same read powers a -`DeployTrackingReady: VersionMismatch` warning when the bucket pointer and the -CR disagree — but only for fleets using `secretRef` credentials; the operator -does not read the bucket for `iamRole` fleets. Tracking reads use the fleet's -`secretRef` credentials, or the operator's ambient AWS identity otherwise, -and support `s3://` buckets only: a `gs://` or `az://` fleet must pin -`appVersion` (`DeployTrackingReady: UnsupportedStore` says so). +Use `deploymentPolicy: InPlace` to adopt without restarting on expected-version +changes. `status.deployments` reports live versions/generations; rollout completion +requires all pods to serve the expectation with no draining or swapping. + +For automatic tracking, set `appVersion: auto` and `deployTrackingSecretRef` to +a dedicated read-only S3 Secret. No ambient manager credentials are used. +Pinned mode can supply the same Secret for drift warnings. GCS and Azure auto +tracking are rejected. See the [credential contract](docs/production.md). A deployment can also need a newer celld than the fleet runs: the manifest names the features it uses. v0.4 adds Workers KV, Queues, Workflows, and R2 @@ -237,14 +234,12 @@ SIGKILL. | --- | --- | --- | | v0.1 ↔ v0.2 | No, either direction | Mixed fleets break: ownership records changed address semantics and block objects changed format | | v0.2.1 → v0.3.0 | **Yes** | A v0.3 node that cannot replicate to a v0.2 peer falls back to bucket proofs until the peer upgrades | -| v0.3 → v0.2 | No | A v0.2 binary cannot read writes still waiting in v0.3's replicated log or bundle objects, so the downgrade can lose acknowledged writes. `Recreate` drains every node first, which seals each log (`node-log close: sealed epoch` in the shutdown log); check for that line before trusting the downgrade | +| v0.3 → v0.2 | Blocked, including Recreate | A v0.2 binary cannot read writes still waiting in v0.3 replicated logs or bundle objects; pod disappearance does not prove sealing | | v0.3 ↔ v0.4 | No, either direction | v0.4's versioned peer tunnel refuses v0.3 peers, and v0.3 cannot read v0.4's epoch-qualified large Workers KV value references. Use `Recreate`: stop every v0.3 node before starting v0.4 | -celld 0.3 also changed the default write-acknowledgement proof from `bucket` -to `fleet` (two follower nodes hold the write on disk, the bucket upload -trails): a rolling upgrade switches each node as it restarts, and -`spec.durability: bucket` pins the old behavior if you want write latency -and durability to keep depending on the bucket alone. +The operator defaults to `durability: bucket` independently of celld's default. +Fleet acknowledgement requires retained PVCs, at least three replicas and +required host separation; optional required zone separation is available. ## Autoscaling @@ -261,6 +256,7 @@ celld has no metrics endpoint yet, so the operator polls each pod's internal | `celld_rss_bytes` / `celld_in_use_bytes` | Process RSS and RSS minus allocator retention | | `celld_cgroup_working_set_bytes` / `celld_cgroup_current_bytes` | v0.4 active cgroup memory used by pressure shedding, and the complete cgroup charge used by the absolute cap; absent outside a readable Linux cgroup | | `celld_container_restarts` / `celld_self_fenced` | Kubelet restart count, and 1 if the last exit was a celld self-fence (exit code 3) — a fence loop means the store or the bucket credential is broken | +| `celld_fleet_complete` / `celld_fleet_sample_timestamp_seconds` | Complete-census safety gate and sample time; scaling requires a complete sample younger than 60s | | `celld_state_up` | 1 if `/state` answered the last poll | With `spec.autoscaling.enabled`, the operator materializes a KEDA @@ -278,19 +274,14 @@ them wrong via templates because there are no templates: - **Liveness is TCP-only.** celld's health path answers 503 during a graceful drain; an HTTP liveness probe there would kill nodes mid-handoff. -- **Termination grace (40s) exceeds the drain bound** (`CELLD_SHUTDOWN_DRAIN_MS`, - 25s), so the kubelet never SIGKILLs a draining node, and the node-log seal - that follows the drain gets its headroom. +- **Termination grace is 60s**, exceeding the configured 40s total shutdown bound. Forced termination or infrastructure failure can still interrupt sealing. - **`CELLD_MAX_RSS_MB` is set explicitly** to ~80% of the container memory limit, so the ceiling is visible in the pod spec rather than inferred (celld derives the same 80% from the cgroup limit on its own). celld applies it to the memory the cells hold and keeps its own absolute cap at 95% of the limit on the process RSS; 80% stays under that cap, so shedding can still recover the node. -- **Fleet pods spread across hosts** (soft hostname topology spread). With - celld 0.3's fleet durability an acknowledged write lives on two follower - disks until the bucket upload lands; co-located followers would make one - host failure lose it. +- **Persistent fleet pods require different hosts**, with optional required zone separation. Fleet durability requires persistent storage and at least three replicas. - **Self-fenced nodes come back.** celld exits with code 3 after a `SELF-FENCE:` and requires a supervisor that restarts it without limit, spacing attempts by at least one lease lifetime (10s); the kubelet's @@ -298,13 +289,11 @@ them wrong via templates because there are no templates: `celld_self_fenced` makes a fence loop visible. - **The internal listener stays internal.** `:8081` (peer protocol plus an *unauthenticated* operator API) is reachable only from fleet pods and the - operator's namespace via NetworkPolicy, reinforced by an Istio + selected manager pods in the operator namespace via NetworkPolicy, reinforced by an Istio AuthorizationPolicy when Istio is installed, and is never routed. - **Peers can reach draining pods** (`publishNotReadyAddresses` on the headless service) so cell handoff works while a node reports unready. -- **Drain 503s are retried at the gateway**, making rollouts invisible to - clients; WebSocket routes get their request timeout disabled so hibernated - sockets aren't severed. +- **Gateway retries depend on the ingress implementation.** HTTPRoute retries require opt-in and experimental CRDs; clients must otherwise tolerate drain 503s. WebSocket routes disable request timeouts. - **PDB `maxUnavailable: 1`** keeps node maintenance as serialized as rollouts. diff --git a/api/v1alpha1/workerapp_types.go b/api/v1alpha1/workerapp_types.go index c687ca0..e22ba8e 100644 --- a/api/v1alpha1/workerapp_types.go +++ b/api/v1alpha1/workerapp_types.go @@ -29,7 +29,7 @@ import ( // UpdateStrategy selects how a celld version change rolls through the fleet. // Rolling is the partition-stepped, restoring-gated path (docs/celld-behaviors.md); // Recreate scales to zero first, for upstream releases that forbid mixed -// fleets or flag a downgrade as lossy. A Rolling request across a +// fleets. Downgrades from v0.3+ to older runtimes remain blocked. A Rolling request across a // known-breaking celld boundary is refused (F8). // +kubebuilder:validation:Enum=Rolling;Recreate type UpdateStrategy string @@ -41,9 +41,10 @@ const ( // CelldSpec pins the celld runtime for the fleet. type CelldSpec struct { - // image is the celld container image, tag included. Mixed-version fleets - // are never created; changing this triggers the strategy below. + // image is an explicitly versioned celld image. Known incompatible versions + // require Recreate; compatible releases roll one pod at a time. // +required + // +kubebuilder:validation:MinLength=1 Image string `json:"image"` // updateStrategy selects the rollout path for celld version changes. @@ -55,6 +56,7 @@ type CelldSpec struct { // BucketCredentials selects how the fleet authenticates to its bucket // prefix. Exactly one mechanism applies; iamRole is preferred because the // bucket credential is fleet-admin authority and static keys spread. +// +kubebuilder:validation:XValidation:rule="[has(self.iamRole) && self.iamRole.size() > 0, has(self.secretRef) && self.secretRef.size() > 0, has(self.azureClientID) && self.azureClientID.size() > 0].filter(x, x).size() <= 1",message="select at most one credential family" type BucketCredentials struct { // iamRole is an IAM role ARN assumed via the pod's service account // (IRSA / Workload Identity), or the literal "auto" to have the @@ -67,6 +69,7 @@ type BucketCredentials struct { // AZURE_STORAGE_ACCOUNT_KEY for an az:// container. For stores where // identity-based auth is unavailable. // +optional + // +kubebuilder:validation:MinLength=1 SecretRef string `json:"secretRef,omitempty"` // azureClientID is the client ID of a Microsoft Entra workload identity @@ -83,6 +86,9 @@ type BucketCredentials struct { // BucketSpec locates the fleet's slice of the object store. // +kubebuilder:validation:XValidation:rule="!self.name.startsWith('az://') || (has(self.storageAccount) && self.storageAccount.size() > 0)",message="an az:// bucket requires storageAccount (the Azure storage account; the bucket name is the container)" // +kubebuilder:validation:XValidation:rule="!has(self.endpoint) || self.endpoint.size() == 0 || self.name.startsWith('s3://')",message="endpoint applies to s3:// buckets only; celld rejects an endpoint for gs:// and az://" +// +kubebuilder:validation:XValidation:rule="self.name == oldSelf.name",message="bucket identity is immutable; migrate to a new WorkerApp" +// +kubebuilder:validation:XValidation:rule="has(self.storageAccount) == has(oldSelf.storageAccount) && (!has(self.storageAccount) || self.storageAccount == oldSelf.storageAccount)",message="storage account is immutable" +// +kubebuilder:validation:XValidation:rule="has(self.endpoint) == has(oldSelf.endpoint) && (!has(self.endpoint) || self.endpoint == oldSelf.endpoint)",message="bucket endpoint is immutable" type BucketSpec struct { // name is the fleet bucket and prefix, e.g. "s3://platform-cells/apps/chat", // "gs://platform-cells/apps/chat", or "az://platform-cells/apps/chat" @@ -91,7 +97,7 @@ type BucketSpec struct { // contract (conditional create/overwrite, read-after-write); see // docs/celld-behaviors.md for the qualified list. // +required - // +kubebuilder:validation:Pattern=`^(s3|gs|az)://.+` + // +kubebuilder:validation:Pattern=`^(s3|gs|az)://[a-zA-Z0-9][a-zA-Z0-9._-]*(/[a-zA-Z0-9._/-]+)?$` // +kubebuilder:validation:MaxLength=1024 Name string `json:"name"` @@ -126,10 +132,22 @@ type BucketSpec struct { // applies that threshold to the memory its cells hold and keeps its own // absolute cap at 95% of the limit on the process RSS (docs/celld-behaviors.md F10). type ResourcesSpec struct { + // cpuMillis is the CPU request per pod. + // +optional + // +kubebuilder:default=500 + // +kubebuilder:validation:Minimum=1 + CPUMillis int32 `json:"cpuMillis,omitempty"` + // ephemeralStorageGi bounds writable temporary storage. + // +optional + // +kubebuilder:default=10 + // +kubebuilder:validation:Minimum=1 + EphemeralStorageGi int32 `json:"ephemeralStorageGi,omitempty"` + // memoryGi is the container memory limit per pod, in GiB. // +optional // +kubebuilder:default=8 // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65536 MemoryGi int32 `json:"memoryGi,omitempty"` // maxResidentCells is the hard per-node resident-cell admission limit @@ -146,6 +164,7 @@ type VarsSpec struct { // CELLD_VARS_FILE. Rotation is a Secret update plus an ordinary gated // rollout; values are never baked into bundles. // +required + // +kubebuilder:validation:MinLength=1 SecretRef string `json:"secretRef"` } @@ -193,8 +212,10 @@ type AutoscalingTargets struct { // operator materializes it as a KEDA ScaledObject over its own // /state-derived Prometheus metrics, and pauses it during rollouts so the // scaler and the partition controller never fight over replica count. +// +kubebuilder:validation:XValidation:rule="self.minReplicas <= self.maxReplicas",message="minReplicas must not exceed maxReplicas" type AutoscalingSpec struct { // +optional + // +kubebuilder:default=false Enabled bool `json:"enabled,omitempty"` // minReplicas is the scale floor; keep >= 2 for HA. @@ -276,8 +297,54 @@ const ( DurabilityBucket Durability = "bucket" ) +// StorageSpec provisions retained local state per ordinal. Changing storage +// configuration requires a new WorkerApp; existing PVCs are never deleted. +type StorageSpec struct { + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65536 + SizeGi int32 `json:"sizeGi"` + // +optional + StorageClassName *string `json:"storageClassName,omitempty"` +} + +// SchedulingSpec controls placement. Persistent fleets always require distinct +// hosts; spreadAcrossZones additionally requires distinct zones. +type SchedulingSpec struct { + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // +optional + SpreadAcrossZones bool `json:"spreadAcrossZones,omitempty"` +} + // WorkerAppSpec defines the desired state of WorkerApp. +// +kubebuilder:validation:XValidation:rule="self.appVersion != 'auto' || (self.bucket.name.startsWith('s3://') && has(self.deployTrackingSecretRef))",message="auto requires an s3 bucket and a deployTrackingSecretRef" +// +kubebuilder:validation:XValidation:rule="!has(self.durability) || self.durability != 'fleet' || (has(self.storage) && self.replicas >= 3 && (!has(self.autoscaling) || !self.autoscaling.enabled || self.autoscaling.minReplicas >= 3))",message="fleet durability requires persistent storage and at least three replicas" +// +kubebuilder:validation:XValidation:rule="has(self.storage) == has(oldSelf.storage) && (!has(self.storage) || self.storage == oldSelf.storage)",message="storage is immutable; migrate to a new WorkerApp" + type WorkerAppSpec struct { + // deployTrackingSecretRef is a dedicated read-only S3 credential for + // deploy/current.json. The operator never uses its ambient cloud identity. + // +optional + // +kubebuilder:validation:MinLength=1 + DeployTrackingSecretRef string `json:"deployTrackingSecretRef,omitempty"` + // deploymentPolicy selects adoption in place or an additional gated restart. + // +optional + // +kubebuilder:default=Restart + // +kubebuilder:validation:Enum=Restart;InPlace + DeploymentPolicy string `json:"deploymentPolicy,omitempty"` + // clusterDomain is the Kubernetes DNS suffix. + // +optional + // +kubebuilder:default="cluster.local" + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$` + ClusterDomain string `json:"clusterDomain,omitempty"` + // +optional + Storage *StorageSpec `json:"storage,omitempty"` + // +optional + Scheduling SchedulingSpec `json:"scheduling,omitzero"` + // hostnames route to this app. One route object is reconciled in the // app's namespace carrying every hostname; what kind depends on the // operator's --ingress-mode. @@ -285,12 +352,9 @@ type WorkerAppSpec struct { // +listType=set Hostnames []string `json:"hostnames,omitempty"` - // appVersion names the application deployment in the fleet bucket - // (written there by `celld deploy`). Nodes load a deployment at startup - // only, so changing this triggers the gated rollout (docs/celld-behaviors.md). - // The sentinel "auto" makes the operator follow the bucket's - // deploy/current.json instead: `celld deploy` alone rolls the fleet, - // within one poll interval, with no CR edit. + // appVersion is the expected deployment, not a runtime pin: celld v0.4 + // adopts the bucket pointer in place. auto reads the pointer using the + // dedicated tracking credential. Status verifies live node versions. // +required // +kubebuilder:validation:MinLength=1 AppVersion string `json:"appVersion"` @@ -335,9 +399,10 @@ type WorkerAppSpec struct { // durability selects how celld proves a write before acknowledging it: // fleet (celld's default: two follower nodes fsync it, the bucket // upload follows) or bucket (the write is in the bucket first). Unset - // leaves celld's default. Changing it restarts the fleet through the + // selects bucket acknowledgement for safety with ephemeral storage. Changing it restarts the fleet through the // ordinary gated rollout. // +optional + // +kubebuilder:default=bucket Durability Durability `json:"durability,omitempty"` // trustForwardedHeaders lets X-Forwarded-Host and X-Forwarded-Proto @@ -390,8 +455,26 @@ type FleetStatus struct { Restoring int32 `json:"restoring"` } +// PodDeployment reports a live node's adoption state. +type PodDeployment struct { + Name string `json:"name"` + Version string `json:"version"` + Generation int64 `json:"generation"` + Draining int32 `json:"draining"` + Swapping int64 `json:"swapping"` +} + // WorkerAppStatus defines the observed state of WorkerApp. type WorkerAppStatus struct { + // expectedAppVersion is the resolved deployment target. + // +optional + ExpectedAppVersion string `json:"expectedAppVersion,omitempty"` + // deployments are fresh observations; missing nodes are not inferred. + // +optional + // +listType=map + // +listMapKey=name + Deployments []PodDeployment `json:"deployments,omitempty"` + // +optional Phase WorkerAppPhase `json:"phase,omitempty"` @@ -414,6 +497,7 @@ type WorkerAppStatus struct { } // +kubebuilder:object:root=true +// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) <= 48",message="WorkerApp name must not exceed 48 characters" // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` // +kubebuilder:printcolumn:name="App",type=string,JSONPath=`.status.rolledOutAppVersion` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2aeba1f..56da224 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,8 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -127,6 +128,21 @@ func (in *FleetStatus) DeepCopy() *FleetStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDeployment) DeepCopyInto(out *PodDeployment) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDeployment. +func (in *PodDeployment) DeepCopy() *PodDeployment { + if in == nil { + return nil + } + out := new(PodDeployment) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourcesSpec) DeepCopyInto(out *ResourcesSpec) { *out = *in @@ -157,6 +173,35 @@ func (in *RolloutStatus) DeepCopy() *RolloutStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingSpec) DeepCopyInto(out *SchedulingSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingSpec. +func (in *SchedulingSpec) DeepCopy() *SchedulingSpec { + if in == nil { + return nil + } + out := new(SchedulingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = *in @@ -179,6 +224,26 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { + *out = *in + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageSpec. +func (in *StorageSpec) DeepCopy() *StorageSpec { + if in == nil { + return nil + } + out := new(StorageSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TelemetrySpec) DeepCopyInto(out *TelemetrySpec) { *out = *in @@ -276,6 +341,12 @@ func (in *WorkerAppList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkerAppSpec) DeepCopyInto(out *WorkerAppSpec) { *out = *in + if in.Storage != nil { + in, out := &in.Storage, &out.Storage + *out = new(StorageSpec) + (*in).DeepCopyInto(*out) + } + in.Scheduling.DeepCopyInto(&out.Scheduling) if in.Hostnames != nil { in, out := &in.Hostnames, &out.Hostnames *out = make([]string, len(*in)) @@ -316,11 +387,16 @@ func (in *WorkerAppSpec) DeepCopy() *WorkerAppSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkerAppStatus) DeepCopyInto(out *WorkerAppStatus) { *out = *in + if in.Deployments != nil { + in, out := &in.Deployments, &out.Deployments + *out = make([]PodDeployment, len(*in)) + copy(*out, *in) + } out.Rollout = in.Rollout out.Fleet = in.Fleet if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/cmd/main.go b/cmd/main.go index e693f97..6a8447a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -19,10 +19,14 @@ package main import ( "crypto/tls" "flag" + "fmt" "os" + "slices" "strings" "time" + "sigs.k8s.io/controller-runtime/pkg/cache" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -117,11 +121,33 @@ func main() { var deployPollInterval time.Duration flag.DurationVar(&deployPollInterval, "deploy-poll-interval", 60*time.Second, "How often fleet buckets' deploy/current.json is polled for appVersion \"auto\" tracking.") + var watchNamespace, imagePrefixes, iamRoles, azureIDs, bucketEndpoints string + var routeRetries bool + flag.StringVar(&watchNamespace, "watch-namespace", "", + "Restrict reconciliation to this namespace; pair with namespaced RBAC.") + flag.StringVar(&imagePrefixes, "allowed-image-prefixes", "ghcr.io/denoland/celld:", + "Comma-separated allowed image prefixes.") + flag.StringVar(&iamRoles, "allowed-iam-roles", "", "Comma-separated allowed IAM role ARNs.") + flag.StringVar(&azureIDs, "allowed-azure-client-ids", "", "Comma-separated allowed Azure workload identities.") + flag.StringVar(&bucketEndpoints, "allowed-bucket-endpoints", "", + "Allowed HTTPS storage endpoints; empty permits AWS S3 only.") + flag.BoolVar(&routeRetries, "httproute-retries", false, + "Requires experimental Gateway API CRDs.") opts := zap.Options{ - Development: true, + Development: false, } opts.BindFlags(flag.CommandLine) flag.Parse() + validModes := []string{controller.IngressModeHTTPRoute, controller.IngressModeIngress, + controller.IngressModeVirtualService, controller.IngressModeNone} + if !slices.Contains(validModes, ingressMode) || statePollInterval <= 0 || deployPollInterval <= 0 { + fmt.Fprintln(os.Stderr, "Invalid ingress mode or nonpositive poll interval") + os.Exit(1) + } + cacheOptions := cache.Options{} + if watchNamespace != "" { + cacheOptions.DefaultNamespaces = map[string]cache.Config{watchNamespace: {}} + } ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) @@ -194,6 +220,7 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + Cache: cacheOptions, Metrics: metricsServerOptions, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, @@ -203,7 +230,9 @@ func main() { // rollouts) but must not be cached: a cached read would start a // cluster-wide Secret informer and hold every Secret in memory. Client: client.Options{ - Cache: &client.CacheOptions{DisableFor: []client.Object{&corev1.Secret{}}}, + Cache: &client.CacheOptions{DisableFor: []client.Object{ + &corev1.Secret{}, &gatewayv1.HTTPRoute{}, + }}, }, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the @@ -228,19 +257,24 @@ func main() { istioGatewayList = strings.Split(istioGateways, ",") } if err := (&controller.WorkerAppReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - State: stateClient, - Deploys: controller.NewDeployTracker(deployPollInterval), - IngressMode: ingressMode, - IstioGateways: istioGatewayList, - IngressClassName: ingressClass, - ClusterIssuer: clusterIssuer, - GatewayName: gatewayName, - GatewayNamespace: gatewayNamespace, - PrometheusURL: prometheusURL, - OperatorNamespace: operatorNamespace, - OperatorPrincipal: operatorPrincipal, + AllowedImagePrefixes: splitList(imagePrefixes), + AllowedIAMRoles: splitList(iamRoles), + AllowedAzureClientIDs: splitList(azureIDs), + AllowedBucketEndpoints: splitList(bucketEndpoints), + HTTPRouteRetries: routeRetries, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + State: stateClient, + Deploys: controller.NewDeployTracker(deployPollInterval), + IngressMode: ingressMode, + IstioGateways: istioGatewayList, + IngressClassName: ingressClass, + ClusterIssuer: clusterIssuer, + GatewayName: gatewayName, + GatewayNamespace: gatewayNamespace, + PrometheusURL: prometheusURL, + OperatorNamespace: operatorNamespace, + OperatorPrincipal: operatorPrincipal, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "workerapp") os.Exit(1) @@ -272,3 +306,13 @@ func main() { os.Exit(1) } } + +func splitList(value string) []string { + var result []string + for item := range strings.SplitSeq(value, ",") { + if item = strings.TrimSpace(item); item != "" { + result = append(result, item) + } + } + return result +} diff --git a/config/crd/bases/celld-operator.io_workerapps.yaml b/config/crd/bases/celld-operator.io_workerapps.yaml index fb6fcd6..dc3782c 100644 --- a/config/crd/bases/celld-operator.io_workerapps.yaml +++ b/config/crd/bases/celld-operator.io_workerapps.yaml @@ -57,12 +57,9 @@ spec: properties: appVersion: description: |- - appVersion names the application deployment in the fleet bucket - (written there by `celld deploy`). Nodes load a deployment at startup - only, so changing this triggers the gated rollout (docs/celld-behaviors.md). - The sentinel "auto" makes the operator follow the bucket's - deploy/current.json instead: `celld deploy` alone rolls the fleet, - within one poll interval, with no CR edit. + appVersion is the expected deployment, not a runtime pin: celld v0.4 + adopts the bucket pointer in place. auto reads the pointer using the + dedicated tracking credential. Status verifies live node versions. minLength: 1 type: string autoscaling: @@ -73,6 +70,7 @@ spec: scaler and the partition controller never fight over replica count. properties: enabled: + default: false type: boolean maxReplicas: default: 10 @@ -112,6 +110,9 @@ spec: type: integer type: object type: object + x-kubernetes-validations: + - message: minReplicas must not exceed maxReplicas + rule: self.minReplicas <= self.maxReplicas bucket: description: BucketSpec locates the fleet's slice of the object store. properties: @@ -140,8 +141,14 @@ spec: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY for an s3:// bucket, or AZURE_STORAGE_ACCOUNT_KEY for an az:// container. For stores where identity-based auth is unavailable. + minLength: 1 type: string type: object + x-kubernetes-validations: + - message: select at most one credential family + rule: '[has(self.iamRole) && self.iamRole.size() > 0, has(self.secretRef) + && self.secretRef.size() > 0, has(self.azureClientID) && self.azureClientID.size() + > 0].filter(x, x).size() <= 1' endpoint: description: |- endpoint is the S3-compatible endpoint URL, when not AWS S3. @@ -157,7 +164,7 @@ spec: contract (conditional create/overwrite, read-after-write); see docs/celld-behaviors.md for the qualified list. maxLength: 1024 - pattern: ^(s3|gs|az)://.+ + pattern: ^(s3|gs|az)://[a-zA-Z0-9][a-zA-Z0-9._-]*(/[a-zA-Z0-9._/-]+)?$ type: string region: description: region is the storage region, when it cannot be inferred. @@ -182,13 +189,22 @@ spec: - message: endpoint applies to s3:// buckets only; celld rejects an endpoint for gs:// and az:// rule: '!has(self.endpoint) || self.endpoint.size() == 0 || self.name.startsWith(''s3://'')' + - message: bucket identity is immutable; migrate to a new WorkerApp + rule: self.name == oldSelf.name + - message: storage account is immutable + rule: has(self.storageAccount) == has(oldSelf.storageAccount) && + (!has(self.storageAccount) || self.storageAccount == oldSelf.storageAccount) + - message: bucket endpoint is immutable + rule: has(self.endpoint) == has(oldSelf.endpoint) && (!has(self.endpoint) + || self.endpoint == oldSelf.endpoint) celld: description: CelldSpec pins the celld runtime for the fleet. properties: image: description: |- - image is the celld container image, tag included. Mixed-version fleets - are never created; changing this triggers the strategy below. + image is an explicitly versioned celld image. Known incompatible versions + require Recreate; compatible releases roll one pod at a time. + minLength: 1 type: string updateStrategy: default: Rolling @@ -201,12 +217,33 @@ spec: required: - image type: object + clusterDomain: + default: cluster.local + description: clusterDomain is the Kubernetes DNS suffix. + maxLength: 253 + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$ + type: string + deployTrackingSecretRef: + description: |- + deployTrackingSecretRef is a dedicated read-only S3 credential for + deploy/current.json. The operator never uses its ambient cloud identity. + minLength: 1 + type: string + deploymentPolicy: + default: Restart + description: deploymentPolicy selects adoption in place or an additional + gated restart. + enum: + - Restart + - InPlace + type: string durability: + default: bucket description: |- durability selects how celld proves a write before acknowledging it: fleet (celld's default: two follower nodes fsync it, the bucket upload follows) or bucket (the write is in the bucket first). Unset - leaves celld's default. Changing it restarts the fleet through the + selects bucket acknowledgement for safety with ephemeral storage. Changing it restarts the fleet through the ordinary gated rollout. enum: - fleet @@ -238,6 +275,18 @@ spec: applies that threshold to the memory its cells hold and keeps its own absolute cap at 95% of the limit on the process RSS (docs/celld-behaviors.md F10). properties: + cpuMillis: + default: 500 + description: cpuMillis is the CPU request per pod. + format: int32 + minimum: 1 + type: integer + ephemeralStorageGi: + default: 10 + description: ephemeralStorageGi bounds writable temporary storage. + format: int32 + minimum: 1 + type: integer maxResidentCells: default: 1000 description: |- @@ -251,9 +300,61 @@ spec: description: memoryGi is the container memory limit per pod, in GiB. format: int32 + maximum: 65536 minimum: 1 type: integer type: object + scheduling: + description: |- + SchedulingSpec controls placement. Persistent fleets always require distinct + hosts; spreadAcrossZones additionally requires distinct zones. + properties: + nodeSelector: + additionalProperties: + type: string + type: object + spreadAcrossZones: + type: boolean + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object service: description: |- service shapes the serving Service (type, annotations) for internal @@ -277,6 +378,21 @@ spec: - NodePort type: string type: object + storage: + description: |- + StorageSpec provisions retained local state per ordinal. Changing storage + configuration requires a new WorkerApp; existing PVCs are never deleted. + properties: + sizeGi: + format: int32 + maximum: 65536 + minimum: 1 + type: integer + storageClassName: + type: string + required: + - sizeGi + type: object telemetry: description: TelemetrySpec controls celld's built-in tracing (CELLD_OTEL). properties: @@ -324,6 +440,7 @@ spec: secretRef names a Secret whose data is mounted and passed via CELLD_VARS_FILE. Rotation is a Secret update plus an ordinary gated rollout; values are never baked into bundles. + minLength: 1 type: string required: - secretRef @@ -340,6 +457,18 @@ spec: - bucket - celld type: object + x-kubernetes-validations: + - message: auto requires an s3 bucket and a deployTrackingSecretRef + rule: self.appVersion != 'auto' || (self.bucket.name.startsWith('s3://') + && has(self.deployTrackingSecretRef)) + - message: fleet durability requires persistent storage and at least three + replicas + rule: '!has(self.durability) || self.durability != ''fleet'' || (has(self.storage) + && self.replicas >= 3 && (!has(self.autoscaling) || !self.autoscaling.enabled + || self.autoscaling.minReplicas >= 3))' + - message: storage is immutable; migrate to a new WorkerApp + rule: has(self.storage) == has(oldSelf.storage) && (!has(self.storage) + || self.storage == oldSelf.storage) status: description: status defines the observed state of WorkerApp properties: @@ -404,6 +533,39 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + deployments: + description: deployments are fresh observations; missing nodes are + not inferred. + items: + description: PodDeployment reports a live node's adoption state. + properties: + draining: + format: int32 + type: integer + generation: + format: int64 + type: integer + name: + type: string + swapping: + format: int64 + type: integer + version: + type: string + required: + - draining + - generation + - name + - swapping + - version + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + expectedAppVersion: + description: expectedAppVersion is the resolved deployment target. + type: string fleet: description: |- FleetStatus aggregates the per-pod /state the operator polls. The fields @@ -455,6 +617,9 @@ spec: required: - spec type: object + x-kubernetes-validations: + - message: WorkerApp name must not exceed 48 characters + rule: size(self.metadata.name) <= 48 served: true storage: true subresources: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 39a5518..fd41cbf 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -92,10 +92,10 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 512Mi requests: cpu: 10m - memory: 64Mi + memory: 128Mi volumeMounts: [] volumes: [] serviceAccountName: controller-manager diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index 9eeea37..fd6cad0 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -11,6 +11,7 @@ metadata: spec: endpoints: - path: /metrics + honorLabels: true port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d19cf64..2082c1e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -50,6 +50,14 @@ rules: - patch - update - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch - apiGroups: - celld-operator.io resources: @@ -76,6 +84,12 @@ rules: - get - patch - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get - apiGroups: - gateway.networking.k8s.io resources: diff --git a/config/samples/celld-operator_v1alpha1_workerapp.yaml b/config/samples/celld-operator_v1alpha1_workerapp.yaml index a0e39fa..a472b96 100644 --- a/config/samples/celld-operator_v1alpha1_workerapp.yaml +++ b/config/samples/celld-operator_v1alpha1_workerapp.yaml @@ -28,7 +28,7 @@ spec: vars: secretRef: chat-vars websockets: true - # durability: fleet # celld's default; "bucket" waits for the upload before acking + durability: bucket # fleet requires storage and at least three replicas # trustForwardedHeaders: true # only behind a proxy that replaces X-Forwarded-Host and -Proto autoscaling: enabled: true diff --git a/dist/chart/templates/crd/celld-operator.io_workerapps.yaml b/dist/chart/templates/crd/celld-operator.io_workerapps.yaml index 9cd701d..68992b0 100644 --- a/dist/chart/templates/crd/celld-operator.io_workerapps.yaml +++ b/dist/chart/templates/crd/celld-operator.io_workerapps.yaml @@ -64,12 +64,9 @@ spec: properties: appVersion: description: |- - appVersion names the application deployment in the fleet bucket - (written there by `celld deploy`). Nodes load a deployment at startup - only, so changing this triggers the gated rollout (docs/celld-behaviors.md). - The sentinel "auto" makes the operator follow the bucket's - deploy/current.json instead: `celld deploy` alone rolls the fleet, - within one poll interval, with no CR edit. + appVersion is the expected deployment, not a runtime pin: celld v0.4 + adopts the bucket pointer in place. auto reads the pointer using the + dedicated tracking credential. Status verifies live node versions. minLength: 1 type: string autoscaling: @@ -80,6 +77,7 @@ spec: scaler and the partition controller never fight over replica count. properties: enabled: + default: false type: boolean maxReplicas: default: 10 @@ -119,6 +117,9 @@ spec: type: integer type: object type: object + x-kubernetes-validations: + - message: minReplicas must not exceed maxReplicas + rule: self.minReplicas <= self.maxReplicas bucket: description: BucketSpec locates the fleet's slice of the object store. properties: @@ -147,8 +148,14 @@ spec: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY for an s3:// bucket, or AZURE_STORAGE_ACCOUNT_KEY for an az:// container. For stores where identity-based auth is unavailable. + minLength: 1 type: string type: object + x-kubernetes-validations: + - message: select at most one credential family + rule: '[has(self.iamRole) && self.iamRole.size() > 0, has(self.secretRef) + && self.secretRef.size() > 0, has(self.azureClientID) && self.azureClientID.size() + > 0].filter(x, x).size() <= 1' endpoint: description: |- endpoint is the S3-compatible endpoint URL, when not AWS S3. @@ -164,7 +171,7 @@ spec: contract (conditional create/overwrite, read-after-write); see docs/celld-behaviors.md for the qualified list. maxLength: 1024 - pattern: ^(s3|gs|az)://.+ + pattern: ^(s3|gs|az)://[a-zA-Z0-9][a-zA-Z0-9._-]*(/[a-zA-Z0-9._/-]+)?$ type: string region: description: region is the storage region, when it cannot be inferred. @@ -189,13 +196,22 @@ spec: - message: endpoint applies to s3:// buckets only; celld rejects an endpoint for gs:// and az:// rule: '!has(self.endpoint) || self.endpoint.size() == 0 || self.name.startsWith(''s3://'')' + - message: bucket identity is immutable; migrate to a new WorkerApp + rule: self.name == oldSelf.name + - message: storage account is immutable + rule: has(self.storageAccount) == has(oldSelf.storageAccount) && + (!has(self.storageAccount) || self.storageAccount == oldSelf.storageAccount) + - message: bucket endpoint is immutable + rule: has(self.endpoint) == has(oldSelf.endpoint) && (!has(self.endpoint) + || self.endpoint == oldSelf.endpoint) celld: description: CelldSpec pins the celld runtime for the fleet. properties: image: description: |- - image is the celld container image, tag included. Mixed-version fleets - are never created; changing this triggers the strategy below. + image is an explicitly versioned celld image. Known incompatible versions + require Recreate; compatible releases roll one pod at a time. + minLength: 1 type: string updateStrategy: default: Rolling @@ -208,12 +224,33 @@ spec: required: - image type: object + clusterDomain: + default: cluster.local + description: clusterDomain is the Kubernetes DNS suffix. + maxLength: 253 + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$ + type: string + deployTrackingSecretRef: + description: |- + deployTrackingSecretRef is a dedicated read-only S3 credential for + deploy/current.json. The operator never uses its ambient cloud identity. + minLength: 1 + type: string + deploymentPolicy: + default: Restart + description: deploymentPolicy selects adoption in place or an additional + gated restart. + enum: + - Restart + - InPlace + type: string durability: + default: bucket description: |- durability selects how celld proves a write before acknowledging it: fleet (celld's default: two follower nodes fsync it, the bucket upload follows) or bucket (the write is in the bucket first). Unset - leaves celld's default. Changing it restarts the fleet through the + selects bucket acknowledgement for safety with ephemeral storage. Changing it restarts the fleet through the ordinary gated rollout. enum: - fleet @@ -245,6 +282,18 @@ spec: applies that threshold to the memory its cells hold and keeps its own absolute cap at 95% of the limit on the process RSS (docs/celld-behaviors.md F10). properties: + cpuMillis: + default: 500 + description: cpuMillis is the CPU request per pod. + format: int32 + minimum: 1 + type: integer + ephemeralStorageGi: + default: 10 + description: ephemeralStorageGi bounds writable temporary storage. + format: int32 + minimum: 1 + type: integer maxResidentCells: default: 1000 description: |- @@ -258,9 +307,61 @@ spec: description: memoryGi is the container memory limit per pod, in GiB. format: int32 + maximum: 65536 minimum: 1 type: integer type: object + scheduling: + description: |- + SchedulingSpec controls placement. Persistent fleets always require distinct + hosts; spreadAcrossZones additionally requires distinct zones. + properties: + nodeSelector: + additionalProperties: + type: string + type: object + spreadAcrossZones: + type: boolean + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object service: description: |- service shapes the serving Service (type, annotations) for internal @@ -284,6 +385,21 @@ spec: - NodePort type: string type: object + storage: + description: |- + StorageSpec provisions retained local state per ordinal. Changing storage + configuration requires a new WorkerApp; existing PVCs are never deleted. + properties: + sizeGi: + format: int32 + maximum: 65536 + minimum: 1 + type: integer + storageClassName: + type: string + required: + - sizeGi + type: object telemetry: description: TelemetrySpec controls celld's built-in tracing (CELLD_OTEL). properties: @@ -331,6 +447,7 @@ spec: secretRef names a Secret whose data is mounted and passed via CELLD_VARS_FILE. Rotation is a Secret update plus an ordinary gated rollout; values are never baked into bundles. + minLength: 1 type: string required: - secretRef @@ -347,6 +464,18 @@ spec: - bucket - celld type: object + x-kubernetes-validations: + - message: auto requires an s3 bucket and a deployTrackingSecretRef + rule: self.appVersion != 'auto' || (self.bucket.name.startsWith('s3://') + && has(self.deployTrackingSecretRef)) + - message: fleet durability requires persistent storage and at least three + replicas + rule: '!has(self.durability) || self.durability != ''fleet'' || (has(self.storage) + && self.replicas >= 3 && (!has(self.autoscaling) || !self.autoscaling.enabled + || self.autoscaling.minReplicas >= 3))' + - message: storage is immutable; migrate to a new WorkerApp + rule: has(self.storage) == has(oldSelf.storage) && (!has(self.storage) + || self.storage == oldSelf.storage) status: description: status defines the observed state of WorkerApp properties: @@ -411,6 +540,39 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + deployments: + description: deployments are fresh observations; missing nodes are + not inferred. + items: + description: PodDeployment reports a live node's adoption state. + properties: + draining: + format: int32 + type: integer + generation: + format: int64 + type: integer + name: + type: string + swapping: + format: int64 + type: integer + version: + type: string + required: + - draining + - generation + - name + - swapping + - version + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + expectedAppVersion: + description: expectedAppVersion is the resolved deployment target. + type: string fleet: description: |- FleetStatus aggregates the per-pod /state the operator polls. The fields @@ -462,6 +624,9 @@ spec: required: - spec type: object + x-kubernetes-validations: + - message: WorkerApp name must not exceed 48 characters + rule: size(self.metadata.name) <= 48 served: true storage: true subresources: diff --git a/dist/chart/templates/manager/manager.yaml b/dist/chart/templates/manager/manager.yaml index 7601ba7..325c9d0 100644 --- a/dist/chart/templates/manager/manager.yaml +++ b/dist/chart/templates/manager/manager.yaml @@ -29,8 +29,22 @@ spec: - name: manager args: {{- range .Values.controllerManager.container.args }} - - {{ . }} + {{- if not (or (hasPrefix "--metrics-bind-address" .) (hasPrefix "--metrics-secure" .) (hasPrefix "--metrics-cert-path" .)) }} + - {{ . | quote }} {{- end }} + {{- end }} + - --metrics-bind-address={{ if .Values.metrics.enable }}:{{ .Values.metrics.port }}{{ else }}0{{ end }} + - --metrics-secure={{ not .Values.metrics.insecure }} + {{- if and .Values.certmanager.enable .Values.metrics.enable }} + {{- if .Values.metrics.insecure }}{{ fail "certmanager requires secure metrics" }}{{ end }} + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + {{- end }} + - --watch-namespace={{ .Values.operator.watchNamespace }} + - {{ printf "--allowed-image-prefixes=%s" (join "," .Values.operator.allowedImagePrefixes) | quote }} + - {{ printf "--allowed-iam-roles=%s" (join "," .Values.operator.allowedIAMRoles) | quote }} + - {{ printf "--allowed-azure-client-ids=%s" (join "," .Values.operator.allowedAzureClientIDs) | quote }} + - {{ printf "--allowed-bucket-endpoints=%s" (join "," .Values.operator.allowedBucketEndpoints) | quote }} + - --httproute-retries={{ .Values.operator.httpRouteRetries }} - --ingress-mode={{ .Values.operator.ingressMode }} - --gateway-name={{ .Values.operator.gatewayName }} - --gateway-namespace={{ .Values.operator.gatewayNamespace }} diff --git a/dist/chart/templates/prometheus/monitor.yaml b/dist/chart/templates/prometheus/monitor.yaml index 779c1bc..bb526c4 100644 --- a/dist/chart/templates/prometheus/monitor.yaml +++ b/dist/chart/templates/prometheus/monitor.yaml @@ -1,5 +1,5 @@ # To integrate with Prometheus. -{{- if .Values.prometheus.enable }} +{{- if and .Values.prometheus.enable .Values.metrics.enable }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -11,8 +11,10 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https + port: {{ if .Values.metrics.insecure }}http{{ else }}https{{ end }} + scheme: {{ if .Values.metrics.insecure }}http{{ else }}https{{ end }} + honorLabels: true + {{- if not .Values.metrics.insecure }} bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: {{- if .Values.certmanager.enable }} @@ -34,6 +36,7 @@ spec: # Development/Test mode (insecure configuration) insecureSkipVerify: true {{- end }} + {{- end }} selector: matchLabels: control-plane: controller-manager diff --git a/dist/chart/templates/rbac/role.yaml b/dist/chart/templates/rbac/role.yaml index badeccf..ee54256 100644 --- a/dist/chart/templates/rbac/role.yaml +++ b/dist/chart/templates/rbac/role.yaml @@ -1,8 +1,11 @@ {{- if .Values.rbac.enable }} --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: {{ if .Values.operator.watchNamespace }}Role{{ else }}ClusterRole{{ end }} metadata: + {{- with .Values.operator.watchNamespace }} + namespace: {{ . }} + {{- end }} labels: {{- include "chart.labels" . | nindent 4 }} name: celld-operator-manager-role @@ -54,21 +57,17 @@ rules: - update - watch - apiGroups: - - gateway.networking.k8s.io + - autoscaling resources: - - httproutes + - horizontalpodautoscalers verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - - keda.sh + - celld-operator.io resources: - - scaledobjects + - workerapps verbs: - create - delete @@ -78,22 +77,29 @@ rules: - update - watch - apiGroups: - - networking.istio.io + - celld-operator.io resources: - - virtualservices + - workerapps/finalizers + verbs: + - update +- apiGroups: + - celld-operator.io + resources: + - workerapps/status verbs: - - create - - delete - get - - list - patch - update - - watch - apiGroups: - - networking.k8s.io + - gateway.networking.k8s.io resources: - - ingresses - - networkpolicies + - gateways + verbs: + - get +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes verbs: - create - delete @@ -103,9 +109,9 @@ rules: - update - watch - apiGroups: - - celld-operator.io + - keda.sh resources: - - workerapps + - scaledobjects verbs: - create - delete @@ -115,19 +121,30 @@ rules: - update - watch - apiGroups: - - celld-operator.io + - networking.istio.io resources: - - workerapps/finalizers + - virtualservices verbs: + - create + - delete + - get + - list + - patch - update + - watch - apiGroups: - - celld-operator.io + - networking.k8s.io resources: - - workerapps/status + - ingresses + - networkpolicies verbs: + - create + - delete - get + - list - patch - update + - watch - apiGroups: - policy resources: @@ -152,4 +169,4 @@ rules: - patch - update - watch -{{- end -}} +{{- end }} diff --git a/dist/chart/templates/rbac/role_binding.yaml b/dist/chart/templates/rbac/role_binding.yaml index 87b3faa..cd353b0 100644 --- a/dist/chart/templates/rbac/role_binding.yaml +++ b/dist/chart/templates/rbac/role_binding.yaml @@ -1,13 +1,16 @@ {{- if .Values.rbac.enable }} apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: {{ if .Values.operator.watchNamespace }}RoleBinding{{ else }}ClusterRoleBinding{{ end }} metadata: + {{- with .Values.operator.watchNamespace }} + namespace: {{ . }} + {{- end }} labels: {{- include "chart.labels" . | nindent 4 }} name: celld-operator-manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole + kind: {{ if .Values.operator.watchNamespace }}Role{{ else }}ClusterRole{{ end }} name: celld-operator-manager-role subjects: - kind: ServiceAccount diff --git a/dist/chart/values.yaml b/dist/chart/values.yaml index e59d28a..106a69e 100644 --- a/dist/chart/values.yaml +++ b/dist/chart/values.yaml @@ -15,10 +15,10 @@ controllerManager: resources: limits: cpu: 500m - memory: 128Mi + memory: 512Mi requests: cpu: 10m - memory: 64Mi + memory: 128Mi livenessProbe: initialDelaySeconds: 15 periodSeconds: 20 @@ -45,6 +45,12 @@ controllerManager: # [OPERATOR]: celld-operator behavior; rendered as manager flags. operator: + watchNamespace: "" + allowedImagePrefixes: ["ghcr.io/denoland/celld:"] + allowedIAMRoles: [] + allowedAzureClientIDs: [] + allowedBucketEndpoints: [] + httpRouteRetries: false # How WorkerApp hostnames are routed: # httproute (Gateway API) | virtualservice (classic Istio) | # ingress (networking.k8s.io/v1) | none @@ -87,12 +93,7 @@ crd: keep: true # [METRICS]: Set to true to generate manifests for exporting metrics. -# To disable metrics export set false, and ensure that the -# ControllerManager argument "--metrics-bind-address=:8443" is removed. -# port/insecure must agree with the --metrics-bind-address and -# --metrics-secure manager args above (e.g. a plain-HTTP scrape target for -# a minimal Prometheus: port 8080, insecure true, and args -# --metrics-bind-address=:8080 --metrics-secure=false). +# Manager flags and monitor transport are derived from these values. metrics: enable: true port: 8443 diff --git a/docs/celld-behaviors.md b/docs/celld-behaviors.md index b63f9a9..7e2fe21 100644 --- a/docs/celld-behaviors.md +++ b/docs/celld-behaviors.md @@ -11,16 +11,16 @@ in this repo and revisit them. | F1 | Two listeners: public (`--listen`, serves the Worker, reserves only `/.well-known/celld/health`) and internal (`--internal-listen`, versioned peer protocol + alpha operator API). The operator API is unauthenticated except its command-specific checks. | Separate Services; the internal listener gets NetworkPolicy + (where Istio enforces) AuthorizationPolicy, and is never routed. | | F2 | Peers dial each other at the `--advertise` address; peer requests carry an HMAC, body signature, clock limit and replay protection, but celld terminates no TLS; docs require a trusted network or encrypted overlay. | Stable per-pod DNS via the headless Service; encryption is the cluster's job (mesh or CNI). | | F3 | One fleet runs one application deployment. | Fleet-per-app tenancy; one WorkerApp = one fleet = one bucket prefix with its own credentials, so a full runtime compromise reaches only that app. | -| F4 | v0.4 reads `deploy/current.json` at startup, polls it every 30 seconds (`CELLD_DEPLOY_POLL_S`), and adopts a new deployment in place; `POST /reload` adopts immediately. Existing requests finish on the old generation and resident Durable Objects move at safe points. The manifest names the runtime features it needs. | `appVersion` remains the CR's expected deployment and `auto` follows the pointer. The pod-template annotation retains a conservative gated restart as an explicit convergence fence, although v0.4 normally adopts before it. Upgrade `celld.image` before publishing an app that requires new runtime features. | +| F4 | v0.4 reads `deploy/current.json` at startup, polls it every 30 seconds (`CELLD_DEPLOY_POLL_S`), and adopts a new deployment in place; `POST /reload` adopts immediately. Existing requests finish on the old generation and resident Durable Objects move at safe points. The manifest names the runtime features it needs. | `appVersion` remains the CR's expected deployment and `auto` follows the pointer. Restart policy retains a conservative gated restart; InPlace policy avoids a restart when only the expected version changes. Completion requires observed deployment convergence. Upgrade `celld.image` before publishing an app that requires new runtime features. | | F5 | For runtime rollouts, v0.4 paces handoff inside the stopping node and gates a replacement's first healthy response on fleet capacity, pressure, and restore backlog. `/state.restoring` remains the activation backlog (`activating` + `activation_waiting` + `capacity_waiting`). | The operator still owns the StatefulSet partition and steps one ordinal at a time, conservatively requiring a live fleet-wide `/state` sweep with `restoring=0` in addition to the replacement's readiness gate. | -| F6 | SIGTERM starts a graceful drain: health flips to 503, new requests get 503+close, and cells hand off in batches (`CELLD_RELEASES`, default 8). `CELLD_DRAIN_TOKEN_WAIT_MS` (default 30000) serializes simultaneous donors; `CELLD_SHUTDOWN_DRAIN_MS` (default 25000) is the no-progress interval; `CELLD_SHUTDOWN_TOTAL_MS` (default 40000) bounds the complete stop. After handoff the node seals its log. | Pin the 40 s celld total bound and set Kubernetes termination grace to 60 s; readiness uses the health path; liveness is TCP because an HTTP probe would kill a draining node. Recreate scales to zero only after every old pod exits. | +| F6 | SIGTERM starts a graceful drain: health flips to 503, new requests get 503+close, and cells hand off in batches (`CELLD_RELEASES`, default 8). `CELLD_DRAIN_TOKEN_WAIT_MS` (default 30000) serializes simultaneous donors; `CELLD_SHUTDOWN_DRAIN_MS` (default 25000) is the no-progress interval; `CELLD_SHUTDOWN_TOTAL_MS` (default 40000) bounds the complete stop. After handoff the node seals its log. | Pin the 40 s celld total bound and set Kubernetes termination grace to 60 s; readiness uses the health path; liveness is TCP because an HTTP probe would kill a draining node. Recreate scales to zero, then waits for every old pod to exit before starting replacements. | | F7 | The bucket must provide conditional create (`If-None-Match: *`), conditional overwrite (`If-Match`/ETag CAS), and read-after-write consistency, enforced atomically. celld speaks the S3, GCS and (since v0.3.0) Azure Blob dialects (`bucket.rs`). Qualified: Amazon S3, Cloudflare R2, Google Cloud Storage, Tigris, Azure Blob Storage; not: MinIO CE, Backblaze B2, Hetzner, DigitalOcean Spaces, Azurite. An `az://` name is a *container*; the account comes from `AZURE_STORAGE_ACCOUNT_NAME` and exactly one credential family (account key, VM/AKS-node managed identity via IMDS, or AKS workload identity); `--endpoint` is rejected for `gs://` and `az://`. Bucket credentials are fleet-admin authority. | Qualified-store guidance in the README; `celld diagnose` + `hack/cas-hammer` for anything else (F14); per-fleet credential scoping. `spec.bucket.storageAccount` renders `AZURE_STORAGE_ACCOUNT_NAME`, `credentialsFrom.azureClientID` wires AKS workload identity, `secretRef` injects an account key; the CRD refuses `az://` without an account and an endpoint on `gs://`/`az://`. | -| F8 | v0.1 ↔ v0.2 cannot mix. v0.2.1 → v0.3.0 is rolling-safe, but v0.3 → v0.2 can lose acknowledged writes unless each v0.3 node sealed its log. v0.3 ↔ v0.4 cannot mix: the v0.4 peer tunnel refuses v0.3 and v0.3 cannot read v0.4 epoch-qualified large Workers KV references. Security fixes land on the latest release only. | `breakingBoundaries` checks every crossed boundary and direction. Rolling across a known hazard is refused with the upstream reason unless the CR says `Recreate`; v0.3 ↔ v0.4 therefore stops every old node before any new one starts. | +| F8 | v0.1 ↔ v0.2 cannot mix. v0.2.1 → v0.3.0 is rolling-safe, but v0.3 → v0.2 can lose acknowledged writes unless each v0.3 node sealed its log. v0.3 ↔ v0.4 cannot mix: the v0.4 peer tunnel refuses v0.3 and v0.3 cannot read v0.4 epoch-qualified large Workers KV references. Security fixes land on the latest release only. | `breakingBoundaries` checks every crossed boundary and direction. Downgrades from v0.3+ to older runtimes are blocked even with Recreate because pod disappearance cannot prove sealing. Other known rolling hazards require `Recreate`; v0.3 ↔ v0.4 therefore stops every old node before any new one starts. | | F9 | Telemetry (`CELLD_OTEL=1`) writes Parquet traces/logs to the fleet bucket or OTLP; W3C `traceparent` propagates both ways. celld exports no metrics endpoint. | Telemetry env wired from `spec.telemetry`; the operator polls `/state` and re-exports Prometheus metrics itself. | | F10 | No rebalancing on node join; placement is traffic-driven. v0.4's pressure threshold (`CELLD_MAX_RSS_MB`, default 80% of available memory) uses the greater of allocator-adjusted RSS (`in_use_bytes`) and active cgroup memory (`cgroup_working_set_bytes`). The absolute 95% cap uses complete cgroup charge (`cgroup_current_bytes`), falling back to process RSS when cgroup data is unavailable. `0` disables both. | Set `CELLD_MAX_RSS_MB` to 80% of the container limit. Export all four memory measurements; autoscaling scales up early and treats shedding as the hard out-of-capacity trigger. | | F11 | `celld deploy` = esbuild + `wrangler.jsonc`/`.json` with a strict key allowlist; v0.4 adds Workers KV, Queues, Workflows, R2 bindings, and broader compatibility settings. Unknown keys fail loudly; `--dry-run` bundles without writing. | CI-validated deploys; the operator never touches the build. | | F12 | Self-fencing: a node whose lease lapses, or whose core actor or replication process dies, logs a `SELF-FENCE:` line and exits with code **3**. The state is terminal, and upstream **requires** a supervisor that restarts the process without an attempt limit and waits at least one lease lifetime (`CELLD_TTL_MS`, default 10 s) between attempts. | Kubernetes restarts the container without a limit, but CrashLoopBackOff is sufficient only when the effective kubelet delay before each retry is at least `CELLD_TTL_MS`. The default 10 s initial delay covers the default lease lifetime; a larger `CELLD_TTL_MS` requires a correspondingly longer kubelet backoff or an explicit supervisor delay. The operator exports `celld_container_restarts` and `celld_self_fenced` (last exit code 3) so a fence loop is visible rather than silent. | -| F13 | Fleet durability (`CELLD_DURABILITY`, default `fleet` since v0.3.0; `bucket` is the pre-0.3 behavior): a write is acknowledged when **two follower nodes** hold it on local disk (write-all, ack-all) or the bucket upload proves it, whichever first; the bucket remains the tiering target. A follower failure degrades the leader to bucket proofs until it re-recruits; a takeover of a dead leader recovers from at least one reachable sealed follower before it restores. A one-node fleet behaves as `bucket`. | `spec.durability` pins the proof; a soft hostname topology spread keeps a leader and its followers off one host, since until the upload lands an acknowledged write exists only on their emptyDirs; the per-pod `CELLD_WATCH` emptyDir is otherwise unchanged (its loss is the node-failure case upstream designs for). | +| F13 | Fleet durability (`CELLD_DURABILITY`, default `fleet` since v0.3.0; `bucket` is the pre-0.3 behavior): a write is acknowledged when **two follower nodes** hold it on local disk (write-all, ack-all) or the bucket upload proves it, whichever first; the bucket remains the tiering target. A follower failure degrades the leader to bucket proofs until it re-recruits; a takeover of a dead leader recovers from at least one reachable sealed follower before it restores. A one-node fleet behaves as `bucket`. | The operator defaults to bucket acknowledgement. Fleet mode requires PVCs, at least three replicas, and required host separation; optional required zone separation is available. Claims are retained across scale-down and deletion. | | F14 | Startup storage probe (`CELLD_STORAGE_PROBE`, default on): every node writes and deletes one small object under `probe/` to test the four conditional-write transitions (create, reject-create, update, reject-stale) and stops if the store fails any — a store that ignores the precondition would otherwise make the node self-fence in a loop. `celld diagnose` runs the same test (`--read-only` skips it). celld reserves `probe/`, `cells/`, `nodes/`, `node-cells/`, `fleet/`, `deploy/`, `deploy-blobs/`, `wake/` and `telemetry/` under the prefix. | Per-fleet credentials must allow put and delete under the prefix (they already must for cells); README qualification guidance is `celld diagnose` for the sequential contract and `hack/cas-hammer` for atomicity under racers (S3 API only). | ## Wire-format notes (learned from source, not docs) diff --git a/docs/production-audit.md b/docs/production-audit.md index e917629..18b9bc3 100644 --- a/docs/production-audit.md +++ b/docs/production-audit.md @@ -1,234 +1,59 @@ -# Production audit — 2026-09-10 - -Scope: operator source at `1905ee7`, CRD/API, Helm and Kustomize packaging, -CI, tests, and the storage qualification helper. The celld integration was -cross-checked against the adjacent celld checkout at tag `v0.4.0`, notably -`actor.rs`, `generation.rs`, `main.rs`, and `docs/guarantees.md`. No production -cluster or application bucket was accessed. This is a source review and test -report, not a certification of storage, CNI, mesh, or runtime behavior. - -## Cleanup included - -- Upgrade `golang.org/x/text` to v0.39.0 and gRPC to v1.82.1 (with their - required transitive updates). The initial govulncheck v1.8.0 scan reported - reachable affected symbols for [GO-2026-5970](https://pkg.go.dev/vuln/GO-2026-5970) - and [GO-2026-6061](https://pkg.go.dev/vuln/GO-2026-6061). These are static - call-graph findings; exploitability was not reproduced. - -- Reject updates to typed resources, optional integration resources, and - StatefulSets unless their controller owner UID is the current WorkerApp. - ScaledObject cleanup also checks ownership and uses UID/resource-version - delete preconditions. Previously, name collisions could overwrite or delete - unrelated objects, including resources left by a previous WorkerApp UID. - Existing installations with unowned children now get an error: inspect and - resolve the collision rather than having the operator silently adopt it. -- Skip reconciliation of deleting WorkerApps and skip unchanged status writes. - The latter avoids status events repeatedly waking the primary resource watch. -- Preserve allocated Service node ports and foreign ServiceAccount annotations. - In `iamRole: auto` mode, preserve the externally provisioned identity promised - by the existing condition message; explicit identity configuration still wins. -- Watch owned networking.k8s.io Ingress objects, retry failed ScaledObject - cleanup, distinguish a missing map key from an empty value, handle IPv6 - addresses in the state client, and tolerate nonpositive poll intervals. - IPv6 URL handling alone does not provide IPv6-only fleet support: listener - addresses still bind IPv4 and cluster DNS configuration is hardcoded. -- Correct misleading internal comments about deployment polling and settle time. -- Fix `cas-hammer` to read the seed ETag once before releasing writers. Reading - an ETag independently in each writer permitted valid sequential writes to - appear as multiple winners. Reject zero rounds or fewer than two writers. - -## Feature changes recommended before production - -Priorities below describe impact when the stated trigger applies. They are -not claims that every installation will experience the failure. - -### P1: Protect acknowledged writes from correlated node loss - -[`buildPodTemplate`](../internal/controller/fleet_resources.go) uses `emptyDir` -for local state and `ScheduleAnyway` hostname spread. With fleet durability, -acknowledged writes may still reside only on the owner/follower disks before -bucket upload. Soft placement permits those copies on one host, and there is -no zone constraint. A correlated loss can remove every unuploaded copy. - -Add an explicit production storage/scheduling profile: persistent local state, -required host/zone placement with a documented minimum topology, configurable -resources and ephemeral-storage limits, and a tested recovery procedure. For -workloads that need bucket-backed acknowledgement regardless of placement, -use the existing `spec.durability: bucket` and measure the latency tradeoff. -Test host loss and fleet restart while writes are being acknowledged. Upstream's -[durability contract](https://github.com/denoland/celld/blob/v0.4.0/docs/guarantees.md) -should define the acceptance criteria. - -### P1: Make KEDA pause an acknowledged step before changing the fleet - -[`Reconcile`](../internal/controller/workerapp_controller.go) changes the -StatefulSet before calling `ensureScaledObject`. A pause error therefore occurs -after a template or replica change. Even a successful annotation write does not -prove KEDA/HPA has stopped changing replicas. `recreateStep` also relies on -StatefulSet status replica count to decide that old pods have exited; it does -not list old pods or check status freshness at that transition. - -Persist a rollout phase that requests pause, observes KEDA's paused state, -then changes the StatefulSet. Confirm old pods (including terminating pods) -are absent before starting an incompatible runtime. Resume scaling only after -convergence. Cover delayed KEDA reconciliation, API write failures, concurrent -scale-up/down, controller restart, and rapid spec changes in integration tests. -Treat lease/durability shutdown verification for lossy downgrades separately: -`Recreate` by itself does not prove every old node successfully sealed its log. - -### P1: Scale only from complete, fresh fleet measurements - -[`StatePoller.sweep`](../internal/controller/fleetstate.go) resets gauges before -sequentially polling every pod. Scrapes can observe an empty or partial fleet; -unreachable pods disappear from utilization averages. Poll time can grow by -three seconds per unreachable pod. [`promTrigger`](../internal/controller/fleet_resources.go) -does not set `ignoreNullValues`, so KEDA's default tolerates empty results. -The latency query also filters service name without destination namespace, -mixing identically named apps in different namespaces. - -Publish complete snapshots atomically, bound concurrent polling, expose sample -age/completeness, and prohibit scale-down on missing or stale data. Set -`ignoreNullValues: "false"`, namespace-scope latency queries, and test Prometheus -outage, partial pod failure, and leader transitions. See the -[KEDA Prometheus scaler contract](https://keda.sh/docs/2.18/scalers/prometheus/). - -### P1: Fail closed on invalid state responses at rollout gates - -[`StateClient.Fetch`](../internal/controller/fleetstate.go) accepts any JSON -object that decodes into `PodState`. Missing counters become zero: `{}` or an -error object is a successful sample with `restoring=0`. celld's `actor.rs` -can return an `actor_stopped` error JSON. The rollout sweep also skips pods -that lack an IP or are not Running, and does not require a complete census -of expected old ordinals before releasing a replacement. - -Validate the required safety fields and explicit runtime errors, bound response -size, and represent unavailable state distinctly from zero. Define which -unhealthy pods may safely be skipped; gate on expected membership, readiness, -and observed revision. Test malformed responses, actor failure, missing pods, -and overlapping pod termination. Introduce a persisted settle deadline if a -minimum dwell time is required: `RequeueAfter` cannot enforce one because -watch events may arrive earlier. - -### P1: Define and enforce the tenant/security boundary - -The fleet container has no security context and the celld v0.4 image has no -non-root USER. Pods do not meet Restricted Pod Security requirements and mount -a service-account token by default. WorkerApp writers can select arbitrary -images, reference namespace Secrets, choose IAM role annotations, and supply an -S3 endpoint. Deploy tracking can use the operator's ambient AWS credentials -against that endpoint. NetworkPolicy permits the entire operator namespace to -access the unauthenticated internal API; same-namespace fleet labels are not -an identity boundary for users allowed to create arbitrary pods. - -Treat WorkerApp creation as privileged workload deployment until admission and -RBAC boundaries are implemented. Add non-root writable-volume support, seccomp, -capability drops, token opt-out with workload-identity compatibility, restricted -image/role/endpoint policies, namespace-scoped operator deployments where -needed, and narrowly selected operator ingress. Verify CNI enforcement and -encrypted peer transport in the target cluster. Do not grant broad ambient -bucket authority to an operator serving untrusted WorkerApp authors. - -### P2: Separate expected deployment from observed serving version - -[`updateStatus`](../internal/controller/workerapp_controller.go) sets -`rolledOutAppVersion` from the desired annotation once StatefulSet replica -counts converge. It never reads `/state.deployment.version`. celld v0.4 polls -`deploy/current.json` and adopts in place, so a pinned CR value does not pin -the running application. A stalled adoption or a second publication can make -status disagree with reality. Some API comments still describe startup-only -loading or assert that all pods serve the reported version. - -Expose expected and per-pod observed deployment versions/generations, including -draining/swapping state; report convergence only after verifying live nodes. -Offer a documented in-place adoption policy versus a conservative restart -policy. Align API descriptions and generated chart schemas with that contract. -See [celld deployment behavior](https://github.com/denoland/celld/blob/v0.4.0/docs/README.md). - -### P2: Make deploy tracking bounded and credential-correct - -[`DeployTracker`](../internal/controller/deploytracker.go) caches by namespace -and name, without UID or bucket configuration. Deletion/recreation or a bucket -change can reuse an unrelated cached version; entries are never evicted. -Failures after TTL retry on every reconcile without a failure backoff. S3 reads -have no operator-level deadline or JSON size bound. With no static Secret the -tracker uses the operator's ambient identity, not the fleet ServiceAccount's -IRSA/AKS identity. GCS and Azure tracking are unsupported. - -Key cache entries by resource identity and relevant configuration, evict deleted -apps, bound requests and response size, back off failures, and choose an explicit -read-only tracking identity per fleet. Implement store adapters or make unsupported -auto tracking an admission error. Test credential rotation and bucket migration; -consider making bucket identity immutable unless a migration workflow is selected. - -### P2: Reconcile removal and actual readiness of integrations - -`ensureIngress` returns early when hostnames are cleared or mode is `none`, -leaving old routes live. Changing ingress mode leaves the previous resource kind. -Merged annotations retain removed WebSocket/certificate/LB settings. Conditions -not emitted on a later pass remain in status (for example a resolved pinned -version mismatch or disabled autoscaling). Successful object creation is labeled -IngressReady without checking Gateway acceptance, references, or programming. - -Track owned integration fields/resources, remove obsolete routes with ownership -checks, retire no-longer-applicable conditions, and derive readiness from actual -integration status. Avoid repeatedly updating HTTPRoute retry fields that the -standard-channel CRD prunes. Add removal/mode-switch and rejected-route tests. - -### P2: Validate API invariants and runtime configuration - -The API allows names too long for derived Services/labels, empty image/Secret -references, contradictory credential families, and inconsistent autoscaling -bounds. Very large `memoryGi` can overflow the int32 RSS calculation. Unknown -image tags bypass the compatibility table. CLI ingress typos silently select -HTTPRoute, and DNS assumes `cluster.local`. - -Add CEL/schema checks and a documented supported image/version policy; expose -cluster domain and scheduling/startup-probe configuration. Handle existing invalid -objects through actionable conditions before tightening admission on upgrades. -Test startup under slow storage; the current TCP liveness probe has no startup -probe to protect lengthy initialization. - -### P2: Make packaging and CI exercise the configuration users install - -The Helm ServiceMonitor always uses HTTPS even when `metrics.insecure` selects -an HTTP Service port. Enabling cert-manager mounts certificates but does not -pass `--metrics-cert-path` to the manager, so it can continue serving a self-signed -certificate while the monitor verifies the configured CA. The chart CI builds -and loads `celld-operator:v0.1.0`, then installs the chart without overriding its -image and without waiting for readiness. It can pass without running the build. - -Unify metrics TLS settings and flags; test HTTP, self-signed development, and -CA-verified configurations. Install the locally built image with `--wait` and -assert the running image. Add a real WorkerApp e2e suite covering admission, -rollout, storage recovery, NetworkPolicy, and autoscaling: the existing e2e suite -checks manager startup and metrics only. Add dependency/container vulnerability -scanning and pin build/CI tools by immutable versions. The e2e command logger also prints a temporary bearer token inside the curl -command; use mounted credentials and redact command output before reusing that -pattern outside disposable clusters. Manager memory limits -(128 MiB) and serial polling need load qualification for the intended fleet count. - -## Validation - -Regression tests cover API-backed no-op status/Service updates, preserved IAM -annotations, resource ownership collisions (including prior WorkerApp UIDs), -IPv4/IPv6 state URLs, and concurrent CAS writers sharing one seed ETag. -`make test` passed (Kubernetes 1.36.2 envtest), `make lint-fix` reported -zero issues, and `helm lint dist/chart` passed. The existing e2e suite passed -both tests on the dedicated `celld-operator-audit-20260910` Kind cluster with -an isolated kubeconfig; cert-manager installation was skipped. The cluster was -removed afterward. Helm certificate rendering was also inspected. -No real application durability, external identity provider, mesh, KEDA, or -production-scale load test was performed in this audit. - -The final govulncheck v1.8.0 scan (Go 1.26.8) reports zero reachable affected -symbols. It still lists these advisories without detected affected call paths; -track compatible upgrades, since this result is not a proof of non-exploitability: - -| Dependency | Remaining advisory | First fixed version | -| --- | --- | --- | -| cel-go v0.26.0 | [GO-2026-6094](https://pkg.go.dev/vuln/GO-2026-6094) | v0.30.0 | -| OpenTelemetry v1.43.0 | [GO-2026-5158](https://pkg.go.dev/vuln/GO-2026-5158) | v1.44.0 | -| x/mod v0.37.0 | [GO-2026-6180](https://pkg.go.dev/vuln/GO-2026-6180), [GO-2026-6179](https://pkg.go.dev/vuln/GO-2026-6179) | v0.40.0 | - -`go test -race ./hack/cas-hammer` also passed. Container/base-image and Rust -runtime dependency vulnerability scans were not part of this audit. +# Production audit and hardening — 2026-09-10 + +The audit covered the operator, its celld v0.4.0 integration, API, Helm and +Kustomize packaging, CI, and storage qualification helper. The initial cleanup +was PR #4. The subsequent production hardening addresses the remaining source +findings together. No production cluster or application bucket was accessed. + +## Changes + +| Finding | Implemented protection | +| --- | --- | +| Correlated loss of ephemeral fleet logs | Default to bucket acknowledgement. Fleet durability requires persistent storage and at least three replicas, including the autoscaling minimum. Persistent fleets require different hosts; optional required zone separation is available. PVCs survive scale-down and deletion. | +| KEDA races with rollouts | Request pause and observe KEDA's paused condition and absence of targeting HPAs before mutating an existing fleet. Recreate persists its transition and waits for an uncached census of old pods to be empty. Unsafe pre-v0.3 downgrades are refused even with Recreate. | +| Partial/stale scaling data | Publish immutable metric snapshots atomically, bound concurrent polling and sweep duration, export completeness and timestamp, and gate every scaling query on a complete sample younger than 60 seconds. Empty Prometheus results fail the scaler. Latency queries include destination namespace. | +| Invalid state unlocks rollout | Reject missing required counters, runtime errors, negative counters, oversized responses and trailing JSON. Require every expected ordinal to be owned, ready and reachable. Persist a 10-second settle deadline between partition releases. | +| Excessive workload/credential authority | Use non-root pods, read-only roots, explicit writable volumes, seccomp, dropped capabilities and no automatic service-account token. Restrict runtime images, IAM roles, Azure identities and bucket endpoints through operator allowlists. Support namespace-scoped operation and restrict internal ingress to selected operator pods. | +| Desired version presented as observed | Separate expected version from per-pod observed version, generation, draining and swapping. Report rollout completion only after live convergence. Offer Restart and InPlace deployment policies. | +| Unbounded or wrong-identity tracking | Require an explicit tracking Secret for auto mode, with no ambient credential fallback. Bound requests, pointer size and cache size; include app and credential identity in cache keys, evict on deletion and back off failures. Reject unsupported auto stores. | +| Stale routes, metadata and conditions | Delete obsolete owned routes, track and remove managed annotations, retire obsolete conditions, and check fresh HTTPRoute acceptance/reference resolution plus Gateway programming. Retry configuration is opt-in; detected schema pruning stops repeated writes and reports failure. | +| Weak configuration validation | Add schema/CEL and runtime checks for credentials, names, replica bounds, resource sizes, storage and image tags. Make bucket identity and storage configuration immutable. Add cluster domain, CPU/ephemeral budgets, scheduling and startup probes. Reject invalid CLI modes and intervals. | +| Packaging and supply chain gaps | Synchronize generated chart schemas/RBAC, derive consistent metrics TLS flags, test chart modes, install the actual CI-built image with readiness waiting, isolate e2e clusters, stop logging command arguments containing credentials, pin build tools/images, and add module and container vulnerability scans. Upgrade the remaining affected modules. | + +## Upgrade and operational requirements + +Read [production operation and migration](production.md) before upgrading. +Admission and security defaults intentionally reject configurations previously +accepted. In particular, existing fleet-durability deployments without PVCs +need an explicit migration, custom images and identities need allowlists, and +auto tracking needs a dedicated read-only Secret. + +The operator does not make WorkerApp authors untrusted tenants. They can deploy +workloads and reference Secrets in their namespace. Namespace RBAC, scoped +bucket credentials, storage behavior, encrypted peer networking and CNI policy +enforcement remain platform responsibilities. IPv6 state URLs are supported; +the runtime listeners currently require IPv4-capable pod networking. + +## Validation and limits + +Unit and envtest regressions cover malformed state, incomplete fleets, persisted +settle gates, KEDA pause acknowledgement and live HPAs, recreate pod disappearance, +observed deployment convergence, credential rotation/cache isolation, resource +ownership, route/annotation cleanup, PVC/security profiles and guarded queries. +Helm rendering tests cover HTTP, self-signed TLS, cert-manager TLS, disabled +metrics and namespace-scoped RBAC. + +The dedicated Kind lifecycle suite passed three tests: manager readiness, +authenticated metrics, and WorkerApp admission, image replacement and PVC +persistence. The last test uses a deterministic HTTP runtime fixture; it verifies +operator behavior, not celld log recovery. Its cluster and kubeconfig were removed. + +The module vulnerability scan reports no known vulnerabilities in the resolved +module graph. The local Trivy scan of the rebuilt image reports zero HIGH/CRITICAL findings +in its OS and Go binary after upgrading gRPC to v1.83.2; CI repeats that scan. +Race checks passed for the controller and CAS helper. These results do not cover the separately deployed Rust runtime. + +Real celld host/zone loss under acknowledged writes, cloud workload identity, +CNI/mesh enforcement, live KEDA/Prometheus outages, CA-verified scraping and +production-scale load still require qualification on the intended platform. +Fake-client pause tests and rendered TLS tests do not substitute for those tests. diff --git a/docs/production.md b/docs/production.md new file mode 100644 index 0000000..a91302b --- /dev/null +++ b/docs/production.md @@ -0,0 +1,148 @@ +# Production operation and migration + +## Storage and rollout policy + +New WorkerApps default to `durability: bucket`: acknowledgement depends on the +bucket rather than the availability of replicated local logs. Local state may +use emptyDir in this mode. To opt into fleet acknowledgement: + +```yaml +spec: + durability: fleet + replicas: 3 + storage: + sizeGi: 100 + storageClassName: your-durable-class + scheduling: + spreadAcrossZones: true + resources: + memoryGi: 8 + cpuMillis: 2000 + ephemeralStorageGi: 10 + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 10 +``` + +Persistent fleets require distinct hosts. With `spreadAcrossZones`, each pod +also requires a distinct zone: provision enough zones for the maximum replica +count, or leave zone separation disabled and accept that failure domain. +Use a storage class whose failure/recovery guarantees satisfy your workload; +a PVC backed by node-local storage does not survive permanent node loss. +PVCs are retained after scale-down and WorkerApp deletion. Inventory and delete +them only after an explicit data-retention decision. Reusing an application +name can reuse its retained claims; do not do so for an unrelated application. + +Bucket identity and storage configuration are immutable. Existing ephemeral +StatefulSets cannot gain volume claim templates through an in-place update. +Before upgrading an existing fleet-mode app, plan a maintenance migration: +stop writes, verify the runtime has uploaded/sealed acknowledged data, stop the +old fleet, provision a new WorkerApp with the intended storage, and only then +resume traffic. Never overlap fleets sharing a bucket prefix. Do not delete +local state as a substitute for verifying bucket recovery. + +When `spec.autoscaling.enabled` is true, autoscaling requires KEDA v2.12.0 or +newer for the `Paused=True` acknowledgement. The operator pauses KEDA and waits +for its acknowledgement and HPA removal before rolling changes; a stalled pause +blocks rollout. Non-autoscaled WorkerApps use `spec.replicas` and require neither +a KEDA ScaledObject nor a `Paused=True` acknowledgement. Unmanaged HPAs targeting +the same StatefulSet still block rollout. Recreate waits for all old pods, including +terminating pods, to disappear. It cannot prove a killed node sealed its log; +downgrades from v0.3+ to older runtimes are therefore blocked even with Recreate. + +A TCP startup probe allows ten minutes before liveness takes over. Termination +grace is 60 seconds around celld's 40-second total shutdown bound. Slow storage, +startup and failure recovery need testing with your actual runtime and store. + +## Application deployments + +`appVersion` is the expected bucket deployment, not a pin on what celld serves. +celld v0.4 independently follows the bucket pointer. `deploymentPolicy: Restart` +(the default) rolls pods when the expectation changes; `InPlace` lets celld +adopt without that restart. Other pod configuration changes still roll. +`status.expectedAppVersion` records the expectation; `status.deployments` records +observed per-pod versions/generations and transition counts. +`rolledOutAppVersion` is set only when every expected pod serves the expectation +and no generations are draining or swapping. Older runtimes without deployment +state cannot satisfy that gate; use a qualified v0.4 runtime for this contract. + +For auto mode, supply a dedicated read-only S3 tracking credential: + +```yaml +spec: + appVersion: auto + deployTrackingSecretRef: chat-deploy-reader +``` + +The Secret uses `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally +`AWS_SESSION_TOKEN`. Grant read access to the fleet's `deploy/current.json`. +Tracking never uses the manager's ambient identity or assumes the fleet's IRSA +role. Pinned mode can use the same Secret for drift warnings. GCS/Azure fleets +must use an explicit expected version; auto mode is rejected. + +## Security and installation + +Runtime image tags must contain a numeric major.minor.patch version, including +when digest-pinned. Default image policy permits `ghcr.io/denoland/celld:`; +configure trusted prefixes for mirrors or other builds. Unknown live versions +require an explicit Recreate transition. Explicit cloud identities and custom +HTTPS bucket endpoints require exact allowlists: + +```yaml +operator: + watchNamespace: tenant-acme + allowedImagePrefixes: ["ghcr.io/denoland/celld:"] + allowedIAMRoles: ["arn:aws:iam::123456789012:role/celld-chat"] + allowedAzureClientIDs: [] + allowedBucketEndpoints: ["https://ACCOUNT.r2.cloudflarestorage.com"] +``` + +`watchNamespace` scopes the cache and the chart's workload Role. For a shared +Gateway in another namespace, additionally grant the manager ServiceAccount +`get` on that Gateway through a Role/RoleBinding there; otherwise readiness is +Unknown. The manager still needs its metrics authentication and leader-election +permissions. An empty namespace retains cluster-wide workload reconciliation. + +WorkerApp authors are trusted workload deployers. Restrict their Kubernetes +permissions and isolate unrelated tenants in namespaces with separate bucket +credentials. Image allowlists do not sandbox arbitrary Worker code. Pods use +UID/GID 10001, read-only root filesystems and writable `/tmp`/watch volumes; +qualify your runtime image, CSI permissions and identity injector accordingly. +No default service-account token is mounted, but identity webhooks may inject +purpose-specific tokens. NetworkPolicy requires selected manager labels; retain +`control-plane: controller-manager` and `app.kubernetes.io/name: celld-operator` +on manager pods. Verify CNI enforcement and peer-network encryption separately. + +`spec.clusterDomain` supports non-default cluster DNS suffixes. Set the Istio +operator principal separately when its trust domain differs. Runtime listeners +still bind IPv4; IPv6-only pod networking is not currently supported. + +## Ingress, metrics and capacity + +HTTPRoute readiness requires fresh Accepted and ResolvedRefs conditions and a +Programmed Gateway. Ingress/VirtualService creation reports Unknown readiness, +because object creation alone does not prove traffic can flow. Removing hosts +or changing ingress modes removes obsolete owned routes. + +HTTPRoute retries default off for standard Gateway API CRDs. Enable +`operator.httpRouteRetries` only with experimental CRDs and a supporting gateway. +If the API prunes retries, the controller reports failure and stops rewriting +them. Support is retried automatically after a persisted five-minute delay, so upgrading +the CRDs does not require manually clearing the retry latch. Clients +must tolerate rollout 503s unless their ingress is configured to retry them; +WebSocket connections may still close during node replacement. + +The chart derives manager and ServiceMonitor TLS settings from `metrics` and +`certmanager`. Insecure mode uses HTTP; cert-manager mode supplies the certificate +path and CA verification. Do not combine cert-manager with insecure metrics. +The monitor preserves exported namespace/pod labels with `honorLabels`. + +Scaling queries require `celld_fleet_complete = 1` and a +`celld_fleet_sample_timestamp_seconds` younger than 60 seconds. Partial data or +an unavailable Prometheus produces a scaler error instead of a low utilization +sample. Alert on these errors and stale/absent metrics. Keep polling intervals +comfortably below 60 seconds. Polls have 16-way concurrency and a 30-second sweep +budget; large or unhealthy fleets may fail completeness rather than extending +that deadline. Size manager CPU/memory and replica limits using a load test; +the chart's 512 MiB limit is a starting configuration, not a capacity guarantee. diff --git a/go.mod b/go.mod index 1984562..332c1ec 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ezgamehost/celld-operator -go 1.26.0 +go 1.26.8 require ( github.com/aws/aws-sdk-go-v2 v1.43.5 @@ -11,18 +11,20 @@ require ( github.com/onsi/ginkgo/v2 v2.27.4 github.com/onsi/gomega v1.39.0 github.com/prometheus/client_golang v1.23.2 + golang.org/x/sync v0.22.0 k8s.io/api v0.36.1 k8s.io/apimachinery v0.36.1 k8s.io/client-go v0.36.1 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api v1.6.1 + sigs.k8s.io/yaml v1.6.0 ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 // indirect @@ -64,7 +66,7 @@ require ( github.com/go-openapi/swag/typeutils v0.26.0 // indirect github.com/go-openapi/swag/yamlutils v0.26.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/google/cel-go v0.26.0 // indirect + github.com/google/cel-go v0.30.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect @@ -81,35 +83,33 @@ require ( github.com/prometheus/procfs v0.19.2 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.49.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.82.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -123,5 +123,4 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 9c86796..e8ef809 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aws/aws-sdk-go-v2 v1.43.5 h1:yKT5GYnFWhuDo+DqKvE5ZPwVn3RjC4MAeBtZGlh6AVM= github.com/aws/aws-sdk-go-v2 v1.43.5/go.mod h1:wZjAJppCntyOGgVSmgVTfDyRJK5PHOasO6Wsy8U7Axk= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 h1:mn+Vxb9zgz/FE/yDTcFim3DZ1qpcrxR+qBQkBrl6bzA= @@ -118,8 +118,8 @@ github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= -github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/cel-go v0.30.0 h1:ll54AkzKunWkBn9wSoiUXbFZXYZTkdJGNXTBXUoolGo= +github.com/google/cel-go v0.30.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -184,17 +184,10 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -211,20 +204,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -239,34 +232,34 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -276,7 +269,6 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnf gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= diff --git a/hack/sync-chart.py b/hack/sync-chart.py new file mode 100644 index 0000000..1f435e9 --- /dev/null +++ b/hack/sync-chart.py @@ -0,0 +1,12 @@ +"""Copy controller-gen schemas/RBAC into the customized Helm envelopes.""" +from pathlib import Path + +for source, target, key in [ + ("config/crd/bases/celld-operator.io_workerapps.yaml", "dist/chart/templates/crd/celld-operator.io_workerapps.yaml", "spec:"), + ("config/rbac/role.yaml", "dist/chart/templates/rbac/role.yaml", "rules:"), +]: + generated = Path(source).read_text() + template = Path(target).read_text() + prefix = template.split("\n" + key, 1)[0] + body = generated.split("\n" + key, 1)[1] + Path(target).write_text(prefix + "\n" + key + body.rstrip() + "\n{{- end }}\n") diff --git a/internal/controller/deploytracker.go b/internal/controller/deploytracker.go index 969a45e..0e1c243 100644 --- a/internal/controller/deploytracker.go +++ b/internal/controller/deploytracker.go @@ -18,14 +18,16 @@ package controller import ( "context" + "crypto/sha256" "encoding/json" "fmt" + "io" + "net/http" "strings" "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" appsv1 "k8s.io/api/apps/v1" @@ -55,6 +57,8 @@ type deployPointer struct { type trackedVersion struct { version string fetchedAt time.Time + identity string + failure error } // DeployTracker caches per-fleet bucket-pointer reads so reconciles don't @@ -74,8 +78,7 @@ func NewDeployTracker(interval time.Duration) *DeployTracker { } // currentBucketVersion reads deploy/current.json from the fleet's bucket -// using the fleet's own credentials (static keys from the secretRef, or the -// operator's ambient AWS chain otherwise). +// using an explicit read-only tracking Secret; never the ambient AWS chain. func (r *WorkerAppReconciler) currentBucketVersion(ctx context.Context, app *platformv1alpha1.WorkerApp) (string, error) { spec := app.Spec.Bucket rest, ok := strings.CutPrefix(spec.Name, "s3://") @@ -88,27 +91,27 @@ func (r *WorkerAppReconciler) currentBucketVersion(ctx context.Context, app *pla key = strings.TrimSuffix(prefix, "/") + "/" + key } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + ref := app.Spec.DeployTrackingSecretRef + if ref == "" { + return "", fmt.Errorf("deployTrackingSecretRef is required; ambient cloud credentials are never used") + } + var secret corev1.Secret + if err := r.Reader.Get(ctx, types.NamespacedName{Namespace: app.Namespace, Name: ref}, &secret); err != nil { + return "", fmt.Errorf("reading tracking credentials: %w", err) + } + if len(secret.Data["AWS_ACCESS_KEY_ID"]) == 0 || len(secret.Data["AWS_SECRET_ACCESS_KEY"]) == 0 { + return "", fmt.Errorf("tracking Secret requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") + } region := spec.Region if region == "" { - region = defaultBucketRegion - } - loadOptions := []func(*awsconfig.LoadOptions) error{awsconfig.WithRegion(region)} - if ref := spec.CredentialsFrom.SecretRef; ref != "" { - var secret corev1.Secret - if err := r.Reader.Get(ctx, types.NamespacedName{Namespace: app.Namespace, Name: ref}, &secret); err != nil { - return "", fmt.Errorf("reading bucket credentials %s: %w", ref, err) + region = "us-east-1" + if spec.Endpoint != "" { + region = defaultBucketRegion } - loadOptions = append(loadOptions, awsconfig.WithCredentialsProvider( - credentials.NewStaticCredentialsProvider( - string(secret.Data["AWS_ACCESS_KEY_ID"]), - string(secret.Data["AWS_SECRET_ACCESS_KEY"]), - string(secret.Data["AWS_SESSION_TOKEN"]), - ))) - } - cfg, err := awsconfig.LoadDefaultConfig(ctx, loadOptions...) - if err != nil { - return "", err } + cfg := aws.Config{Region: region, Credentials: credentials.NewStaticCredentialsProvider(string(secret.Data["AWS_ACCESS_KEY_ID"]), string(secret.Data["AWS_SECRET_ACCESS_KEY"]), string(secret.Data["AWS_SESSION_TOKEN"])), RetryMaxAttempts: 2, HTTPClient: &http.Client{Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}} client := s3.NewFromConfig(cfg, func(o *s3.Options) { if spec.Endpoint != "" { o.BaseEndpoint = aws.String(spec.Endpoint) @@ -121,8 +124,15 @@ func (r *WorkerAppReconciler) currentBucketVersion(ctx context.Context, app *pla return "", fmt.Errorf("reading %s: %w", key, err) } defer func() { _ = out.Body.Close() }() + raw, err := io.ReadAll(io.LimitReader(out.Body, (64<<10)+1)) + if err != nil { + return "", err + } + if len(raw) > 64<<10 { + return "", fmt.Errorf("deploy pointer exceeds 64 KiB") + } var pointer deployPointer - if err := json.NewDecoder(out.Body).Decode(&pointer); err != nil { + if err := json.Unmarshal(raw, &pointer); err != nil { return "", fmt.Errorf("decoding %s: %w", key, err) } if pointer.Version == "" { @@ -138,24 +148,43 @@ func (r *WorkerAppReconciler) pointerVersion(ctx context.Context, app *platformv key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name} tracker := r.Deploys + var secret corev1.Secret + if ref := app.Spec.DeployTrackingSecretRef; ref != "" { + if err := r.Reader.Get(ctx, types.NamespacedName{Namespace: app.Namespace, Name: ref}, &secret); err != nil { + return "", false, err + } + } + identityBytes, _ := json.Marshal([]any{app.UID, app.Spec.Bucket, app.Spec.DeployTrackingSecretRef, secret.UID, secret.ResourceVersion}) + identity := fmt.Sprintf("%x", sha256.Sum256(identityBytes)) tracker.mu.Lock() cached, have := tracker.cache[key] + have = have && cached.identity == identity tracker.mu.Unlock() if have && time.Since(cached.fetchedAt) < tracker.Interval { - return cached.version, false, nil + return cached.version, cached.failure != nil && cached.version != "", cached.failure } - fresh, err := r.currentBucketVersion(ctx, app) - if err != nil { - if have { - return cached.version, true, err - } - return "", false, err + if err != nil && have { + fresh = cached.version } tracker.mu.Lock() - tracker.cache[key] = trackedVersion{version: fresh, fetchedAt: time.Now()} + // Bound memory even when deletes are missed while the manager is offline. + if len(tracker.cache) >= 1024 { + for k, v := range tracker.cache { + if time.Since(v.fetchedAt) >= tracker.Interval { + delete(tracker.cache, k) + } + } + if len(tracker.cache) >= 1024 { + for k := range tracker.cache { + delete(tracker.cache, k) + break + } + } + } + tracker.cache[key] = trackedVersion{version: fresh, fetchedAt: time.Now(), identity: identity, failure: err} tracker.mu.Unlock() - return fresh, false, nil + return fresh, err != nil && fresh != "", err } // trackableBucket reports whether the operator can read the fleet's deploy @@ -188,7 +217,7 @@ func (r *WorkerAppReconciler) resolveAppVersion(ctx context.Context, app *platfo // static credentials: nodes load what current.json names, so a // pinned version that disagrees with the bucket is a lie waiting // to be served. - if app.Spec.Bucket.CredentialsFrom.SecretRef != "" { + if app.Spec.DeployTrackingSecretRef != "" { if current, _, err := r.pointerVersion(ctx, app); err == nil && current != app.Spec.AppVersion { *conditions = append(*conditions, metav1.Condition{ Type: condDeployTrackingReady, Status: metav1.ConditionFalse, @@ -234,3 +263,12 @@ func (r *WorkerAppReconciler) liveTemplateVersion(ctx context.Context, app *plat } return sts.Spec.Template.Annotations[appVersionAnnotation] } + +func (t *DeployTracker) Forget(key types.NamespacedName) { + if t == nil { + return + } + t.mu.Lock() + delete(t.cache, key) + t.mu.Unlock() +} diff --git a/internal/controller/fleet_resources.go b/internal/controller/fleet_resources.go index 6e70095..84a2583 100644 --- a/internal/controller/fleet_resources.go +++ b/internal/controller/fleet_resources.go @@ -45,6 +45,9 @@ import ( const ( celldContainerName = "celld" + watchVolumeName = "watch" + varsVolumeName = "vars" + annotationTrue = "true" // iamRoleAuto asks the operator to provision the role (not implemented; // surfaced as a condition). @@ -150,9 +153,10 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st // changes. It stays under celld's own absolute RSS cap (95% of the // limit), so the cell-memory threshold keeps the recovery property of // shedding (docs/celld-behaviors.md F10). - rssMb := memGi * 1024 * 4 / 5 + rssMb := int64(memGi) * 1024 * 4 / 5 env := []corev1.EnvVar{ + {Name: "HOME", Value: "/tmp"}, {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, }}, @@ -161,8 +165,8 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st // Stable per-pod DNS via the headless service; peers resolve it to // the internal listener (F2). {Name: "CELLD_ADVERTISE", Value: fmt.Sprintf( - "$(POD_NAME).%s.%s.svc.cluster.local:%d", - internalServiceName(app), app.Namespace, internalPort)}, + "$(POD_NAME).%s.%s.svc.%s:%d", + internalServiceName(app), app.Namespace, clusterDomain(app), internalPort)}, {Name: "CELLD_WATCH", Value: watchDir}, {Name: "CELLD_SHUTDOWN_DRAIN_MS", Value: fmt.Sprintf("%d", shutdownDrainMs)}, {Name: "CELLD_SHUTDOWN_TOTAL_MS", Value: fmt.Sprintf("%d", shutdownTotalMs)}, @@ -182,10 +186,14 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st // field for az://, so the pod spec is the one source of truth. env = append(env, corev1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT_NAME", Value: app.Spec.Bucket.StorageAccount}) } - if app.Spec.Durability != "" { + durability := app.Spec.Durability + if durability == "" { + durability = platformv1alpha1.DurabilityBucket + } + if durability != "" { // Unset keeps celld's default (fleet since v0.3.0), so the fleet // follows upstream unless the CR pins a proof (F13). - env = append(env, corev1.EnvVar{Name: "CELLD_DURABILITY", Value: string(app.Spec.Durability)}) + env = append(env, corev1.EnvVar{Name: "CELLD_DURABILITY", Value: string(durability)}) } if app.Spec.TrustForwardedHeaders { // Off unless asked: celld reads the last value of each header, so @@ -213,19 +221,19 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st } volumes := []corev1.Volume{{ - Name: "watch", + Name: watchVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, }} - mounts := []corev1.VolumeMount{{Name: "watch", MountPath: watchDir}} + mounts := []corev1.VolumeMount{{Name: watchVolumeName, MountPath: watchDir}} if app.Spec.Vars != nil { volumes = append(volumes, corev1.Volume{ - Name: "vars", + Name: varsVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ SecretName: app.Spec.Vars.SecretRef, }}, }) - mounts = append(mounts, corev1.VolumeMount{Name: "vars", MountPath: varsDir, ReadOnly: true}) + mounts = append(mounts, corev1.VolumeMount{Name: varsVolumeName, MountPath: varsDir, ReadOnly: true}) env = append(env, corev1.EnvVar{Name: "CELLD_VARS_FILE", Value: varsDir + "/" + varsKey}) } @@ -238,9 +246,16 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st }) } + if app.Spec.Storage != nil { + volumes = volumes[1:] + } + volumes = append(volumes, corev1.Volume{Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + mounts = append(mounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) container := corev1.Container{ - Name: celldContainerName, - Image: app.Spec.Celld.Image, + SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: ptr.To(false), ReadOnlyRootFilesystem: ptr.To(true), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, + StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(publicPort)}}, PeriodSeconds: 5, FailureThreshold: 120}, + Name: celldContainerName, + Image: app.Spec.Celld.Image, Args: []string{ "--bucket", app.Spec.Bucket.Name, "--listen", fmt.Sprintf("0.0.0.0:%d", publicPort), @@ -274,18 +289,19 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceCPU: *resource.NewMilliQuantity(int64(defaultPositive(app.Spec.Resources.CPUMillis, 500)), resource.DecimalSI), corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", memGi)), }, Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", memGi)), + corev1.ResourceEphemeralStorage: resource.MustParse(fmt.Sprintf("%dGi", defaultPositive(app.Spec.Resources.EphemeralStorageGi, 10))), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", memGi)), }, }, VolumeMounts: mounts, } annotations := map[string]string{} - if appVersion != "" { + if appVersion != "" && app.Spec.DeploymentPolicy != "InPlace" { // The declarative rollout trigger (F4): `celld deploy` publishes // to the bucket, then this annotation changing (a spec bump, or // the tracked pointer moving) rolls the fleet so nodes restart @@ -300,7 +316,7 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st if app.Spec.Bucket.CredentialsFrom.AzureClientID != "" { // Pod-only label (never part of the selector): it is what makes // the AKS webhook mutate the pod. - labels[azureWorkloadIdentityUseLabel] = "true" + labels[azureWorkloadIdentityUseLabel] = annotationTrue } return corev1.PodTemplateSpec{ @@ -309,6 +325,11 @@ func buildPodTemplate(app *platformv1alpha1.WorkerApp, configHash, appVersion st Annotations: annotations, }, Spec: corev1.PodSpec{ + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptr.To(true), RunAsUser: ptr.To(int64(10001)), RunAsGroup: ptr.To(int64(10001)), FSGroup: ptr.To(int64(10001)), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + NodeSelector: app.Spec.Scheduling.NodeSelector, + Tolerations: app.Spec.Scheduling.Tolerations, + Affinity: fleetAffinity(app), ServiceAccountName: fleetName(app), TerminationGracePeriodSeconds: ptr.To(int64(terminationGraceS)), Containers: []corev1.Container{container}, @@ -355,10 +376,12 @@ func buildStatefulSet(app *platformv1alpha1.WorkerApp, configHash, appVersion st }, }, Spec: appsv1.StatefulSetSpec{ - ServiceName: internalServiceName(app), - Replicas: ptr.To(desiredReplicas(app)), - Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(app)}, - Template: template, + VolumeClaimTemplates: fleetClaims(app), + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType}, + ServiceName: internalServiceName(app), + Replicas: ptr.To(desiredReplicas(app)), + Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(app)}, + Template: template, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ Type: appsv1.RollingUpdateStatefulSetStrategyType, RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ @@ -460,7 +483,7 @@ func buildNetworkPolicy(app *platformv1alpha1.WorkerApp, operatorNamespace strin }}, From: []networkingv1.NetworkPolicyPeer{ {PodSelector: &metav1.LabelSelector{MatchLabels: selectorLabels(app)}}, - {NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + {PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"control-plane": "controller-manager", "app.kubernetes.io/name": "celld-operator"}}, NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ "kubernetes.io/metadata.name": operatorNamespace, }}}, }, @@ -654,7 +677,7 @@ func buildVirtualService(app *platformv1alpha1.WorkerApp, gateways []string) *un "gateways": gws, "http": []any{map[string]any{ "route": []any{map[string]any{"destination": map[string]any{ - "host": fmt.Sprintf("%s.%s.svc.cluster.local", fleetName(app), app.Namespace), + "host": fmt.Sprintf("%s.%s.svc.%s", fleetName(app), app.Namespace, clusterDomain(app)), "port": map[string]any{"number": int64(publicPort)}, }}}, "retries": map[string]any{ @@ -731,11 +754,16 @@ func buildScaledObject(app *platformv1alpha1.WorkerApp, prometheusURL string, pa } if as.Targets.P95LatencyMs != nil { triggers = append(triggers, promTrigger(prometheusURL, - fmt.Sprintf(`histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{destination_service_name=%q,reporter="destination"}[5m])) by (le))`, - fleetName(app)), + fmt.Sprintf(`histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{destination_service_name=%q,destination_service_namespace=%q,reporter="destination"}[5m])) by (le))`, + fleetName(app), app.Namespace), fmt.Sprintf("%d", *as.Targets.P95LatencyMs))) } + for _, trigger := range triggers { + metadata := trigger.(map[string]any)["metadata"].(map[string]any) + metadata["query"] = fmt.Sprintf("(%s) and on() (min(celld_fleet_complete{namespace=%q,workerapp=%q}) == 1) and on() (time() - min(celld_fleet_sample_timestamp_seconds{namespace=%q,workerapp=%q}) < 60)", metadata["query"], app.Namespace, app.Name, app.Namespace, app.Name) + } + // Scale down slowly and one pod at a time: removing a pod hands off its // cells (cold restores on peers) and closes its WebSockets; WebSocket // fleets wait even longer (docs/celld-behaviors.md). @@ -784,9 +812,10 @@ func promTrigger(serverAddress, query, threshold string) map[string]any { "type": "prometheus", "metricType": "Value", "metadata": map[string]any{ - "serverAddress": serverAddress, - "query": query, - "threshold": threshold, + "serverAddress": serverAddress, + "query": query, + "threshold": threshold, + "ignoreNullValues": "false", }, } } @@ -798,3 +827,36 @@ func toAnyMap(in map[string]string) map[string]any { } return out } + +func clusterDomain(app *platformv1alpha1.WorkerApp) string { + if app.Spec.ClusterDomain != "" { + return app.Spec.ClusterDomain + } + return "cluster.local" +} +func defaultPositive(value, fallback int32) int32 { + if value > 0 { + return value + } + return fallback +} +func fleetAffinity(app *platformv1alpha1.WorkerApp) *corev1.Affinity { + if app.Spec.Storage == nil { + return nil + } + keys := []string{"kubernetes.io/hostname"} + if app.Spec.Scheduling.SpreadAcrossZones { + keys = append(keys, "topology.kubernetes.io/zone") + } + terms := make([]corev1.PodAffinityTerm, 0, len(keys)) + for _, key := range keys { + terms = append(terms, corev1.PodAffinityTerm{TopologyKey: key, LabelSelector: &metav1.LabelSelector{MatchLabels: selectorLabels(app)}}) + } + return &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: terms}} +} +func fleetClaims(app *platformv1alpha1.WorkerApp) []corev1.PersistentVolumeClaim { + if app.Spec.Storage == nil { + return nil + } + return []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: watchVolumeName}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: app.Spec.Storage.StorageClassName, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(fmt.Sprintf("%dGi", app.Spec.Storage.SizeGi))}}}}} +} diff --git a/internal/controller/fleetstate.go b/internal/controller/fleetstate.go index a026d03..06f3b8c 100644 --- a/internal/controller/fleetstate.go +++ b/internal/controller/fleetstate.go @@ -17,14 +17,22 @@ limitations under the License. package controller import ( + "bytes" "context" "encoding/json" "fmt" + "io" "net" "net/http" "strconv" + "sync" "time" + "golang.org/x/sync/errgroup" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,80 +55,46 @@ const ( labelPod = "pod" ) -var ( - stateLabels = []string{labelNamespace, labelWorkerApp, labelPod} +// A collector publishes immutable snapshots so a scrape cannot observe half +// of a sweep. Replacing the snapshot also removes deleted pod/fleet series. +type stateSnapshot struct { + mu sync.RWMutex + samples []prometheus.Metric +} - metricResidentCells = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_resident_cells", - Help: "Resident (occupied) cells on a fleet pod, from celld /state.", - }, stateLabels) - metricEvicting = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_evicting", - Help: "Cells the pod is currently evicting, from celld /state.", - }, stateLabels) - metricRestoring = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_restoring", - Help: "Cold routes holding or awaiting an activation permit, from celld /state.", - }, stateLabels) - metricUtilization = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_resident_cell_utilization", - Help: "occupied / CELLD_MAX_RESIDENT_CELLS per pod (0..1); the primary autoscaling signal.", - }, stateLabels) - metricShedding = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_shedding", - Help: "1 while the pod reports pressure shedding; the hard out-of-capacity signal.", - }, stateLabels) - metricUp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_state_up", - Help: "1 if the pod's internal /state endpoint answered the last poll.", - }, stateLabels) - metricRSSBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_rss_bytes", - Help: "Resident set size of the celld process, from /state.", - }, stateLabels) - metricInUseBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_in_use_bytes", - Help: "Process RSS minus allocator retention, from /state.", - }, stateLabels) - metricCgroupWorkingSetBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_cgroup_working_set_bytes", - Help: "Active cgroup memory used by celld's v0.4 pressure threshold, from /state.", - }, stateLabels) - metricCgroupCurrentBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_cgroup_current_bytes", - Help: "Complete cgroup charge used by celld's v0.4 absolute memory cap, from /state.", - }, stateLabels) - metricRestarts = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_container_restarts", - Help: "Kubelet restart count of the celld container; celld relies on the supervisor restarting it after a self-fence.", - }, stateLabels) - metricSelfFenced = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "celld_self_fenced", - Help: "1 if the celld container's last termination was a self-fence (exit code 3): it lost its lease or its storage probe failed.", - }, stateLabels) -) +var fleetMetrics = &stateSnapshot{} + +func (s *stateSnapshot) Describe(ch chan<- *prometheus.Desc) { prometheus.DescribeByCollect(s, ch) } +func (s *stateSnapshot) Collect(ch chan<- prometheus.Metric) { + s.mu.RLock() + samples := s.samples + s.mu.RUnlock() + for _, sample := range samples { + ch <- sample + } +} +func (s *stateSnapshot) publish(samples []prometheus.Metric) { + s.mu.Lock() + s.samples = samples + s.mu.Unlock() +} +func init() { metrics.Registry.MustRegister(fleetMetrics) } -// selfFenceExitCode is what celld exits with after logging "SELF-FENCE:" -// (celld docs/fencing.md); the kubelet restarts it, and this is the one -// signal that distinguishes a fence from any other crash. const selfFenceExitCode = 3 -func init() { - metrics.Registry.MustRegister( - metricResidentCells, metricEvicting, metricRestoring, - metricUtilization, metricShedding, metricUp, - metricRSSBytes, metricInUseBytes, - metricCgroupWorkingSetBytes, metricCgroupCurrentBytes, - metricRestarts, metricSelfFenced, - ) +// DeploymentState reports the runtime deployment currently observed on a node. +type DeploymentState struct { + Version string `json:"version"` + Generation int64 `json:"generation"` + Draining []json.RawMessage `json:"draining"` + Swapping int64 `json:"swapping"` } -// PodState is one pod's /state sample. The response schema is celld's alpha -// operator API (actor.rs state_json): keep parsing tolerant, fail per-pod -// not per-fleet, and pin operator and celld releases together -// (docs/celld-behaviors.md). Fields added across releases decode as zero -// or nil so an older node does not break the whole fleet sweep. +// PodState is one /state sample. Safety counters are required; optional +// diagnostic fields remain compatible with older response schemas. type PodState struct { + Deployment *DeploymentState `json:"deployment"` + Occupied int64 `json:"occupied"` Evicting int64 `json:"evicting"` // Restoring is state_json's activation backlog: every cold route that @@ -159,6 +133,8 @@ func NewStateClient() *StateClient { } func (c *StateClient) Fetch(ctx context.Context, podIP string) (*PodState, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() url := "http://" + net.JoinHostPort(podIP, strconv.Itoa(internalPort)) + "/state" req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -172,10 +148,36 @@ func (c *StateClient) Fetch(ctx context.Context, podIP string) (*PodState, error if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("/state returned %d", resp.StatusCode) } + raw, err := io.ReadAll(io.LimitReader(resp.Body, (4<<20)+1)) + if err != nil { + return nil, err + } + if len(raw) > 4<<20 { + return nil, fmt.Errorf("/state exceeds 4 MiB") + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, fmt.Errorf("decoding /state: %w", err) + } + if _, failed := fields["error"]; failed { + return nil, fmt.Errorf("/state reports a runtime error") + } + for _, key := range []string{"occupied", "restoring", "shedding"} { + value, present := fields[key] + if !present || (key != "shedding" && bytes.Equal(bytes.TrimSpace(value), []byte("null"))) { + return nil, fmt.Errorf("/state missing %s", key) + } + } var state PodState - if err := json.NewDecoder(resp.Body).Decode(&state); err != nil { + if err := json.Unmarshal(raw, &state); err != nil { return nil, fmt.Errorf("decoding /state: %w", err) } + if state.Occupied < 0 || state.Restoring < 0 || state.Evicting < 0 { + return nil, fmt.Errorf("/state has negative counters") + } + if d := state.Deployment; d != nil && (d.Version == "" || d.Generation < 1 || d.Swapping < 0) { + return nil, fmt.Errorf("invalid deployment state") + } return &state, nil } @@ -186,19 +188,28 @@ func (c *StateClient) Fetch(ctx context.Context, podIP string) (*PodState, error func (c *StateClient) FleetSweep(ctx context.Context, pods []corev1.Pod) (map[string]*PodState, int64, error) { states := make(map[string]*PodState, len(pods)) var restoring int64 + var mu sync.Mutex + var group errgroup.Group + group.SetLimit(16) for i := range pods { - pod := &pods[i] - if pod.Status.PodIP == "" || pod.Status.Phase != corev1.PodRunning { - continue - } - state, err := c.Fetch(ctx, pod.Status.PodIP) - if err != nil { - return states, restoring, fmt.Errorf("pod %s: %w", pod.Name, err) - } - states[pod.Name] = state - restoring += state.Restoring + pod := pods[i] + group.Go(func() error { + if pod.Status.PodIP == "" || pod.Status.Phase != corev1.PodRunning || !pod.DeletionTimestamp.IsZero() { + return fmt.Errorf("pod %s is unavailable", pod.Name) + } + state, err := c.Fetch(ctx, pod.Status.PodIP) + if err != nil { + return fmt.Errorf("pod %s: %w", pod.Name, err) + } + mu.Lock() + states[pod.Name] = state + restoring += state.Restoring + mu.Unlock() + return nil + }) } - return states, restoring, nil + err := group.Wait() + return states, restoring, err } // StatePoller is a manager Runnable that continuously exports fleet metrics. @@ -212,6 +223,7 @@ type StatePoller struct { func (p *StatePoller) NeedLeaderElection() bool { return true } func (p *StatePoller) Start(ctx context.Context) error { + defer fleetMetrics.publish(nil) log := logf.FromContext(ctx).WithName("state-poller") interval := p.Interval if interval <= 0 { @@ -232,68 +244,68 @@ func (p *StatePoller) Start(ctx context.Context) error { } func (p *StatePoller) sweep(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() var apps platformv1alpha1.WorkerAppList if err := p.Client.List(ctx, &apps); err != nil { return err } - // Reset then repopulate: pods and fleets come and go, and stale series - // would keep autoscaling on dead data. Single writer (leader-only). - metricResidentCells.Reset() - metricEvicting.Reset() - metricRestoring.Reset() - metricUtilization.Reset() - metricShedding.Reset() - metricUp.Reset() - metricRSSBytes.Reset() - metricInUseBytes.Reset() - metricCgroupWorkingSetBytes.Reset() - metricCgroupCurrentBytes.Reset() - metricRestarts.Reset() - metricSelfFenced.Reset() - + var samples []prometheus.Metric + add := func(name string, value float64, labels ...string) { + keys := []string{labelNamespace, labelWorkerApp} + if len(labels) == 3 { + keys = append(keys, labelPod) + } + samples = append(samples, prometheus.MustNewConstMetric(prometheus.NewDesc(name, name, keys, nil), prometheus.GaugeValue, value, labels...)) + } for i := range apps.Items { app := &apps.Items[i] + sampledAt := time.Now() + var sts appsv1.StatefulSet var pods corev1.PodList - if err := p.Client.List(ctx, &pods, - client.InNamespace(app.Namespace), - client.MatchingLabels(selectorLabels(app))); err != nil { - return err + err := p.Client.Get(ctx, client.ObjectKey{Namespace: app.Namespace, Name: fleetName(app)}, &sts) + if err == nil { + err = p.Client.List(ctx, &pods, client.InNamespace(app.Namespace), client.MatchingLabels(selectorLabels(app))) + } + complete := err == nil && ptr.Deref(sts.Spec.Replicas, 0) > 0 && int32(len(pods.Items)) == ptr.Deref(sts.Spec.Replicas, 0) && sts.Status.ObservedGeneration == sts.Generation + for j := range pods.Items { + if !metav1.IsControlledBy(&pods.Items[j], &sts) || !podReady(&pods.Items[j]) { + complete = false + } } - maxCells := float64(maxResidentCells(app)) + states, _, sweepErr := p.State.FleetSweep(ctx, pods.Items) + complete = complete && sweepErr == nil && len(states) == len(pods.Items) + add("celld_fleet_complete", boolToGauge(complete), app.Namespace, app.Name) + add("celld_fleet_sample_timestamp_seconds", float64(sampledAt.Unix()), app.Namespace, app.Name) for j := range pods.Items { pod := &pods.Items[j] - if pod.Status.PodIP == "" || pod.Status.Phase != corev1.PodRunning { + labels := []string{app.Namespace, app.Name, pod.Name} + restarts, fenced := celldRestarts(pod) + add("celld_container_restarts", float64(restarts), labels...) + add("celld_self_fenced", boolToGauge(fenced), labels...) + state, up := states[pod.Name] + add("celld_state_up", boolToGauge(up), labels...) + if !up { continue } - labels := prometheus.Labels{ - labelNamespace: app.Namespace, labelWorkerApp: app.Name, labelPod: pod.Name, + values := map[string]float64{ + "celld_resident_cells": float64(state.Occupied), "celld_evicting": float64(state.Evicting), + "celld_restoring": float64(state.Restoring), "celld_shedding": boolToGauge(state.IsShedding()), + "celld_resident_cell_utilization": float64(state.Occupied) / float64(maxResidentCells(app)), + "celld_rss_bytes": float64(state.RSSBytes), "celld_in_use_bytes": float64(state.InUseBytes), } - // Restart bookkeeping comes from the pod, not /state, so it is - // reported even while the node is down (F12). - restarts, selfFenced := celldRestarts(pod) - metricRestarts.With(labels).Set(float64(restarts)) - metricSelfFenced.With(labels).Set(boolToGauge(selfFenced)) - state, err := p.State.Fetch(ctx, pod.Status.PodIP) - if err != nil { - metricUp.With(labels).Set(0) - continue + if state.CgroupCurrentBytes != nil { + values["celld_cgroup_current_bytes"] = float64(*state.CgroupCurrentBytes) } - metricUp.With(labels).Set(1) - metricResidentCells.With(labels).Set(float64(state.Occupied)) - metricEvicting.With(labels).Set(float64(state.Evicting)) - metricRestoring.With(labels).Set(float64(state.Restoring)) - metricUtilization.With(labels).Set(float64(state.Occupied) / maxCells) - metricShedding.With(labels).Set(boolToGauge(state.IsShedding())) - metricRSSBytes.With(labels).Set(float64(state.RSSBytes)) - metricInUseBytes.With(labels).Set(float64(state.InUseBytes)) if state.CgroupWorkingSetBytes != nil { - metricCgroupWorkingSetBytes.With(labels).Set(float64(*state.CgroupWorkingSetBytes)) + values["celld_cgroup_working_set_bytes"] = float64(*state.CgroupWorkingSetBytes) } - if state.CgroupCurrentBytes != nil { - metricCgroupCurrentBytes.With(labels).Set(float64(*state.CgroupCurrentBytes)) + for name, value := range values { + add(name, value, labels...) } } } + fleetMetrics.publish(samples) return nil } diff --git a/internal/controller/production_test.go b/internal/controller/production_test.go new file mode 100644 index 0000000..db0ace5 --- /dev/null +++ b/internal/controller/production_test.go @@ -0,0 +1,501 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + platformv1alpha1 "github.com/ezgamehost/celld-operator/api/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +const testWorkerAppKind = "WorkerApp" +const testAppHostname = "app.example.com" + +const healthyState = `{"occupied":4,"restoring":0,"shedding":null,"deployment":{"version":"deploy-a","generation":1,"draining":[],"swapping":0}}` + +func productionFixture(t *testing.T) (*WorkerAppReconciler, *platformv1alpha1.WorkerApp, *appsv1.StatefulSet) { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, appsv1.AddToScheme, autoscalingv2.AddToScheme, platformv1alpha1.AddToScheme, gatewayv1.Install, networkingv1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + app := &platformv1alpha1.WorkerApp{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "default", UID: "app"}, Spec: platformv1alpha1.WorkerAppSpec{AppVersion: "deploy-a", Celld: platformv1alpha1.CelldSpec{Image: testCelldImage}, Bucket: platformv1alpha1.BucketSpec{Name: "s3://bucket/app"}}} + sts := buildStatefulSet(app, "", app.Spec.AppVersion) + sts.UID = "sts" + sts.Generation = 1 + sts.OwnerReferences = []metav1.OwnerReference{{APIVersion: platformv1alpha1.SchemeGroupVersion.String(), Kind: testWorkerAppKind, Name: app.Name, UID: app.UID, Controller: ptr.To(true)}} + sts.Status = appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 3, Replicas: 3, UpdateRevision: "new"} + objects := make([]client.Object, 0, 5) + objects = append(objects, app, sts) + for i := range 3 { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%d", sts.Name, i), Namespace: app.Namespace, Labels: selectorLabels(app), OwnerReferences: []metav1.OwnerReference{{APIVersion: appsv1.SchemeGroupVersion.String(), Kind: testStatefulSetKind, Name: sts.Name, UID: sts.UID, Controller: ptr.To(true)}}}, Status: corev1.PodStatus{PodIP: fmt.Sprintf("10.0.0.%d", i+1), Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + pod.Labels["controller-revision-hash"] = "new" + objects = append(objects, pod) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithStatusSubresource(app, sts, &corev1.Pod{}).Build() + r := &WorkerAppReconciler{Client: c, Reader: c, Scheme: scheme, State: responseClient(healthyState), Deploys: NewDeployTracker(time.Minute), IngressMode: IngressModeNone} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + return r, app, sts +} +func responseClient(body string) *StateClient { + return &StateClient{HTTP: &http.Client{Transport: stateRoundTripper(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil + })}} +} +func TestStateValidationFailsClosed(t *testing.T) { + for _, body := range []string{`{}`, `{"error":"actor_stopped"}`, `{"occupied":1,"restoring":null,"shedding":null}`, `{"occupied":1,"restoring":-1,"shedding":null}`, healthyState + `{}`, strings.Repeat(" ", 4<<20) + healthyState} { + if _, err := responseClient(body).Fetch(context.Background(), "10.0.0.1"); err == nil { + t.Fatalf("accepted invalid state %.80s", body) + } + } +} +func TestRollingGateRequiresCompleteFleetAndSettle(t *testing.T) { + r, app, sts := productionFixture(t) + setStsPartition(sts, 3) + sts.Annotations[settleAnnotation] = time.Now().Add(time.Minute).Format(time.RFC3339Nano) + if err := r.Update(context.Background(), sts); err != nil { + t.Fatal(err) + } + out, err := r.rolloutStep(context.Background(), app, sts) + if err != nil || out.Partition != 3 { + t.Fatalf("settle gate moved: %+v %v", out, err) + } + sts.Annotations[settleAnnotation] = time.Now().Add(-time.Minute).Format(time.RFC3339Nano) + if err := r.Update(context.Background(), sts); err != nil { + t.Fatal(err) + } + r.State = responseClient(`{"error":"actor_stopped"}`) + out, err = r.rolloutStep(context.Background(), app, sts) + if err != nil || out.Partition != 3 { + t.Fatalf("invalid state released ordinal: %+v %v", out, err) + } + r.State = responseClient(healthyState) + out, err = r.rolloutStep(context.Background(), app, sts) + if err != nil || out.Partition != 2 { + t.Fatalf("healthy fleet did not progress: %+v %v", out, err) + } + var pod corev1.Pod + if err := r.Get(context.Background(), client.ObjectKey{Namespace: app.Namespace, Name: sts.Name + "-0"}, &pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(context.Background(), &pod); err != nil { + t.Fatal(err) + } + if _, _, err := r.observeFleet(context.Background(), app, sts); err == nil { + t.Fatal("accepted a missing ordinal") + } +} +func TestKEDAPauseWaitsForAcknowledgementAndHPA(t *testing.T) { + r, app, sts := productionFixture(t) + app.Spec.Autoscaling = &platformv1alpha1.AutoscalingSpec{Enabled: true, MinReplicas: 2, MaxReplicas: 10} + paused, err := r.pauseScaling(context.Background(), app) + if err != nil || paused { + t.Fatalf("pause request: %v %v", paused, err) + } + paused, err = r.pauseScaling(context.Background(), app) + if err != nil || paused { + t.Fatalf("pause not acknowledged: %v %v", paused, err) + } + obj := buildScaledObject(app, "", true) + if err := r.Get(context.Background(), client.ObjectKeyFromObject(obj), obj); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal([]metav1.Condition{{Type: "Paused", Status: metav1.ConditionTrue}}) + var conditions []any + if err := json.Unmarshal(raw, &conditions); err != nil { + t.Fatal(err) + } + obj.Object["status"] = map[string]any{"conditions": conditions} + if err := r.Update(context.Background(), obj); err != nil { + t.Fatal(err) + } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "hpa", Namespace: app.Namespace}, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{Kind: testStatefulSetKind, Name: sts.Name}}} + if err := r.Create(context.Background(), hpa); err != nil { + t.Fatal(err) + } + paused, err = r.pauseScaling(context.Background(), app) + if err != nil || paused { + t.Fatalf("live HPA ignored: %v %v", paused, err) + } + if err := r.Delete(context.Background(), hpa); err != nil { + t.Fatal(err) + } + paused, err = r.pauseScaling(context.Background(), app) + if err != nil || !paused { + t.Fatalf("acknowledged pause blocked: %v %v", paused, err) + } +} +func TestRecreateDoesNotTrustStaleReplicaStatus(t *testing.T) { + r, app, sts := productionFixture(t) + sts.Spec.Replicas = ptr.To(int32(0)) + sts.Status.Replicas = 0 + if err := r.Update(context.Background(), sts); err != nil { + t.Fatal(err) + } + desired := buildStatefulSet(app, "changed", "deploy-b") + out, err := r.recreateStep(context.Background(), app, sts, desired, "") + if err != nil || *sts.Spec.Replicas != 0 || out.WaitingOn == "starting new fleet" { + t.Fatalf("started with old pods present: %+v %v", out, err) + } +} +func TestServingVersionMustBeObserved(t *testing.T) { + r, app, sts := productionFixture(t) + out, err := r.steadyFleet(context.Background(), app, sts, "deploy-b") + if err != nil || out.RolledOut { + t.Fatalf("claimed wrong version: %+v %v", out, err) + } + out, err = r.steadyFleet(context.Background(), app, sts, "deploy-a") + if err != nil || !out.RolledOut { + t.Fatalf("did not observe convergence: %+v %v", out, err) + } + app.Spec.DeploymentPolicy = "InPlace" + if templateHash(buildPodTemplate(app, "", "a")) != templateHash(buildPodTemplate(app, "", "b")) { + t.Fatal("in-place publication restarted pods") + } +} +func TestStorageAndSecurityProfile(t *testing.T) { + _, app, _ := productionFixture(t) + app.Spec.Storage = &platformv1alpha1.StorageSpec{SizeGi: 20} + app.Spec.Scheduling.SpreadAcrossZones = true + app.Spec.Vars = &platformv1alpha1.VarsSpec{SecretRef: varsVolumeName} + sts := buildStatefulSet(app, "", "deploy-a") + pod := sts.Spec.Template.Spec + if len(sts.Spec.VolumeClaimTemplates) != 1 || *pod.AutomountServiceAccountToken || !*pod.SecurityContext.RunAsNonRoot || len(pod.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 2 { + t.Fatal("incomplete production profile") + } + foundVars := false + for _, v := range pod.Volumes { + if v.Name == watchVolumeName { + t.Fatal("PVC shadowed by emptyDir") + } + if v.Name == varsVolumeName { + foundVars = true + } + } + if !foundVars { + t.Fatal("PVC removed variables volume") + } + if !*pod.Containers[0].SecurityContext.ReadOnlyRootFilesystem || pod.Containers[0].StartupProbe == nil { + t.Fatal("container is not hardened") + } +} +func TestMetricsSnapshotAndIncompleteFleet(t *testing.T) { + r, app, sts := productionFixture(t) + defer fleetMetrics.publish(nil) + poller := &StatePoller{Client: r.Client, State: r.State} + if err := poller.sweep(context.Background()); err != nil { + t.Fatal(err) + } + registry := prometheus.NewRegistry() + registry.MustRegister(fleetMetrics) + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + complete := false + for _, family := range families { + if family.GetName() == "celld_fleet_complete" { + complete = family.Metric[0].GetGauge().GetValue() == 1 + } + } + if !complete { + t.Fatal("healthy snapshot not complete") + } + var pod corev1.Pod + if err := r.Get(context.Background(), client.ObjectKey{Namespace: app.Namespace, Name: sts.Name + "-0"}, &pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(context.Background(), &pod); err != nil { + t.Fatal(err) + } + if err := poller.sweep(context.Background()); err != nil { + t.Fatal(err) + } + families, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() == "celld_fleet_complete" && family.Metric[0].GetGauge().GetValue() != 0 { + t.Fatal("partial fleet marked complete") + } + } +} +func TestRemovedRoutesAndAnnotations(t *testing.T) { + r, app, _ := productionFixture(t) + app.Spec.Hostnames = []string{testAppHostname} + ingress := buildIngress(app, "", "issuer") + if err := r.ensureObject(context.Background(), app, ingress, func(client.Object, client.Object) {}); err != nil { + t.Fatal(err) + } + if err := r.removeObsoleteRoutes(context.Background(), app); err != nil { + t.Fatal(err) + } + if err := r.Get(context.Background(), client.ObjectKeyFromObject(ingress), &networkingv1.Ingress{}); err == nil { + t.Fatal("obsolete ingress survived") + } + service := buildPublicService(app) + reconcileAnnotations(service, map[string]string{"cloud.example/internal": "true"}) + service.Annotations["foreign"] = "foreign-value" + reconcileAnnotations(service, nil) + if _, exists := service.Annotations["cloud.example/internal"]; exists || service.Annotations["foreign"] != "foreign-value" { + t.Fatal("annotation ownership was not preserved") + } +} +func TestTrackingCredentialRotationAndFailureBackoff(t *testing.T) { + r, app, _ := productionFixture(t) + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + _, _ = io.WriteString(w, `{"version":"deploy-a"}`) + })) + defer server.Close() + app.Spec.Bucket.Endpoint = server.URL + app.Spec.DeployTrackingSecretRef = "tracking" + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "tracking", Namespace: app.Namespace}, Data: map[string][]byte{"AWS_ACCESS_KEY_ID": []byte("id"), "AWS_SECRET_ACCESS_KEY": []byte("secret")}} + if err := r.Create(context.Background(), secret); err != nil { + t.Fatal(err) + } + for range 2 { + if _, _, err := r.pointerVersion(context.Background(), app); err != nil { + t.Fatal(err) + } + } + if calls.Load() != 1 { + t.Fatal("successful read bypassed cache") + } + secret.Data["AWS_SECRET_ACCESS_KEY"] = []byte("rotated") + if err := r.Update(context.Background(), secret); err != nil { + t.Fatal(err) + } + if _, _, err := r.pointerVersion(context.Background(), app); err != nil { + t.Fatal(err) + } + if calls.Load() != 2 { + t.Fatal("rotation did not invalidate cache") + } + app.UID = "new-app" + if _, _, err := r.pointerVersion(context.Background(), app); err != nil { + t.Fatal(err) + } + if calls.Load() != 3 { + t.Fatal("new resource reused old identity") + } + server.Close() + app.Spec.Bucket.Endpoint = "http://127.0.0.1:1" + for range 2 { + if version, _, err := r.pointerVersion(context.Background(), app); err == nil || version != "" { + t.Fatal("new bucket failure reused stale version") + } + } + app.Spec.DeployTrackingSecretRef = "" + if _, err := r.currentBucketVersion(context.Background(), app); err == nil { + t.Fatal("ambient credentials accepted") + } +} +func TestScaleQueriesRequireFreshCompleteData(t *testing.T) { + _, app, _ := productionFixture(t) + app.Spec.Autoscaling = &platformv1alpha1.AutoscalingSpec{Enabled: true, Targets: platformv1alpha1.AutoscalingTargets{P95LatencyMs: ptr.To(int32(100))}} + object := buildScaledObject(app, "http://prometheus", false) + triggers, _, err := unstructured.NestedSlice(object.Object, "spec", "triggers") + if err != nil { + t.Fatal(err) + } + if len(triggers) == 0 { + t.Fatal("missing scaling triggers") + } + for _, item := range triggers { + metadata := item.(map[string]any)["metadata"].(map[string]any) + query := metadata["query"].(string) + if metadata["ignoreNullValues"] != "false" || !strings.Contains(query, "celld_fleet_complete") || !strings.Contains(query, "time()") { + t.Fatal("unsafe scaler query") + } + if strings.Contains(query, "histogram_quantile") && !strings.Contains(query, "destination_service_namespace") { + t.Fatal("latency query crosses namespaces") + } + } +} + +func TestProductionConfigurationValidation(t *testing.T) { + r, base, _ := productionFixture(t) + for _, host := range []string{testAppHostname, "*.example.com"} { + app := base.DeepCopy() + app.Spec.Hostnames = []string{host} + if err := r.validateApp(app); err != nil { + t.Fatalf("valid host %s: %v", host, err) + } + } + cases := map[string]func(*platformv1alpha1.WorkerApp){ + "hostname": func(a *platformv1alpha1.WorkerApp) { a.Spec.Hostnames = []string{"https://example.com"} }, + "image": func(a *platformv1alpha1.WorkerApp) { a.Spec.Celld.Image = "ghcr.io/denoland/celld:latest" }, + "untrusted image": func(a *platformv1alpha1.WorkerApp) { a.Spec.Celld.Image = "example.com/celld:v0.4.0" }, + "ambient tracking": func(a *platformv1alpha1.WorkerApp) { a.Spec.AppVersion = AppVersionAuto }, + "endpoint": func(a *platformv1alpha1.WorkerApp) { a.Spec.Bucket.Endpoint = "https://untrusted.example" }, + "credentials": func(a *platformv1alpha1.WorkerApp) { + a.Spec.Bucket.CredentialsFrom.IAMRole = "auto" + a.Spec.Bucket.CredentialsFrom.SecretRef = "secret" + }, + "ephemeral fleet": func(a *platformv1alpha1.WorkerApp) { a.Spec.Durability = platformv1alpha1.DurabilityFleet }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + app := base.DeepCopy() + mutate(app) + if err := r.validateApp(app); err == nil { + t.Fatal("invalid configuration accepted") + } + }) + } +} + +func TestPrunedHTTPRouteRetryDoesNotRewrite(t *testing.T) { + r, app, _ := productionFixture(t) + r.HTTPRouteRetries = true + r.GatewayName, r.GatewayNamespace = "edge", "infra" + app.Spec.Hostnames = []string{testAppHostname} + route := buildHTTPRoute(app, r.GatewayName, r.GatewayNamespace) + route.OwnerReferences = []metav1.OwnerReference{{APIVersion: platformv1alpha1.SchemeGroupVersion.String(), Kind: testWorkerAppKind, Name: app.Name, UID: app.UID, Controller: ptr.To(true)}} + for i := range route.Spec.Rules { + route.Spec.Rules[i].Retry = nil + } + reconcileAnnotations(route, map[string]string{retryRequestedAnnotation: annotationTrue, retryRecheckAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Create(context.Background(), route); err != nil { + t.Fatal(err) + } + var conditions []metav1.Condition + if retry := r.ensureHTTPRoute(context.Background(), app, &conditions); !retry { + t.Fatal("missing bounded support recheck") + } + var live gatewayv1.HTTPRoute + if err := r.Get(context.Background(), client.ObjectKeyFromObject(route), &live); err != nil { + t.Fatal(err) + } + if live.ResourceVersion != route.ResourceVersion || live.Spec.Rules[0].Retry != nil { + t.Fatal("pruned retry caused a write") + } + if len(conditions) != 1 || conditions[0].Status != metav1.ConditionFalse { + t.Fatal("missing unsupported retry condition") + } +} + +func TestFleetSweepRetainsHealthySamplesAfterPodFailure(t *testing.T) { + // Fill the concurrency limit with unavailable pods before scheduling a healthy + // one, making errgroup cancellation deterministic in the regression case. + pods := make([]corev1.Pod, 17) + for i := range pods { + pods[i].Name = fmt.Sprintf("pod-%d", i) + } + pods[16].Status = corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"} + state := &StateClient{HTTP: &http.Client{Transport: stateRoundTripper(func(req *http.Request) (*http.Response, error) { + if err := req.Context().Err(); err != nil { + return nil, err + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(healthyState)), Header: make(http.Header)}, nil + })}} + states, _, err := state.FleetSweep(context.Background(), pods) + if err == nil || len(states) != 1 || states["pod-16"] == nil { + t.Fatalf("lost healthy sample: states=%v error=%v", states, err) + } +} + +type noKEDAReader struct{ client.Reader } + +func (r noKEDAReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if obj.GetObjectKind().GroupVersionKind().Group == "keda.sh" { + return &meta.NoKindMatchError{GroupKind: obj.GetObjectKind().GroupVersionKind().GroupKind()} + } + return r.Reader.Get(ctx, key, obj, opts...) +} +func TestMissingKEDAStillChecksResidualHPA(t *testing.T) { + r, app, sts := productionFixture(t) + app.Spec.Autoscaling = &platformv1alpha1.AutoscalingSpec{Enabled: true} + r.Reader = noKEDAReader{r.Reader} + paused, err := r.pauseScaling(context.Background(), app) + if err != nil || !paused { + t.Fatalf("missing KEDA blocks fleet: paused=%v err=%v", paused, err) + } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "residual", Namespace: app.Namespace}, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{Kind: testStatefulSetKind, Name: sts.Name}}} + if err := r.Create(context.Background(), hpa); err != nil { + t.Fatal(err) + } + paused, err = r.pauseScaling(context.Background(), app) + if err != nil || paused { + t.Fatalf("residual HPA ignored: paused=%v err=%v", paused, err) + } +} +func TestInvalidSpecRetiresIntegrationConditions(t *testing.T) { + r, app, _ := productionFixture(t) + app.Spec.Celld.Image = "untrusted:v0.4.0" + if err := r.Update(context.Background(), app); err != nil { + t.Fatal(err) + } + for _, kind := range []string{condIngressReady, condAutoscalingReady, condDeployTrackingReady} { + app.Status.Conditions = append(app.Status.Conditions, metav1.Condition{Type: kind, Status: metav1.ConditionTrue, Reason: "Old"}) + } + if err := r.Status().Update(context.Background(), app); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(app)}); err != nil { + t.Fatal(err) + } + if err := r.Get(context.Background(), client.ObjectKeyFromObject(app), app); err != nil { + t.Fatal(err) + } + for _, kind := range []string{condIngressReady, condAutoscalingReady, condDeployTrackingReady} { + if meta.FindStatusCondition(app.Status.Conditions, kind) != nil { + t.Fatalf("stale %s", kind) + } + } + if app.Status.Phase != platformv1alpha1.PhaseDegraded || !meta.IsStatusConditionFalse(app.Status.Conditions, "SpecValid") { + t.Fatal("invalid status lost") + } +} +func TestHTTPRouteRetryLatchExpires(t *testing.T) { + r, app, _ := productionFixture(t) + r.HTTPRouteRetries = true + route := buildHTTPRoute(app, "edge", "infra") + for i := range route.Spec.Rules { + route.Spec.Rules[i].Retry = nil + } + reconcileAnnotations(route, map[string]string{retryRequestedAnnotation: annotationTrue, retryRecheckAnnotation: time.Now().Add(-time.Minute).Format(time.RFC3339Nano)}) + if err := r.Create(context.Background(), route); err != nil { + t.Fatal(err) + } + desired := buildHTTPRoute(app, "edge", "infra") + if !r.configureHTTPRouteRetries(context.Background(), desired) || desired.Annotations[retryRequestedAnnotation] != "" || desired.Spec.Rules[0].Retry != nil { + t.Fatal("expired latch not cleared while retaining pruned spec") + } + reconcileAnnotations(route, desired.Annotations) + if err := r.Update(context.Background(), route); err != nil { + t.Fatal(err) + } + desired = buildHTTPRoute(app, "edge", "infra") + if r.configureHTTPRouteRetries(context.Background(), desired) || desired.Spec.Rules[0].Retry == nil { + t.Fatal("support not retried after latch expiry") + } +} diff --git a/internal/controller/reconciliation_test.go b/internal/controller/reconciliation_test.go index 1b6172d..0968ba1 100644 --- a/internal/controller/reconciliation_test.go +++ b/internal/controller/reconciliation_test.go @@ -58,7 +58,7 @@ func TestOwnershipCollisions(t *testing.T) { obj = newUnstructuredObject("keda.sh/v1alpha1", kind, fleetName(app), app.Namespace, nil, nil, map[string]any{"sentinel": "preserve"}) } if ownedByOther { - obj.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: platformv1alpha1.SchemeGroupVersion.String(), Kind: "WorkerApp", Name: app.Name, UID: "previous-app-uid", Controller: ptr.To(true)}}) + obj.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: platformv1alpha1.SchemeGroupVersion.String(), Kind: testWorkerAppKind, Name: app.Name, UID: "previous-app-uid", Controller: ptr.To(true)}}) } c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() r := &WorkerAppReconciler{Client: c, Reader: c, Scheme: scheme} @@ -103,7 +103,7 @@ func TestStateClientPodAddresses(t *testing.T) { if req.URL.Host != address.host || req.URL.Path != "/state" { t.Fatalf("unexpected URL %s", req.URL) } - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"occupied":2,"restoring":1}`)), Header: make(http.Header)}, nil + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"occupied":2,"restoring":1,"shedding":null}`)), Header: make(http.Header)}, nil })}} state, err := c.Fetch(context.Background(), address.ip) if err != nil { diff --git a/internal/controller/rollout.go b/internal/controller/rollout.go index 9b42e96..f42a49d 100644 --- a/internal/controller/rollout.go +++ b/internal/controller/rollout.go @@ -28,8 +28,12 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -294,6 +298,21 @@ func (r *WorkerAppReconciler) reconcileFleet(ctx context.Context, app *platformv oldImage := stsContainerImage(&existing) imageChanged := oldImage != "" && oldImage != app.Spec.Celld.Image + if templateChanged || stsPartition(&existing) > 0 || existing.Annotations[recreateAnnotation] == annotationTrue || (!autoscalingEnabled(app) && ptr.Deref(existing.Spec.Replicas, 0) != desiredReplicas(app)) { + paused, err := r.pauseScaling(ctx, app) + if err != nil { + return fleetOutcome{}, err + } + if !paused { + return fleetOutcome{Phase: platformv1alpha1.PhaseRollingOut, WaitingOn: "waiting for KEDA pause and HPA removal", Requeue: 5 * time.Second}, nil + } + } + if refusal := runtimeChangeRefusal(app, oldImage); refusal != "" { + return fleetOutcome{Phase: platformv1alpha1.PhaseDegraded, WaitingOn: refusal, Requeue: time.Minute}, nil + } + if existing.Annotations[recreateAnnotation] == annotationTrue { + return r.recreateStep(ctx, app, &existing, desired, "") + } if templateChanged { var breaking string if imageChanged { @@ -310,7 +329,7 @@ func (r *WorkerAppReconciler) reconcileFleet(ctx context.Context, app *platformv oldImage, app.Spec.Celld.Image, breaking), }, nil } - if imageChanged && app.Spec.Celld.UpdateStrategy == platformv1alpha1.UpdateStrategyRecreate { + if app.Spec.Celld.UpdateStrategy == platformv1alpha1.UpdateStrategyRecreate { return r.recreateStep(ctx, app, &existing, desired, breaking) } // Start a gated rolling update: apply the new template frozen at @@ -321,6 +340,7 @@ func (r *WorkerAppReconciler) reconcileFleet(ctx context.Context, app *platformv existing.Annotations = map[string]string{} } existing.Annotations[templateHashAnnotation] = desiredHash + existing.Annotations[settleAnnotation] = time.Now().Add(10 * time.Second).UTC().Format(time.RFC3339Nano) setStsPartition(&existing, liveReplicas) if conflict, err := r.updateFleet(ctx, &existing); err != nil { return fleetOutcome{}, err @@ -338,29 +358,7 @@ func (r *WorkerAppReconciler) reconcileFleet(ctx context.Context, app *platformv return r.rolloutStep(ctx, app, &existing) } - // Steady state. Scale is ours only when KEDA does not own it. - if !autoscalingEnabled(app) && ptr.Deref(existing.Spec.Replicas, 0) != desiredReplicas(app) { - existing.Spec.Replicas = ptr.To(desiredReplicas(app)) - if conflict, err := r.updateFleet(ctx, &existing); err != nil { - return fleetOutcome{}, err - } else if conflict { - return conflictOutcome(app), nil - } - return fleetOutcome{Phase: platformv1alpha1.PhasePending, Requeue: 10 * time.Second}, nil - } - - liveReplicas := ptr.Deref(existing.Spec.Replicas, 0) - converged := existing.Status.ReadyReplicas == liveReplicas && - existing.Status.UpdatedReplicas == liveReplicas && - existing.Status.ObservedGeneration == existing.Generation - if !converged { - return fleetOutcome{ - Phase: platformv1alpha1.PhasePending, - WaitingOn: fmt.Sprintf("fleet: %d/%d ready", existing.Status.ReadyReplicas, liveReplicas), - Requeue: 15 * time.Second, - }, nil - } - return fleetOutcome{Phase: platformv1alpha1.PhaseReady, RolledOut: true, Requeue: 5 * time.Minute}, nil + return r.steadyFleet(ctx, app, &existing, appVersion) } // rolloutStep advances a gated rolling update by at most one ordinal. @@ -380,6 +378,15 @@ func (r *WorkerAppReconciler) rolloutStep(ctx context.Context, app *platformv1al } out := fleetOutcome{Phase: platformv1alpha1.PhaseRollingOut, Partition: partition, Requeue: 10 * time.Second} + if partition <= 0 || sts.Status.ObservedGeneration != sts.Generation { + out.WaitingOn = "waiting for StatefulSet observation" + return out, nil + } + if deadline, err := time.Parse(time.RFC3339Nano, sts.Annotations[settleAnnotation]); err == nil && time.Now().Before(deadline) { + out.WaitingOn = "waiting for rollout settle interval" + out.Requeue = time.Until(deadline) + return out, nil + } // Gate 1: every already-released ordinal (>= partition) runs the new // revision and is Ready. if sts.Status.UpdateRevision == "" { @@ -403,42 +410,21 @@ func (r *WorkerAppReconciler) rolloutStep(ctx context.Context, app *platformv1al } } - // Gate 2: fleet-wide restoring == 0, from a live sweep of every pod's - // internal /state. The cold work lands on the peers, so the whole fleet - // is polled. A pod that is Ready but unreachable holds the gate — never - // step on missing data. A pod that is unreachable AND not Ready holds - // no cells (celld is not serving) and is skipped: otherwise a fleet - // that was never healthy could never roll out the fix for what broke - // it, and the gate becomes a deadlock. - var pods corev1.PodList - if err := r.List(ctx, &pods, client.InNamespace(app.Namespace), client.MatchingLabels(selectorLabels(app))); err != nil { - return fleetOutcome{}, err - } - var restoring int64 - for i := range pods.Items { - pod := &pods.Items[i] - if pod.Status.PodIP == "" || pod.Status.Phase != corev1.PodRunning { - continue - } - state, err := r.State.Fetch(ctx, pod.Status.PodIP) - if err != nil { - if podReady(pod) { - out.WaitingOn = fmt.Sprintf("state unreachable: pod %s: %v", pod.Name, err) - out.Requeue = 15 * time.Second - return out, nil - } - continue - } - restoring += state.Restoring - } - if restoring > 0 { - out.WaitingOn = fmt.Sprintf("fleet: restoring=%d", restoring) - out.Requeue = 15 * time.Second + // Require every expected ordinal, including old pods, to answer a valid + // live state request. Recreate is the explicit recovery path for a fleet + // too unhealthy to satisfy the rolling safety contract. + _, restoring, err := r.observeFleet(ctx, app, sts) + if err != nil || restoring != 0 { + out.WaitingOn = fmt.Sprintf("fleet state: restoring=%d, error=%v", restoring, err) return out, nil } // Both gates pass: release the next ordinal. partition-- + if sts.Annotations == nil { + sts.Annotations = map[string]string{} + } + sts.Annotations[settleAnnotation] = time.Now().Add(10 * time.Second).UTC().Format(time.RFC3339Nano) setStsPartition(sts, partition) if conflict, err := r.updateFleet(ctx, sts); err != nil { return fleetOutcome{}, err @@ -466,6 +452,10 @@ func (r *WorkerAppReconciler) recreateStep(ctx context.Context, app *platformv1a // Phase A: scale the OLD template to zero and let every node drain. if ptr.Deref(existing.Spec.Replicas, 0) != 0 { existing.Spec.Replicas = ptr.To(int32(0)) + if existing.Annotations == nil { + existing.Annotations = map[string]string{} + } + existing.Annotations[recreateAnnotation] = annotationTrue if conflict, err := r.updateFleet(ctx, existing); err != nil { return fleetOutcome{}, err } else if conflict { @@ -477,7 +467,11 @@ func (r *WorkerAppReconciler) recreateStep(ctx context.Context, app *platformv1a } return out, nil } - if existing.Status.Replicas > 0 { + var oldPods corev1.PodList + if err := r.Reader.List(ctx, &oldPods, client.InNamespace(app.Namespace), client.MatchingLabels(selectorLabels(app))); err != nil { + return out, err + } + if existing.Status.ObservedGeneration != existing.Generation || len(oldPods.Items) > 0 || existing.Status.Replicas > 0 { out.WaitingOn = fmt.Sprintf("%d pods draining", existing.Status.Replicas) return out, nil } @@ -488,6 +482,7 @@ func (r *WorkerAppReconciler) recreateStep(ctx context.Context, app *platformv1a existing.Annotations = map[string]string{} } existing.Annotations[templateHashAnnotation] = desired.Annotations[templateHashAnnotation] + delete(existing.Annotations, recreateAnnotation) existing.Spec.Replicas = ptr.To(desiredReplicas(app)) setStsPartition(existing, 0) if conflict, err := r.updateFleet(ctx, existing); err != nil { @@ -500,6 +495,9 @@ func (r *WorkerAppReconciler) recreateStep(ctx context.Context, app *platformv1a } func podReady(pod *corev1.Pod) bool { + if !pod.DeletionTimestamp.IsZero() { + return false + } for _, cond := range pod.Status.Conditions { if cond.Type == corev1.PodReady { return cond.Status == corev1.ConditionTrue @@ -507,3 +505,129 @@ func podReady(pod *corev1.Pod) bool { } return false } + +const settleAnnotation = "celld-operator.io/settle-until" +const recreateAnnotation = "celld-operator.io/recreating" + +func (r *WorkerAppReconciler) observeFleet(ctx context.Context, app *platformv1alpha1.WorkerApp, sts *appsv1.StatefulSet) (map[string]*PodState, int64, error) { + var pods corev1.PodList + if err := r.Reader.List(ctx, &pods, client.InNamespace(app.Namespace), client.MatchingLabels(selectorLabels(app))); err != nil { + return nil, 0, err + } + expected := ptr.Deref(sts.Spec.Replicas, 0) + if expected < 1 || int32(len(pods.Items)) != expected { + return nil, 0, fmt.Errorf("expected %d pods, found %d", expected, len(pods.Items)) + } + names := map[string]bool{} + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, sts) || !podReady(pod) { + return nil, 0, fmt.Errorf("pod %s is not ready or not owned by the fleet", pod.Name) + } + names[pod.Name] = true + } + for i := range expected { + if !names[fmt.Sprintf("%s-%d", sts.Name, i)] { + return nil, 0, fmt.Errorf("missing ordinal %d", i) + } + } + return r.State.FleetSweep(ctx, pods.Items) +} + +// KEDA v2.12.0 or newer is required for the Paused=True acknowledgement. +// KEDA acknowledges a full pause only after stopping its scale loop and +// deleting its HPA. Independently check that no HPA still targets this fleet. +func (r *WorkerAppReconciler) pauseScaling(ctx context.Context, app *platformv1alpha1.WorkerApp) (bool, error) { + obj := newUnstructuredObject("keda.sh/v1alpha1", "ScaledObject", fleetName(app), app.Namespace, nil, nil, nil) + err := r.Reader.Get(ctx, client.ObjectKeyFromObject(obj), obj) + if !autoscalingEnabled(app) { + if err == nil { + var conditions []metav1.Condition + if r.ensureScaledObject(ctx, app, fleetOutcome{}, &conditions) { + return false, fmt.Errorf("could not remove disabled scaler") + } + return false, nil + } + if !apierrors.IsNotFound(err) && !meta.IsNoMatchError(err) { + return false, err + } + } else if !meta.IsNoMatchError(err) { + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + if apierrors.IsNotFound(err) || obj.GetAnnotations()[kedaPausedAnnotation] != annotationTrue { + desired := buildScaledObject(app, r.PrometheusURL, true) + return false, r.ensureUnstructured(ctx, app, desired) + } + if err := requireOwnership(app, obj); err != nil { + return false, err + } + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + paused := false + for _, item := range conditions { + if cond, ok := item.(map[string]any); ok && cond["type"] == "Paused" && cond["status"] == "True" { + paused = true + } + } + if !paused { + return false, nil + } + } + var hpas autoscalingv2.HorizontalPodAutoscalerList + if err := r.Reader.List(ctx, &hpas, client.InNamespace(app.Namespace)); err != nil { + return false, err + } + for _, hpa := range hpas.Items { + if hpa.Spec.ScaleTargetRef.Kind == "StatefulSet" && hpa.Spec.ScaleTargetRef.Name == fleetName(app) { + return false, nil + } + } + return true, nil +} + +func (r *WorkerAppReconciler) steadyFleet(ctx context.Context, app *platformv1alpha1.WorkerApp, existing *appsv1.StatefulSet, appVersion string) (fleetOutcome, error) { + // Steady state. Scale is ours only when KEDA does not own it. + if !autoscalingEnabled(app) && ptr.Deref(existing.Spec.Replicas, 0) != desiredReplicas(app) { + existing.Spec.Replicas = ptr.To(desiredReplicas(app)) + if conflict, err := r.updateFleet(ctx, existing); err != nil { + return fleetOutcome{}, err + } else if conflict { + return conflictOutcome(app), nil + } + return fleetOutcome{Phase: platformv1alpha1.PhasePending, Requeue: 10 * time.Second}, nil + } + + liveReplicas := ptr.Deref(existing.Spec.Replicas, 0) + converged := existing.Status.ReadyReplicas == liveReplicas && + existing.Status.UpdatedReplicas == liveReplicas && + existing.Status.ObservedGeneration == existing.Generation + if !converged { + return fleetOutcome{ + Phase: platformv1alpha1.PhasePending, + WaitingOn: fmt.Sprintf("fleet: %d/%d ready", existing.Status.ReadyReplicas, liveReplicas), + Requeue: 15 * time.Second, + }, nil + } + states, restoring, err := r.observeFleet(ctx, app, existing) + if err != nil || restoring != 0 { + return fleetOutcome{Phase: platformv1alpha1.PhasePending, WaitingOn: fmt.Sprintf("waiting for complete fleet state: restoring=%d, error=%v", restoring, err), Requeue: 15 * time.Second}, nil + } + for name, state := range states { + if state.Deployment == nil || state.Deployment.Version != appVersion || len(state.Deployment.Draining) > 0 || state.Deployment.Swapping != 0 { + return fleetOutcome{Phase: platformv1alpha1.PhasePending, WaitingOn: name + ": deployment not converged", Requeue: 15 * time.Second}, nil + } + } + return fleetOutcome{Phase: platformv1alpha1.PhaseReady, RolledOut: true, Requeue: 30 * time.Second}, nil +} + +func runtimeChangeRefusal(app *platformv1alpha1.WorkerApp, oldImage string) string { + from, fromOK := minorOf(oldImage) + to, toOK := minorOf(app.Spec.Celld.Image) + if fromOK && toOK && from.cmp(minorVersion{0, 3}) >= 0 && to.cmp(minorVersion{0, 3}) < 0 { + return "downgrade requires offline durability verification and migration to a new WorkerApp" + } + if oldImage != app.Spec.Celld.Image && (!fromOK || !toOK) && app.Spec.Celld.UpdateStrategy != platformv1alpha1.UpdateStrategyRecreate { + return "unrecognized live image requires Recreate" + } + return "" +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 3ed099f..dbbd566 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" platformv1alpha1 "github.com/ezgamehost/celld-operator/api/v1alpha1" // +kubebuilder:scaffold:imports @@ -63,6 +64,7 @@ var _ = BeforeSuite(func() { err = platformv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + Expect(gatewayv1.Install(scheme.Scheme)).To(Succeed()) // +kubebuilder:scaffold:scheme By("bootstrapping test environment") diff --git a/internal/controller/validation.go b/internal/controller/validation.go new file mode 100644 index 0000000..c113ca6 --- /dev/null +++ b/internal/controller/validation.go @@ -0,0 +1,98 @@ +package controller + +import ( + "fmt" + "net/url" + "regexp" + "slices" + "strings" + + platformv1alpha1 "github.com/ezgamehost/celld-operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/util/validation" +) + +var runtimeImageTag = regexp.MustCompile(`^v?[0-9]+\.[0-9]+\.[0-9]+([-.+][a-zA-Z0-9.-]+)?$`) + +// Validate before creating children, including objects admitted by older CRDs. +func (r *WorkerAppReconciler) validateApp(app *platformv1alpha1.WorkerApp) error { + if len(app.Name) > 48 || len(validation.IsDNS1123Label(app.Name)) > 0 { + return fmt.Errorf("name must be a DNS label of at most 48 characters") + } + for _, host := range app.Spec.Hostnames { + if len(validation.IsDNS1123Subdomain(host)) > 0 && len(validation.IsWildcardDNS1123Subdomain(host)) > 0 { + return fmt.Errorf("invalid hostname %q", host) + } + } + if app.Spec.AppVersion == "" { + return fmt.Errorf("appVersion is required") + } + image, _, _ := strings.Cut(app.Spec.Celld.Image, "@") + _, tag, tagged := strings.Cut(image[strings.LastIndex(image, "/")+1:], ":") + if !tagged || !runtimeImageTag.MatchString(tag) { + return fmt.Errorf("celld image must include a numeric major.minor.patch tag, including when digest-pinned") + } + creds := app.Spec.Bucket.CredentialsFrom + if app.Spec.AppVersion == AppVersionAuto && (!trackableBucket(app) || app.Spec.DeployTrackingSecretRef == "") { + return fmt.Errorf("auto requires an s3 bucket and deployTrackingSecretRef") + } + for _, ref := range []string{creds.SecretRef, app.Spec.DeployTrackingSecretRef} { + if ref != "" && len(validation.IsDNS1123Subdomain(ref)) > 0 { + return fmt.Errorf("invalid Secret reference") + } + } + if app.Spec.Vars != nil && (app.Spec.Vars.SecretRef == "" || len(validation.IsDNS1123Subdomain(app.Spec.Vars.SecretRef)) > 0) { + return fmt.Errorf("vars.secretRef is required and must be a Secret name") + } + if desiredReplicas(app) < 1 || app.Spec.Resources.MemoryGi < 0 || app.Spec.Resources.MemoryGi > 65536 { + return fmt.Errorf("invalid replica or memory configuration") + } + if as := app.Spec.Autoscaling; as != nil && as.Enabled && defaultPositive(as.MinReplicas, 2) > defaultPositive(as.MaxReplicas, 10) { + return fmt.Errorf("minReplicas exceeds maxReplicas") + } + if app.Spec.Durability == platformv1alpha1.DurabilityFleet && (app.Spec.Storage == nil || desiredReplicas(app) < 3 || (autoscalingEnabled(app) && defaultPositive(app.Spec.Autoscaling.MinReplicas, 2) < 3)) { + return fmt.Errorf("fleet durability requires persistent storage and at least three replicas") + } + if len(validation.IsDNS1123Subdomain(clusterDomain(app))) > 0 { + return fmt.Errorf("invalid clusterDomain") + } + return r.validateTrust(app) +} + +func (r *WorkerAppReconciler) validateTrust(app *platformv1alpha1.WorkerApp) error { + prefixes := r.AllowedImagePrefixes + if len(prefixes) == 0 { + prefixes = []string{"ghcr.io/denoland/celld:"} + } + allowed := false + for _, prefix := range prefixes { + if strings.HasPrefix(app.Spec.Celld.Image, prefix) { + allowed = true + } + } + if !allowed { + return fmt.Errorf("celld image is not in the operator image allowlist") + } + creds := app.Spec.Bucket.CredentialsFrom + count := 0 + for _, value := range []string{creds.IAMRole, creds.SecretRef, creds.AzureClientID} { + if value != "" { + count++ + } + } + if count > 1 { + return fmt.Errorf("select only one bucket credential family") + } + if creds.IAMRole != "" && creds.IAMRole != iamRoleAuto && !slices.Contains(r.AllowedIAMRoles, creds.IAMRole) { + return fmt.Errorf("IAM role is not in the operator allowlist") + } + if creds.AzureClientID != "" && !slices.Contains(r.AllowedAzureClientIDs, creds.AzureClientID) { + return fmt.Errorf("azure identity is not in the operator allowlist") + } + if endpoint := app.Spec.Bucket.Endpoint; endpoint != "" { + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || !slices.Contains(r.AllowedBucketEndpoints, endpoint) { + return fmt.Errorf("bucket endpoint must be an explicitly allowed HTTPS URL without credentials, query, or fragment") + } + } + return nil +} diff --git a/internal/controller/workerapp_controller.go b/internal/controller/workerapp_controller.go index bae5959..db9e652 100644 --- a/internal/controller/workerapp_controller.go +++ b/internal/controller/workerapp_controller.go @@ -18,9 +18,11 @@ package controller import ( "context" + "encoding/json" "fmt" "maps" "reflect" + "slices" "time" appsv1 "k8s.io/api/apps/v1" @@ -72,6 +74,13 @@ const ( // around the unauthenticated internal listener, the PDB, the HTTPRoute on // the shared Gateway, and the KEDA ScaledObject (docs/celld-behaviors.md). type WorkerAppReconciler struct { + AllowedImagePrefixes []string + AllowedIAMRoles []string + AllowedAzureClientIDs []string + AllowedBucketEndpoints []string + // HTTPRouteRetries must be enabled only with experimental Gateway API CRDs. + HTTPRouteRetries bool + client.Client Scheme *runtime.Scheme State *StateClient @@ -113,6 +122,7 @@ type WorkerAppReconciler struct { // +kubebuilder:rbac:groups=celld-operator.io,resources=workerapps,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=celld-operator.io,resources=workerapps/status,verbs=get;update;patch // +kubebuilder:rbac:groups=celld-operator.io,resources=workerapps/finalizers,verbs=update +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=services;serviceaccounts,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch @@ -120,6 +130,7 @@ type WorkerAppReconciler struct { // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies;ingresses,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=keda.sh,resources=scaledobjects,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=security.istio.io,resources=authorizationpolicies,verbs=get;list;watch;create;update;patch;delete @@ -127,19 +138,41 @@ type WorkerAppReconciler struct { // Reconcile drives one WorkerApp toward its spec. func (r *WorkerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() log := logf.FromContext(ctx) app := &platformv1alpha1.WorkerApp{} if err := r.Get(ctx, req.NamespacedName, app); err != nil { + if apierrors.IsNotFound(err) { + r.Deploys.Forget(req.NamespacedName) + } // Deleted: children are owned and garbage-collected. return ctrl.Result{}, client.IgnoreNotFound(err) } if !app.DeletionTimestamp.IsZero() { + r.Deploys.Forget(req.NamespacedName) return ctrl.Result{}, nil } - var conditions []metav1.Condition + conditions := make([]metav1.Condition, 0, 8) + if err := r.validateApp(app); err != nil { + condition := metav1.Condition{Type: "SpecValid", Status: metav1.ConditionFalse, Reason: "InvalidConfiguration", Message: err.Error(), ObservedGeneration: app.Generation} + before := app.DeepCopy() + for _, kind := range []string{condIngressReady, condAutoscalingReady, condDeployTrackingReady} { + meta.RemoveStatusCondition(&app.Status.Conditions, kind) + } + meta.SetStatusCondition(&app.Status.Conditions, condition) + app.Status.Phase = platformv1alpha1.PhaseDegraded + app.Status.RolledOutAppVersion = "" + meta.SetStatusCondition(&app.Status.Conditions, metav1.Condition{Type: "Available", Status: metav1.ConditionFalse, Reason: "InvalidConfiguration", ObservedGeneration: app.Generation}) + if !apiequality.Semantic.DeepEqual(before.Status, app.Status) { + return ctrl.Result{}, r.Status().Update(ctx, app) + } + return ctrl.Result{RequeueAfter: time.Minute}, nil + } + conditions = append(conditions, metav1.Condition{Type: "SpecValid", Status: metav1.ConditionTrue, Reason: "Validated"}) // Foundation objects first: the StatefulSet references the service // account and headless service by name. @@ -247,7 +280,7 @@ func (r *WorkerAppReconciler) ensureFoundation(ctx context.Context, app *platfor l.Spec.Type = d.Spec.Type // Cloud LB configuration rides on annotations; merge so the cloud // controller's own bookkeeping annotations survive. - l.SetAnnotations(mergeStringMaps(l.GetAnnotations(), d.GetAnnotations())) + reconcileAnnotations(l, d.GetAnnotations()) }); err != nil { return fmt.Errorf("public service: %w", err) } @@ -278,6 +311,12 @@ func (r *WorkerAppReconciler) ensureFoundation(ctx context.Context, app *platfor // objects, and unconditional writes amplify every pod-churn burst into an // apiserver write storm. func (r *WorkerAppReconciler) ensureObject(ctx context.Context, app *platformv1alpha1.WorkerApp, desired client.Object, mutate func(live, desired client.Object)) error { + if _, ok := desired.(*corev1.Service); ok { + reconcileAnnotations(desired, desired.GetAnnotations()) + } + if _, ok := desired.(*networkingv1.Ingress); ok { + reconcileAnnotations(desired, desired.GetAnnotations()) + } if err := ctrl.SetControllerReference(app, desired, r.Scheme); err != nil { return err } @@ -313,6 +352,10 @@ func requireOwnership(app *platformv1alpha1.WorkerApp, obj client.Object) error // fast requeue (anything but a missing CRD, which only changes when a // human installs something). func (r *WorkerAppReconciler) ensureIngress(ctx context.Context, app *platformv1alpha1.WorkerApp, conditions *[]metav1.Condition) bool { + if err := r.removeObsoleteRoutes(ctx, app); err != nil { + *conditions = append(*conditions, metav1.Condition{Type: condIngressReady, Status: metav1.ConditionFalse, Reason: reasonRouteError, Message: err.Error()}) + return true + } if len(app.Spec.Hostnames) == 0 { return false } @@ -335,7 +378,7 @@ func (r *WorkerAppReconciler) ensureV1Ingress(ctx context.Context, app *platform l.Spec = d.Spec // Annotations carry the route policy; merge so other controllers' // bookkeeping survives. - l.SetAnnotations(mergeStringMaps(l.GetAnnotations(), d.GetAnnotations())) + reconcileAnnotations(l, d.GetAnnotations()) }) if err != nil { *conditions = append(*conditions, metav1.Condition{ @@ -345,7 +388,7 @@ func (r *WorkerAppReconciler) ensureV1Ingress(ctx context.Context, app *platform return true } *conditions = append(*conditions, metav1.Condition{ - Type: condIngressReady, Status: metav1.ConditionTrue, Reason: "IngressReconciled", + Type: condIngressReady, Status: metav1.ConditionUnknown, Reason: "IngressReconciled", Message: "Ingress configuration reconciled; check ingress controller readiness", }) return false } @@ -356,7 +399,7 @@ func (r *WorkerAppReconciler) ensureVirtualService(ctx context.Context, app *pla switch { case err == nil: *conditions = append(*conditions, metav1.Condition{ - Type: condIngressReady, Status: metav1.ConditionTrue, Reason: "VirtualServiceReconciled", + Type: condIngressReady, Status: metav1.ConditionUnknown, Reason: "VirtualServiceReconciled", Message: "VirtualService reconciled; verify gateway programming", }) case meta.IsNoMatchError(err): *conditions = append(*conditions, metav1.Condition{ @@ -376,6 +419,7 @@ func (r *WorkerAppReconciler) ensureVirtualService(ctx context.Context, app *pla func (r *WorkerAppReconciler) ensureHTTPRoute(ctx context.Context, app *platformv1alpha1.WorkerApp, conditions *[]metav1.Condition) bool { route := buildHTTPRoute(app, r.GatewayName, r.GatewayNamespace) + retryDropped := r.configureHTTPRouteRetries(ctx, route) err := r.ensureObject(ctx, app, route, func(live, desired client.Object) { l, d := live.(*gatewayv1.HTTPRoute), desired.(*gatewayv1.HTTPRoute) l.Spec = d.Spec @@ -385,16 +429,16 @@ func (r *WorkerAppReconciler) ensureHTTPRoute(ctx context.Context, app *platform // Standard-channel Gateway API CRDs silently drop the experimental // retry field; per the fail-loud rule, say so rather than let the // drain-503 retry policy vanish quietly. - if dropped, checkErr := r.httpRouteRetryDropped(ctx, route); checkErr == nil && dropped { + if dropped, checkErr := r.httpRouteRetryDropped(ctx, route); retryDropped || (checkErr == nil && dropped) { *conditions = append(*conditions, metav1.Condition{ - Type: condIngressReady, Status: metav1.ConditionTrue, + Type: condIngressReady, Status: metav1.ConditionFalse, Reason: "RouteReconciledRetryDropped", Message: "cluster Gateway API CRDs dropped the retry field (standard channel); drain 503s are not retried at the gateway", }) - return false + return true // Use the bounded transient requeue while waiting to retry support } *conditions = append(*conditions, metav1.Condition{ - Type: condIngressReady, Status: metav1.ConditionTrue, Reason: "RouteReconciled", + Type: condIngressReady, Status: r.httpRouteReady(ctx, route), Reason: "RouteObserved", }) case meta.IsNoMatchError(err): *conditions = append(*conditions, metav1.Condition{ @@ -559,10 +603,13 @@ func stringMapSubset(sub, of map[string]string) bool { func (r *WorkerAppReconciler) updateStatus(ctx context.Context, app *platformv1alpha1.WorkerApp, outcome fleetOutcome, appVersion string, conditions []metav1.Condition) error { before := app.DeepCopy().Status app.Status.Phase = outcome.Phase + app.Status.ExpectedAppVersion = appVersion + app.Status.Deployments = nil app.Status.Rollout = platformv1alpha1.RolloutStatus{ Partition: outcome.Partition, WaitingOn: outcome.WaitingOn, } + app.Status.RolledOutAppVersion = "" if outcome.RolledOut { app.Status.RolledOutAppVersion = appVersion } @@ -571,14 +618,26 @@ func (r *WorkerAppReconciler) updateStatus(ctx context.Context, app *platformv1a if err := r.Get(ctx, client.ObjectKey{Namespace: app.Namespace, Name: fleetName(app)}, &sts); err == nil { app.Status.Fleet.Ready = sts.Status.ReadyReplicas } - // Fleet restoring is best-effort in status: a partial sweep (fleet mid- - // change, pod restarting) reports what answered. The rollout gate does - // its own strict sweep. - var pods corev1.PodList - if err := r.List(ctx, &pods, client.InNamespace(app.Namespace), client.MatchingLabels(selectorLabels(app))); err == nil { - if _, restoring, err := r.State.FleetSweep(ctx, pods.Items); err == nil { - app.Status.Fleet.Restoring = int32(restoring) + if states, restoring, err := r.observeFleet(ctx, app, &sts); err == nil { + conditions = append(conditions, metav1.Condition{Type: "FleetStateReady", Status: metav1.ConditionTrue, Reason: "Observed"}) + app.Status.Fleet.Restoring = int32(restoring) + for name, state := range states { + if d := state.Deployment; d != nil { + app.Status.Deployments = append(app.Status.Deployments, platformv1alpha1.PodDeployment{Name: name, Version: d.Version, Generation: d.Generation, Draining: int32(len(d.Draining)), Swapping: d.Swapping}) + } } + slices.SortFunc(app.Status.Deployments, func(a, b platformv1alpha1.PodDeployment) int { + if a.Name < b.Name { + return -1 + } + if a.Name > b.Name { + return 1 + } + return 0 + }) + } else { + app.Status.Fleet.Restoring = 0 + conditions = append(conditions, metav1.Condition{Type: "FleetStateReady", Status: metav1.ConditionFalse, Reason: "Unavailable", Message: err.Error()}) } progressing := outcome.Phase == platformv1alpha1.PhasePending || @@ -603,6 +662,15 @@ func (r *WorkerAppReconciler) updateStatus(ctx context.Context, app *platformv1a Message: outcome.WaitingOn, }, ) + types := map[string]bool{} + for _, cond := range conditions { + types[cond.Type] = true + } + for _, kind := range []string{condIngressReady, condAutoscalingReady, condDeployTrackingReady} { + if !types[kind] { + meta.RemoveStatusCondition(&app.Status.Conditions, kind) + } + } for _, cond := range conditions { cond.ObservedGeneration = app.Generation meta.SetStatusCondition(&app.Status.Conditions, cond) @@ -644,3 +712,145 @@ func (r *WorkerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { Named("workerapp"). Complete(r) } + +const managedAnnotationsKey = "celld-operator.io/managed-annotations" + +func reconcileAnnotations(obj client.Object, desired map[string]string) { + annotations := mergeStringMaps(obj.GetAnnotations(), nil) + if annotations == nil { + annotations = map[string]string{} + } + var keys []string + _ = json.Unmarshal([]byte(annotations[managedAnnotationsKey]), &keys) + // Known legacy Ingress keys can be safely removed without a previous inventory. + if _, ok := obj.(*networkingv1.Ingress); ok { + keys = append(keys, "nginx.ingress.kubernetes.io/proxy-read-timeout", "nginx.ingress.kubernetes.io/proxy-send-timeout", "cert-manager.io/cluster-issuer") + } + for _, key := range keys { + if _, found := desired[key]; !found { + delete(annotations, key) + } + } + keys = nil + for key, value := range desired { + if key != managedAnnotationsKey { + annotations[key] = value + keys = append(keys, key) + } + } + slices.Sort(keys) + raw, _ := json.Marshal(keys) + annotations[managedAnnotationsKey] = string(raw) + obj.SetAnnotations(annotations) +} +func (r *WorkerAppReconciler) removeObsoleteRoutes(ctx context.Context, app *platformv1alpha1.WorkerApp) error { + mode := r.IngressMode + if mode == "" { + mode = IngressModeHTTPRoute + } + if len(app.Spec.Hostnames) == 0 { + mode = IngressModeNone + } + for kind, obj := range map[string]client.Object{ + IngressModeIngress: &networkingv1.Ingress{}, IngressModeHTTPRoute: &gatewayv1.HTTPRoute{}, + IngressModeVirtualService: newUnstructuredObject("networking.istio.io/v1", "VirtualService", fleetName(app), app.Namespace, nil, nil, nil), + } { + if kind == mode { + continue + } + err := r.Reader.Get(ctx, client.ObjectKey{Namespace: app.Namespace, Name: fleetName(app)}, obj) + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + continue + } + if err != nil { + return err + } + // Only delete our own obsolete routes. Foreign names are unrelated. + if !metav1.IsControlledBy(obj, app) { + continue + } + uid, rv := obj.GetUID(), obj.GetResourceVersion() + if err := r.Delete(ctx, obj, client.Preconditions{UID: &uid, ResourceVersion: &rv}); client.IgnoreNotFound(err) != nil { + return err + } + } + return nil +} +func (r *WorkerAppReconciler) httpRouteReady(ctx context.Context, desired *gatewayv1.HTTPRoute) metav1.ConditionStatus { + var live gatewayv1.HTTPRoute + if err := r.Reader.Get(ctx, client.ObjectKeyFromObject(desired), &live); err != nil { + return metav1.ConditionUnknown + } + for _, parent := range live.Status.Parents { + if parent.ParentRef.Name != gatewayv1.ObjectName(r.GatewayName) { + continue + } + namespace := live.Namespace + if parent.ParentRef.Namespace != nil { + namespace = string(*parent.ParentRef.Namespace) + } + if namespace != r.GatewayNamespace { + continue + } + accepted, resolved := false, false + for _, c := range parent.Conditions { + if c.ObservedGeneration != live.Generation { + continue + } + if c.Type == "Accepted" { + accepted = c.Status == metav1.ConditionTrue + } + if c.Type == "ResolvedRefs" { + resolved = c.Status == metav1.ConditionTrue + } + } + if accepted && resolved { + var gateway gatewayv1.Gateway + if err := r.Reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: string(parent.ParentRef.Name)}, &gateway); err != nil { + return metav1.ConditionUnknown + } + for _, cond := range gateway.Status.Conditions { + if cond.Type == "Programmed" && cond.ObservedGeneration == gateway.Generation && cond.Status == metav1.ConditionTrue { + return metav1.ConditionTrue + } + } + } + } + return metav1.ConditionFalse +} + +const retryRequestedAnnotation = "celld-operator.io/retry-requested" +const retryRecheckAnnotation = "celld-operator.io/retry-recheck-after" + +// Preserve pruned retries until the persisted deadline, then clear the latch. +// The next reconcile probes support again; watch events cannot bypass the delay. +func (r *WorkerAppReconciler) configureHTTPRouteRetries(ctx context.Context, route *gatewayv1.HTTPRoute) bool { + dropped := false + if r.HTTPRouteRetries { + if route.Annotations == nil { + route.Annotations = map[string]string{} + } + route.Annotations[retryRequestedAnnotation] = annotationTrue + var live gatewayv1.HTTPRoute + if err := r.Get(ctx, client.ObjectKeyFromObject(route), &live); err == nil { + dropped = live.Annotations[retryRequestedAnnotation] == annotationTrue && len(live.Spec.Rules) > 0 && live.Spec.Rules[0].Retry == nil + if dropped { + deadline, err := time.Parse(time.RFC3339Nano, live.Annotations[retryRecheckAnnotation]) + if err != nil { + deadline = time.Now().Add(5 * time.Minute) + } + if time.Now().Before(deadline) { + route.Annotations[retryRecheckAnnotation] = deadline.UTC().Format(time.RFC3339Nano) + } else { + delete(route.Annotations, retryRequestedAnnotation) + } + } + } + } + if !r.HTTPRouteRetries || dropped { + for i := range route.Spec.Rules { + route.Spec.Rules[i].Retry = nil + } + } + return dropped +} diff --git a/internal/controller/workerapp_controller_test.go b/internal/controller/workerapp_controller_test.go index f8f9bcc..e28ed31 100644 --- a/internal/controller/workerapp_controller_test.go +++ b/internal/controller/workerapp_controller_test.go @@ -18,6 +18,8 @@ package controller import ( "context" + "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -112,7 +114,7 @@ var _ = Describe("WorkerApp Controller", func() { // the next spec starts clean. for _, obj := range []client.Object{ &appsv1.StatefulSet{}, &corev1.Service{}, &corev1.ServiceAccount{}, - &networkingv1.NetworkPolicy{}, &policyv1.PodDisruptionBudget{}, + &networkingv1.NetworkPolicy{}, &policyv1.PodDisruptionBudget{}, &networkingv1.Ingress{}, } { if err := k8sClient.Get(ctx, fleetKey, obj); err == nil { Expect(k8sClient.Delete(ctx, obj)).To(Succeed()) @@ -124,6 +126,46 @@ var _ = Describe("WorkerApp Controller", func() { } }) + It("should validate DNS subdomain label and total length boundaries", func() { + valid := []string{"cluster.local", "a", strings.Repeat("a", 63) + ".local", strings.Repeat(strings.Repeat("a", 63)+".", 3) + strings.Repeat("b", 61)} + invalid := []string{"", "a..b", "a.-b", "a-.b", "a.", ".a", strings.Repeat("a", 64) + ".local", strings.Repeat(strings.Repeat("a", 63)+".", 3) + strings.Repeat("b", 62)} + for _, domain := range valid { + app := &platformv1alpha1.WorkerApp{} + Expect(k8sClient.Get(ctx, typeNamespacedName, app)).To(Succeed()) + app.Spec.ClusterDomain = domain + Expect(k8sClient.Update(ctx, app)).To(Succeed(), domain) + } + for _, domain := range invalid { + // A raw patch retains empty strings rather than omitting the field. + app := &platformv1alpha1.WorkerApp{} + Expect(k8sClient.Get(ctx, typeNamespacedName, app)).To(Succeed()) + patch := client.RawPatch(types.MergePatchType, fmt.Appendf(nil, `{"spec":{"clusterDomain":%q}}`, domain)) + Expect(errors.IsInvalid(k8sClient.Patch(ctx, app, patch))).To(BeTrue(), domain) + } + }) + + It("should reject unsafe admission and immutable migrations", func() { + cases := map[string]func(*platformv1alpha1.WorkerApp){ + "bucket migration": func(a *platformv1alpha1.WorkerApp) { a.Spec.Bucket.Name = "s3://other/app" }, + "storage migration": func(a *platformv1alpha1.WorkerApp) { a.Spec.Storage = &platformv1alpha1.StorageSpec{SizeGi: 10} }, + "uncredentialed auto": func(a *platformv1alpha1.WorkerApp) { a.Spec.AppVersion = AppVersionAuto }, + "contradictory credentials": func(a *platformv1alpha1.WorkerApp) { + a.Spec.Bucket.CredentialsFrom.IAMRole = iamRoleAuto + a.Spec.Bucket.CredentialsFrom.SecretRef = "credentials" + }, + "autoscaling bounds": func(a *platformv1alpha1.WorkerApp) { + a.Spec.Autoscaling = &platformv1alpha1.AutoscalingSpec{Enabled: true, MinReplicas: 10, MaxReplicas: 3} + }, + } + for name, mutate := range cases { + By(name) + app := &platformv1alpha1.WorkerApp{} + Expect(k8sClient.Get(ctx, typeNamespacedName, app)).To(Succeed()) + mutate(app) + Expect(errors.IsInvalid(k8sClient.Update(ctx, app))).To(BeTrue(), name) + } + }) + It("should leave a steady reconcile unchanged and preserve Service metadata", func() { r := reconciler() resource := &platformv1alpha1.WorkerApp{} @@ -201,7 +243,7 @@ var _ = Describe("WorkerApp Controller", func() { Expect(env["CELLD_ADVERTISE"]).To(ContainSubstring("test-resource-celld-internal")) // Unset durability and forwarded-header trust leave celld's // defaults alone rather than pinning them. - Expect(env).NotTo(HaveKey("CELLD_DURABILITY")) + Expect(env).To(HaveKeyWithValue("CELLD_DURABILITY", "bucket")) Expect(env).NotTo(HaveKey("CELLD_TRUST_FORWARDED_HEADERS")) // Fleet durability keeps acknowledged writes on follower disks // until tiered (F13): spread the pods across hosts, softly. @@ -309,8 +351,8 @@ var _ = Describe("WorkerApp Controller", func() { resource := &platformv1alpha1.WorkerApp{} Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) resource.Spec.AppVersion = AppVersionAuto - // Point at an unreachable endpoint so the pointer read fails. - resource.Spec.Bucket.Endpoint = "http://127.0.0.1:1" + // A missing dedicated credential makes the pointer read fail safely. + resource.Spec.DeployTrackingSecretRef = "missing-tracking" resource.Spec.Bucket.CredentialsFrom.SecretRef = "" Expect(k8sClient.Update(ctx, resource)).To(Succeed()) @@ -343,9 +385,12 @@ var _ = Describe("WorkerApp Controller", func() { } resource.Spec.Durability = platformv1alpha1.DurabilityBucket resource.Spec.TrustForwardedHeaders = true - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + resource.ObjectMeta = metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace} + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) r := reconciler() + r.AllowedAzureClientIDs = []string{"11111111-2222-3333-4444-555555555555"} _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) Expect(err).NotTo(HaveOccurred()) @@ -374,14 +419,7 @@ var _ = Describe("WorkerApp Controller", func() { By("refusing appVersion auto, which reads the pointer over the S3 API only") Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) resource.Spec.AppVersion = AppVersionAuto - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - updated := &platformv1alpha1.WorkerApp{} - Expect(k8sClient.Get(ctx, typeNamespacedName, updated)).To(Succeed()) - cond := meta.FindStatusCondition(updated.Status.Conditions, condDeployTrackingReady) - Expect(cond).NotTo(BeNil()) - Expect(cond.Reason).To(Equal("UnsupportedStore")) + Expect(errors.IsInvalid(k8sClient.Update(ctx, resource))).To(BeTrue()) // And the fleet keeps the version it was serving. Expect(k8sClient.Get(ctx, fleetKey, sts)).To(Succeed()) Expect(sts.Spec.Template.Annotations).To(HaveKeyWithValue(appVersionAnnotation, "sha-test")) @@ -448,7 +486,7 @@ var _ = Describe("WorkerApp Controller", func() { Expect(sts.Spec.Template.Spec.Containers[0].Image).To(Equal(testCelldImageOld)) }) - It("should refuse a rolling downgrade from v0.4 to v0.2 and surface every hazard on Recreate", func() { + It("should refuse a rolling downgrade from v0.4 to v0.2 even when Recreate is requested", func() { r := reconciler() _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) Expect(err).NotTo(HaveOccurred()) @@ -464,23 +502,22 @@ var _ = Describe("WorkerApp Controller", func() { updated := &platformv1alpha1.WorkerApp{} Expect(k8sClient.Get(ctx, typeNamespacedName, updated)).To(Succeed()) Expect(updated.Status.Phase).To(Equal(platformv1alpha1.PhaseDegraded)) - Expect(updated.Status.Rollout.WaitingOn).To(ContainSubstring("sealed epoch")) - Expect(updated.Status.Rollout.WaitingOn).To(ContainSubstring("peer tunnel")) + Expect(updated.Status.Rollout.WaitingOn).To(ContainSubstring("offline durability verification")) sts := &appsv1.StatefulSet{} Expect(k8sClient.Get(ctx, fleetKey, sts)).To(Succeed()) Expect(sts.Spec.Template.Spec.Containers[0].Image).To(Equal(testCelldImage)) - By("proceeding only once the CR says Recreate, and saying what the stop protects") + By("refusing the unsafe downgrade even when Recreate is requested") Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) resource.Spec.Celld.UpdateStrategy = platformv1alpha1.UpdateStrategyRecreate Expect(k8sClient.Update(ctx, resource)).To(Succeed()) _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient.Get(ctx, typeNamespacedName, updated)).To(Succeed()) - Expect(updated.Status.Phase).To(Equal(platformv1alpha1.PhaseRecreating)) - Expect(updated.Status.Rollout.WaitingOn).To(ContainSubstring("sealed epoch")) + Expect(updated.Status.Phase).To(Equal(platformv1alpha1.PhaseDegraded)) + Expect(updated.Status.Rollout.WaitingOn).To(ContainSubstring("offline durability verification")) Expect(k8sClient.Get(ctx, fleetKey, sts)).To(Succeed()) - Expect(*sts.Spec.Replicas).To(BeZero()) + Expect(*sts.Spec.Replicas).To(Equal(int32(3))) }) }) }) diff --git a/test/chart/chart_test.go b/test/chart/chart_test.go new file mode 100644 index 0000000..8e18301 --- /dev/null +++ b/test/chart/chart_test.go @@ -0,0 +1,96 @@ +package chart + +import ( + "os/exec" + "slices" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +const httpMode = "http" +const certificateMode = "cert-manager" +const disabledMode = "disabled" + +func TestMetricsAndNamespaceRendering(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is required for chart rendering tests") + } + for _, mode := range []string{httpMode, "self-signed", certificateMode, disabledMode} { + t.Run(mode, func(t *testing.T) { + args := []string{"template", "test", "../../dist/chart", "--namespace", "operator", + "--set", "prometheus.enable=true", "--set", "operator.watchNamespace=tenant"} + switch mode { + case httpMode: + args = append(args, "--set", "metrics.insecure=true", "--set", "metrics.port=8080") + case certificateMode: + args = append(args, "--set", "certmanager.enable=true") + case disabledMode: + args = append(args, "--set", "metrics.enable=false") + } + output, err := exec.Command("helm", args...).CombinedOutput() + if err != nil { + t.Fatalf("helm: %v %s", err, output) + } + foundManager, foundMonitor, foundRole := false, false, false + for doc := range strings.SplitSeq(string(output), "\n---") { + var object unstructured.Unstructured + if err := yaml.Unmarshal([]byte(doc), &object.Object); err != nil { + t.Fatal(err) + } + switch object.GetKind() { + case "Deployment": + foundManager = true + assertManager(t, object, mode) + case "ServiceMonitor": + foundMonitor = true + endpoints, _, _ := unstructured.NestedSlice(object.Object, "spec", "endpoints") + endpoint := endpoints[0].(map[string]any) + if endpoint["honorLabels"] != true { + t.Fatal("fleet namespace/pod labels would be rewritten") + } + if mode == httpMode && (endpoint["scheme"] != httpMode || + endpoint["port"] != httpMode || endpoint["tlsConfig"] != nil) { + t.Fatal("HTTP scrape misconfigured") + } + case "Role": + if object.GetName() == "celld-operator-manager-role" { + foundRole = true + if object.GetNamespace() != "tenant" { + t.Fatal("manager Role has wrong scope") + } + } + case "ClusterRole": + if object.GetName() == "celld-operator-manager-role" { + t.Fatal("namespace-scoped operator has cluster-wide workload RBAC") + } + } + } + if !foundManager || !foundRole || (foundMonitor == (mode == disabledMode)) { + t.Fatalf("unexpected resources: manager=%v role=%v monitor=%v", foundManager, foundRole, foundMonitor) + } + }) + } +} + +func assertManager(t *testing.T, object unstructured.Unstructured, mode string) { + t.Helper() + containers, _, _ := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "containers") + manager := containers[0].(map[string]any) + flags, _, _ := unstructured.NestedStringSlice(manager, "args") + joined := strings.Join(flags, " ") + if strings.Count(joined, "--metrics-bind-address=") != 1 || !strings.Contains(joined, "--watch-namespace=tenant") { + t.Fatalf("bad flags: %s", joined) + } + if mode == httpMode && !strings.Contains(joined, "--metrics-secure=false") { + t.Fatal("HTTP still uses TLS") + } + if mode == certificateMode && !strings.Contains(joined, "--metrics-cert-path=") { + t.Fatal("certificate not passed to manager") + } + if mode == disabledMode && !slices.Contains(flags, "--metrics-bind-address=0") { + t.Fatal("metrics still enabled") + } +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0ed0fe9..07d8a8c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "os/exec" + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -45,6 +46,12 @@ var ( // By default, kuberc is disabled to ensure consistent test behavior across different environments. // To skip CertManager installation, set: CERT_MANAGER_INSTALL_SKIP=true func TestE2E(t *testing.T) { + cluster := os.Getenv("KIND_CLUSTER") + current, err := exec.Command("kubectl", "config", "current-context").Output() + if err != nil || (!strings.HasPrefix(cluster, "celld-operator-test-") && !strings.HasPrefix(cluster, "celld-operator-audit-")) || strings.TrimSpace(string(current)) != "kind-"+cluster { + t.Fatal("e2e requires the dedicated Kind context selected by make test-e2e") + } + RegisterFailHandler(Fail) _, _ = fmt.Fprintf(GinkgoWriter, "Starting celld-operator e2e test suite\n") RunSpecs(t, "e2e suite") @@ -62,6 +69,12 @@ var _ = BeforeSuite(func() { err = utils.LoadImageToKindClusterWithName(managerImage) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") + By("building and loading the deterministic lifecycle fixture") + cmd = exec.Command("docker", "build", "-f", "test/fixtures/runtime/Dockerfile", "-t", fixtureImage, "-t", fixtureNextImage, ".") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(utils.LoadImageToKindClusterWithName(fixtureImage)).To(Succeed()) + Expect(utils.LoadImageToKindClusterWithName(fixtureNextImage)).To(Succeed()) configureKubectlKubeRC() setupCertManager() }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 37a8e33..5a5df7c 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -217,13 +217,13 @@ var _ = Describe("Manager", Ordered, func() { By("creating the curl-metrics pod to access the metrics endpoint") cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", "--namespace", namespace, - "--image=curlimages/curl:latest", + "--image=curlimages/curl:8.19.0", "--overrides", fmt.Sprintf(`{ "spec": { "containers": [{ "name": "curl", - "image": "curlimages/curl:latest", + "image": "curlimages/curl:8.19.0", "command": ["/bin/sh", "-c"], "args": [ "for i in $(seq 1 30); do curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics && exit 0 || sleep 2; done; exit 1" @@ -270,15 +270,8 @@ var _ = Describe("Manager", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO: Customize the e2e test suite with scenarios specific to your project. - // Consider applying sample/CR(s) and check their status and/or verifying - // the reconciliation by using the metrics, i.e.: - // metricsOutput, err := getMetricsOutput() - // Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") - // Expect(metricsOutput).To(ContainSubstring( - // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, - // strings.ToLower(), - // )) + It("should roll a WorkerApp and retain local state", verifyWorkerAppLifecycle) + }) }) diff --git a/test/e2e/workerapp_test.go b/test/e2e/workerapp_test.go new file mode 100644 index 0000000..aa584e9 --- /dev/null +++ b/test/e2e/workerapp_test.go @@ -0,0 +1,75 @@ +//go:build e2e + +package e2e + +import ( + "os/exec" + "strings" + "time" + + "github.com/ezgamehost/celld-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const fixtureImage = "example.com/celld-fixture:v0.4.0" +const fixtureNextImage = "example.com/celld-fixture:v0.4.1" + +func verifyWorkerAppLifecycle() { + By("allowing the deterministic test runtime on this isolated operator") + patch := `[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--allowed-image-prefixes=ghcr.io/denoland/celld:,example.com/celld-fixture:"}]` + _, err := utils.Run(exec.Command("kubectl", "patch", "deployment", "celld-operator-controller-manager", "-n", namespace, "--type=json", "-p", patch)) + Expect(err).NotTo(HaveOccurred()) + _, err = utils.Run(exec.Command("kubectl", "rollout", "status", "deployment/celld-operator-controller-manager", "-n", namespace, "--timeout=120s")) + Expect(err).NotTo(HaveOccurred()) + manifest := `apiVersion: celld-operator.io/v1alpha1 +kind: WorkerApp +metadata: + name: lifecycle + namespace: celld-operator-system +spec: + appVersion: deploy-a + celld: + image: example.com/celld-fixture:v0.4.0 + bucket: + name: s3://unused-by-test-fixture + replicas: 1 + resources: + memoryGi: 1 + cpuMillis: 10 + storage: + sizeGi: 1 +` + apply := exec.Command("kubectl", "apply", "-f", "-") + apply.Stdin = strings.NewReader(manifest) + _, err = utils.Run(apply) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _, _ = utils.Run(exec.Command("kubectl", "delete", "workerapp", "lifecycle", "-n", namespace, "--ignore-not-found")) + _, _ = utils.Run(exec.Command("kubectl", "delete", "pvc", "watch-lifecycle-celld-0", "-n", namespace, "--ignore-not-found")) + }) + ready := func(g Gomega) { + out, err := utils.Run(exec.Command("kubectl", "get", "workerapp", "lifecycle", "-n", namespace, "-o", "jsonpath={.status.phase}:{.status.rolledOutAppVersion}")) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(Equal("Ready:deploy-a")) + } + Eventually(ready, 180*time.Second, time.Second).Should(Succeed()) + logs, err := utils.Run(exec.Command("kubectl", "logs", "lifecycle-celld-0", "-n", namespace)) + Expect(err).NotTo(HaveOccurred()) + Expect(logs).To(ContainSubstring("Boot 1")) + By("rolling a real StatefulSet and retaining its volume across replacement") + _, err = utils.Run(exec.Command("kubectl", "patch", "workerapp", "lifecycle", "-n", namespace, "--type=merge", "-p", `{"spec":{"celld":{"image":"example.com/celld-fixture:v0.4.1"}}}`)) + Expect(err).NotTo(HaveOccurred()) + Eventually(func(g Gomega) { + out, err := utils.Run(exec.Command("kubectl", "get", "pod", "lifecycle-celld-0", "-n", namespace, "-o", "jsonpath={.spec.containers[0].image}")) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(Equal(fixtureNextImage)) + }, 180*time.Second, time.Second).Should(Succeed()) + Eventually(ready, 180*time.Second, time.Second).Should(Succeed()) + logs, err = utils.Run(exec.Command("kubectl", "logs", "lifecycle-celld-0", "-n", namespace)) + Expect(err).NotTo(HaveOccurred()) + Expect(logs).To(ContainSubstring("Boot 2")) + By("rejecting an unsafe fleet durability configuration at admission") + _, err = utils.Run(exec.Command("kubectl", "patch", "workerapp", "lifecycle", "-n", namespace, "--type=merge", "-p", `{"spec":{"durability":"fleet"}}`)) + Expect(err).To(HaveOccurred()) +} diff --git a/test/fixtures/runtime/Dockerfile b/test/fixtures/runtime/Dockerfile new file mode 100644 index 0000000..d8483c4 --- /dev/null +++ b/test/fixtures/runtime/Dockerfile @@ -0,0 +1,7 @@ +FROM golang:1.26.8@sha256:3c3e25a4da13fd0478eed2df1eb35a0e667094a7124d3993a6a1d30f71c17e79 AS build +WORKDIR /src +COPY test/fixtures/runtime/main.go main.go +RUN CGO_ENABLED=0 go build -tags=e2e -o /runtime main.go +FROM scratch +COPY --from=build /runtime /runtime +ENTRYPOINT ["/runtime"] diff --git a/test/fixtures/runtime/main.go b/test/fixtures/runtime/main.go new file mode 100644 index 0000000..e02f68b --- /dev/null +++ b/test/fixtures/runtime/main.go @@ -0,0 +1,46 @@ +//go:build e2e + +// This deterministic runtime fixture tests the operator's Kubernetes lifecycle. +// It intentionally does not emulate celld's storage or replication protocol. +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" +) + +func main() { + path := os.Getenv("CELLD_WATCH") + "/boots" + previous, _ := os.ReadFile(path) + boots, _ := strconv.Atoi(strings.TrimSpace(string(previous))) + if err := os.WriteFile(path, fmt.Appendf(nil, "%d", boots+1), 0600); err != nil { + log.Fatal(err) + } + log.Printf("Boot %d", boots+1) + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) }) + state := http.NewServeMux() + state.HandleFunc("/state", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"occupied":1,"restoring":0,"shedding":null,"deployment":{"version":"deploy-a","generation":1,"draining":[],"swapping":0}}`)) + }) + public := &http.Server{Addr: ":8080", Handler: mux, ReadHeaderTimeout: time.Second} + internal := &http.Server{Addr: ":8081", Handler: state, ReadHeaderTimeout: time.Second} + go func() { _ = public.ListenAndServe() }() + go func() { _ = internal.ListenAndServe() }() + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer cancel() + <-ctx.Done() + stop, done := context.WithTimeout(context.Background(), 5*time.Second) + defer done() + _ = public.Shutdown(stop) + _ = internal.Shutdown(stop) +} diff --git a/test/utils/utils.go b/test/utils/utils.go index a408630..8d4c13b 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -49,11 +49,11 @@ func Run(cmd *exec.Cmd) (string, error) { } cmd.Env = append(os.Environ(), "GO111MODULE=on") - command := strings.Join(cmd.Args, " ") + command := cmd.Path _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) output, err := cmd.CombinedOutput() if err != nil { - return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) + return string(output), fmt.Errorf("%q failed: %w", command, err) } return string(output), nil