From 87870f59d86810113b8ba575f3bd5bcd09c2b88c Mon Sep 17 00:00:00 2001 From: AhmedElBanna80 Date: Thu, 22 Oct 2026 17:44:54 +0200 Subject: [PATCH 1/3] fix(security): scale-zero-pg compute pods meet Pod Security restricted on every container (#2564) Every container of the rendered compute pods (writer + read replica, init containers included) now carries allowPrivilegeEscalation=false, drop ALL, seccomp RuntimeDefault and runAsNonRoot. The images declare USER by name (postgres / neon, both 1000:1000 in the pinned digests), which the kubelet cannot verify as non-root, so runAsUser/runAsGroup 1000 are set explicitly. Guarded by the pod-security-admission restricted evaluator over the real render and the break-glass template, with a negative control. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0198oVxWkEEDJJnVhES6ULnx --- .../deploy/compute-app.template.yaml | 17 +++ packages/scale-zero-pg/docs/operations.md | 26 ++++ packages/scale-zero-pg/gateway/go.mod | 2 + packages/scale-zero-pg/gateway/go.sum | 4 + .../gateway/internal/appdb/render.go | 69 ++++++--- .../appdb/render_pod_security_test.go | 135 ++++++++++++++++++ 6 files changed, 237 insertions(+), 16 deletions(-) create mode 100644 packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go diff --git a/packages/scale-zero-pg/deploy/compute-app.template.yaml b/packages/scale-zero-pg/deploy/compute-app.template.yaml index 6f23571ab..8e6f7f934 100644 --- a/packages/scale-zero-pg/deploy/compute-app.template.yaml +++ b/packages/scale-zero-pg/deploy/compute-app.template.yaml @@ -72,6 +72,16 @@ spec: initContainers: - name: wait-timeline image: docker.io/neondatabase/neon:8464@sha256:7a4f124917bb929964b2d696d710f19584f80bb9bd51b2af4a6e2425434c761f + # Pod Security `restricted`. Both images name their USER (neon / postgres, + # uid 1000:1000 in these digests), which the kubelet cannot verify as + # non-root, so the numeric id is spelled out. + securityContext: + allowPrivilegeEscalation: false + capabilities: { drop: ["ALL"] } + seccompProfile: { type: RuntimeDefault } + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 envFrom: - configMapRef: { name: compute-config-__APP__ } command: ["/bin/sh", "-c"] @@ -87,6 +97,13 @@ spec: - name: compute image: docker.io/neondatabase/compute-node-v17:8464@sha256:13ab146d3e7bbabb25a8532f315ac443e7512351d1ede0bab586def5c70e26c3 # PINNED pair with storage (ops "Upgrades") imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: { drop: ["ALL"] } + seccompProfile: { type: RuntimeDefault } + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 command: ["/bin/sh", "/compute-files/entrypoint.sh"] envFrom: - configMapRef: { name: compute-config-__APP__ } diff --git a/packages/scale-zero-pg/docs/operations.md b/packages/scale-zero-pg/docs/operations.md index c93588e76..76e627715 100644 --- a/packages/scale-zero-pg/docs/operations.md +++ b/packages/scale-zero-pg/docs/operations.md @@ -2381,6 +2381,32 @@ that `compute:55433` is unreachable from a non-gateway pod and **fails** (not wa if it is not — so a regression in the policies, or a CNI that only partially enforces, is caught instead of passing silently. +## Pod Security: per-app compute pods meet `restricted` + +Every container of the per-app compute pods the operator renders (the writer and +the read-replica pool, init containers included) carries a container +`securityContext` that satisfies the Kubernetes Pod Security `restricted` level, so +the pods are admitted in a namespace labelled +`pod-security.kubernetes.io/enforce=restricted`: + +- `allowPrivilegeEscalation: false` +- `capabilities.drop: [ALL]` (nothing is added back; compute_ctl and Postgres run + unprivileged on ports above 1024) +- `seccompProfile: RuntimeDefault` +- `runAsNonRoot: true`, with `runAsUser` / `runAsGroup` set to `1000` + +The numeric uid matters: the pinned compute image declares its user by name +(`postgres`) and the init image as `neon`, and the kubelet refuses `runAsNonRoot` +for a named user it cannot resolve to a number. Both are uid/gid 1000 in the pinned +digests, which is the identity the pods already ran as, so file ownership and the +boot sequence are unchanged. **If you bump either image digest, check its +`/etc/passwd` and update the uid** (`crane export - | tar -xO etc/passwd`). +`readOnlyRootFilesystem` is not set: Postgres writes under the container filesystem. + +The break-glass template (`deploy/compute-app.template.yaml`) carries the same +fields. The base-tier manifests (`20-compute.yaml`, `25-compute-warm.yaml`, +`26-compute-ro.yaml`) still set only the pod-level seccomp profile. + ## Tenant isolation controls (issue #112 / #113 / #114) Because NetworkPolicy is **inert on flannel** (above), the tenant boundary must not diff --git a/packages/scale-zero-pg/gateway/go.mod b/packages/scale-zero-pg/gateway/go.mod index f060ce489..0d6ebc12d 100644 --- a/packages/scale-zero-pg/gateway/go.mod +++ b/packages/scale-zero-pg/gateway/go.mod @@ -9,6 +9,7 @@ require ( k8s.io/api v0.34.1 k8s.io/apimachinery v0.34.1 k8s.io/client-go v0.34.1 + k8s.io/pod-security-admission v0.34.1 sigs.k8s.io/yaml v1.6.0 ) @@ -47,6 +48,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-base v0.34.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect diff --git a/packages/scale-zero-pg/gateway/go.sum b/packages/scale-zero-pg/gateway/go.sum index e091739ec..34f26b01a 100644 --- a/packages/scale-zero-pg/gateway/go.sum +++ b/packages/scale-zero-pg/gateway/go.sum @@ -154,10 +154,14 @@ k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= +k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/pod-security-admission v0.34.1 h1:XsP5eh8qCj69hK0a5TBMU4Ed7Ckn8JEmmbk/iepj+XM= +k8s.io/pod-security-admission v0.34.1/go.mod h1:87yY36Gxc8Hjx24FxqAD5zMY4k0tP0u7Mu/XuwXEbmg= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= diff --git a/packages/scale-zero-pg/gateway/internal/appdb/render.go b/packages/scale-zero-pg/gateway/internal/appdb/render.go index 0fddbf187..95286fb7e 100644 --- a/packages/scale-zero-pg/gateway/internal/appdb/render.go +++ b/packages/scale-zero-pg/gateway/internal/appdb/render.go @@ -33,6 +33,38 @@ func DefaultRenderConfig(ns string) RenderConfig { } } +// computeImageUID is the numeric uid/gid every container of a rendered compute +// pod runs as. Measured from /etc/passwd of the pinned digests: the compute image +// declares USER `postgres` (1000:1000) and the init image USER `neon` (1000:1000). +// Both are NAMES in the image config, which the kubelet cannot verify as +// non-root, so runAsNonRoot needs the numeric id spelled out here. It equals the +// uid the pods already ran as, so nothing about file ownership or the boot +// changes. Bumping either image digest means re-reading it (crane export + the +// image's /etc/passwd): a different uid is a silent boot failure otherwise. +const computeImageUID int64 = 1000 + +// restrictedContainerSecurityContext is the container securityContext every +// container of a gateway-rendered compute pod carries, so the pod is admitted in +// a namespace enforcing the Kubernetes Pod Security `restricted` level: +// allowPrivilegeEscalation false, capabilities drop ALL (nothing added back - +// compute_ctl and Postgres run unprivileged on ports above 1024 and need none), +// seccomp RuntimeDefault, runAsNonRoot with the numeric uid/gid above. +// readOnlyRootFilesystem is deliberately not set: Postgres and compute_ctl write +// under the container filesystem (PGDATA, the staged TLS key), and it is not part +// of the restricted profile. +func restrictedContainerSecurityContext() *corev1.SecurityContext { + no, yes := false, true + uid, gid := computeImageUID, computeImageUID + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: &no, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + RunAsNonRoot: &yes, + RunAsUser: &uid, + RunAsGroup: &gid, + } +} + func labelsFor(app string) map[string]string { return map[string]string{"app": "compute-" + app, "tier": "apps", "plane": "compute"} } @@ -155,16 +187,18 @@ echo "timeline ready"` SecurityContext: &corev1.PodSecurityContext{SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, TerminationGracePeriodSeconds: &grace, InitContainers: []corev1.Container{{ - Name: "wait-timeline", - Image: c.InitImage, - EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "compute-config-" + s.App}}}}, - Command: []string{"/bin/sh", "-c"}, - Args: []string{waitScript}, + Name: "wait-timeline", + Image: c.InitImage, + SecurityContext: restrictedContainerSecurityContext(), + EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "compute-config-" + s.App}}}}, + Command: []string{"/bin/sh", "-c"}, + Args: []string{waitScript}, }}, Containers: []corev1.Container{{ Name: "compute", Image: c.ComputeImage, ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), Command: []string{"/bin/sh", "/compute-files/entrypoint.sh"}, EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "compute-config-" + s.App}}}}, Env: []corev1.EnvVar{{ @@ -330,25 +364,28 @@ echo "resolved tip LSN = ${LSN:-}"` TerminationGracePeriodSeconds: &grace, InitContainers: []corev1.Container{ { - Name: "wait-timeline", - Image: c.InitImage, - EnvFrom: cmRef, - Command: []string{"/bin/sh", "-c"}, - Args: []string{waitScript}, + Name: "wait-timeline", + Image: c.InitImage, + SecurityContext: restrictedContainerSecurityContext(), + EnvFrom: cmRef, + Command: []string{"/bin/sh", "-c"}, + Args: []string{waitScript}, }, { - Name: "resolve-lsn", - Image: c.InitImage, - EnvFrom: cmRef, - Command: []string{"/bin/sh", "-c"}, - Args: []string{resolveLSNScript}, - VolumeMounts: []corev1.VolumeMount{{Name: "ro-lsn", MountPath: "/ro-lsn"}}, + Name: "resolve-lsn", + Image: c.InitImage, + SecurityContext: restrictedContainerSecurityContext(), + EnvFrom: cmRef, + Command: []string{"/bin/sh", "-c"}, + Args: []string{resolveLSNScript}, + VolumeMounts: []corev1.VolumeMount{{Name: "ro-lsn", MountPath: "/ro-lsn"}}, }, }, Containers: []corev1.Container{{ Name: "compute", Image: c.ComputeImage, ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), Command: []string{"/bin/sh", "/compute-files/entrypoint-ro.sh"}, EnvFrom: cmRef, // Same security env as the per-app WRITER (RenderDeployment): both are diff --git a/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go b/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go new file mode 100644 index 000000000..29f6db67f --- /dev/null +++ b/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go @@ -0,0 +1,135 @@ +package appdb + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + podsecurityapi "k8s.io/pod-security-admission/api" + "k8s.io/pod-security-admission/policy" +) + +// POD SECURITY `restricted` ON EVERY CONTAINER THE GATEWAY RENDERS. +// +// The per-app compute pods (writer and read replica) have to be admitted in a +// namespace labelled pod-security.kubernetes.io/enforce=restricted. The check is +// the Kubernetes project's own policy evaluator (the library the admission +// plugin runs) over the REAL render, not a hand-written list of the four fields +// the profile mostly cares about. +// +// The images name their user, not a number (`postgres` / `neon`, both uid 1000 +// in the pinned digests, read from each image's /etc/passwd). The kubelet +// refuses runAsNonRoot on a non-numeric image user ("cannot verify user is +// non-root"), so the render pins the numeric uid/gid explicitly. + +func restrictedViolations(t *testing.T, spec *corev1.PodSpec) []string { + t.Helper() + evaluator, err := policy.NewEvaluator(policy.DefaultChecks()) + if err != nil { + t.Fatalf("building the Pod Security evaluator: %v", err) + } + lv := podsecurityapi.LevelVersion{Level: podsecurityapi.LevelRestricted, Version: podsecurityapi.LatestVersion()} + var out []string + for _, r := range evaluator.EvaluatePod(lv, &metav1.ObjectMeta{Name: "p", Namespace: "n"}, spec) { + if !r.Allowed { + out = append(out, r.ForbiddenReason+": "+r.ForbiddenDetail) + } + } + return out +} + +// requireContainerRestricted asserts the fields the profile turns on, by name, +// so removing any one of them names itself instead of hiding in the aggregate. +func requireContainerRestricted(t *testing.T, c corev1.Container) { + t.Helper() + sc := c.SecurityContext + if sc == nil { + t.Errorf("container %q has no securityContext", c.Name) + return + } + if sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation { + t.Errorf("container %q: allowPrivilegeEscalation = %v, want false", c.Name, sc.AllowPrivilegeEscalation) + } + if sc.Capabilities == nil || len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" || len(sc.Capabilities.Add) != 0 { + t.Errorf("container %q: capabilities = %+v, want drop [ALL] and nothing added", c.Name, sc.Capabilities) + } + if sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Errorf("container %q: seccompProfile = %+v, want RuntimeDefault", c.Name, sc.SeccompProfile) + } + if sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot { + t.Errorf("container %q: runAsNonRoot = %v, want true", c.Name, sc.RunAsNonRoot) + } + // Numeric uid/gid: the images' USER is a name, which the kubelet cannot + // verify as non-root. + if sc.RunAsUser == nil || *sc.RunAsUser != 1000 { + t.Errorf("container %q: runAsUser = %v, want 1000 (the postgres/neon uid in the pinned images)", c.Name, sc.RunAsUser) + } + if sc.RunAsGroup == nil || *sc.RunAsGroup != 1000 { + t.Errorf("container %q: runAsGroup = %v, want 1000", c.Name, sc.RunAsGroup) + } +} + +func renderedComputePods(t *testing.T) map[string]*corev1.PodSpec { + t.Helper() + c := DefaultRenderConfig("scale-zero-pg") + w := c.RenderDeployment(ComputeSpec{App: "shop", TenantID: "tenant-x", TimelineID: "timeline-y"}) + r := c.RenderRODeployment(ROComputeSpec{App: "shop", TenantID: "tenant-x", TimelineID: "timeline-y", MaxReplicas: 4}) + return map[string]*corev1.PodSpec{ + "writer": &w.Spec.Template.Spec, + "read replica": &r.Spec.Template.Spec, + } +} + +func TestComputePodsMeetPodSecurityRestricted(t *testing.T) { + for name, spec := range renderedComputePods(t) { + t.Run(name, func(t *testing.T) { + if len(spec.InitContainers) == 0 || len(spec.Containers) == 0 { + t.Fatalf("render has %d init / %d main containers; the guard would pass vacuously", len(spec.InitContainers), len(spec.Containers)) + } + for _, c := range append(append([]corev1.Container{}, spec.InitContainers...), spec.Containers...) { + requireContainerRestricted(t, c) + } + if v := restrictedViolations(t, spec); len(v) != 0 { + t.Fatalf("pod fails Pod Security restricted:\n - %s", strings.Join(v, "\n - ")) + } + }) + } +} + +// The evaluator must actually be able to fail: the pre-hardening pod shape +// (pod-level seccomp only, no container securityContext) has to be reported, or +// the test above is decoration. +func TestRestrictedEvaluatorRejectsUnhardenedPod(t *testing.T) { + for name, spec := range renderedComputePods(t) { + t.Run(name, func(t *testing.T) { + bare := spec.DeepCopy() + for i := range bare.InitContainers { + bare.InitContainers[i].SecurityContext = nil + } + for i := range bare.Containers { + bare.Containers[i].SecurityContext = nil + } + if len(restrictedViolations(t, bare)) == 0 { + t.Fatal("a pod with no container securityContext passed restricted: the evaluator wiring is broken") + } + }) + } +} + +// The break-glass path (provision-app.sh renders deploy/compute-app.template.yaml) +// must meet the same profile, field for field, as the operator's render: the +// structural parity guard only sees that a securityContext is PRESENT. +func TestTemplateComputePodMeetsPodSecurityRestricted(t *testing.T) { + dep := parseTemplateComputeDeployment(t) + spec := &dep.Spec.Template.Spec + if len(spec.InitContainers) == 0 || len(spec.Containers) == 0 { + t.Fatalf("template has %d init / %d main containers; the guard would pass vacuously", len(spec.InitContainers), len(spec.Containers)) + } + for _, c := range append(append([]corev1.Container{}, spec.InitContainers...), spec.Containers...) { + requireContainerRestricted(t, c) + } + if v := restrictedViolations(t, spec); len(v) != 0 { + t.Fatalf("template pod fails Pod Security restricted:\n - %s", strings.Join(v, "\n - ")) + } +} From 11de7fe420eea41f2f800e146c76ea42db013370 Mon Sep 17 00:00:00 2001 From: AhmedElBanna80 Date: Thu, 22 Oct 2026 18:11:36 +0200 Subject: [PATCH 2/3] fix(security): make compute pod uid/gid configurable per image and tie defaults to pinned digests (#2564) The run-as identity was a hardcoded 1000 although the operator lets the compute and init images be overridden. RenderConfig now carries ComputeUID/GID and InitUID/GID (default 1000), overridable with APPDB_COMPUTE_UID/GID and APPDB_INIT_UID/GID, validated as numeric and > 0 at start-up. A hand-maintained digest -> uid table, enforced against the render defaults and every digest pinned under deploy/, goes red when a digest is bumped without recording its uid. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0198oVxWkEEDJJnVhES6ULnx --- packages/scale-zero-pg/docs/operations.md | 31 ++++- .../gateway/cmd/appdb-operator/main.go | 51 +++++++- .../gateway/cmd/appdb-operator/main_test.go | 87 +++++++++++++ .../internal/appdb/image_uid_table_test.go | 120 ++++++++++++++++++ .../gateway/internal/appdb/render.go | 48 ++++--- .../appdb/render_pod_security_test.go | 26 ++++ 6 files changed, 340 insertions(+), 23 deletions(-) create mode 100644 packages/scale-zero-pg/gateway/internal/appdb/image_uid_table_test.go diff --git a/packages/scale-zero-pg/docs/operations.md b/packages/scale-zero-pg/docs/operations.md index 76e627715..560923dd4 100644 --- a/packages/scale-zero-pg/docs/operations.md +++ b/packages/scale-zero-pg/docs/operations.md @@ -2393,15 +2393,38 @@ the pods are admitted in a namespace labelled - `capabilities.drop: [ALL]` (nothing is added back; compute_ctl and Postgres run unprivileged on ports above 1024) - `seccompProfile: RuntimeDefault` -- `runAsNonRoot: true`, with `runAsUser` / `runAsGroup` set to `1000` +- `runAsNonRoot: true`, with a numeric `runAsUser` / `runAsGroup` (default `1000`, see below to override) The numeric uid matters: the pinned compute image declares its user by name (`postgres`) and the init image as `neon`, and the kubelet refuses `runAsNonRoot` for a named user it cannot resolve to a number. Both are uid/gid 1000 in the pinned digests, which is the identity the pods already ran as, so file ownership and the -boot sequence are unchanged. **If you bump either image digest, check its -`/etc/passwd` and update the uid** (`crane export - | tar -xO etc/passwd`). -`readOnlyRootFilesystem` is not set: Postgres writes under the container filesystem. +boot sequence are unchanged. `readOnlyRootFilesystem` is not set: Postgres writes +under the container filesystem. + +**Using your own compute or init image.** The operator can be pointed at other +images with `APPDB_COMPUTE_IMAGE` and `APPDB_INIT_IMAGE`. The uid above is only +right for the default digests, so an image that runs as a different user needs its +identity supplied with it, on the operator Deployment: + +| Variable | Applies to | Default | +| --- | --- | --- | +| `APPDB_COMPUTE_UID` / `APPDB_COMPUTE_GID` | the `compute` container | `1000` / `1000` | +| `APPDB_INIT_UID` / `APPDB_INIT_GID` | the `wait-timeline` and `resolve-lsn` init containers | `1000` / `1000` | + +The compute and init images are separate images with their own users, which is why +each has its own pair. Read the real value from the image rather than guessing: +`crane export - | tar -xO etc/passwd`. Each value must be a decimal number +from 1 to 2147483647; `0` is refused because `runAsNonRoot` forbids root, and a +user name is refused because the kubelet cannot verify a name as non-root. A bad +value stops the operator at start-up with a message naming the variable. If the +uid is wrong for the image, the container starts as the wrong user and the compute +fails to boot (typically a permission error on its data directory), so set it +whenever you override an image. + +**If you bump the default image digests in the manifests**, record each new +image's uid first: the repository's unit tests keep a hand-maintained table of +every pinned digest and its uid and fail when a digest has no entry. The break-glass template (`deploy/compute-app.template.yaml`) carries the same fields. The base-tier manifests (`20-compute.yaml`, `25-compute-warm.yaml`, diff --git a/packages/scale-zero-pg/gateway/cmd/appdb-operator/main.go b/packages/scale-zero-pg/gateway/cmd/appdb-operator/main.go index b22fa9bdc..0824c4c6f 100644 --- a/packages/scale-zero-pg/gateway/cmd/appdb-operator/main.go +++ b/packages/scale-zero-pg/gateway/cmd/appdb-operator/main.go @@ -16,6 +16,7 @@ import ( "encoding/hex" "fmt" "log" + "math" "net/http" "os" "os/signal" @@ -49,6 +50,51 @@ func gatewayHostFromEnv() string { return env("APPDB_GATEWAY_HOST", appdb.DefaultGatewayHost) } +// applyImageEnv applies the image overrides (APPDB_COMPUTE_IMAGE, APPDB_INIT_IMAGE) +// together with the numeric identity those images run as. The compute pods run +// every container with runAsNonRoot and an explicit runAsUser/runAsGroup, because +// the images declare their user by name and the kubelet cannot verify a name as +// non-root. The compute image and the init image are different images with their +// own users, so each has its own pair: APPDB_COMPUTE_UID/GID and +// APPDB_INIT_UID/GID. Unset keeps the pinned defaults (right only for the pinned +// digests); an image override whose user is not that default MUST set its pair, +// or the container starts as the wrong user. A malformed value is an error here +// (start-up), never a silent fallback. +func applyImageEnv(render *appdb.RenderConfig) error { + render.ComputeImage = env("APPDB_COMPUTE_IMAGE", render.ComputeImage) + render.InitImage = env("APPDB_INIT_IMAGE", render.InitImage) + for _, f := range []struct { + key string + dst *int64 + }{ + {"APPDB_COMPUTE_UID", &render.ComputeUID}, + {"APPDB_COMPUTE_GID", &render.ComputeGID}, + {"APPDB_INIT_UID", &render.InitUID}, + {"APPDB_INIT_GID", &render.InitGID}, + } { + v, err := idFromEnv(f.key, *f.dst) + if err != nil { + return err + } + *f.dst = v + } + return nil +} + +// idFromEnv reads a numeric uid/gid: unset or empty -> def; otherwise a decimal +// integer in [1, 2147483647]. 0 is refused because runAsNonRoot forbids root. +func idFromEnv(key string, def int64) (int64, error) { + raw := os.Getenv(key) + if raw == "" { + return def, nil + } + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil || n < 1 || n > math.MaxInt32 { + return 0, fmt.Errorf("%s=%q is invalid: want a numeric id between 1 and %d (runAsNonRoot forbids 0, and the container image's user name cannot be used)", key, raw, math.MaxInt32) + } + return n, nil +} + func envInt(key string, def int) int { if v := os.Getenv(key); v != "" { if n, err := strconv.Atoi(v); err == nil { @@ -112,8 +158,9 @@ func main() { render.PageserverHost = pageserverHost render.PGVersion = strconv.Itoa(pgVersion) render.RolePrefix = rolePrefix - render.ComputeImage = env("APPDB_COMPUTE_IMAGE", render.ComputeImage) - render.InitImage = env("APPDB_INIT_IMAGE", render.InitImage) + if err := applyImageEnv(&render); err != nil { + logger.Fatalf("[appdb] %v", err) + } cluster := appdb.NewK8sCluster(cs, dyn, namespace, render, reclaimCM, logger) // The DB warm-hold actuator (knext #388, ADR-0030 addendum; #777): the diff --git a/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go b/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go index e5950cf82..5d4dbabd8 100644 --- a/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go +++ b/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go @@ -1,11 +1,98 @@ package main import ( + "os" + "strings" "testing" "github.com/alpheya/scale-zero-pg/gateway/internal/appdb" ) +var identityEnv = []string{ + "APPDB_COMPUTE_IMAGE", "APPDB_INIT_IMAGE", + "APPDB_COMPUTE_UID", "APPDB_COMPUTE_GID", "APPDB_INIT_UID", "APPDB_INIT_GID", +} + +func clearIdentityEnv(t *testing.T) { + t.Helper() + for _, k := range identityEnv { + t.Setenv(k, "") + } +} + +// With nothing set, the image overrides and uids stay at the pinned defaults. +func TestApplyImageEnvDefaults(t *testing.T) { + clearIdentityEnv(t) + got := appdb.DefaultRenderConfig("ns") + if err := applyImageEnv(&got); err != nil { + t.Fatalf("applyImageEnv with no env: %v", err) + } + if want := appdb.DefaultRenderConfig("ns"); got != want { + t.Fatalf("no env changed the config:\n got %+v\nwant %+v", got, want) + } +} + +// The images and the identity they run as are overridden together, each image +// with its own pair. +func TestApplyImageEnvOverrides(t *testing.T) { + clearIdentityEnv(t) + t.Setenv("APPDB_COMPUTE_IMAGE", "registry.example.com/pg@sha256:aa") + t.Setenv("APPDB_INIT_IMAGE", "registry.example.com/init@sha256:bb") + t.Setenv("APPDB_COMPUTE_UID", "26") + t.Setenv("APPDB_COMPUTE_GID", "27") + t.Setenv("APPDB_INIT_UID", "65532") + t.Setenv("APPDB_INIT_GID", "65533") + c := appdb.DefaultRenderConfig("ns") + if err := applyImageEnv(&c); err != nil { + t.Fatalf("applyImageEnv: %v", err) + } + if c.ComputeImage != "registry.example.com/pg@sha256:aa" || c.InitImage != "registry.example.com/init@sha256:bb" { + t.Errorf("image overrides not applied: %q / %q", c.ComputeImage, c.InitImage) + } + if c.ComputeUID != 26 || c.ComputeGID != 27 || c.InitUID != 65532 || c.InitGID != 65533 { + t.Errorf("identity overrides not applied: compute %d:%d init %d:%d", c.ComputeUID, c.ComputeGID, c.InitUID, c.InitGID) + } +} + +// A bad value fails at start-up and names the variable and the value: runAsNonRoot +// forbids uid 0 and a non-numeric id cannot be rendered. +func TestApplyImageEnvRejectsBadIdentity(t *testing.T) { + for _, key := range []string{"APPDB_COMPUTE_UID", "APPDB_COMPUTE_GID", "APPDB_INIT_UID", "APPDB_INIT_GID"} { + for _, bad := range []string{"0", "-1", "postgres", "1000x", "1.5", "99999999999999999999"} { + clearIdentityEnv(t) + t.Setenv(key, bad) + c := appdb.DefaultRenderConfig("ns") + err := applyImageEnv(&c) + if err == nil { + t.Errorf("%s=%q was accepted, want a start-up error", key, bad) + continue + } + if !strings.Contains(err.Error(), key) || !strings.Contains(err.Error(), bad) { + t.Errorf("%s=%q: error %q does not name the variable and the value", key, bad, err) + } + } + } +} + +// main() itself is not unit-testable, so scan it: the image overrides must flow +// through applyImageEnv (which also carries their uid/gid), never be read raw +// again, or an overridden image would silently run as the default uid. +func TestMainAppliesImageOverridesOnlyThroughApplyImageEnv(t *testing.T) { + src, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + s := string(src) + if n := strings.Count(s, "applyImageEnv(&render)"); n != 1 { + t.Errorf("main() must call applyImageEnv(&render) exactly once, found %d", n) + } + for _, key := range []string{"APPDB_COMPUTE_IMAGE", "APPDB_INIT_IMAGE"} { + if n := strings.Count(s, `"`+key+`"`); n != 1 { + t.Errorf("%s must be read in exactly one place (applyImageEnv), found %d", key, n) + } + } +} + // Half 1: with no APPDB_GATEWAY_HOST set, the operator mints DSNs against the // ROOTED apps-gateway name (trailing dot) — the only form that skips the ndots:5 // search-path walk on a fresh pod's first resolutions; a 4-dot name without the dot diff --git a/packages/scale-zero-pg/gateway/internal/appdb/image_uid_table_test.go b/packages/scale-zero-pg/gateway/internal/appdb/image_uid_table_test.go new file mode 100644 index 000000000..50d73b615 --- /dev/null +++ b/packages/scale-zero-pg/gateway/internal/appdb/image_uid_table_test.go @@ -0,0 +1,120 @@ +package appdb + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// PINNED IMAGE -> RUN-AS UID TABLE. +// +// The compute pods run every container with a numeric runAsUser (the images +// declare their user by NAME, which the kubelet cannot verify as non-root). The +// default uid in DefaultRenderConfig is only right for the digests it ships with, +// so each pinned digest has its uid recorded below, by hand, next to the digest. +// +// Bumping a digest therefore turns this test red until someone has re-read the +// new image's /etc/passwd and recorded the uid: +// +// crane export --platform linux/amd64 - | tar -xO etc/passwd +// +// There is deliberately no network access here: the table is maintained by hand +// and this test only enforces that it is COMPLETE and consistent with the +// defaults. +var pinnedImageUIDs = map[string]struct{ uid, gid int64 }{ + // compute image: config User "postgres", /etc/passwd postgres:x:1000:1000 + "neondatabase/compute-node-v17:8464@sha256:13ab146d3e7bbabb25a8532f315ac443e7512351d1ede0bab586def5c70e26c3": {1000, 1000}, + // init image (also the storage-plane image): config User "neon", /etc/passwd neon:x:1000:1000 + "neondatabase/neon:8464@sha256:7a4f124917bb929964b2d696d710f19584f80bb9bd51b2af4a6e2425434c761f": {1000, 1000}, +} + +var pinnedNeonImage = regexp.MustCompile(`neondatabase/(?:compute-node-v[0-9]+|neon):[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}`) + +// pinnedRef strips the registry host so the table key is registry-independent. +func pinnedRef(image string) string { + return pinnedNeonImage.FindString(image) +} + +func TestDefaultImagesHaveARecordedUID(t *testing.T) { + c := DefaultRenderConfig("ns") + for _, tc := range []struct { + name, image string + uid, gid int64 + }{ + {"compute", c.ComputeImage, c.ComputeUID, c.ComputeGID}, + {"init", c.InitImage, c.InitUID, c.InitGID}, + } { + ref := pinnedRef(tc.image) + if ref == "" { + t.Fatalf("%s image %q is not a digest-pinned neondatabase image; the uid table cannot cover it", tc.name, tc.image) + } + want, ok := pinnedImageUIDs[ref] + if !ok { + t.Errorf("%s image %s has NO recorded uid in pinnedImageUIDs. Read the new image's /etc/passwd "+ + "(crane export - | tar -xO etc/passwd) and add it, then update the default uid.", tc.name, ref) + continue + } + if tc.uid != want.uid || tc.gid != want.gid { + t.Errorf("%s image %s: default uid:gid = %d:%d but the image runs as %d:%d", tc.name, ref, tc.uid, tc.gid, want.uid, want.gid) + } + } +} + +// Every digest pinned in the deploy manifests must also be in the table: the +// static manifests and the break-glass template pin the same images, and a digest +// bumped there alone must not skip the uid re-check. +func TestEveryPinnedNeonImageInDeployHasARecordedUID(t *testing.T) { + root := filepath.Join("..", "..", "..", "deploy") + found := map[string][]string{} + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == "testdata" { + return fs.SkipDir + } + return nil + } + b, rerr := os.ReadFile(path) + if rerr != nil { + return rerr + } + for _, m := range pinnedNeonImage.FindAllString(string(b), -1) { + found[m] = append(found[m], filepath.Base(path)) + } + return nil + }) + if err != nil { + t.Fatalf("walking %s: %v", root, err) + } + if len(found) == 0 { + t.Fatal("found no pinned neondatabase digests under deploy/: the scan would pass vacuously") + } + var refs []string + for r := range found { + refs = append(refs, r) + } + sort.Strings(refs) + for _, ref := range refs { + if _, ok := pinnedImageUIDs[ref]; !ok { + t.Errorf("deploy/ pins %s (in %s) but pinnedImageUIDs has no uid for it", ref, strings.Join(dedupe(found[ref]), ", ")) + } + } +} + +func dedupe(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} diff --git a/packages/scale-zero-pg/gateway/internal/appdb/render.go b/packages/scale-zero-pg/gateway/internal/appdb/render.go index 95286fb7e..f5d8ff691 100644 --- a/packages/scale-zero-pg/gateway/internal/appdb/render.go +++ b/packages/scale-zero-pg/gateway/internal/appdb/render.go @@ -19,6 +19,12 @@ type RenderConfig struct { RolePrefix string // "app_" ComputeImage string // neondatabase/compute-node-v17:8464 InitImage string // neondatabase/neon:8464 + // ComputeUID/ComputeGID is the numeric identity the compute container runs as; + // InitUID/InitGID the one the init containers run as. They are the identity + // the matching IMAGE declares, so an overridden image needs its own values + // (see DefaultImageUID). Must be > 0: runAsNonRoot forbids 0. + ComputeUID, ComputeGID int64 + InitUID, InitGID int64 } // DefaultRenderConfig fills a RenderConfig from the template's pinned values. @@ -30,31 +36,39 @@ func DefaultRenderConfig(ns string) RenderConfig { RolePrefix: "app_", ComputeImage: "docker.io/neondatabase/compute-node-v17:8464@sha256:13ab146d3e7bbabb25a8532f315ac443e7512351d1ede0bab586def5c70e26c3", InitImage: "docker.io/neondatabase/neon:8464@sha256:7a4f124917bb929964b2d696d710f19584f80bb9bd51b2af4a6e2425434c761f", + ComputeUID: DefaultImageUID, + ComputeGID: DefaultImageUID, + InitUID: DefaultImageUID, + InitGID: DefaultImageUID, } } -// computeImageUID is the numeric uid/gid every container of a rendered compute -// pod runs as. Measured from /etc/passwd of the pinned digests: the compute image -// declares USER `postgres` (1000:1000) and the init image USER `neon` (1000:1000). -// Both are NAMES in the image config, which the kubelet cannot verify as -// non-root, so runAsNonRoot needs the numeric id spelled out here. It equals the -// uid the pods already ran as, so nothing about file ownership or the boot -// changes. Bumping either image digest means re-reading it (crane export + the -// image's /etc/passwd): a different uid is a silent boot failure otherwise. -const computeImageUID int64 = 1000 +// DefaultImageUID is the numeric uid/gid the PINNED default images run as. +// Measured from /etc/passwd of the pinned digests: the compute image declares +// USER `postgres` (1000:1000) and the init image USER `neon` (1000:1000). Both +// are NAMES in the image config, which the kubelet cannot verify as non-root, so +// runAsNonRoot needs the numeric id spelled out. It equals the uid the pods +// already ran as, so nothing about file ownership or the boot changes. +// +// It is only right for those digests. Bumping a digest means re-reading the new +// image's /etc/passwd and recording the uid in pinnedImageUIDs +// (image_uid_table_test.go), which fails until it is done; an image supplied at +// run time (APPDB_COMPUTE_IMAGE / APPDB_INIT_IMAGE) needs its identity supplied +// with it (APPDB_COMPUTE_UID/GID, APPDB_INIT_UID/GID). +const DefaultImageUID int64 = 1000 // restrictedContainerSecurityContext is the container securityContext every // container of a gateway-rendered compute pod carries, so the pod is admitted in // a namespace enforcing the Kubernetes Pod Security `restricted` level: // allowPrivilegeEscalation false, capabilities drop ALL (nothing added back - // compute_ctl and Postgres run unprivileged on ports above 1024 and need none), -// seccomp RuntimeDefault, runAsNonRoot with the numeric uid/gid above. +// seccomp RuntimeDefault, runAsNonRoot with the numeric uid/gid of the image the +// container runs (RenderConfig.ComputeUID/InitUID). // readOnlyRootFilesystem is deliberately not set: Postgres and compute_ctl write // under the container filesystem (PGDATA, the staged TLS key), and it is not part // of the restricted profile. -func restrictedContainerSecurityContext() *corev1.SecurityContext { +func restrictedContainerSecurityContext(uid, gid int64) *corev1.SecurityContext { no, yes := false, true - uid, gid := computeImageUID, computeImageUID return &corev1.SecurityContext{ AllowPrivilegeEscalation: &no, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, @@ -189,7 +203,7 @@ echo "timeline ready"` InitContainers: []corev1.Container{{ Name: "wait-timeline", Image: c.InitImage, - SecurityContext: restrictedContainerSecurityContext(), + SecurityContext: restrictedContainerSecurityContext(c.InitUID, c.InitGID), EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "compute-config-" + s.App}}}}, Command: []string{"/bin/sh", "-c"}, Args: []string{waitScript}, @@ -198,7 +212,7 @@ echo "timeline ready"` Name: "compute", Image: c.ComputeImage, ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: restrictedContainerSecurityContext(), + SecurityContext: restrictedContainerSecurityContext(c.ComputeUID, c.ComputeGID), Command: []string{"/bin/sh", "/compute-files/entrypoint.sh"}, EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "compute-config-" + s.App}}}}, Env: []corev1.EnvVar{{ @@ -366,7 +380,7 @@ echo "resolved tip LSN = ${LSN:-}"` { Name: "wait-timeline", Image: c.InitImage, - SecurityContext: restrictedContainerSecurityContext(), + SecurityContext: restrictedContainerSecurityContext(c.InitUID, c.InitGID), EnvFrom: cmRef, Command: []string{"/bin/sh", "-c"}, Args: []string{waitScript}, @@ -374,7 +388,7 @@ echo "resolved tip LSN = ${LSN:-}"` { Name: "resolve-lsn", Image: c.InitImage, - SecurityContext: restrictedContainerSecurityContext(), + SecurityContext: restrictedContainerSecurityContext(c.InitUID, c.InitGID), EnvFrom: cmRef, Command: []string{"/bin/sh", "-c"}, Args: []string{resolveLSNScript}, @@ -385,7 +399,7 @@ echo "resolved tip LSN = ${LSN:-}"` Name: "compute", Image: c.ComputeImage, ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: restrictedContainerSecurityContext(), + SecurityContext: restrictedContainerSecurityContext(c.ComputeUID, c.ComputeGID), Command: []string{"/bin/sh", "/compute-files/entrypoint-ro.sh"}, EnvFrom: cmRef, // Same security env as the per-app WRITER (RenderDeployment): both are diff --git a/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go b/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go index 29f6db67f..eaa6698b9 100644 --- a/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go +++ b/packages/scale-zero-pg/gateway/internal/appdb/render_pod_security_test.go @@ -117,6 +117,32 @@ func TestRestrictedEvaluatorRejectsUnhardenedPod(t *testing.T) { } } +// An operator that overrides the compute or init image can override the uid the +// containers run as: the compute image's identity applies to the compute +// container only and the init image's to the init containers only, in both the +// writer and the read replica. +func TestRenderHonoursImageIdentityOverrides(t *testing.T) { + c := DefaultRenderConfig("scale-zero-pg") + c.ComputeUID, c.ComputeGID = 2001, 2002 + c.InitUID, c.InitGID = 3001, 3002 + w := c.RenderDeployment(ComputeSpec{App: "shop", TenantID: "t", TimelineID: "l"}) + r := c.RenderRODeployment(ROComputeSpec{App: "shop", TenantID: "t", TimelineID: "l", MaxReplicas: 2}) + for name, spec := range map[string]*corev1.PodSpec{"writer": &w.Spec.Template.Spec, "read replica": &r.Spec.Template.Spec} { + for _, ic := range spec.InitContainers { + sc := ic.SecurityContext + if sc == nil || sc.RunAsUser == nil || *sc.RunAsUser != 3001 || sc.RunAsGroup == nil || *sc.RunAsGroup != 3002 { + t.Errorf("%s init container %q: runAsUser/runAsGroup = %+v, want 3001/3002 (InitUID/InitGID)", name, ic.Name, sc) + } + } + for _, mc := range spec.Containers { + sc := mc.SecurityContext + if sc == nil || sc.RunAsUser == nil || *sc.RunAsUser != 2001 || sc.RunAsGroup == nil || *sc.RunAsGroup != 2002 { + t.Errorf("%s container %q: runAsUser/runAsGroup = %+v, want 2001/2002 (ComputeUID/ComputeGID)", name, mc.Name, sc) + } + } + } +} + // The break-glass path (provision-app.sh renders deploy/compute-app.template.yaml) // must meet the same profile, field for field, as the operator's render: the // structural parity guard only sees that a securityContext is PRESENT. From 13a32aec5b1cbd7601d7d2b97fd16acf045e4887 Mon Sep 17 00:00:00 2001 From: AhmedElBanna80 Date: Thu, 22 Oct 2026 18:15:13 +0200 Subject: [PATCH 3/3] test: pin the uid upper bound in the operator identity env validation (#2564) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0198oVxWkEEDJJnVhES6ULnx --- packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go b/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go index 5d4dbabd8..8148c6159 100644 --- a/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go +++ b/packages/scale-zero-pg/gateway/cmd/appdb-operator/main_test.go @@ -58,7 +58,7 @@ func TestApplyImageEnvOverrides(t *testing.T) { // forbids uid 0 and a non-numeric id cannot be rendered. func TestApplyImageEnvRejectsBadIdentity(t *testing.T) { for _, key := range []string{"APPDB_COMPUTE_UID", "APPDB_COMPUTE_GID", "APPDB_INIT_UID", "APPDB_INIT_GID"} { - for _, bad := range []string{"0", "-1", "postgres", "1000x", "1.5", "99999999999999999999"} { + for _, bad := range []string{"0", "-1", "postgres", "1000x", "1.5", "2147483648", "99999999999999999999"} { clearIdentityEnv(t) t.Setenv(key, bad) c := appdb.DefaultRenderConfig("ns")