diff --git a/k8s/README.md b/k8s/README.md index 8081541..9d7579a 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -4,7 +4,7 @@ How each folder gets onto the cluster, and who may apply it. | Folder | What's in it | Applied by | |---|---|---| -| `k8s/platform/` | Cluster setup: Let's Encrypt issuers, Traefik settings, the `relay` namespace and the `deployer` account | One of us, by hand, after the PR is reviewed | +| `k8s/platform/` | Cluster setup: Let's Encrypt issuers, Traefik settings, the `relay` namespace (with its Pod Security level), the `deployer` account and the Service policy for `relay` | One of us, by hand, after the PR is reviewed | | `k8s/app/` (from step 1.7) | Our app in namespace `relay`, the API and the dashboard: Deployments, Services, Ingresses, ConfigMaps | Only the pipeline, through `relay-deploy` | ## Rules @@ -28,6 +28,18 @@ How each folder gets onto the cluster, and who may apply it. - **Server and platform changes are applied by the one who merges, right after merging**, and the check output is posted on the PR (for example `auth can-i` and the script `diff`). Otherwise git and the server drift apart, and it only shows when something fails. +- **Two guards on namespace `relay`**, checked by the API server before anything is saved, + whoever sends it (the pipeline or one of us): + - **Pod Security `restricted`** (labels in `deployer.yaml`): no pod runs as root, + privileged, or with host folders, network or ports. Every pod spec needs + `runAsNonRoot`, `allowPrivilegeEscalation: false`, `capabilities: drop: ["ALL"]` and the + `RuntimeDefault` seccomp profile, as `k8s/app/deployment.yaml` shows. + - **Services stay inside the cluster** (`relay-service-policy.yaml`): only type ClusterIP + (headless included), never `externalIPs`. ufw can't filter ports that Kubernetes + publishes, so nothing in `relay`, and above all Postgres, may publish one. +- **The issuers' challenge Services are ClusterIP** (`cluster-issuers.yaml`). cert-manager's + default is NodePort, which the Service policy would refuse in `relay`. Apply the issuers + before the policy. - **No Secrets in git.** They are created on the server with `kubectl create secret`. - **Git and the cluster must match.** Everything applied by hand is committed in the same PR. diff --git a/k8s/platform/cluster-issuers.yaml b/k8s/platform/cluster-issuers.yaml index f86793d..5a23bad 100644 --- a/k8s/platform/cluster-issuers.yaml +++ b/k8s/platform/cluster-issuers.yaml @@ -1,6 +1,12 @@ # Let's Encrypt issuers for the whole cluster (cert-manager). # staging: loose rate limits, untrusted certificates. Use while testing. # prod: trusted certificates, strict rate limits. Use once staging works. +# +# To prove we own a domain, cert-manager starts a small "solver" pod plus a Service for it, +# in the namespace of the certificate. Left alone, that Service is a NodePort: a port opened +# on the server itself, which ufw can't filter. "serviceType: ClusterIP" keeps it inside the +# cluster; Traefik reaches it like any other Service. Namespace relay only allows ClusterIP +# Services (relay-service-policy.yaml), so this setting is also what lets renewals work there. apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: @@ -14,6 +20,7 @@ spec: - http01: ingress: ingressClassName: traefik # prove domain ownership through Traefik on port 80 + serviceType: ClusterIP # the solver's Service stays inside the cluster --- apiVersion: cert-manager.io/v1 kind: ClusterIssuer @@ -28,3 +35,4 @@ spec: - http01: ingress: ingressClassName: traefik + serviceType: ClusterIP diff --git a/k8s/platform/deployer.yaml b/k8s/platform/deployer.yaml index e16dc7b..df1e321 100644 --- a/k8s/platform/deployer.yaml +++ b/k8s/platform/deployer.yaml @@ -4,10 +4,25 @@ # Applied by hand by one of us (platform files are never applied by the pipeline). # The folder our app lives in, apart from system parts like kube-system and cert-manager. +# +# Pod Security (step 2.0): the API server refuses any pod in relay that doesn't meet the +# "restricted" standard: no root, no privileged mode, no host folders, network or ports, no +# extra Linux capabilities. Without it, one merged manifest could run a privileged pod, which +# means control of the whole server, and from Phase 2 the database lives here. +# enforce: refuse the pod. warn: kubectl prints why. audit: noted in the audit log. +# -version: the rules of Kubernetes 1.36, so an upgrade can't tighten them by surprise. +# The deployer account can't change Namespace labels, so it can't switch this off. apiVersion: v1 kind: Namespace metadata: name: relay + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.36 + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: v1.36 + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: v1.36 --- # An identity for a program (the deploy script), not for a person. apiVersion: v1 diff --git a/k8s/platform/relay-service-policy.yaml b/k8s/platform/relay-service-policy.yaml new file mode 100644 index 0000000..4922e9d --- /dev/null +++ b/k8s/platform/relay-service-policy.yaml @@ -0,0 +1,49 @@ +# Guard: in namespace relay, every Service stays inside the cluster (step 2.0). +# +# A Service of type NodePort or LoadBalancer, or one with externalIPs, opens a port on the +# server itself. ufw can't filter those ports (Kubernetes publishes them past it), so only +# one.com's firewall would stand between Postgres and the internet. This policy makes the API +# server refuse such a Service before it's saved, whoever sends it: the pipeline's "deployer" +# account or one of us. The deployer can't change or remove this policy. +# +# Applied by hand by one of us (a platform file), after cluster-issuers.yaml: cert-manager's +# certificate Services must already be ClusterIP, or renewals in relay would be refused. + +# The rule: what an allowed Service looks like. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: relay-services-clusterip-only +spec: + # If the rule can't be evaluated (for example it reads a missing field), refuse. + # A guard that lets things through when it breaks isn't a guard. + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + # UPDATE too: otherwise an existing ClusterIP Service could be changed into a NodePort. + operations: ["CREATE", "UPDATE"] + resources: ["services"] + validations: + # "object" is the Service as it would be saved. Kubernetes has already filled in + # spec.type ("ClusterIP" when our YAML leaves it out). spec.externalIPs is optional. + # !has(...) comes first: an unset field can't be read (it's an error, not null), and "||" + # stops at the first true side, so size() only runs when externalIPs exists. + - expression: "object.spec.type == 'ClusterIP' && (!has(object.spec.externalIPs) || size(object.spec.externalIPs) == 0)" + message: "In namespace relay a Service must be ClusterIP without externalIPs: nothing may be published outside the cluster." +--- +# Where the rule applies, and what happens when it fails. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: relay-services-clusterip-only +spec: + policyName: relay-services-clusterip-only + validationActions: [Deny] # refuse the request (Warn or Audit would only report it) + matchResources: + namespaceSelector: + matchLabels: + # Every namespace carries its own name as this label. Only relay is guarded; + # Traefik's LoadBalancer Service in kube-system is untouched. + kubernetes.io/metadata.name: relay