refactor(k8s): split manifests by lifecycle so ArgoCD stops reapplying secrets

.gitignore

@@ -99,8 +99,10 @@ local_settings.py
 
 .env
 db.sqlite3
-k8s/app/regcred.yaml
-k8s/app/namespace-admin.yaml
+# Real bootstrap secrets live only in the cluster. Only *.example.yaml is tracked.
+k8s/bootstrap/secrets/*.yaml
+!k8s/bootstrap/secrets/*.example.yaml
+k8s/bootstrap/namespace-admin.yaml
 *-kubeconfig.yaml
 
 custom/

.woodpecker/sync.yaml

@@ -1,8 +1,11 @@
+# Only k8s/app is auto-synced. k8s/platform (stateful infra, ingress) and
+# k8s/bootstrap (secrets, RBAC, one-shot jobs) are applied deliberately, not
+# on every push — see k8s/bootstrap/README.md.
 when:
   - event: push
     branch: prod
     path:
-      - "k8s/**"
+      - "k8s/app/**"
 
 steps:
   argocd-sync:

argocd/application-platform.yaml

@@ -0,0 +1,23 @@
+# Stateful infrastructure (Postgres, MinIO, Redis) and the shared ingress.
+# Deliberately NOT synced by CI: these change on their own schedule, and a
+# careless sync here can disrupt storage or routing. Sync manually from the
+# ArgoCD UI/CLI after reviewing the diff.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: attendee-platform
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: git@github.com:wisflux/attendee.git
+    targetRevision: refs/heads/prod
+    path: k8s/platform
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: meeting-utility
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=false

argocd/application.yaml

@@ -1,3 +1,6 @@
+# Application workloads — the only path CI auto-syncs on every deploy.
+# Contains nothing whose lifecycle differs from a code push: no secrets,
+# no RBAC, no namespace, no one-shot jobs. Those live in k8s/bootstrap.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
@@ -9,8 +12,10 @@ spec:
   project: default
   source:
     repoURL: git@github.com:wisflux/attendee.git
-    targetRevision: prod
-    path: k8s
+    # Fully qualified: a tag named "prod" also exists and CI force-pushes it,
+    # so a bare "prod" is an ambiguous ref.
+    targetRevision: refs/heads/prod
+    path: k8s/app
   destination:
     server: https://kubernetes.default.svc
     namespace: meeting-utility

k8s/bootstrap/README.md

@@ -0,0 +1,86 @@
+# Bootstrap
+
+Resources here are **applied by hand, once, when standing up a namespace**. ArgoCD does
+not track this directory — no Application points at it, and nothing in CI applies it.
+
+That is the entire point. Everything in here either holds a real secret value or is a
+one-shot operation, so re-applying it on every deploy is at best pointless and at worst
+destructive. Previously these lived under `k8s/app`, which ArgoCD re-synced on every
+image-tag push — meaning the placeholder secrets in git overwrote the real ones in the
+cluster on every deploy.
+
+## Layout
+
+| Path | Why it is not in `k8s/app` |
+|---|---|
+| `namespace.yaml` | Created once; the Applications set `CreateNamespace=false`. |
+| `rbac.yaml` | ServiceAccount + Role the app uses to launch bot pods. Changes rarely. |
+| `namespace-admin.yaml` | Admin SA whose token backs the kubeconfig. Gitignored. |
+| `secrets/*.example.yaml` | Templates. The real `*.yaml` are gitignored and live only in the cluster. |
+| `jobs/` | One-shot init Jobs. Job specs are immutable — re-applying a changed one fails. |
+
+## Standing up a fresh namespace
+
+Run in order; each step depends on the previous.
+
+```sh
+# 1. Namespace
+kubectl apply -f k8s/bootstrap/namespace.yaml
+
+# 2. Secrets — copy each example, fill in real values, then apply.
+#    See the header comment in each file for how to generate values.
+cp k8s/bootstrap/secrets/attendee-secret.example.yaml k8s/bootstrap/secrets/attendee-secret.yaml
+cp k8s/bootstrap/secrets/postgres-secret.example.yaml k8s/bootstrap/secrets/postgres-secret.yaml
+cp k8s/bootstrap/secrets/minio-secret.example.yaml    k8s/bootstrap/secrets/minio-secret.yaml
+$EDITOR k8s/bootstrap/secrets/*.yaml
+kubectl apply -f k8s/bootstrap/secrets/attendee-secret.yaml \
+               -f k8s/bootstrap/secrets/postgres-secret.yaml \
+               -f k8s/bootstrap/secrets/minio-secret.yaml
+
+# 3. Image pull secret — generated, not hand-written.
+#    See secrets/regcred.example.yaml for the command.
+
+# 4. RBAC
+kubectl apply -f k8s/bootstrap/rbac.yaml
+kubectl apply -f k8s/bootstrap/namespace-admin.yaml
+
+# 5. Platform (Postgres/MinIO/Redis/ingress) — sync the attendee-platform
+#    ArgoCD Application, then wait for Postgres and MinIO to be Ready.
+kubectl -n meeting-utility rollout status deploy/postgres deploy/minio
+
+# 6. Init jobs — only after Postgres and MinIO are up.
+kubectl apply -f k8s/bootstrap/jobs/
+
+# 7. App — sync the attendee ArgoCD Application.
+```
+
+## Rotating a secret
+
+Edit the real (gitignored) file and re-apply, then restart consumers — pods read env
+from Secrets at start and will not pick up changes on their own:
+
+```sh
+kubectl apply -f k8s/bootstrap/secrets/attendee-secret.yaml
+kubectl -n meeting-utility rollout restart deploy/attendee-web deploy/attendee-worker \
+        deploy/attendee-scheduler deploy/attendee-webpage-streamer
+```
+
+Changing `postgres-secret` or `minio-secret` after data exists does **not** re-key the
+running Postgres/MinIO — their passwords are set at first init from the PVC. Rotating
+those means changing the credential inside the service too.
+
+## Re-running an init job
+
+Jobs are immutable. Delete before re-applying:
+
+```sh
+kubectl -n meeting-utility delete job postgres-init-db minio-init-buckets --ignore-not-found
+kubectl apply -f k8s/bootstrap/jobs/
+```
+
+## Longer term
+
+The migration to the Vault-equipped cluster should replace `secrets/` with External
+Secrets Operator `ExternalSecret` resources. Those hold only Vault *references*, not
+values, so they can live in `k8s/app` and sync freely — which removes the manual step
+this directory exists to protect.

k8s/bootstrap/secrets/attendee-secret.example.yaml

@@ -0,0 +1,24 @@
+# Template only — NOT applied by ArgoCD.
+# Copy to attendee-secret.yaml (gitignored), fill in real values, apply once:
+#   kubectl apply -f k8s/bootstrap/secrets/attendee-secret.yaml
+#
+# Generate the two key values with:
+#   DJANGO_SECRET_KEY:          python -c 'import secrets; print(secrets.token_hex(50))'
+#   CREDENTIALS_ENCRYPTION_KEY: python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'
+#
+# POSTGRES_* must match postgres-secret.yaml.
+# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY must match MINIO_ROOT_USER /
+# MINIO_ROOT_PASSWORD in minio-secret.yaml — the app talks to MinIO with these.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: attendee-secret
+  namespace: meeting-utility
+type: Opaque
+stringData:
+  DJANGO_SECRET_KEY: "REPLACE_ME"
+  CREDENTIALS_ENCRYPTION_KEY: "REPLACE_ME"
+  AWS_ACCESS_KEY_ID: "REPLACE_ME"
+  AWS_SECRET_ACCESS_KEY: "REPLACE_ME"
+  POSTGRES_USER: "REPLACE_ME"
+  POSTGRES_PASSWORD: "REPLACE_ME"

k8s/bootstrap/secrets/minio-secret.example.yaml

@@ -0,0 +1,18 @@
+# Template only — NOT applied by ArgoCD.
+# Copy to minio-secret.yaml (gitignored), fill in real values, apply once.
+# MINIO_ROOT_USER / MINIO_ROOT_PASSWORD must match AWS_ACCESS_KEY_ID /
+# AWS_SECRET_ACCESS_KEY in attendee-secret.yaml.
+#
+# MINIO_BROWSER_REDIRECT_URL is not sensitive — it lives here because the MinIO
+# deployment loads this Secret with envFrom. It must match the /console path on
+# attendee-store in k8s/platform/ingress/ingress.yaml.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+  namespace: meeting-utility
+type: Opaque
+stringData:
+  MINIO_ROOT_USER: "REPLACE_ME"
+  MINIO_ROOT_PASSWORD: "REPLACE_ME"
+  MINIO_BROWSER_REDIRECT_URL: "https://attendee-store.apps.wisflux.com/console"

k8s/bootstrap/secrets/postgres-secret.example.yaml

@@ -0,0 +1,12 @@
+# Template only — NOT applied by ArgoCD.
+# Copy to postgres-secret.yaml (gitignored), fill in real values, apply once.
+# These must match POSTGRES_USER / POSTGRES_PASSWORD in attendee-secret.yaml.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secret
+  namespace: meeting-utility
+type: Opaque
+stringData:
+  POSTGRES_USER: "REPLACE_ME"
+  POSTGRES_PASSWORD: "REPLACE_ME"

k8s/bootstrap/secrets/regcred.example.yaml

@@ -0,0 +1,22 @@
+# Template only — NOT applied by ArgoCD.
+# Do not hand-write this one. Generate it from a Docker Hub access token:
+#
+#   kubectl create secret docker-registry regcred \
+#     --namespace=meeting-utility \
+#     --docker-server=https://index.docker.io/v1/ \
+#     --docker-username=<dockerhub-user> \
+#     --docker-password=<dockerhub-pat>
+#
+# To keep a local copy for reference (gitignored):
+#   kubectl -n meeting-utility get secret regcred -o yaml > k8s/bootstrap/secrets/regcred.yaml
+#
+# Pods reference this via imagePullSecrets, and bot pods via
+# BOT_POD_IMAGE_PULL_SECRET_NAME in k8s/app/configmap.yaml.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: regcred
+  namespace: meeting-utility
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: REPLACE_ME_BASE64_DOCKERCONFIGJSON