From 2902166de9d5262173eb4ee2b31fdd6f90fa1d6a Mon Sep 17 00:00:00 2001 From: hd1801 Date: Fri, 17 Jul 2026 16:16:36 +0530 Subject: [PATCH] refactor(k8s): split manifests by lifecycle so ArgoCD stops reapplying secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArgoCD tracked the whole k8s/ tree, and CI commits an image tag into k8s/app on every build, which triggers a full sync. That meant every deploy re-applied k8s/app/secret.yaml, overwriting the real cluster secret with its CHANGEME_ placeholders. Split the tree by how often a resource legitimately changes: k8s/app/ per-deploy workloads — the only auto-synced path k8s/platform/ Postgres/MinIO/Redis + ingress, manual sync k8s/bootstrap/ secrets, RBAC, namespace, one-shot jobs — untracked by ArgoCD, applied by hand once (see its README) Real secrets are now gitignored and live only in the cluster; only *.example.yaml templates are tracked. Scope the sync pipeline to k8s/app/** so platform changes don't auto-fire. Also fix an ambiguous ArgoCD targetRevision: "prod" is both a branch and a CI-force-pushed tag pointing at a different commit. Fully qualify it as refs/heads/prod. All manifest moves are pure renames — no spec content changed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 6 +- .woodpecker/sync.yaml | 5 +- argocd/application-platform.yaml | 23 +++++ argocd/application.yaml | 9 +- k8s/app/secret.yaml | 13 --- k8s/bootstrap/README.md | 86 +++++++++++++++++++ .../jobs}/init-buckets-job.yaml | 0 .../jobs}/init-db-job.yaml | 0 k8s/{ => bootstrap}/namespace.yaml | 0 k8s/{app => bootstrap}/rbac.yaml | 0 .../secrets/attendee-secret.example.yaml | 24 ++++++ .../secrets/minio-secret.example.yaml | 18 ++++ .../secrets/postgres-secret.example.yaml | 12 +++ k8s/bootstrap/secrets/regcred.example.yaml | 22 +++++ k8s/infra/minio/secret.yaml | 10 --- k8s/infra/postgres/secret.yaml | 9 -- k8s/{ => platform}/ingress/ingress.yaml | 0 k8s/{infra => platform}/minio/deployment.yaml | 0 k8s/{infra => platform}/minio/pvc.yaml | 0 k8s/{infra => platform}/minio/service.yaml | 0 .../postgres/deployment.yaml | 0 k8s/{infra => platform}/postgres/pvc.yaml | 0 k8s/{infra => platform}/postgres/service.yaml | 0 k8s/{infra => platform}/redis/deployment.yaml | 0 k8s/{infra => platform}/redis/pvc.yaml | 0 k8s/{infra => platform}/redis/service.yaml | 0 26 files changed, 200 insertions(+), 37 deletions(-) create mode 100644 argocd/application-platform.yaml delete mode 100644 k8s/app/secret.yaml create mode 100644 k8s/bootstrap/README.md rename k8s/{infra/minio => bootstrap/jobs}/init-buckets-job.yaml (100%) rename k8s/{infra/postgres => bootstrap/jobs}/init-db-job.yaml (100%) rename k8s/{ => bootstrap}/namespace.yaml (100%) rename k8s/{app => bootstrap}/rbac.yaml (100%) create mode 100644 k8s/bootstrap/secrets/attendee-secret.example.yaml create mode 100644 k8s/bootstrap/secrets/minio-secret.example.yaml create mode 100644 k8s/bootstrap/secrets/postgres-secret.example.yaml create mode 100644 k8s/bootstrap/secrets/regcred.example.yaml delete mode 100644 k8s/infra/minio/secret.yaml delete mode 100644 k8s/infra/postgres/secret.yaml rename k8s/{ => platform}/ingress/ingress.yaml (100%) rename k8s/{infra => platform}/minio/deployment.yaml (100%) rename k8s/{infra => platform}/minio/pvc.yaml (100%) rename k8s/{infra => platform}/minio/service.yaml (100%) rename k8s/{infra => platform}/postgres/deployment.yaml (100%) rename k8s/{infra => platform}/postgres/pvc.yaml (100%) rename k8s/{infra => platform}/postgres/service.yaml (100%) rename k8s/{infra => platform}/redis/deployment.yaml (100%) rename k8s/{infra => platform}/redis/pvc.yaml (100%) rename k8s/{infra => platform}/redis/service.yaml (100%) diff --git a/.gitignore b/.gitignore index 179aaffd0..e6ee18fc7 100644 --- a/.gitignore +++ b/.gitignore @@ -99,8 +99,10 @@ local_settings.py .env db.sqlite3 -k8s/app/regcred.yaml -k8s/app/namespace-admin.yaml +# Real bootstrap secrets live only in the cluster. Only *.example.yaml is tracked. +k8s/bootstrap/secrets/*.yaml +!k8s/bootstrap/secrets/*.example.yaml +k8s/bootstrap/namespace-admin.yaml *-kubeconfig.yaml custom/ diff --git a/.woodpecker/sync.yaml b/.woodpecker/sync.yaml index d253bae26..751e4b387 100644 --- a/.woodpecker/sync.yaml +++ b/.woodpecker/sync.yaml @@ -1,8 +1,11 @@ +# Only k8s/app is auto-synced. k8s/platform (stateful infra, ingress) and +# k8s/bootstrap (secrets, RBAC, one-shot jobs) are applied deliberately, not +# on every push — see k8s/bootstrap/README.md. when: - event: push branch: prod path: - - "k8s/**" + - "k8s/app/**" steps: argocd-sync: diff --git a/argocd/application-platform.yaml b/argocd/application-platform.yaml new file mode 100644 index 000000000..f8bdb9ebf --- /dev/null +++ b/argocd/application-platform.yaml @@ -0,0 +1,23 @@ +# Stateful infrastructure (Postgres, MinIO, Redis) and the shared ingress. +# Deliberately NOT synced by CI: these change on their own schedule, and a +# careless sync here can disrupt storage or routing. Sync manually from the +# ArgoCD UI/CLI after reviewing the diff. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: attendee-platform + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: git@github.com:wisflux/attendee.git + targetRevision: refs/heads/prod + path: k8s/platform + destination: + server: https://kubernetes.default.svc + namespace: meeting-utility + syncPolicy: + syncOptions: + - CreateNamespace=false diff --git a/argocd/application.yaml b/argocd/application.yaml index 0cfdee33f..d04c6f17e 100644 --- a/argocd/application.yaml +++ b/argocd/application.yaml @@ -1,3 +1,6 @@ +# Application workloads — the only path CI auto-syncs on every deploy. +# Contains nothing whose lifecycle differs from a code push: no secrets, +# no RBAC, no namespace, no one-shot jobs. Those live in k8s/bootstrap. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -9,8 +12,10 @@ spec: project: default source: repoURL: git@github.com:wisflux/attendee.git - targetRevision: prod - path: k8s + # Fully qualified: a tag named "prod" also exists and CI force-pushes it, + # so a bare "prod" is an ambiguous ref. + targetRevision: refs/heads/prod + path: k8s/app destination: server: https://kubernetes.default.svc namespace: meeting-utility diff --git a/k8s/app/secret.yaml b/k8s/app/secret.yaml deleted file mode 100644 index 3192e931a..000000000 --- a/k8s/app/secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: attendee-secret - namespace: meeting-utility -type: Opaque -stringData: - DJANGO_SECRET_KEY: "CHANGEME_generate_with_python_-c_import_secrets_print_secrets.token_hex_50" - CREDENTIALS_ENCRYPTION_KEY: "CHANGEME_generate_with_python_-c_from_cryptography.fernet_import_Fernet_print_Fernet.generate_key().decode()" - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "CHANGEME_strong_password" - POSTGRES_USER: "pgadmin" - POSTGRES_PASSWORD: "CHANGEME_strong_password" diff --git a/k8s/bootstrap/README.md b/k8s/bootstrap/README.md new file mode 100644 index 000000000..89411de97 --- /dev/null +++ b/k8s/bootstrap/README.md @@ -0,0 +1,86 @@ +# Bootstrap + +Resources here are **applied by hand, once, when standing up a namespace**. ArgoCD does +not track this directory — no Application points at it, and nothing in CI applies it. + +That is the entire point. Everything in here either holds a real secret value or is a +one-shot operation, so re-applying it on every deploy is at best pointless and at worst +destructive. Previously these lived under `k8s/app`, which ArgoCD re-synced on every +image-tag push — meaning the placeholder secrets in git overwrote the real ones in the +cluster on every deploy. + +## Layout + +| Path | Why it is not in `k8s/app` | +|---|---| +| `namespace.yaml` | Created once; the Applications set `CreateNamespace=false`. | +| `rbac.yaml` | ServiceAccount + Role the app uses to launch bot pods. Changes rarely. | +| `namespace-admin.yaml` | Admin SA whose token backs the kubeconfig. Gitignored. | +| `secrets/*.example.yaml` | Templates. The real `*.yaml` are gitignored and live only in the cluster. | +| `jobs/` | One-shot init Jobs. Job specs are immutable — re-applying a changed one fails. | + +## Standing up a fresh namespace + +Run in order; each step depends on the previous. + +```sh +# 1. Namespace +kubectl apply -f k8s/bootstrap/namespace.yaml + +# 2. Secrets — copy each example, fill in real values, then apply. +# See the header comment in each file for how to generate values. +cp k8s/bootstrap/secrets/attendee-secret.example.yaml k8s/bootstrap/secrets/attendee-secret.yaml +cp k8s/bootstrap/secrets/postgres-secret.example.yaml k8s/bootstrap/secrets/postgres-secret.yaml +cp k8s/bootstrap/secrets/minio-secret.example.yaml k8s/bootstrap/secrets/minio-secret.yaml +$EDITOR k8s/bootstrap/secrets/*.yaml +kubectl apply -f k8s/bootstrap/secrets/attendee-secret.yaml \ + -f k8s/bootstrap/secrets/postgres-secret.yaml \ + -f k8s/bootstrap/secrets/minio-secret.yaml + +# 3. Image pull secret — generated, not hand-written. +# See secrets/regcred.example.yaml for the command. + +# 4. RBAC +kubectl apply -f k8s/bootstrap/rbac.yaml +kubectl apply -f k8s/bootstrap/namespace-admin.yaml + +# 5. Platform (Postgres/MinIO/Redis/ingress) — sync the attendee-platform +# ArgoCD Application, then wait for Postgres and MinIO to be Ready. +kubectl -n meeting-utility rollout status deploy/postgres deploy/minio + +# 6. Init jobs — only after Postgres and MinIO are up. +kubectl apply -f k8s/bootstrap/jobs/ + +# 7. App — sync the attendee ArgoCD Application. +``` + +## Rotating a secret + +Edit the real (gitignored) file and re-apply, then restart consumers — pods read env +from Secrets at start and will not pick up changes on their own: + +```sh +kubectl apply -f k8s/bootstrap/secrets/attendee-secret.yaml +kubectl -n meeting-utility rollout restart deploy/attendee-web deploy/attendee-worker \ + deploy/attendee-scheduler deploy/attendee-webpage-streamer +``` + +Changing `postgres-secret` or `minio-secret` after data exists does **not** re-key the +running Postgres/MinIO — their passwords are set at first init from the PVC. Rotating +those means changing the credential inside the service too. + +## Re-running an init job + +Jobs are immutable. Delete before re-applying: + +```sh +kubectl -n meeting-utility delete job postgres-init-db minio-init-buckets --ignore-not-found +kubectl apply -f k8s/bootstrap/jobs/ +``` + +## Longer term + +The migration to the Vault-equipped cluster should replace `secrets/` with External +Secrets Operator `ExternalSecret` resources. Those hold only Vault *references*, not +values, so they can live in `k8s/app` and sync freely — which removes the manual step +this directory exists to protect. diff --git a/k8s/infra/minio/init-buckets-job.yaml b/k8s/bootstrap/jobs/init-buckets-job.yaml similarity index 100% rename from k8s/infra/minio/init-buckets-job.yaml rename to k8s/bootstrap/jobs/init-buckets-job.yaml diff --git a/k8s/infra/postgres/init-db-job.yaml b/k8s/bootstrap/jobs/init-db-job.yaml similarity index 100% rename from k8s/infra/postgres/init-db-job.yaml rename to k8s/bootstrap/jobs/init-db-job.yaml diff --git a/k8s/namespace.yaml b/k8s/bootstrap/namespace.yaml similarity index 100% rename from k8s/namespace.yaml rename to k8s/bootstrap/namespace.yaml diff --git a/k8s/app/rbac.yaml b/k8s/bootstrap/rbac.yaml similarity index 100% rename from k8s/app/rbac.yaml rename to k8s/bootstrap/rbac.yaml diff --git a/k8s/bootstrap/secrets/attendee-secret.example.yaml b/k8s/bootstrap/secrets/attendee-secret.example.yaml new file mode 100644 index 000000000..509030fa5 --- /dev/null +++ b/k8s/bootstrap/secrets/attendee-secret.example.yaml @@ -0,0 +1,24 @@ +# Template only — NOT applied by ArgoCD. +# Copy to attendee-secret.yaml (gitignored), fill in real values, apply once: +# kubectl apply -f k8s/bootstrap/secrets/attendee-secret.yaml +# +# Generate the two key values with: +# DJANGO_SECRET_KEY: python -c 'import secrets; print(secrets.token_hex(50))' +# CREDENTIALS_ENCRYPTION_KEY: python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())' +# +# POSTGRES_* must match postgres-secret.yaml. +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY must match MINIO_ROOT_USER / +# MINIO_ROOT_PASSWORD in minio-secret.yaml — the app talks to MinIO with these. +apiVersion: v1 +kind: Secret +metadata: + name: attendee-secret + namespace: meeting-utility +type: Opaque +stringData: + DJANGO_SECRET_KEY: "REPLACE_ME" + CREDENTIALS_ENCRYPTION_KEY: "REPLACE_ME" + AWS_ACCESS_KEY_ID: "REPLACE_ME" + AWS_SECRET_ACCESS_KEY: "REPLACE_ME" + POSTGRES_USER: "REPLACE_ME" + POSTGRES_PASSWORD: "REPLACE_ME" diff --git a/k8s/bootstrap/secrets/minio-secret.example.yaml b/k8s/bootstrap/secrets/minio-secret.example.yaml new file mode 100644 index 000000000..f62eb916b --- /dev/null +++ b/k8s/bootstrap/secrets/minio-secret.example.yaml @@ -0,0 +1,18 @@ +# Template only — NOT applied by ArgoCD. +# Copy to minio-secret.yaml (gitignored), fill in real values, apply once. +# MINIO_ROOT_USER / MINIO_ROOT_PASSWORD must match AWS_ACCESS_KEY_ID / +# AWS_SECRET_ACCESS_KEY in attendee-secret.yaml. +# +# MINIO_BROWSER_REDIRECT_URL is not sensitive — it lives here because the MinIO +# deployment loads this Secret with envFrom. It must match the /console path on +# attendee-store in k8s/platform/ingress/ingress.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret + namespace: meeting-utility +type: Opaque +stringData: + MINIO_ROOT_USER: "REPLACE_ME" + MINIO_ROOT_PASSWORD: "REPLACE_ME" + MINIO_BROWSER_REDIRECT_URL: "https://attendee-store.apps.wisflux.com/console" diff --git a/k8s/bootstrap/secrets/postgres-secret.example.yaml b/k8s/bootstrap/secrets/postgres-secret.example.yaml new file mode 100644 index 000000000..a6d5076ec --- /dev/null +++ b/k8s/bootstrap/secrets/postgres-secret.example.yaml @@ -0,0 +1,12 @@ +# Template only — NOT applied by ArgoCD. +# Copy to postgres-secret.yaml (gitignored), fill in real values, apply once. +# These must match POSTGRES_USER / POSTGRES_PASSWORD in attendee-secret.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + namespace: meeting-utility +type: Opaque +stringData: + POSTGRES_USER: "REPLACE_ME" + POSTGRES_PASSWORD: "REPLACE_ME" diff --git a/k8s/bootstrap/secrets/regcred.example.yaml b/k8s/bootstrap/secrets/regcred.example.yaml new file mode 100644 index 000000000..d0a327602 --- /dev/null +++ b/k8s/bootstrap/secrets/regcred.example.yaml @@ -0,0 +1,22 @@ +# Template only — NOT applied by ArgoCD. +# Do not hand-write this one. Generate it from a Docker Hub access token: +# +# kubectl create secret docker-registry regcred \ +# --namespace=meeting-utility \ +# --docker-server=https://index.docker.io/v1/ \ +# --docker-username= \ +# --docker-password= +# +# To keep a local copy for reference (gitignored): +# kubectl -n meeting-utility get secret regcred -o yaml > k8s/bootstrap/secrets/regcred.yaml +# +# Pods reference this via imagePullSecrets, and bot pods via +# BOT_POD_IMAGE_PULL_SECRET_NAME in k8s/app/configmap.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: regcred + namespace: meeting-utility +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: REPLACE_ME_BASE64_DOCKERCONFIGJSON diff --git a/k8s/infra/minio/secret.yaml b/k8s/infra/minio/secret.yaml deleted file mode 100644 index 83a2c3adb..000000000 --- a/k8s/infra/minio/secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: minio-secret - namespace: meeting-utility -type: Opaque -stringData: - MINIO_ROOT_USER: "minioadmin" - MINIO_ROOT_PASSWORD: "CHANGEME_strong_password" - MINIO_BROWSER_REDIRECT_URL: "https://attendee-store.apps.wisflux.com/console" diff --git a/k8s/infra/postgres/secret.yaml b/k8s/infra/postgres/secret.yaml deleted file mode 100644 index d2ab71d02..000000000 --- a/k8s/infra/postgres/secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: postgres-secret - namespace: meeting-utility -type: Opaque -stringData: - POSTGRES_USER: "pgadmin" - POSTGRES_PASSWORD: "CHANGEME_strong_password" diff --git a/k8s/ingress/ingress.yaml b/k8s/platform/ingress/ingress.yaml similarity index 100% rename from k8s/ingress/ingress.yaml rename to k8s/platform/ingress/ingress.yaml diff --git a/k8s/infra/minio/deployment.yaml b/k8s/platform/minio/deployment.yaml similarity index 100% rename from k8s/infra/minio/deployment.yaml rename to k8s/platform/minio/deployment.yaml diff --git a/k8s/infra/minio/pvc.yaml b/k8s/platform/minio/pvc.yaml similarity index 100% rename from k8s/infra/minio/pvc.yaml rename to k8s/platform/minio/pvc.yaml diff --git a/k8s/infra/minio/service.yaml b/k8s/platform/minio/service.yaml similarity index 100% rename from k8s/infra/minio/service.yaml rename to k8s/platform/minio/service.yaml diff --git a/k8s/infra/postgres/deployment.yaml b/k8s/platform/postgres/deployment.yaml similarity index 100% rename from k8s/infra/postgres/deployment.yaml rename to k8s/platform/postgres/deployment.yaml diff --git a/k8s/infra/postgres/pvc.yaml b/k8s/platform/postgres/pvc.yaml similarity index 100% rename from k8s/infra/postgres/pvc.yaml rename to k8s/platform/postgres/pvc.yaml diff --git a/k8s/infra/postgres/service.yaml b/k8s/platform/postgres/service.yaml similarity index 100% rename from k8s/infra/postgres/service.yaml rename to k8s/platform/postgres/service.yaml diff --git a/k8s/infra/redis/deployment.yaml b/k8s/platform/redis/deployment.yaml similarity index 100% rename from k8s/infra/redis/deployment.yaml rename to k8s/platform/redis/deployment.yaml diff --git a/k8s/infra/redis/pvc.yaml b/k8s/platform/redis/pvc.yaml similarity index 100% rename from k8s/infra/redis/pvc.yaml rename to k8s/platform/redis/pvc.yaml diff --git a/k8s/infra/redis/service.yaml b/k8s/platform/redis/service.yaml similarity index 100% rename from k8s/infra/redis/service.yaml rename to k8s/platform/redis/service.yaml