diff --git a/infra/server/relay-deploy.sh b/infra/server/relay-deploy.sh index af2fa71..3d5ed40 100644 --- a/infra/server/relay-deploy.sh +++ b/infra/server/relay-deploy.sh @@ -10,46 +10,96 @@ # can run it but not change it. Everything it prints shows up in the GitHub Actions log. set -euo pipefail -REPO=/home/deploy/relay # our repo, cloned by setup-deploy-user.sh -export KUBECONFIG=/home/deploy/.kube/config # the "deployer" badge +REPO=/home/deploy/relay # our repo, cloned by setup-deploy-user.sh +LAST_GOOD_FILE=/home/deploy/.relay-last-good # the last commit that deployed successfully +export KUBECONFIG=/home/deploy/.kube/config # the "deployer" badge fail() { echo "relay-deploy: $*" >&2; exit 1; } -# 0. One deploy at a time: a second deploy waits here for up to 5 minutes. -# The lock is released by itself when the script ends, even if it crashes. -exec 9>/home/deploy/.relay-deploy.lock -flock --wait 300 9 || fail "another deploy is still running" - SHA="${SSH_ORIGINAL_COMMAND:-}" # 1. Gate 1: the input must be a full commit SHA, exactly 40 characters of 0-9 and a-f. -# It runs first, because $SHA is used in git and sed below. +# It runs first, because $SHA is used in git and sed below. It needs no lock. if ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then fail "invalid commit SHA: $SHA" fi -# 2. Gate 2: the commit must be on main, which means it was merged through a reviewed PR. +# 2. One deploy at a time: a second deploy waits here for up to 5 minutes. +# The lock is released by itself when the script ends, even if it crashes. +exec 9>/home/deploy/.relay-deploy.lock +flock --wait 300 9 || fail "another deploy is still running" + +# 3. Gate 2: the commit must be on main, which means it was merged through a reviewed PR. git -C "$REPO" fetch --quiet origin main if ! git -C "$REPO" merge-base --is-ancestor "$SHA" "origin/main"; then fail "commit $SHA is not part of origin/main's history" fi -# 3. Move our copy of the repo to exactly that commit, so the manifests match the image. -# --force: this copy is only for deploys, so any stray local change is thrown away. -git -C "$REPO" checkout --quiet --force --detach "$SHA" -[ -d "$REPO/k8s/app" ] || fail "no k8s/app folder at commit $SHA" - -# 4. Apply every manifest in k8s/app, with the placeholder __GIT_SHA__ replaced by the commit. -# The "---" line between files keeps them separate YAML documents. -echo "deploying $SHA" -for file in "$REPO"/k8s/app/*.yaml; do - printf '\n---\n' - sed "s/__GIT_SHA__/$SHA/g" "$file" -done | kubectl apply -f - - -# 5. Wait until the new pods are ready. If they never are, go back to the previous version. -if ! kubectl rollout status deployment/relay-api --timeout=180s; then - kubectl rollout undo deployment/relay-api || echo "relay-deploy: nothing to roll back to" - fail "rollout of $SHA failed" +# When a rollout fails, show why in the CI log: the pods' states and Kubernetes' latest +# events (an image that can't be pulled, a failing probe, a crash). "|| true": printing +# them must never change the result. +show_state() { + echo "--- pods" + kubectl get pods -o wide || true + echo "--- latest events" + kubectl get events --sort-by=.lastTimestamp 2>&1 | tail -n 20 || true +} + +# Deploys one commit: its manifests in k8s/app/ with its images, then waits until every +# Deployment it touched is ready. Returns 0 on success and 1 on any failure. +# Every step ends with "|| return 1": inside a function that is called from "if", set -e +# is switched off, so without it a failing command would be ignored. +deploy_commit() { + local commit="$1" applied deployment + + # Move our copy of the repo to exactly that commit, so the manifests match the images. + # --force: this copy is only for deploys, so any stray local change is thrown away. + git -C "$REPO" checkout --quiet --force --detach "$commit" || return 1 + if [ ! -d "$REPO/k8s/app" ]; then + echo "relay-deploy: no k8s/app folder at commit $commit" >&2 + return 1 + fi + + # Apply every manifest, with the placeholder __GIT_SHA__ replaced by the commit. + # The "---" line between files keeps them separate YAML documents. + # -o name prints what was applied, for example "deployment.apps/relay-api". + echo "deploying $commit" + applied=$(for file in "$REPO"/k8s/app/*.yaml; do + printf '\n---\n' + sed "s/__GIT_SHA__/$commit/g" "$file" + done | kubectl apply -f - -o name) || return 1 + echo "$applied" + + # Wait until each Deployment it touched has its new pods ready (the API, the dashboard, + # and later the worker), so one that never gets ready can't pass as a success. + for deployment in $(echo "$applied" | grep '^deployment'); do + if ! kubectl rollout status "$deployment" --timeout=180s; then + show_state + return 1 + fi + done +} + +# 4. Deploy the commit. On success, remember it as the last good one. +if deploy_commit "$SHA"; then + echo "$SHA" > "$LAST_GOOD_FILE" + echo "deployed $SHA" + exit 0 +fi + +# 5. The deploy failed: put the last good commit back, manifests and images. +if [ -f "$LAST_GOOD_FILE" ]; then + last_good=$(cat "$LAST_GOOD_FILE") + # The note's content goes into git and sed below, just like $SHA, so it gets gate 1's check. + if ! [[ "$last_good" =~ ^[0-9a-f]{40}$ ]]; then + fail "deploy of $SHA failed, and the last-good note is damaged ('$last_good'): a human is needed" + fi + echo "rolling back to last good commit $last_good" + deploy_commit "$last_good" || fail "rollback to $last_good failed" + echo "rolled back to $last_good" +else + echo "no last good commit to roll back to" fi -echo "deployed $SHA" + +# Safety net: whatever happens above, a failed deploy must never end with exit 0. +fail "deploy of $SHA failed" diff --git a/infra/server/test-relay-deploy.sh b/infra/server/test-relay-deploy.sh new file mode 100644 index 0000000..ef9bf70 --- /dev/null +++ b/infra/server/test-relay-deploy.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Tests relay-deploy's decisions with fake git, kubectl and flock: no server, no cluster. +# The script calls git and kubectl by name, so fakes placed first in PATH replace them. +# Usage (from the repo root): bash infra/server/test-relay-deploy.sh infra/server/relay-deploy.sh +set -uo pipefail +SCRIPT="${1:?usage: test-relay-deploy.sh path/to/relay-deploy.sh}" +GOOD=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +BAD=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +passed=0; failed=0 + +# A fresh fake world: the script with /home/deploy pointed at a temp folder, and fakes in PATH. +setup() { + T=$(mktemp -d) + mkdir -p "$T/bin" "$T/home/relay/k8s/app" + echo "kind: Deployment" > "$T/home/relay/k8s/app/api.yaml" + sed "s#/home/deploy#$T/home#g" "$SCRIPT" > "$T/relay-deploy" + # git: "checkout" notes which commit is checked out. The main check (merge-base) says + # "not on main" when not-on-main exists. Everything else (fetch) passes. + cat > "$T/bin/git" < "$T/checked-out" ;; + *merge-base*) [ -e "$T/not-on-main" ] && exit 1 ;; +esac +exit 0 +EOF + # kubectl: apply notes that it ran and prints one Deployment (or fails if apply-fails exists); + # rollout status fails when the checked-out commit is listed in "broken"; + # get (pods, events) prints a line, as the real one would. + cat > "$T/bin/kubectl" < /dev/null; touch "$T/applied"; [ -e "$T/apply-fails" ] && exit 1; echo deployment.apps/relay-api ;; + "rollout status"*) grep -qx "\$(cat "$T/checked-out")" "$T/broken" && exit 1; exit 0 ;; + get*) echo "fake kubectl \$*" ;; +esac +EOF + printf '#!/usr/bin/env bash\nexit 0\n' > "$T/bin/flock" + chmod +x "$T/bin/"* + : > "$T/broken" +} + +run() { SSH_ORIGINAL_COMMAND="$1" PATH="$T/bin:$PATH" bash "$T/relay-deploy" > "$T/out" 2>&1; echo $?; } + +check() { # check "" "" "" + if [ "$2" = "$3" ]; then passed=$((passed + 1)); echo " ok $1" + else failed=$((failed + 1)); echo " FAIL $1: got '$2', expected '$3'"; sed 's/^/ | /' "$T/out"; fi +} + +echo "1. A good deploy" +setup; code=$(run $GOOD) +check "exits 0" "$code" 0 +check "remembers it as last good" "$(cat "$T/home/.relay-last-good" 2>/dev/null)" $GOOD + +echo "2. A broken deploy, with a last good commit" +setup; echo $GOOD > "$T/home/.relay-last-good"; echo $BAD > "$T/broken"; code=$(run $BAD) +check "exits 1 (CI turns red)" "$code" 1 +check "puts the last good commit back" "$(cat "$T/checked-out")" $GOOD +check "last good stays the same" "$(cat "$T/home/.relay-last-good")" $GOOD + +echo "3. A broken first deploy (nothing to go back to)" +setup; echo $BAD > "$T/broken"; code=$(run $BAD) +check "exits 1" "$code" 1 +check "doesn't invent a last good" "$(cat "$T/home/.relay-last-good" 2>/dev/null || echo none)" none + +echo "4. The rollback fails too" +setup; echo $GOOD > "$T/home/.relay-last-good"; printf '%s\n%s\n' $BAD $GOOD > "$T/broken"; code=$(run $BAD) +check "exits 1" "$code" 1 +check "did try the last good commit" "$(cat "$T/checked-out")" $GOOD + +echo "5. kubectl apply fails (the set -e trap inside a function)" +setup; touch "$T/apply-fails"; code=$(run $GOOD) +check "exits 1" "$code" 1 +check "doesn't record it as last good" "$(cat "$T/home/.relay-last-good" 2>/dev/null || echo none)" none + +exists() { [ -e "$1" ] && echo yes || echo no; } + +echo "6. Gate 1 refuses anything that isn't a full commit SHA" +for input in "whoami" "$GOOD; touch /tmp/pwned" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" "$GOOD "; do + setup; code=$(run "$input") + check "refuses '$input'" "$code" 1 + check "checks nothing out for '$input'" "$(exists "$T/checked-out")" no +done + +echo "7. Gate 2 refuses a commit that isn't on main" +setup; touch "$T/not-on-main"; code=$(run $GOOD) +check "exits 1" "$code" 1 +check "checks nothing out" "$(exists "$T/checked-out")" no +check "applies nothing" "$(exists "$T/applied")" no + +echo "8. A failed rollout shows why in the log" +setup; echo $BAD > "$T/broken"; code=$(run $BAD) +check "prints the pods" "$(grep -q -- '--- pods' "$T/out" && echo yes || echo no)" yes +check "prints the latest events" "$(grep -q -- '--- latest events' "$T/out" && echo yes || echo no)" yes + +echo "9. A damaged last-good note is never used" +setup; echo "not a commit; touch /tmp/pwned" > "$T/home/.relay-last-good"; echo $BAD > "$T/broken"; code=$(run $BAD) +check "exits 1" "$code" 1 +check "never checks out what's in the note" "$(cat "$T/checked-out")" $BAD + +echo; echo "$passed passed, $failed failed" +[ "$failed" -eq 0 ] diff --git a/k8s/README.md b/k8s/README.md index f82ca19..448fb30 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -5,7 +5,7 @@ How each folder gets onto the cluster, and who may apply it. | Folder | What's in it | Applied by | |---|---|---| | `k8s/platform/` | Cluster setup: Let's Encrypt issuers, Traefik settings, the `relay` namespace and the `deployer` account | One of us, by hand, after the PR is reviewed | -| `k8s/app/` (from step 1.7) | Our app in namespace `relay`: Deployments, Services, Ingresses, ConfigMaps | Only the pipeline, through `relay-deploy` | +| `k8s/app/` (from step 1.7) | Our app in namespace `relay`, the API and the dashboard: Deployments, Services, Ingresses, ConfigMaps | Only the pipeline, through `relay-deploy` | | `k8s/hello/` | The temporary hello-world test app | By hand. Removed when the API takes over in step 1.7 | ## Rules @@ -15,9 +15,14 @@ How each folder gets onto the cluster, and who may apply it. - **`k8s/app/` holds only what the `deployer` account may change:** Deployments, Services, Ingresses and ConfigMaps. A Namespace or a Secret there would make the deploy fail with "forbidden". -- **Images in `k8s/app/` use the tag `__GIT_SHA__`.** `relay-deploy` replaces it with the - commit it deploys, so manifests and image always come from the same commit. Don't apply - these files by hand: the tag would be wrong. +- **Images in `k8s/app/` use the tag `__GIT_SHA__`** (`relay-api` and `relay-dashboard` + alike). `relay-deploy` replaces it with the commit it deploys, so manifests and images + always come from the same commit. That's why every merge to main builds every image, even + the ones that didn't change: the tag must exist for each commit. Don't apply these files + by hand: the tag would be wrong. +- **Removing a file from `k8s/app/` doesn't remove it from the cluster.** The `deployer` + account has no delete, so one of us deletes the resource by hand: + `kubectl -n relay delete /`. - **No Secrets in git.** They are created on the server with `kubectl create secret`. - **Git and the cluster must match.** Everything applied by hand is committed in the same PR. @@ -42,13 +47,21 @@ VPS Linux user "deploy": its key may only start /usr/local/bin/relay-deploy │ relay-deploy: must be 40 hex characters and part of main's history, then check out that commit, apply k8s/app/ with the tag, - wait for the rollout, or undo it if it fails + and wait until every Deployment is ready. + If one isn't: deploy the last good commit again. │ every kubectl call uses the "deployer" account: │ namespace relay only, no Secrets ▼ - Deployment relay-api runs the new pods + the Deployments (API, dashboard) run the new pods ``` +**Rollback.** After every successful deploy, `relay-deploy` writes the commit to +`/home/deploy/.relay-last-good`. If a later rollout fails, it deploys that commit again the +same way, so manifests and images go back together. `kubectl rollout undo` would only +revert the pods' image and leave the failed commit's ConfigMaps and Ingresses behind. The +job still turns red after a rollback, because the commit we asked for isn't live. Limit: +a resource that only the failed commit *added* stays, because there's no delete. + The server-side parts live in `infra/server/` and are installed by hand, by an admin: | File | What it does | Install or update | diff --git a/k8s/platform/deployer.yaml b/k8s/platform/deployer.yaml index 7df4811..e16dc7b 100644 --- a/k8s/platform/deployer.yaml +++ b/k8s/platform/deployer.yaml @@ -38,6 +38,11 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] + # Read-only: when a rollout fails, relay-deploy prints the latest events (an image that + # can't be pulled, a failing probe) into the CI log, so nobody has to log in to see why. + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["get", "list"] # The stable name in front of the pods, and the app's non-secret settings. - apiGroups: [""] resources: ["services", "configmaps"]