infra: deploy script follow-ups from the #14 review

infra/server/relay-deploy.sh

@@ -13,6 +13,8 @@ set -euo pipefail
 REPO=/home/deploy/relay                        # our repo, cloned by setup-deploy-user.sh
 LAST_GOOD_FILE=/home/deploy/.relay-last-good   # the last commit that deployed successfully
 export KUBECONFIG=/home/deploy/.kube/config    # the "deployer" badge
+# No kubectl call below says "-n relay": the namespace comes from this kubeconfig's context,
+# which setup-deploy-user.sh sets to "relay". If that ever changes, these commands follow it.
 
 fail() { echo "relay-deploy: $*" >&2; exit 1; }
 
@@ -24,10 +26,13 @@ if ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
   fail "invalid commit SHA: $SHA"
 fi
 
-# 2. One deploy at a time: a second deploy waits here for up to 5 minutes.
+# 2. One deploy at a time: a second deploy waits here for up to 15 minutes. That's longer
+#    than the worst case (deploy + rollback, 180 s per Deployment each). In CI this rarely
+#    waits at all, because the pipeline already runs main's deploys one after the other;
+#    it protects against an admin deploying by hand at the same moment.
 #    The lock is released by itself when the script ends, even if it crashes.
 exec 9>/home/deploy/.relay-deploy.lock
-flock --wait 300 9 || fail "another deploy is still running"
+flock --wait 900 9 || fail "another deploy is still running after 15 minutes: nothing was changed, re-run this job"
 
 # 3. Gate 2: the commit must be on main, which means it was merged through a reviewed PR.
 git -C "$REPO" fetch --quiet origin main
@@ -78,6 +83,7 @@ deploy_commit() {
       return 1
     fi
   done
+  return 0 # says "success" on purpose, not by luck of what happens to run last
 }
 
 # 4. Deploy the commit. On success, remember it as the last good one.

infra/server/test-relay-deploy.sh

@@ -35,7 +35,8 @@ case "\$*" in
   get*) echo "fake kubectl \$*" ;;
 esac
 EOF
-  printf '#!/usr/bin/env bash\nexit 0\n' > "$T/bin/flock"
+  # flock: gets the lock, unless lock-busy exists (another deploy holds it the whole time).
+  printf '#!/usr/bin/env bash\n[ -e "%s/lock-busy" ] && exit 1\nexit 0\n' "$T" > "$T/bin/flock"
   chmod +x "$T/bin/"*
   : > "$T/broken"
 }
@@ -98,5 +99,11 @@ setup; echo "not a commit; touch /tmp/pwned" > "$T/home/.relay-last-good"; echo
 check "exits 1" "$code" 1
 check "never checks out what's in the note" "$(cat "$T/checked-out")" $BAD
 
+echo "10. Another deploy still holds the lock"
+setup; touch "$T/lock-busy"; code=$(run $GOOD)
+check "exits 1" "$code" 1
+check "says to re-run" "$(grep -q 're-run this job' "$T/out" && echo yes || echo no)" yes
+check "changes nothing" "$(exists "$T/checked-out")" no
+
 echo; echo "$passed passed, $failed failed"
 [ "$failed" -eq 0 ]

k8s/README.md

@@ -22,7 +22,13 @@ How each folder gets onto the cluster, and who may apply it.
   by hand: the tag would be wrong.
 - **Removing a file from `k8s/app/` doesn't remove it from the cluster.** The `deployer`
   account has no delete, so one of us deletes the resource by hand:
-  `kubectl -n relay delete <kind>/<name>`.
+  `kubectl -n relay delete <kind>/<name>`. **Renaming is the dangerous case:** the new name is
+  created, but the old object stays and keeps serving (two Services or two Ingresses for the
+  same thing). A rename = the new name in the PR, plus deleting the old object right after it
+  merges.
+- **Server and platform changes are applied by the one who merges, right after merging**, and
+  the check output is posted on the PR (for example `auth can-i` and the script `diff`).
+  Otherwise git and the server drift apart, and it only shows when something fails.
 - **No Secrets in git.** They are created on the server with `kubectl create secret`.
 - **Git and the cluster must match.** Everything applied by hand is committed in the same PR.