From eb82e649b74b221893f4fdcb950bffdba03c1a42 Mon Sep 17 00:00:00 2001 From: Devinder Singh Date: Thu, 29 Oct 2026 12:20:26 +0100 Subject: [PATCH 1/5] infra: deploy every merge to main and check that it's live --- .github/workflows/api.yml | 67 ------------------ .github/workflows/pipeline.yml | 121 +++++++++++++++++++++++++++++++++ k8s/hello/whoami.yaml | 61 ----------------- 3 files changed, 121 insertions(+), 128 deletions(-) delete mode 100644 .github/workflows/api.yml create mode 100644 .github/workflows/pipeline.yml delete mode 100644 k8s/hello/whoami.yaml diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml deleted file mode 100644 index ca48924..0000000 --- a/.github/workflows/api.yml +++ /dev/null @@ -1,67 +0,0 @@ -# The API pipeline. -# Every pull request: test the API and check that its image builds (nothing is pushed). -# Every merge to main that touches the API: test again, then build and push the image -# to GHCR, tagged with the commit. Deployment comes later (step 1.8). -name: api - -on: - pull_request: - # No path filter here on purpose: "api-test" is a required check, and a required - # check that never runs (for example on a dashboard-only PR) would block the merge forever. - push: - branches: [main] - paths: ["api/**", ".github/workflows/api.yml"] - -# By default no job may write anything. Jobs ask for more only where they need it. -permissions: - contents: read - -# A newer push to the same PR cancels the older, now pointless run. Runs on main always finish. -concurrency: - group: api-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - test: - name: api-test - runs-on: ubuntu-24.04 - defaults: - run: - working-directory: api - steps: - # Actions are pinned to a full commit SHA, so a moved or hijacked tag can't run new code. - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - with: - bun-version-file: api/.bun-version - - run: bun install --frozen-lockfile - - run: bun run typecheck - - run: bun test - - image: - name: api-image - needs: test - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write # only this job may push to GHCR, and it only pushes on main - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - name: Log in to GHCR - if: github.event_name == 'push' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build (and on main, push) the image - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: api - platforms: linux/amd64 # the VPS is amd64; GitHub's runners build that natively - build-args: GIT_SHA=${{ github.sha }} - tags: ghcr.io/${{ github.repository_owner }}/relay-api:${{ github.sha }} - push: ${{ github.event_name == 'push' }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..e91152f --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,121 @@ +# The pipeline (steps 1.5 and 1.8). +# Every pull request: test the API and check that its image builds. Nothing is pushed or deployed. +# Every merge to main: test, build and push the image tagged with the commit, deploy that +# commit to the VPS, then check from outside that it's live. +# Every merge is a release, so main has no path filter: a deploy needs an image tagged with +# exactly the merged commit, even when only k8s/app/ or the docs changed. +name: pipeline + +on: + pull_request: + # No path filter here on purpose: "api-test" is a required check, and a required + # check that never runs (for example on a dashboard-only PR) would block the merge forever. + push: + branches: [main] + +# By default no job may write anything. Jobs ask for more only where they need it. +permissions: + contents: read + +# A newer push to the same PR cancels the older, now pointless run. +# On main, runs queue up one at a time and never cancel a running deploy. If several merges +# wait, only the newest runs: its commit already contains the others. +concurrency: + group: pipeline-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + test: + name: api-test + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: api + steps: + # Actions are pinned to a full commit SHA, so a moved or hijacked tag can't run new code. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: api/.bun-version + - run: bun install --frozen-lockfile + - run: bun run typecheck + - run: bun test + + image: + name: api-image + needs: test + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write # only this job may push to GHCR, and it only pushes on main + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Log in to GHCR + if: github.event_name == 'push' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build (and on main, push) the image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: api + platforms: linux/amd64 # the VPS is amd64; GitHub's runners build that natively + build-args: GIT_SHA=${{ github.sha }} + tags: ghcr.io/${{ github.repository_owner }}/relay-api:${{ github.sha }} + push: ${{ github.event_name == 'push' }} + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + name: deploy + needs: image + if: github.event_name == 'push' # only after a merge to main, never on a PR + runs-on: ubuntu-24.04 + # The environment hands over DEPLOY_SSH_KEY, and only to workflows running on main. + environment: + name: production + url: https://api.myrelay.tech + permissions: {} # needs no GitHub access at all: it only talks to our server + # Worst case: 5 min waiting for the server's lock + 3 min rollout + 3 min rollback. + timeout-minutes: 20 + steps: + - name: Deploy the commit over SSH + # Values arrive as environment variables, never pasted into the command itself. + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_HOST: ${{ vars.DEPLOY_HOST }} + DEPLOY_KNOWN_HOSTS: ${{ vars.DEPLOY_KNOWN_HOSTS }} + run: | + install -d -m 700 ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + # Our server's ID card: SSH refuses any server that doesn't match it. + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + # The server runs relay-deploy whatever we send, so we send only the commit. + # Its exit code becomes this step's: a failed or rolled-back deploy turns the job red. + ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o BatchMode=yes -o ConnectTimeout=20 \ + "deploy@$DEPLOY_HOST" "$GITHUB_SHA" + + - name: Check that the new version is live + # relay-deploy already waited for ready pods inside the cluster. This checks the whole + # public path a customer uses (DNS, certificate, Traefik, Service, pod) from outside. + # An old pod can still answer for a few seconds after the rollout, so we ask a few times. + shell: bash # GitHub then adds -e and pipefail, so a failed curl isn't hidden behind jq + run: | + for attempt in $(seq 1 12); do + # -f: an error answer (404, 503) counts as a failure. --max-time: never hang. + live=$(curl -fsS --max-time 10 https://api.myrelay.tech/version | jq -r .sha) \ + || live="no answer" + echo "attempt $attempt: live is $live" + if [ "$live" = "$GITHUB_SHA" ]; then + echo "$GITHUB_SHA is live" + exit 0 + fi + sleep 5 + done + echo "::error::api.myrelay.tech still doesn't run $GITHUB_SHA after 12 tries" + exit 1 diff --git a/k8s/hello/whoami.yaml b/k8s/hello/whoami.yaml deleted file mode 100644 index 8b6e134..0000000 --- a/k8s/hello/whoami.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Temporary test app for step 0.13: proves DNS -> firewalls -> Traefik -> HTTPS -> pod. -# traefik/whoami answers every request with details about the request and which pod served it. -# Replaced by the real API in Phase 2 (step 2.7). - -# Deployment: "keep 1 copy of the whoami container running, and restart it if it dies." -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hello -spec: - replicas: 1 - selector: - matchLabels: - app: hello # manage the pods that carry this label... - template: - metadata: - labels: - app: hello # ...and give every new pod this label - spec: - containers: - - name: whoami - image: traefik/whoami:v1.12.0 # pinned version, never "latest" - ports: - - containerPort: 80 ---- -# Service: a stable name ("hello") inside the cluster that sends traffic to the pods labelled app=hello. -apiVersion: v1 -kind: Service -metadata: - name: hello -spec: - selector: - app: hello - ports: - - port: 80 # the Service listens on 80... - targetPort: 80 # ...and forwards to port 80 in the pod ---- -# Ingress: expose the Service to the outside world. -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: hello - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: traefik - tls: - - hosts: - - api.myrelay.tech - secretName: api-myrelay-tech-tls - rules: - - host: api.myrelay.tech - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: hello - port: - number: 80 From 86a421346a048118842ff8cbf255795ba527874d Mon Sep 17 00:00:00 2001 From: Devinder Singh Date: Thu, 29 Oct 2026 13:18:22 +0100 Subject: [PATCH 2/5] infra: give the first deploy's new certificate time before the version check fails --- .github/workflows/pipeline.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index e91152f..0b37679 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -103,10 +103,11 @@ jobs: - name: Check that the new version is live # relay-deploy already waited for ready pods inside the cluster. This checks the whole # public path a customer uses (DNS, certificate, Traefik, Service, pod) from outside. - # An old pod can still answer for a few seconds after the rollout, so we ask a few times. + # An old pod can still answer for a few seconds after the rollout, and the first deploy + # also waits for a brand-new certificate, so we ask for up to 36 × 5 s ≈ 3 minutes. shell: bash # GitHub then adds -e and pipefail, so a failed curl isn't hidden behind jq run: | - for attempt in $(seq 1 12); do + for attempt in $(seq 1 36); do # -f: an error answer (404, 503) counts as a failure. --max-time: never hang. live=$(curl -fsS --max-time 10 https://api.myrelay.tech/version | jq -r .sha) \ || live="no answer" @@ -117,5 +118,5 @@ jobs: fi sleep 5 done - echo "::error::api.myrelay.tech still doesn't run $GITHUB_SHA after 12 tries" + echo "::error::api.myrelay.tech still doesn't run $GITHUB_SHA after 36 tries" exit 1 From b0929c21b3ad888889a6f54fc947edca9ff8f58f Mon Sep 17 00:00:00 2001 From: Devinder Singh Date: Thu, 29 Oct 2026 14:48:31 +0100 Subject: [PATCH 3/5] infra: stop hung pipeline jobs after 10 minutes --- .github/workflows/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 0b37679..9d25e87 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -28,6 +28,7 @@ jobs: test: name: api-test runs-on: ubuntu-24.04 + timeout-minutes: 10 # a hung step stops here instead of GitHub's 6-hour default defaults: run: working-directory: api @@ -45,6 +46,7 @@ jobs: name: api-image needs: test runs-on: ubuntu-24.04 + timeout-minutes: 10 permissions: contents: read packages: write # only this job may push to GHCR, and it only pushes on main From 67b29ddd309649cb66bdf060d0d8ab1287516622 Mon Sep 17 00:00:00 2001 From: Devinder Singh Date: Thu, 29 Oct 2026 15:23:24 +0100 Subject: [PATCH 4/5] infra: test the deploy script in CI, a 30-minute deploy budget, drop the hello row --- .github/workflows/pipeline.yml | 19 ++++++++++++++++--- k8s/README.md | 1 - 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 9d25e87..f7b425b 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -71,9 +71,22 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + deploy-script: + name: deploy-script-test + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # relay-deploy guards production, so its gates and rollback are tested on every PR, + # with fake git and kubectl (no server needed). shellcheck ships on GitHub's runner. + - name: Test relay-deploy's decisions + run: bash infra/server/test-relay-deploy.sh infra/server/relay-deploy.sh + - name: Lint the server scripts + run: shellcheck infra/server/*.sh + deploy: name: deploy - needs: image + needs: [image, deploy-script] # a broken deploy script never reaches the server if: github.event_name == 'push' # only after a merge to main, never on a PR runs-on: ubuntu-24.04 # The environment hands over DEPLOY_SSH_KEY, and only to workflows running on main. @@ -81,8 +94,8 @@ jobs: name: production url: https://api.myrelay.tech permissions: {} # needs no GitHub access at all: it only talks to our server - # Worst case: 5 min waiting for the server's lock + 3 min rollout + 3 min rollback. - timeout-minutes: 20 + # Worst case: 15 min waiting for the server's lock + 6 min rollout + 6 min rollback. + timeout-minutes: 30 steps: - name: Deploy the commit over SSH # Values arrive as environment variables, never pasted into the command itself. diff --git a/k8s/README.md b/k8s/README.md index 448fb30..84d298b 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -6,7 +6,6 @@ How each folder gets onto the cluster, and who may apply it. |---|---|---| | `k8s/platform/` | Cluster setup: Let's Encrypt issuers, Traefik settings, the `relay` namespace and the `deployer` account | One of us, by hand, after the PR is reviewed | | `k8s/app/` (from step 1.7) | Our app in namespace `relay`, the API and the dashboard: Deployments, Services, Ingresses, ConfigMaps | Only the pipeline, through `relay-deploy` | -| `k8s/hello/` | The temporary hello-world test app | By hand. Removed when the API takes over in step 1.7 | ## Rules From d00063daeb2ccbf6bc2c5d5af8b41267c3a6e184 Mon Sep 17 00:00:00 2001 From: Devinder Singh Date: Thu, 29 Oct 2026 15:52:01 +0100 Subject: [PATCH 5/5] infra: allow time for public version verification --- .github/workflows/pipeline.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index f7b425b..7eb5086 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -94,8 +94,10 @@ jobs: name: production url: https://api.myrelay.tech permissions: {} # needs no GitHub access at all: it only talks to our server - # Worst case: 15 min waiting for the server's lock + 6 min rollout + 6 min rollback. - timeout-minutes: 30 + # Budget for two Deployments: 15 min lock + 6 min rollout + 9 min public check = 30 min. + # Failure path: 15 min lock + 6 min rollout + 6 min rollback = 27 min; no public check. + # Allow 5 extra minutes above the longer path for SSH, git, kubectl and job overhead. + timeout-minutes: 35 steps: - name: Deploy the commit over SSH # Values arrive as environment variables, never pasted into the command itself. @@ -119,7 +121,8 @@ jobs: # relay-deploy already waited for ready pods inside the cluster. This checks the whole # public path a customer uses (DNS, certificate, Traefik, Service, pod) from outside. # An old pod can still answer for a few seconds after the rollout, and the first deploy - # also waits for a brand-new certificate, so we ask for up to 36 × 5 s ≈ 3 minutes. + # also waits for a brand-new certificate. Budget up to 36 × (10 s curl + 5 s sleep) + # = 9 minutes for retries, plus command overhead. shell: bash # GitHub then adds -e and pipefail, so a failed curl isn't hidden behind jq run: | for attempt in $(seq 1 36); do