From 3b9f691fbbaeec0c67b4e06ef27be67e5df6caed Mon Sep 17 00:00:00 2001 From: Babissimo Date: Mon, 14 Sep 2026 17:13:28 +0100 Subject: [PATCH] Hold staging for the whole deploy-and-verify sequence, not just the deploy `deploy-staging` was the only job in the `staging-deploy` concurrency group. `staging-smoke-tests` and `e2e-staging` carried no concurrency at all, so the deploy was serialised while everything that verifies the deploy was not: a following run's deploy was free to land between one run's smoke tests and its E2E, restarting the box underneath a suite that was testing it. Three consecutive merges failed that way on 2026-09-10 (#348, #351, #352), and four runs within three minutes produced the same thing on 2026-08-21 (#223). None of them failed for a fault in the change. The failure does not present as a flake either: a test hitting a half-swapped application gets a specific, plausible assertion failure, which reads as a content regression and costs someone real time before they conclude otherwise. It also blocks releases, as a red staging E2E skips `deploy-production`. Adding the group to the two verification jobs does not close it. Concurrency is acquired and released per job, so a deploy can still slot in between one run's smoke tests and its E2E. The environment has to stay held across the sequence. So the three jobs move to `staging-deploy-verify.yml` and ci.yml calls it from a single `Staging` job that carries the group. A calling job is not complete until every job in the called workflow has finished, so the group is held from the first byte of the deploy to the last E2E assertion. The step bodies move unchanged; only the job headers differ. Two things the move forced: - `env` does not cross a `workflow_call` boundary and `with:` cannot read the `env` context, so APP_DIR is re-exported as a `changes` output rather than restated in the new file, keeping one definition of the deploy directory. - `playwright-image` joins the staging job's prerequisites. An unpublished image now fails the run before the droplet is touched rather than after a deploy and a smoke suite have landed on it. Two comments that named the old job IDs move with them: the DNS-tolerance notes in frontend/e2e/dashboard.spec.ts and deploy/staging-smoke-test.sh both explained themselves by their job being a direct `needs:` of deploy-production, which is now indirect through the calling job. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 292 ++++---------------- .github/workflows/staging-deploy-verify.yml | 277 +++++++++++++++++++ ONBOARDING.md | 12 + deploy/staging-smoke-test.sh | 10 +- docs/architecture.md | 6 +- frontend/e2e/dashboard.spec.ts | 6 +- 6 files changed, 357 insertions(+), 246 deletions(-) create mode 100644 .github/workflows/staging-deploy-verify.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3f8243cf..f947db1e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -291,61 +291,6 @@ jobs: fi echo "Resolved and verified ${IMAGE}" - e2e-staging: - name: Playwright E2E (staging) - runs-on: ubuntu-latest - # Only the pull and npm ci sit outside the test step's own cap now, and an - # image pull is minutes at worst. See the production job for why the job - # cap is a backstop rather than the primary bound. - timeout-minutes: 30 - needs: [staging-smoke-tests, playwright-image] - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - # The browsers and their shared libraries ship in the image, so there is no - # install step and neither apt nor Playwright's CDN is in this path. Both - # broke repeatedly on 2026-08-18 and 2026-08-19; see the production job. - container: - image: ${{ needs.playwright-image.outputs.image }} - # `npm ci` runs @playwright/test's postinstall, which fetches browsers. The - # image sets PLAYWRIGHT_BROWSERS_PATH=/ms-playwright and the tag is derived - # to match the lockfile, so it finds the right revision already there and - # does nothing. This makes that a guarantee rather than an expectation: if - # the browsers were ever NOT found, the postinstall would quietly download - # them from the CDN this job exists to avoid, and the only symptom would be - # a slow green run. With this set it fails loudly at the browser instead, - # pointing at the cause. See microsoft/playwright#4283. - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - # Kept despite the image shipping its own Node: this pins the version the - # tests run under to the same 20 as every other job, rather than letting - # it drift with whatever the Playwright image happens to carry, and keeps - # the npm cache. Only browser provisioning changes here. - - uses: actions/setup-node@v4 - with: - node-version: "20" - cache: npm - cache-dependency-path: frontend/package-lock.json - - run: npm ci --legacy-peer-deps - - name: Run E2E tests (staging) - # Has to stay above what nodes.spec.ts's two retried beforeAll hooks can - # reach, so a node that never propagates fails an assertion rather than - # timing the step out and losing the report. That spec derives its hook - # budgets from the server's refresh cadence; raise this if it widens. - timeout-minutes: 20 - run: E2E_ENV=staging npx playwright test - env: - CI: "true" - RADAR_API_KEY: ${{ secrets.STAGING_RADAR_API_KEY }} - # Passage through Access for the admin surface. Playwright follows the - # edge's 302, so without these the admin tests would parse Cloudflare's - # login page as the app. Empty secrets render empty and send nothing. - CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} - CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - e2e-prod: name: Playwright E2E (production) runs-on: ubuntu-latest @@ -512,10 +457,10 @@ jobs: # Does this push touch anything the droplets serve? A docs-only merge would # otherwise redeploy staging AND production and run both Playwright suites for # files no vhost serves, with the prod deploy queued behind the whole staging - # chain to do it. Gating deploy-staging alone is enough: a skipped - # dependency skips its dependents whatever their own `if:` says, so - # staging-smoke-tests, e2e-staging, deploy-production and everything past it - # fall out with it. The rollback jobs are keyed to `result == 'failure'`, and a + # chain to do it. Gating the `staging` job alone is enough: a skipped + # dependency skips its dependents whatever their own `if:` says, so the whole + # called workflow, deploy-production and everything past it fall out with + # it. The rollback jobs are keyed to `result == 'failure'`, and a # skipped deploy is not a failed one, so they stay dormant. # # Only the deploy chain is gated. The test matrix above still runs on a @@ -534,7 +479,25 @@ jobs: if: github.ref == 'refs/heads/main' && github.event_name == 'push' outputs: code: ${{ steps.filter.outputs.code }} + app_dir: ${{ steps.deploy_dir.outputs.app_dir }} steps: + # Workflow-level `env` does not reach a called workflow, and `with:` on a + # calling job cannot read the `env` context either. Re-exported through an + # output so APP_DIR above stays the one definition. + - name: Re-export the deploy directory + id: deploy_dir + run: | + set -euo pipefail + # Refuse to hand the staging workflow an empty directory. `cd ""` + # succeeds into $HOME, and the deploy's own pre-flights all pass there + # (right hostname, no .env, disk fine) before `git fetch` fails in a + # directory nothing is served from. Caught here, where the cause is + # legible, rather than as a confusing abort on the droplet. + if [ -z "${APP_DIR:-}" ]; then + echo "::error::APP_DIR is empty. It is set at the top of this workflow and the staging deploy cds into it; nothing below can run without it." + exit 1 + fi + echo "app_dir=${APP_DIR}" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v4 with: # The push's whole range has to be diffable and `before` can be @@ -593,11 +556,18 @@ jobs: echo "code=false" >> "$GITHUB_OUTPUT" fi - deploy-staging: - name: Deploy to staging - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [changes, backend-tests, frontend-build, dashboard-build, docker-build, env-parity, tower-service-contract] + # Staging's deploy and both of its verification suites, called as one job. + # A calling job is not complete until every job in the called workflow has + # finished, so the concurrency group below is held across the whole sequence. + # Three grouped jobs here would not do it: the group is acquired and released + # per job, which let a following run's deploy land between one run's smoke + # tests and its E2E. See the called workflow. + staging: + name: Staging + # `playwright-image` joins the deploy's own prerequisites so an unpublished + # image fails the run before the droplet is touched, rather than after a + # deploy and a smoke suite have already landed on it. + needs: [changes, backend-tests, frontend-build, dashboard-build, docker-build, env-parity, tower-service-contract, playwright-image] if: github.ref == 'refs/heads/main' && github.event_name == 'push' && needs.changes.outputs.code == 'true' concurrency: group: staging-deploy @@ -605,172 +575,23 @@ jobs: # `git reset --hard` but before `docker compose up -d --build`, leaving the # box on fresh code with stale running containers (config/image changes # silently never take effect). Matches deploy-production. + # + # At most one run sits pending per group, so a third merge arriving while + # one run holds staging and another is queued cancels the queued one. That + # is the right outcome — main is linear, so the newer run deploys a + # superset of what the cancelled one would have — but it makes a cancelled + # `Staging` job an expected state during a burst rather than a fault. cancel-in-progress: false - steps: - - name: Deploy to staging via SSH - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.STAGING_HOST }} - username: root - key: ${{ secrets.STAGING_SSH_KEY }} - command_timeout: 20m - envs: APP_DIR - script: | - set -e - cd ${{ env.APP_DIR }} - - # Same two wrong-environment checks production runs, first and in the - # same order, for the same reasons. The hostname is the only signal - # that holds on a box with no .env at all, which is exactly the state - # a droplet awaiting its first deploy is in; see the production block - # for why it is an exact match rather than a pattern. - if [ "$(hostname)" != "retina-staging" ]; then - echo "::error::Refusing to deploy staging onto '$(hostname)' — expected retina-staging. If this droplet was rebuilt, finish provisioning it (hostnamectl set-hostname retina-staging); otherwise this job is pointed at the wrong box." - exit 1 - fi - - # A .env naming another environment means this job is pointed - # at a box it does not own. Without this the write further down simply - # takes that box over — overwriting its .env and running `down` then - # `up` would bring production back on staging's hostnames, fleet size - # and container names. A MISSING .env is allowed through; that is the - # normal state of a box awaiting its first deploy, and we write it - # below. - if [ -f .env ] && ! grep -q '^COMPOSE_FILE=.*docker-compose\.staging\.yml' .env; then - echo "::error::${{ env.APP_DIR }}/.env exists but does not select the staging overlay — this host is not staging. Inspect it; do not deploy." - exit 1 - fi - - # Pre-flight: Cloudflare's origin-pull CA, checked BEFORE anything - # stops or rebuilds the stack. nginx renders `ssl_verify_client on` - # (deploy/nginx/snippets/tls.conf) and start.sh refuses to boot - # without the CA, so on a droplet where the human half of this work - # is not done — enable Authenticated Origin Pulls for the zone, put - # the CA on the box — a deploy would run `docker compose down`, - # rebuild, fail to start, and leave the environment DOWN. Refusing - # here converts that outage into a refused deploy: everything above - # this line only reads the box. - CF_CA=/etc/ssl/cloudflare/origin-pull-ca.pem - if [ ! -f "$CF_CA" ]; then - echo "::error::${CF_CA} is missing on $(hostname). nginx requires Cloudflare's origin-pull CA and will not start without it. Enable Authenticated Origin Pulls for the zone, then fetch the CA onto this droplet (curl -fsS --create-dirs -o ${CF_CA} https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem) and re-run. Refusing to deploy; the running stack is untouched." - exit 1 - fi - if ! openssl x509 -in "$CF_CA" -noout >/dev/null 2>&1; then - echo "::error::${CF_CA} exists on $(hostname) but is not a parseable PEM certificate — a truncated download or an error page saved to the path would look exactly like this. nginx would fail to start. Replace it and re-run. Refusing to deploy; the running stack is untouched." - exit 1 - fi - echo "Origin-pull CA present and parseable: ${CF_CA}" - - # Pre-flight: refuse to deploy when the disk is near full. Without - # this, git fetch silently writes a partial pack, the source tree - # never advances, and `docker compose --build` happily caches every - # layer and "succeeds" with yesterday's image (run 25923117075). - AVAIL_MB=$(df -m / | awk 'NR==2 {print $4}') - if [ "${AVAIL_MB:-0}" -lt 2048 ]; then - echo "::error::Staging disk only ${AVAIL_MB}MB free (< 2GB threshold). Aborting deploy." - df -h / - docker system df 2>/dev/null || true - exit 1 - fi - - # Hard-reset rather than `git pull` — see the production deploy - # block for the full incident description; same failure mode would - # surface here if the staging server ever picked up a local commit. - # `set -e` above guarantees a failed fetch/reset aborts the deploy. - git fetch origin main - git reset --hard origin/main - git submodule update --init --recursive - # The host's ./.env sets COMPOSE_FILE (base + staging overlay), so - # every command below is character-identical to the production deploy - # block. Written from the tree we just checked out rather than assumed - # to be present: this job knows perfectly well which environment it is - # deploying, so making that depend on someone having copied the right - # file by hand only creates a way to get it wrong. A rebuilt droplet, - # a restored snapshot or a hand-mangled file now self-heals here. - cp deploy/env.staging.example .env - # Top up with this host's own CARTO basemap key — see the note in - # deploy/env..example. Guarded rather than chained with &&: a host - # with no key must build watermarked tiles, not fail its deploy. - if [ -f /root/.secrets/carto.env ]; then cat /root/.secrets/carto.env >> .env; fi - docker compose down --timeout 30 - # Cap build cache at ~10GB (LRU eviction) rather than wiping it all: - # keeps recent layers so incremental builds stay fast, and bounds - # total size so cache growth can't trip the disk pre-flight above. - # The stale-layer failure the old blanket wipe guarded against was - # rooted in disk-full (partial git pack -> source never advanced -> - # cache reused); the disk pre-flight now catches that root cause, so a - # full wipe is redundant and only forced slow from-scratch rebuilds. - docker builder prune -f --keep-storage 10GB - # Deploy the app FIRST, on its own — building only `server` - # means a broken Dockerfile.fleet can't abort the staging app deploy - # (which would block the whole pipeline). Fleet is a separate step below. - # --remove-orphans because this deploy renamed the compose - # service. container_name is pinned, so the container kept by the - # old service key is an orphan holding the exact name the new one - # needs, and compose leaves orphans alone by default: the create - # then fails with a name conflict and the stack does not come up. - # Verified on retina-test before this landed. It stays afterwards - # as the general guard for a removed or renamed service. - docker compose up -d --build --remove-orphans server - # Wait for server to become healthy (up to 90s) - for i in $(seq 1 18); do - if docker compose exec -T server \ - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health')" 2>/dev/null; then - echo "Staging server healthy after ~$((i*5))s" - break - fi - echo "Waiting for health check... attempt $i/18" - sleep 5 - if [ "$i" = "18" ]; then - echo "Staging health check failed after 90s" - docker compose logs --tail=50 - exit 1 - fi - done - # Now the fleet, separately — a fleet build/start failure surfaces - # loudly here without taking the already-healthy app down with it. - docker compose up -d --build --no-deps fleet - - staging-smoke-tests: - name: Staging smoke tests - runs-on: ubuntu-latest - timeout-minutes: 35 - needs: [deploy-staging] - steps: - - uses: actions/checkout@v4 - # The curls in this loop carry no --max-time, so against a hung origin - # the 24 iterations run far past the job cap. Bounded here rather than by - # rewriting the loop, which deploy/staging-smoke-test.sh mirrors. - - name: Wait for fleet to connect - timeout-minutes: 10 - run: | - echo "Waiting for fleet nodes to connect..." - # Through the edge, not at the droplet's address: the origin now requires - # Cloudflare's client certificate (ssl_verify_client in - # deploy/nginx/snippets/tls.conf) and the DOCKER-USER boundary drops - # anything on 443 that is not from a Cloudflare range, so a direct-to-IP - # request can only ever time out or 400. `-k` does not help — that flag - # governs verification of the SERVER's certificate, not presentation of - # a client one. Same hostname deploy/staging-smoke-test.sh uses for this - # endpoint (BASE_URL there), so both agree on what "staging" means. - for i in $(seq 1 24); do - COUNT=$(curl -s https://staging-towers.retina.fm/api/test/dashboard \ - | python3 -c "import sys,json; print(json.load(sys.stdin)['nodes']['active'])" 2>/dev/null) || COUNT=0 - if [ "${COUNT:-0}" -ge 1 ]; then - echo "Fleet ready: ${COUNT} node(s) connected after ~$((i*5))s" - break - fi - echo "Attempt $i/24: ${COUNT} nodes — waiting 5s..." - sleep 5 - done - - name: Run smoke tests against staging - timeout-minutes: 15 - # Same service token as production: one token, admitted by a Service Auth - # policy on each admin application. Inert while the secrets are unset. - env: - CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} - CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - run: bash deploy/staging-smoke-test.sh + uses: ./.github/workflows/staging-deploy-verify.yml + with: + app_dir: ${{ needs.changes.outputs.app_dir }} + playwright_image: ${{ needs.playwright-image.outputs.image }} + secrets: + STAGING_HOST: ${{ secrets.STAGING_HOST }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + STAGING_RADAR_API_KEY: ${{ secrets.STAGING_RADAR_API_KEY }} + CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} + CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} deploy-production: name: Deploy to production @@ -778,12 +599,11 @@ jobs: # Above the ssh-action's own command_timeout below, so a stuck deploy # surfaces as that step's error rather than as a bare job timeout. timeout-minutes: 30 - # Gate prod on BOTH the staging smoke tests and the full staging Playwright - # E2E suite (`e2e-staging`). Previously only staging-smoke-tests gated this, - # so the ~127-test E2E ran in parallel with the prod deploy and couldn't - # stop a bad build reaching prod. This serialises the pipeline (E2E is now - # on the critical path before prod) in exchange for real pre-prod coverage. - needs: [staging-smoke-tests, e2e-staging] + # Gate prod on the whole staging sequence: its deploy, its smoke tests and + # its ~127-test Playwright suite. The E2E used to run in parallel with the + # prod deploy and so could not stop a bad build reaching prod. This + # serialises the pipeline in exchange for real pre-prod coverage. + needs: [staging] if: github.ref == 'refs/heads/main' && github.event_name == 'push' concurrency: group: production-deploy @@ -1057,7 +877,7 @@ jobs: cancel-in-progress: false # `always() && ...result == 'failure'` rather than a bare `failure()`. # failure() is true when ANY ancestor job failed, and deploy-production has - # ancestors: an e2e-staging failure skips the deploy entirely, and this job + # ancestors: a staging failure skips the deploy entirely, and this job # would then SSH into production to consider rolling back a deploy that # never ran. Keyed to the deploy job's own result, it cannot. # diff --git a/.github/workflows/staging-deploy-verify.yml b/.github/workflows/staging-deploy-verify.yml new file mode 100644 index 000000000..262cc8e79 --- /dev/null +++ b/.github/workflows/staging-deploy-verify.yml @@ -0,0 +1,277 @@ +name: Staging deploy and verify + +# Deploy staging, then prove it, as one unit. ci.yml calls this from a single +# job that carries the `staging-deploy` concurrency group, and a calling job is +# not complete until every job below has finished, so the group is held from +# the first byte of the deploy to the last E2E assertion. +# +# That holding is the whole point of the file, and it is why the three jobs +# cannot simply live in ci.yml with a group each. Concurrency is acquired and +# released per job, so a following run's deploy was free to land between one +# run's smoke tests and its E2E — which is how three merges on 2026-09-10 all +# failed, none of them for a fault in the change: the box was restarting +# underneath a suite that was testing it. +# +# Splitting these back into three jobs in the caller reopens that race, +# whatever concurrency group each of them carries. + +on: + workflow_call: + inputs: + app_dir: + description: >- + Deploy directory on the droplet. Threaded in from ci.yml's + `env.APP_DIR` rather than restated here: workflow-level `env` does not + cross a `workflow_call` boundary, and a second definition is a second + thing to forget when the directory moves. + required: true + type: string + playwright_image: + description: >- + Playwright container image for the E2E job, derived from the lockfile + by ci.yml's `playwright-image` job so the browsers cannot disagree + with the client that drives them. + required: true + type: string + secrets: + STAGING_HOST: + required: true + STAGING_SSH_KEY: + required: true + STAGING_RADAR_API_KEY: + required: true + # Inert while unset: the smoke script and Playwright both render them + # empty and send no Access headers, which is the state of an environment + # with no Service Auth policy in front of its admin surface. + CF_ACCESS_CLIENT_ID: + required: false + CF_ACCESS_CLIENT_SECRET: + required: false + +jobs: + deploy: + name: Deploy to staging + runs-on: ubuntu-latest + timeout-minutes: 30 + # Carries the deploy directory as an environment variable rather than only + # interpolating it: `envs: APP_DIR` below forwards it to the droplet, where + # deploy/pre-deploy.sh and deploy/rollback.sh read it. + env: + APP_DIR: ${{ inputs.app_dir }} + steps: + - name: Deploy to staging via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.STAGING_HOST }} + username: root + key: ${{ secrets.STAGING_SSH_KEY }} + command_timeout: 20m + envs: APP_DIR + script: | + set -e + cd ${{ env.APP_DIR }} + + # Same two wrong-environment checks production runs, first and in the + # same order, for the same reasons. The hostname is the only signal + # that holds on a box with no .env at all, which is exactly the state + # a droplet awaiting its first deploy is in; see the production block + # for why it is an exact match rather than a pattern. + if [ "$(hostname)" != "retina-staging" ]; then + echo "::error::Refusing to deploy staging onto '$(hostname)' — expected retina-staging. If this droplet was rebuilt, finish provisioning it (hostnamectl set-hostname retina-staging); otherwise this job is pointed at the wrong box." + exit 1 + fi + + # A .env naming another environment means this job is pointed + # at a box it does not own. Without this the write further down simply + # takes that box over — overwriting its .env and running `down` then + # `up` would bring production back on staging's hostnames, fleet size + # and container names. A MISSING .env is allowed through; that is the + # normal state of a box awaiting its first deploy, and we write it + # below. + if [ -f .env ] && ! grep -q '^COMPOSE_FILE=.*docker-compose\.staging\.yml' .env; then + echo "::error::${{ env.APP_DIR }}/.env exists but does not select the staging overlay — this host is not staging. Inspect it; do not deploy." + exit 1 + fi + + # Pre-flight: Cloudflare's origin-pull CA, checked BEFORE anything + # stops or rebuilds the stack. nginx renders `ssl_verify_client on` + # (deploy/nginx/snippets/tls.conf) and start.sh refuses to boot + # without the CA, so on a droplet where the human half of this work + # is not done — enable Authenticated Origin Pulls for the zone, put + # the CA on the box — a deploy would run `docker compose down`, + # rebuild, fail to start, and leave the environment DOWN. Refusing + # here converts that outage into a refused deploy: everything above + # this line only reads the box. + CF_CA=/etc/ssl/cloudflare/origin-pull-ca.pem + if [ ! -f "$CF_CA" ]; then + echo "::error::${CF_CA} is missing on $(hostname). nginx requires Cloudflare's origin-pull CA and will not start without it. Enable Authenticated Origin Pulls for the zone, then fetch the CA onto this droplet (curl -fsS --create-dirs -o ${CF_CA} https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem) and re-run. Refusing to deploy; the running stack is untouched." + exit 1 + fi + if ! openssl x509 -in "$CF_CA" -noout >/dev/null 2>&1; then + echo "::error::${CF_CA} exists on $(hostname) but is not a parseable PEM certificate — a truncated download or an error page saved to the path would look exactly like this. nginx would fail to start. Replace it and re-run. Refusing to deploy; the running stack is untouched." + exit 1 + fi + echo "Origin-pull CA present and parseable: ${CF_CA}" + + # Pre-flight: refuse to deploy when the disk is near full. Without + # this, git fetch silently writes a partial pack, the source tree + # never advances, and `docker compose --build` happily caches every + # layer and "succeeds" with yesterday's image (run 25923117075). + AVAIL_MB=$(df -m / | awk 'NR==2 {print $4}') + if [ "${AVAIL_MB:-0}" -lt 2048 ]; then + echo "::error::Staging disk only ${AVAIL_MB}MB free (< 2GB threshold). Aborting deploy." + df -h / + docker system df 2>/dev/null || true + exit 1 + fi + + # Hard-reset rather than `git pull` — see the production deploy + # block for the full incident description; same failure mode would + # surface here if the staging server ever picked up a local commit. + # `set -e` above guarantees a failed fetch/reset aborts the deploy. + git fetch origin main + git reset --hard origin/main + git submodule update --init --recursive + # The host's ./.env sets COMPOSE_FILE (base + staging overlay), so + # every command below is character-identical to the production deploy + # block. Written from the tree we just checked out rather than assumed + # to be present: this job knows perfectly well which environment it is + # deploying, so making that depend on someone having copied the right + # file by hand only creates a way to get it wrong. A rebuilt droplet, + # a restored snapshot or a hand-mangled file now self-heals here. + cp deploy/env.staging.example .env + # Top up with this host's own CARTO basemap key — see the note in + # deploy/env..example. Guarded rather than chained with &&: a host + # with no key must build watermarked tiles, not fail its deploy. + if [ -f /root/.secrets/carto.env ]; then cat /root/.secrets/carto.env >> .env; fi + docker compose down --timeout 30 + # Cap build cache at ~10GB (LRU eviction) rather than wiping it all: + # keeps recent layers so incremental builds stay fast, and bounds + # total size so cache growth can't trip the disk pre-flight above. + # The stale-layer failure the old blanket wipe guarded against was + # rooted in disk-full (partial git pack -> source never advanced -> + # cache reused); the disk pre-flight now catches that root cause, so a + # full wipe is redundant and only forced slow from-scratch rebuilds. + docker builder prune -f --keep-storage 10GB + # Deploy the app FIRST, on its own — building only `server` + # means a broken Dockerfile.fleet can't abort the staging app deploy + # (which would block the whole pipeline). Fleet is a separate step below. + # --remove-orphans because this deploy renamed the compose + # service. container_name is pinned, so the container kept by the + # old service key is an orphan holding the exact name the new one + # needs, and compose leaves orphans alone by default: the create + # then fails with a name conflict and the stack does not come up. + # Verified on retina-test before this landed. It stays afterwards + # as the general guard for a removed or renamed service. + docker compose up -d --build --remove-orphans server + # Wait for server to become healthy (up to 90s) + for i in $(seq 1 18); do + if docker compose exec -T server \ + python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health')" 2>/dev/null; then + echo "Staging server healthy after ~$((i*5))s" + break + fi + echo "Waiting for health check... attempt $i/18" + sleep 5 + if [ "$i" = "18" ]; then + echo "Staging health check failed after 90s" + docker compose logs --tail=50 + exit 1 + fi + done + # Now the fleet, separately — a fleet build/start failure surfaces + # loudly here without taking the already-healthy app down with it. + docker compose up -d --build --no-deps fleet + smoke-tests: + name: Staging smoke tests + runs-on: ubuntu-latest + timeout-minutes: 35 + needs: [deploy] + steps: + - uses: actions/checkout@v4 + # The curls in this loop carry no --max-time, so against a hung origin + # the 24 iterations run far past the job cap. Bounded here rather than by + # rewriting the loop, which deploy/staging-smoke-test.sh mirrors. + - name: Wait for fleet to connect + timeout-minutes: 10 + run: | + echo "Waiting for fleet nodes to connect..." + # Through the edge, not at the droplet's address: the origin now requires + # Cloudflare's client certificate (ssl_verify_client in + # deploy/nginx/snippets/tls.conf) and the DOCKER-USER boundary drops + # anything on 443 that is not from a Cloudflare range, so a direct-to-IP + # request can only ever time out or 400. `-k` does not help — that flag + # governs verification of the SERVER's certificate, not presentation of + # a client one. Same hostname deploy/staging-smoke-test.sh uses for this + # endpoint (BASE_URL there), so both agree on what "staging" means. + for i in $(seq 1 24); do + COUNT=$(curl -s https://staging-towers.retina.fm/api/test/dashboard \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['nodes']['active'])" 2>/dev/null) || COUNT=0 + if [ "${COUNT:-0}" -ge 1 ]; then + echo "Fleet ready: ${COUNT} node(s) connected after ~$((i*5))s" + break + fi + echo "Attempt $i/24: ${COUNT} nodes — waiting 5s..." + sleep 5 + done + - name: Run smoke tests against staging + timeout-minutes: 15 + # Same service token as production: one token, admitted by a Service Auth + # policy on each admin application. Inert while the secrets are unset. + env: + CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} + CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} + run: bash deploy/staging-smoke-test.sh + e2e: + name: Playwright E2E (staging) + runs-on: ubuntu-latest + # Only the pull and npm ci sit outside the test step's own cap now, and an + # image pull is minutes at worst. See the production job for why the job + # cap is a backstop rather than the primary bound. + timeout-minutes: 30 + needs: [smoke-tests] + # The browsers and their shared libraries ship in the image, so there is no + # install step and neither apt nor Playwright's CDN is in this path. Both + # broke repeatedly on 2026-08-18 and 2026-08-19; see the production job. + container: + image: ${{ inputs.playwright_image }} + # `npm ci` runs @playwright/test's postinstall, which fetches browsers. The + # image sets PLAYWRIGHT_BROWSERS_PATH=/ms-playwright and the tag is derived + # to match the lockfile, so it finds the right revision already there and + # does nothing. This makes that a guarantee rather than an expectation: if + # the browsers were ever NOT found, the postinstall would quietly download + # them from the CDN this job exists to avoid, and the only symptom would be + # a slow green run. With this set it fails loudly at the browser instead, + # pointing at the cause. See microsoft/playwright#4283. + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + # Kept despite the image shipping its own Node: this pins the version the + # tests run under to the same 20 as every other job, rather than letting + # it drift with whatever the Playwright image happens to carry, and keeps + # the npm cache. Only browser provisioning changes here. + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci --legacy-peer-deps + - name: Run E2E tests (staging) + # Has to stay above what nodes.spec.ts's two retried beforeAll hooks can + # reach, so a node that never propagates fails an assertion rather than + # timing the step out and losing the report. That spec derives its hook + # budgets from the server's refresh cadence; raise this if it widens. + timeout-minutes: 20 + run: E2E_ENV=staging npx playwright test + env: + CI: "true" + RADAR_API_KEY: ${{ secrets.STAGING_RADAR_API_KEY }} + # Passage through Access for the admin surface. Playwright follows the + # edge's 302, so without these the admin tests would parse Cloudflare's + # login page as the app. Empty secrets render empty and send nothing. + CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} + CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} diff --git a/ONBOARDING.md b/ONBOARDING.md index 38db18912..23cf8990a 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -233,12 +233,24 @@ CI runs on every PR, on push to `main`, and on demand through `dashboard-build`, `docker-build`, `env-parity`, plus an automated review. 2. Merge to `main` → deploy to **staging** → staging smoke + Playwright E2E → deploy to **production** → prod smoke + Playwright E2E. A markdown-only merge skips that chain; the `changes` job has the exceptions. + The staging third of it is a called workflow, + `.github/workflows/staging-deploy-verify.yml`, invoked from one `Staging` + job so that job's concurrency group is held across the deploy and both + suites. Adding a staging step means editing that file, not `ci.yml`. So merging to `main` deploys to production automatically. Work on a feature branch, open a PR, get it green, then merge. ## Things that will bite you +- **A cancelled `Staging` job on a burst of merges is expected, not a fault.** + Only one run may sit pending on the `staging-deploy` group, so when a third + merge arrives while one run holds staging and another is queued, the queued + one is cancelled. `main` is linear, so the run that replaces it deploys a + superset of what was dropped. What it does mean is that the cancelled + commit's own run never reaches production: the following run carries it. + The last merge in a burst has no successor, so check it landed. + - **The CARTO basemap key lives on the droplet, not in this repo.** Every deploy appends `/root/.secrets/carto.env` to `./.env` after copying `deploy/env..example`, and `docker-compose.yml` interpolates it into the diff --git a/deploy/staging-smoke-test.sh b/deploy/staging-smoke-test.sh index c67ea7260..3f25b9a6e 100644 --- a/deploy/staging-smoke-test.sh +++ b/deploy/staging-smoke-test.sh @@ -103,11 +103,11 @@ check_contract() { NO_DNS_EXPECTED="" # Vhosts whose record exists and is expected to, but whose absence must not -# fail the run: staging-smoke-tests is a `needs:` of deploy-production -# (ci.yml), so a hard failure here would let a Cloudflare wobble block every -# release. Reported as WARN and tallied separately, because a deleted record -# must still be visible: skipping it silently would retire the only check on a -# vhost nothing else monitors. +# fail the run: this runs inside the `staging` job that deploy-production needs +# (ci.yml calls staging-deploy-verify.yml), so a hard failure here would let a +# Cloudflare wobble block every release. Reported as WARN and tallied +# separately, because a deleted record must still be visible: skipping it +# silently would retire the only check on a vhost nothing else monitors. DNS_NOT_DEPLOY_BLOCKING="staging-admin.retina.fm staging-data.retina.fm" # Decides what to do about $1 not resolving, prints it, and returns 0 when the diff --git a/docs/architecture.md b/docs/architecture.md index a97d67413..1dcb029ca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,6 +144,8 @@ data to a user's own nodes. ## Deploy `.github/workflows/ci.yml`: push to `main` → build/test → deploy staging → -staging smoke + E2E → deploy production → prod smoke + E2E. Deploy is an SSH -`git reset --hard origin/main` + `docker compose up -d --build`, gated by a +staging smoke + E2E → deploy production → prod smoke + E2E. The three staging +steps live in `staging-deploy-verify.yml` and are called as a single job, so one +run holds the environment until its own verification has finished. Deploy is an +SSH `git reset --hard origin/main` + `docker compose up -d --build`, gated by a free-disk pre-flight. Operational detail is in [`runbook.md`](runbook.md). diff --git a/frontend/e2e/dashboard.spec.ts b/frontend/e2e/dashboard.spec.ts index aa487b587..bf2ee3a9b 100644 --- a/frontend/e2e/dashboard.spec.ts +++ b/frontend/e2e/dashboard.spec.ts @@ -104,9 +104,9 @@ test.describe("Dashboard — unauthenticated access (real auth mode)", () => { /** * Whether a hostname resolves at all, as distinct from what it answers. * - * e2e-staging is a `needs:` of deploy-production exactly as staging-smoke-tests - * is, so this suite can block a release for the same reason that one can. That - * one reports an unresolvable name as a warning rather than a failure, because + * This suite runs inside the `staging` job that deploy-production needs, exactly + * as the smoke tests do, so it can block a release for the same reason they can. + * They report an unresolvable name as a warning rather than a failure, because * staging-admin.retina.fm's record is young and nothing monitors it; without * the same tolerance here a DNS wobble still holds up every deploy, through the * sibling gate.