diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3f8243cf..f947db1e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -291,61 +291,6 @@ jobs: fi echo "Resolved and verified ${IMAGE}" - e2e-staging: - name: Playwright E2E (staging) - runs-on: ubuntu-latest - # Only the pull and npm ci sit outside the test step's own cap now, and an - # image pull is minutes at worst. See the production job for why the job - # cap is a backstop rather than the primary bound. - timeout-minutes: 30 - needs: [staging-smoke-tests, playwright-image] - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - # The browsers and their shared libraries ship in the image, so there is no - # install step and neither apt nor Playwright's CDN is in this path. Both - # broke repeatedly on 2026-08-18 and 2026-08-19; see the production job. - container: - image: ${{ needs.playwright-image.outputs.image }} - # `npm ci` runs @playwright/test's postinstall, which fetches browsers. The - # image sets PLAYWRIGHT_BROWSERS_PATH=/ms-playwright and the tag is derived - # to match the lockfile, so it finds the right revision already there and - # does nothing. This makes that a guarantee rather than an expectation: if - # the browsers were ever NOT found, the postinstall would quietly download - # them from the CDN this job exists to avoid, and the only symptom would be - # a slow green run. With this set it fails loudly at the browser instead, - # pointing at the cause. See microsoft/playwright#4283. - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - # Kept despite the image shipping its own Node: this pins the version the - # tests run under to the same 20 as every other job, rather than letting - # it drift with whatever the Playwright image happens to carry, and keeps - # the npm cache. Only browser provisioning changes here. - - uses: actions/setup-node@v4 - with: - node-version: "20" - cache: npm - cache-dependency-path: frontend/package-lock.json - - run: npm ci --legacy-peer-deps - - name: Run E2E tests (staging) - # Has to stay above what nodes.spec.ts's two retried beforeAll hooks can - # reach, so a node that never propagates fails an assertion rather than - # timing the step out and losing the report. That spec derives its hook - # budgets from the server's refresh cadence; raise this if it widens. - timeout-minutes: 20 - run: E2E_ENV=staging npx playwright test - env: - CI: "true" - RADAR_API_KEY: ${{ secrets.STAGING_RADAR_API_KEY }} - # Passage through Access for the admin surface. Playwright follows the - # edge's 302, so without these the admin tests would parse Cloudflare's - # login page as the app. Empty secrets render empty and send nothing. - CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} - CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - e2e-prod: name: Playwright E2E (production) runs-on: ubuntu-latest @@ -512,10 +457,10 @@ jobs: # Does this push touch anything the droplets serve? A docs-only merge would # otherwise redeploy staging AND production and run both Playwright suites for # files no vhost serves, with the prod deploy queued behind the whole staging - # chain to do it. Gating deploy-staging alone is enough: a skipped - # dependency skips its dependents whatever their own `if:` says, so - # staging-smoke-tests, e2e-staging, deploy-production and everything past it - # fall out with it. The rollback jobs are keyed to `result == 'failure'`, and a + # chain to do it. Gating the `staging` job alone is enough: a skipped + # dependency skips its dependents whatever their own `if:` says, so the whole + # called workflow, deploy-production and everything past it fall out with + # it. The rollback jobs are keyed to `result == 'failure'`, and a # skipped deploy is not a failed one, so they stay dormant. # # Only the deploy chain is gated. The test matrix above still runs on a @@ -534,7 +479,25 @@ jobs: if: github.ref == 'refs/heads/main' && github.event_name == 'push' outputs: code: ${{ steps.filter.outputs.code }} + app_dir: ${{ steps.deploy_dir.outputs.app_dir }} steps: + # Workflow-level `env` does not reach a called workflow, and `with:` on a + # calling job cannot read the `env` context either. Re-exported through an + # output so APP_DIR above stays the one definition. + - name: Re-export the deploy directory + id: deploy_dir + run: | + set -euo pipefail + # Refuse to hand the staging workflow an empty directory. `cd ""` + # succeeds into $HOME, and the deploy's own pre-flights all pass there + # (right hostname, no .env, disk fine) before `git fetch` fails in a + # directory nothing is served from. Caught here, where the cause is + # legible, rather than as a confusing abort on the droplet. + if [ -z "${APP_DIR:-}" ]; then + echo "::error::APP_DIR is empty. It is set at the top of this workflow and the staging deploy cds into it; nothing below can run without it." + exit 1 + fi + echo "app_dir=${APP_DIR}" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v4 with: # The push's whole range has to be diffable and `before` can be @@ -593,11 +556,18 @@ jobs: echo "code=false" >> "$GITHUB_OUTPUT" fi - deploy-staging: - name: Deploy to staging - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [changes, backend-tests, frontend-build, dashboard-build, docker-build, env-parity, tower-service-contract] + # Staging's deploy and both of its verification suites, called as one job. + # A calling job is not complete until every job in the called workflow has + # finished, so the concurrency group below is held across the whole sequence. + # Three grouped jobs here would not do it: the group is acquired and released + # per job, which let a following run's deploy land between one run's smoke + # tests and its E2E. See the called workflow. + staging: + name: Staging + # `playwright-image` joins the deploy's own prerequisites so an unpublished + # image fails the run before the droplet is touched, rather than after a + # deploy and a smoke suite have already landed on it. + needs: [changes, backend-tests, frontend-build, dashboard-build, docker-build, env-parity, tower-service-contract, playwright-image] if: github.ref == 'refs/heads/main' && github.event_name == 'push' && needs.changes.outputs.code == 'true' concurrency: group: staging-deploy @@ -605,172 +575,23 @@ jobs: # `git reset --hard` but before `docker compose up -d --build`, leaving the # box on fresh code with stale running containers (config/image changes # silently never take effect). Matches deploy-production. + # + # At most one run sits pending per group, so a third merge arriving while + # one run holds staging and another is queued cancels the queued one. That + # is the right outcome — main is linear, so the newer run deploys a + # superset of what the cancelled one would have — but it makes a cancelled + # `Staging` job an expected state during a burst rather than a fault. cancel-in-progress: false - steps: - - name: Deploy to staging via SSH - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.STAGING_HOST }} - username: root - key: ${{ secrets.STAGING_SSH_KEY }} - command_timeout: 20m - envs: APP_DIR - script: | - set -e - cd ${{ env.APP_DIR }} - - # Same two wrong-environment checks production runs, first and in the - # same order, for the same reasons. The hostname is the only signal - # that holds on a box with no .env at all, which is exactly the state - # a droplet awaiting its first deploy is in; see the production block - # for why it is an exact match rather than a pattern. - if [ "$(hostname)" != "retina-staging" ]; then - echo "::error::Refusing to deploy staging onto '$(hostname)' — expected retina-staging. If this droplet was rebuilt, finish provisioning it (hostnamectl set-hostname retina-staging); otherwise this job is pointed at the wrong box." - exit 1 - fi - - # A .env naming another environment means this job is pointed - # at a box it does not own. Without this the write further down simply - # takes that box over — overwriting its .env and running `down` then - # `up` would bring production back on staging's hostnames, fleet size - # and container names. A MISSING .env is allowed through; that is the - # normal state of a box awaiting its first deploy, and we write it - # below. - if [ -f .env ] && ! grep -q '^COMPOSE_FILE=.*docker-compose\.staging\.yml' .env; then - echo "::error::${{ env.APP_DIR }}/.env exists but does not select the staging overlay — this host is not staging. Inspect it; do not deploy." - exit 1 - fi - - # Pre-flight: Cloudflare's origin-pull CA, checked BEFORE anything - # stops or rebuilds the stack. nginx renders `ssl_verify_client on` - # (deploy/nginx/snippets/tls.conf) and start.sh refuses to boot - # without the CA, so on a droplet where the human half of this work - # is not done — enable Authenticated Origin Pulls for the zone, put - # the CA on the box — a deploy would run `docker compose down`, - # rebuild, fail to start, and leave the environment DOWN. Refusing - # here converts that outage into a refused deploy: everything above - # this line only reads the box. - CF_CA=/etc/ssl/cloudflare/origin-pull-ca.pem - if [ ! -f "$CF_CA" ]; then - echo "::error::${CF_CA} is missing on $(hostname). nginx requires Cloudflare's origin-pull CA and will not start without it. Enable Authenticated Origin Pulls for the zone, then fetch the CA onto this droplet (curl -fsS --create-dirs -o ${CF_CA} https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem) and re-run. Refusing to deploy; the running stack is untouched." - exit 1 - fi - if ! openssl x509 -in "$CF_CA" -noout >/dev/null 2>&1; then - echo "::error::${CF_CA} exists on $(hostname) but is not a parseable PEM certificate — a truncated download or an error page saved to the path would look exactly like this. nginx would fail to start. Replace it and re-run. Refusing to deploy; the running stack is untouched." - exit 1 - fi - echo "Origin-pull CA present and parseable: ${CF_CA}" - - # Pre-flight: refuse to deploy when the disk is near full. Without - # this, git fetch silently writes a partial pack, the source tree - # never advances, and `docker compose --build` happily caches every - # layer and "succeeds" with yesterday's image (run 25923117075). - AVAIL_MB=$(df -m / | awk 'NR==2 {print $4}') - if [ "${AVAIL_MB:-0}" -lt 2048 ]; then - echo "::error::Staging disk only ${AVAIL_MB}MB free (< 2GB threshold). Aborting deploy." - df -h / - docker system df 2>/dev/null || true - exit 1 - fi - - # Hard-reset rather than `git pull` — see the production deploy - # block for the full incident description; same failure mode would - # surface here if the staging server ever picked up a local commit. - # `set -e` above guarantees a failed fetch/reset aborts the deploy. - git fetch origin main - git reset --hard origin/main - git submodule update --init --recursive - # The host's ./.env sets COMPOSE_FILE (base + staging overlay), so - # every command below is character-identical to the production deploy - # block. Written from the tree we just checked out rather than assumed - # to be present: this job knows perfectly well which environment it is - # deploying, so making that depend on someone having copied the right - # file by hand only creates a way to get it wrong. A rebuilt droplet, - # a restored snapshot or a hand-mangled file now self-heals here. - cp deploy/env.staging.example .env - # Top up with this host's own CARTO basemap key — see the note in - # deploy/env..example. Guarded rather than chained with &&: a host - # with no key must build watermarked tiles, not fail its deploy. - if [ -f /root/.secrets/carto.env ]; then cat /root/.secrets/carto.env >> .env; fi - docker compose down --timeout 30 - # Cap build cache at ~10GB (LRU eviction) rather than wiping it all: - # keeps recent layers so incremental builds stay fast, and bounds - # total size so cache growth can't trip the disk pre-flight above. - # The stale-layer failure the old blanket wipe guarded against was - # rooted in disk-full (partial git pack -> source never advanced -> - # cache reused); the disk pre-flight now catches that root cause, so a - # full wipe is redundant and only forced slow from-scratch rebuilds. - docker builder prune -f --keep-storage 10GB - # Deploy the app FIRST, on its own — building only `server` - # means a broken Dockerfile.fleet can't abort the staging app deploy - # (which would block the whole pipeline). Fleet is a separate step below. - # --remove-orphans because this deploy renamed the compose - # service. container_name is pinned, so the container kept by the - # old service key is an orphan holding the exact name the new one - # needs, and compose leaves orphans alone by default: the create - # then fails with a name conflict and the stack does not come up. - # Verified on retina-test before this landed. It stays afterwards - # as the general guard for a removed or renamed service. - docker compose up -d --build --remove-orphans server - # Wait for server to become healthy (up to 90s) - for i in $(seq 1 18); do - if docker compose exec -T server \ - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health')" 2>/dev/null; then - echo "Staging server healthy after ~$((i*5))s" - break - fi - echo "Waiting for health check... attempt $i/18" - sleep 5 - if [ "$i" = "18" ]; then - echo "Staging health check failed after 90s" - docker compose logs --tail=50 - exit 1 - fi - done - # Now the fleet, separately — a fleet build/start failure surfaces - # loudly here without taking the already-healthy app down with it. - docker compose up -d --build --no-deps fleet - - staging-smoke-tests: - name: Staging smoke tests - runs-on: ubuntu-latest - timeout-minutes: 35 - needs: [deploy-staging] - steps: - - uses: actions/checkout@v4 - # The curls in this loop carry no --max-time, so against a hung origin - # the 24 iterations run far past the job cap. Bounded here rather than by - # rewriting the loop, which deploy/staging-smoke-test.sh mirrors. - - name: Wait for fleet to connect - timeout-minutes: 10 - run: | - echo "Waiting for fleet nodes to connect..." - # Through the edge, not at the droplet's address: the origin now requires - # Cloudflare's client certificate (ssl_verify_client in - # deploy/nginx/snippets/tls.conf) and the DOCKER-USER boundary drops - # anything on 443 that is not from a Cloudflare range, so a direct-to-IP - # request can only ever time out or 400. `-k` does not help — that flag - # governs verification of the SERVER's certificate, not presentation of - # a client one. Same hostname deploy/staging-smoke-test.sh uses for this - # endpoint (BASE_URL there), so both agree on what "staging" means. - for i in $(seq 1 24); do - COUNT=$(curl -s https://staging-towers.retina.fm/api/test/dashboard \ - | python3 -c "import sys,json; print(json.load(sys.stdin)['nodes']['active'])" 2>/dev/null) || COUNT=0 - if [ "${COUNT:-0}" -ge 1 ]; then - echo "Fleet ready: ${COUNT} node(s) connected after ~$((i*5))s" - break - fi - echo "Attempt $i/24: ${COUNT} nodes — waiting 5s..." - sleep 5 - done - - name: Run smoke tests against staging - timeout-minutes: 15 - # Same service token as production: one token, admitted by a Service Auth - # policy on each admin application. Inert while the secrets are unset. - env: - CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} - CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - run: bash deploy/staging-smoke-test.sh + uses: ./.github/workflows/staging-deploy-verify.yml + with: + app_dir: ${{ needs.changes.outputs.app_dir }} + playwright_image: ${{ needs.playwright-image.outputs.image }} + secrets: + STAGING_HOST: ${{ secrets.STAGING_HOST }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + STAGING_RADAR_API_KEY: ${{ secrets.STAGING_RADAR_API_KEY }} + CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} + CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} deploy-production: name: Deploy to production @@ -778,12 +599,11 @@ jobs: # Above the ssh-action's own command_timeout below, so a stuck deploy # surfaces as that step's error rather than as a bare job timeout. timeout-minutes: 30 - # Gate prod on BOTH the staging smoke tests and the full staging Playwright - # E2E suite (`e2e-staging`). Previously only staging-smoke-tests gated this, - # so the ~127-test E2E ran in parallel with the prod deploy and couldn't - # stop a bad build reaching prod. This serialises the pipeline (E2E is now - # on the critical path before prod) in exchange for real pre-prod coverage. - needs: [staging-smoke-tests, e2e-staging] + # Gate prod on the whole staging sequence: its deploy, its smoke tests and + # its ~127-test Playwright suite. The E2E used to run in parallel with the + # prod deploy and so could not stop a bad build reaching prod. This + # serialises the pipeline in exchange for real pre-prod coverage. + needs: [staging] if: github.ref == 'refs/heads/main' && github.event_name == 'push' concurrency: group: production-deploy @@ -1057,7 +877,7 @@ jobs: cancel-in-progress: false # `always() && ...result == 'failure'` rather than a bare `failure()`. # failure() is true when ANY ancestor job failed, and deploy-production has - # ancestors: an e2e-staging failure skips the deploy entirely, and this job + # ancestors: a staging failure skips the deploy entirely, and this job # would then SSH into production to consider rolling back a deploy that # never ran. Keyed to the deploy job's own result, it cannot. # diff --git a/.github/workflows/staging-deploy-verify.yml b/.github/workflows/staging-deploy-verify.yml new file mode 100644 index 000000000..262cc8e79 --- /dev/null +++ b/.github/workflows/staging-deploy-verify.yml @@ -0,0 +1,277 @@ +name: Staging deploy and verify + +# Deploy staging, then prove it, as one unit. ci.yml calls this from a single +# job that carries the `staging-deploy` concurrency group, and a calling job is +# not complete until every job below has finished, so the group is held from +# the first byte of the deploy to the last E2E assertion. +# +# That holding is the whole point of the file, and it is why the three jobs +# cannot simply live in ci.yml with a group each. Concurrency is acquired and +# released per job, so a following run's deploy was free to land between one +# run's smoke tests and its E2E — which is how three merges on 2026-09-10 all +# failed, none of them for a fault in the change: the box was restarting +# underneath a suite that was testing it. +# +# Splitting these back into three jobs in the caller reopens that race, +# whatever concurrency group each of them carries. + +on: + workflow_call: + inputs: + app_dir: + description: >- + Deploy directory on the droplet. Threaded in from ci.yml's + `env.APP_DIR` rather than restated here: workflow-level `env` does not + cross a `workflow_call` boundary, and a second definition is a second + thing to forget when the directory moves. + required: true + type: string + playwright_image: + description: >- + Playwright container image for the E2E job, derived from the lockfile + by ci.yml's `playwright-image` job so the browsers cannot disagree + with the client that drives them. + required: true + type: string + secrets: + STAGING_HOST: + required: true + STAGING_SSH_KEY: + required: true + STAGING_RADAR_API_KEY: + required: true + # Inert while unset: the smoke script and Playwright both render them + # empty and send no Access headers, which is the state of an environment + # with no Service Auth policy in front of its admin surface. + CF_ACCESS_CLIENT_ID: + required: false + CF_ACCESS_CLIENT_SECRET: + required: false + +jobs: + deploy: + name: Deploy to staging + runs-on: ubuntu-latest + timeout-minutes: 30 + # Carries the deploy directory as an environment variable rather than only + # interpolating it: `envs: APP_DIR` below forwards it to the droplet, where + # deploy/pre-deploy.sh and deploy/rollback.sh read it. + env: + APP_DIR: ${{ inputs.app_dir }} + steps: + - name: Deploy to staging via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.STAGING_HOST }} + username: root + key: ${{ secrets.STAGING_SSH_KEY }} + command_timeout: 20m + envs: APP_DIR + script: | + set -e + cd ${{ env.APP_DIR }} + + # Same two wrong-environment checks production runs, first and in the + # same order, for the same reasons. The hostname is the only signal + # that holds on a box with no .env at all, which is exactly the state + # a droplet awaiting its first deploy is in; see the production block + # for why it is an exact match rather than a pattern. + if [ "$(hostname)" != "retina-staging" ]; then + echo "::error::Refusing to deploy staging onto '$(hostname)' — expected retina-staging. If this droplet was rebuilt, finish provisioning it (hostnamectl set-hostname retina-staging); otherwise this job is pointed at the wrong box." + exit 1 + fi + + # A .env naming another environment means this job is pointed + # at a box it does not own. Without this the write further down simply + # takes that box over — overwriting its .env and running `down` then + # `up` would bring production back on staging's hostnames, fleet size + # and container names. A MISSING .env is allowed through; that is the + # normal state of a box awaiting its first deploy, and we write it + # below. + if [ -f .env ] && ! grep -q '^COMPOSE_FILE=.*docker-compose\.staging\.yml' .env; then + echo "::error::${{ env.APP_DIR }}/.env exists but does not select the staging overlay — this host is not staging. Inspect it; do not deploy." + exit 1 + fi + + # Pre-flight: Cloudflare's origin-pull CA, checked BEFORE anything + # stops or rebuilds the stack. nginx renders `ssl_verify_client on` + # (deploy/nginx/snippets/tls.conf) and start.sh refuses to boot + # without the CA, so on a droplet where the human half of this work + # is not done — enable Authenticated Origin Pulls for the zone, put + # the CA on the box — a deploy would run `docker compose down`, + # rebuild, fail to start, and leave the environment DOWN. Refusing + # here converts that outage into a refused deploy: everything above + # this line only reads the box. + CF_CA=/etc/ssl/cloudflare/origin-pull-ca.pem + if [ ! -f "$CF_CA" ]; then + echo "::error::${CF_CA} is missing on $(hostname). nginx requires Cloudflare's origin-pull CA and will not start without it. Enable Authenticated Origin Pulls for the zone, then fetch the CA onto this droplet (curl -fsS --create-dirs -o ${CF_CA} https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem) and re-run. Refusing to deploy; the running stack is untouched." + exit 1 + fi + if ! openssl x509 -in "$CF_CA" -noout >/dev/null 2>&1; then + echo "::error::${CF_CA} exists on $(hostname) but is not a parseable PEM certificate — a truncated download or an error page saved to the path would look exactly like this. nginx would fail to start. Replace it and re-run. Refusing to deploy; the running stack is untouched." + exit 1 + fi + echo "Origin-pull CA present and parseable: ${CF_CA}" + + # Pre-flight: refuse to deploy when the disk is near full. Without + # this, git fetch silently writes a partial pack, the source tree + # never advances, and `docker compose --build` happily caches every + # layer and "succeeds" with yesterday's image (run 25923117075). + AVAIL_MB=$(df -m / | awk 'NR==2 {print $4}') + if [ "${AVAIL_MB:-0}" -lt 2048 ]; then + echo "::error::Staging disk only ${AVAIL_MB}MB free (< 2GB threshold). Aborting deploy." + df -h / + docker system df 2>/dev/null || true + exit 1 + fi + + # Hard-reset rather than `git pull` — see the production deploy + # block for the full incident description; same failure mode would + # surface here if the staging server ever picked up a local commit. + # `set -e` above guarantees a failed fetch/reset aborts the deploy. + git fetch origin main + git reset --hard origin/main + git submodule update --init --recursive + # The host's ./.env sets COMPOSE_FILE (base + staging overlay), so + # every command below is character-identical to the production deploy + # block. Written from the tree we just checked out rather than assumed + # to be present: this job knows perfectly well which environment it is + # deploying, so making that depend on someone having copied the right + # file by hand only creates a way to get it wrong. A rebuilt droplet, + # a restored snapshot or a hand-mangled file now self-heals here. + cp deploy/env.staging.example .env + # Top up with this host's own CARTO basemap key — see the note in + # deploy/env..example. Guarded rather than chained with &&: a host + # with no key must build watermarked tiles, not fail its deploy. + if [ -f /root/.secrets/carto.env ]; then cat /root/.secrets/carto.env >> .env; fi + docker compose down --timeout 30 + # Cap build cache at ~10GB (LRU eviction) rather than wiping it all: + # keeps recent layers so incremental builds stay fast, and bounds + # total size so cache growth can't trip the disk pre-flight above. + # The stale-layer failure the old blanket wipe guarded against was + # rooted in disk-full (partial git pack -> source never advanced -> + # cache reused); the disk pre-flight now catches that root cause, so a + # full wipe is redundant and only forced slow from-scratch rebuilds. + docker builder prune -f --keep-storage 10GB + # Deploy the app FIRST, on its own — building only `server` + # means a broken Dockerfile.fleet can't abort the staging app deploy + # (which would block the whole pipeline). Fleet is a separate step below. + # --remove-orphans because this deploy renamed the compose + # service. container_name is pinned, so the container kept by the + # old service key is an orphan holding the exact name the new one + # needs, and compose leaves orphans alone by default: the create + # then fails with a name conflict and the stack does not come up. + # Verified on retina-test before this landed. It stays afterwards + # as the general guard for a removed or renamed service. + docker compose up -d --build --remove-orphans server + # Wait for server to become healthy (up to 90s) + for i in $(seq 1 18); do + if docker compose exec -T server \ + python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health')" 2>/dev/null; then + echo "Staging server healthy after ~$((i*5))s" + break + fi + echo "Waiting for health check... attempt $i/18" + sleep 5 + if [ "$i" = "18" ]; then + echo "Staging health check failed after 90s" + docker compose logs --tail=50 + exit 1 + fi + done + # Now the fleet, separately — a fleet build/start failure surfaces + # loudly here without taking the already-healthy app down with it. + docker compose up -d --build --no-deps fleet + smoke-tests: + name: Staging smoke tests + runs-on: ubuntu-latest + timeout-minutes: 35 + needs: [deploy] + steps: + - uses: actions/checkout@v4 + # The curls in this loop carry no --max-time, so against a hung origin + # the 24 iterations run far past the job cap. Bounded here rather than by + # rewriting the loop, which deploy/staging-smoke-test.sh mirrors. + - name: Wait for fleet to connect + timeout-minutes: 10 + run: | + echo "Waiting for fleet nodes to connect..." + # Through the edge, not at the droplet's address: the origin now requires + # Cloudflare's client certificate (ssl_verify_client in + # deploy/nginx/snippets/tls.conf) and the DOCKER-USER boundary drops + # anything on 443 that is not from a Cloudflare range, so a direct-to-IP + # request can only ever time out or 400. `-k` does not help — that flag + # governs verification of the SERVER's certificate, not presentation of + # a client one. Same hostname deploy/staging-smoke-test.sh uses for this + # endpoint (BASE_URL there), so both agree on what "staging" means. + for i in $(seq 1 24); do + COUNT=$(curl -s https://staging-towers.retina.fm/api/test/dashboard \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['nodes']['active'])" 2>/dev/null) || COUNT=0 + if [ "${COUNT:-0}" -ge 1 ]; then + echo "Fleet ready: ${COUNT} node(s) connected after ~$((i*5))s" + break + fi + echo "Attempt $i/24: ${COUNT} nodes — waiting 5s..." + sleep 5 + done + - name: Run smoke tests against staging + timeout-minutes: 15 + # Same service token as production: one token, admitted by a Service Auth + # policy on each admin application. Inert while the secrets are unset. + env: + CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} + CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} + run: bash deploy/staging-smoke-test.sh + e2e: + name: Playwright E2E (staging) + runs-on: ubuntu-latest + # Only the pull and npm ci sit outside the test step's own cap now, and an + # image pull is minutes at worst. See the production job for why the job + # cap is a backstop rather than the primary bound. + timeout-minutes: 30 + needs: [smoke-tests] + # The browsers and their shared libraries ship in the image, so there is no + # install step and neither apt nor Playwright's CDN is in this path. Both + # broke repeatedly on 2026-08-18 and 2026-08-19; see the production job. + container: + image: ${{ inputs.playwright_image }} + # `npm ci` runs @playwright/test's postinstall, which fetches browsers. The + # image sets PLAYWRIGHT_BROWSERS_PATH=/ms-playwright and the tag is derived + # to match the lockfile, so it finds the right revision already there and + # does nothing. This makes that a guarantee rather than an expectation: if + # the browsers were ever NOT found, the postinstall would quietly download + # them from the CDN this job exists to avoid, and the only symptom would be + # a slow green run. With this set it fails loudly at the browser instead, + # pointing at the cause. See microsoft/playwright#4283. + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + # Kept despite the image shipping its own Node: this pins the version the + # tests run under to the same 20 as every other job, rather than letting + # it drift with whatever the Playwright image happens to carry, and keeps + # the npm cache. Only browser provisioning changes here. + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci --legacy-peer-deps + - name: Run E2E tests (staging) + # Has to stay above what nodes.spec.ts's two retried beforeAll hooks can + # reach, so a node that never propagates fails an assertion rather than + # timing the step out and losing the report. That spec derives its hook + # budgets from the server's refresh cadence; raise this if it widens. + timeout-minutes: 20 + run: E2E_ENV=staging npx playwright test + env: + CI: "true" + RADAR_API_KEY: ${{ secrets.STAGING_RADAR_API_KEY }} + # Passage through Access for the admin surface. Playwright follows the + # edge's 302, so without these the admin tests would parse Cloudflare's + # login page as the app. Empty secrets render empty and send nothing. + CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} + CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} diff --git a/ONBOARDING.md b/ONBOARDING.md index 38db18912..23cf8990a 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -233,12 +233,24 @@ CI runs on every PR, on push to `main`, and on demand through `dashboard-build`, `docker-build`, `env-parity`, plus an automated review. 2. Merge to `main` → deploy to **staging** → staging smoke + Playwright E2E → deploy to **production** → prod smoke + Playwright E2E. A markdown-only merge skips that chain; the `changes` job has the exceptions. + The staging third of it is a called workflow, + `.github/workflows/staging-deploy-verify.yml`, invoked from one `Staging` + job so that job's concurrency group is held across the deploy and both + suites. Adding a staging step means editing that file, not `ci.yml`. So merging to `main` deploys to production automatically. Work on a feature branch, open a PR, get it green, then merge. ## Things that will bite you +- **A cancelled `Staging` job on a burst of merges is expected, not a fault.** + Only one run may sit pending on the `staging-deploy` group, so when a third + merge arrives while one run holds staging and another is queued, the queued + one is cancelled. `main` is linear, so the run that replaces it deploys a + superset of what was dropped. What it does mean is that the cancelled + commit's own run never reaches production: the following run carries it. + The last merge in a burst has no successor, so check it landed. + - **The CARTO basemap key lives on the droplet, not in this repo.** Every deploy appends `/root/.secrets/carto.env` to `./.env` after copying `deploy/env..example`, and `docker-compose.yml` interpolates it into the diff --git a/deploy/staging-smoke-test.sh b/deploy/staging-smoke-test.sh index c67ea7260..3f25b9a6e 100644 --- a/deploy/staging-smoke-test.sh +++ b/deploy/staging-smoke-test.sh @@ -103,11 +103,11 @@ check_contract() { NO_DNS_EXPECTED="" # Vhosts whose record exists and is expected to, but whose absence must not -# fail the run: staging-smoke-tests is a `needs:` of deploy-production -# (ci.yml), so a hard failure here would let a Cloudflare wobble block every -# release. Reported as WARN and tallied separately, because a deleted record -# must still be visible: skipping it silently would retire the only check on a -# vhost nothing else monitors. +# fail the run: this runs inside the `staging` job that deploy-production needs +# (ci.yml calls staging-deploy-verify.yml), so a hard failure here would let a +# Cloudflare wobble block every release. Reported as WARN and tallied +# separately, because a deleted record must still be visible: skipping it +# silently would retire the only check on a vhost nothing else monitors. DNS_NOT_DEPLOY_BLOCKING="staging-admin.retina.fm staging-data.retina.fm" # Decides what to do about $1 not resolving, prints it, and returns 0 when the diff --git a/docs/architecture.md b/docs/architecture.md index a97d67413..1dcb029ca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,6 +144,8 @@ data to a user's own nodes. ## Deploy `.github/workflows/ci.yml`: push to `main` → build/test → deploy staging → -staging smoke + E2E → deploy production → prod smoke + E2E. Deploy is an SSH -`git reset --hard origin/main` + `docker compose up -d --build`, gated by a +staging smoke + E2E → deploy production → prod smoke + E2E. The three staging +steps live in `staging-deploy-verify.yml` and are called as a single job, so one +run holds the environment until its own verification has finished. Deploy is an +SSH `git reset --hard origin/main` + `docker compose up -d --build`, gated by a free-disk pre-flight. Operational detail is in [`runbook.md`](runbook.md). diff --git a/frontend/e2e/dashboard.spec.ts b/frontend/e2e/dashboard.spec.ts index aa487b587..bf2ee3a9b 100644 --- a/frontend/e2e/dashboard.spec.ts +++ b/frontend/e2e/dashboard.spec.ts @@ -104,9 +104,9 @@ test.describe("Dashboard — unauthenticated access (real auth mode)", () => { /** * Whether a hostname resolves at all, as distinct from what it answers. * - * e2e-staging is a `needs:` of deploy-production exactly as staging-smoke-tests - * is, so this suite can block a release for the same reason that one can. That - * one reports an unresolvable name as a warning rather than a failure, because + * This suite runs inside the `staging` job that deploy-production needs, exactly + * as the smoke tests do, so it can block a release for the same reason they can. + * They report an unresolvable name as a warning rather than a failure, because * staging-admin.retina.fm's record is young and nothing monitors it; without * the same tolerance here a DNS wobble still holds up every deploy, through the * sibling gate.