Repository navigation
201 lines (184 loc) · 8.93 KB
/
Copy pathdeploy.yml
File metadata and controls
201 lines (184 loc) · 8.93 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
name: Deploy to Railway
on:
# Manual-only prod deploys. The automatic push-to-main trigger was removed
# so merges (including migration PRs such as v141 / #3717) cannot ship to
# Railway without an explicit workflow_dispatch. Operators must type
# confirm=deploy-prod when running the workflow. No GitHub Environment
# protection rules are used here (avoids paid settings).
# The wait-for-ci job below still blocks any SHA without a green Backend
# CI Status Gate. The deploy job uses concurrency group deploy-main with
# cancel-in-progress: false so a prod migration mid-run is never cancelled.
workflow_dispatch:
inputs:
confirm:
description: 'Type deploy-prod to confirm a production Railway deploy (migrations included)'
required: true
type: string
permissions:
checks: read
contents: read
jobs:
# Fail closed unless the operator typed the confirm phrase. A wrong/empty
# value must not silently skip into a green workflow.
confirm-gate:
name: Confirm production deploy
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Require confirm=deploy-prod
env:
CONFIRM: ${{ github.event.inputs.confirm }}
run: |
if [ "$CONFIRM" != "deploy-prod" ]; then
echo "::error::Refusing deploy: confirm input must be exactly 'deploy-prod'."
exit 1
fi
echo "Confirm input accepted; proceeding to CI wait."
# Gate: manual deploys must wait for a green CI Status Gate check on the
# selected ref before Railway deployment can start.
wait-for-ci:
name: Wait for CI
needs: [confirm-gate]
runs-on: ubuntu-latest
timeout-minutes: 50
steps:
- name: Require backend CI green
uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1
with:
ref: ${{ github.sha }}
running-workflow-name: 'Wait for CI'
repo-token: ${{ secrets.GITHUB_TOKEN }}
# The deploy ships the backend monolith image, so it must wait on the
# BACKEND gate specifically. Backend and frontend used to share the
# name "CI Status Gate", so a frontend-only SHA (whose only run was
# frontend-ci) satisfied this wait with the backend never verified.
# The gates are now "Backend CI Status Gate" / "Frontend CI Status
# Gate"; matching the backend name means a SHA with no backend run has
# no matching check, so this step times out (checks-discovery-timeout)
# instead of passing — deploy does not proceed. (#2537)
# GitHub Checks API returns job names, not "workflow / job" format.
# Backend CI can take longer than 15m to create its summary gate, so
# keep discovery patient.
check-name: 'Backend CI Status Gate'
wait-interval: 30
checks-discovery-timeout: 2700
# Only a passing backend gate clears the deploy. `skipped` is NOT in
# this list (the action default is `success,skipped`), so a skipped
# gate cannot green-light a deploy.
allowed-conclusions: success
# If no "Backend CI Status Gate" check exists for the dispatched SHA
# (e.g. a frontend-only SHA never ran backend-ci), fail rather than
# treat "no matching check" as success — deploy must not proceed on a
# ref the backend never verified (#2537).
fail-on-no-checks: true
deploy:
name: Deploy RegEngine Service
needs: [wait-for-ci]
# Serialize prod deploys; never cancel mid-migration (Railway migrate).
concurrency:
group: deploy-main
cancel-in-progress: false
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
- name: Setup Node.js
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6
with:
node-version-file: .nvmrc
- name: Install Railway CLI
run: |
npm install -g "@railway/cli@4.44.0"
railway --version
- name: Validate Railway deployment secrets
id: validate_railway
env:
RAILWAY_API_TOKEN: ${{ secrets.RAILWAY_API_TOKEN }}
RAILWAY_PROJECT_ID: ${{ secrets.RAILWAY_PROJECT_ID }}
RAILWAY_ENVIRONMENT_ID: ${{ secrets.RAILWAY_ENVIRONMENT_ID }}
RAILWAY_SERVICE_ID: ${{ secrets.RAILWAY_SERVICE_ID }}
run: |
missing=()
[ -z "$RAILWAY_API_TOKEN" ] && missing+=("RAILWAY_API_TOKEN")
[ -z "$RAILWAY_PROJECT_ID" ] && missing+=("RAILWAY_PROJECT_ID")
[ -z "$RAILWAY_ENVIRONMENT_ID" ] && missing+=("RAILWAY_ENVIRONMENT_ID")
[ -z "$RAILWAY_SERVICE_ID" ] && missing+=("RAILWAY_SERVICE_ID")
if [ ${#missing[@]} -gt 0 ]; then
echo "::error::Railway deployment secrets are missing: ${missing[*]}"
exit 1
fi
# Stamp the commit into the build context. .dockerignore excludes .git,
# so without this the image has no way to say which commit it is —
# which is what made the class of failure below undetectable.
- name: Stamp build identity
run: |
printf '%s\n' "${{ github.sha }}" > build-sha.txt
echo "Stamped build-sha.txt with ${{ github.sha }}"
- name: Deploy to Railway
env:
# Use the account/workspace token path. The previous project token
# secret was unauthorized for this project/service.
RAILWAY_API_TOKEN: ${{ secrets.RAILWAY_API_TOKEN }}
RAILWAY_PROJECT_ID: ${{ secrets.RAILWAY_PROJECT_ID }}
RAILWAY_ENVIRONMENT_ID: ${{ secrets.RAILWAY_ENVIRONMENT_ID }}
RAILWAY_SERVICE_ID: ${{ secrets.RAILWAY_SERVICE_ID }}
run: |
railway up \
--ci \
--project "$RAILWAY_PROJECT_ID" \
--environment "$RAILWAY_ENVIRONMENT_ID" \
--service "$RAILWAY_SERVICE_ID"
# `railway up --ci` returns once the upload is accepted, not once the new
# image is serving. railway.toml sets restartPolicyType = "ON_FAILURE"
# with a /readiness healthcheck, so a rollout that fails its healthcheck
# reverts to the previous image *after* this workflow has already gone
# green. Nothing downstream could notice: every probe asserts behaviour
# an older image satisfies perfectly.
#
# This closes that hole by asserting identity rather than behaviour —
# the deployed /health must report the commit we just shipped.
- name: Verify the deployed commit is the one we shipped
env:
BACKEND_URL: https://regengine-production.up.railway.app
EXPECTED_SHA: ${{ github.sha }}
run: |
set -uo pipefail
# Rollout budget. railway.toml allows healthcheckTimeout=300 plus up
# to 3 ON_FAILURE restarts, so a slow-but-succeeding rollout must not
# be called a failure here.
deadline=$(( SECONDS + 900 ))
attempt=0
last_sha=""
last_code=""
while [ "$SECONDS" -lt "$deadline" ]; do
attempt=$(( attempt + 1 ))
body=$(mktemp)
last_code=$(curl -sS --max-time 20 -o "$body" -w '%{http_code}' "${BACKEND_URL}/health" || true)
last_code="${last_code:-000}"
if [ "$last_code" = "200" ]; then
# `// empty` also covers a payload with no build key at all,
# which is what an image predating build stamping returns.
last_sha=$(jq -r '.build.commit_sha // empty' "$body" 2>/dev/null || true)
if [ "$last_sha" = "$EXPECTED_SHA" ]; then
rm -f "$body"
echo "Deployed commit matches: ${EXPECTED_SHA}"
exit 0
fi
fi
rm -f "$body"
echo "attempt ${attempt}: status=${last_code} deployed_sha=${last_sha:-(none)} expected=${EXPECTED_SHA}"
sleep 15
done
echo "::error::Deploy did not converge. /health last reported status=${last_code} commit_sha=${last_sha:-(none)}, expected ${EXPECTED_SHA}."
# Three distinct causes, three distinct messages. A single generic
# failure string would send whoever reads this at 3am down the wrong
# path, which is most of what makes an alert useless.
if [ "$last_code" != "200" ]; then
echo "::error::/health never returned 200 (last status ${last_code}). The service is not serving — check the Railway deploy logs for a crashed or unhealthy rollout."
elif [ -z "$last_sha" ]; then
echo "::error::/health answered but carried no commit_sha. Either the rollout reverted to an image predating build stamping, or build-sha.txt did not reach the image."
else
echo "::error::A different commit is serving — the rollout reverted (railway.toml restartPolicyType=ON_FAILURE) or another deploy raced this one."
fi
exit 1