From 15e48b8ac4ca65452fad8985684de8dd6d5fd521 Mon Sep 17 00:00:00 2001 From: Christopher Sellers <1@christophersellers.com> Date: Thu, 29 Oct 2026 23:54:28 -0700 Subject: [PATCH 1/5] feat(deploy): stamp commit SHA on hand-run Railway deploys Add scripts/railway/deploy_prod.sh so a manual `railway up` ships a real commit identity the same way deploy.yml does. The script refuses a dirty working tree, writes `git rev-parse HEAD` into build-sha.txt, runs `railway up --ci` against the linked service, restores the placeholder `unknown` (also on failure, via an EXIT trap), then polls /health until build.commit_sha equals the stamped SHA, with the same three failure diagnoses as the workflow. `--dry-run` does everything except the upload and the poll. Since 2026-10-28 the Railway service has no GitHub source and deploy.yml is disabled, so the 2026-10-29 hand deploy shipped the placeholder and /health reported commit_sha: null. Document in the release runbook that manual deploys must go through this script, and why. The tracked build-sha.txt stays the placeholder so a plain checkout never carries a stale SHA. Co-Authored-By: Claude Fable 5.1 --- docs/runbooks/RELEASE_BRANCH_DEPLOYS.md | 24 ++++ scripts/railway/deploy_prod.sh | 180 ++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100755 scripts/railway/deploy_prod.sh diff --git a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md index 6ca30ee3e..40057b460 100644 --- a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md +++ b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md @@ -217,3 +217,27 @@ run the audit-chain integrity check (section 6). Approve or reject this model, and pick the cadence: on demand, or at most two or three times per weekday. Until then, production keeps deploying from `main` through `deploy.yml`. + +## 11. Manual deploys today (in effect, not part of the proposal) + +Since 2026-10-28 the Railway production service has no GitHub source and +`deploy.yml` is disabled, so every production deploy is a hand-run +`railway up` from a local checkout. A hand deploy must go through +`scripts/railway/deploy_prod.sh`, never a bare `railway up`. + +Reason: the image has no `.git` (`.dockerignore` excludes it), so the only +commit identity it carries is `build-sha.txt`, which the `Dockerfile` copies +to `/app/build-sha.txt`. The tracked file is the placeholder `unknown`. +`deploy.yml` overwrites it with the commit before `railway up`; a bare +`railway up` ships the placeholder and `/health` reports `commit_sha: null`. +The 2026-10-29 deploy (Railway deployment `44c0486a`) did exactly that, and +the live commit could only be identified by hashing files inside the +container over `railway ssh`. + +The script refuses a dirty tree, stamps `git rev-parse HEAD` into +`build-sha.txt`, runs `railway up --ci` against the linked service, restores +the placeholder (also on failure), then polls `https://api.regengine.co/health` +until `build.commit_sha` equals the stamped commit, with the same three +failure diagnoses as `deploy.yml`. `--dry-run` does everything except the +upload and the poll. Set `RAILWAY_SERVICE` or `RAILWAY_ENVIRONMENT` to +override what `railway link` chose. diff --git a/scripts/railway/deploy_prod.sh b/scripts/railway/deploy_prod.sh new file mode 100755 index 000000000..52b05e93e --- /dev/null +++ b/scripts/railway/deploy_prod.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# Hand-run production deploy that stamps the commit the same way +# .github/workflows/deploy.yml does. +# +# Why this exists: the image has no .git (.dockerignore excludes it), so the +# only way /health can report which commit is serving is the build-sha.txt +# the Dockerfile copies to /app/build-sha.txt. The tracked file is the +# placeholder "unknown" so a plain checkout never carries a stale SHA. The +# workflow overwrites it with github.sha before `railway up`; a bare +# `railway up` from a laptop ships the placeholder and /health answers +# commit_sha: null (that is what the 2026-10-29 deploy did). This script +# closes that gap for manual deploys: +# +# 1. refuse to run with a dirty working tree +# 2. write `git rev-parse HEAD` into build-sha.txt +# 3. `railway up --ci` against the linked Railway service +# 4. restore the placeholder (also on failure, via trap) +# 5. poll /health until build.commit_sha equals the stamped SHA +# +# Usage: +# scripts/railway/deploy_prod.sh # real deploy +# scripts/railway/deploy_prod.sh --dry-run # everything except railway up +# # and the health poll +# +# Environment (all optional): +# BACKEND_URL health base URL (default https://api.regengine.co) +# RAILWAY_SERVICE passed as `railway up --service` when set +# RAILWAY_ENVIRONMENT passed as `railway up --environment` when set +# DEPLOY_TIMEOUT_SECS health poll budget (default 900, mirrors deploy.yml) +set -euo pipefail + +PLACEHOLDER="unknown" +STAMP_FILE="build-sha.txt" +BACKEND_URL="${BACKEND_URL:-https://api.regengine.co}" +DEPLOY_TIMEOUT_SECS="${DEPLOY_TIMEOUT_SECS:-900}" +DRY_RUN=0 + +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=1 ;; + -h|--help) + sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "error: unknown argument: $arg (try --help)" >&2 + exit 2 + ;; + esac +done + +die() { + echo "error: $*" >&2 + exit 1 +} + +command -v git >/dev/null 2>&1 || die "git is required but not found in PATH" + +# Everything below runs from the repository root: the Dockerfile and the +# stamp file live there, and `railway up` uploads the current directory. +repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || die "not inside a git checkout" +cd "$repo_root" + +# --- Preconditions. Nothing is touched until all of these pass. ------------- + +dirty="$(git status --porcelain --untracked-files=all)" +if [ -n "$dirty" ]; then + echo "error: working tree is not clean. A deploy must ship exactly one" >&2 + echo "commit, and \`railway up\` uploads whatever is on disk, tracked or not." >&2 + echo "Commit, discard, or move these first:" >&2 + printf '%s\n' "$dirty" >&2 + exit 1 +fi + +[ -f "$STAMP_FILE" ] || die "$STAMP_FILE is missing from the checkout" +if [ "$(cat "$STAMP_FILE")" != "$PLACEHOLDER" ]; then + die "$STAMP_FILE is tracked as '$(cat "$STAMP_FILE")', expected the placeholder '$PLACEHOLDER'. Fix the tracked file before deploying." +fi + +if [ "$DRY_RUN" -eq 0 ]; then + for tool in railway curl jq; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not found in PATH" + done +fi + +sha="$(git rev-parse HEAD)" +branch="$(git rev-parse --abbrev-ref HEAD)" + +echo "checkout: $repo_root" +echo "branch: $branch" +echo "commit: $sha" +if [ "$branch" != "main" ]; then + echo "warning: HEAD is not on main. Production normally ships main; continuing because you asked." >&2 +fi + +# --- Stamp, with a trap so the placeholder always comes back. ---------------- + +restore_stamp() { + # `git checkout --` restores the exact tracked bytes, which is what keeps + # `git status` clean afterwards. The printf fallback covers a checkout + # whose index is unreadable mid-failure. + git checkout --quiet -- "$STAMP_FILE" 2>/dev/null || printf '%s\n' "$PLACEHOLDER" > "$STAMP_FILE" +} +trap restore_stamp EXIT + +printf '%s\n' "$sha" > "$STAMP_FILE" +echo "stamped: $STAMP_FILE = $(cat "$STAMP_FILE")" + +# --- Deploy. ----------------------------------------------------------------- + +up_args=(up --ci) +[ -n "${RAILWAY_SERVICE:-}" ] && up_args+=(--service "$RAILWAY_SERVICE") +[ -n "${RAILWAY_ENVIRONMENT:-}" ] && up_args+=(--environment "$RAILWAY_ENVIRONMENT") + +if [ "$DRY_RUN" -eq 1 ]; then + echo "dry run: would run: railway ${up_args[*]}" + echo "dry run: would poll ${BACKEND_URL}/health for build.commit_sha == $sha (budget ${DEPLOY_TIMEOUT_SECS}s)" + restore_stamp + trap - EXIT + if [ -n "$(git status --porcelain --untracked-files=all)" ]; then + die "dry run left the tree dirty; this is a bug in $0" + fi + echo "restored: $STAMP_FILE = $(cat "$STAMP_FILE"); git status clean" + exit 0 +fi + +echo "target:" +railway status || die "railway status failed. Run \`railway login\` and \`railway link\` first." + +railway "${up_args[@]}" + +# Restore as soon as the upload is accepted; the trap is only the safety net. +restore_stamp +trap - EXIT +echo "restored: $STAMP_FILE = $(cat "$STAMP_FILE")" + +# --- Verify identity, not behaviour (mirrors deploy.yml). -------------------- +# +# `railway up --ci` returns once the upload is accepted, not once the new +# image is serving. railway.toml sets restartPolicyType = "ON_FAILURE" with a +# /readiness healthcheck, so a rollout that fails its healthcheck reverts to +# the previous image after this script would otherwise have exited 0. The +# only check that cannot be satisfied by an older image is the commit itself. + +deadline=$(( SECONDS + DEPLOY_TIMEOUT_SECS )) +attempt=0 +last_sha="" +last_code="" + +while [ "$SECONDS" -lt "$deadline" ]; do + attempt=$(( attempt + 1 )) + body="$(mktemp)" + last_code="$(curl -sS --max-time 20 -o "$body" -w '%{http_code}' "${BACKEND_URL}/health" || true)" + last_code="${last_code:-000}" + + if [ "$last_code" = "200" ]; then + # `// empty` also covers a payload with no build key at all, which is + # what an image predating build stamping returns. + last_sha="$(jq -r '.build.commit_sha // empty' "$body" 2>/dev/null || true)" + if [ "$last_sha" = "$sha" ]; then + rm -f "$body" + echo "deployed: ${BACKEND_URL}/health reports commit_sha $sha" + exit 0 + fi + fi + rm -f "$body" + + echo "attempt ${attempt}: status=${last_code} deployed_sha=${last_sha:-(none)} expected=${sha}" + sleep 15 +done + +echo "error: deploy did not converge within ${DEPLOY_TIMEOUT_SECS}s. /health last reported status=${last_code} commit_sha=${last_sha:-(none)}, expected ${sha}." >&2 +if [ "$last_code" != "200" ]; then + echo "error: /health never returned 200 (last status ${last_code}). The service is not serving; check the Railway deploy logs for a crashed or unhealthy rollout." >&2 +elif [ -z "$last_sha" ]; then + echo "error: /health answered but carried no commit_sha. Either the rollout reverted to an image predating build stamping, or build-sha.txt did not reach the image." >&2 +else + echo "error: a different commit is serving. The rollout reverted (railway.toml restartPolicyType=ON_FAILURE) or another deploy raced this one." >&2 +fi +exit 1 From 2945742966849dc03899aed91d59ccacc9e7021a Mon Sep 17 00:00:00 2001 From: Christopher Sellers <1@christophersellers.com> Date: Fri, 30 Oct 2026 00:00:25 -0700 Subject: [PATCH 2/5] fix(deploy): refuse to run deploy_prod.sh from a git worktree `railway up` fails at "Indexing... prefix not found" when `.git` is a worktree pointer file rather than a directory (observed on the 2026-10-29 hand deploy). Check for that right after resolving the repo root, before anything is stamped, with a message that says where to run it instead. Co-Authored-By: Claude Fable 5.1 --- scripts/railway/deploy_prod.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/railway/deploy_prod.sh b/scripts/railway/deploy_prod.sh index 52b05e93e..a90c2feb3 100755 --- a/scripts/railway/deploy_prod.sh +++ b/scripts/railway/deploy_prod.sh @@ -61,6 +61,13 @@ command -v git >/dev/null 2>&1 || die "git is required but not found in PATH" repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || die "not inside a git checkout" cd "$repo_root" +# `railway up` indexes the upload relative to the git toplevel and fails with +# "Indexing... prefix not found" when `.git` is a worktree pointer file instead +# of a directory (observed 2026-10-29). Refuse here rather than after stamping. +if [ ! -d "$repo_root/.git" ]; then + die "$repo_root is a git worktree (.git is a file). \`railway up\` cannot index a worktree; run this from the primary checkout." +fi + # --- Preconditions. Nothing is touched until all of these pass. ------------- dirty="$(git status --porcelain --untracked-files=all)" From 301b8c75b74f965a97df79dd02cb90eb835fd249 Mon Sep 17 00:00:00 2001 From: Christopher Sellers <1@christophersellers.com> Date: Fri, 30 Oct 2026 00:33:42 -0700 Subject: [PATCH 3/5] fix(deploy): name the production target explicitly; runbook status deploy_prod.sh now passes --project, --service and --environment to `railway up` with production defaults (overridable through RAILWAY_PROJECT, RAILWAY_SERVICE, RAILWAY_ENVIRONMENT) instead of trusting whatever this directory was last linked to, so a checkout linked to staging cannot upload there while the health poll watches production. The login check is `railway whoami`, since link state no longer matters. RELEASE_BRANCH_DEPLOYS.md sections 1 and 10 described deploy.yml as still deploying on push to main; that ended 2026-10-28 and section 11 has the manual process. Both now say so. Review: coderabbitai threads on deploy_prod.sh:137 and RELEASE_BRANCH_DEPLOYS.md:225. Co-Authored-By: Claude Fable 5.1 --- docs/runbooks/RELEASE_BRANCH_DEPLOYS.md | 25 ++++++++++++++----------- scripts/railway/deploy_prod.sh | 24 +++++++++++++++--------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md index 40057b460..46db2e171 100644 --- a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md +++ b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md @@ -7,12 +7,13 @@ workflow, Railway setting, or branch rule changes with this file. ## 1. Status -Production still deploys on every backend-relevant push to `main`. - -`.github/workflows/deploy.yml` ("Deploy to Railway") runs on `push` to -`main` for the backend path list, and on `workflow_dispatch`. It waits for -a green **Backend CI Status Gate** on that commit, then runs `railway up --ci` -against the production Railway service. Vercel deploys the frontend separately. +Until 2026-10-28, production deployed on every backend-relevant push to +`main`: `.github/workflows/deploy.yml` ("Deploy to Railway") ran on `push` +to `main` for the backend path list, and on `workflow_dispatch`, waited for +a green **Backend CI Status Gate** on that commit, then ran `railway up --ci` +against the production Railway service. That workflow is now disabled and +production deploys are manual; section 11 has the current process. Vercel +deploys the frontend separately. Nothing changes until the founder approves this model and a later pull request lands. @@ -215,8 +216,8 @@ run the audit-chain integrity check (section 6). ## 10. Decision needed Approve or reject this model, and pick the cadence: on demand, or at most -two or three times per weekday. Until then, production keeps deploying -from `main` through `deploy.yml`. +two or three times per weekday. Until then, production deploys by hand as +section 11 describes. ## 11. Manual deploys today (in effect, not part of the proposal) @@ -235,9 +236,11 @@ the live commit could only be identified by hashing files inside the container over `railway ssh`. The script refuses a dirty tree, stamps `git rev-parse HEAD` into -`build-sha.txt`, runs `railway up --ci` against the linked service, restores +`build-sha.txt`, runs `railway up --ci` against the production project, +service and environment named on the command line, restores the placeholder (also on failure), then polls `https://api.regengine.co/health` until `build.commit_sha` equals the stamped commit, with the same three failure diagnoses as `deploy.yml`. `--dry-run` does everything except the -upload and the poll. Set `RAILWAY_SERVICE` or `RAILWAY_ENVIRONMENT` to -override what `railway link` chose. +upload and the poll. `RAILWAY_PROJECT`, `RAILWAY_SERVICE` and +`RAILWAY_ENVIRONMENT` override the production defaults; `railway link` +state is ignored so a checkout linked to staging cannot redirect it. diff --git a/scripts/railway/deploy_prod.sh b/scripts/railway/deploy_prod.sh index a90c2feb3..377072806 100755 --- a/scripts/railway/deploy_prod.sh +++ b/scripts/railway/deploy_prod.sh @@ -13,7 +13,8 @@ # # 1. refuse to run with a dirty working tree # 2. write `git rev-parse HEAD` into build-sha.txt -# 3. `railway up --ci` against the linked Railway service +# 3. `railway up --ci` against the production project, environment and +# service, named explicitly so a stray `railway link` cannot redirect it # 4. restore the placeholder (also on failure, via trap) # 5. poll /health until build.commit_sha equals the stamped SHA # @@ -22,16 +23,20 @@ # scripts/railway/deploy_prod.sh --dry-run # everything except railway up # # and the health poll # -# Environment (all optional): +# Environment (all optional; the defaults are production): # BACKEND_URL health base URL (default https://api.regengine.co) -# RAILWAY_SERVICE passed as `railway up --service` when set -# RAILWAY_ENVIRONMENT passed as `railway up --environment` when set +# RAILWAY_PROJECT `railway up --project` (default RegEngine-Prod id) +# RAILWAY_SERVICE `railway up --service` (default RegEngine) +# RAILWAY_ENVIRONMENT `railway up --environment` (default production) # DEPLOY_TIMEOUT_SECS health poll budget (default 900, mirrors deploy.yml) set -euo pipefail PLACEHOLDER="unknown" STAMP_FILE="build-sha.txt" BACKEND_URL="${BACKEND_URL:-https://api.regengine.co}" +RAILWAY_PROJECT="${RAILWAY_PROJECT:-4e1d16d9-0d88-4b88-8eb9-5c0ae38e2b51}" +RAILWAY_SERVICE="${RAILWAY_SERVICE:-RegEngine}" +RAILWAY_ENVIRONMENT="${RAILWAY_ENVIRONMENT:-production}" DEPLOY_TIMEOUT_SECS="${DEPLOY_TIMEOUT_SECS:-900}" DRY_RUN=0 @@ -96,6 +101,7 @@ branch="$(git rev-parse --abbrev-ref HEAD)" echo "checkout: $repo_root" echo "branch: $branch" echo "commit: $sha" +echo "target: project=$RAILWAY_PROJECT service=$RAILWAY_SERVICE environment=$RAILWAY_ENVIRONMENT" if [ "$branch" != "main" ]; then echo "warning: HEAD is not on main. Production normally ships main; continuing because you asked." >&2 fi @@ -115,9 +121,10 @@ echo "stamped: $STAMP_FILE = $(cat "$STAMP_FILE")" # --- Deploy. ----------------------------------------------------------------- -up_args=(up --ci) -[ -n "${RAILWAY_SERVICE:-}" ] && up_args+=(--service "$RAILWAY_SERVICE") -[ -n "${RAILWAY_ENVIRONMENT:-}" ] && up_args+=(--environment "$RAILWAY_ENVIRONMENT") +# Always name the target. `railway up` otherwise uploads to whatever this +# directory was last linked to, which may be staging or another project, +# while the health poll below would still watch production. +up_args=(up --ci --project "$RAILWAY_PROJECT" --service "$RAILWAY_SERVICE" --environment "$RAILWAY_ENVIRONMENT") if [ "$DRY_RUN" -eq 1 ]; then echo "dry run: would run: railway ${up_args[*]}" @@ -131,8 +138,7 @@ if [ "$DRY_RUN" -eq 1 ]; then exit 0 fi -echo "target:" -railway status || die "railway status failed. Run \`railway login\` and \`railway link\` first." +railway whoami >/dev/null 2>&1 || die "not logged in to Railway. Run \`railway login\` first." railway "${up_args[@]}" From 7d7863b55f7a380152c36ce762e5547239a294fd Mon Sep 17 00:00:00 2001 From: Christopher Sellers <1@christophersellers.com> Date: Fri, 30 Oct 2026 00:44:24 -0700 Subject: [PATCH 4/5] fix(deploy): drop the railway whoami preflight A project-scoped RAILWAY_TOKEN can run `railway up` but cannot answer `railway whoami`, so the preflight blocked a valid token-based deploy. `railway up` reports a missing or wrong credential itself, and the trap restores the stamp when it fails. Review: coderabbitai thread on deploy_prod.sh:141. Co-Authored-By: Claude Fable 5.1 --- scripts/railway/deploy_prod.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/railway/deploy_prod.sh b/scripts/railway/deploy_prod.sh index 377072806..e1e9c8db0 100755 --- a/scripts/railway/deploy_prod.sh +++ b/scripts/railway/deploy_prod.sh @@ -138,8 +138,9 @@ if [ "$DRY_RUN" -eq 1 ]; then exit 0 fi -railway whoami >/dev/null 2>&1 || die "not logged in to Railway. Run \`railway login\` first." - +# No identity preflight: a project-scoped RAILWAY_TOKEN can deploy but cannot +# answer `railway whoami`. `railway up` reports a missing or wrong credential +# itself, and the trap restores the stamp when it does. railway "${up_args[@]}" # Restore as soon as the upload is accepted; the trap is only the safety net. From a2b8dba9784ef61e7a26009badc0b0e5373c7292 Mon Sep 17 00:00:00 2001 From: Christopher Sellers <1@christophersellers.com> Date: Fri, 30 Oct 2026 00:53:22 -0700 Subject: [PATCH 5/5] fix(deploy): CI gate, main-only default, full poll budget, accurate comments From Muse's review of this branch: - require a `success` "Backend CI Status Gate" check run on the commit before stamping, as deploy.yml's wait-for-ci did; `--skip-ci-gate` overrides knowingly, and a missing or non-success gate refuses with the reason - refuse a commit that is not on main unless `--allow-non-main` is passed, instead of warning and continuing - reset SECONDS before the health poll so the build's own duration does not eat the 900 s budget - state that `railway up --ci` exits when the build completes, not when the upload is accepted; note that api.regengine.co fronts the same deployment deploy.yml polls by Railway hostname, and that the CLI resolves service and environment by name where deploy.yml passes ids - runbook section 11 now lists the worktree refusal, the CI gate, the two override flags and the BACKEND_URL override Co-Authored-By: Claude Fable 5.1 --- docs/runbooks/RELEASE_BRANCH_DEPLOYS.md | 23 ++++++---- scripts/railway/deploy_prod.sh | 61 +++++++++++++++++++++---- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md index 46db2e171..73b25766d 100644 --- a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md +++ b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md @@ -235,12 +235,17 @@ The 2026-10-29 deploy (Railway deployment `44c0486a`) did exactly that, and the live commit could only be identified by hashing files inside the container over `railway ssh`. -The script refuses a dirty tree, stamps `git rev-parse HEAD` into -`build-sha.txt`, runs `railway up --ci` against the production project, -service and environment named on the command line, restores -the placeholder (also on failure), then polls `https://api.regengine.co/health` -until `build.commit_sha` equals the stamped commit, with the same three -failure diagnoses as `deploy.yml`. `--dry-run` does everything except the -upload and the poll. `RAILWAY_PROJECT`, `RAILWAY_SERVICE` and -`RAILWAY_ENVIRONMENT` override the production defaults; `railway link` -state is ignored so a checkout linked to staging cannot redirect it. +The script refuses a dirty tree, a git worktree (`railway up` cannot index +one), and a commit that is not on `main` unless `--allow-non-main` is passed. +It then requires a `success` **Backend CI Status Gate** check run on the +commit, as `deploy.yml` did (`--skip-ci-gate` overrides, knowingly), stamps +`git rev-parse HEAD` into `build-sha.txt`, runs `railway up --ci` against the +production project, service and environment named on the command line, +restores the placeholder (also on failure), then polls `BACKEND_URL/health` +(default `https://api.regengine.co`, the same deployment `deploy.yml` polls +by its Railway hostname) until `build.commit_sha` equals the stamped commit, +with the same three failure diagnoses as `deploy.yml`. `--dry-run` does +everything except the upload and the poll. `RAILWAY_PROJECT`, +`RAILWAY_SERVICE` and `RAILWAY_ENVIRONMENT` override the production +defaults; `railway link` state is ignored so a checkout linked to staging +cannot redirect it. diff --git a/scripts/railway/deploy_prod.sh b/scripts/railway/deploy_prod.sh index e1e9c8db0..5e060999a 100755 --- a/scripts/railway/deploy_prod.sh +++ b/scripts/railway/deploy_prod.sh @@ -19,32 +19,49 @@ # 5. poll /health until build.commit_sha equals the stamped SHA # # Usage: -# scripts/railway/deploy_prod.sh # real deploy -# scripts/railway/deploy_prod.sh --dry-run # everything except railway up -# # and the health poll +# scripts/railway/deploy_prod.sh # real deploy +# scripts/railway/deploy_prod.sh --dry-run # everything except railway up +# # and the health poll +# scripts/railway/deploy_prod.sh --allow-non-main # ship a commit that is not +# # on main (otherwise refused) +# scripts/railway/deploy_prod.sh --skip-ci-gate # do not require a passing +# # "Backend CI Status Gate" # # Environment (all optional; the defaults are production): -# BACKEND_URL health base URL (default https://api.regengine.co) +# BACKEND_URL health base URL (default https://api.regengine.co, +# the custom domain; deploy.yml polls the raw Railway +# hostname, which fronts the same deployment) # RAILWAY_PROJECT `railway up --project` (default RegEngine-Prod id) # RAILWAY_SERVICE `railway up --service` (default RegEngine) # RAILWAY_ENVIRONMENT `railway up --environment` (default production) # DEPLOY_TIMEOUT_SECS health poll budget (default 900, mirrors deploy.yml) +# CI_GATE_NAME check run that must be success (default +# "Backend CI Status Gate", as deploy.yml waits for) set -euo pipefail PLACEHOLDER="unknown" STAMP_FILE="build-sha.txt" BACKEND_URL="${BACKEND_URL:-https://api.regengine.co}" +# deploy.yml passes Railway ids from secrets; the CLI also resolves service +# and environment by name within the project (the 2026-10-29 hand deploy used +# exactly these names). The RAILWAY_*_ID variables deploy.yml reads are not +# consulted here: set RAILWAY_PROJECT / RAILWAY_SERVICE / RAILWAY_ENVIRONMENT. RAILWAY_PROJECT="${RAILWAY_PROJECT:-4e1d16d9-0d88-4b88-8eb9-5c0ae38e2b51}" RAILWAY_SERVICE="${RAILWAY_SERVICE:-RegEngine}" RAILWAY_ENVIRONMENT="${RAILWAY_ENVIRONMENT:-production}" DEPLOY_TIMEOUT_SECS="${DEPLOY_TIMEOUT_SECS:-900}" +CI_GATE_NAME="${CI_GATE_NAME:-Backend CI Status Gate}" DRY_RUN=0 +ALLOW_NON_MAIN=0 +SKIP_CI_GATE=0 for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=1 ;; + --allow-non-main) ALLOW_NON_MAIN=1 ;; + --skip-ci-gate) SKIP_CI_GATE=1 ;; -h|--help) - sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' + sed -n '2,39p' "$0" | sed 's/^# \{0,1\}//' exit 0 ;; *) @@ -94,6 +111,11 @@ if [ "$DRY_RUN" -eq 0 ]; then command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not found in PATH" done fi +if [ "$SKIP_CI_GATE" -eq 0 ]; then + for tool in gh jq; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required for the CI gate check (or pass --skip-ci-gate)" + done +fi sha="$(git rev-parse HEAD)" branch="$(git rev-parse --abbrev-ref HEAD)" @@ -102,8 +124,25 @@ echo "checkout: $repo_root" echo "branch: $branch" echo "commit: $sha" echo "target: project=$RAILWAY_PROJECT service=$RAILWAY_SERVICE environment=$RAILWAY_ENVIRONMENT" -if [ "$branch" != "main" ]; then - echo "warning: HEAD is not on main. Production normally ships main; continuing because you asked." >&2 +if [ "$branch" != "main" ] && [ "$ALLOW_NON_MAIN" -eq 0 ]; then + die "HEAD is on '$branch', not main. Production ships main; pass --allow-non-main to ship this commit knowingly." +fi + +# --- CI gate (mirrors deploy.yml's wait-for-ci step). ------------------------ +# +# deploy.yml refused to upload a commit whose "Backend CI Status Gate" was not +# success (skipped does not count). A hand deploy must not be looser. + +if [ "$SKIP_CI_GATE" -eq 0 ]; then + gate_json="$(gh api -X GET "repos/{owner}/{repo}/commits/${sha}/check-runs" -f check_name="$CI_GATE_NAME" -f per_page=100 2>/dev/null)" \ + || die "could not read check runs for $sha from GitHub. Is the commit pushed and is \`gh auth status\` ok? Pass --skip-ci-gate to override knowingly." + gate_count="$(printf '%s' "$gate_json" | jq -r '.check_runs | length')" + [ "$gate_count" -gt 0 ] || die "no \"$CI_GATE_NAME\" check run exists for $sha. Let CI finish on this commit first, or pass --skip-ci-gate to override knowingly." + gate_conclusion="$(printf '%s' "$gate_json" | jq -r '.check_runs | sort_by(.completed_at // "") | last | .conclusion // "pending"')" + [ "$gate_conclusion" = "success" ] || die "\"$CI_GATE_NAME\" for $sha is '$gate_conclusion', not success. Only a passing gate clears a deploy; pass --skip-ci-gate to override knowingly." + echo "ci gate: $CI_GATE_NAME = success" +else + echo "ci gate: skipped (--skip-ci-gate)" fi # --- Stamp, with a trap so the placeholder always comes back. ---------------- @@ -143,19 +182,21 @@ fi # itself, and the trap restores the stamp when it does. railway "${up_args[@]}" -# Restore as soon as the upload is accepted; the trap is only the safety net. +# Restore as soon as `railway up` returns; the trap is only the safety net. restore_stamp trap - EXIT echo "restored: $STAMP_FILE = $(cat "$STAMP_FILE")" # --- Verify identity, not behaviour (mirrors deploy.yml). -------------------- # -# `railway up --ci` returns once the upload is accepted, not once the new -# image is serving. railway.toml sets restartPolicyType = "ON_FAILURE" with a +# `railway up --ci` streams the build and exits when the build completes, +# before the new image has passed its healthcheck and taken traffic. +# railway.toml sets restartPolicyType = "ON_FAILURE" with a # /readiness healthcheck, so a rollout that fails its healthcheck reverts to # the previous image after this script would otherwise have exited 0. The # only check that cannot be satisfied by an older image is the commit itself. +SECONDS=0 # the build above already consumed some; the poll gets its full budget deadline=$(( SECONDS + DEPLOY_TIMEOUT_SECS )) attempt=0 last_sha=""