diff --git a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md index 6ca30ee3e..73b25766d 100644 --- a/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md +++ b/docs/runbooks/RELEASE_BRANCH_DEPLOYS.md @@ -7,12 +7,13 @@ workflow, Railway setting, or branch rule changes with this file. ## 1. Status -Production still deploys on every backend-relevant push to `main`. - -`.github/workflows/deploy.yml` ("Deploy to Railway") runs on `push` to -`main` for the backend path list, and on `workflow_dispatch`. It waits for -a green **Backend CI Status Gate** on that commit, then runs `railway up --ci` -against the production Railway service. Vercel deploys the frontend separately. +Until 2026-10-28, production deployed on every backend-relevant push to +`main`: `.github/workflows/deploy.yml` ("Deploy to Railway") ran on `push` +to `main` for the backend path list, and on `workflow_dispatch`, waited for +a green **Backend CI Status Gate** on that commit, then ran `railway up --ci` +against the production Railway service. That workflow is now disabled and +production deploys are manual; section 11 has the current process. Vercel +deploys the frontend separately. Nothing changes until the founder approves this model and a later pull request lands. @@ -215,5 +216,36 @@ run the audit-chain integrity check (section 6). ## 10. Decision needed Approve or reject this model, and pick the cadence: on demand, or at most -two or three times per weekday. Until then, production keeps deploying -from `main` through `deploy.yml`. +two or three times per weekday. Until then, production deploys by hand as +section 11 describes. + +## 11. Manual deploys today (in effect, not part of the proposal) + +Since 2026-10-28 the Railway production service has no GitHub source and +`deploy.yml` is disabled, so every production deploy is a hand-run +`railway up` from a local checkout. A hand deploy must go through +`scripts/railway/deploy_prod.sh`, never a bare `railway up`. + +Reason: the image has no `.git` (`.dockerignore` excludes it), so the only +commit identity it carries is `build-sha.txt`, which the `Dockerfile` copies +to `/app/build-sha.txt`. The tracked file is the placeholder `unknown`. +`deploy.yml` overwrites it with the commit before `railway up`; a bare +`railway up` ships the placeholder and `/health` reports `commit_sha: null`. +The 2026-10-29 deploy (Railway deployment `44c0486a`) did exactly that, and +the live commit could only be identified by hashing files inside the +container over `railway ssh`. + +The script refuses a dirty tree, a git worktree (`railway up` cannot index +one), and a commit that is not on `main` unless `--allow-non-main` is passed. +It then requires a `success` **Backend CI Status Gate** check run on the +commit, as `deploy.yml` did (`--skip-ci-gate` overrides, knowingly), stamps +`git rev-parse HEAD` into `build-sha.txt`, runs `railway up --ci` against the +production project, service and environment named on the command line, +restores the placeholder (also on failure), then polls `BACKEND_URL/health` +(default `https://api.regengine.co`, the same deployment `deploy.yml` polls +by its Railway hostname) until `build.commit_sha` equals the stamped commit, +with the same three failure diagnoses as `deploy.yml`. `--dry-run` does +everything except the upload and the poll. `RAILWAY_PROJECT`, +`RAILWAY_SERVICE` and `RAILWAY_ENVIRONMENT` override the production +defaults; `railway link` state is ignored so a checkout linked to staging +cannot redirect it. diff --git a/scripts/railway/deploy_prod.sh b/scripts/railway/deploy_prod.sh new file mode 100755 index 000000000..5e060999a --- /dev/null +++ b/scripts/railway/deploy_prod.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# Hand-run production deploy that stamps the commit the same way +# .github/workflows/deploy.yml does. +# +# Why this exists: the image has no .git (.dockerignore excludes it), so the +# only way /health can report which commit is serving is the build-sha.txt +# the Dockerfile copies to /app/build-sha.txt. The tracked file is the +# placeholder "unknown" so a plain checkout never carries a stale SHA. The +# workflow overwrites it with github.sha before `railway up`; a bare +# `railway up` from a laptop ships the placeholder and /health answers +# commit_sha: null (that is what the 2026-10-29 deploy did). This script +# closes that gap for manual deploys: +# +# 1. refuse to run with a dirty working tree +# 2. write `git rev-parse HEAD` into build-sha.txt +# 3. `railway up --ci` against the production project, environment and +# service, named explicitly so a stray `railway link` cannot redirect it +# 4. restore the placeholder (also on failure, via trap) +# 5. poll /health until build.commit_sha equals the stamped SHA +# +# Usage: +# scripts/railway/deploy_prod.sh # real deploy +# scripts/railway/deploy_prod.sh --dry-run # everything except railway up +# # and the health poll +# scripts/railway/deploy_prod.sh --allow-non-main # ship a commit that is not +# # on main (otherwise refused) +# scripts/railway/deploy_prod.sh --skip-ci-gate # do not require a passing +# # "Backend CI Status Gate" +# +# Environment (all optional; the defaults are production): +# BACKEND_URL health base URL (default https://api.regengine.co, +# the custom domain; deploy.yml polls the raw Railway +# hostname, which fronts the same deployment) +# RAILWAY_PROJECT `railway up --project` (default RegEngine-Prod id) +# RAILWAY_SERVICE `railway up --service` (default RegEngine) +# RAILWAY_ENVIRONMENT `railway up --environment` (default production) +# DEPLOY_TIMEOUT_SECS health poll budget (default 900, mirrors deploy.yml) +# CI_GATE_NAME check run that must be success (default +# "Backend CI Status Gate", as deploy.yml waits for) +set -euo pipefail + +PLACEHOLDER="unknown" +STAMP_FILE="build-sha.txt" +BACKEND_URL="${BACKEND_URL:-https://api.regengine.co}" +# deploy.yml passes Railway ids from secrets; the CLI also resolves service +# and environment by name within the project (the 2026-10-29 hand deploy used +# exactly these names). The RAILWAY_*_ID variables deploy.yml reads are not +# consulted here: set RAILWAY_PROJECT / RAILWAY_SERVICE / RAILWAY_ENVIRONMENT. +RAILWAY_PROJECT="${RAILWAY_PROJECT:-4e1d16d9-0d88-4b88-8eb9-5c0ae38e2b51}" +RAILWAY_SERVICE="${RAILWAY_SERVICE:-RegEngine}" +RAILWAY_ENVIRONMENT="${RAILWAY_ENVIRONMENT:-production}" +DEPLOY_TIMEOUT_SECS="${DEPLOY_TIMEOUT_SECS:-900}" +CI_GATE_NAME="${CI_GATE_NAME:-Backend CI Status Gate}" +DRY_RUN=0 +ALLOW_NON_MAIN=0 +SKIP_CI_GATE=0 + +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=1 ;; + --allow-non-main) ALLOW_NON_MAIN=1 ;; + --skip-ci-gate) SKIP_CI_GATE=1 ;; + -h|--help) + sed -n '2,39p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "error: unknown argument: $arg (try --help)" >&2 + exit 2 + ;; + esac +done + +die() { + echo "error: $*" >&2 + exit 1 +} + +command -v git >/dev/null 2>&1 || die "git is required but not found in PATH" + +# Everything below runs from the repository root: the Dockerfile and the +# stamp file live there, and `railway up` uploads the current directory. +repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || die "not inside a git checkout" +cd "$repo_root" + +# `railway up` indexes the upload relative to the git toplevel and fails with +# "Indexing... prefix not found" when `.git` is a worktree pointer file instead +# of a directory (observed 2026-10-29). Refuse here rather than after stamping. +if [ ! -d "$repo_root/.git" ]; then + die "$repo_root is a git worktree (.git is a file). \`railway up\` cannot index a worktree; run this from the primary checkout." +fi + +# --- Preconditions. Nothing is touched until all of these pass. ------------- + +dirty="$(git status --porcelain --untracked-files=all)" +if [ -n "$dirty" ]; then + echo "error: working tree is not clean. A deploy must ship exactly one" >&2 + echo "commit, and \`railway up\` uploads whatever is on disk, tracked or not." >&2 + echo "Commit, discard, or move these first:" >&2 + printf '%s\n' "$dirty" >&2 + exit 1 +fi + +[ -f "$STAMP_FILE" ] || die "$STAMP_FILE is missing from the checkout" +if [ "$(cat "$STAMP_FILE")" != "$PLACEHOLDER" ]; then + die "$STAMP_FILE is tracked as '$(cat "$STAMP_FILE")', expected the placeholder '$PLACEHOLDER'. Fix the tracked file before deploying." +fi + +if [ "$DRY_RUN" -eq 0 ]; then + for tool in railway curl jq; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not found in PATH" + done +fi +if [ "$SKIP_CI_GATE" -eq 0 ]; then + for tool in gh jq; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required for the CI gate check (or pass --skip-ci-gate)" + done +fi + +sha="$(git rev-parse HEAD)" +branch="$(git rev-parse --abbrev-ref HEAD)" + +echo "checkout: $repo_root" +echo "branch: $branch" +echo "commit: $sha" +echo "target: project=$RAILWAY_PROJECT service=$RAILWAY_SERVICE environment=$RAILWAY_ENVIRONMENT" +if [ "$branch" != "main" ] && [ "$ALLOW_NON_MAIN" -eq 0 ]; then + die "HEAD is on '$branch', not main. Production ships main; pass --allow-non-main to ship this commit knowingly." +fi + +# --- CI gate (mirrors deploy.yml's wait-for-ci step). ------------------------ +# +# deploy.yml refused to upload a commit whose "Backend CI Status Gate" was not +# success (skipped does not count). A hand deploy must not be looser. + +if [ "$SKIP_CI_GATE" -eq 0 ]; then + gate_json="$(gh api -X GET "repos/{owner}/{repo}/commits/${sha}/check-runs" -f check_name="$CI_GATE_NAME" -f per_page=100 2>/dev/null)" \ + || die "could not read check runs for $sha from GitHub. Is the commit pushed and is \`gh auth status\` ok? Pass --skip-ci-gate to override knowingly." + gate_count="$(printf '%s' "$gate_json" | jq -r '.check_runs | length')" + [ "$gate_count" -gt 0 ] || die "no \"$CI_GATE_NAME\" check run exists for $sha. Let CI finish on this commit first, or pass --skip-ci-gate to override knowingly." + gate_conclusion="$(printf '%s' "$gate_json" | jq -r '.check_runs | sort_by(.completed_at // "") | last | .conclusion // "pending"')" + [ "$gate_conclusion" = "success" ] || die "\"$CI_GATE_NAME\" for $sha is '$gate_conclusion', not success. Only a passing gate clears a deploy; pass --skip-ci-gate to override knowingly." + echo "ci gate: $CI_GATE_NAME = success" +else + echo "ci gate: skipped (--skip-ci-gate)" +fi + +# --- Stamp, with a trap so the placeholder always comes back. ---------------- + +restore_stamp() { + # `git checkout --` restores the exact tracked bytes, which is what keeps + # `git status` clean afterwards. The printf fallback covers a checkout + # whose index is unreadable mid-failure. + git checkout --quiet -- "$STAMP_FILE" 2>/dev/null || printf '%s\n' "$PLACEHOLDER" > "$STAMP_FILE" +} +trap restore_stamp EXIT + +printf '%s\n' "$sha" > "$STAMP_FILE" +echo "stamped: $STAMP_FILE = $(cat "$STAMP_FILE")" + +# --- Deploy. ----------------------------------------------------------------- + +# Always name the target. `railway up` otherwise uploads to whatever this +# directory was last linked to, which may be staging or another project, +# while the health poll below would still watch production. +up_args=(up --ci --project "$RAILWAY_PROJECT" --service "$RAILWAY_SERVICE" --environment "$RAILWAY_ENVIRONMENT") + +if [ "$DRY_RUN" -eq 1 ]; then + echo "dry run: would run: railway ${up_args[*]}" + echo "dry run: would poll ${BACKEND_URL}/health for build.commit_sha == $sha (budget ${DEPLOY_TIMEOUT_SECS}s)" + restore_stamp + trap - EXIT + if [ -n "$(git status --porcelain --untracked-files=all)" ]; then + die "dry run left the tree dirty; this is a bug in $0" + fi + echo "restored: $STAMP_FILE = $(cat "$STAMP_FILE"); git status clean" + exit 0 +fi + +# No identity preflight: a project-scoped RAILWAY_TOKEN can deploy but cannot +# answer `railway whoami`. `railway up` reports a missing or wrong credential +# itself, and the trap restores the stamp when it does. +railway "${up_args[@]}" + +# Restore as soon as `railway up` returns; the trap is only the safety net. +restore_stamp +trap - EXIT +echo "restored: $STAMP_FILE = $(cat "$STAMP_FILE")" + +# --- Verify identity, not behaviour (mirrors deploy.yml). -------------------- +# +# `railway up --ci` streams the build and exits when the build completes, +# before the new image has passed its healthcheck and taken traffic. +# railway.toml sets restartPolicyType = "ON_FAILURE" with a +# /readiness healthcheck, so a rollout that fails its healthcheck reverts to +# the previous image after this script would otherwise have exited 0. The +# only check that cannot be satisfied by an older image is the commit itself. + +SECONDS=0 # the build above already consumed some; the poll gets its full budget +deadline=$(( SECONDS + DEPLOY_TIMEOUT_SECS )) +attempt=0 +last_sha="" +last_code="" + +while [ "$SECONDS" -lt "$deadline" ]; do + attempt=$(( attempt + 1 )) + body="$(mktemp)" + last_code="$(curl -sS --max-time 20 -o "$body" -w '%{http_code}' "${BACKEND_URL}/health" || true)" + last_code="${last_code:-000}" + + if [ "$last_code" = "200" ]; then + # `// empty` also covers a payload with no build key at all, which is + # what an image predating build stamping returns. + last_sha="$(jq -r '.build.commit_sha // empty' "$body" 2>/dev/null || true)" + if [ "$last_sha" = "$sha" ]; then + rm -f "$body" + echo "deployed: ${BACKEND_URL}/health reports commit_sha $sha" + exit 0 + fi + fi + rm -f "$body" + + echo "attempt ${attempt}: status=${last_code} deployed_sha=${last_sha:-(none)} expected=${sha}" + sleep 15 +done + +echo "error: deploy did not converge within ${DEPLOY_TIMEOUT_SECS}s. /health last reported status=${last_code} commit_sha=${last_sha:-(none)}, expected ${sha}." >&2 +if [ "$last_code" != "200" ]; then + echo "error: /health never returned 200 (last status ${last_code}). The service is not serving; check the Railway deploy logs for a crashed or unhealthy rollout." >&2 +elif [ -z "$last_sha" ]; then + echo "error: /health answered but carried no commit_sha. Either the rollout reverted to an image predating build stamping, or build-sha.txt did not reach the image." >&2 +else + echo "error: a different commit is serving. The rollout reverted (railway.toml restartPolicyType=ON_FAILURE) or another deploy raced this one." >&2 +fi +exit 1