From e5098e82895910acf1c29f6b99034c87476fc6c2 Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 04:02:48 +0200 Subject: [PATCH] feat(deploy): stamp both dashboard images with their git revision (#3315) Deploy verification is inference today: compare `docker images` creation time with the merge time, or grep a string out of the shipped binary. `apiary-backend:latest` had `Labels: null`, so every merge since the last rebuild looked identical to the one before it, and docs/ARCANE-GIT-SYNC.md already names the state this produces -- "green, healthy, running the old code". Both images now carry the revision three ways: - backend-service compiles it in (build.rs: GIT_SHA -> APIARY_GIT_SHA) and /healthz returns it as `revision`, so the question becomes a curl rather than a comparison of timestamps. build.rs also emits `cargo:rerun-if-env-changed=GIT_SHA`; without that, cargo reuses a cached build and the second build of an identical tree reports the first revision -- the exact failure this fixes, reproduced by the fix. - frontend-next bakes it into a static /build.json, generated in the build stage before `npm run build` so vite copies it into the output. A static file rather than a route because Nitro serving from public/ is already proven in this image (the container healthcheck reads /static/theme.css the same way). - both Dockerfiles declare ARG GIT_SHA per stage and set org.opencontainers.image.{revision,source,created}. Both tiers normalize to a bare lowercase hex object name or "unknown", and are held to one shared table (backend-service/src/revision-corpus.json) so two normalizers in two languages, in two CI lanes, cannot drift -- the failure mode of their disagreeing is a deploy mismatch that is not one. scripts/verify-deploy.sh compares the live /healthz revision and the image labels against an expected revision, defaulting to origin/main. Its exit codes are the point: 0 agrees, 1 is a finding, and 2 is "could not tell" -- deliberately distinct, so an unreadable body or an image that is not on the host can never read as a stale deploy. diagnostics.yml runs it in --warn-only mode, against $GITHUB_SHA, because the home runner has no clone to measure against (deploy.yml rsyncs into /opt/stacks/apiary with --exclude .git/). Unset stays the honest default: nothing on the homeserver writes GIT_SHA yet, so the first run reports "not stamped", which is true, and the docs say how to turn it on. --- .github/workflows/containers.yml | 22 + .github/workflows/diagnostics.yml | 49 ++ .../honeypot-dashboard-backend/.env.example | 23 + .../honeypot-dashboard-backend/compose.yml | 20 +- arcane/home/honeypot-dashboard/.env.example | 13 + .../backend-service/Dockerfile | 36 +- .../backend-service/build.rs | 25 +- .../backend-service/src/main.rs | 120 +++- .../backend-service/src/revision-corpus.json | 23 + arcane/home/honeypot-dashboard/compose.yml | 14 +- .../frontend-next/.gitignore | 6 + .../frontend-next/Dockerfile | 39 ++ .../scripts/write-build-info.mjs | 84 +++ docs/ARCANE-GIT-SYNC.md | 112 ++++ scripts/tests/test_3315_image_revision.py | 576 ++++++++++++++++++ scripts/verify-deploy.sh | 374 ++++++++++++ 16 files changed, 1528 insertions(+), 8 deletions(-) create mode 100644 arcane/home/honeypot-dashboard/backend-service/src/revision-corpus.json create mode 100644 arcane/home/honeypot-dashboard/frontend-next/scripts/write-build-info.mjs create mode 100644 scripts/tests/test_3315_image_revision.py create mode 100755 scripts/verify-deploy.sh diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 5ffdc9942..39111e27d 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -66,6 +66,15 @@ jobs: # into the digest-keyed CycloneDX + Trivy steps below. Absent on # the other sixteen rows, so they build exactly as before. sbom: true + # #3315: the two dashboard images that now report which commit + # they were built from -- one compiled into the binary (where + # /healthz answers it as `revision`) or baked into the served + # /build.json, and both on the image's + # org.opencontainers.image.revision label. Only these two + # Dockerfiles declare `ARG GIT_SHA`, so only these two rows pass + # it; the other sixteen would take an unused-build-arg warning + # per build for an arg they do not read. + stamp: true - image: cisco-asa-honeypot context: arcane/home/honeypot-cisco-asa-honeypot/cisco-asa-honeypot - image: citrix-honeypot @@ -75,6 +84,7 @@ jobs: - image: dashboard-next context: arcane/home/honeypot-dashboard/frontend-next sbom: true + stamp: true - image: dicompot context: arcane/home/honeypot-dicompot/dicompot - image: dionaea @@ -236,6 +246,18 @@ jobs: load: false tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} + # #3315: the revision this image is built from, for the two rows + # whose Dockerfile declares `ARG GIT_SHA` (matrix.stamp). It has to + # be a build arg and not only a label: backend-service compiles it + # into the binary, and nothing in the shipped image can be edited + # after the fact without changing the image id. Note that + # metadata-action already emits its own org.opencontainers.revision + # for the same image, and both are wired from the same + # `${{ github.sha }}` here -- so whichever key wins the merge, the + # label and the compiled-in value cannot disagree about which commit + # this is. scripts/verify-deploy.sh compares them against origin/main + # and against the live /healthz. + build-args: ${{ matrix.stamp == true && format('GIT_SHA={0}', github.sha) || '' }} # Scope the layer cache per image (#2771). Without an explicit # scope every matrix row defaults to `buildkit`, so all 18 # concurrent builds read and write ONE shared cache index per diff --git a/.github/workflows/diagnostics.yml b/.github/workflows/diagnostics.yml index 60268acbe..dad665304 100644 --- a/.github/workflows/diagnostics.yml +++ b/.github/workflows/diagnostics.yml @@ -145,6 +145,55 @@ jobs: alert "healthz: **unreachable** — the dashboard container is down or not bound" fi + # #3315: "green, healthy, running the old code" is indistinguishable + # from a correct deploy by looking at it -- a fresh container, a + # passing healthcheck and a recent image build all describe the + # machinery, not the code. Both dashboard images now carry their git + # revision (#/healthz's `revision` field and + # org.opencontainers.image.revision), and scripts/verify-deploy.sh + # compares what is running against what is newest. + # + # The expected revision is $GITHUB_SHA rather than a clone's + # origin/main because this host has no usable one: deploy.yml's + # rsync into $stack carries --exclude .git/ (see its own step), and + # Arcane's directory sync has no .git either. On a schedule trigger + # GITHUB_SHA is the default branch's head, which is exactly the + # "newest merge" this has to be measured against. For the same + # reason --behind-days is 0 here: the day-count needs a clone to + # measure commit age against, and without one it would exit 2 + # ("could not tell") on every run. Degrade to "is this the tip of + # main" and say so, rather than shipping a check that can only + # report that it failed to run. + # + # Report-only, like the Elasticsearch section: a stale deploy is a + # scheduling fact, not one of the failures #2222 lists. --warn-only + # stops the tool's own exit 1 from reddening a scheduled run; its + # exit 2 is still separated out below, because folding "could not + # tell" into a pass is the one outcome that would make this section + # worse than not having it. + section "Deployed revision" + verify=$stack/scripts/verify-deploy.sh + if [ ! -x "$verify" ]; then + report "not checked: $verify is not on this host yet (it arrives with the merge that adds it)" + else + if stamp=$("$verify" --warn-only --repo-root "$stack" \ + --healthz-exec 'docker exec hp-apiary-backend curl -sf http://127.0.0.1:8081/healthz' \ + --image apiary-backend:latest --behind-days 0 "$GITHUB_SHA" 2>&1); then + printf '%s\n' "$stamp" | tee -a "$GITHUB_STEP_SUMMARY" + else + stamp_rc=$? + printf '%s\n' "$stamp" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$stamp_rc" -eq 2 ]; then + report "could not tell: the deployed revision was not readable (exit 2), which is not a pass" + else + report "findings above: the running revision is not verifiably the expected one" + fi + fi + report 'PASS means the running binary was built from the current main tip.' + report 'A finding means it was not; "not stamped" means the image was built' + report 'without --build-arg GIT_SHA. Neither reddens this run.' + fi + section "Sensor to Elasticsearch to Dashboard" # #2096: the Go dashboard's Prometheus /metrics died with it # (#1659) -- frontend-next owns :19090 now and answers 307 diff --git a/arcane/home/honeypot-dashboard-backend/.env.example b/arcane/home/honeypot-dashboard-backend/.env.example index 576359e3a..cc481c6b8 100644 --- a/arcane/home/honeypot-dashboard-backend/.env.example +++ b/arcane/home/honeypot-dashboard-backend/.env.example @@ -83,3 +83,26 @@ LLM_MODEL=qwen3:14b # the server default would evict a worker that runs continuously to serve # one that does not. LLM_KEEP_ALIVE=5m + +# #3315 -- the git revision apiary-backend is built from. Optional: unset +# means the image is built with no revision, /healthz reports +# {"revision":"unknown"} and the image's org.opencontainers.image.revision +# label is "unknown", which scripts/verify-deploy.sh reports as an unstamped +# deploy rather than passing silently. +# +# Take the value from the checkout you are actually deploying -- the commit +# that was merged, not the branch name. Not from /opt/stacks/apiary, which +# looks like a clone but is a `git ls-files`/rsync of the working tree with +# .git/ excluded (see .github/workflows/deploy.yml's own sync step), so +# `git -C /opt/stacks/apiary rev-parse HEAD` there fails. From a real clone: +# echo "GIT_SHA=$(git rev-parse HEAD)" >> .env +# For an Arcane-driven build that is the only place it has to be set: the +# synced directory has no .git, and compose will not invent one. For a manual +# `docker compose build` in the synced directory, the environment works too: +# GIT_SHA=$(git rev-parse HEAD) docker compose build backend-service +# +# It is a revision, not a branch: the whole point is that "is the merge on the +# host?" becomes a curl of /healthz rather than an inference from image +# timestamps. Never a branch name, a tag, or a hand-typed guess -- a value that +# is not an object name is normalized to "unknown" by the binary itself. +GIT_SHA= diff --git a/arcane/home/honeypot-dashboard-backend/compose.yml b/arcane/home/honeypot-dashboard-backend/compose.yml index 23575aa38..1caa8766a 100644 --- a/arcane/home/honeypot-dashboard-backend/compose.yml +++ b/arcane/home/honeypot-dashboard-backend/compose.yml @@ -79,7 +79,25 @@ services: # ------------------------------------------------------------------ backend-service: <<: *runtime-defaults - build: ../honeypot-dashboard/backend-service + build: + context: ../honeypot-dashboard/backend-service + # #3315: the revision this image is built from. build.rs compiles it into + # the binary (where /healthz reports it as `revision`) and the Dockerfile + # stamps it on the image as org.opencontainers.image.revision, so the two + # can be compared against each other and against main by + # scripts/verify-deploy.sh. + # + # Read from this stack's .env, not invented here: compose cannot run + # `git rev-parse`, and Arcane's directory sync explicitly excludes .git + # from the tree it materializes (docs/ARCANE-GIT-SYNC.md), so the synced + # checkout has no repository to ask. Set it from the checkout you are + # deploying, before the build -- see this stack's .env.example. + # + # Unset is the honest default and yields "unknown" in /healthz and on the + # label rather than a build failure. A build that refuses to produce an + # unstamped image is a build that gets reverted; a stale one is reported. + args: + GIT_SHA: ${GIT_SHA:-} image: apiary-backend:latest # build: this image is never pushed to any registry -- Arcane's # redeploy does an unconditional `docker compose pull` across every diff --git a/arcane/home/honeypot-dashboard/.env.example b/arcane/home/honeypot-dashboard/.env.example index 03bef6dda..c4d061988 100644 --- a/arcane/home/honeypot-dashboard/.env.example +++ b/arcane/home/honeypot-dashboard/.env.example @@ -265,3 +265,16 @@ DASHBOARD_BFF_LOG_FILE=/logs/dashboard-bff/app.jsonl # value into this stack's .env. Set it by hand only for a manual stand-up: # getent group deploy-runner | cut -d: -f3 DEPLOY_RUNNER_GID=980 + +# #3315 -- the git revision apiary-dashboard-next is built from. Optional and +# empty by default; see the identical block in +# ../honeypot-dashboard-backend/.env.example for the full reasoning. Briefly: +# it lands in the image's /build.json and its +# org.opencontainers.image.revision label, and +# scripts/verify-deploy.sh reports a missing one as an unstamped deploy +# instead of passing silently. +# +# The two dashboard stacks carry the variable independently: a build of one +# says nothing about the other, so set it in both .env files. +# echo "GIT_SHA=$(git rev-parse HEAD)" >> .env +GIT_SHA= diff --git a/arcane/home/honeypot-dashboard/backend-service/Dockerfile b/arcane/home/honeypot-dashboard/backend-service/Dockerfile index d47d4e24b..0123d7f0d 100644 --- a/arcane/home/honeypot-dashboard/backend-service/Dockerfile +++ b/arcane/home/honeypot-dashboard/backend-service/Dockerfile @@ -3,12 +3,22 @@ # (no OpenSSL); ca-certificates + curl for TLS trust and the container # healthcheck. FROM rust:1-slim-bookworm@sha256:94e9efa4033213dbb70d4f665527e7ece3944ddb7ba1dd2e43f6fd6e2490af58 AS build +# #3315: which revision of the repository this binary is built from. Declared +# in the build stage as well as the runtime one because build.rs reads it from +# the environment to compile it in (/healthz's `revision` field) and the +# runtime stage's LABEL cannot reach back into the compiler. Unset is normal, +# not an error: a developer building this by hand gets "unknown" rather than a +# broken build, and the whole point is that the answer is honest when it is +# missing. Arcane passes it from the synced checkout via the stack's own +# `build.args`; CI passes ${{ github.sha }}. +ARG GIT_SHA WORKDIR /src COPY Cargo.toml Cargo.lock* ./ # build.rs stamps the binary with its compile time (see the file). It has to # be copied explicitly: cargo only runs a build script that is actually in # the crate root, and without it env!("APIARY_BUILD_EPOCH") is a compile -# error rather than a missing value. +# error rather than a missing value. build.rs also carries GIT_SHA through as +# APIARY_GIT_SHA since #3315. COPY build.rs ./ COPY src ./src # report_pdf.rs's two include_bytes!("../assets_pdf/...") calls need this @@ -20,6 +30,30 @@ COPY assets_pdf ./assets_pdf RUN cargo build --release FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 +# Redeclared per stage: Docker scopes an ARG to the stage that declares it, so +# the value the build stage compiled in is not in scope here on its own. +ARG GIT_SHA +# Static, so it is written here rather than taken from a build arg: the +# repository URL is the same for every build of this image and there is no +# reason to let a caller point the label somewhere else. +ARG GIT_SOURCE=https://github.com/Xore/APIARY +# Build time is the one label a build cannot honestly invent -- unlike the +# revision, there is no earlier source of truth to copy it from. So it stays +# "unknown" unless the builder supplies it, and CI does (containers.yml passes +# metadata-action's labels, which carry a real created/source pair and take +# precedence over these). An "unknown" here reads as "nobody told us", which is +# true; a plausible-looking wrong timestamp would not be. +ARG BUILD_DATE +# #3315: `docker images` shows no revision at all without these. `Labels: null` +# on a stale image is the exact state that let every merge since a redeploy go +# undeployed without anything saying so (see the issue's homeserver example). +# Read with: docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' +LABEL org.opencontainers.image.revision="${GIT_SHA:-unknown}" \ + org.opencontainers.image.source="${GIT_SOURCE}" \ + org.opencontainers.image.created="${BUILD_DATE:-unknown}" \ + org.opencontainers.image.title="apiary-backend" \ + org.opencontainers.image.description="APIARY dashboard backend service (Rust, axum)" + RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl \ && rm -rf /var/lib/apt/lists/* diff --git a/arcane/home/honeypot-dashboard/backend-service/build.rs b/arcane/home/honeypot-dashboard/backend-service/build.rs index 8737a15b7..db023f400 100644 --- a/arcane/home/honeypot-dashboard/backend-service/build.rs +++ b/arcane/home/honeypot-dashboard/backend-service/build.rs @@ -12,7 +12,9 @@ // A compile timestamp answers the question that actually gets asked after a // deploy: is the running binary newer than the merge? Git metadata would say // more, but the Docker build context is the crate directory alone and -// carries no .git, so it cannot be read where it would need to be. +// carries no .git, so it cannot be read where it would need to be -- it has +// to be handed in as a build arg (GIT_SHA) instead, which is what #3315 added +// once a timestamp stopped being enough to answer "which code is this?". use std::time::{SystemTime, UNIX_EPOCH}; fn main() { @@ -20,10 +22,31 @@ fn main() { // the code moves underneath it. println!("cargo:rerun-if-changed=src"); println!("cargo:rerun-if-changed=Cargo.toml"); + // ...and whenever GIT_SHA changes, which is the case this did not cover + // before #3315. cargo caches a build script's output by its inputs, and an + // environment variable is not one of them unless it is declared here: a + // rebuild of an identical tree with a different GIT_SHA would otherwise + // reuse the previous revision's compiled-in value and report itself as + // that revision. That is the "green, healthy, running the old code" shape + // this stamp exists to end, reproduced by the stamp itself. + println!("cargo:rerun-if-env-changed=GIT_SHA"); let epoch = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|elapsed| elapsed.as_secs()) .unwrap_or(0); println!("cargo:rustc-env=APIARY_BUILD_EPOCH={epoch}"); + + // Only the first whitespace-delimited token survives, and only because the + // cargo directive protocol is newline-separated: a GIT_SHA carrying a + // newline would otherwise inject arbitrary directives (extra + // rustc-envs, a rerun-if-changed pointing anywhere) into this build. + // Whether what came out is a plausible object name is decided by + // main.rs's normalize_revision, which is a plain function and therefore + // unit-testable; this is only the transport. + let sha = std::env::var("GIT_SHA") + .ok() + .and_then(|value| value.split_whitespace().next().map(str::to_string)) + .unwrap_or_default(); + println!("cargo:rustc-env=APIARY_GIT_SHA={sha}"); } diff --git a/arcane/home/honeypot-dashboard/backend-service/src/main.rs b/arcane/home/honeypot-dashboard/backend-service/src/main.rs index be89218c1..049c212fa 100644 --- a/arcane/home/honeypot-dashboard/backend-service/src/main.rs +++ b/arcane/home/honeypot-dashboard/backend-service/src/main.rs @@ -109,11 +109,20 @@ pub struct AppState { struct Health { ok: bool, es: bool, + /// #3315: which revision of the repository this binary was compiled from, + /// or "unknown". Unauthenticated and deliberately so — this is the field + /// that turns "is the running binary newer than the merge?" from a manual + /// inference into a curl, and /healthz is already the one open probe on + /// this service (the token middleware covers /api/v1 only, see + /// require_service_token). A git revision names no secret: it is the same + /// string the image carries in org.opencontainers.image.revision and that + /// ghcr shows on the tag. + revision: String, } async fn healthz(State(state): State) -> Json { let es_ok = state.es.ping().await; - Json(Health { ok: true, es: es_ok }) + Json(Health { ok: true, es: es_ok, revision: git_revision() }) } /// A boot refusal carries the code the cutover doc and dashboards grep @@ -227,6 +236,44 @@ pub fn build_stamp() -> String { } } +/// The honest answer when no revision was baked in. Same word build_stamp +/// uses, deliberately: a value that reads as a plausible time or a plausible +/// object name is worse than one that says nothing. +pub const REVISION_UNKNOWN: &str = "unknown"; + +/// A git object name is 7-64 hex characters, optionally `sha256:`-prefixed +/// (git's own object-format naming) — anything else is not a revision. +/// +/// This is a filter, not a format preference. `APIARY_GIT_SHA` arrives from a +/// `docker build --build-arg`, and a value that is not an object name is +/// either a mistake or something injected; either way it must not be echoed +/// back out of /healthz verbatim and read as "this is the deployed commit". +/// Case is normalized because GitHub, `git rev-parse` and the OCI label +/// convention each spell it differently, and a spelling difference must not +/// read as a deployed-revision mismatch. +pub fn normalize_revision(raw: &str) -> String { + let candidate = raw.strip_prefix("sha256:").unwrap_or(raw); + if (7..=64).contains(&candidate.len()) && candidate.bytes().all(|b| b.is_ascii_hexdigit()) { + candidate.to_ascii_lowercase() + } else { + REVISION_UNKNOWN.to_string() + } +} + +/// The revision this binary was compiled from, as the image build supplied it. +/// +/// `option_env!` for the same reason build_stamp() uses it: a missing stamp is +/// a diagnostic field, not a reason to fail a build. It is set unconditionally +/// by build.rs (an absent GIT_SHA becomes the empty string), so the None arm +/// only fires for a crate built by some path that skipped build.rs entirely — +/// which must read as "unknown", never as a guess. +pub fn git_revision() -> String { + match option_env!("APIARY_GIT_SHA") { + Some(raw) => normalize_revision(raw), + None => REVISION_UNKNOWN.to_string(), + } +} + #[tokio::main] async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt() @@ -495,8 +542,16 @@ async fn main() -> anyhow::Result<()> { // `built` is the one thing that makes a deploy verifiable from outside. // Compare it against the merge time; anything else -- a fresh image id, a // recreated container, `{"done":true}` -- says the machinery ran, not - // that this code is what is running. See build.rs. - tracing::info!(%addr, %es_url, built = %build_stamp(), "apiary-backend listening"); + // that this code is what is running. See build.rs. `revision` (#3315) is + // the same claim without the inference: a timestamp can only be compared, + // an object name can be looked up. + tracing::info!( + %addr, + %es_url, + built = %build_stamp(), + revision = %git_revision(), + "apiary-backend listening" + ); let listener = tokio::net::TcpListener::bind(addr).await?; axum::serve(listener, app).await?; Ok(()) @@ -504,7 +559,7 @@ async fn main() -> anyhow::Result<()> { #[cfg(test)] mod build_stamp_tests { - use super::build_stamp; + use super::{build_stamp, git_revision, normalize_revision, REVISION_UNKNOWN}; #[test] fn build_stamp_is_a_real_recent_timestamp() { @@ -527,6 +582,63 @@ mod build_stamp_tests { "build stamp {stamp} is not a plausible build time (age {age})", ); } + + // ---- #3315: the revision /healthz reports (main.rs's `revision` field) ---- + + #[test] + fn a_full_object_name_survives_verbatim() { + assert_eq!(normalize_revision("3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab"), + "3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab"); + } + + /// The table is a file rather than a literal list so that the other + /// implementation of this rule -- dashboard-next's normalizeRevision, in + /// JavaScript, in a different CI lane -- can be driven from exactly the + /// same cases. `scripts/tests/test_3315_image_revision.py` runs both. + /// Written inline, the two lists would drift the first time either gained + /// a case, and the failure would be a deploy disagreement that is not one: + /// two tiers stamping different strings for the same build, which + /// scripts/verify-deploy.sh would report as a mismatch. + #[test] + fn the_shared_corpus_normalizes_as_the_other_tier_does() { + let corpus: serde_json::Value = + serde_json::from_str(include_str!("revision-corpus.json")).expect("corpus is valid JSON"); + let unknown = corpus["unknown"].as_str().expect("corpus names its unknown value"); + let cases = corpus["cases"].as_array().expect("corpus carries a cases array"); + assert!(!cases.is_empty(), "the corpus is empty, so this test proves nothing"); + for case in cases { + let pair = case.as_array().expect("each case is a [input, expected] pair"); + let (raw, expected) = ( + pair[0].as_str().expect("case input is a string"), + pair[1].as_str().expect("case expectation is a string"), + ); + assert_eq!( + &normalize_revision(raw), + expected, + "normalize_revision({raw:?}) disagrees with the shared corpus" + ); + } + // The one value the whole design turns on: a build with no revision + // says so rather than reporting something that reads as an answer. + assert_eq!(normalize_revision(""), unknown); + } + + #[test] + fn this_test_binary_reports_a_revision_or_says_unknown() { + // A `cargo test` run has GIT_SHA unset unless the caller exported it, + // so the honest value here is "unknown" -- and both are acceptable. + // What must never happen is a third thing: a revision that does not + // look like an object name coming out of the real accessor. + let revision = git_revision(); + assert!(!revision.is_empty(), "the revision field must never be empty"); + if revision != REVISION_UNKNOWN { + assert_eq!( + revision, + normalize_revision(&revision), + "git_revision() returned {revision:?}, which its own normalizer would not accept" + ); + } + } } #[cfg(test)] diff --git a/arcane/home/honeypot-dashboard/backend-service/src/revision-corpus.json b/arcane/home/honeypot-dashboard/backend-service/src/revision-corpus.json new file mode 100644 index 000000000..42a399eef --- /dev/null +++ b/arcane/home/honeypot-dashboard/backend-service/src/revision-corpus.json @@ -0,0 +1,23 @@ +{ + "note": "Shared normalization corpus for the #3315 git-revision stamp. backend-service/src/main.rs's revision test and frontend-next/scripts/write-build-info.mjs's normalizeRevision are the same rule in two languages, in two CI lanes, with nothing at review time comparing them. This file is the comparison: whichever side changes, the other's cases are unchanged, and scripts/tests/test_3315_image_revision.py fails until both sides agree on the table below. Kept as JSON rather than a Rust const and a JS array so the two can be byte-identical. The cases that must stay pinned are the ones whose disagreement is silent: a value that normalizes to 'unknown' on one side and to itself on the other is reported as a deploy disagreement that is not one.", + "unknown": "unknown", + "cases": [ + ["3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab", "3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab"], + ["3dca445", "3dca445"], + ["sha256:3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab", "3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab"], + ["3DCA4457F1B2C0D4E5A69788796A5B4C3D2E1F0AB", "3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab"], + ["a3ac9f1b0e2d4c6f8a0b2c4e6f8a0b2c4e6f8a0b2", "a3ac9f1b0e2d4c6f8a0b2c4e6f8a0b2c4e6f8a0b2"], + ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"], + ["", "unknown"], + [" ", "unknown"], + ["unknown", "unknown"], + ["main", "unknown"], + ["HEAD", "unknown"], + ["v1.2.3", "unknown"], + ["123456", "unknown"], + ["3dca4457 and some other text", "unknown"], + ["3dca4457\nLABEL injected=true", "unknown"], + ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "unknown"], + ["ggggggggggggggggggggggggggggggggggggggggg", "unknown"] + ] +} diff --git a/arcane/home/honeypot-dashboard/compose.yml b/arcane/home/honeypot-dashboard/compose.yml index fca5c032e..abd6caf53 100644 --- a/arcane/home/honeypot-dashboard/compose.yml +++ b/arcane/home/honeypot-dashboard/compose.yml @@ -453,7 +453,19 @@ services: dashboard-next: <<: *runtime-defaults - build: ./frontend-next + build: + context: ./frontend-next + # #3315: the revision this image is built from, baked into the served + # /build.json and the image's org.opencontainers.image.revision label. + # Read from this stack's .env, not invented here -- compose cannot run + # `git rev-parse`, and this stack's synced directory carries no .git (see + # docs/ARCANE-GIT-SYNC.md), so whatever builds the image has to say which + # commit it is. Unset is the honest default and produces + # {"revision":"unknown"} plus an `unknown` label rather than a build + # failure: scripts/verify-deploy.sh reports an unstamped image as its own + # finding, which is more use than a red build that gets reverted. + args: + GIT_SHA: ${GIT_SHA:-} image: apiary-dashboard-next:latest # build: same "never pushed to any registry, Arcane's redeploy pulls # unconditionally" issue as apiary-backend:latest above -- this one has diff --git a/arcane/home/honeypot-dashboard/frontend-next/.gitignore b/arcane/home/honeypot-dashboard/frontend-next/.gitignore index b85468516..d486fa56e 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/.gitignore +++ b/arcane/home/honeypot-dashboard/frontend-next/.gitignore @@ -5,6 +5,12 @@ node_modules/ dist/ .env +# #3315: generated at image-build time by scripts/write-build-info.mjs from the +# GIT_SHA build arg and served at /build.json. Never committed -- a checked-in +# copy would be a revision claim about nobody's build, and the one in the tree +# is the one a local `npm run build` would serve. +public/build.json + # #1828: the design lab symlinks variant stylesheets in here at start-up # and clears it on exit. Never content, always disposable. public/static/lab/ diff --git a/arcane/home/honeypot-dashboard/frontend-next/Dockerfile b/arcane/home/honeypot-dashboard/frontend-next/Dockerfile index b6d3c2a76..e923289b7 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/Dockerfile +++ b/arcane/home/honeypot-dashboard/frontend-next/Dockerfile @@ -2,13 +2,52 @@ # Nitro's node-server output is self-contained (.output holds server + # client assets + public/static, including the vendored theme.css copy). FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 AS build +# #3315: which revision of the repository this image is built from, stamped +# into the served /build.json below. Unset is normal, not an error: the +# generated file then says "unknown" rather than the build failing or the +# value being invented. Arcane passes it from the synced checkout via the +# stack's own `build.args`; CI passes ${{ github.sha }}. +ARG GIT_SHA +ARG GIT_SOURCE=https://github.com/Xore/APIARY WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci COPY . . +# Written after the source copy (it needs scripts/write-build-info.mjs) and +# before the build (vite copies public/ into .output/public, so a file added +# afterwards would never be served). Two consecutive RUNs because the stamp +# must exist even if `npm run build` is what fails: an image with no +# /build.json at all is a silent regression of this feature, and the error +# says nothing about which step dropped it. +# hadolint ignore=DL3059 +RUN node scripts/write-build-info.mjs --out public/build.json RUN npm run build FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 +# Redeclared per stage: Docker scopes an ARG to the stage that declares it, so +# the value the build stage wrote into public/build.json is not in scope here +# on its own. +ARG GIT_SHA +# Static, so it is written here rather than taken from a build arg: the +# repository URL is the same for every build of this image and there is no +# reason to let a caller point the label somewhere else. +ARG GIT_SOURCE=https://github.com/Xore/APIARY +# Build time is the one label a build cannot honestly invent. So it stays +# "unknown" unless the builder supplies it, and CI does (containers.yml passes +# metadata-action's labels, which carry a real created/source pair and take +# precedence over these). An "unknown" here reads as "nobody told us", which +# is true; a plausible-looking wrong timestamp would not be. +ARG BUILD_DATE +# #3315: `docker images` showed `Labels: null` for this image, so nothing said +# which commit a running dashboard was built from. Same labels as +# backend-service/Dockerfile, and they are read the same way: +# docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' +LABEL org.opencontainers.image.revision="${GIT_SHA:-unknown}" \ + org.opencontainers.image.source="${GIT_SOURCE}" \ + org.opencontainers.image.created="${BUILD_DATE:-unknown}" \ + org.opencontainers.image.title="apiary-dashboard-next" \ + org.opencontainers.image.description="APIARY dashboard frontend/BFF (TanStack Start, Nitro)" + WORKDIR /app COPY --from=build /app/.output ./.output COPY server ./server diff --git a/arcane/home/honeypot-dashboard/frontend-next/scripts/write-build-info.mjs b/arcane/home/honeypot-dashboard/frontend-next/scripts/write-build-info.mjs new file mode 100644 index 000000000..c7d12aa3a --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/scripts/write-build-info.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +// #3315: write the build-stamp file this tier serves at /build.json. +// +// The problem: nothing could say which commit a running dashboard-next was +// built from. `docker images` showed no labels at all for the image, and the +// container's /healthz answers a fixed "ok", so "did the merge that closed +// #3310 actually reach the host?" was answered by comparing image creation +// times against merge times by eye -- the inference docs/ARCANE-GIT-SYNC.md +// calls out as the reason a green, healthy container can be running old code. +// +// A static file rather than a server route, on purpose. Nitro serves the +// build's `public/` directory at the server root, and this image already +// depends on that: the container HEALTHCHECK fetches /static/theme.css, which +// is public/static/theme.css. /build.json rides the same, already-exercised +// path, so it cannot be a route that the file-based generator fails to +// register, fails to give a clean path for (a literal `.` in a TanStack file +// route's name is not something to bet a deploy check on), or shadows behind +// SSR. There is nothing to import and no server code to run. +// +// Usage: write-build-info.mjs [--out ] +// GIT_SHA the revision, from `docker build --build-arg GIT_SHA=...`. +// Absent is normal -- a developer building this by hand gets +// "unknown", never a broken build and never a plausible lie. +// GIT_SOURCE repository URL recorded alongside it. Defaults to this repo. +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, resolve } from 'node:path' + +const DEFAULT_OUT = 'public/build.json' +const DEFAULT_SOURCE = 'https://github.com/Xore/APIARY' +export const UNKNOWN_REVISION = 'unknown' + +/** + * A git object name is 7-64 hex characters, optionally `sha256:`-prefixed + * (git's own object-format naming); anything else is not a revision. + * + * Deliberately the same rule as backend-service's `normalize_revision` + * (src/main.rs) rather than a looser "trim and hope": the two stamps are + * compared against each other by scripts/verify-deploy.sh, and a value that + * reads as a revision in one tier and as junk in the other is a disagreement + * the operator has to debug. Keep the two in step -- + * scripts/tests/test_3315_image_revision.py pins one corpus against both. + */ +export function normalizeRevision(raw) { + const candidate = (raw ?? '').trim().replace(/^sha256:/, '') + const isObjectName = candidate.length >= 7 && candidate.length <= 64 && + /^[0-9a-fA-F]+$/.test(candidate) + return isObjectName ? candidate.toLowerCase() : UNKNOWN_REVISION +} + +function parseArgs(argv) { + let out = DEFAULT_OUT + for (let i = 0; i < argv.length; i += 1) { + if (argv[i] === '--out') { + const value = argv[i + 1] + if (!value) { + console.error('write-build-info: --out needs a value') + process.exit(2) + } + out = value + i += 1 + } else { + console.error(`write-build-info: unknown argument: ${argv[i]}`) + process.exit(2) + } + } + return { out } +} + +const { out } = parseArgs(process.argv.slice(2)) +const doc = { + // toISOString() is UTC by definition, which is why it and not a hand-rolled + // local-time format: scripts/check-timestamp-utc.py polices every Z-suffixed + // stamp in this repo and a wall-clock one would be wrong twice a year. + built: new Date().toISOString(), + revision: normalizeRevision(process.env.GIT_SHA), + source: (process.env.GIT_SOURCE ?? '').trim() || DEFAULT_SOURCE, +} + +const target = resolve(out) +mkdirSync(dirname(target), { recursive: true }) +// Key order is the read order: what a human opens /build.json for is `revision` +// first, and alphabetical would bury it under `built`. +writeFileSync(target, `${JSON.stringify(doc, null, 2)}\n`, 'utf8') +console.log(`write-build-info: ${target} revision=${doc.revision}`) diff --git a/docs/ARCANE-GIT-SYNC.md b/docs/ARCANE-GIT-SYNC.md index 6b26dddc7..7ba46f1ec 100644 --- a/docs/ARCANE-GIT-SYNC.md +++ b/docs/ARCANE-GIT-SYNC.md @@ -504,6 +504,118 @@ manual process, because it is unattended. scope — the #1507 activation itself was never done, and #2858 (below) decided explicitly not to do it in this round either. +## Proving which revision is deployed (#3315) + +The failure this section exists for is quoted above: a content-change +redeploy leaves the dashboard *"green, healthy, running the old code."* +Everything else about that state is a lie by omission — a fresh container +id, a passing healthcheck and a recent image build all say the machinery +ran, not that this code is what is running. So both dashboard images now +carry the git revision they were built from, in three places, and +`scripts/verify-deploy.sh` compares them. + +**Where the stamp is.** + +- **Backend** — compiled in. `backend-service/build.rs` reads the `GIT_SHA` + build arg and re-exports it as `APIARY_GIT_SHA`; `/healthz` returns it as + the `revision` field (`{"ok":true,"es":true,"revision":"3dca445…"}`) and + the boot log line carries it. `build.rs` also emits + `cargo:rerun-if-env-changed=GIT_SHA`, which is load-bearing: without it + cargo reuses a cached build and the second build of an identical tree + keeps the *first* revision, producing the exact failure this fixes. +- **Frontend** — a static `public/build.json` (`{"revision":…,"built":…, + "source":…}`), written by `frontend-next/scripts/write-build-info.mjs` + into the build stage before `npm run build` so vite copies it into the + output. Served by Nitro from `public/`, the same mechanism the image + healthcheck already uses for `/static/theme.css`. It is gitignored: a + committed `build.json` would be a revision claim about nobody's build. +- **Both** — `org.opencontainers.image.revision` (plus `.source` and + `.created`) as image labels, readable without running anything: + `docker image inspect apiary-backend:latest --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}'`. + +**Where the revision comes from.** The stacks' compose files take +`build.args.GIT_SHA: ${GIT_SHA:-}`, and `.env.example` documents the knob. +It is a variable rather than an inline `git rev-parse` because Arcane's +directory sync explicitly excludes `.git` — there is no repository on the +host for compose to ask. Unset is normal, not an error: a hand-run build +gets `unknown` and a developer never has to pass the arg to get a working +image. CI (`.github/workflows/containers.yml`) passes `github.sha` for +exactly the two Dockerfiles that declare the arg. + +**Nothing on the homeserver sets it yet, and that is deliberate to state +plainly.** Arcane reads `${GIT_SHA:-}` out of the stack's own `.env`, and no +repo-tracked file, deploy step or installer writes that variable — the +`.env` files are root-owned `0600` and are provisioned outside this +repository (the same root-owned `.env` the #3312 token helper exists for). +It also cannot be computed in place: `/opt/stacks/apiary` looks like a +checkout but is a working-tree rsync with `.git/` excluded (`deploy.yml`'s +own sync step), and Arcane's synced directory has no `.git` either, so +there is no repository on the host to ask. So the first +`verify-deploy.sh` run after this lands reports **not stamped**, correctly: +the live image was built before the knob existed. Turning it on is one line +per stack, from whatever clone you are deploying — + +``` +echo "GIT_SHA=$(git rev-parse HEAD)" \ + >> /var/dockge/stacks/honeypot-dashboard-backend/.env +``` + +— followed by the `POST /projects/{id}/build` the "Two facts" section above +says a content-change sync does *not* do. The two dashboard stacks carry +the variable independently, so it goes in both. Reporting the gap honestly +beats having the image invent a revision. + +Both tiers normalize the value to a bare lowercase hex object name +(optional `sha256:` prefix stripped, 7–64 hex characters) or `unknown`, and +both are held to the same table — `backend-service/src/revision-corpus.json`, +read by a cargo test and by `scripts/tests/test_3315_image_revision.py` — +because the two implementations are in different languages in different CI +lanes and the failure mode of their disagreeing is silent: the same build +stamped two ways, which the verification tool would then report as a +mismatch that is not one. + +**How to check a deploy.** The tool compares the running revision, the image +label, and an expected revision — which it takes from `origin/main` in a +clone, or from an explicit argument: + +``` +# On a host with a real clone of apiary, the full check: +scripts/verify-deploy.sh --healthz-exec \ + 'docker exec hp-apiary-backend curl -sf http://127.0.0.1:8081/healthz' \ + --image apiary-backend:latest + +# Without one, name the expected revision yourself: +scripts/verify-deploy.sh --behind-days 0 "$(git rev-parse origin/main)" \ + --healthz-exec 'docker exec hp-apiary-backend curl -sf http://127.0.0.1:8081/healthz' +``` + +`--healthz-url http://host:19090/build.json` is the frontend form. Exit +codes are the point: **0** the deployed revision matches, **1** a finding +(missing, unstamped, malformed, or stale past `--behind-days`), **2** +"could not tell" — deliberately distinct, so an unreadable body or an image +that is not on this host can never be mistaken for a stale deploy. Add +`--image` to check a label against the running container, which catches the +commoner variant where a `compose up` recreated from an older image than +the one whose label you just read. + +The lag check measures the age of the *deployed commit*, not of the +container, so it survives a container that has been up since before the +merge it is missing — but it needs a clone to measure against, which is why +`--behind-days 0` is the form that works without one. A short revision +(`git rev-parse --short HEAD`) is accepted and reported as an abbreviation +rather than as a mismatch — otherwise the tool would cry wolf on the first +thing an operator tries. + +**`diagnostics.yml` runs it in `--warn-only` mode**, in its own "Deployed +revision" section: a scheduled diagnostics red X is the alert, and a stale +deploy is not on #2222's list of things that should redden a run. It passes +`$GITHUB_SHA` as the expected revision and `--behind-days 0`, for the +reason above — the home runner has no clone to measure against (`deploy.yml` +rsyncs into `/opt/stacks/apiary` with `--exclude .git/`, and diagnostics.yml +deliberately has no `actions/checkout`), so the section degrades to "is the +running revision the current tip of `main`" and says so in the report. Its +exit 2 is still reported as **could not tell** rather than as a pass. + ## autoSync decision (#2858) Re-measured 2026-09-03 against `GET /environments/0/gitops-syncs?limit=100` diff --git a/scripts/tests/test_3315_image_revision.py b/scripts/tests/test_3315_image_revision.py new file mode 100644 index 000000000..50a59d4ba --- /dev/null +++ b/scripts/tests/test_3315_image_revision.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +"""Exercise the #3315 revision stamp end to end, without a live host. + +Three things this covers, and why each needs its own harness: + +* `frontend-next/scripts/write-build-info.mjs` -- run for real with node, over + the corpus of values a `--build-arg GIT_SHA` could plausibly carry. The + point is not that it writes JSON (any shell printf does that) but what it + does with a value that is not an object name, and that it agrees with + backend-service's own normalizer, which is a different language in a + different CI lane and can only be kept in step by a shared corpus. + +* The Dockerfile/compose wiring -- asserted as text. Neither cargo nor vitest + can see that `ARG GIT_SHA` is declared in *both* stages of a multi-stage + build, that the label lands on the runtime image rather than the discarded + build stage, or that the file the build writes is written before the build + that has to copy it. Those are the three ways this feature can exist in the + repository and not exist in any image, and all three are silent. + +* `scripts/verify-deploy.sh` -- driven against a real throwaway git repo with + stub `curl`/`docker` binaries, because the script's whole value is the + verdict it returns, and a verdict is not worth much if the only thing anyone + ever checks is that the file parses. +""" +from __future__ import annotations + +import datetime +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "verify-deploy.sh" +WRITER = ROOT / "arcane" / "home" / "honeypot-dashboard" / "frontend-next" / "scripts" / "write-build-info.mjs" +FRONTEND_DOCKERFILE = ROOT / "arcane" / "home" / "honeypot-dashboard" / "frontend-next" / "Dockerfile" +BACKEND_DOCKERFILE = ROOT / "arcane" / "home" / "honeypot-dashboard" / "backend-service" / "Dockerfile" +FRONTEND_COMPOSE = ROOT / "arcane" / "home" / "honeypot-dashboard" / "compose.yml" +BACKEND_COMPOSE = ROOT / "arcane" / "home" / "honeypot-dashboard-backend" / "compose.yml" +FRONTEND_ENV_EXAMPLE = ROOT / "arcane" / "home" / "honeypot-dashboard" / ".env.example" +BACKEND_ENV_EXAMPLE = ROOT / "arcane" / "home" / "honeypot-dashboard-backend" / ".env.example" +BUILD_RS = ROOT / "arcane" / "home" / "honeypot-dashboard" / "backend-service" / "build.rs" +MAIN_RS = ROOT / "arcane" / "home" / "honeypot-dashboard" / "backend-service" / "src" / "main.rs" +CORPUS_JSON = ROOT / "arcane" / "home" / "honeypot-dashboard" / "backend-service" / "src" / "revision-corpus.json" +CONTAINERS_YML = ROOT / ".github" / "workflows" / "containers.yml" + +FULL = "3dca4457f1b2c0d4e5a69788796a5b4c3d2e1f0ab" + + +def load_corpus() -> tuple[str, list[tuple[str, str]]]: + """The one table both normalizers are held to, read from the repo. + + backend-service's normalize_revision is Rust and lives in a cargo test + lane; normalizeRevision is JS run from this file. Nothing at build or + review time compares them, so the corpus is the comparison -- a fixture + rather than a second literal list, because a second literal list is a + second thing that can drift. + """ + doc = json.loads(CORPUS_JSON.read_text(encoding="utf-8")) + return doc["unknown"], [tuple(case) for case in doc["cases"]] + + +UNKNOWN, CORPUS = load_corpus() + + +def run(*args: str, env: dict[str, str] | None = None, cwd: Path | None = None): + merged = dict(os.environ) + for leaked in ("CURL_BIN", "DOCKER_BIN", "PYTHON_BIN", "GIT_BIN", "GIT_SHA", "GIT_SOURCE"): + merged.pop(leaked, None) + if env: + merged.update(env) + return subprocess.run( + [str(a) for a in args], capture_output=True, text=True, env=merged, cwd=cwd, check=False + ) + + +def write_exec(path: Path, body: str) -> Path: + path.write_text(body, encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + return path + + +class WriteBuildInfo(unittest.TestCase): + """The generator the frontend image runs, executed for real.""" + + @classmethod + def setUpClass(cls) -> None: + if shutil.which("node") is None: + raise unittest.SkipTest("node is not on PATH; the frontend image's generator needs it") + + def generate(self, tmp: Path, git_sha: str | None = None, **extra: str) -> dict: + out = tmp / "build.json" + env = dict(extra) + if git_sha is not None: + env["GIT_SHA"] = git_sha + res = run("node", WRITER, "--out", out, env=env) + self.assertEqual(res.returncode, 0, res.stderr) + return json.loads(out.read_text(encoding="utf-8")) + + def test_a_real_revision_is_carried_through(self) -> None: + with tempfile.TemporaryDirectory() as raw: + doc = self.generate(Path(raw), FULL) + self.assertEqual(doc["revision"], FULL) + + def test_no_build_arg_says_unknown_rather_than_failing_the_build(self) -> None: + # A developer running `docker build` by hand must get a working image + # that admits it does not know its revision. Refusing to build would + # get the Dockerfile reverted. + with tempfile.TemporaryDirectory() as raw: + doc = self.generate(Path(raw)) + self.assertEqual(doc["revision"], UNKNOWN) + + def test_every_corpus_case_normalizes_as_the_table_says(self) -> None: + with tempfile.TemporaryDirectory() as raw: + tmp = Path(raw) + for raw_value, expected in CORPUS: + with self.subTest(git_sha=raw_value): + self.assertEqual(self.generate(tmp, raw_value)["revision"], expected) + + def test_a_multiline_build_arg_cannot_inject_a_second_json_document(self) -> None: + # The stamp is served as a static file and read by whatever can reach + # the dashboard, so it has to be exactly one JSON document whatever the + # build arg contained. + with tempfile.TemporaryDirectory() as raw: + tmp = Path(raw) + doc = self.generate(tmp, 'x"}\n{"revision":"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef') + self.assertEqual(doc["revision"], UNKNOWN) + self.assertNotIn("deadbeef", json.dumps(doc)) + + def test_the_document_carries_a_utc_build_time_and_the_source(self) -> None: + with tempfile.TemporaryDirectory() as raw: + tmp = Path(raw) + doc = self.generate(tmp, FULL) + # `built` is the frontend's counterpart to the backend's compile + # timestamp; it is a Z-suffixed stamp, so it has to be real UTC -- + # toISOString() is, and scripts/check-timestamp-utc.py polices + # every other emitter of one in this repo. + self.assertRegex(doc["built"], r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$") + self.assertEqual(doc["source"], "https://github.com/Xore/APIARY") + overridden = self.generate(tmp, FULL, GIT_SOURCE=" ") + self.assertEqual(overridden["source"], "https://github.com/Xore/APIARY") + + def test_the_output_directory_is_created(self) -> None: + # The Dockerfile points this at public/, which exists -- but the script + # is also the thing a person runs by hand, and mkdir -p is cheaper than + # a second failure mode. + with tempfile.TemporaryDirectory() as raw: + out = Path(raw) / "nested" / "deeper" / "build.json" + res = run("node", WRITER, "--out", out, env={"GIT_SHA": FULL}) + self.assertEqual(res.returncode, 0, res.stderr) + self.assertTrue(out.is_file()) + + def test_an_unknown_flag_is_refused_rather_than_ignored(self) -> None: + # Silently ignoring a typo'd flag would write a stamp the caller did + # not ask for and report success. + res = run("node", WRITER, "--outt", "/dev/null") + self.assertEqual(res.returncode, 2) + self.assertIn("unknown argument", res.stderr) + + +class NormalizerParity(unittest.TestCase): + """The JS and Rust normalizers are one rule in two languages.""" + + def test_the_rust_side_is_held_to_the_same_table(self) -> None: + # backend-service/src/main.rs is a different CI lane with a different + # toolchain, and its own test module is the only thing that pins + # normalize_revision. What makes the two sides comparable is that both + # read the same file: if the Rust test ever drifts to a literal list of + # its own, the disagreement becomes invisible again. The failure mode + # is subtle -- the two tiers stamping different strings for the same + # build, which verify-deploy.sh would then report as a disagreement + # that is not one. + self.assertIn( + 'include_str!("revision-corpus.json")', + MAIN_RS.read_text(encoding="utf-8"), + "the Rust normalizer is no longer pinned to the shared corpus", + ) + # A stubbed-out corpus would pass every assertion above it. + self.assertGreaterEqual(len(CORPUS), 10, "the shared corpus has been hollowed out") + self.assertEqual(UNKNOWN, "unknown", "both sides spell the sentinel the same way") + self.assertIn(f'pub const REVISION_UNKNOWN: &str = "{UNKNOWN}"', MAIN_RS.read_text(encoding="utf-8")) + # Internal coherence: every expectation is either the sentinel or a + # value the rule accepts unchanged. Anything else is a case one side + # could not produce and the table would be describing a third + # normalizer nobody implements. + hexdigits = set("0123456789abcdef") + for raw_value, expected in CORPUS: + with self.subTest(git_sha=raw_value): + self.assertTrue( + expected == UNKNOWN or (7 <= len(expected) <= 64 and set(expected) <= hexdigits), + f"corpus expects {expected!r}, which is neither the sentinel nor an object name", + ) + + def test_the_rust_build_script_passes_the_arg_through(self) -> None: + build_rs = BUILD_RS.read_text(encoding="utf-8") + # Without rerun-if-env-changed, a second build of an identical tree with + # a different GIT_SHA reuses the first one's compiled-in revision -- + # the exact "green, running the old code" shape this issue is about, + # produced by the fix itself. + self.assertIn("cargo:rerun-if-env-changed=GIT_SHA", build_rs) + self.assertIn("APIARY_GIT_SHA", build_rs) + + +class DockerfileWiring(unittest.TestCase): + """The three ways this can be in the repo and in no image.""" + + @staticmethod + def stages(text: str) -> list[str]: + """One entry per `FROM` line, preamble excluded. + + Splitting on "\\nFROM " is wrong for a real Dockerfile: the header + comment block means the text before the first FROM is a real, non-empty + chunk, so the naive split counts a stage that does not exist and every + index is off by one. + """ + parts = re.split(r"(?m)^FROM\s+", text) + return parts[1:] + + def test_backend_declares_the_arg_in_both_stages(self) -> None: + # Docker scopes an ARG to the stage that declares it. The build stage + # needs it (build.rs compiles it in); the runtime stage needs it again + # for the label. Declaring it once is the mistake this pins. + for name, path in (("backend", BACKEND_DOCKERFILE), ("frontend", FRONTEND_DOCKERFILE)): + with self.subTest(image=name): + parts = self.stages(path.read_text(encoding="utf-8")) + self.assertEqual(len(parts), 2, f"{name}: expected a two-stage build") + for index, part in enumerate(parts): + self.assertRegex( + part, r"(?m)^ARG GIT_SHA$", f"{name}: stage {index} does not declare ARG GIT_SHA" + ) + + def test_both_images_carry_the_oci_revision_label(self) -> None: + # `Labels: null` on apiary-backend:latest is the state this issue was + # filed about; the label is what replaces it. + for name, path in (("backend", BACKEND_DOCKERFILE), ("frontend", FRONTEND_DOCKERFILE)): + with self.subTest(image=name): + text = path.read_text(encoding="utf-8") + # On the runtime stage, not the build stage: a label on a stage + # that is never pushed is a label nobody can read. + runtime = self.stages(text)[-1] + self.assertIn('org.opencontainers.image.revision="${GIT_SHA:-unknown}"', runtime) + self.assertIn('org.opencontainers.image.source="${GIT_SOURCE}"', runtime) + self.assertIn('org.opencontainers.image.created="${BUILD_DATE:-unknown}"', runtime) + + def test_the_frontend_writes_its_stamp_before_the_build_that_copies_it(self) -> None: + # vite copies public/ into .output/public during `npm run build`. A + # file written after that is never served, and the image looks fine. + text = FRONTEND_DOCKERFILE.read_text(encoding="utf-8") + write_at = text.index("write-build-info.mjs --out public/build.json") + build_at = text.index("RUN npm run build") + self.assertLess(write_at, build_at, "build.json is written after the build that serves it") + self.assertIn("COPY . .", text[:write_at], "the generator is invoked before the source is copied") + + def test_the_frontend_stamp_is_not_a_tracked_file(self) -> None: + # A committed build.json would be a revision claim about nobody's + # build, and the one in the tree is the one a local `npm run build` + # would serve. + gitignore = (FRONTEND_DOCKERFILE.parent / ".gitignore").read_text(encoding="utf-8") + self.assertIn("public/build.json", gitignore) + res = run("git", "-C", ROOT, "ls-files", "--error-unmatch", "public/build.json", + cwd=FRONTEND_DOCKERFILE.parent) + self.assertNotEqual(res.returncode, 0, "public/build.json is tracked but is a build artefact") + + +class ComposeWiring(unittest.TestCase): + def test_both_stacks_pass_the_arg_to_their_build(self) -> None: + for name, path in (("honeypot-dashboard", FRONTEND_COMPOSE), ("honeypot-dashboard-backend", BACKEND_COMPOSE)): + with self.subTest(stack=name): + self.assertIn("GIT_SHA: ${GIT_SHA:-}", path.read_text(encoding="utf-8")) + + def test_both_env_examples_document_the_knob(self) -> None: + # scripts/check-compose-env-docs.py already gates this; asserting it + # here too means the gate's own failure explains what the knob is for. + for name, path in (("honeypot-dashboard", FRONTEND_ENV_EXAMPLE), ("honeypot-dashboard-backend", BACKEND_ENV_EXAMPLE)): + with self.subTest(stack=name): + self.assertRegex(path.read_text(encoding="utf-8"), r"(?m)^GIT_SHA=") + + def test_the_short_form_compose_still_validates(self) -> None: + if shutil.which("docker") is None: + self.skipTest("docker is not available to render compose config") + for name, path, workdir in ( + ("honeypot-dashboard", FRONTEND_COMPOSE, FRONTEND_COMPOSE.parent), + ("honeypot-dashboard-backend", BACKEND_COMPOSE, BACKEND_COMPOSE.parent), + ): + with self.subTest(stack=name): + res = run("docker", "compose", "-f", path, "config", "--quiet", cwd=workdir) + self.assertEqual(res.returncode, 0, res.stderr) + + def test_ci_passes_the_same_sha_the_labels_already_carry(self) -> None: + # metadata-action already stamps an org.opencontainers.revision on + # every image this workflow pushes. Passing GIT_SHA as well is what + # makes the compiled-in /healthz value and the image label come from + # one source instead of two that can drift. + text = CONTAINERS_YML.read_text(encoding="utf-8") + self.assertIn("stamp: true", text) + self.assertEqual( + text.count("stamp: true"), + 2, + "exactly the two dashboard Dockerfiles declare ARG GIT_SHA", + ) + self.assertIn("format('GIT_SHA={0}', github.sha)", text) + + +class VerifyDeploy(unittest.TestCase): + """verify-deploy.sh, against a real repo and stub curl/docker.""" + + @classmethod + def setUpClass(cls) -> None: + if shutil.which("git") is None: + raise unittest.SkipTest("git is not on PATH") + if shutil.which("python3") is None: + raise unittest.SkipTest("python3 is not on PATH") + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.repo = self.tmp / "repo" + self.repo.mkdir() + self.git("init", "-q", "-b", "main") + self.git("config", "user.email", "test@example.invalid") + self.git("config", "user.name", "verify-deploy test") + self.write_file("a", "one") + self.git("add", "-A") + # The first commit is backdated, because the lag check measures the age + # of the *deployed commit* rather than the age of the container. A + # fixture made entirely of commits written "now" has no age to measure, + # so every lag assertion here would be testing a zero. + thirty_days_ago = ( + datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=30) + ).strftime("%Y-%m-%dT%H:%M:%S+00:00") + self.git( + "commit", "-qm", "one", + env={"GIT_AUTHOR_DATE": thirty_days_ago, "GIT_COMMITTER_DATE": thirty_days_ago}, + ) + self.first = self.rev() + self.write_file("a", "two") + self.git("commit", "-qam", "two") + self.head = self.rev() + self.git("update-ref", "refs/remotes/origin/main", self.head) + + def git(self, *args: str, env: dict[str, str] | None = None) -> str: + res = run("git", "-C", self.repo, *args, env=env or {}) + self.assertEqual(res.returncode, 0, res.stderr) + return res.stdout.strip() + + def rev(self) -> str: + return self.git("rev-parse", "HEAD") + + def write_file(self, name: str, body: str) -> None: + (self.repo / name).write_text(body, encoding="utf-8") + + def healthz(self, body: str) -> str: + """A --healthz-exec that answers `body` verbatim.""" + return f"printf '%s' {json.dumps(body)}" + + def check(self, *args: str, env: dict[str, str] | None = None): + return run( + SCRIPT, "--repo-root", self.repo, *args, env=env or {}, cwd=self.tmp + ) + + def docker_stub(self, label: str, exists: bool = True) -> Path: + return write_exec( + self.tmp / "docker", + f"""#!/usr/bin/env bash +# Stub for the two `docker image inspect` shapes verify-deploy.sh uses: the +# --format label read, and the plain existence probe behind it. +if [ "${{1:-}}" = "image" ] && [ "${{2:-}}" = "inspect" ]; then + if [ "${{3:-}}" = "--format" ]; then + printf '%s\\n' {json.dumps(label)} + exit 0 + fi + exit {0 if exists else 1} +fi +exit 1 +""", + ) + + # ---- the verdict is the product ---- + + def test_a_deploy_at_main_passes(self) -> None: + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}'), "--behind-days", "0") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertIn("PASS", res.stdout) + + def test_a_stale_deploy_fails_and_says_by_how_much(self) -> None: + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{self.first}"}}'), "--behind-days", "0") + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn(f"deployed {self.first}", res.stdout) + self.assertIn(f"expected {self.head}", res.stdout) + + def test_a_short_revision_is_not_a_mismatch(self) -> None: + # `git rev-parse --short HEAD` is what an operator reaches for first. + # Reporting it as a wrong deploy on every run would teach people to + # ignore the tool. + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{self.head[:7]}"}}'), "--behind-days", "0") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertIn("abbreviation", res.stdout) + + def test_an_unstamped_deploy_fails_loudly(self) -> None: + # The state this whole issue was filed about, stated as a failure + # rather than as a pass with nothing to report. + res = self.check("--healthz-exec", self.healthz('{"revision":"unknown"}'), "--behind-days", "0") + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn("not stamped", res.stdout) + + def test_a_malformed_revision_is_distinguished_from_a_stale_one(self) -> None: + # A value that is not an object name is a broken stamp, and telling an + # operator their deploy is merely out of date sends them to rebuild + # instead of to find whoever passed "main" as a revision. + res = self.check("--healthz-exec", self.healthz('{"revision":"main"}'), "--behind-days", "0") + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn("not a git object name", res.stdout) + + def test_a_stale_but_matching_deploy_is_still_flagged_by_the_lag_check(self) -> None: + # Pinning an explicit expected revision and finding it deployed exactly + # is a legitimate "yes, that is the release we chose" answer. Lag is + # then the only thing left to say, so it has to be measured from the + # commit rather than from the container's creation time. + old = self.git("rev-parse", self.first) + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{old}"}}'), old, "--behind-days", "3") + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn("commits behind", res.stdout) + + def test_a_deploy_at_main_is_not_behind(self) -> None: + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}')) + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertIn("at or ahead", res.stdout) + + def test_a_revision_this_clone_has_never_seen_is_reported_not_guessed(self) -> None: + absent = "deadbeef" * 5 + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{absent}"}}')) + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn("not in this clone", res.stdout) + # No invented commit count: the honest answer is that it cannot be + # measured, not that it is current. + self.assertNotIn("commits behind", res.stdout) + + def test_lag_can_be_switched_off(self) -> None: + # A host that has not fetched origin, or a caller measuring one image + # against a fixed expected revision, should not be forced to have a + # lag answer. + absent = "deadbeef" * 5 + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{absent}"}}'), absent, "--behind-days", "0") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertNotIn("not in this clone", res.stdout) + + def test_warn_only_reports_without_failing(self) -> None: + # The mode diagnostics.yml uses: this workflow's own red X is the + # alert, and a stale deploy is not on #2222's list of things that + # should redden a scheduled run. + res = self.check( + "--healthz-exec", self.healthz('{"revision":"unknown"}'), "--behind-days", "0", "--warn-only" + ) + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertIn("not stamped", res.stdout) + + # ---- "could not tell" must never read as "it is fine" ---- + + def test_an_unreadable_body_exits_two_not_one(self) -> None: + # Traefik's 401 page, an HTML error from a wrong port, a plain "ok" + # from a different service: all of them are this script being pointed + # at the wrong thing. Exiting 1 would report them as a stale deploy. + for body in ("ok", "nope", '{"ok":true,"es":true}', "[]"): + with self.subTest(body=body): + res = self.check("--healthz-exec", self.healthz(body), "--behind-days", "0") + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + + def test_an_unreachable_endpoint_exits_two(self) -> None: + res = self.check("--healthz-url", "http://127.0.0.1:1/healthz", "--behind-days", "0") + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + + def test_no_endpoint_at_all_is_a_usage_error(self) -> None: + res = self.check() + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + self.assertIn("nothing to check", res.stderr) + + def test_a_branch_name_is_refused_as_the_expected_revision(self) -> None: + # "Is main deployed?" is the question; "main" is not the answer to it, + # and resolving it here would make the tool's own comparison circular. + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}'), "main") + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + self.assertIn("not a git object name", res.stderr) + + def test_a_missing_origin_main_says_how_to_fix_it(self) -> None: + self.git("update-ref", "-d", "refs/remotes/origin/main") + res = self.check("--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}')) + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + self.assertIn("fetch origin main", res.stderr) + + def test_a_nonsense_lag_threshold_is_refused(self) -> None: + res = self.check("--healthz-exec", self.healthz("{}"), "--behind-days", "soon") + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + self.assertIn("whole number of days", res.stderr) + + def test_two_expected_revisions_are_refused(self) -> None: + res = self.check("--healthz-exec", self.healthz("{}"), self.head, self.first) + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + self.assertIn("only one expected revision", res.stderr) + + def test_help_prints_the_headers_own_usage(self) -> None: + # A help text that has drifted from the flags is worse than none, and + # this one is generated from the header so it cannot. + res = run(SCRIPT, "--help") + self.assertEqual(res.returncode, 0) + for flag in ("--healthz-url", "--healthz-exec", "--image", "--repo-root", "--behind-days", "--warn-only"): + self.assertIn(flag, res.stdout) + + # ---- the image label half ---- + + def test_the_image_label_is_checked_too(self) -> None: + res = self.check( + "--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}'), + "--image", "apiary-backend:latest", "--behind-days", "0", + env={"DOCKER_BIN": str(self.docker_stub(self.head))}, + ) + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + + def test_an_image_whose_label_disagrees_with_the_running_container_is_a_finding(self) -> None: + # The failure this catches is real and cheap to miss by eye: a + # `docker compose up` that recreated a container from an older image + # than the one whose label an operator just read. + res = self.check( + "--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}'), + "--image", "apiary-backend:latest", "--behind-days", "0", + env={"DOCKER_BIN": str(self.docker_stub(self.first))}, + ) + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn(f"deployed {self.first}", res.stdout) + + def test_an_image_with_no_revision_label_names_the_missing_build_arg(self) -> None: + res = self.check( + "--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}'), + "--image", "apiary-backend:latest", "--behind-days", "0", + env={"DOCKER_BIN": str(self.docker_stub(""))}, + ) + self.assertEqual(res.returncode, 1, res.stdout) + self.assertIn("GIT_SHA", res.stdout) + + def test_a_missing_image_is_not_reported_as_an_unstamped_one(self) -> None: + # Two different facts with the same evidence (an empty label): the + # image is not on this host, versus it was built without the arg. The + # first is a wrong flag, the second is a rebuild. + res = self.check( + "--healthz-exec", self.healthz(f'{{"revision":"{self.head}"}}'), + "--image", "nope:latest", "--behind-days", "0", + env={"DOCKER_BIN": str(self.docker_stub("", exists=False))}, + ) + self.assertEqual(res.returncode, 2, res.stdout + res.stderr) + self.assertIn("no such image", res.stderr) + + def test_the_curl_path_is_driven_through_the_same_comparison(self) -> None: + # --healthz-exec is the form this fleet needs (backend-service + # publishes no port), but the HTTP form is what a proxied dashboard + # gives, and both must reach the same verdict. + stub = write_exec( + self.tmp / "curl", + f"""#!/usr/bin/env bash +# Stub: the real curl is called with -sf --max-time 10 . +printf '%s\\n' {json.dumps(f'{{"revision":"{self.head}"}}')} +""", + ) + res = self.check( + "--healthz-url", "http://dashboard.invalid/healthz", "--behind-days", "0", + env={"CURL_BIN": str(stub)}, + ) + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-deploy.sh b/scripts/verify-deploy.sh new file mode 100755 index 000000000..fd4e97fdb --- /dev/null +++ b/scripts/verify-deploy.sh @@ -0,0 +1,374 @@ +#!/usr/bin/env bash +# verify-deploy.sh -- #3315: prove mechanically that what is deployed is the +# code we think is deployed. +# +# The problem this closes, in the issue's own words: deploy verification was +# "manual inference -- compare `docker images` creation time with the merge +# time, or grep a string out of the shipped binary". docs/ARCANE-GIT-SYNC.md +# documents what that inference costs: apiary-backend:latest on the homeserver +# carried `Labels: null` and a creation time of 2026-09-08, every merge since +# was undeployed, and nothing surfaced it. "Green, healthy, running the old +# code" is indistinguishable from a good deploy by every signal the stack +# reports. +# +# Since #3315 there are three stamps, all from the same build arg (GIT_SHA): +# +# backend-service the revision is compiled into the binary by build.rs and +# answered by /healthz as {"revision": ...} +# dashboard-next the revision is baked into public/build.json, served at +# /build.json +# both images org.opencontainers.image.revision on the image config +# +# This script reads them and compares them against a git ref, so the question +# becomes "is the deployed revision the one I asked for?" instead of a reading +# of two timestamps. It also measures the lag -- commits behind, and the age of +# the deployed commit -- which is the number that says "this is fine, it is just +# three weeks old" or "this is not fine". +# +# Every comparison is a pure function over values already in hand (compare_ +# revision, evaluate_lag) so the decisions can be tested without a live host, +# the same split scripts/arcane-verify-recreate.sh uses. +# +# Usage: +# verify-deploy.sh [options] [] +# +# the revision that should be deployed. Defaults to +# origin/main in --repo-root. +# --healthz-url URL HTTP endpoint whose JSON body carries `revision`. +# --healthz-exec CMD command whose stdout is that same JSON. Needed for +# backend-service, whose /healthz is internal-only: the +# image publishes no port, so from the host it is only +# reachable with `docker exec`. Example: +# +# --healthz-exec 'docker exec hp-apiary-backend sh -c +# "curl -sf http://127.0.0.1:${LISTEN_ADDR##*:}/healthz"' +# +# (The `${LISTEN_ADDR##*:}` is the image's own HEALTHCHECK +# trick: the port is derived at check time, not +# hardcoded, because several stacks mount this image on +# a non-default LISTEN_ADDR.) +# --image REF also read REF's org.opencontainers.image.revision +# label, and compare it to the same expected revision. +# The image carrying one revision while the container +# running from it reports another is its own finding. +# --repo-root DIR clone to read origin/main from (default: cwd). +# --behind-days N report lag once the deployed commit is more than N +# days old (default 3; 0 disables the lag check). +# --warn-only print every finding, exit 0 anyway. For callers that +# report rather than gate (diagnostics.yml). +# -h, --help this header. +# +# Exit codes: +# 0 every stamp agrees with the expected revision, or --warn-only. +# 1 a stamp is missing, unstamped, disagrees, or is stale past --behind-days. +# 2 the check could not be run at all: bad usage, no origin/main to compare +# against, or the healthz endpoint did not answer. Distinct from 1 on +# purpose -- "we could not tell" must never be read as "it is fine". +# +# Environment overrides (also what scripts/tests stubs, mirroring the +# SYFT_BIN/TRIVY_BIN convention in generate-image-sbom.sh): +# CURL_BIN, DOCKER_BIN, PYTHON_BIN, GIT_BIN +set -euo pipefail + +readonly UNKNOWN="unknown" +readonly DEFAULT_BEHIND_DAYS=3 + +die() { + echo "verify-deploy: $*" >&2 + exit 2 +} + +usage() { + # Print this file's own leading comment block, minus the shebang: a help + # text that drifts out of date with the code below it is worse than none. + awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0" +} + +# ---------------------------------------------------------------- pure ---- + +# Lowercase, and tolerate the two spellings the same object arrives in: +# `git rev-parse`'s bare name and the OCI spec's `sha256:`-prefixed one. +# A short name is kept short -- compare_revision decides prefix equivalence, +# which it can only do if the short form stays recognisable. +normalize_sha() { + local raw=${1:-} + raw=${raw#sha256:} + printf '%s' "${raw,,}" +} + +# Is this string a plausible git object name at all? Same 7-64 hex rule the +# two tiers apply to the value they stamp (backend-service's +# normalize_revision, frontend-next's normalizeRevision) -- one rule in three +# places is a smell, but a value that is not an object name must never be +# compared as if it were, and this is the place that decides. +is_object_name() { + local candidate + candidate=$(normalize_sha "${1:-}") + [ ${#candidate} -ge 7 ] && [ ${#candidate} -le 64 ] && + [[ $candidate =~ ^[0-9a-f]+$ ]] +} + +# compare_revision -- prints one verdict line, returns 0 +# when the two agree. +# +# The prefix case is the common one and must not be a finding: `git rev-parse +# --short HEAD` and a full `github.sha` are the same commit, and an operator +# who set GIT_SHA from the short form would otherwise be told their deploy is +# wrong on every run. +compare_revision() { + local observed expected short_len + observed=$(normalize_sha "${1:-}") + expected=$(normalize_sha "${2:-}") + + if [ -z "$observed" ] || [ "$observed" = "$UNKNOWN" ]; then + echo "FAIL not stamped: this build carries no revision, so nothing here can say which code is deployed" + return 1 + fi + if ! is_object_name "$observed"; then + echo "FAIL '$observed' is not a git object name -- the stamp is malformed, not merely stale" + return 1 + fi + if [ "$observed" = "$expected" ]; then + echo "PASS $observed matches the expected revision" + return 0 + fi + if is_object_name "$expected"; then + # A short deployed name against a longer expected one. The length guards + # are what keep this from accepting a truncated *prefix* of a different + # commit: both names are already >= 7 hex chars, which is the shortest + # unambiguous abbreviation, and 7 of 40 shared hex characters is a real + # possibility, not a theoretical one. + if [ ${#observed} -le ${#expected} ] && [ "${expected:0:${#observed}}" = "$observed" ]; then + echo "PASS $observed is an abbreviation of $expected" + return 0 + fi + if [ ${#expected} -le ${#observed} ] && [ "${observed:0:${#expected}}" = "$expected" ]; then + echo "PASS $expected is an abbreviation of $observed" + return 0 + fi + fi + echo "FAIL deployed $observed, expected $expected" + return 1 +} + +# evaluate_lag [subject] +# +# Only ever called once the deployed revision has been resolved to a real +# commit in the repo, so both numbers are measured rather than estimated. The +# age is of the *commit*, not of the container: an image rebuilt yesterday from +# a month-old revision is a month-old deploy, and reading its container creation +# time is the very inference this script exists to replace. +evaluate_lag() { + local behind=$1 age=$2 threshold=$3 subject=${4:-deploy} + [ "$threshold" -gt 0 ] || return 0 + if [ "$behind" -le 0 ]; then + echo "PASS $subject is at or ahead of the compared ref (0 commits behind)" + return 0 + fi + if [ "$age" -gt "$threshold" ]; then + echo "FAIL $subject is $behind commits behind and its commit is ${age}d old (threshold ${threshold}d) -- a merge has been sitting undeployed" + return 1 + fi + echo "WARN $subject is $behind commits behind, commit ${age}d old (threshold ${threshold}d)" + return 0 +} + +# ------------------------------------------------------------------ io ---- + +CURL_BIN=${CURL_BIN:-curl} +DOCKER_BIN=${DOCKER_BIN:-docker} +PYTHON_BIN=${PYTHON_BIN:-python3} +GIT_BIN=${GIT_BIN:-git} + +# `safe.directory` on every git call rather than trusting the caller's config: +# the clone this is meant to read is usually owned by another user (the deploy +# runner's), and git refuses to operate on it otherwise -- the same reason +# diagnostics.yml passes it explicitly. +git_at() { + local root=$1 + shift + "$GIT_BIN" -c safe.directory="$root" -C "$root" "$@" +} + +# Pull `revision` out of a JSON body. python3 rather than jq because every +# other script in this directory that needs JSON parsing (generate-image-sbom.sh) +# already depends on python3, while jq is not installed on the hosts these run +# on. Non-JSON and a missing key are both errors, not silent empties: this +# script's entire value is that its answer came from the field it names. +extract_revision() { + "$PYTHON_BIN" -c ' +import json, sys +try: + doc = json.load(sys.stdin) +except ValueError as error: + sys.exit(f"not JSON ({error})") +if not isinstance(doc, dict) or "revision" not in doc: + sys.exit("no 'revision' field in the response (keys: %s)" % (sorted(doc) if isinstance(doc, dict) else type(doc).__name__)) +print(doc["revision"]) +' +} + +read_healthz_revision() { + local url=$1 exec_cmd=$2 body + if [ -n "$url" ]; then + # -f so an HTTP error page cannot be parsed as a stamp; --max-time so an + # endpoint behind a wedged network fails the check instead of hanging it. + body=$("$CURL_BIN" -sf --max-time 10 "$url") || + die "could not read $url -- is the service up?" + else + # A command, deliberately, not a container name: reaching an internal-only + # /healthz needs `docker exec` with the port derived from the container's own + # LISTEN_ADDR, and hardcoding either half here would be wrong for every + # stack that is not the default. + # shellcheck disable=SC2086 # the caller's own quoting, passed through verbatim + body=$(eval "$exec_cmd") || + die "the --healthz-exec command failed: $exec_cmd" + fi + [ -n "$body" ] || die "the healthz endpoint answered with an empty body" + # A body that is not the JSON this script was promised is "we could not + # tell", not "it disagrees" -- exit 2, never 1. Conflating them would make a + # rewired or auth-gated endpoint look like a stale deploy. + if ! printf '%s' "$body" | extract_revision; then + die "could not read a 'revision' out of the healthz body (see the message above)" + fi +} + +read_image_revision() { + local ref=$1 + "$DOCKER_BIN" image inspect --format \ + '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$ref" 2>/dev/null | + head -1 +} + +# ---------------------------------------------------------------- main ---- + +main() { + local healthz_url="" healthz_exec="" image="" repo_root="$PWD" + local behind_days=$DEFAULT_BEHIND_DAYS warn_only=0 expected="" + + while [ $# -gt 0 ]; do + case "$1" in + --healthz-url) + healthz_url=${2:?--healthz-url needs a value} + shift 2 + ;; + --healthz-exec) + healthz_exec=${2:?--healthz-exec needs a value} + shift 2 + ;; + --image) + image=${2:?--image needs a value} + shift 2 + ;; + --repo-root) + repo_root=${2:?--repo-root needs a value} + shift 2 + ;; + --behind-days) + behind_days=${2:?--behind-days needs a value} + shift 2 + ;; + --warn-only) + warn_only=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + -*) die "unknown option: $1 (try --help)" ;; + *) + [ -z "$expected" ] || die "only one expected revision may be given (got '$expected' and '$1')" + expected=$1 + shift + ;; + esac + done + + [ -n "$healthz_url" ] || [ -n "$healthz_exec" ] || + die "nothing to check: pass --healthz-url, --healthz-exec, or both (try --help)" + [[ $behind_days =~ ^[0-9]+$ ]] || die "--behind-days wants a whole number of days, got '$behind_days'" + [ -d "$repo_root" ] || die "--repo-root $repo_root is not a directory" + command -v "$GIT_BIN" >/dev/null || die "git is required to resolve origin/main" + command -v "$PYTHON_BIN" >/dev/null || die "python3 is required to read the JSON body" + + local failures=0 observed="" + + # What the deploy should be. An explicit argument wins; otherwise main. Either + # way it has to resolve to a commit in *this* clone, because that is the only + # thing the lag check can measure against. + if [ -z "$expected" ]; then + expected=$(git_at "$repo_root" rev-parse --verify --quiet origin/main) || + die "no origin/main in $repo_root -- run 'git -C $repo_root fetch origin main', or pass the expected revision explicitly" + fi + if ! is_object_name "$expected"; then + die "'$expected' is not a git object name -- pass a revision, not a branch or a tag" + fi + local main_revision + main_revision=$(git_at "$repo_root" rev-parse --verify --quiet origin/main 2>/dev/null || echo "") + echo "INFO expecting $expected${main_revision:+ (origin/main is $main_revision)}" + + # Read once. A second fetch could land after a redeploy and make the two + # halves of this report describe different deploys, which is the one thing a + # verification script must not do. `|| exit 2` rather than relying on + # set -e: read_healthz_revision's die() runs in this command substitution's + # subshell, and the "we could not tell" code has to survive that hop. + observed=$(read_healthz_revision "$healthz_url" "$healthz_exec") || exit 2 + compare_revision "$observed" "$expected" || failures=$((failures + 1)) + + if [ -n "$image" ]; then + command -v "$DOCKER_BIN" >/dev/null || + die "docker is required for --image (or drop the flag to check only the endpoint)" + local label + label=$(read_image_revision "$image" || true) + if [ -z "$label" ]; then + # A missing image and an image with no labels are different facts, and + # the second is the one this issue is about. Say which one this is. + if "$DOCKER_BIN" image inspect "$image" >/dev/null 2>&1; then + echo "FAIL $image carries no org.opencontainers.image.revision label -- built without --build-arg GIT_SHA" + failures=$((failures + 1)) + else + die "no such image: $image" + fi + else + compare_revision "$label" "$expected" || failures=$((failures + 1)) + fi + fi + + # Lag, measured from the endpoint's revision. This is the number the issue + # asks diagnostics to warn on, and it is the one an operator can act on + # without a rollback: a deploy three weeks behind main is a scheduling fact, + # not a broken image. + if [ "$behind_days" -gt 0 ]; then + local deployed_commit behind age_days commit_epoch now_epoch + if deployed_commit=$(git_at "$repo_root" rev-parse --verify --quiet "${observed}^{commit}" 2>/dev/null); then + behind=$(git_at "$repo_root" rev-list --count "${deployed_commit}..origin/main" 2>/dev/null || echo "") + commit_epoch=$(git_at "$repo_root" log -1 --format=%ct "$deployed_commit" 2>/dev/null || echo "") + now_epoch=$(date -u +%s) + if [ -n "$behind" ] && [ -n "$commit_epoch" ]; then + age_days=$(( (now_epoch - commit_epoch) / 86400 )) + # A negative age means the deployed commit is dated in the future, which + # is a clock problem, not a stale deploy. Report it as 0 rather than + # letting a -3d read as "very stale". + [ "$age_days" -ge 0 ] || age_days=0 + evaluate_lag "$behind" "$age_days" "$behind_days" "deployed revision" || failures=$((failures + 1)) + else + echo "INFO could not measure how far behind the deployed revision is (no rev-list/log for it in $repo_root)" + fi + else + echo "WARN deployed revision $observed is not in this clone -- cannot measure lag; fetch it, or compare by hand" + fi + fi + + if [ "$failures" -gt 0 ]; then + if [ "$warn_only" -eq 1 ]; then + echo "INFO $failures finding(s) reported; --warn-only, so not failing the run" + exit 0 + fi + echo "FAIL $failures finding(s): the deploy is not verifiably the expected revision" + exit 1 + fi + echo "PASS every stamp checked agrees with $expected" +} + +main "$@"