feat(deploy): stamp both dashboard images with their git revision (#3315)

.github/workflows/containers.yml

@@ -77,6 +77,15 @@ jobs:
             # into the digest-keyed CycloneDX + Trivy steps below. Absent on
             # the other sixteen rows, so they build exactly as before.
             sbom: true
+            # #3315: the two dashboard images that now report which commit
+            # they were built from -- one compiled into the binary (where
+            # /livez and its /healthz alias answer it as `revision`) or baked
+            # into the served /build.json, and both on the image's
+            # org.opencontainers.image.revision label. Only these two
+            # Dockerfiles declare `ARG GIT_SHA`, so only these two rows pass
+            # it; the other sixteen would take an unused-build-arg warning
+            # per build for an arg they do not read.
+            stamp: true
             # #3316: ...and the two images that are actually deployed as
             # long-lived HTTP services are also started and waited on
             # below. `boot_smoke: true` is the reason a build row needs a
@@ -98,6 +107,7 @@ jobs:
           - image: dashboard-next
             context: arcane/home/honeypot-dashboard/frontend-next
             sbom: true
+            stamp: true
             boot_smoke: true
           - image: dicompot
             context: arcane/home/honeypot-dicompot/dicompot
@@ -368,6 +378,18 @@ jobs:
           # boot-smoke rows: on a push it is redundant (the digest names the
           # image) and the manifest should not carry a CI run id.
           labels: ${{ steps.metadata.outputs.labels }}${{ matrix.boot_smoke == true && format('\napiary.ci.build-row={0}-{1}-{2}', matrix.image, github.run_id, github.run_attempt) || '' }}
+          # #3315: the revision this image is built from, for the two rows
+          # whose Dockerfile declares `ARG GIT_SHA` (matrix.stamp). It has to
+          # be a build arg and not only a label: backend-service compiles it
+          # into the binary, and nothing in the shipped image can be edited
+          # after the fact without changing the image id. Note that
+          # metadata-action already emits its own org.opencontainers.revision
+          # for the same image, and both are wired from the same
+          # `${{ github.sha }}` here -- so whichever key wins the merge, the
+          # label and the compiled-in value cannot disagree about which commit
+          # this is. scripts/verify-deploy.sh compares them against origin/main
+          # and against the live /healthz.
+          build-args: ${{ matrix.stamp == true && format('GIT_SHA={0}', github.sha) || '' }}
           # Scope the layer cache per image (#2771). Without an explicit
           # scope every matrix row defaults to `buildkit`, so all 18
           # concurrent builds read and write ONE shared cache index per

.github/workflows/diagnostics.yml

@@ -145,6 +145,55 @@ jobs:
             alert "healthz: **unreachable** — the dashboard container is down or not bound"
           fi
 
+          # #3315: "green, healthy, running the old code" is indistinguishable
+          # from a correct deploy by looking at it -- a fresh container, a
+          # passing healthcheck and a recent image build all describe the
+          # machinery, not the code. Both dashboard images now carry their git
+          # revision (#/healthz's `revision` field and
+          # org.opencontainers.image.revision), and scripts/verify-deploy.sh
+          # compares what is running against what is newest.
+          #
+          # The expected revision is $GITHUB_SHA rather than a clone's
+          # origin/main because this host has no usable one: deploy.yml's
+          # rsync into $stack carries --exclude .git/ (see its own step), and
+          # Arcane's directory sync has no .git either. On a schedule trigger
+          # GITHUB_SHA is the default branch's head, which is exactly the
+          # "newest merge" this has to be measured against. For the same
+          # reason --behind-days is 0 here: the day-count needs a clone to
+          # measure commit age against, and without one it would exit 2
+          # ("could not tell") on every run. Degrade to "is this the tip of
+          # main" and say so, rather than shipping a check that can only
+          # report that it failed to run.
+          #
+          # Report-only, like the Elasticsearch section: a stale deploy is a
+          # scheduling fact, not one of the failures #2222 lists. --warn-only
+          # stops the tool's own exit 1 from reddening a scheduled run; its
+          # exit 2 is still separated out below, because folding "could not
+          # tell" into a pass is the one outcome that would make this section
+          # worse than not having it.
+          section "Deployed revision"
+          verify=$stack/scripts/verify-deploy.sh
+          if [ ! -x "$verify" ]; then
+            report "not checked: $verify is not on this host yet (it arrives with the merge that adds it)"
+          else
+            if stamp=$("$verify" --warn-only --repo-root "$stack" \
+              --healthz-exec 'docker exec hp-apiary-backend curl -sf http://127.0.0.1:8081/healthz' \
+              --image apiary-backend:latest --behind-days 0 "$GITHUB_SHA" 2>&1); then
+              printf '%s\n' "$stamp" | tee -a "$GITHUB_STEP_SUMMARY"
+            else
+              stamp_rc=$?
+              printf '%s\n' "$stamp" | tee -a "$GITHUB_STEP_SUMMARY"
+              if [ "$stamp_rc" -eq 2 ]; then
+                report "could not tell: the deployed revision was not readable (exit 2), which is not a pass"
+              else
+                report "findings above: the running revision is not verifiably the expected one"
+              fi
+            fi
+            report 'PASS means the running binary was built from the current main tip.'
+            report 'A finding means it was not; "not stamped" means the image was built'
+            report 'without --build-arg GIT_SHA. Neither reddens this run.'
+          fi
+
           section "Sensor to Elasticsearch to Dashboard"
           # #2096: the Go dashboard's Prometheus /metrics died with it
           # (#1659) -- frontend-next owns :19090 now and answers 307

arcane/home/honeypot-dashboard-backend/.env.example

@@ -83,3 +83,26 @@ LLM_MODEL=qwen3:14b
 # the server default would evict a worker that runs continuously to serve
 # one that does not.
 LLM_KEEP_ALIVE=5m
+
+# #3315 -- the git revision apiary-backend is built from. Optional: unset
+# means the image is built with no revision, /healthz reports
+# {"revision":"unknown"} and the image's org.opencontainers.image.revision
+# label is "unknown", which scripts/verify-deploy.sh reports as an unstamped
+# deploy rather than passing silently.
+#
+# Take the value from the checkout you are actually deploying -- the commit
+# that was merged, not the branch name. Not from /opt/stacks/apiary, which
+# looks like a clone but is a `git ls-files`/rsync of the working tree with
+# .git/ excluded (see .github/workflows/deploy.yml's own sync step), so
+# `git -C /opt/stacks/apiary rev-parse HEAD` there fails. From a real clone:
+#   echo "GIT_SHA=$(git rev-parse HEAD)" >> .env
+# For an Arcane-driven build that is the only place it has to be set: the
+# synced directory has no .git, and compose will not invent one. For a manual
+# `docker compose build` in the synced directory, the environment works too:
+#   GIT_SHA=$(git rev-parse HEAD) docker compose build backend-service
+#
+# It is a revision, not a branch: the whole point is that "is the merge on the
+# host?" becomes a curl of /healthz rather than an inference from image
+# timestamps. Never a branch name, a tag, or a hand-typed guess -- a value that
+# is not an object name is normalized to "unknown" by the binary itself.
+GIT_SHA=

arcane/home/honeypot-dashboard-backend/compose.yml

@@ -79,7 +79,25 @@ services:
   # ------------------------------------------------------------------
   backend-service:
     <<: *runtime-defaults
-    build: ../honeypot-dashboard/backend-service
+    build:
+      context: ../honeypot-dashboard/backend-service
+      # #3315: the revision this image is built from. build.rs compiles it into
+      # the binary (where /healthz reports it as `revision`) and the Dockerfile
+      # stamps it on the image as org.opencontainers.image.revision, so the two
+      # can be compared against each other and against main by
+      # scripts/verify-deploy.sh.
+      #
+      # Read from this stack's .env, not invented here: compose cannot run
+      # `git rev-parse`, and Arcane's directory sync explicitly excludes .git
+      # from the tree it materializes (docs/ARCANE-GIT-SYNC.md), so the synced
+      # checkout has no repository to ask. Set it from the checkout you are
+      # deploying, before the build -- see this stack's .env.example.
+      #
+      # Unset is the honest default and yields "unknown" in /healthz and on the
+      # label rather than a build failure. A build that refuses to produce an
+      # unstamped image is a build that gets reverted; a stale one is reported.
+      args:
+        GIT_SHA: ${GIT_SHA:-}
     image: apiary-backend:latest
     # build: this image is never pushed to any registry -- Arcane's
     # redeploy does an unconditional `docker compose pull` across every

arcane/home/honeypot-dashboard/.env.example

@@ -265,3 +265,16 @@ DASHBOARD_BFF_LOG_FILE=/logs/dashboard-bff/app.jsonl
 # value into this stack's .env. Set it by hand only for a manual stand-up:
 #   getent group deploy-runner | cut -d: -f3
 DEPLOY_RUNNER_GID=980
+
+# #3315 -- the git revision apiary-dashboard-next is built from. Optional and
+# empty by default; see the identical block in
+# ../honeypot-dashboard-backend/.env.example for the full reasoning. Briefly:
+# it lands in the image's /build.json and its
+# org.opencontainers.image.revision label, and
+# scripts/verify-deploy.sh reports a missing one as an unstamped deploy
+# instead of passing silently.
+#
+# The two dashboard stacks carry the variable independently: a build of one
+# says nothing about the other, so set it in both .env files.
+#   echo "GIT_SHA=$(git rev-parse HEAD)" >> .env
+GIT_SHA=

arcane/home/honeypot-dashboard/backend-service/Dockerfile

@@ -3,12 +3,22 @@
 # (no OpenSSL); ca-certificates + curl for TLS trust and the container
 # healthcheck.
 FROM rust:1-slim-bookworm@sha256:94e9efa4033213dbb70d4f665527e7ece3944ddb7ba1dd2e43f6fd6e2490af58 AS build
+# #3315: which revision of the repository this binary is built from. Declared
+# in the build stage as well as the runtime one because build.rs reads it from
+# the environment to compile it in (/healthz's `revision` field) and the
+# runtime stage's LABEL cannot reach back into the compiler. Unset is normal,
+# not an error: a developer building this by hand gets "unknown" rather than a
+# broken build, and the whole point is that the answer is honest when it is
+# missing. Arcane passes it from the synced checkout via the stack's own
+# `build.args`; CI passes ${{ github.sha }}.
+ARG GIT_SHA
 WORKDIR /src
 COPY Cargo.toml Cargo.lock* ./
 # build.rs stamps the binary with its compile time (see the file). It has to
 # be copied explicitly: cargo only runs a build script that is actually in
 # the crate root, and without it env!("APIARY_BUILD_EPOCH") is a compile
-# error rather than a missing value.
+# error rather than a missing value. build.rs also carries GIT_SHA through as
+# APIARY_GIT_SHA since #3315.
 COPY build.rs ./
 COPY src ./src
 # report_pdf.rs's two include_bytes!("../assets_pdf/...") calls need this
@@ -20,6 +30,30 @@ COPY assets_pdf ./assets_pdf
 RUN cargo build --release
 
 FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171
+# Redeclared per stage: Docker scopes an ARG to the stage that declares it, so
+# the value the build stage compiled in is not in scope here on its own.
+ARG GIT_SHA
+# Static, so it is written here rather than taken from a build arg: the
+# repository URL is the same for every build of this image and there is no
+# reason to let a caller point the label somewhere else.
+ARG GIT_SOURCE=https://github.com/Xore/APIARY
+# Build time is the one label a build cannot honestly invent -- unlike the
+# revision, there is no earlier source of truth to copy it from. So it stays
+# "unknown" unless the builder supplies it, and CI does (containers.yml passes
+# metadata-action's labels, which carry a real created/source pair and take
+# precedence over these). An "unknown" here reads as "nobody told us", which is
+# true; a plausible-looking wrong timestamp would not be.
+ARG BUILD_DATE
+# #3315: `docker images` shows no revision at all without these. `Labels: null`
+# on a stale image is the exact state that let every merge since a redeploy go
+# undeployed without anything saying so (see the issue's homeserver example).
+# Read with: docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}'
+LABEL org.opencontainers.image.revision="${GIT_SHA:-unknown}" \
+      org.opencontainers.image.source="${GIT_SOURCE}" \
+      org.opencontainers.image.created="${BUILD_DATE:-unknown}" \
+      org.opencontainers.image.title="apiary-backend" \
+      org.opencontainers.image.description="APIARY dashboard backend service (Rust, axum)"
+
 RUN apt-get update \
     && apt-get install -y --no-install-recommends ca-certificates curl \
     && rm -rf /var/lib/apt/lists/*

arcane/home/honeypot-dashboard/backend-service/build.rs

@@ -12,18 +12,41 @@
 // A compile timestamp answers the question that actually gets asked after a
 // deploy: is the running binary newer than the merge? Git metadata would say
 // more, but the Docker build context is the crate directory alone and
-// carries no .git, so it cannot be read where it would need to be.
+// carries no .git, so it cannot be read where it would need to be -- it has
+// to be handed in as a build arg (GIT_SHA) instead, which is what #3315 added
+// once a timestamp stopped being enough to answer "which code is this?".
 use std::time::{SystemTime, UNIX_EPOCH};
 
 fn main() {
     // Re-run whenever the crate changes, so the stamp cannot go stale while
     // the code moves underneath it.
     println!("cargo:rerun-if-changed=src");
     println!("cargo:rerun-if-changed=Cargo.toml");
+    // ...and whenever GIT_SHA changes, which is the case this did not cover
+    // before #3315. cargo caches a build script's output by its inputs, and an
+    // environment variable is not one of them unless it is declared here: a
+    // rebuild of an identical tree with a different GIT_SHA would otherwise
+    // reuse the previous revision's compiled-in value and report itself as
+    // that revision. That is the "green, healthy, running the old code" shape
+    // this stamp exists to end, reproduced by the stamp itself.
+    println!("cargo:rerun-if-env-changed=GIT_SHA");
 
     let epoch = SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .map(|elapsed| elapsed.as_secs())
         .unwrap_or(0);
     println!("cargo:rustc-env=APIARY_BUILD_EPOCH={epoch}");
+
+    // Only the first whitespace-delimited token survives, and only because the
+    // cargo directive protocol is newline-separated: a GIT_SHA carrying a
+    // newline would otherwise inject arbitrary directives (extra
+    // rustc-envs, a rerun-if-changed pointing anywhere) into this build.
+    // Whether what came out is a plausible object name is decided by
+    // main.rs's normalize_revision, which is a plain function and therefore
+    // unit-testable; this is only the transport.
+    let sha = std::env::var("GIT_SHA")
+        .ok()
+        .and_then(|value| value.split_whitespace().next().map(str::to_string))
+        .unwrap_or_default();
+    println!("cargo:rustc-env=APIARY_GIT_SHA={sha}");
 }