diff --git a/.agents/notes/implemented/2026-10-12-a-credential-nobody-can-measure-is-not-a-posture.md b/.agents/notes/implemented/2026-10-12-a-credential-nobody-can-measure-is-not-a-posture.md new file mode 100644 index 00000000..86bf693d --- /dev/null +++ b/.agents/notes/implemented/2026-10-12-a-credential-nobody-can-measure-is-not-a-posture.md @@ -0,0 +1,54 @@ +--- +title: A credential nobody can measure is not a posture +status: implemented +date: 2026-10-12 +scope: stack +--- + +## Decision + +A mounted credential whose measuring tool is not in the image gets its own +verdict, `unmeasurable`, and it is never a pass: + +- a `readonly` node refuses to start under `HOOKPROBE_POSTURE_CHECK=enforce` + and says which tool is missing; +- a writing node (`danger-only`) starts, records the verdict, and pages on + `HOOKPROBE_ALARM_URL` at once; +- the hourly selftest's posture check fails until the tool is back; +- what WAS measured still outranks it: a violation found on one credential is + `wider-than-declared` whether or not the other credential's tool exists. + +And the shape that caused it is refused at the gate: services that share an +image tag must build it alike (`scripts/assert_shared_images_build_alike.py`, +in `scripts/gate.sh` and `ci-stack`). The work compose's `probe-watch` build now +passes `KUBECTL_VERSION` exactly as `probe-plan` and `probe-work` do. + +## Why + +The work stack's three probes share `hookprobe:work`. A compose builds every +service that has a `build:` block, and the tag holds whichever build ran last; +`probe-watch`'s build passed no kubectl version. From 2026-09-29 the image had +no kubectl. At every boot after that both credentialed nodes recorded +`can-i --list: kubectl: not found` — the planner's verdict `unverifiable`, the +work node's `recorded` with `mutating: []` — and both started. The hourly check +asks one question, "wider than declared?", and answered held every hour. The +stack ran this way until it was stopped on 2026-10-09; nothing paged, no gate +went red, and the record read like a cluster with no mutating grants. + +`unverifiable` was built for an identity that may not ask about itself. A +missing binary is a different silence: the far side was never asked, and the +operator can fix it with a rebuild — which is exactly why it must be loud. + +## Consequences + +- A rebuild that drops a tool now stops a readonly node at boot instead of + degrading its planner quietly; a writing node keeps running but says so. +- The posture record carries `tool_missing` on the credential it could not + measure, and `GET /v1/posture` and the audit cite it. +- `assert_guards_are_tested.py` carries the mutation: turning `unmeasurable` + back into `recorded` must fail `tests/test_posture.py`. +- The new gate check parses the composes by indentation and has a floor; a + compose written in another shape, or one that stops building something, + needs the floor lowered on purpose. +- Already-built images are not rebuilt by this change; the work image needs + `docker compose ... build` before the next recreate. diff --git a/.github/workflows/ci-stack.yml b/.github/workflows/ci-stack.yml index 63a8ab1e..cad9df64 100644 --- a/.github/workflows/ci-stack.yml +++ b/.github/workflows/ci-stack.yml @@ -133,6 +133,11 @@ jobs: # in is a comment. - name: the pipe and the brain are inside their stated ceilings run: python3 scripts/assert_weight.py + # Three services build one image tag, and the tag is whichever build ran + # last: a build arg one of them does not pass is an image the others did + # not mean. The work image shipped without kubectl this way (2026-09-29). + - name: services sharing an image tag build it alike + run: python3 scripts/assert_shared_images_build_alike.py # A live production component on the card-callback path, outside every # service package and therefore outside every component gate. It was # covered by nothing at all until it was edited and the question was asked. diff --git a/deploy/docker-compose.work.yml b/deploy/docker-compose.work.yml index 2469c474..c9d3beb5 100644 --- a/deploy/docker-compose.work.yml +++ b/deploy/docker-compose.work.yml @@ -491,6 +491,14 @@ services: context: ../hookprobe args: APT_EXTRAS: ${PROBE_PLAN_APT_EXTRAS:-git awscli} + # ARG for ARG the planner's build: the three probes share one image tag + # and the tag is whichever build ran last. This block passed no kubectl + # version until 2026-10-12, so from 2026-09-29 the tag was this + # service's kubectl-less build and the planner and the work node started + # every day unable to measure their cluster side (posture.py says what + # that cost). scripts/assert_shared_images_build_alike.py keeps the + # three builds alike. + KUBECTL_VERSION: ${PROBE_PLAN_KUBECTL_VERSION:-v1.31.4} image: hookprobe:work container_name: work-probe-watch restart: unless-stopped @@ -685,7 +693,11 @@ services: build: context: ../hookprobe args: - APT_EXTRAS: ${PROBE_WORK_APT_EXTRAS:-git awscli} + # The planner's variables, not a PROBE_WORK_ pair: the three probes share + # one image tag, so one set of build args is the only honest shape. A + # second variable with the same default would let .env fork the image + # and leave the tag holding whichever build ran last. + APT_EXTRAS: ${PROBE_PLAN_APT_EXTRAS:-git awscli} KUBECTL_VERSION: ${PROBE_PLAN_KUBECTL_VERSION:-v1.31.4} image: hookprobe:work container_name: work-probe-work diff --git a/docs/containment.md b/docs/containment.md index c1850f3e..e721ae25 100644 --- a/docs/containment.md +++ b/docs/containment.md @@ -17,7 +17,10 @@ a real model cannot be steered into emitting a harmful one. > (`hookprobe/posture.py`): what `kubectl auth can-i` and `iam simulate-principal-policy` say the identity may do, against > the posture it declares. Declared `readonly` with credentials that can mutate refuses to start (`HOOKPROBE_POSTURE_CHECK=enforce`); > a `danger-only` runner records its blast radius; an identity that cannot be asked is reported *unverifiable*, never confirmed. -> The measurement is served on `GET /v1/posture` and cited in every run's audit record. +> A credential whose measuring tool is not in the image is *unmeasurable* (2026-10-12): a readonly runner refuses to start +> under `enforce`, a writing runner starts and pages, and the hourly selftest fails until the tool is back — ten days of +> `kubectl: not found` had passed as `unverifiable` and `recorded` before this. The measurement is served on `GET /v1/posture` +> and cited in every run's audit record. ## What holds, and what only helps diff --git a/hookprobe/README.md b/hookprobe/README.md index 4045d4eb..9fe2851b 100644 --- a/hookprobe/README.md +++ b/hookprobe/README.md @@ -83,7 +83,7 @@ caller ── POST /hooks/agent ───────────────▶ | `GET /v1/automation` | Each class of automation, its declared ceiling, its record of proposals and the human decisions on them, and the tier the record WOULD support. Counters, never agreement rates. The page an operator reads before deciding a class has earned a step up. | | `POST /v1/automation/{class}/{id}/regret` | A sampling review saying an auto-applied action was wrong — the one event that resets a class's argument for its tier, and how an unattended deployment gets its human-in-the-loop back asynchronously. Operator token only; no run can label its own work. | | `POST /v1/runs/{key}/ruling` | Rule one run useful/useless (or clear) from the sessions page, and be told what it DID: a useless that withdraws the runbook this run distilled says so. The only ruling surface that needs no chat bridge — the card buttons were the only other one, and a deployment without a bridge has none, which is why the loops that feed on a ruling went unfed. `record_ruling` runs in-process here, so the consequence is reported synchronously rather than enqueued and forgotten. | -| `GET /v1/posture` | what the credentials could do when this runner started, measured against the declared posture — kube rules that mutate, AWS actions that would be allowed, or `unverifiable` when the identity may not ask; the record behind "this ran read-only" (see `HOOKPROBE_POSTURE_CHECK`) | +| `GET /v1/posture` | what the credentials could do when this runner started, measured against the declared posture — kube rules that mutate, AWS actions that would be allowed, `unverifiable` when the identity may not ask, or `unmeasurable` when the tool that would ask is not in the image; the record behind "this ran read-only" (see `HOOKPROBE_POSTURE_CHECK`) | | `GET /v1/work` | the board: one row per piece of WORK rather than per run, with what is blocked, what is in flight and what closed without anybody stepping in. Its `counts` also answer the two questions an on-call opens a board with — `median_answer_seconds`, how long until the work was any use, and `signals` against the items they became, which is how much noise the door folded. Both are `null` rather than `0` when nothing has answered yet, because those are different facts. Derived on every request from the run records, the proposals and the memory queue — nothing is stored twice ([work.py](hookprobe/work.py)) | | `POST /v1/runs/{key}/retry` | human takeover: try a failed investigation again. Continues the engine session when the failure left one — the second attempt starts from what the first gathered — and otherwise re-asks the question the run opened with. Not budget-gated, for the reason `/hooks/agent` is not | | `GET /v1/agent` | what this node is: name, role, runtime adapter and model, the policy it runs under (guard, budget, escalation bar, who may speak to it from chat) and its health. Settings this process resolved, answered in one place so a board can name the agent that did the work; no secret values, like `/v1/config` | @@ -175,7 +175,9 @@ always reach. Two things about the word *default*, before the layers: actually do (`kubectl auth can-i --list`, `aws iam simulate-principal-policy` over the dangerous actions) and compares the answer with the declaration: `readonly-confirmed`, `wider-than-declared` (refuses to start under the - default `HOOKPROBE_POSTURE_CHECK=enforce`), `unverifiable`, `recorded`, + default `HOOKPROBE_POSTURE_CHECK=enforce`), `unverifiable`, `unmeasurable` + (the measuring tool is not in the image: a readonly node refuses to start + under `enforce`, a writing node starts and pages), `recorded`, `no-credentials`. The verdict is at `GET /v1/posture` and at the top of every run's audit, so "read-only" is a measurement with a date rather than a sentence in this file. diff --git a/hookprobe/hookprobe/app.py b/hookprobe/hookprobe/app.py index 7390b063..4dab819d 100644 --- a/hookprobe/hookprobe/app.py +++ b/hookprobe/hookprobe/app.py @@ -223,7 +223,11 @@ async def lifespan(_: FastAPI): # Before the first run: are the credentials as narrow as the posture # says? Under enforce a wider-than-declared runner does not come up. await posture.on_startup( - settings.workdir, settings.bash_guard, settings.posture_check, radius=settings.blast_radius + settings.workdir, + settings.bash_guard, + settings.posture_check, + radius=settings.blast_radius, + alarm=service.alarm, ) service.recover_orphans() service.sweep_interrupted_remediations() diff --git a/hookprobe/hookprobe/posture.py b/hookprobe/hookprobe/posture.py index bfece8fc..e0ae4847 100644 --- a/hookprobe/hookprobe/posture.py +++ b/hookprobe/hookprobe/posture.py @@ -28,11 +28,22 @@ Unverifiable is a verdict, not an error: a read-only AWS identity often may not call SimulatePrincipalPolicy on itself. That is reported as such — loudly — and never mistaken for confirmed. + +Unmeasurable is the verdict for a credential whose measuring tool is not in the +image, and it is a different silence: the far side was never asked. It happened. +The work stack's three probes share one image tag, one of their builds passed no +kubectl version, and from 2026-09-29 to 10-09 the planner and the work node +started every day with `kubectl: not found` in their records — the planner +`unverifiable`, the writer `recorded` with an empty radius — and nothing paged, +because both verdicts start and the hourly check only asks "wider?". Now a +readonly node refuses to start under `enforce`, a writing node starts and pages +through the alarm channel, and the hourly selftest fails until the tool is back. """ from __future__ import annotations import asyncio +import contextlib import json import logging import os @@ -74,7 +85,8 @@ class PostureViolation(RuntimeError): - """The credentials are wider than the declared posture, and the mode is enforce.""" + """The posture cannot be held as declared under enforce: the credentials are + wider than it, or could not be measured at all.""" async def run_cli(argv: list[str]) -> tuple[int, str, str]: @@ -157,6 +169,12 @@ async def check_kube(run: Runner) -> dict[str, Any]: return {"present": False} record: dict[str, Any] = {"present": True, "kubeconfig": str(path), "mutating": [], "errors": []} code, out, err = await run(["kubectl", "auth", "can-i", "--list"]) + if code == 127: + # The binary is not in the image. Four more questions of it would be + # four more copies of this line; the verdict needs the one. + record["tool_missing"] = "kubectl" + record["errors"].append(f"can-i --list: {(err or out).strip()[:200]}") + return record if code != 0: record["errors"].append(f"can-i --list: {(err or out).strip()[:200]}") else: @@ -180,6 +198,8 @@ async def check_aws(run: Runner) -> dict[str, Any]: if code != 0: record["errors"].append(f"get-caller-identity: {(err or out).strip()[:200]}") record["unverifiable"] = True + if code == 127: + record["tool_missing"] = "aws" return record try: arn = str(json.loads(out).get("Arn") or "") @@ -243,7 +263,18 @@ def beyond(kube: dict[str, Any], aws: dict[str, Any], declared: set[str] | None) return sorted(item for item in measured if item not in declared) +def tools_missing(kube: dict[str, Any], aws: dict[str, Any]) -> list[str]: + """The measuring tools a mounted credential could not be asked with.""" + return [ + str(record["tool_missing"]) for record in (kube, aws) if record.get("present") and record.get("tool_missing") + ] + + def verdict(bash_guard: str, kube: dict[str, Any], aws: dict[str, Any], declared: set[str] | None = None) -> str: + # A credential nobody could measure is not a posture of any kind. What WAS + # measured still outranks it: a violation found on one credential is a + # violation whether or not the other one's tool is installed. + missing = tools_missing(kube, aws) if bash_guard != "readonly": # A writing posture is ALLOWED to write — the question is never "can it" # but "is it wider than what somebody agreed to". Undeclared, there is @@ -251,12 +282,16 @@ def verdict(bash_guard: str, kube: dict[str, Any], aws: dict[str, Any], declared # what it was for a year. Declared, `enforce` means the same thing on # both postures: the credential may not exceed the declaration. if declared is None: - return "recorded" - return "wider-than-declared" if beyond(kube, aws, declared) else "within-declared-radius" + return "unmeasurable" if missing else "recorded" + if beyond(kube, aws, declared): + return "wider-than-declared" + return "unmeasurable" if missing else "within-declared-radius" if not kube.get("present") and not aws.get("present"): return "no-credentials" if kube.get("mutating") or aws.get("allowed"): return "wider-than-declared" + if missing: + return "unmeasurable" if kube.get("errors") or aws.get("unverifiable"): return "unverifiable" return "readonly-confirmed" @@ -287,11 +322,13 @@ def summary(record: dict[str, Any]) -> str: parts.append( f"kube mutating={len(kube.get('mutating') or [])}" + (f" errors={len(kube['errors'])}" if kube.get("errors") else "") + + (f" tool-missing={kube['tool_missing']}" if kube.get("tool_missing") else "") ) if aws.get("present"): parts.append( f"aws identity={aws.get('identity') or '?'} dangerous-allowed={len(aws.get('allowed') or [])}" + (" unverifiable" if aws.get("unverifiable") else "") + + (f" tool-missing={aws['tool_missing']}" if aws.get("tool_missing") else "") ) return " · ".join(parts) @@ -314,6 +351,7 @@ async def on_startup( mode: str, run: Runner = run_cli, radius: Path | None = None, + alarm: Callable[[str], Awaitable[bool]] | None = None, ) -> dict[str, Any] | None: """Check, record, announce — and under `enforce`, refuse to start a runner whose credentials are wider than it declares. @@ -322,6 +360,11 @@ async def on_startup( `danger-only` runner has always started whatever it held, because "allowed to write" left nothing to compare against; with it, `enforce` means the same sentence on both postures. + + `alarm` is the node's road around the pipe (`HOOKPROBE_ALARM_URL`). A + credential that could not be measured is said there at once, on both + postures: the readonly node is about to refuse and loop, and a writing node + is about to start with a radius nobody measured. """ if mode not in MODES: mode = "enforce" # a typo in the switch that decides whether to refuse must fail closed @@ -346,6 +389,18 @@ async def on_startup( logger.error("REFUSING TO START — credentials wider than declared posture: %s · %s", line, detail) raise PostureViolation(f"credentials wider than declared posture {bash_guard!r}: {detail}") logger.warning("credentials wider than declared posture: %s · %s", line, detail) + elif record["verdict"] == "unmeasurable": + missing = ", ".join(tools_missing(record["kube"], record["aws"])) + text = f"posture cannot be measured: {missing} not in this image · {line}" + if alarm is not None: + with contextlib.suppress(Exception): # the page is a courtesy; the verdict below is the decision + await alarm(f"posture UNMEASURABLE — {text}") + if bash_guard == "readonly" and mode == "enforce": + logger.error("REFUSING TO START — %s", text) + raise PostureViolation( + f"posture cannot be measured as declared {bash_guard!r}: {missing} not in this image" + ) + logger.error("%s", text) elif record["verdict"] == "unverifiable": logger.warning( "posture could not be fully verified: %s · %s", diff --git a/hookprobe/hookprobe/selftest.py b/hookprobe/hookprobe/selftest.py index 4c92586e..b1a1f051 100644 --- a/hookprobe/hookprobe/selftest.py +++ b/hookprobe/hookprobe/selftest.py @@ -213,9 +213,15 @@ async def posture_still_holds(settings: Settings) -> dict[str, Any]: beyond = record.get("beyond_declared") or [] if verdict == "wider-than-declared" and beyond: detail = f"{detail} · beyond: {', '.join(str(item) for item in beyond)}"[:280] + if verdict == "unmeasurable": + # The tool that would measure it is not in the image. Ten days of + # `kubectl: not found` passed this check hourly (2026-09-29 to 10-09) + # because the only question asked was "wider?". + missing = ", ".join(posture.tools_missing(record.get("kube") or {}, record.get("aws") or {})) + detail = f"{detail} · cannot measure: {missing} not in this image"[:280] return _check( "credentials are no wider than the declared posture", - verdict != "wider-than-declared", + verdict not in ("wider-than-declared", "unmeasurable"), detail, "a credential whose far side cannot be asked — `unverifiable` is a verdict, not a pass", ) diff --git a/hookprobe/tests/test_posture.py b/hookprobe/tests/test_posture.py index 27e7344c..9ae31852 100644 --- a/hookprobe/tests/test_posture.py +++ b/hookprobe/tests/test_posture.py @@ -222,3 +222,64 @@ async def fake_run(argv, **kw): "recorded even though it refused — the refusal must be explainable" ) assert record["beyond_declared"] == ["configmaps: patch update"], "the record names the excess, not everything held" + + +def _with_aws_too(tmp_path, monkeypatch) -> None: + creds = tmp_path / "aws" / "credentials" + creds.parent.mkdir(parents=True, exist_ok=True) + creds.write_text("[default]\n") + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(creds)) + + +def _pager(): + pages: list[str] = [] + + async def alarm(text: str) -> bool: + pages.append(text) + return True + + return alarm, pages + + +def test_a_readonly_node_without_kubectl_refuses_and_pages(tmp_path, monkeypatch) -> None: + """2026-09-29 to 10-09: the shared work image had no kubectl, both nodes + recorded `kubectl: not found` at every boot, nothing refused, nothing paged.""" + _with_kube(tmp_path, monkeypatch) + run = _runner({"kubectl": (127, "", "kubectl: not found")}) + alarm, pages = _pager() + with pytest.raises(posture.PostureViolation, match="kubectl"): + asyncio.run(posture.on_startup(tmp_path, "readonly", "enforce", run, alarm=alarm)) + record = posture.read(tmp_path) + assert record["verdict"] == "unmeasurable" and record["kube"]["tool_missing"] == "kubectl" + assert len([a for a in run.calls if a[0] == "kubectl"]) == 1, "one answer is the answer, not four more copies" + assert pages and "kubectl" in pages[0], "said on the alarm channel before the refusal loops" + + +def test_a_writing_node_without_kubectl_starts_and_pages(tmp_path, monkeypatch) -> None: + _with_kube(tmp_path, monkeypatch) + run = _runner({"kubectl": (127, "", "kubectl: not found")}) + alarm, pages = _pager() + record = asyncio.run(posture.on_startup(tmp_path, "danger-only", "enforce", run, alarm=alarm)) + assert record["verdict"] == "unmeasurable", "not `recorded`: an empty radius nobody measured is not a radius" + assert pages and "kubectl" in pages[0] + + +def test_warn_starts_a_readonly_node_without_its_tool_and_pages(tmp_path, monkeypatch) -> None: + _with_kube(tmp_path, monkeypatch) + run = _runner({"kubectl": (127, "", "kubectl: not found")}) + alarm, pages = _pager() + record = asyncio.run(posture.on_startup(tmp_path, "readonly", "warn", run, alarm=alarm)) + assert record["verdict"] == "unmeasurable" and pages + + +def test_a_missing_aws_cli_is_named_and_a_measured_violation_outranks_it(tmp_path, monkeypatch) -> None: + _with_kube(tmp_path, monkeypatch) + _with_aws_too(tmp_path, monkeypatch) + run = _runner({"kubectl auth can-i --list": (0, CAN_I_LIST_READONLY, ""), "aws": (127, "", "aws: not found")}) + with pytest.raises(posture.PostureViolation, match="aws"): + asyncio.run(posture.on_startup(tmp_path, "readonly", "enforce", run)) + assert posture.read(tmp_path)["aws"]["tool_missing"] == "aws" + run = _runner({"kubectl auth can-i --list": (0, CAN_I_LIST_ADMIN, ""), "aws": (127, "", "aws: not found")}) + with pytest.raises(posture.PostureViolation, match=r"\*"): + asyncio.run(posture.on_startup(tmp_path, "readonly", "enforce", run)) + assert posture.read(tmp_path)["verdict"] == "wider-than-declared", "what was measured outranks what could not be" diff --git a/hookprobe/tests/test_selftest.py b/hookprobe/tests/test_selftest.py index 33bd45d3..c0982e6d 100644 --- a/hookprobe/tests/test_selftest.py +++ b/hookprobe/tests/test_selftest.py @@ -447,3 +447,27 @@ async def fake_check(bash_guard, run=None, declared=None): result = asyncio.run(selftest.posture_still_holds(settings)) assert seen["declared"] is None assert result["held"] is True + + +def test_the_hourly_check_fails_when_the_measuring_tool_is_gone(tmp_path, monkeypatch) -> None: + """The shared work image had no kubectl from 2026-09-29 to 10-09 and this + check said held every hour, because `recorded` and `unverifiable` both + answer "no wider" and the only question asked was that one.""" + + async def fake_check(bash_guard, run=None, declared=None): + return { + "bash_guard": bash_guard, + "verdict": "unmeasurable", + "kube": { + "present": True, + "mutating": [], + "errors": ["can-i --list: kubectl: not found"], + "tool_missing": "kubectl", + }, + "aws": {"present": False}, + } + + monkeypatch.setattr(selftest.posture, "check", fake_check) + settings = make_settings(tmp_path, workdir=tmp_path, bash_guard="danger-only", blast_radius=None) + result = asyncio.run(selftest.posture_still_holds(settings)) + assert result["held"] is False and "kubectl" in result["detail"] diff --git a/scripts/assert_guards_are_tested.py b/scripts/assert_guards_are_tested.py index 28864e17..47357ef2 100755 --- a/scripts/assert_guards_are_tested.py +++ b/scripts/assert_guards_are_tested.py @@ -160,11 +160,18 @@ " if False:", ("tests/test_selftest.py",), ), + ( + "a credential nobody could measure passes as recorded", + "hookprobe/hookprobe/posture.py", + ' return "unmeasurable" if missing else "recorded"', + ' return "recorded"', + ("tests/test_posture.py",), + ), ( "a writing node's declared blast radius is not checked", "hookprobe/hookprobe/posture.py", - ' return "wider-than-declared" if beyond(kube, aws, declared) else "within-declared-radius"', - ' return "within-declared-radius"', + ' if beyond(kube, aws, declared):\n return "wider-than-declared"\n', + ' if False:\n return "wider-than-declared"\n', ("tests/test_posture.py",), ), ( diff --git a/scripts/assert_shared_images_build_alike.py b/scripts/assert_shared_images_build_alike.py new file mode 100755 index 00000000..f5fb6898 --- /dev/null +++ b/scripts/assert_shared_images_build_alike.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Services that build the same image tag must build it the same way. + +A compose builds every service that has a `build:` block, and two services that +name the same `image:` both tag their result with it — so the tag ends up +holding whichever build ran last, and nothing says which. The work stack's +three probes share `hookprobe:work`; probe-plan's build passed a kubectl version +and probe-watch's did not, so from 2026-09-29 the tag was the kubectl-less build, +and the planner and the work node started every day with `kubectl: not found` +in their posture records, measured nothing on their cluster side, and nothing +paged (hookprobe/posture.py says what that cost). + + python3 scripts/assert_shared_images_build_alike.py # every compose + python3 scripts/assert_shared_images_build_alike.py --list # what builds what + +Two services may share a tag only when their build context, dockerfile and args +are the same TEXT — the same `${VAR:-default}` expression, not merely the same +value today. Compared across every compose in the repository, because a tag is +global to the engine, not to a file. + +Parsed by indentation rather than a YAML library: this runs under the system +interpreter in CI, which has none, and the composes here are written in the one +shape (two-space indents, `services:` at the top) that makes a small reader +enough. A floor keeps the reader honest — fewer build blocks than the composes +are known to hold means the parse lost a file, and that fails rather than +passing on a short list. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +COMPOSES = sorted(ROOT.glob("deploy/docker-compose*.yml")) + sorted(ROOT.glob("docker-compose*.yml")) +# What the composes hold today: 15 build blocks, two tags built by more than one +# service (`hookprobe:work` by three, `lark-bridge:shadow` by two). A parse that +# finds fewer has lost a file. Lower these on purpose, with the reason, when a +# compose stops building something. +FLOOR_BUILDS = 8 +FLOOR_SHARED = 2 + + +def _strip(raw: str) -> str: + """The line without its comment; a `#` counts only at the start or after a space.""" + line = raw.rstrip("\n") + if line.lstrip().startswith("#"): + return "" + head, _, _ = line.partition(" #") + return head.rstrip() + + +def services_of(text: str) -> dict[str, dict]: + """Each service's `image:` and `build:` block, by name. Only the keys this + check compares are read; everything else in a service is skipped.""" + out: dict[str, dict] = {} + in_services = False + service: str | None = None + build: dict | None = None + for raw in text.splitlines(): + line = _strip(raw) + if not line.strip(): + continue + indent = len(line) - len(line.lstrip(" ")) + body = line.strip() + if indent == 0: + in_services = body == "services:" + service, build = None, None + continue + if not in_services: + continue + if indent == 2 and body.endswith(":"): + service = body[:-1] + out[service] = {"image": None, "build": None} + build = None + continue + if service is None: + continue + key, _, value = body.partition(":") + value = value.strip() + if indent == 4: + if key == "image": + out[service]["image"] = value + elif key == "build": + build = {"context": value or None, "dockerfile": None, "args": {}} + out[service]["build"] = build + else: + build = None # left the build block + elif build is not None and indent == 6 and key in ("context", "dockerfile"): + build[key] = value + elif build is not None and indent == 8: + build["args"][key.strip()] = value + return out + + +def builders() -> dict[str, list[tuple[str, str, dict]]]: + """image tag → [(compose file, service, build spec)] for every service that builds.""" + found: dict[str, list[tuple[str, str, dict]]] = {} + for compose in COMPOSES: + for name, spec in services_of(compose.read_text(encoding="utf-8")).items(): + if spec["image"] and spec["build"] is not None: + found.setdefault(spec["image"], []).append((str(compose.relative_to(ROOT)), name, spec["build"])) + return found + + +def differences(a: dict, b: dict) -> list[str]: + out = [f"{key}: {a.get(key)!r} vs {b.get(key)!r}" for key in ("context", "dockerfile") if a.get(key) != b.get(key)] + for arg in sorted(set(a["args"]) | set(b["args"])): + if a["args"].get(arg) != b["args"].get(arg): + out.append(f"args.{arg}: {a['args'].get(arg)!r} vs {b['args'].get(arg)!r}") + return out + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--list", action="store_true", help="print every image tag and the services that build it") + args = parser.parse_args() + found = builders() + total = sum(len(v) for v in found.values()) + shared = {tag: who for tag, who in found.items() if len(who) > 1} + if args.list: + for tag, who in sorted(found.items()): + print(f"{tag}") + for compose, service, spec in who: + print(f" {compose} · {service} · context={spec['context']} args={spec['args']}") + return 0 + if total < FLOOR_BUILDS or len(shared) < FLOOR_SHARED: + print( + f"assert_shared_images_build_alike: parsed {total} build block(s) and {len(shared)} shared tag(s); " + f"the floor is {FLOOR_BUILDS} and {FLOOR_SHARED} — the reader has lost a compose, which is not a pass", + file=sys.stderr, + ) + return 1 + problems: list[str] = [] + for tag, who in sorted(shared.items()): + first_compose, first_service, first_spec = who[0] + for compose, service, spec in who[1:]: + for diff in differences(first_spec, spec): + problems.append(f"{tag}: {first_compose}/{first_service} vs {compose}/{service} — {diff}") + if problems: + print("services that share an image tag build it differently; the tag is whichever ran last:", file=sys.stderr) + for problem in problems: + print(f" {problem}", file=sys.stderr) + return 1 + print(f"shared images: {len(shared)} tag(s) built by {sum(len(v) for v in shared.values())} service(s), all alike") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate.sh b/scripts/gate.sh index 482b6cca..185415d1 100755 --- a/scripts/gate.sh +++ b/scripts/gate.sh @@ -99,6 +99,11 @@ python3 scripts/assert_weight.py # service because the composes are, and because the same mistake landed in two # services on one afternoon. python3 scripts/assert_knobs_are_reachable.py +# Three services build one image tag, and the tag is whichever build ran last. +# probe-watch's build passed no kubectl version, so from 2026-09-29 the work +# image had no kubectl and two nodes measured nothing on their cluster side for +# ten days while every gate was green. +python3 scripts/assert_shared_images_build_alike.py # Cross-service for the same reason again: it compares one helper's body against # the copy of it living in another service. `Live.watcher_count` had already gone # missing from one of three copies, and verify_signature had stripped its