From 77b946b35e522d6d988314733ffc9444bdcf932e Mon Sep 17 00:00:00 2001 From: mecattaf Date: Wed, 16 Sep 2026 22:41:45 +0200 Subject: [PATCH] update-adopt: name a 404 no-candidate, refuse to guess at root-only state, and write the --no-check-sigs note (#354) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Item 4 of #354's 2026-09-14 status comment, and the two CLI answers beside it that read as "fine" when they are not. 1. `status` exits 1 when it cannot read what it is reporting on. /var/lib/update-adopt is 0700 root, so a run as tom read nothing and printed load_state()'s defaults — `"state": "idle"`, every other field null, exit 0 — which is indistinguishable from a healthy host that has nothing staged. It now prints the facts /run and /nix still give it, sets `state_unreadable`, leaves the state-derived fields null, and exits 1. Nothing staged yet on a readable directory is still a real answer and still exits 0. fleet-status already collects every host as root over ssh (modules/fleet-status.nix); if it ever ran the verb unprivileged it would now grade the answer unknown instead of "up to date". 2. A 404 manifest is `no-candidate`, not `fetch-failed`. curl gets `-w %{http_code}`, so "the NAS answered, with nothing published for this host" and "the NAS did not answer" are different journal lines. Both still exit 0: neither is this host's failure. MEASURED against the live NAS — an unpublished host is rc 22 / 404, a real one rc 0 / 200, an unreachable endpoint rc 7 / 000. 3. docs/fleet-candidate-adoption.md writes down when `--no-check-sigs` on the `nix copy` publish step is legitimate: a locally built closure is unsigned, the receiving daemon honours the flag only for a trusted user (modules/common.nix), and the trust in that transfer is the SSH channel and who built the bytes. It also names the three places the flag is never legitimate, starting with substitution from the fleet cache, where the signing key is the whole proof. tests/update-adopt gains three cases (404 -> no-candidate, an unreadable state directory -> rc 1, no state yet -> rc 0) and its fake curl now models a 404 and an unreachable NAS separately. 33 tests pass, in-tree and through `nix build .#checks.x86_64-linux.update-adopt`. Co-Authored-By: Claude Opus 5 (1M context) --- docs/README.md | 1 + docs/fleet-candidate-adoption.md | 100 ++++++++++++++++++++++++ flake.nix | 5 +- modules/update-adopt.py | 58 ++++++++++++-- tests/update-adopt/test_update_adopt.py | 71 ++++++++++++++++- 5 files changed, 225 insertions(+), 10 deletions(-) create mode 100644 docs/fleet-candidate-adoption.md diff --git a/docs/README.md b/docs/README.md index 28b298d92..229b82867 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,4 +24,5 @@ it. | [`util-01-sampler-reconcile.md`](util-01-sampler-reconcile.md) | **How the two-commit UTIL-01 sampler branch reached `main` with its semantics unchanged** — the one overlapping import line, the digests the card locks, the two `[E]` probe rows that ship RED on purpose, and the U-D17 oracle. | | [`old/`](old/) | Archival stub: an index of the retired documentation set and how to read it back from Git history. | | [`local-ai/wanted-packages/`](local-ai/wanted-packages/) | Findings the fleet turned up but has not acted on — one page per finding, each naming what stopped, what is measured, and what is inference. | +| [`fleet-candidate-adoption.md`](fleet-candidate-adoption.md) | **Publishing a candidate the NAS did not build, and why that `nix copy` needs `--no-check-sigs`** — where the trust in that transfer actually comes from, the three places the flag is never legitimate, and how to read `update-adopt status` (why it exits 1 as tom, and `no-candidate` against `fetch-failed`). | | [`runbook-conventions.md`](runbook-conventions.md) | **How a runbook suppresses a unit on NixOS** — why a plain mask fails on the read-only `/etc/systemd/system` store symlink and `mask --runtime` silently does nothing, and the intent → wrong → right table (runtime drop-in, `enable = false` + rebuild, move the data, stop the timer). | diff --git a/docs/fleet-candidate-adoption.md b/docs/fleet-candidate-adoption.md new file mode 100644 index 000000000..8110f32d8 --- /dev/null +++ b/docs/fleet-candidate-adoption.md @@ -0,0 +1,100 @@ +# Publishing a fleet candidate by hand, and reading an endpoint's answer + +Written 2026-09-16 for #354 (the four items left on its 2026-09-14 status +comment). The nightly path needs no operator: `update-center` on the NAS +resolves `main` once, builds each enrolled host, pushes to Attic and publishes +a signed per-host manifest, and each endpoint's `update-adopt` timers stage and +(on the rolling policy) activate it. This page is for the two times an operator +is in the loop — publishing a closure built somewhere else, and reading +`update-adopt status` — and for the one flag in that path that looks like a +safety bypass and is not. + +The Nix files that decide the behaviour here are `hosts/nas/update-center.nix` +(with its body in `hosts/nas/update-center.sh`) and `modules/update-adopt.nix` +(body in `modules/update-adopt.py`). + +## Publishing a closure the NAS did not build + +The NAS publishes only what is already in its own store and already in Attic: +`update-center --publish-only HOST STORE_PATH FLAKEREF` refuses a path that is +not valid locally, and publishes no pointer unless `attic push` succeeds +first. So a closure built on the coordinator has to be copied to the NAS first: + +```sh +# on the coordinator, as tom +nix copy --to ssh-ng://root@nas --no-check-sigs /nix/store/-nixos-system-worker- +# on the NAS, as root +update-center --publish-only worker /nix/store/-nixos-system-worker- \ + github:mecattaf/dotfiles/<40-char rev> +``` + +### Why `--no-check-sigs` belongs on that line + +A path you just built is unsigned. Nothing signs a local build: signatures are +added by a cache when it serves the path, so `attic push` signs with the +`fleet:` key on the way into the cache and nothing signs it on the way into the +NAS's store. The receiving daemon enforces `require-sigs` unless the client +asks it not to, so the copy above fails with *"cannot add path … because it +lacks a signature by a trusted key"* until `--no-check-sigs` is passed. + +The flag is honoured only for a trusted user — `trusted-users = [ "root" +"@wheel" ]` in `modules/common.nix` — which is what makes it work as root to +the NAS and what makes it useless to anyone else. + +It is legitimate here for one reason: **the trust in this transfer comes from +the transport and from who built the bytes, not from a signature.** You built +the closure on a fleet host, and you are handing it over an authenticated SSH +channel to a store you administer. A signature would only re-prove a fact you +already have. + +It is **not** legitimate anywhere the signature is the only evidence: + +- never on an endpoint substituting from a cache. `http://nas:8080/fleet` is + trusted by its public key (`extra-trusted-public-keys` in + `modules/common.nix`), and that key is the whole proof that the bytes are the + ones the NAS built; +- never to silence a signature error whose cause you have not found. An + unsigned path from a cache is a broken cache or a wrong key, and copying it + anyway hides that; +- never in a unit or a timer, and never as a habit on `nix copy`. The seed + timer in `home/update-center-seed.nix` copies the lock's private source trees + to the NAS nightly and does *not* pass it: those paths are narHash-addressed, + so their own hash is the proof and no signature is wanted. A path that needs + the flag is a path whose only possible proof is who you are. + +Adoption's own trust is separate from all of this and is not weakened by the +flag: the endpoint verifies the manifest's `ssh-keygen -Y` signature in the +`fleet-update` namespace against an allowed-signers file holding the NAS's host +key from `modules/mesh-registry.nix` (rendered into the store by +`modules/update-adopt.nix`), checks +that `store_path` names *this* host's system closure, and re-reads the +revision from the closure's own `fleet-revision.json` rather than trusting the +manifest's copy of it. + +## Reading `update-adopt status` + +Two answers that used to read as "fine" now say what they are. + +**`status` as tom exits 1.** The state directory is `/var/lib/update-adopt`, +mode 0700 root. A run that cannot read it prints the facts it can still see +(`current`, `booted`, `profile` and the running generation's revision), sets +`state_unreadable` to the reason, leaves `state`, `candidate`, `last_refusal`, +`last_known_good` and `pending_reboot` at `null`, and exits 1. It no longer +prints `"state": "idle"` for a host it never looked at. Run it under `sudo`. +`fleet-status` collects every host, the coordinator included, as root over ssh +for exactly this reason (`modules/fleet-status.nix`); if it ever runs the verb +as a non-root user it grades the answer `unknown`, not "up to date". + +**A 404 manifest is `no-candidate`, not `fetch-failed`.** Both still exit 0 — +neither is this host's failure — but they are different facts and the journal +now names them differently: + +| Journal line | What happened | What to do | +|---|---|---| +| `reason=no-candidate` | The NAS answered, with 404: it has published nothing for this host. Normal for a host that is not enrolled, and for any host before the first successful nightly | Check `update-center` on the NAS, not the endpoint | +| `reason=fetch-failed` | No HTTP answer at all: the NAS is down, or off the mesh from here | Check reachability | + +Read them with +`journalctl -t update-adopt -o cat` (the identifier is set by the script, so a +unit glob is not needed), or from the last refusal in `sudo update-adopt +status --json`. diff --git a/flake.nix b/flake.nix index 96c26c0eb..6e716882a 100644 --- a/flake.nix +++ b/flake.nix @@ -2223,7 +2223,10 @@ # unknown gates, probe-failure rollback, switch-failure rollback, # kernel → boot + pending-reboot marker, bad signature, host mismatch, # manual and stage-only policies, downgrade refusal before and after - # download, dirty/unknown current, --force, the activation lock) and + # download, dirty/unknown current, --force, the activation lock, a 404 + # manifest as `no-candidate` against an unreachable NAS as + # `fetch-failed`, and a `status` that cannot read the root-only state + # exiting 1 rather than printing an empty one) and # each gate verb of update-adopt-gates.sh, with fake nix/systemctl and a # REAL ssh-keygen. The asserts pin the policy table: NAS not enrolled # (2026-08-21 ruling), client manual (R-18), both units immune to the diff --git a/modules/update-adopt.py b/modules/update-adopt.py index 22a9a28dd..46a9409cd 100644 --- a/modules/update-adopt.py +++ b/modules/update-adopt.py @@ -8,9 +8,13 @@ status print the freshness facts as JSON Exit codes are a contract with failure-surfacing: a busy host, a newer local -generation, a manual policy or an unreachable NAS exit 0 (a receipt, not a -failure); a bad signature, a failed realise, or a failed probe (after the -local rollback) exit 1, so the unit fails and a marker is written. +generation, a manual policy, an unreachable NAS or a host the NAS has not +published a candidate for exit 0 (a receipt, not a failure); a bad signature, +a failed realise, or a failed probe (after the local rollback) exit 1, so the +unit fails and a marker is written. `status` exits 1 when it cannot read the +state it is reporting on — a run as tom against the root-only state directory +prints what /run and /nix show and says the rest is unknown, instead of an +empty state that reads like a healthy idle host. Test seams (all unset in the real unit; see modules/update-adopt.nix): UPDATE_ADOPT_CONFIG JSON config rendered by Nix (required) @@ -110,6 +114,27 @@ def journal(state, reason, **fields): pass +def state_unreadable(): + """Why this user cannot read the host's adoption state, or None. + + /var/lib/update-adopt is 0700 root, so `update-adopt status` run as tom + opens neither the directory nor state.json — and load_state() would hand + back an empty dict that prints exactly like a healthy idle host. Nothing + staged YET is a real answer (no state file on a directory this user can + read, or no directory at all on a host that never ran stage); state this + user is not allowed to see is not. + """ + try: + with open(os.path.join(STATE_DIR, "state.json")): + return None + except FileNotFoundError: + if os.path.isdir(STATE_DIR) and not os.access(STATE_DIR, os.R_OK | os.X_OK): + return f"{STATE_DIR} is not readable by uid {os.geteuid()}" + return None + except OSError as error: + return str(error) + + def load_state(): try: with open(os.path.join(STATE_DIR, "state.json")) as stream: @@ -261,8 +286,18 @@ def fetch_candidate(cfg, workdir): manifest_path = os.path.join(workdir, "manifest.json") sig_path = manifest_path + ".sig" for url, dest in ((base + "/manifest.json", manifest_path), (base + "/manifest.json.sig", sig_path)): - result = run(["curl", "-fsS", "-m", "10", "-o", dest, url], timeout=30) + # -w prints the status line's code (000 when no HTTP answer arrived), + # so "the NAS published nothing for this host" is told apart from "the + # NAS did not answer". Both exit 0; only their names differ, and the + # name is what an operator reads in the journal. + result = run(["curl", "-fsS", "-m", "10", "-w", "%{http_code}", "-o", dest, url], timeout=30) if result.returncode != 0: + if result.stdout.strip() == "404": + raise Refusal( + "no-candidate", + f"{url}: HTTP 404 — the NAS has published no candidate for this host", + rc=0, + ) raise Refusal("fetch-failed", f"{url}: {result.stderr.strip()}"[:300], rc=0) with open(manifest_path, "rb") as stream: raw = stream.read() @@ -753,9 +788,13 @@ def _activate(cfg, force): def status(cfg): - state = load_state() + unreadable = state_unreadable() + # Report no state at all rather than load_state()'s defaults: an "idle" + # this process never read is a claim, not a fact. + state = {} if unreadable else load_state() current = resolve(CURRENT) out = { + "state_unreadable": unreadable, "host": cfg["host"], "policy": cfg["policy"], "state": state.get("state"), @@ -774,6 +813,15 @@ def status(cfg): } json.dump(out, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") + if unreadable: + print( + f"update-adopt: {unreadable}; the state, candidate, refusal and " + "known-good fields above are unknown, not empty. Run it as root " + "(`sudo update-adopt status --json`); fleet-status collects it " + "over ssh as root for this reason.", + file=sys.stderr, + ) + return 1 return 0 diff --git a/tests/update-adopt/test_update_adopt.py b/tests/update-adopt/test_update_adopt.py index 4e5575348..ccce4c2c1 100644 --- a/tests/update-adopt/test_update_adopt.py +++ b/tests/update-adopt/test_update_adopt.py @@ -22,14 +22,27 @@ GATES = os.path.join(REPO, "modules", "update-adopt-gates.sh") FAKES = { + # Models the two failures that look alike from the outside: a NAS that + # answers 404 (nothing published for this host) and a NAS that does not + # answer at all (curl exit 7, no status line, %{http_code} = 000). "curl": r"""#!/bin/sh -dest=""; url="" +dest=""; url=""; code="" while [ $# -gt 0 ]; do - case "$1" in -o) dest="$2"; shift 2 ;; -m) shift 2 ;; -*) shift ;; *) url="$1"; shift ;; esac + case "$1" in -o) dest="$2"; shift 2 ;; -m|-w) shift 2 ;; -*) shift ;; *) url="$1"; shift ;; esac done +if [ -e "$FIXTURE/nas-down" ]; then + printf 000 + echo "curl: (7) Failed to connect to nas.test port 8734" >&2 + exit 7 +fi src="$FIXTURE/www/${url#http://nas.test/}" -[ -f "$src" ] || exit 22 +if [ ! -f "$src" ]; then + printf 404 + echo "curl: (22) The requested URL returned error: 404" >&2 + exit 22 +fi if [ -n "$dest" ]; then cp "$src" "$dest"; else cat "$src"; fi +printf 200 """, "nix-store": r"""#!/bin/sh echo "nix-store $*" >> "$FIXTURE/calls.log" @@ -475,8 +488,30 @@ def test_host_mismatch_refused(self): self.assertNotIn("--realise", self.fx.calls()) def test_unreachable_nas_is_not_a_failure(self): - self.assertOk(self.fx.run("stage")) + open(os.path.join(self.fx.tmp, "nas-down"), "w").close() + result = self.fx.run("stage") + self.assertOk(result) self.assertEqual(self.fx.state()["last_refusal"]["reason"], "fetch-failed") + self.assertIn("reason=fetch-failed", result.stdout) + + # A NAS that answers 404 has published nothing for this host. That is not + # a fetch that failed, and the journal must not say it was (#354 item 4). + def test_missing_manifest_is_no_candidate(self): + result = self.fx.run("stage") + self.assertOk(result) + self.assertIn("reason=no-candidate", result.stdout) + refusal = self.fx.state()["last_refusal"] + self.assertEqual(refusal["reason"], "no-candidate") + self.assertIn("404", refusal["detail"]) + self.assertEqual(self.fx.receipts()[-1]["reason"], "no-candidate") + self.assertNotIn("--realise", self.fx.calls()) + # And it is still a 404 once the host HAS a candidate that is unpublished + # again: the honest name does not depend on prior state. + self.fx.publish(self.fx.system("b", last_modified=2000)) + self.assertOk(self.fx.run("stage")) + os.unlink(os.path.join(self.fx.tmp, "www/candidates/worker/manifest.json")) + self.assertOk(self.fx.run("stage")) + self.assertEqual(self.fx.state()["last_refusal"]["reason"], "no-candidate") # (6) manual policy → never realises def test_manual_policy_only_reports(self): @@ -569,6 +604,34 @@ def test_status_reports_freshness(self): self.assertIn(key, out) self.assertEqual(out["candidate"]["store_path"], b) self.assertEqual(out["current_revision"]["lastModified"], 1000) + self.assertIsNone(out["state_unreadable"]) + + # The real state directory is 0700 root; a run that cannot read it must + # not print an empty state that looks like a healthy idle host (#354). + @unittest.skipIf(os.geteuid() == 0, "root reads a 0700 directory anyway") + def test_status_without_access_to_state_exits_nonzero(self): + self.fx.publish(self.fx.system("b", last_modified=2000)) + self.assertOk(self.fx.run("stage")) + os.chmod(self.fx.state_dir, 0o000) + try: + result = self.fx.run("status", "--json") + finally: + os.chmod(self.fx.state_dir, 0o700) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + out = json.loads(result.stdout) + self.assertIsNotNone(out["state_unreadable"]) + for key in ("state", "candidate", "last_refusal", "last_known_good", "pending_reboot"): + self.assertIsNone(out[key], key) + # The facts it CAN read are still reported, and still true. + self.assertEqual(out["current"], self.fx.a) + self.assertIn("unknown, not empty", result.stderr) + + def test_status_with_no_state_yet_is_not_an_error(self): + result = self.fx.run("status", "--json") + self.assertOk(result) + out = json.loads(result.stdout) + self.assertIsNone(out["state_unreadable"]) + self.assertEqual(out["state"], "idle") def test_receipts_are_capped(self): for _ in range(60):