diff --git a/modules/checks/stibnite-access-wiring.nix b/modules/checks/stibnite-access-wiring.nix new file mode 100644 index 000000000..620e5d924 --- /dev/null +++ b/modules/checks/stibnite-access-wiring.nix @@ -0,0 +1,115 @@ +# Structural check for stibnite's aarch64-darwin build access. +# +# It guards the pairing between two machines' configurations, which no single +# machine's evaluation can catch. magnetite generates two keypairs and stibnite +# authorizes them under two different accounts; swapping them, or authorizing +# the build key without its forced command, produces a configuration that +# builds and activates and is wrong in the one way that matters — the build +# key would carry an interactive login, or the independently revocable +# identities would be assigned to the wrong accounts. +# +# It also pins the two mechanisms to one account: the nix.buildMachines entry +# (result copied back to the caller) and the remote-store URI (build stays in +# stibnite's store) must name the same user and the same ssh alias, or one of +# them silently reaches an account that authorizes nothing. +{ + self, + lib, + ... +}: +{ + perSystem = + { pkgs, system, ... }: + let + mkCheck = self.lib.mkStructuralCheck pkgs; + + magnetite = self.nixosConfigurations.magnetite.config; + stibnite = self.darwinConfigurations.stibnite.config; + + buildKey = + lib.removeSuffix "\n" + magnetite.clan.core.vars.generators.stibnite-nix-build.files."key.pub".value; + sessionKey = + lib.removeSuffix "\n" + magnetite.clan.core.vars.generators.stibnite-agent-session.files."key.pub".value; + + authorized = user: stibnite.environment.etc."ssh/nix_authorized_keys.d/${user}".text; + authorizedLines = user: builtins.filter (line: line != "") (lib.splitString "\n" (authorized user)); + + darwinEntries = builtins.filter ( + m: builtins.elem "aarch64-darwin" m.systems + ) magnetite.nix.buildMachines; + darwinEntry = builtins.head darwinEntries; + + # The forced command names an activation-time store path, so the oracle + # asserts its shape: the key options, and that the program is the + # nix-daemon of the nix package stibnite activates. + buildLine = builtins.head (authorizedLines "nixbuild"); + forcedCommand = ''restrict,command="${stibnite.nix.package}/bin/nix-daemon --stdio"''; + in + { + checks = + lib.optionalAttrs + (builtins.elem system [ + "x86_64-linux" + "aarch64-darwin" + ]) + { + stibnite-access-wiring = mkCheck { + name = "stibnite-access-wiring"; + actual = { + darwinBuilderCount = builtins.length darwinEntries; + darwinBuilder = { + inherit (darwinEntry) + hostName + sshUser + protocol + systems + maxJobs + supportedFeatures + ; + }; + + # Both mechanisms, one account and one alias. + storeUriNamesBuildAccount = + magnetite.environment.etc."nix/stibnite-store-uri".text + == "ssh-ng://${darwinEntry.sshUser}@${darwinEntry.hostName}?ssh-key=${darwinEntry.sshKey}\n"; + + # Key separation, asserted in both directions. + buildAccountAuthorizes = authorizedLines "nixbuild"; + sessionKeyIsNotABuildKey = !(lib.hasInfix sessionKey (authorized "nixbuild")); + buildKeyIsNotASessionKey = !(lib.hasInfix buildKey (authorized "crs58")); + sessionKeyAuthorizedForSessions = lib.hasInfix sessionKey (authorized "crs58"); + + # The build key carries a forced command and nothing else. + buildLineIsRestrictedToTheProtocol = buildLine == "${forcedCommand} ${buildKey}"; + + buildAccountIsTrusted = builtins.elem "nixbuild" stibnite.nix.settings.trusted-users; + buildAccountIsManaged = builtins.elem "nixbuild" stibnite.users.knownUsers; + }; + expected = { + darwinBuilderCount = 1; + darwinBuilder = { + hostName = "stibnite-builder"; + sshUser = "nixbuild"; + protocol = "ssh-ng"; + systems = [ "aarch64-darwin" ]; + maxJobs = 4; + supportedFeatures = [ + "apple-virt" + "big-parallel" + ]; + }; + storeUriNamesBuildAccount = true; + buildAccountAuthorizes = [ "${forcedCommand} ${buildKey}" ]; + sessionKeyIsNotABuildKey = true; + buildKeyIsNotASessionKey = true; + sessionKeyAuthorizedForSessions = true; + buildLineIsRestrictedToTheProtocol = true; + buildAccountIsTrusted = true; + buildAccountIsManaged = true; + }; + }; + }; + }; +} diff --git a/modules/machines/darwin/stibnite/default.nix b/modules/machines/darwin/stibnite/default.nix index 3e520d7b5..69283d935 100644 --- a/modules/machines/darwin/stibnite/default.nix +++ b/modules/machines/darwin/stibnite/default.nix @@ -40,6 +40,7 @@ in zt-dns zt-services-trust magnetite-builder + stibnite-build-host # Not importing users module (defines testuser at UID 550) # stibnite defines its own user (crs58) ]); @@ -224,6 +225,27 @@ in # Offload native x86_64-linux builds to magnetite over ZeroTier. services.magnetite-builder.enable = true; + # Inbound side of the same asymmetry: stibnite is the fleet's only + # aarch64-darwin machine, so it serves darwin builds to hosts that cannot + # perform them. The two keypairs come from magnetite and are authorized + # separately for independent revocation and rotation. The build key is + # confined at the SSH boundary to the nix protocol. The session key is + # broader: it logs in as admin-group crs58, a Nix trusted user, so it has + # build authority plus shell access. + # Both encrypted private halves and public values are committed under + # vars/per-machine/magnetite/. `clan vars generate magnetite` populates + # them; only magnetite and authorized users can decrypt the private + # halves. + services.stibnite-build-host = { + enable = true; + buildKeys = [ + inputs.self.nixosConfigurations.magnetite.config.clan.core.vars.generators.stibnite-nix-build.files."key.pub".value + ]; + sessionKeys = [ + inputs.self.nixosConfigurations.magnetite.config.clan.core.vars.generators.stibnite-agent-session.files."key.pub".value + ]; + }; + # Colima for OCI container management (complementary to nix-rosetta-builder) services.colima = { enable = true; diff --git a/modules/machines/nixos/magnetite/default.nix b/modules/machines/nixos/magnetite/default.nix index f0740f288..f22ebef5c 100644 --- a/modules/machines/nixos/magnetite/default.nix +++ b/modules/machines/nixos/magnetite/default.nix @@ -41,6 +41,8 @@ in hm-sops-bridge niks3 ssh-known-hosts + stibnite-builder + stibnite-session buildbot nixbot gitea @@ -82,6 +84,30 @@ in ]; }; + # magnetite is x86_64-linux and cannot build aarch64-darwin derivations, + # so darwin work is dispatched to stibnite, the fleet's only machine of + # that system. Two mechanisms, two callers: + # nix.buildMachines below — a local nix build by an operator who wants + # the darwin result in magnetite's store. + # /etc/nix/stibnite-store-uri — a caller that wants the build to happen + # entirely in stibnite's store with nothing copied back. + # nixbot.toml sets attribute = "checks.x86_64-linux", which prevents CI + # from evaluating or requesting aarch64-darwin work and makes this builder + # unreachable from CI. modules/nixos/nixbot.nix and + # modules/nixos/buildbot.nix each set buildSystems = [ "x86_64-linux" ] + # as an independent second layer. These controls remain because stibnite + # is a laptop without guaranteed availability and a sleeping machine + # could gate CI. + services.stibnite-builder.enable = true; + nix.buildMachines = config.services.stibnite-builder.buildMachines; + + # The build and session keys are separately authorized for independent + # revocation and rotation. Only the build key is confined to the Nix + # protocol by `restrict` and a forced command. The session key is broader: + # it grants an unrestricted login as admin-group crs58, already a Nix + # trusted user, with build authority plus shell. + services.stibnite-session.enable = true; + # User configuration managed via clan inventory users service (modules/clan/inventory/services/users/cameron.nix). security.sudo.wheelNeedsPassword = false; diff --git a/modules/system/stibnite-access.nix b/modules/system/stibnite-access.nix new file mode 100644 index 000000000..678fd5744 --- /dev/null +++ b/modules/system/stibnite-access.nix @@ -0,0 +1,361 @@ +# Access to stibnite (aarch64-darwin) from the fleet, in three separable pieces. +# +# stibnite is the fleet's only aarch64-darwin machine, and until this module +# nothing declared it as a build target: magnetite is x86_64-linux and cannot +# build darwin derivations, and stibnite's own nix.buildMachines list carries +# only the rosetta VM and magnetite. The two mechanisms below are the two ways +# a caller can reach stibnite's store, and they are not interchangeable. +# +# services.stibnite-builder — remote BUILDER (nix.buildMachines). +# The caller's nix daemon copies the input closure to stibnite, builds there, +# and copies the output closure back, so the result exists in the CALLER's +# store. Callers: developers and operators who need an aarch64-darwin result +# locally, for example to build or test Darwin configurations from magnetite. +# +# services.stibnite-builder.storeUri — remote STORE (--store ssh-ng://). +# The whole build happens inside stibnite's store; evaluation stays with the +# caller and no closure is copied back. Callers: a machine whose own store is +# empty and stays empty — an ephemeral CI runner, a fresh container, an +# installer image — where a remote builder would spend the whole job +# populating a store that is about to be discarded. Nothing lands locally, so +# a caller that needs the output path locally wants the builder above. +# +# services.stibnite-session — an unrestricted login as admin-group crs58, who +# is already a Nix trusted user, so it grants build authority plus shell. Its +# key is separately authorized for independent revocation and rotation; only +# the build key is confined to the Nix protocol by `restrict` and a forced +# command. +# +# services.stibnite-build-host — the stibnite side: the account restricted to +# nix-daemon at the SSH boundary and trusted with store-root-equivalent +# authority, plus the authorization for both keypairs. +{ lib, ... }: +let + # Deterministic ZeroTier IPv6, matching modules/system/ssh-known-hosts.nix + # and modules/machines/nixos/cinnabar/zt-dns.nix. + stibniteZt = "fddb:4344:343b:14b9:399:9324:19d9:3451"; + + # clan's public var values carry the trailing newline of the file they were + # read from, and an authorized_keys entry is one line. + trimKey = key: lib.removeSuffix "\n" key; + + mkSshBlock = args: '' + Host ${args.hostAlias} + HostName ${stibniteZt} + User ${args.sshUser} + IdentityFile ${args.sshKeyPath} + IdentitiesOnly yes + HostKeyAlias stibnite.zt + ''; + + builderOptions = + config: + let + cfg = config.services.stibnite-builder; + sshKeyPath = config.clan.core.vars.generators.stibnite-nix-build.files.key.path; + in + { + options.services.stibnite-builder = { + enable = lib.mkEnableOption "build access to stibnite, the fleet's aarch64-darwin machine"; + + systems = lib.mkOption { + type = lib.types.listOf lib.types.str; + # aarch64-darwin only. `nix config show extra-platforms` on stibnite + # reports aarch64-darwin and nothing else, so advertising + # x86_64-darwin would route derivations the machine refuses to build. + default = [ "aarch64-darwin" ]; + description = "Systems stibnite can build for."; + }; + + maxJobs = lib.mkOption { + type = lib.types.int; + # stibnite has 18 logical cores (12 performance + 6 efficiency) and + # 64 GiB of memory. It is also a laptop in interactive use that + # commits 12 cores and 48 GiB to the rosetta builder VM and the same + # again to colima when either is running, so the remote share is + # deliberately a minority of the machine: 4 concurrent jobs, each + # free to use every core through the daemon's own `cores` setting. + default = 4; + description = "Maximum simultaneous build jobs dispatched to stibnite."; + }; + + speedFactor = lib.mkOption { + type = lib.types.int; + # Inert while stibnite is the only aarch64-darwin builder: nix + # compares speedFactor among machines that can build the same system, + # and there is no second darwin machine to compare against. 1 is the + # neutral value to raise if one appears. + default = 1; + description = "Scheduler weight, compared only against other aarch64-darwin builders."; + }; + + supportedFeatures = lib.mkOption { + type = lib.types.listOf lib.types.str; + # Measured with `nix config show system-features` on stibnite, minus + # what the value means rather than what nix prints. apple-virt is + # real and stibnite is the only machine in the fleet that has it. + # big-parallel is real on 18 cores. "benchmark" is dropped because + # timings taken on a laptop under interactive load are not + # measurements, and "nixos-test" is dropped because it is a Linux + # sandbox capability that nix lists unconditionally. + default = [ + "apple-virt" + "big-parallel" + ]; + description = "Build features stibnite advertises to the scheduler."; + }; + + sshUser = lib.mkOption { + type = lib.types.str; + default = "nixbuild"; + description = "Build account restricted to nix-daemon at the SSH boundary and trusted with store-root-equivalent authority; see services.stibnite-build-host.user."; + }; + + hostAlias = lib.mkOption { + type = lib.types.str; + # Distinct from the "stibnite" / "stibnite.zt" aliases an interactive + # session uses, so the build identity and the session identity cannot + # be reached through each other's alias by accident. + default = "stibnite-builder"; + description = "ssh Host alias the nix daemon resolves to stibnite's ZeroTier address."; + }; + + buildMachines = lib.mkOption { + type = lib.types.listOf (lib.types.attrsOf lib.types.anything); + default = + if cfg.enable then + [ + { + hostName = cfg.hostAlias; + sshUser = cfg.sshUser; + protocol = "ssh-ng"; + sshKey = sshKeyPath; + systems = cfg.systems; + maxJobs = cfg.maxJobs; + speedFactor = cfg.speedFactor; + supportedFeatures = cfg.supportedFeatures; + mandatoryFeatures = [ ]; + } + ] + else + [ ]; + defaultText = lib.literalExpression "single-element stibnite buildMachines entry when enabled, else []"; + description = "Computed nix.buildMachines entry. Consumers splice this into their own nix.buildMachines; this module never sets nix.buildMachines itself."; + }; + + storeUri = lib.mkOption { + type = lib.types.str; + default = "ssh-ng://${cfg.sshUser}@${cfg.hostAlias}?ssh-key=${sshKeyPath}"; + defaultText = lib.literalExpression "ssh-ng URI naming the build account, the ssh alias and the build key"; + readOnly = true; + description = '' + Remote-store URI for `nix build --store`, the other mechanism. + Materialized at /etc/nix/stibnite-store-uri so a caller reads the + key path from configuration rather than retyping it. The key is + root-owned, so a non-root caller needs `sudo -E`. + ''; + }; + }; + }; + + # Generated rather than operator-populated: no plaintext private half leaves + # magnetite. Clan commits the encrypted private half and public value, and + # only magnetite and authorized users can decrypt the private half. This + # avoids manual transcription. The operator step is `clan vars generate + # ` plus committing both outputs; stibnite reads the public value at + # evaluation time. + mkBuildKeyGenerator = pkgs: { + clan.core.vars.generators.stibnite-nix-build = { + files.key = { }; + files."key.pub".secret = false; + runtimeInputs = [ pkgs.openssh ]; + script = '' + ssh-keygen -t ed25519 -N "" -C "stibnite-nix-build" -f "$out"/key + ''; + }; + }; + + sessionOptions = { + options.services.stibnite-session = { + enable = lib.mkEnableOption "interactive ssh access to stibnite under a dedicated keypair"; + + sshUser = lib.mkOption { + type = lib.types.str; + default = "crs58"; + description = "Account on stibnite this key logs in as."; + }; + + hostAlias = lib.mkOption { + type = lib.types.str; + default = "stibnite-session"; + description = "ssh Host alias for the session identity, distinct from the builder alias."; + }; + }; + }; + + mkSessionKeyGenerator = pkgs: { + clan.core.vars.generators.stibnite-agent-session = { + files.key = { }; + files."key.pub".secret = false; + runtimeInputs = [ pkgs.openssh ]; + script = '' + ssh-keygen -t ed25519 -N "" -C "stibnite-agent-session" -f "$out"/key + ''; + }; + }; +in +{ + flake.modules.nixos.stibnite-builder = + { config, pkgs, ... }: + let + cfg = config.services.stibnite-builder; + sshKeyPath = config.clan.core.vars.generators.stibnite-nix-build.files.key.path; + in + (builderOptions config) + // { + config = lib.mkIf cfg.enable ( + (mkBuildKeyGenerator pkgs) + // { + nix.distributedBuilds = true; + + # stibnite subscribes to the same binary cache, so a dependency it can + # substitute itself is not worth shipping to it over ZeroTier. + nix.settings.builders-use-substitutes = true; + + programs.ssh.extraConfig = mkSshBlock { + inherit (cfg) hostAlias sshUser; + inherit sshKeyPath; + }; + + environment.etc."nix/stibnite-store-uri".text = "${cfg.storeUri}\n"; + } + ); + }; + + flake.modules.nixos.stibnite-session = + { config, pkgs, ... }: + let + cfg = config.services.stibnite-session; + sshKeyPath = config.clan.core.vars.generators.stibnite-agent-session.files.key.path; + in + sessionOptions + // { + config = lib.mkIf cfg.enable ( + (mkSessionKeyGenerator pkgs) + // { + programs.ssh.extraConfig = mkSshBlock { + inherit (cfg) hostAlias sshUser; + inherit sshKeyPath; + }; + } + ); + }; + + # stibnite's own side: one account both build mechanisms land in, and the + # authorization for the two keypairs above. + flake.modules.darwin.stibnite-build-host = + { config, lib, ... }: + let + cfg = config.services.stibnite-build-host; + in + { + options.services.stibnite-build-host = { + enable = lib.mkEnableOption "the remote build account restricted to nix-daemon at the SSH boundary and trusted with store-root-equivalent authority"; + + user = lib.mkOption { + type = lib.types.str; + default = "nixbuild"; + description = "Account the nix build protocol is served under."; + }; + + uid = lib.mkOption { + type = lib.types.int; + # Free on stibnite as of 2026-08-31: 501 crs58, 502 runner, + # 535 _dnscrypt-proxy are the only accounts at or above 500. + default = 530; + description = "UID for the build account."; + }; + + buildKeys = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Public keys authorized to speak the nix build protocol. Each is + installed with a forced command and `restrict`, so it can run the + protocol and nothing else at the SSH boundary. This restricts which + program the key starts, not the nix daemon's authority. The account + is a Nix trusted user with store-root-equivalent authority: it can + cause arbitrary paths to enter the store and influence what the + daemon trusts. This grant is required to receive caller-evaluated + unsigned paths. + ''; + }; + + sessionUser = lib.mkOption { + type = lib.types.str; + default = "crs58"; + description = "Account the session keys below log in as."; + }; + + sessionKeys = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Public keys authorized for an ordinary interactive login, with no forced command."; + }; + + authorizeSshAccessGroup = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Add the build account to macOS's `com.apple.access_ssh` service + ACL at activation. On stibnite that ACL nests only the admin + group, and the build account is deliberately not an admin, so + without this every dispatch fails as `Permission denied + (publickey)` with a correct key installed. Set false to manage the + ACL by hand with + `dseditgroup -o edit -a -t user com.apple.access_ssh`. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + users.users.${cfg.user} = { + inherit (cfg) uid; + description = "Remote nix build account"; + # sshd runs a forced command through the account's shell, so + # /usr/bin/false (nix-darwin's default for shell = null) would break + # the build protocol rather than harden it. Interactive use is + # foreclosed by the key options below, which sshd applies before the + # shell ever runs. + shell = "/bin/sh"; + # nix-daemon --stdio is exactly what an ssh-ng caller would have + # invoked (`remote-program` defaults to nix-daemon), so forcing it + # serves both the remote-builder and the remote-store mechanism and + # discards anything else the client asks for. Legacy `ssh://`, which + # would run `nix-store --serve`, is deliberately not served. + openssh.authorizedKeys.keys = map ( + key: ''restrict,command="${config.nix.package}/bin/nix-daemon --stdio" ${trimKey key}'' + ) cfg.buildKeys; + }; + + users.knownUsers = [ cfg.user ]; + + # This deliberate trusted-user grant is store-root-equivalent on + # stibnite: the account can cause arbitrary paths to enter the store and + # influence what the daemon trusts. The forced SSH command restricts + # which program the key starts, not that program's authority. The grant + # is required because an untrusted build account cannot receive unsigned + # caller-evaluated paths, matching magnetite's builder account. + nix.settings.trusted-users = [ cfg.user ]; + + users.users.${cfg.sessionUser}.openssh.authorizedKeys.keys = map trimKey cfg.sessionKeys; + + system.activationScripts.postActivation.text = lib.mkIf cfg.authorizeSshAccessGroup '' + if ! /usr/sbin/dseditgroup -o checkmember -m ${cfg.user} com.apple.access_ssh >/dev/null 2>&1; then + echo "Adding ${cfg.user} to the com.apple.access_ssh service ACL..." + /usr/sbin/dseditgroup -o edit -a ${cfg.user} -t user com.apple.access_ssh + fi + ''; + }; + }; +} diff --git a/packages/docs/src/content/docs/concepts/darwin-build-access.md b/packages/docs/src/content/docs/concepts/darwin-build-access.md new file mode 100644 index 000000000..32f32f0c9 --- /dev/null +++ b/packages/docs/src/content/docs/concepts/darwin-build-access.md @@ -0,0 +1,94 @@ +--- +title: Darwin build access +description: How the fleet reaches stibnite's aarch64-darwin store, and why the remote builder and the remote store are two mechanisms rather than one +sidebar: + order: 9 +--- + +Nix evaluation is portable and nix building is not. +Magnetite, which carries both CI services, is x86_64-linux, so it can evaluate an aarch64-darwin derivation and cannot build one. +Stibnite is the fleet's only aarch64-darwin machine, and until `modules/system/stibnite-access.nix` nothing declared it as a build target: the only `nix.buildMachines` entries anywhere named the rosetta VM and magnetite, both of them Linux. +Darwin derivations therefore had no build target at all, which is a different condition from having a slow one. + +## Two mechanisms, two callers + +The two ways to reach another machine's store are not interchangeable, and treating one as a synonym for the other is the mistake this page exists to prevent. + +A remote builder is `nix.buildMachines` on the caller, or `--builders` on one invocation. +The caller's nix daemon copies the input closure out to stibnite, builds there, and copies the output closure back, so the result exists in the *caller's* store. +That is what a caller wants when the verdict is wanted locally: a CI service that will sign the output and push it to the binary cache, or an operator on magnetite who feeds the result to a later local step. + +A remote store build is `--store ssh-ng://…`, or `--eval-store` for the mirror-image split. +Evaluation stays with the caller and the entire build happens inside stibnite's store: the derivation and its source inputs are copied to stibnite, while the output closure is not copied back. +That is what a caller wants when its own store is empty and stays empty — an ephemeral CI runner, a fresh container, an installer image — where a remote builder would spend the job populating a store that is about to be discarded. +Nothing lands locally, so a caller that needs the output path locally wants the builder instead. + +| | Remote builder | Remote store | +|---|---|---| +| Configured as | `services.stibnite-builder.buildMachines`, spliced into `nix.buildMachines` | `services.stibnite-builder.storeUri`, written to `/etc/nix/stibnite-store-uri` | +| Evaluation | caller | caller | +| Build | stibnite | stibnite | +| Output closure | copied back to the caller | stays in stibnite's store | +| Intended caller | a developer or operator who needs an aarch64-darwin result locally, such as building or testing Darwin configurations from magnetite | a machine whose store is empty and stays empty | + +Both mechanisms speak `ssh-ng` to the same account through the same ssh alias, which is what the `stibnite-access-wiring` check pins. +Legacy `ssh://`, which would run `nix-store --serve` on the far side, is deliberately not served. + +## What the builder entry claims, and why + +`systems` is `aarch64-darwin` alone. +`nix config show extra-platforms` on stibnite reports `aarch64-darwin` and nothing else, so advertising `x86_64-darwin` would route derivations the machine refuses to build. + +`maxJobs` is 4. +Stibnite has 18 logical cores, 12 performance and 6 efficiency, and 64 GiB of memory, and it is also a laptop in interactive use that commits 12 cores and 48 GiB to the rosetta builder VM and the same again to colima when either runs. +The remote share is deliberately a minority of the machine. + +`speedFactor` is 1 and is inert. +Nix compares speed factors only among machines that can build the same system, and there is no second aarch64-darwin machine to compare against. + +`supportedFeatures` is `apple-virt` and `big-parallel`. +Both are real on this hardware. +`benchmark` is dropped because timings taken on a laptop under interactive load are not measurements, and `nixos-test` is dropped because it is a Linux sandbox capability that nix lists unconditionally. + +## Two keys, two authorizations + +The build and session keys are separately authorized so they can be revoked or rotated independently. + +Magnetite generates both keypairs as `clan.core.vars` generators, `stibnite-nix-build` and `stibnite-agent-session`. +No plaintext private key material is committed. +Each private half is committed age/SOPS-encrypted under `vars/per-machine/magnetite/` and is decryptable only by magnetite and the authorized users recorded beside it. +The public halves are committed under the same path, and stibnite's configuration reads them at evaluation time. + +The build key is authorized on `nixbuild`, a non-admin account on stibnite that exists only to serve the build protocol. +Its authorized-keys entry carries `restrict` and a forced command of `nix-daemon --stdio`, which restricts the key to starting that program with no pty, forwarding, shell, or other command. +The forced command does not bound the nix daemon's authority. +`nixbuild` deliberately belongs to Nix's `trusted-users` because an untrusted account cannot receive unsigned store paths that a caller evaluated itself. +That membership is store-root-equivalent on stibnite: the account can cause arbitrary paths to enter the store and influence what the daemon trusts. + +The session key is the broader credential: it authorizes an ordinary login as `crs58`, an admin-group member who is already a Nix trusted user through `@admin`. +It therefore includes build authority plus shell access, while giving automated sessions an identity that can be revoked or rotated independently of the human's keys and the build key. + +The `stibnite-access-wiring` check asserts the separation in both directions: the session key does not appear in the build account's authorized keys, the build key does not appear in the session account's, and the build key's line is exactly its forced command plus the key. + +## What an operator must do + +Two steps are the operator's. + +Run `clan vars generate magnetite` when either keypair is rotated or first created, and commit the resulting encrypted private half and public value under `vars/per-machine/magnetite/`. +Stibnite's authorization reads those values at evaluation time, so an ungenerated key is an evaluation failure rather than a silent grant. + +Activate both ends: `clan machines update magnetite` for the builder entry, the ssh alias and the store URI, and `just activate` on stibnite for the account and its authorization. +Stibnite's activation also adds `nixbuild` to macOS's `com.apple.access_ssh` service ACL, which on this machine nests only the admin group; without that the build account is refused by sshd before the key is ever consulted, and the failure reads as `Permission denied (publickey)` with a correct key installed. +Creating a macOS account requires Full Disk Access when `darwin-rebuild` runs over ssh, so the first activation carrying this change should run in a graphical session on the machine. + +## What is deliberately not enabled + +`nixbot.toml` sets `attribute = "checks.x86_64-linux"`, which prevents CI from evaluating or requesting aarch64-darwin work and makes the builder unreachable from CI. +The configurations in `modules/nixos/nixbot.nix` and `modules/nixos/buildbot.nix` each set `buildSystems = [ "x86_64-linux" ]` as an independent second layer. +These controls remain because stibnite is a laptop without guaranteed availability and a sleeping machine could gate CI. + +## Related + +- [Build service topology](/concepts/build-service-topology/) — the two CI services kept separate from this development builder +- [Clan Integration](/concepts/clan-integration/) — how `clan machines update` deploys these ends +- [Secrets management](/guides/secrets-management/) — how `clan.core.vars` credentials reach a service diff --git a/packages/docs/src/content/docs/concepts/index.md b/packages/docs/src/content/docs/concepts/index.md index 162179457..fc5285c2c 100644 --- a/packages/docs/src/content/docs/concepts/index.md +++ b/packages/docs/src/content/docs/concepts/index.md @@ -20,3 +20,8 @@ Conceptual documentation to help you understand the architecture and design patt - [Deferred module composition](/concepts/deferred-module-composition/) - Understanding deferred module composition where every Nix file is a module organized by aspect - [Clan Integration](/concepts/clan-integration/) - Multi-machine coordination with clan and clear boundaries with other tools - [System-user integration](/concepts/system-user-integration/) - Understanding admin users with integrated home-manager vs non-admin standalone users + +## Build and CI + +- [Build service topology](/concepts/build-service-topology/) - Two CI build services coexisting on magnetite, and the boundaries that keep them apart +- [Darwin build access](/concepts/darwin-build-access/) - How the fleet reaches stibnite's aarch64-darwin store, and why the remote builder and the remote store are two mechanisms diff --git a/vars/per-machine/magnetite/stibnite-agent-session/key.pub/value b/vars/per-machine/magnetite/stibnite-agent-session/key.pub/value new file mode 100644 index 000000000..636532e33 --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-agent-session/key.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMZubXhStfGXDJuE1PXKai8YHGPN6XDFkErjyle9WjOy stibnite-agent-session diff --git a/vars/per-machine/magnetite/stibnite-agent-session/key/machines/magnetite b/vars/per-machine/magnetite/stibnite-agent-session/key/machines/magnetite new file mode 120000 index 000000000..41bd9646c --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-agent-session/key/machines/magnetite @@ -0,0 +1 @@ +../../../../../../sops/machines/magnetite \ No newline at end of file diff --git a/vars/per-machine/magnetite/stibnite-agent-session/key/secret b/vars/per-machine/magnetite/stibnite-agent-session/key/secret new file mode 100644 index 000000000..746ddde05 --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-agent-session/key/secret @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:mD2pXSyWuDLVvDz6LEJmYDYbdIgeWVNv47DA3/LeXzgT5bps8BhPNbzUERh6vzVWb1SpIjVdlzklnp303xQpIOrEf8MEaZ9dtPaBMH6Idu+Y38QGK97j0J1QTjReikqJ6p7952U1bUwsWuBkcgRduAEuOQ3Duqd5gNL8Bg8YcLFRg7msWp5gO+a9xIMAInXsCFqRPETDPfMcQVIBkBAhD6cY0k6T9ggRFiicPNjTYA6kvkdugEC8Rrtdwya9MBKqOxvrZxuyB/LyxNgGusKFf/UW/rB3mOdj0rdIxT4uT4Fv5wVyhN3wr36Geiwbd6/2sVLQPJiP46TdgbknZd8gs0BOmtviPBxnhRYgV4LeR7LAFKtKQrZLPqQMjIvpt/XUTygxJDskADhAGnDHpsEWyOqqQMLOE68KP1Hx8YAfGoebsSn9PM/tJ0MgYp8bOcdB5ddC2nzqBpS3yiHJp4+ScNSch3LynXO8uVfB3hWS69fMplITayd+a2QI1ERB+gVUk2lPesj1oey/6RcCnZHaro0ocFUzK5Yaab+BB11ffPWy4Yg=,iv:HcQ1nE9oyHiZEIFOwiQf8Ex4B6yUMX6zdMZ2TF/HEzA=,tag:VFhONshwQ00ibvEI218DFw==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2TjNuVk1VMklmbU9TUkYv\nRFl6UkxGeFg3OEVKQXgwOC9SOVpDZllUdFhzClRsRGl6dHNybHF0dHJwbnBlYXVZ\nNjVRNUJyVWtmbTRZcWxWaFVuL21GbjgKLS0tIHQ0ZHJPcnd3bnVKbnBuUWZjL096\naklEYkdqdGZaeVZwSGhJYjQyWFdWeXMKF+mhdQxb9RIysw/YN5lSUr8Lc/gFHvLc\nhEBYWRlFnDL5AUDrFHwVKQqZ3F86jIA6niCOQqugWTPyuZOY6j7LVQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1a7a70qcpjemlvk6q4uaf4k77p9eq7lj7wcal5jdj3xuetznyqdrs3mfnsf" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFZXJLSjI1ekJXQTZaMXRH\nNHVjU2ZQSjJhcmorY3NjMUtUN1doZWNUZGowCjA0NnNjdHVOV1F5RUdJR09DS0dz\nSTVaUGtaZjlrRzloakZCT0Q1bWllV0kKLS0tIEIyTkNEaWZBOEtGN0Zpc0NKUXVE\nMEhsYmpwY04vTVRHenlNZ2J1S1I3UVEK31BLQJXTUE0WtQojy07KuLUa/dTFR3Zb\n0Jjm+CZnwRIytIsoROTpaOnL/FGbn6RazzhN508ZFEgU229YN0SdAg==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1vn8fpkmkzkjttcuc3prq3jrp7t5fsrdqey74ydu5p88keqmcupvs8jtmv8" + } + ], + "lastmodified": "2026-08-31T23:17:22Z", + "mac": "ENC[AES256_GCM,data:juMELRuv9Vl/kcfS4xYLFjGgLAPhTcKdUezcdwvs4X7rUqoNzra1q4N3nIh3cCStozh87adJPMrZ2QZXyJ9P/x7/dFiTDpSIruPg9NtPg39WYTh6PS/0+AZHqJQGew+5/v7lClgtHo7O/rUya07LWvqjbkVwFW8rrcEB6oRICII=,iv:2feuuTK6uzEIigpTN3usN2LvTsxnRKZAl/ZX426lq1E=,tag:xStuYm5s6W2cRwAxtyMrpA==,type:str]", + "version": "3.13.3" + } +} diff --git a/vars/per-machine/magnetite/stibnite-agent-session/key/users/cameron b/vars/per-machine/magnetite/stibnite-agent-session/key/users/cameron new file mode 120000 index 000000000..015130152 --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-agent-session/key/users/cameron @@ -0,0 +1 @@ +../../../../../../sops/users/cameron \ No newline at end of file diff --git a/vars/per-machine/magnetite/stibnite-nix-build/key.pub/value b/vars/per-machine/magnetite/stibnite-nix-build/key.pub/value new file mode 100644 index 000000000..2385599f1 --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-nix-build/key.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyo39zRk8tI0IZ238Vk2iOk7K80Rf9IEeOfbD/NkfQf stibnite-nix-build diff --git a/vars/per-machine/magnetite/stibnite-nix-build/key/machines/magnetite b/vars/per-machine/magnetite/stibnite-nix-build/key/machines/magnetite new file mode 120000 index 000000000..41bd9646c --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-nix-build/key/machines/magnetite @@ -0,0 +1 @@ +../../../../../../sops/machines/magnetite \ No newline at end of file diff --git a/vars/per-machine/magnetite/stibnite-nix-build/key/secret b/vars/per-machine/magnetite/stibnite-nix-build/key/secret new file mode 100644 index 000000000..2954cd0e1 --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-nix-build/key/secret @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:uizSA5+21CSLLu425FO1Z/lzwD8U35pRq+KwfSOhkM0FoQbUxHkjUre7fMVQwAQjwVYltcyVNkw//VecTXNJ6CaS6d2HHN3mVrXfy3puSiptU3eSPdiCCY4g3HZtZtBQbD8uzpWQqjif3msyDN5oA8ZcUpa7lrnSQPhtx4WyfsMx2JYpOEEc23wWsEg055irfCIR/QXu5z6w2mUTFCQF5rsP82ySR0o9a1X5czbBS9XHlfUDSQBY3Zf3UWLIoWTel6QoeZVpfJGQ2bl+59uHDtxOoD8Vqwml1lIcZyecuaquqr5P4EdShWQixTB2N1hdqx2xkZYYEXPtud4EFR/RVnR4W271wJ5jhaGNH61NDf+FUK9mea7elnznqbGVOLd7UCC1Xx+H36vpxzMfPkLVcT/grmuMJUq9zc38GF3TLezdoLsRiLrHhWQ4E3z1Vt+0ME5WaRhJb4QXSxZ0vl/rif32wnX1/ODS3mEsRDRTOEdR+ytL4ztvgcR8j652iEQZOuT1AbsicWVsyy1GdrpNjUaPolQPOS88fwqB,iv:xVyTJJ+BYxiTXldjAfinkUz8z/6MOQCuuluZgLoV+3Y=,tag:1+dKhn0da8NuLFYAMh6ucA==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByQlFkaFFZQ3gzTFBqb3My\nQk1Ob2dIQU1VQ20xSjRsN0syUHZMSTJrRmxJCkdjK0wvZklTMDlFOHhuV1VwVE91\nMXQ5N1ZNcGozQzczUVA5OFZhcEptWjAKLS0tIFloMFFqbDJOaXJjSDg5SlNmTTVR\nTVNsSlZwRzRTbE9GV2FBakRGbVFURmcKjpGxx+RVZSo4IQZL1yZjBC4u2n9iKtqh\nuUv0UiHPt4u+rDxrdY7oOtk921EMTfrhc2h9JbSFMnl2eAeKgRsHVw==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1a7a70qcpjemlvk6q4uaf4k77p9eq7lj7wcal5jdj3xuetznyqdrs3mfnsf" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYLysvZUZwQ1lRMGgvS2Zn\ncUIwRUU4SWNSWGI5c2FNMURocG9ZbVJvUGxVCjZkSFhuaElndW90dmlGT1BiKzJi\nZEljV1NMUTVDRmVVQ2FXMjlFWDNOQXMKLS0tIDA2RzVJckVxM3VJV0UrQS94d1FU\nOE9DRDBYelpQY1ZCVks4WjhobXlSa0UKInEmnwU/bokc3njKkQ0SNxfOnvDXMeCz\ne1tcmBXK2h7LIwoCoFMjgd1qG8JrgSaQdk/ReRZS9z0fC0cK2TB4AQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1vn8fpkmkzkjttcuc3prq3jrp7t5fsrdqey74ydu5p88keqmcupvs8jtmv8" + } + ], + "lastmodified": "2026-08-31T23:17:11Z", + "mac": "ENC[AES256_GCM,data:a/T5NopbVUpctCMzw50W2a6xleWgkyhIZ6pMXA5nk/+B82P7ayQT5kzZJpSGHgjc2OOmreHavSt6ALnSSJ2hWao9oEji9aeDKJBiU28iozI2bCEFJ825h32bZXv8wEQ8nMqE2affX+9MFhLZUfwvPRE/xeIPyHpdgKPrmrnIZwY=,iv:liPe+XEDXffVqDxGfZlPvRvEtDa16Po2psWFGya5mEo=,tag:yiYxIVfKBa5DVkn+/X/Jew==,type:str]", + "version": "3.13.3" + } +} diff --git a/vars/per-machine/magnetite/stibnite-nix-build/key/users/cameron b/vars/per-machine/magnetite/stibnite-nix-build/key/users/cameron new file mode 120000 index 000000000..015130152 --- /dev/null +++ b/vars/per-machine/magnetite/stibnite-nix-build/key/users/cameron @@ -0,0 +1 @@ +../../../../../../sops/users/cameron \ No newline at end of file