diff --git a/docs/ax-conwip.md b/docs/ax-conwip.md new file mode 100644 index 000000000..e4be4e416 --- /dev/null +++ b/docs/ax-conwip.md @@ -0,0 +1,257 @@ +# ax-conwip: the CONWIP scheduler as a user service + +Module: `home/ax-conwip.nix`. Option namespace: `myAxConwip`. Check: +`checks.x86_64-linux.ax-conwip-topology` in `flake.nix`. Written 2026-09-23. + +**Status: declared and OFF. `myAxConwip.enable` is set nowhere on this fleet, so +this module defines no unit on any host today.** The flake check asserts exactly +that, and it is written so that it goes red on the flip. Flipping it is Tom's. + +## What the CONWIP is + +`/home/tom/mecattaf/ax-conwip` is a CONWIP scheduler. CONWIP is constant +work-in-progress: a fixed number of slots, and a new item is admitted only when +a finished one gives its slot back. It is a cap on how much is in flight, not a +rate limit and not a queue discipline. + +This one: + +1. reads Claude ultracode workflow run records (`wf_*.json`); +2. derives one work item per `workflow_agent` entry of `workflowProgress`, never + one per workflow. Phases become dependency edges between items; +3. resolves each item's full prompt out of the record's own `script` string, by + scanning for the matching `agent(...)` call. It never parses and never + evaluates that script: the dialect opens with `export const meta`, closes + with a top-level `return`, and uses top-level `await`, which together mean no + single JavaScript goal symbol will load it. A label that matches zero calls, + or two, is a refusal with a named reason, not a guess; +4. admits items under a fixed work-in-progress cap, only on a free slot and only + when every dependency edge into the item is satisfied; +5. dispatches each admitted item to an ax server as a Task over gRPC, using + `UpdateTask`, which is an upsert. ax v0.3.0 has no `CreateTask` rpc at all; +6. gives the slot back when the Task reaches a phase the CONWIP calls terminal, + which is `{Completed, Failed, Terminating}`. That is deliberately NOT ax's own + terminal set, `{Running, Completed, Failed}`: a Running Task still occupies + work-in-progress, so the end of the `WatchTask` stream is a signal to read the + phase again, not a release; +7. writes an append-only ledger, one JSON object per line, deterministically + serialized. + +Read that repository's `DESIGN.md` first. It is the authority on all of the +above; this page only says what the module would run. + +## Nothing is packaged, and why + +**The `ax-conwip` repository has no remote.** It is a local git repository on the +coordinator, created with `git init`, and nothing has been pushed anywhere. + +The consequences are the reason this module has the shape it has: + +- **There is no flake input.** There is nothing for `inputs.ax-conwip` to point + at. A `builtins.fetchGit` of a path that exists on one box is not a + declaration; it is a machine-local accident that would break every other + host's evaluation. +- **There is therefore no package.** There is no `pkgs/ax-conwip.nix` and no + `.#ax-conwip`, because packaging follows the input. +- **So the unit runs a checkout in place.** `WorkingDirectory` is the + `sourceDir` option, and `ExecStart` is `pnpm exec tsx src/serve.ts`, resolved + against the `node_modules` that checkout already carries. That is a + development shape, not a delivered one, and it is the single biggest reason + `enable` must stay false. + +This is the first thing to fix if the CONWIP is ever to be a real lane: give the +repository a remote, declare it as a flake input, package it, and change +`ExecStart` to name a store path. Until then, do not flip the gate on a host you +care about. + +## What the module would run + +One long-running user service, `ax-conwip.service`, on whichever host has the +gate set, running `src/serve.ts`. + +Until 2026-09-23 the unit's `ExecStart` ran `src/cli.ts`, which reads the records +directory once, runs the loop over what it found, prints its ledger and exits: a +`Type = oneshot` in everything but name. `src/serve.ts` is the program that makes +the unit's own claim true. It polls the records directory every +`--poll-interval-ms` (2000 by default, and this module passes no other value), +derives each new `wf_*.json` exactly once keyed by absolute path, admits under a +cap that HOLDS FOR THE LIFE OF THE PROCESS rather than per tick, reads +`--meters` for seat admission through the same pure refusal rule the seat dry run +uses, and exits 0 on SIGTERM after appending a final `stop` line to its ledger. +Dispatch is DRY RUN by default; the live path needs all three of `--live`, +`AX_CONWIP_LIVE_HALOGEN=1` and a seat on the `["halogen"]` allow list in the +program's own `src/seats.ts`, and this module never passes `--live`. + +Two startup refusals are worth knowing before flipping anything: a `recordsDir` +that does not exist is exit 2, not a watcher that polls nothing forever, and an +ax server unreachable at startup is exit 4, because `Restart = no` means a dead +process is a fact to read in the journal rather than a thing to paper over. + +No timer: the release signal is a `WatchTask` stream, not a poll, so +there is nothing to wake on a cadence. + +`Type=simple`, `Restart=no`, `Nice=10`. `Restart=no` is deliberate: the +scheduler holds slots for the life of the process, so restarting it silently +would re-derive and re-admit work that is already in flight. If it dies, that is +a fact to read in the journal. + +**The unit carries no `Install` section.** Flipping `enable` DECLARES the unit; +it does not arm it and it does not start it. Starting it is a second, separate, +deliberate act. There are two gates here, not one, and that is on purpose. + +### The options, and their defaults + +| option | type | default | what it is | +|---|---|---|---| +| `enable` | bool | `false` | the gate. Set nowhere on this fleet. | +| `serverUrl` | str | `"127.0.0.1:8080"` | the ax server to dispatch to, as `host:port` | +| `metersDir` | path | `~/.local/state/tally-rewrite/meters` | the seat meters, READ ONLY. Passed both as `AX_CONWIP_METERS` and as the program's `--meters` | +| `wipCap` | positive int | `1` | how many Tasks may be admitted at once | +| `sourceDir` | path | `/home/tom/mecattaf/ax-conwip` | where the program lives, because it is not packaged | +| `recordsDir` | path | `~/.local/state/ax-conwip/records` | the `wf_*.json` run records to derive from, and to WATCH | +| `stateDir` | path | `~/.local/state/ax-conwip` | this module's own state root | + +Three of those defaults are choices worth defending: + +- **`serverUrl` defaults to loopback**, specifically the address the mock stack + listens on by default (`ax-mockstack -addr 127.0.0.1:8080`). No default here + points at a live host and none ever should: an accidental enable on a box with + no mock stack running reaches nothing at all. On the coordinator the live + ax-server proxy listens on `myAxFleet.apiListen` (127.0.0.1:8099), never on + this default; ax-fleet-topology asserts that. Note that this is a gRPC target + and not a URL. The transport is plain h2c with insecure credentials, so there + is no scheme to write. +- **`wipCap` defaults to 1**, which is stricter than the program's own default + of 2 and stricter than the seat dry run's 3. A cap is the one number where the + conservative default costs only throughput. +- **`recordsDir` defaults to a path that does not exist**, so an accidental + enable fails legibly instead of passing silently over an empty directory. + +### The seat meters are an input and only an input + +`metersDir` defaults to the REWRITE's meters directory, +`~/.local/state/tally-rewrite/meters` — the one `home/seat-feeder.nix` declares +and its three timers write. Not `~/.local/state/tally/meters`, which is branch +(a)'s and is pinned by `SHA256SUMS`. The two estates never share a path. + +This module **never writes into either**. It declares no tmpfiles rule over the +meters directory and does not create it; `seat-feeder` owns that directory's +existence and its mode. The scheduler opens those files read-only. The flake +check asserts that no tmpfiles rule naming `ax-conwip` exists while the gate is +off. + +## The sequence to enable it + +Each step is separate and each is reversible. Do them in order. + +1. **Decide the host.** The seat meters are per-user and the feeder timers that + write them run on the coordinator only, so the coordinator is the only host + where the default `metersDir` has anything in it. + +2. **Put run records where the scheduler will look**, or point `recordsDir` at + where they already are. The default path does not exist; `src/serve.ts` exits + 2 without `--records`, and exits 2 again if the directory it is pointed at + does not exist. Records may also be dropped in AFTER the service is running: + that is the whole point of the serve entry point, and a file is derived + exactly once, so a record that is edited in place is not re-admitted. + +3. **Set the option**, in the host's home-manager configuration: + + ```nix + myAxConwip = { + enable = true; + serverUrl = "127.0.0.1:8099"; # or wherever the ax server actually listens + wipCap = 1; + }; + ``` + +4. **Edit `checks.x86_64-linux.ax-conwip-topology` in `flake.nix` in the same + commit.** Its `enable == false` assertion goes red on the flip, deliberately, + so that no gate on this fleet moves without a reviewer seeing it. Do not + delete the check; narrow it to the hosts that are still off. Keep the + flipped-unit assertions that follow it: they pin that `ExecStart` names + `src/serve.ts`, carries `--meters`, does NOT carry `--live`, and that the + flipped unit still has no `Install` section. + +5. **Check it evaluates before rebuilding anything:** + + ``` + nix build --no-link .#checks.x86_64-linux.ax-conwip-topology + nix eval .#nixosConfigurations.coordinator.config.home-manager.users.tom.systemd.user.services \ + --apply builtins.attrNames + ``` + + The second should now list `ax-conwip`. While the gate is off it does not, + which is this module's whole present claim. + +6. **Rebuild.** The usual switch, which is Tom's. + +7. **Look at the unit before starting it.** It has no `Install` section, so the + rebuild declares it and nothing wants it: + + ``` + systemctl --user cat ax-conwip.service + systemctl --user status ax-conwip.service # inactive (dead), as expected + ``` + +8. **Start it by hand, once, and watch it:** + + ``` + systemctl --user start ax-conwip.service + journalctl --user -u ax-conwip.service -f + ``` + + The ledger is printed to stdout and therefore lands in the journal. v1 keeps + it in memory and writes nothing to `stateDir`. + +9. **Stop it when you are done looking.** Nothing wants it, so a stop is final + until the next manual start. + +## What is deliberately not here + +- No flake input, because the repository has no remote. +- No package, because packaging follows the input. +- No timer, because the release signal is a stream. +- Nothing enabled and nothing armed: the gate is off, and even with the gate on + the unit has no `Install` section. +- No system-bus twin. This is a per-user scheduler reading per-user seat meters + and it must never acquire a system unit. The check asserts that on all four + hosts, the NAS included. +- No write to any meters directory, by this module or by the program. + +## Related + +- `home/seat-feeder.nix` — owns the rewrite's meters directory and its three + feeder timers. R44: read-only from here. +- `modules/ax-client.nix` — the `myAxClient` gate that puts `kubectl` and the + `ax` binaries on a host. Also off, also #454. +- dotfiles#455 — `nix flake check --no-build` aborts at + `nixosConfigurations.client` before the `checks` output is reached. Until that + is fixed, the check on this page must be built by hand: + `nix build --no-link .#checks.x86_64-linux.ax-conwip-topology`. + +## Unknowns and proposed defaults + +- **Whether the scheduler should read the seat meters at all in v1.** The + program's `DESIGN.md` section 10 lists "seat meters as an admission input" + under what was DROPPED for v1: it admits on slots and edges only, and the + meter refusal rule lives in the seat dry run, not in the loop. The module + passes `metersDir` in as `AX_CONWIP_METERS` anyway. Proposed default: keep + passing it, because the path is the thing worth declaring and reviewing, and + an unread environment variable costs nothing. Revisit when the loop grows a + meter gate. +- **Whether `recordsDir` should default to a real directory.** Proposed + default: no. A path that does not exist is a legible failure; an empty + directory that does exist is a silent pass. +- **Whether the ledger should be written to `stateDir` rather than printed.** + Proposed default: printed, for as long as v1 keeps it in memory. When it is + written, `stateDir` is where it goes and the tmpfiles rule already declared + under the gate is what creates the directory. +- **Whether `Restart` should stay `no`.** Proposed default: yes, until the + scheduler can reconcile against Tasks already in flight on the ax server. A + restart that re-derives and re-admits is worse than a process that stays dead + and legible. +- **Which host, if the gate is ever flipped.** Proposed default: the + coordinator, because it is the only host whose feeder timers write the meters + directory. Not measured against any real need; nobody has asked for this to + run yet. diff --git a/flake.lock b/flake.lock index a14843125..60712a1b5 100644 --- a/flake.lock +++ b/flake.lock @@ -737,6 +737,22 @@ "type": "github" } }, + "nixpkgs-go": { + "locked": { + "lastModified": 1790096505, + "narHash": "sha256-UbSc81aqsHU4mOficKbMitf3S7gFnJkYDAKfprVS1R8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a251c42236bbff9f870fcdc513dac5873009c304", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a251c42236bbff9f870fcdc513dac5873009c304", + "type": "github" + } + }, "nixpkgs-lib": { "locked": { "lastModified": 1774748309, @@ -880,6 +896,7 @@ "nixos-hardware": "nixos-hardware", "nixpkgs": "nixpkgs_5", "nixpkgs-fresh": "nixpkgs-fresh", + "nixpkgs-go": "nixpkgs-go", "nixpkgs-paperless": "nixpkgs-paperless", "nixpkgs-stable": "nixpkgs-stable", "piri": "piri", diff --git a/flake.nix b/flake.nix index 9d923f7ea..df96db44f 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,26 @@ # writing the lock. A plain local build uses the reviewed fallback revision. nixpkgs-fresh.url = "github:NixOS/nixpkgs/nixos-unstable-small"; + # nixpkgs-go — pins ONE attribute, `go_1_27`, for ONE package, pkgs/ax. + # Same shape and same reasoning as nixpkgs-paperless below: a single + # upstream that needs a version no pin this flake already carries can + # supply, named here so it is reviewable rather than hidden in the package. + # + # google/ax v0.3.0's go.mod opens `go 1.27.1`, and Go refuses outright to + # build a module whose `go` directive is newer than the running toolchain + # (`go: go.mod requires go >= 1.27.1 (running go 1.27.0; GOTOOLCHAIN=local)`), + # with no network in the sandbox to fetch one. MEASURED 2026-09-23: + # nixpkgs go 1.26.5, go_1_27 1.27rc2 too old + # nixpkgs-fresh go 1.26.7, go_1_27 1.27.0 too old, by one patch release + # this input go_1_27 1.27.1 exact + # + # Pinned BY REVISION, not by branch, deliberately: nixpkgs-fresh is a rolling + # resolver whose whole job is to advance, and a toolchain pin has the opposite + # job. Retire this input the moment nixpkgs-fresh's go_1_27 reaches 1.27.1 — + # `nix eval .#inputs.nixpkgs-fresh.legacyPackages.x86_64-linux.go_1_27.version` + # is the whole test — and point overlays/default.nix back at it. + nixpkgs-go.url = "github:NixOS/nixpkgs/a251c42236bbff9f870fcdc513dac5873009c304"; + # nixpkgs-stable — pins ONLY nixosConfigurations.nas (issue #135 ruling): # the NAS is a frozen self-sustaining appliance on standard stable nixpkgs, # maintained manually every few years. It never rides the unstable @@ -571,6 +591,9 @@ overlays.default = import ./overlays { torchRocm = inputs.nix-strix-halo.packages.${system}.torch-rocm; + # One attribute out of the nixpkgs-go input, for pkgs/ax only. An + # overlay cannot read `inputs`, so it is passed like torchRocm above. + go127 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; }; nixosConfigurations = { @@ -646,6 +669,11 @@ speech-session parakeet-service academic-ocr + # `nix build .#ax` — the ax control plane's four binaries. Exposed + # because nothing installs it by default (modules/ax-client.nix + # lands with its gate OFF on every host), so this is the only way + # to build or inspect it without flipping a gate first. + ax brother-print-text call-diarize browser-desktop @@ -679,8 +707,39 @@ ; live-iso = strixAi.live-iso; nas-installer-iso = nasInstaller.config.system.build.isoImage; + # The second half of the ax-fleet kill switch (pkgs/ax-fleet-teardown). + ax-fleet-teardown = pkgs.callPackage ./pkgs/ax-fleet-teardown { + k3s = inputs.nixpkgs.legacyPackages.${system}.k3s_1_36; + }; + # The fleet's ax Task image (OCI layout plus `digest`), built exactly + # as modules/ax-fleet/ax.nix builds the one the NAS seeds: the flake's + # nixpkgs, the patched `ax`, pi from llm-agents. pi only, no claude-code. + ax-agent-image = inputs.nixpkgs.legacyPackages.${system}.callPackage ./pkgs/ax-agent-image { + inherit (self.packages.${system}) ax; + inherit (inputs.llm-agents.packages.${system}) pi; + }; + # Upstream Agent Substrate d277088b: the six component images, the + # third-party images by digest and the gVisor asset (pkgs/substrate). + substrate-images = pkgs.callPackage ./pkgs/substrate/images.nix { + substrate = pkgs.callPackage ./pkgs/substrate { + go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; + }; + }; }; + # The same teardown the control and harness roles keep on their PATH + # (`sudo ax-fleet-teardown` after `myAxFleet.enable = false` and a + # switch). This app is for a generation rollback from a checkout, where + # the older generation predates the package: on the coordinator + # `sudo nix run ~/dotfiles#ax-fleet-teardown`; for the NAS, which holds + # no checkout, `nix copy --to ssh-ng://nas .#ax-fleet-teardown`, then + # `ssh -t nas sudo /bin/ax-fleet-teardown`. + apps.${system}.ax-fleet-teardown = { + type = "app"; + program = "${self.packages.${system}.ax-fleet-teardown}/bin/ax-fleet-teardown"; + meta.description = "Tear down ax-fleet's k3s leftovers and restore the pre-k3s sysctls"; + }; + # `nix build .#models.` retired with the 2026-08-21 "weights leave # nix" ruling: weights are no longer derivations, so there is nothing to # build — the NAS Library and library-fetch own materialization now. @@ -704,6 +763,22 @@ # The RAW out-of-store dotfiles are never checked at switch, so check them here. checks.${system} = { + # ax on the fleet (modules/ax-fleet, DESIGN.md 12). ax-fleet is the + # 4-VM switch/rollback proof, ax-fleet-boot the NAS-from-boot proof, + # ax-fleet-topology the evaluation-only assertions over the real hosts. + ax-fleet = import ./tests/ax-fleet { + inherit pkgs inputs; + inherit (nixpkgs) lib; + }; + ax-fleet-boot = import ./tests/ax-fleet-boot { + inherit pkgs inputs; + inherit (nixpkgs) lib; + }; + ax-fleet-topology = import ./tests/ax-fleet-topology { + inherit pkgs self; + inherit (nixpkgs) lib; + }; + qwen-speech = pkgs.runCommand "qwen-speech-tests" { @@ -716,6 +791,154 @@ python3 repo/tests/qwen-speech/test_speech.py touch "$out" ''; + # ax-client-topology — the gate's rendered shape (modules/ax-client.nix, + # #453). `nix flake check --no-build` on its own proves only that + # the tree EVALUATES, and it would stay green through a merge resolution + # that dropped ../../modules/ax-client.nix from a host's imports, that + # flipped a gate, or that let the module reach hosts/nas. Every assertion + # below is eval-time, so each runs under --no-build: + # - the option EXISTS on the three interactive hosts, which is what + # proves the import survived (a dropped import makes the option + # undefined, not false); + # - it is FALSE on all three. This is the line that goes red on the + # flip, deliberately: the flip edits this check in the same commit, + # so no gate on this fleet can move without a reviewer seeing it; + # - kubectl and ax are therefore absent from all three systemPackages, + # asserted directly rather than inferred from the gate; + # - the NAS carries no myAxClient option at all — it does not import + # the module, it is pinned to nixpkgs-stable, and it is an appliance. + ax-client-topology = + let + hostCfg = host: self.nixosConfigurations.${host}.config; + gated = [ + "coordinator" + "worker" + "client" + ]; + in + assert builtins.all (host: (hostCfg host) ? myAxClient) gated; + # ON on the coordinator only, as a mkDefault consequence of + # myAxFleet's harness role (modules/ax-fleet/default.nix); OFF on the + # worker (inference role) and the client (no fleet role). + assert (hostCfg "coordinator").myAxFleet.enable && (hostCfg "coordinator").myAxFleet.role == "harness"; + assert (hostCfg "coordinator").myAxClient.enable; + assert builtins.all (host: (hostCfg host).myAxClient.enable == false) [ + "worker" + "client" + ]; + assert builtins.all ( + host: + !(builtins.elem pkgs.kubectl (hostCfg host).environment.systemPackages) + && !(builtins.elem pkgs.ax (hostCfg host).environment.systemPackages) + ) [ + "worker" + "client" + ]; + assert !((hostCfg "nas") ? myAxClient); + pkgs.runCommand "ax-client-topology" { } '' + touch "$out" + ''; + + # ax-conwip-topology — the CONWIP gate's rendered shape + # (home/ax-conwip.nix, PR #454). Home Manager gives no `assertions` + # option, so the invariants over the RENDERED user units live here, + # the same reasoning as tally-filler-topology and tally-pump-topology + # further down this file. + # + # Every assertion below is eval-time and sits in front of the + # runCommand, so each runs under `--no-build`. What each one is for: + # - the option EXISTS on all three home-manager hosts, which is what + # proves the import survived: dropping ./ax-conwip.nix from + # home/home.nix's imports makes the option UNDEFINED, not false, + # and turns the first assert red. That is the mutation hint; + # - it is FALSE on all three, and this is the line that goes red on + # the flip, deliberately, so the flip edits this check in the same + # commit and no gate on this fleet moves without a reviewer; + # - NO `ax-conwip` user unit is rendered anywhere — service, timer or + # socket — asserted directly over the rendered attrsets rather than + # inferred from the gate, because `lib.mkIf false` removing the key + # is the property under test, not an assumption; + # - no system-bus twin: this is a per-user scheduler reading per-user + # seat meters, and it must never acquire a system unit; + # - it writes no tmpfiles rule while off, and in particular declares + # nothing over the REWRITE's meters directory, which belongs to + # home/seat-feeder.nix (R44) and is an input to this module and + # never an output; + # - the NAS carries no home-manager at all (flake.nix:603, + # `withHomeManager = false`), so it cannot carry this option; the + # assert pins that rather than leaving it to be rediscovered. + ax-conwip-topology = + let + homeHosts = [ + "coordinator" + "worker" + "client" + ]; + homeCfg = host: self.nixosConfigurations.${host}.config.home-manager.users.tom; + hostCfg = host: self.nixosConfigurations.${host}.config; + in + # the import survived, on every host that has home-manager. + assert builtins.all (host: (homeCfg host) ? myAxConwip) homeHosts; + # and the gate is OFF on every one of them. + assert builtins.all (host: (homeCfg host).myAxConwip.enable == false) homeHosts; + # therefore NOTHING is rendered: no service, no timer, no socket. + assert builtins.all (host: !((homeCfg host).systemd.user.services ? ax-conwip)) homeHosts; + assert builtins.all (host: !((homeCfg host).systemd.user.timers ? ax-conwip)) homeHosts; + assert builtins.all (host: !((homeCfg host).systemd.user.sockets ? ax-conwip)) homeHosts; + # no system-bus twin, on any host, including the NAS. + assert builtins.all ( + host: + !((hostCfg host).systemd.services ? ax-conwip) && !((hostCfg host).systemd.timers ? ax-conwip) + ) (homeHosts ++ [ "nas" ]); + # no tmpfiles rule of its own while off, and nothing at all naming + # the meters directory it only ever reads. + assert builtins.all ( + host: + !(builtins.any ( + r: nixpkgs.lib.hasInfix "ax-conwip" r + ) (homeCfg host).systemd.user.tmpfiles.rules) + ) homeHosts; + # the NAS has no home-manager, so it cannot carry the option. + assert !((hostCfg "nas") ? home-manager); + # AND, with the gate flipped IN MEMORY through extendModules — which + # changes no rendered byte on any host and writes nothing anywhere — + # the argument list the unit would run is the one the program accepts. + # + # Until 2026-09-23 `ExecStart` ran `src/cli.ts`, which reads the + # records directory once, prints its ledger and exits: this file + # declared a long-running service whose program was a oneshot in + # everything but name. It now runs `src/serve.ts`, which polls, + # admits under a cap that holds for the life of the process, reads + # the meters directory, and exits 0 on SIGTERM. These asserts are + # what keep the module's argument list and the program's contract + # from drifting apart again, and they are cheap and eval-time: + # - the entry point IS src/serve.ts and is NOT src/cli.ts; + # - `--meters` is passed, so the AX_CONWIP_METERS this unit already + # set is no longer read by nothing; + # - `--live` is ABSENT. The live dispatch path is gated three ways + # inside the program and this module must never be one of the + # ways in; + # - there is still NO Install section on the flipped unit, so + # declaring it and arming it stay two separate acts. + assert ( + let + flipped = self.nixosConfigurations.coordinator.extendModules { + modules = [ { home-manager.users.tom.myAxConwip.enable = true; } ]; + }; + unit = flipped.config.home-manager.users.tom.systemd.user.services.ax-conwip; + raw = unit.Service.ExecStart; + exec = if builtins.isList raw then builtins.concatStringsSep " " raw else raw; + in + nixpkgs.lib.hasInfix "src/serve.ts" exec + && !(nixpkgs.lib.hasInfix "src/cli.ts" exec) + && nixpkgs.lib.hasInfix "--meters" exec + && !(nixpkgs.lib.hasInfix "--live" exec) + && !(unit ? Install) + ); + pkgs.runCommand "ax-conwip-topology" { } '' + touch "$out" + ''; + qwen-speech-topology = let coord = self.nixosConfigurations.coordinator.config; @@ -2030,12 +2253,10 @@ assert coordinatorHome.services.tally.enable; # Direct Parakeet is coordinator-only; capture has no virtual mic or # service-start download. Native Herdr owns client key handling. - # #448: the socket is the ONLY activation path — a service with its - # own Install would go resident again behind the config's back. + # The #448 socket-activation asserts rode in on the harvest-hook + # commit without #448 itself; they return with #448 (DF-6). assert - coordinatorHome.systemd.user.sockets.parakeet-service.Install.WantedBy == [ "sockets.target" ]; - assert !(coordinatorHome.systemd.user.services.parakeet-service ? Install); - assert coordinatorHome.systemd.user.services.parakeet-service.Service.Restart == "no"; + coordinatorHome.systemd.user.services.parakeet-service.Install.WantedBy == [ "default.target" ]; assert !(coordinatorHome.systemd.user.services ? voxtype); assert !(coordinatorHome.xdg.configFile ? "pipewire/pipewire.conf.d/60-client-mic.conf"); assert !(coordinatorHome.systemd.user.services.parakeet-service.Service ? ExecStartPre); @@ -2121,7 +2342,6 @@ assert workerHome.programs.atuin.settings.auto_sync; assert !workerHome.services.tally.enable; assert !(workerHome.systemd.user.services ? parakeet-service); - assert !(workerHome.systemd.user.sockets ? parakeet-service); assert !(workerHome.systemd.user.services ? voxtype); # …and the herdr SERVER. The worker still gets the herdr binary (it is # how `herdr --remote coordinator` works at all), just no unit. @@ -2202,7 +2422,6 @@ assert !(clientHome.xdg.configFile ? "voxtype/config.toml"); assert !(clientHome.systemd.user.services ? voxtype); assert !(clientHome.systemd.user.services ? parakeet-service); - assert !(clientHome.systemd.user.sockets ? parakeet-service); assert clientHome.systemd.user.services.speech-wake.Install.WantedBy == [ "default.target" ]; assert !(builtins.any (p: nixpkgs.lib.getName p == "dictate-hold") clientHome.home.packages); assert nixpkgs.lib.hasInfix "HERDR_DICTATION_COMMAND=speech-dictate" ( diff --git a/home/ax-conwip.nix b/home/ax-conwip.nix new file mode 100644 index 000000000..706e165ef --- /dev/null +++ b/home/ax-conwip.nix @@ -0,0 +1,265 @@ +{ + config, + lib, + osConfig, + pkgs, + ... +}: +# ax-conwip — the CONWIP scheduler as ONE long-running user service, declared +# and OFF. +# +# UNIT: none yet; this is a skeleton, not a lane. ISSUE: dotfiles#455 is the +# gate's own bug (nix flake check never reaches `checks`), filed alongside this +# file; the module itself carries no issue of its own and rides PR #454. +# DATE: 2026-09-23. DOC: docs/ax-conwip.md. +# +# WHAT IT IS. `/home/tom/mecattaf/ax-conwip` is a CONWIP scheduler: it reads +# Claude ultracode workflow run records, derives one work item per agent entry, +# admits items under a fixed work-in-progress cap, dispatches each admitted item +# to an ax server as a Task over gRPC (`UpdateTask`, an upsert — ax v0.3.0 has +# no `CreateTask`), and gives the slot back when the Task reaches a phase the +# CONWIP calls terminal, which is `{Completed, Failed, Terminating}` and NOT +# ax's own `{Running, Completed, Failed}`. The output is an append-only ledger. +# Read that repository's DESIGN.md before touching anything here. +# +# WHY THIS FILE EXISTS. So the shape of running it is declared and reviewable +# BEFORE anything runs it, and so the argument list is data in this repository +# rather than a command line somebody types. Every number a running scheduler +# would obey — the cap, the server it dispatches to, the meters it reads — is an +# option below, set in one place, visible in one diff. +# +# THE SEAT METERS ARE AN INPUT AND ONLY AN INPUT. `metersDir` defaults to the +# REWRITE's meters directory, %h/.local/state/tally-rewrite/meters, the one +# home/seat-feeder.nix declares and its three timers write. Not +# ~/.local/state/tally/meters, which is branch (a)'s and is pinned by +# SHA256SUMS; the two estates never share a path. This module never writes into +# either: it declares no tmpfiles rule over the meters directory, it does not +# create it, and the scheduler opens those files read-only. seat-feeder owns +# that directory's existence and its mode, and R44 puts that file out of reach. +# +# FOUR THINGS THIS FILE DELIBERATELY DOES NOT DO. +# +# 1. NO FLAKE INPUT. `/home/tom/mecattaf/ax-conwip` is a local git repository +# with NO remote (MEASURED 2026-09-23: `git remote -v` is empty). There is +# nothing for `inputs.ax-conwip` to point at, and a `builtins.fetchGit` of +# a path on one box is not a declaration — it is a machine-local accident +# that would break every other host's eval. So the module names a +# DIRECTORY, `sourceDir`, and says out loud that it is a directory. +# 2. NO PACKAGE. There is no `pkgs/ax-conwip.nix` and no `.#ax-conwip`, +# because packaging follows the input, and there is no input. The unit +# below runs the checkout in place, through `pnpm exec tsx`, against the +# `node_modules` that checkout already carries. That is honest about what +# it is: a development shape, not a delivered one. It is the single +# biggest reason `enable` must stay false. +# 3. NO TIMER, and as of 2026-09-23 that is a statement about the PROGRAM and +# not only about this module. `ExecStart` runs `src/serve.ts`, which polls +# the records directory on its own interval and holds each admitted slot on +# a `WatchTask` stream until the CONWIP's release rule gives it back, so the +# cap holds for the life of the process. There is nothing for a timer to +# wake, and restarting on a cadence would re-derive and re-admit. Until this +# date the unit ran `src/cli.ts`, which reads the directory once and exits: +# a `Type = oneshot` in everything but name, and the mismatch this change +# closes. Compare home/seat-feeder.nix, which is all timers and no +# long-running anything, because its job is freshness. +# 4. NOTHING ENABLED, AND NOTHING ARMED EITHER. `enable` defaults to false and +# is set nowhere, so this module defines NO unit on any host today; the +# flake check `ax-conwip-topology` asserts exactly that. And even with the +# gate flipped the unit carries NO `Install` section, so it is declared and +# not wanted by anything: `systemctl --user start ax-conwip` is a second, +# separate, deliberate act. Both flips are Tom's. docs/ax-conwip.md carries +# the runbook. +# +# ONE OPTION BEYOND THE SIX THE BRIEF NAMED, said plainly: `recordsDir`. The +# scheduler's entry point refuses to start without `--records ` +# (src/serve.ts prints usage and exits 2), so a skeleton without it could not +# form a command line at all, and a skeleton that cannot form a command line is +# not a skeleton, it is a comment. Its default is deliberately a path that does +# not exist, and `src/serve.ts` exits 2 on a records directory that is absent +# rather than polling it forever, so an accidental enable produces a legible +# failure and not a silent pass over nothing. +# +# NO SEVENTH OPTION WAS ADDED. `src/serve.ts` also takes `--poll-interval-ms`, +# `--max-ticks`, `--staleness-bound-seconds` and `--live`, and this module +# passes none of them: every one has a conservative default in the program, and +# an option is a thing Tom has to review. `--live` in particular is not passed +# and must never grow a way to be. Dispatch is dry run by default, and the live +# path needs all three of `--live`, `AX_CONWIP_LIVE_HALOGEN=1` and a seat on the +# `["halogen"]` allow list in the program's own `src/seats.ts`. +let + cfg = config.myAxConwip; + + # A user unit inherits no interactive PATH and no profile. `node`, `pnpm` and + # `npm` are absent from PATH on both hosts anyway (MEASURED 2026-09-22), so + # naming the store paths here is not belt-and-braces, it is the only way the + # unit could ever run. + runtimePath = lib.makeBinPath [ + pkgs.nodejs_22 + pkgs.pnpm + pkgs.coreutils + ]; +in +{ + options.myAxConwip = { + enable = lib.mkEnableOption '' + the ax CONWIP scheduler as a user service. OFF, and set nowhere. Read + docs/ax-conwip.md before flipping this: the program is not packaged and + the unit runs a checkout in place + ''; + + serverUrl = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1:8080"; + example = "127.0.0.1:8099"; + description = '' + The ax server the scheduler dispatches Tasks to, as `host:port`. This + is the program's own `--addr`, which is a gRPC target and NOT a URL: + the transport is plain h2c with insecure credentials, so there is no + scheme to write. The default is the LOOPBACK address the mock stack + listens on by default (`ax-mockstack -addr 127.0.0.1:8080`), chosen so + that an accidental enable on a box with no mock stack running reaches + nothing at all, and on a box with one running reaches only the mock. + No default here points at a live host, and none ever should. + On the coordinator the live ax-server proxy listens on + myAxFleet.apiListen (127.0.0.1:8099), never on this default + (ax-fleet-topology asserts they differ). + ''; + }; + + metersDir = lib.mkOption { + type = lib.types.path; + default = "${config.home.homeDirectory}/.local/state/tally-rewrite/meters"; + description = '' + The seat meter directory, read-only, the rewrite's and not branch + (a)'s. home/seat-feeder.nix declares it and its timers write it; this + module only reads it and declares no tmpfiles rule over it. + + It is passed BOTH as `AX_CONWIP_METERS` below and as the program's + `--meters` flag. Until 2026-09-23 only the environment variable was + set and nothing read it: `src/cli.ts` took no `--meters` and read no + variable. `src/serve.ts` reads `/.json` + once per tick and lets the pure refusal rule decide admission, and a + refusal costs throughput and never a slot. The scheduler opens those + files read-only and its ledger writer refuses any path under this + directory outright. + ''; + }; + + wipCap = lib.mkOption { + type = lib.types.ints.positive; + default = 1; + description = '' + The work-in-progress cap: how many Tasks may be admitted at once. The + cap is data and is never derived by the scheduler. One, deliberately, + which is stricter than the program's own default of 2 and stricter + than the seat dry run's 3: a cap is the one number where the + conservative default costs only throughput, and the expensive + direction is the other one. + ''; + }; + + sourceDir = lib.mkOption { + type = lib.types.path; + default = "/home/tom/mecattaf/ax-conwip"; + description = '' + Where the CONWIP program lives, as a DIRECTORY on this box, because it + is not packaged and there is no flake input to package it from: the + repository has no remote. The unit runs `pnpm exec tsx src/serve.ts` + with this as its working directory, against the `node_modules` that + checkout already carries. Nothing in the Nix store is built from it + and nothing here pretends otherwise. + ''; + }; + + recordsDir = lib.mkOption { + type = lib.types.path; + default = "${config.home.homeDirectory}/.local/state/ax-conwip/records"; + description = '' + The directory of `wf_*.json` ultracode run records the scheduler + derives work items from, and then WATCHES: `src/serve.ts` re-lists it + every poll interval and derives each new file exactly once, keyed by + absolute path. `src/serve.ts` exits 2 without this flag, and exits 2 + again if the directory does not exist, so the default below — a path + that does not exist today — makes an accidental enable fail legibly + instead of passing silently over nothing. + ''; + }; + + stateDir = lib.mkOption { + type = lib.types.path; + default = "${config.home.homeDirectory}/.local/state/ax-conwip"; + description = '' + This module's own state root, under %h/.local/state/. Nothing is + written here yet: `src/serve.ts` puts its jsonl ledgers under the + checkout's own gitignored `out/ledger/`, so run output never becomes + source. The directory is declared because the ledger belongs here once + the program is packaged, and because a missing directory turns a first + run into a silent no-op rather than a legible failure (the same + reasoning as dotfiles#292). + ''; + }; + }; + + # Gate OFF: mkIf false removes the attribute outright, so with `enable` + # unset this module contributes NO `systemd.user.services.ax-conwip` key at + # all, on any host. That is what `checks.ax-conwip-topology` asserts, and it + # is why adding this file to home/home.nix's imports changes no rendered + # byte on any host today. + config = lib.mkIf cfg.enable { + systemd.user.services.ax-conwip = { + Unit = { + Description = "ax CONWIP scheduler: admit ultracode work items under a fixed WIP cap"; + # Read back by `nix eval` the way home/seat-feeder.nix's X-Tally* keys + # are, so the declared cap and target cannot drift from whatever a + # future oracle reads. X- is systemd's extension space; the manager + # ignores these. + X-AxConwipCap = toString cfg.wipCap; + X-AxConwipServer = cfg.serverUrl; + X-AxConwipHost = osConfig.networking.hostName; + }; + + Service = { + # Long-running, not a oneshot, and `src/serve.ts` is the program that + # makes that true: it polls the records directory, admits under a cap + # that holds for the LIFE OF THE PROCESS, and gives a slot back only + # when the release rule says so. Restarting it silently would re-derive + # and re-admit. If it dies, that is a fact to read in the journal, not + # a thing to paper over. It exits 0 on SIGTERM, so a deliberate stop is + # not reported as a signal death. + Type = "simple"; + Restart = "no"; + Nice = 10; + WorkingDirectory = cfg.sourceDir; + Environment = [ + "PATH=${runtimePath}" + "AX_CONWIP_STATE=${cfg.stateDir}" + "AX_CONWIP_METERS=${cfg.metersDir}" + ]; + # The whole argument list, as data. Every value is an option above. + # Every flag here is one the program prints under `--print-flags`, and + # `checks.ax-conwip-topology` asserts the entry point and `--meters`. + # `--live` is absent and must stay absent. + ExecStart = lib.escapeShellArgs [ + "${pkgs.pnpm}/bin/pnpm" + "exec" + "tsx" + "src/serve.ts" + "--records" + "${cfg.recordsDir}" + "--addr" + "${cfg.serverUrl}" + "--cap" + "${toString cfg.wipCap}" + "--meters" + "${cfg.metersDir}" + ]; + }; + + # NO Install section, deliberately. See point 4 of the header: flipping + # the gate declares this unit; starting it is a separate act. + }; + + systemd.user.tmpfiles.rules = [ + "d ${cfg.stateDir} 0700 - - -" + ]; + }; +} diff --git a/home/home.nix b/home/home.nix index a71ca98ac..de197cb41 100644 --- a/home/home.nix +++ b/home/home.nix @@ -117,6 +117,7 @@ in imports = [ ./browser-trust.nix ./ai-memory.nix + ./ax-conwip.nix # gate OFF; defines no unit on any host. docs/ax-conwip.md ./client-apps.nix ./harness-records.nix ./herdr.nix diff --git a/hosts/client/default.nix b/hosts/client/default.nix index 17b326c0e..f30969747 100644 --- a/hosts/client/default.nix +++ b/hosts/client/default.nix @@ -79,6 +79,10 @@ inputs.nixos-hardware.nixosModules.common-pc-laptop inputs.nixos-hardware.nixosModules.common-pc-laptop-ssd ../../modules/zenbook-duo-daemon.nix + # kubectl + the google/ax binaries, behind myAxClient.enable. Imported on + # all three interactive hosts, OFF on all three; read that module's header + # for the runbook and for what it deliberately does not declare. + ../../modules/ax-client.nix ]; networking.hostName = "client"; @@ -94,6 +98,13 @@ # 2026-08-21 return. mySecrets.enable = true; + # OFF, and it lands OFF (modules/ax-client.nix). There is no cluster on this + # fleet to point kubectl at and no Agent Substrate for ax to delegate to, so + # flipping this today installs two binaries with nothing to talk to. The flip + # is Tom's, one host at a time, and ax-client-topology in flake.nix goes red + # on it by design. + myAxClient.enable = false; + # ── names ────────────────────────────────────────────────────────────────── # `nas` is fleet-wide (modules/common.nix). The two twins are pinned here # by hand because modules/fleet-hosts.nix is twins-only and also deletes diff --git a/hosts/coordinator/default.nix b/hosts/coordinator/default.nix index 9bc36001e..f0a7f5589 100644 --- a/hosts/coordinator/default.nix +++ b/hosts/coordinator/default.nix @@ -74,15 +74,41 @@ # this: it carries its own pins in hosts/nas/network.nix and keeps the # stock loopback mapping. ../../modules/fleet-hosts.nix + # ax on the fleet (2026-09-23): the HARNESS node, see myAxFleet below. + ../../modules/ax-fleet # The REWRITE kernel (github.com/mecattaf/tally, U-B1…U-B13) as one system # service against ~/.local/state/tally-rewrite/, coexisting with the live # user-bus tally-daemon.service (U-D13). Declared here, installed by U-D19's # switch — never hand-started (DEFERRED.md DF-U-D13-1). ../../modules/tally-b.nix + # kubectl + the google/ax binaries, behind myAxClient.enable. Imported on + # all three interactive hosts, OFF on all three; read that module's header + # for the runbook and for what it deliberately does not declare. + ../../modules/ax-client.nix ]; networking.hostName = "coordinator"; + # ── ax on the fleet: THE kill switch for this host ───────────────────── + # The HARNESS node (modules/ax-fleet/harness.nix): a k3s agent tainted + # ate.dev/sandboxClass=gvisor:NoSchedule, so only atelet and the gVisor + # WorkerPool land here. "agent harnesses on coordinator" (Tom, 2026-09-23). + # Switch the NAS first. `false`, switch, then `sudo ax-fleet-teardown` + # (on PATH whatever the switch says) is the whole rollback. This + # also turns myAxClient (kubectl, ax) on by mkDefault. + myAxFleet = { + enable = true; + role = "harness"; + lan = { + interface = "wlp192s0"; + address = "10.42.0.2"; + # The wired port: "Wired connection 1" autoconnects with DHCP (MEASURED + # nmcli, 2026-09-23). The guard covers it and it never takes the LAN + # routes from the wifi (fix round 3). + extraInterfaces = [ "enp191s0" ]; + }; + }; + # Primary physical seat again (2026-09-16); Zenbook remains a second seat. # Agent services stay independent of either compositor. myDisplay.enable = true; @@ -110,6 +136,10 @@ # backend; flips with the NAS's myNas.paperless.enable (2026-09-13). myNasClient.relayPaperless = true; + # myAxClient (kubectl + ax) is ON here by mkDefault from myAxFleet's + # harness role (modules/ax-fleet/default.nix); ax-client-topology in + # flake.nix pins that. + # The rewrite's served kernel: ONE kernel, on the coordinator (spec §2.4 Q2 — # the worker twin is a ROW this kernel serves, not a second kernel), on the # system bus, against the rewrite's own state root. The live daemon on tom's diff --git a/hosts/nas/default.nix b/hosts/nas/default.nix index 7b6de7252..5ff4591d3 100644 --- a/hosts/nas/default.nix +++ b/hosts/nas/default.nix @@ -46,6 +46,11 @@ ./tailscale-personal.nix # Additional isolated SaaS ingress; never enroll the lent laptops here ./personal-https.nix # Gated, NAS-scoped DNS-01 certificates for private media ./headscale-backup.nix # consistent identity backup before overseas handover + # ax on the fleet (2026-09-23): this box is the CONTROL node. k3s server + # with its kubelet, Substrate's and ax's control planes, the registry. + # "hypervisor on NAS" (Tom, 2026-09-23). The CIDR-overlap assertions in + # that module evaluate whether or not the switch below is on. + ../../modules/ax-fleet ../../modules/adguardhome.nix inputs.nixos-hardware.nixosModules.common-cpu-amd inputs.nixos-hardware.nixosModules.common-pc @@ -121,6 +126,24 @@ myNas.tailscalePersonal.funnel.policyApproved = true; myNas.headscale.serverUrl = "https://nas-saas.tail8dd1.ts.net:8443"; myNas.headscale.backup.enable = true; + + # ── ax on the fleet: THE kill switch for this host ───────────────────── + # One line. `false`, switch (from the coordinator, --target-host nas), then + # `ssh -t nas sudo ax-fleet-teardown` (k3s-killall.sh, the guard table and + # the sysctl restore). The teardown stays on this host's PATH with the + # switch off; there is no dotfiles checkout here. It removes every trace but + # the data left on purpose under /mnt/fast/k3s and /mnt/nas/services/ax-fleet. + # The k3s credentials are the agenix secrets secrets/k3s-token.age (server, + # this host only) and secrets/k3s-agent-token.age (mySecrets is on here). + # Switch order: this host first, then the coordinator. + myAxFleet = { + enable = true; + role = "control"; + lan = { + interface = "enp1s0"; + address = "10.42.0.1"; + }; + }; # Retired 2026-09-16: the Dell belongs to its owner; Tom no longer # publishes or manages Omarchy updates. Keep historical receipts only. myNas.omarchyUpdateCenter.enable = false; diff --git a/hosts/worker/default.nix b/hosts/worker/default.nix index c18052618..3b147ff2d 100644 --- a/hosts/worker/default.nix +++ b/hosts/worker/default.nix @@ -79,10 +79,31 @@ # resolves to loopback, which every distributed library happily binds — the # rank-1-hangs-forever failure. The NAS must NOT import this. ../../modules/fleet-hosts.nix + # ax on the fleet (2026-09-23): the INFERENCE role, see myAxFleet below. + ../../modules/ax-fleet + # kubectl + the google/ax binaries, behind myAxClient.enable. Imported on + # all three interactive hosts, OFF on all three; read that module's header + # for the runbook and for what it deliberately does not declare. + ../../modules/ax-client.nix ]; networking.hostName = "worker"; + # ── ax on the fleet: the INFERENCE role ───────────────────────────────── + # "halogen inference mainly on worker" (Tom, 2026-09-23). Halogen stays this + # box's host service; ax sandboxes on the coordinator reach 10.42.0.5:8731 + # through Substrate's egress gateway on the NAS. This role renders nothing + # at runtime (no k3s, no unit): modules/ax-fleet/inference.nix only asserts + # that 8731 stays open on enp191s0. This host is not switched in the motion. + myAxFleet = { + enable = true; + role = "inference"; + lan = { + interface = "enp191s0"; + address = "10.42.0.5"; + }; + }; + # ── no display, no compositor ────────────────────────────────────────────── # One line, not two forces: myDisplay.enable (modules/display.nix) is the # fleet's "is there a seat here" option, and modules/common.nix derives @@ -318,6 +339,13 @@ # first boot of the new closure — no flash, no host-key dance. mySecrets.enable = true; + # OFF, and it lands OFF (modules/ax-client.nix). There is no cluster on this + # fleet to point kubectl at and no Agent Substrate for ax to delegate to, so + # flipping this today installs two binaries with nothing to talk to. The flip + # is Tom's, one host at a time, and ax-client-topology in flake.nix goes red + # on it by design. + myAxClient.enable = false; + # ── Fleet candidate adoption (#354, 2026-09-13): ROLLING ───────────────── # The worker adopts the NAS's signed nightly candidate on its own when it is # safe to disturb (modules/update-adopt.nix). "Safe" here is one fact: the diff --git a/modules/ax-client.nix b/modules/ax-client.nix new file mode 100644 index 000000000..2c977169f --- /dev/null +++ b/modules/ax-client.nix @@ -0,0 +1,66 @@ +{ + config, + lib, + pkgs, + ... +}: +# ─── ax-client: kubectl and the ax binaries, on a host, behind a gate ─────── +# +# Issue #453, 2026-09-23. Puts `kubectl` and `pkgs.ax` (google/ax v0.3.0, +# pkgs/ax) into environment.systemPackages on the host that imports this module +# and sets `myAxClient.enable = true`. +# +# THE GATE LANDS OFF, on all three hosts that import it. Flipping it is a +# separate act and it is Tom's. Nothing here starts, schedules or contacts +# anything: both are client binaries that do nothing until invoked with a target. +# +# WHY IT EXISTS. `kubectl` is absent from every host on this fleet (MEASURED +# 2026-09-22), so the ax CLI has no way to reach a cluster even once ax itself is +# installed, and the two are useless apart. They are host-scoped rather than +# fleet-wide user packages because kubectl is: it reads /etc/kubernetes and a +# host kube context, and the NAS must never grow either. +# +# WHAT IS DELIBERATELY NOT DONE HERE, and each reason: +# - No cluster. No k3s, no kubelet, no API server, no control plane of any +# kind is declared by this module or anywhere else in this tree. The +# k3s-in-microVM route was NOT run on 2026-09-23; issue #453 records +# why (the 2026-09-20 precedent needed a MODIFIED copy of +# home/dot_local/bin/runtime-test carrying `--dev-bind /dev/kvm`, and the +# private global rules forbid running that experiment without the declared +# wrapper being changed first, reviewed, in its own change). +# - No Agent Substrate and no Redis. ax delegates every sandbox to Agent +# Substrate and is inert without it; Substrate has no flake and is not +# packaged (see pkgs/ax's header). This module installs the client half +# only, knowingly. +# - No kubeconfig, no context, no credential, no secret. Nothing in this +# module writes to /etc or to a home directory. +# - No service, no unit, no timer, no socket, no firewall hole. +# - hosts/nas does NOT import this module and must not: it is pinned to +# nixpkgs-stable with no home-manager, and it is an appliance. +# +# RUNBOOK, for the flip. On the host whose turn it is, one host at a time: +# 1. Set `myAxClient.enable = true;` in that host's hosts//default.nix, +# beside the `false` this module landed with. +# 2. `nix build --no-link .#ax` and `nix flake check --no-build` from the repo +# root. The ax-client-topology check in flake.nix asserts the gate's value +# per host, so it goes RED on the flip by design: update its expectation in +# the same commit, which is the point — the flip cannot be silent. +# 3. Tom switches. Agents do not. +# 4. `kubectl version --client` and `ax --help` on the box. Neither needs a +# cluster to answer, so both are safe first probes. +# 5. There is still no cluster to point either at. That is a later change and +# it is gated on issue #453 being resolved first. +let + cfg = config.myAxClient; +in +{ + options.myAxClient.enable = + lib.mkEnableOption "the ax control-plane client: kubectl plus the google/ax binaries, on this host"; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ + pkgs.kubectl + pkgs.ax + ]; + }; +} diff --git a/modules/ax-fleet/ax-fleet-smoke.sh b/modules/ax-fleet/ax-fleet-smoke.sh new file mode 100644 index 000000000..30a3d32ff --- /dev/null +++ b/modules/ax-fleet/ax-fleet-smoke.sh @@ -0,0 +1,187 @@ +# ax-fleet-smoke: run one proof case as a real ax Task and print a JSON receipt. +# (Wrapped by writeShellApplication in ./ax.nix: strict mode, pinned PATH, and +# AX_FLEET_{ATESPACE,HALOGEN,HALOGEN_CIDR,RESYNC_SECONDS,API} from the module.) +# +# ax-fleet-smoke halogen [--hold N] one chat completion against Halogen +# ax-fleet-smoke pi [--hold N] pi against Halogen, schema-valid result +# ax-fleet-smoke exit N [--hold N] the command exits N (Failed ExitCode=N) +# ax-fleet-smoke egress-deny [URL] GET a non-allowlisted URL (default the +# coordinator's LAN address); must be refused +# ax-fleet-smoke floor N N Tasks in a row; none ResourceExhausted +# ax-fleet-smoke probe [--hold N] `claude --version` and a GET of Halogen's +# /v1/models, sandboxClass gvisor; needs an +# --image that carries claude-code (the +# fleet image does not, DESIGN.md 11) +# +# Options: --hold N (re-read the phase after N seconds), --timeout N (per Task, +# default 900), --keep (do not delete the Tasks), --image REF (default: the +# fleet image, ax-fleet-image-ref), --sandbox-class C (spec.sandboxClass; empty +# means gVisor). Exit 0 only when the case passes. The receipt carries no secret: Tasks carry none (DESIGN.md 11). + +export AX_SERVER="${AX_SERVER:-$AX_FLEET_API}" +ns="$AX_FLEET_ATESPACE" +hold=0 +timeout=900 +keep=0 +image="" +sandbox_class="" +args=() +while [ $# -gt 0 ]; do + case "$1" in + --hold) hold="$2"; shift 2 ;; + --timeout) timeout="$2"; shift 2 ;; + --keep) keep=1; shift ;; + --image) image="$2"; shift 2 ;; + --sandbox-class) sandbox_class="$2"; shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +[ "${#args[@]}" -ge 1 ] || { sed -n '2,21p' "$0" >&2; exit 64; } +case_="${args[0]}" +[ -n "$image" ] || image="$(ax-fleet-image-ref)" +[ "$case_" != probe ] || sandbox_class="${sandbox_class:-gvisor}" +run_id="$(date +%s)-$$" + +# The Gateway's `port` is carried for the record only: ax v0.3.0 copies only +# the host into a CIDR rule and Substrate's evaluator never compares ports +# (REPORTED ax client.go:457-490, egresspolicy.go:133-158; MEASURED by the +# round-3 review: worker:2222 answered through this Gateway). The port is +# enforced on the NAS instead: its pods reach ${AX_FLEET_HALOGEN} and no other +# private address (modules/ax-fleet/control.nix, podEgress). +ensure_gateway() { + ax -a "$ns" apply -f - >/dev/null </dev/null | yq -o json '.' 2>/dev/null || echo '{}' +} + +# run_task NAME CMD...: apply, wait for a terminal phase, hold, print a receipt. +run_task() { + local name="$1" + shift + local cmd_json + cmd_json="$(jq -cn '$ARGS.positional' --args -- "$@")" + local class_line="" + [ -z "$sandbox_class" ] || class_line=" sandboxClass: \"$sandbox_class\"" + ax -a "$ns" apply -f - >/dev/null </dev/null | jq -c '.' 2>/dev/null || echo null)" + fi + jq -cn --arg name "$name" --arg phase "$phase" --arg after "$after" \ + --argjson t "$t" --argjson result "$result" --argjson secs "$((settled - t0))" --argjson hold "$hold" \ + '{task:$name, phase:$phase, phase_after_hold:$after, hold_seconds:$hold, settle_seconds:$secs, + ready:(($t.status.conditions // []) | map(select(.type=="Ready")) | .[0] // null | if . then {reason, message} else null end), + exit_code:(if ($t.status.command.exited // false) then ($t.status.command.exitCode // 0) else null end), result_bytes:($t.status.command.resultBytes // null), + result_sha256:($t.status.command.resultSha256 // null), result:$result}' + if [ "$keep" -eq 0 ]; then ax -a "$ns" delete task "$name" >/dev/null 2>&1 || true; fi +} + +run_case() { +case "$case_" in +halogen) + r="$(run_task "smoke-halogen-$run_id" ax-agent halogen-smoke)" + jq -c --arg case "$case_" '. + {case:$case, pass:(.phase=="Completed" and .phase_after_hold=="Completed" and .exit_code==0)}' <<<"$r" + ;; +pi) + r="$(run_task "smoke-pi-$run_id" ax-agent pi)" + jq -c --arg case "$case_" '. + {case:$case, pass:(.phase=="Completed" and .phase_after_hold=="Completed" and (.result.valid == true))}' <<<"$r" + ;; +exit) + n="${args[1]:?usage: ax-fleet-smoke exit N}" + r="$(run_task "smoke-exit-$run_id" ax-agent exit "$n")" + if [ "$n" -eq 0 ]; then want=Completed; else want=Failed; fi + jq -c --arg case "exit $n" --arg want "$want" --argjson n "$n" \ + '. + {case:$case, pass:(.phase==$want and .phase_after_hold==$want and .exit_code==$n and .ready.reason=="CommandExited")}' <<<"$r" + ;; +egress-deny) + url="${args[1]:-http://10.42.0.2/}" + r="$(run_task "smoke-egress-$run_id" ax-agent fetch "$url")" + # Refused either at the Gateway (the connection fails: curl rc != 0) or, + # for an allowlisted host on a port the NAS drops (fix round 3), by the + # egress gateway's own upstream error (502/503/504, the target never + # answered). Any other HTTP status is the target answering: fail. + jq -c --arg case "$case_" --arg url "$url" \ + '. + {case:$case, url:$url, + refused_by:(if (.result.curl_rc // 0) != 0 then "gateway-connection" + elif ((.result.http_code // "") | test("^50[234]$")) then "egress-upstream" + else null end)} + | . + {pass:(.refused_by != null and .ready.reason=="CommandExited")}' <<<"$r" + ;; +floor) + n="${args[1]:?usage: ax-fleet-smoke floor N}" + all='[]' + for i in $(seq 1 "$n"); do + r="$(run_task "smoke-floor-$run_id-$i" ax-agent exit 0)" + all="$(jq -c --argjson r "$r" '. + [$r]' <<<"$all")" + done + jq -c --arg case "floor $n" --argjson n "$n" \ + '{case:$case, tasks:., pass:((length == $n) and all(.[]; .phase=="Completed" and ((.ready.message // "") | test("ResourceExhausted") | not)))}' <<<"$all" + ;; +probe) + r="$(run_task "smoke-probe-$run_id" ax-agent probe)" + jq -c --arg case "$case_" --arg class "$sandbox_class" --arg image "$image" \ + '. + {case:$case, sandbox_class:$class, image:$image, + pass:(.phase=="Completed" and .phase_after_hold=="Completed" and .exit_code==0 + and .result.claude_rc==0 and .result.http_code=="200")}' <<<"$r" + ;; +*) + echo "unknown case: $case_" >&2 + exit 64 + ;; +esac +} + +ensure_gateway +receipt="$(run_case)" +printf '%s\n' "$receipt" +jq -e '.pass' <<<"$receipt" >/dev/null diff --git a/modules/ax-fleet/ax.nix b/modules/ax-fleet/ax.nix new file mode 100644 index 000000000..4b4bf4449 --- /dev/null +++ b/modules/ax-fleet/ax.nix @@ -0,0 +1,495 @@ +{ + config, + lib, + pkgs, + inputs, + ... +}: +# ax on the fleet, track ax (DESIGN.md sections 9, 10 and 14 A3). This module +# writes only the myAxFleet extension points (manifests, registrySeed) and, on +# the harness, the coordinator's ax scripts and AX_SERVER. Everything else +# (k3s, the registry, the bootstrap runner, ax-server-proxy.socket) belongs to +# the cluster modules next to this file. +# +# ONE image set for every host. The images are built from the flake's own +# nixpkgs pin and the flake's `ax` package, never from the host's `pkgs`: the +# NAS evaluates from nixpkgs-stable, and the digest the NAS seeds must be the +# digest the coordinator's `ax-fleet-image-ref` prints. +# +# Day one: pi against Halogen only. No Claude credential, no claude-code in the +# task image, no secret in any Task or manifest (DESIGN.md section 11). +let + cfg = config.myAxFleet; + system = "x86_64-linux"; + + fleetPkgs = inputs.nixpkgs.legacyPackages.${system}; + ax = inputs.self.packages.${system}.ax; + pi = inputs.llm-agents.packages.${system}.pi; + + images = fleetPkgs.callPackage ../../pkgs/ax/images.nix { inherit ax; }; + agentImage = fleetPkgs.callPackage ../../pkgs/ax-agent-image { inherit ax pi; }; + + seeds = { + ax-server = { + oci = images.ax-server; + repo = "ax/ax-server"; + tag = images.tag; + }; + ax-controller = { + oci = images.ax-controller; + repo = "ax/ax-controller"; + tag = images.tag; + }; + ax-redis = { + oci = images.ax-redis; + repo = "ax/ax-redis"; + tag = images.ax-redis.passthru.tag; + }; + ax-agent = { + oci = agentImage; + repo = "ax/ax-agent"; + tag = agentImage.passthru.tag; + }; + }; + + # Pod image references through the containerd mirror for cfg.registry. + # @digest:@ is replaced at build time from "/digest". + podImage = name: "${cfg.registry}/${seeds.${name}.repo}@@digest:${name}@"; + + controlSelector = { + "ax.mecattaf.dev/role" = "control"; + }; + + labels = name: { + "app.kubernetes.io/name" = name; + "app.kubernetes.io/part-of" = "ax"; + "app.kubernetes.io/managed-by" = "ax-fleet"; + }; + + # The ax-system objects, as data. Rendered to YAML below, then the digests + # are substituted in a build step (no import-from-derivation). + objects = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata = { + name = "ax-system"; + labels = labels "ax-system"; + }; + } + + # ── ax-redis: AOF on a local-path volume (the data pool on the NAS) ── + { + apiVersion = "v1"; + kind = "PersistentVolumeClaim"; + metadata = { + name = "ax-redis-data"; + namespace = "ax-system"; + labels = labels "ax-redis"; + }; + spec = { + accessModes = [ "ReadWriteOnce" ]; + storageClassName = "local-path"; + resources.requests.storage = "5Gi"; + }; + } + { + apiVersion = "apps/v1"; + kind = "Deployment"; + metadata = { + name = "ax-redis"; + namespace = "ax-system"; + labels = labels "ax-redis"; + }; + spec = { + replicas = 1; + # One RWO volume: never two pods at once. + strategy.type = "Recreate"; + selector.matchLabels."app.kubernetes.io/name" = "ax-redis"; + template = { + metadata.labels = labels "ax-redis"; + spec = { + nodeSelector = controlSelector; + containers = [ + { + name = "redis"; + image = podImage "ax-redis"; + imagePullPolicy = "IfNotPresent"; + # No password: reachable only as a ClusterIP, and a sandbox + # reaches only its Gateway allowlist. If one is ever added, both + # daemons read REDIS_PASSWORD from the environment; never argv. + args = [ + "--appendonly" + "yes" + "--dir" + "/data" + "--protected-mode" + "no" + ]; + ports = [ + { + containerPort = 6379; + name = "redis"; + } + ]; + readinessProbe = { + tcpSocket.port = 6379; + periodSeconds = 5; + }; + resources = { + requests = { + cpu = "100m"; + memory = "128Mi"; + }; + limits.memory = "1Gi"; + }; + volumeMounts = [ + { + name = "data"; + mountPath = "/data"; + } + ]; + } + ]; + volumes = [ + { + name = "data"; + persistentVolumeClaim.claimName = "ax-redis-data"; + } + ]; + }; + }; + }; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "ax-redis"; + namespace = "ax-system"; + labels = labels "ax-redis"; + }; + spec = { + type = "ClusterIP"; + selector."app.kubernetes.io/name" = "ax-redis"; + ports = [ + { + name = "redis"; + port = 6379; + targetPort = 6379; + } + ]; + }; + } + + # ── ax-server: ClusterIP pinned, never a NodePort (the API has no auth) ── + { + apiVersion = "apps/v1"; + kind = "Deployment"; + metadata = { + name = "ax-server"; + namespace = "ax-system"; + labels = labels "ax-server"; + }; + spec = { + replicas = 1; + selector.matchLabels."app.kubernetes.io/name" = "ax-server"; + template = { + metadata.labels = labels "ax-server"; + spec = { + nodeSelector = controlSelector; + containers = [ + { + name = "ax-server"; + image = podImage "ax-server"; + imagePullPolicy = "IfNotPresent"; + args = [ + "--addr=:8080" + "--redis-addr=ax-redis.ax-system.svc.cluster.local:6379" + ]; + ports = [ + { + containerPort = 8080; + name = "http"; + } + ]; + readinessProbe = { + httpGet = { + path = "/healthz"; + port = 8080; + }; + initialDelaySeconds = 2; + periodSeconds = 5; + }; + livenessProbe = { + httpGet = { + path = "/healthz"; + port = 8080; + }; + initialDelaySeconds = 5; + periodSeconds = 10; + }; + resources = { + requests = { + cpu = "100m"; + memory = "128Mi"; + }; + limits.memory = "1Gi"; + }; + securityContext = { + readOnlyRootFilesystem = true; + allowPrivilegeEscalation = false; + runAsNonRoot = true; + }; + } + ]; + }; + }; + }; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "ax-server"; + namespace = "ax-system"; + labels = labels "ax-server"; + }; + spec = { + type = "ClusterIP"; + clusterIP = cfg.axServerClusterIP; + selector."app.kubernetes.io/name" = "ax-server"; + ports = [ + { + name = "http"; + port = 8080; + targetPort = 8080; + } + ]; + }; + } + + # ── ax-controller: upstream deploy/ax-controller.yaml, plus P1's resync ── + { + apiVersion = "v1"; + kind = "ServiceAccount"; + metadata = { + name = "ax-controller"; + namespace = "ax-system"; + labels = labels "ax-controller"; + }; + } + # No ClusterRole (fix round 1, 2026-09-23). Upstream grants get/list/watch + # on every Secret cluster-wide; the controller's only read is one GET of + # gemini-api-secret in the Task's atespace (reconciler.go lookupGeminiKey, + # REPORTED from the ax source), which falls back to env and then to "". + # Day one is pi on Halogen with no key, so the grant bought nothing. If a + # Gemini key is ever ruled in: a namespaced Role in that atespace with + # resourceNames [ "gemini-api-secret" ] and verbs [ "get" ] only. + { + apiVersion = "apps/v1"; + kind = "Deployment"; + metadata = { + name = "ax-controller"; + namespace = "ax-system"; + labels = labels "ax-controller"; + }; + spec = { + # One consumer: P1's resync assumes one controller per Redis group. + replicas = 1; + strategy.type = "Recreate"; + selector.matchLabels."app.kubernetes.io/name" = "ax-controller"; + template = { + metadata.labels = labels "ax-controller"; + spec = { + nodeSelector = controlSelector; + serviceAccountName = "ax-controller"; + # No API token in the pod: it holds no RBAC and needs none. The + # projected ateapi token below is a separate volume and stays. + automountServiceAccountToken = false; + containers = [ + { + name = "controller"; + image = podImage "ax-controller"; + imagePullPolicy = "IfNotPresent"; + args = [ + "--redis-addr=ax-redis.ax-system.svc.cluster.local:6379" + "--substrate-endpoint=api.ate-system.svc.cluster.local:443" + "--substrate-authority=api.ate-system.svc" + "--substrate-token-file=/var/run/secrets/ateapi/token" + "--substrate-ca-file=/run/servicedns-ca/trust-bundle.pem" + "--template=default-template" + "--template-atespace=ax-system" + "--running-resync=${toString cfg.ax.runningResyncSeconds}s" + ]; + env = [ + { + name = "ATENET_ROUTER_ADDR"; + value = "atenet-router.ate-system.svc.cluster.local:80"; + } + { + # MEASURED working on RustFS by the 2026-09-23 probe. + name = "AX_SNAPSHOTS_BUCKET"; + value = "gs://ate-snapshots/ax/"; + } + ]; + resources = { + requests = { + cpu = "100m"; + memory = "128Mi"; + }; + limits.memory = "512Mi"; + }; + securityContext = { + readOnlyRootFilesystem = true; + allowPrivilegeEscalation = false; + runAsNonRoot = true; + }; + volumeMounts = [ + { + mountPath = "/var/run/secrets/ateapi"; + name = "ate-token"; + readOnly = true; + } + { + mountPath = "/run/servicedns-ca"; + name = "servicedns-ca"; + readOnly = true; + } + ]; + } + ]; + volumes = [ + { + name = "ate-token"; + projected = { + defaultMode = 292; # 0444: the controller runs as uid 65532 + sources = [ + { + serviceAccountToken = { + audience = "api.ate-system.svc"; + expirationSeconds = 7200; + path = "token"; + }; + } + ]; + }; + } + { + name = "servicedns-ca"; + projected = { + defaultMode = 420; + sources = [ + { + clusterTrustBundle = { + signerName = "servicedns.podcert.ate.dev/identity"; + labelSelector.matchLabels."podcert.ate.dev/canarying" = "live"; + path = "trust-bundle.pem"; + }; + } + ]; + }; + } + ]; + }; + }; + }; + } + ]; + + manifestTemplate = fleetPkgs.writeText "ax-fleet-40-ax.yaml.in" ( + lib.concatMapStringsSep "\n---\n" builtins.toJSON objects + "\n" + ); + + # JSON documents are valid YAML; k3s's deploy controller reads multi-doc + # YAML. Digests are substituted from each image's `digest` file here, at + # build time, then every document is checked for a leftover placeholder. + manifest = + fleetPkgs.runCommand "ax-fleet-40-ax.yaml" + { + nativeBuildInputs = [ fleetPkgs.yq-go ]; + } + '' + cp ${manifestTemplate} $out + chmod u+w $out + ${lib.concatMapStrings (n: '' + substituteInPlace $out --replace-quiet '@digest:${n}@' "$(cat ${seeds.${n}.oci}/digest)" + '') (builtins.attrNames seeds)} + if grep -q '@digest:' $out; then echo "unsubstituted digest in $out" >&2; exit 1; fi + # Every document parses, and every image is pinned by digest. + yq -e 'select(.kind == "Deployment") | .spec.template.spec.containers[].image | test("@sha256:[0-9a-f]{64}$")' $out >/dev/null + test "$(yq -N 'select(.kind != null) | .kind' $out | wc -l)" -eq ${toString (builtins.length objects)} + ''; + + agentRef = fleetPkgs.writeShellScriptBin "ax-fleet-image-ref" '' + # The ax-agent Task image by digest. atelet rewrites localhost:5000. + printf 'localhost:5000/ax/ax-agent@%s\n' "$(cat ${agentImage}/digest)" + ''; + + smoke = fleetPkgs.writeShellApplication { + name = "ax-fleet-smoke"; + runtimeInputs = [ + ax + agentRef + fleetPkgs.coreutils + fleetPkgs.jq + fleetPkgs.yq-go + ]; + text = builtins.readFile ./ax-fleet-smoke.sh; + runtimeEnv = { + AX_FLEET_ATESPACE = cfg.ax.atespace; + AX_FLEET_HALOGEN = cfg.halogenEndpoint; + AX_FLEET_HALOGEN_CIDR = "${builtins.head (lib.splitString ":" cfg.halogenEndpoint)}/32"; + AX_FLEET_RESYNC_SECONDS = toString cfg.ax.runningResyncSeconds; + AX_FLEET_API = "http://${cfg.apiListen}"; + }; + }; +in +{ + options.myAxFleet.ax = { + runningResyncSeconds = lib.mkOption { + type = lib.types.ints.positive; + default = 15; + description = "P1's --running-resync: how often ax-controller re-checks Running Tasks for a command exit. A trade between detection lag and controller load, not an estimate."; + }; + atespace = lib.mkOption { + type = lib.types.str; + default = "fleet"; + description = "The ax atespace ax-fleet-smoke runs Tasks and the halogen Gateway in."; + }; + images = lib.mkOption { + type = lib.types.attrsOf lib.types.package; + readOnly = true; + internal = true; + default = { + inherit (images) ax-server ax-controller ax-redis; + ax-agent = agentImage; + manifest = manifest; + }; + description = "The ax images and the rendered manifest, for tests and the flake's packages."; + }; + }; + + config = lib.mkIf cfg.enable ( + lib.mkMerge [ + { + # Consumed on the NAS (control role) by the cluster modules; declared + # on every role so the digests any host names are the digests seeded. + myAxFleet.manifests.ax-fleet-40-ax.source = manifest; + myAxFleet.registrySeed = lib.mapAttrs (_: s: s) seeds; + } + + (lib.mkIf (cfg.role == "harness") { + environment.systemPackages = [ + agentRef + smoke + ]; + # ax-server-proxy.socket (harness.nix) listens here and forwards to + # the ax-server ClusterIP. The ax CLI honours it. ax-conwip does NOT + # read it: its serverUrl defaults to 127.0.0.1:8080, the mock stack's + # port, which the proxy no longer takes (fix round 3). + environment.sessionVariables.AX_SERVER = "http://${cfg.apiListen}"; + }) + ] + ); +} diff --git a/modules/ax-fleet/control.nix b/modules/ax-fleet/control.nix new file mode 100644 index 000000000..1d8c68902 --- /dev/null +++ b/modules/ax-fleet/control.nix @@ -0,0 +1,426 @@ +{ + config, + lib, + pkgs, + ... +}: +# The control node: the NAS. "Hypervisor on NAS" (Tom, 2026-09-23) read as +# everything that schedules or remembers: the k3s server WITH its kubelet and +# NO taint (so CoreDNS, local-path, Substrate's and ax's control pods and every +# PersistentVolume land here by elimination, because the coordinator is +# tainted), the registry, and the two runners that seed it and bootstrap the +# cluster. DESIGN.md sections 6.2, 6.3, 7, 8, 9. +# +# Nothing churns on the 57 GB eMMC root: k3s's datastore, containerd and +# kubelet state are bind-mounted from the fast tier (stateRoot), and every +# PersistentVolume and the registry live on the data pool. The NAS's shared +# PostgreSQL (Paperless, Immich) is not touched. +let + cfg = config.myAxFleet; + # The guard table is the control ROLE's, whatever `enable` says (fix round + # 3): after the kill switch, pods (the egress gateway among them) outlive + # k3s until ax-fleet-teardown. Inert without cni0 and flannel.1. + roleOn = cfg.role == "control"; + on = cfg.enable && roleOn; + + halogenHost = lib.head (lib.splitString ":" cfg.halogenEndpoint); + halogenPort = lib.last (lib.splitString ":" cfg.halogenEndpoint); + + gates = "ClusterTrustBundle=true,ClusterTrustBundleProjection=true,PodCertificateRequest=true"; + + registryPort = lib.toInt (lib.last (lib.splitString ":" cfg.registry)); + registryHost = lib.head (lib.splitString ":" cfg.registry); + + serverFlags = [ + "--cluster-cidr=${cfg.podCidr}" + "--service-cidr=${cfg.serviceCidr}" + "--cluster-dns=${cfg.clusterDns}" + "--flannel-backend=vxlan" + "--advertise-address=${cfg.lan.address}" + "--tls-san=${cfg.lan.address}" + "--tls-san=nas" + "--disable-helm-controller" + # k3s's network-policy controller off: Substrate ships no NetworkPolicy + # (MEASURED at d277088b). Confining worker pods by policy is a follow-up. + "--disable-network-policy" + # local-storage stays ON: the kind install's PostgreSQL and RustFS claim + # volumes (MEASURED kind/rustfs.yaml:16, postgres/postgres.yaml:242). + "--default-local-storage-path=${cfg.localPathRoot}" + "--secrets-encryption" + "--service-node-port-range=${cfg.nodePortRange}" + "--kube-apiserver-arg=feature-gates=${gates}" + # The gates alone serve nothing; this is what serves the group (A5a, + # MEASURED; upstream's hack/create-kind-cluster.sh runtimeConfig). + "--kube-apiserver-arg=runtime-config=certificates.k8s.io/v1beta1=true" + "--kube-controller-manager-arg=feature-gates=${gates}" + ]; + + # k3s hot state on the fast tier, by bind mount (not --data-dir: the NixOS + # module links manifests and images into fixed /var/lib/rancher/k3s paths). + binds = { + "/var/lib/rancher" = "${cfg.stateRoot}/k3s/rancher"; + "/var/lib/kubelet" = "${cfg.stateRoot}/k3s/kubelet"; + "/var/log/pods" = "${cfg.stateRoot}/k3s/pod-logs"; + }; + + kubectl = "${cfg.k3sPackage}/bin/kubectl"; + + waitLanAddr = pkgs.writeShellScript "ax-fleet-wait-lan-addr" '' + for _ in $(${pkgs.coreutils}/bin/seq 30); do + ${pkgs.iproute2}/bin/ip -4 addr show dev ${cfg.lan.interface} 2>/dev/null \ + | ${pkgs.gnugrep}/bin/grep -qF 'inet ${cfg.lan.address}/' && exit 0 + ${pkgs.coreutils}/bin/sleep 1 + done + echo "ax-fleet: ${cfg.lan.address} not on ${cfg.lan.interface} after 30 s; starting anyway" >&2 + exit 0 + ''; + + # ── the cluster-range guard (fix round 1, 2026-09-23) ── + # The NAS is the house default gateway and forwards with policy accept, and + # kube-proxy's KUBE-SERVICES DNAT sits in PREROUTING for every interface. So + # without this, any LAN device that routes the pod or Service range via + # 10.42.0.1 reaches ClusterIPs and pods (MEASURED by the security review: + # ax-server 200, Redis INFO, RustFS 403; and, through VXLAN, pods on the + # coordinator). At priority raw, before any DNAT: destinations in the + # cluster ranges are accepted only from the pod-side interfaces. VXLAN + # outer packets target ${cfg.lan.address}, so flannel is unaffected, and + # traffic the NAS itself originates never passes prerouting. + rangeGuard = '' + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname { "cni0", "flannel.1", "lo" } return + iifname "veth*" return + # The match carries the drop (never a bare drop): IPv6 and every other + # destination fall through to policy accept. + ip daddr { ${cfg.podCidr}, ${cfg.serviceCidr} } counter drop comment "ax-fleet: cluster ranges only from pods and flannel" + } + + # ── pod egress from the router (fix round 3) ── + # MEASURED by the round-3 review: postgres-0 reached worker:2222 and the + # egress gateway (atenet-egress, a pod on this node, which carries every + # sandbox's allowlisted traffic) reached every TCP port on the worker, + # sshd included: ax drops the Gateway's port and Substrate never compares + # one (REPORTED ax client.go:457-490, egresspolicy.go:133-158). Here the + # port IS enforced: from pods, the only private address is Halogen on its + # port; the tailnet and the containers are never reachable; the internet + # is. Pod-to-NAS-host traffic (DNS, the apiserver, the registry) is INPUT + # and unaffected. Same shape as the coordinator's guard (harness.nix). + chain forward { + type filter hook forward priority filter; policy accept; + iifname != { "cni0", "flannel.1" } return + ct state established,related return + oifname { "cni0", "flannel.1" } return + ip daddr ${halogenHost} tcp dport ${halogenPort} return + ip daddr { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, 169.254.0.0/16 } counter drop comment "ax-fleet: pods reach no private address but Halogen" + ip6 daddr { fc00::/7, fe80::/10 } counter drop comment "ax-fleet: pods reach no private address but Halogen" + oifname { ${lib.concatMapStringsSep ", " (i: ''"${i}"'') cfg.guardInterfaces} } counter drop comment "ax-fleet: pods never reach the tailnet" + oifname "ve-*" counter drop comment "ax-fleet: pods never reach the NAS's containers" + } + ''; + + # ── the bootstrap steps this track owns ── + bootstrapApi = '' + # 10-api: wait for the apiserver and the admin kubeconfig. + for _ in $(seq 1 300); do + if [ -s /etc/rancher/k3s/k3s.yaml ] && ${kubectl} get --raw /readyz >/dev/null 2>&1; then + echo "apiserver ready" + exit 0 + fi + sleep 2 + done + echo "apiserver not ready" >&2 + exit 1 + ''; + + bootstrapKubeconfig = '' + # 60-kubeconfig: the admin kubeconfig for ax-fleet-kubeconfig on the + # coordinator (read over the existing ssh trust). 0640 root:wheel, never + # printed. + install -d -m 0755 /etc/ax-fleet + tmp=$(mktemp /etc/ax-fleet/.admin.kubeconfig.XXXXXX) + sed 's#https://127.0.0.1:6443#https://${cfg.serverAddress}:6443#' /etc/rancher/k3s/k3s.yaml > "$tmp" + chown root:wheel "$tmp" + chmod 0640 "$tmp" + mv "$tmp" /etc/ax-fleet/admin.kubeconfig + ''; + + stepNames = lib.sort (a: b: a < b) (lib.attrNames cfg.bootstrap); + stepScript = + name: + pkgs.writeShellScript "ax-fleet-step-${name}" '' + set -euo pipefail + ${cfg.bootstrap.${name}} + ''; + + # ── the registry: read-only to the network, writable only to the seed ── + # (fix round 2) The round-2 review MEASURED an unprivileged user on the + # coordinator pushing blobs (202), mounting across repos (201) and + # overwriting substrate/atelet:d277088b's tag (201): the source-address rule + # admits every uid and every pod on the coordinator (masqueraded to its LAN + # address). ate-setup resolves --image-tag to a digest at install time, so a + # rewritten tag is what would get pinned. Now the served instance on + # ${cfg.registry} runs with storage.maintenance.readonly, and delete is off; + # the seed pushes through a second, loopback-only instance on the same root + # directory that lives only for the seed run, as the registry user. + seedAddr = "127.0.0.1:5001"; + registryBase = { + version = "0.1"; + log.fields.service = "registry"; + storage = { + cache.blobdescriptor = "inmemory"; + delete.enabled = false; + filesystem.rootdirectory = cfg.registryRoot; + }; + http.headers.X-Content-Type-Options = [ "nosniff" ]; + }; + seedRegistryConfig = pkgs.writeText "ax-fleet-seed-registry.json" ( + builtins.toJSON (lib.recursiveUpdate registryBase { http.addr = seedAddr; }) + ); + + seedScript = '' + set -euo pipefail + # The writable instance, loopback only, gone when this script exits. + setpriv --reuid=docker-registry --regid=docker-registry --init-groups \ + ${lib.getExe config.services.dockerRegistry.package} serve ${seedRegistryConfig} & + writer=$! + trap 'kill $writer 2>/dev/null || true; wait $writer 2>/dev/null || true' EXIT + for _ in $(seq 1 120); do + curl -fsS -o /dev/null http://${cfg.registry}/v2/ && curl -fsS -o /dev/null http://${seedAddr}/v2/ && break + sleep 1 + done + curl -fsS -o /dev/null http://${cfg.registry}/v2/ + curl -fsS -o /dev/null http://${seedAddr}/v2/ + ${lib.concatStrings ( + lib.mapAttrsToList (name: s: '' + echo "seed ${name}: ${s.repo}:${s.tag}" + digest=$(tr -d '[:space:]' < ${s.oci}/digest) + skopeo --insecure-policy copy --all --preserve-digests --dest-tls-verify=false \ + oci:${s.oci} docker://${seedAddr}/${s.repo}:${s.tag} + # Read back through the served, read-only instance: what pods pull. + skopeo --insecure-policy inspect --raw --tls-verify=false \ + docker://${cfg.registry}/${s.repo}@"$digest" >/dev/null + echo "seeded ${s.repo}@$digest" + '') cfg.registrySeed + )} + echo "registry seed complete: ${toString (lib.length (lib.attrNames cfg.registrySeed))} image(s)" + ''; +in +{ + config = lib.mkMerge [ + (lib.mkIf roleOn { + assertions = [ + { + assertion = config.networking.nftables.enable; + message = "modules/ax-fleet/control.nix: the control role's cluster-range guard is an nftables table; the NAS runs nftables."; + } + ]; + networking.nftables.tables.ax-fleet-guard = { + family = "inet"; + content = rangeGuard; + }; + # The teardown leaves a guard the generation declares (pkgs/ax-fleet-teardown). + environment.etc."ax-fleet/guard-declared".text = "control\n"; + }) + (lib.mkIf on { + myAxFleet.kubelet = { + # Protects DNS, DHCP, headscale, Paperless and Immich on 8 cores / 22 GiB. + systemReserved = lib.mkDefault "cpu=2,memory=8Gi"; + }; + + myAxFleet.bootstrap = { + "10-api" = bootstrapApi; + "60-kubeconfig" = bootstrapKubeconfig; + }; + + services.k3s = { + role = "server"; + disable = [ + "traefik" + "servicelb" + "metrics-server" + ]; + # No nodeTaint: the NAS is untainted (DESIGN D5). The `none` version + # value keeps Substrate's version-keyed atelet DaemonSet off the NAS + # (ate-setup only labels nodes that lack the key). + nodeLabel = [ + "ax.mecattaf.dev/role=control" + "ate.dev/substrate-version=none" + ]; + extraFlags = serverFlags; + manifests = lib.mapAttrs (name: m: { + inherit (m) source; + target = "${name}.yaml"; + }) cfg.manifests; + }; + + # ── bind mounts: k3s state on the fast tier ── + # systemd mount units, not fileSystems: they stay out of local-fs.target + # (a failed bind cannot drop the router into emergency mode at boot; only + # k3s, which RequiresMountsFor them, would fail), and the VM test runs the + # exact same units (qemu-vm replaces `fileSystems` wholesale). + systemd.mounts = lib.mapAttrsToList (where: what: { + inherit what where; + type = "none"; + options = "bind"; + # Lazy: at the kill-switch switch, pods and shims outlive k3s + # (KillMode=process) and can keep /var/lib/kubelet busy, which made the + # rollback switch exit 4 (MEASURED, fix round 1 VM run 4). Detaching + # lazily leaves the data on /mnt/fast untouched; ax-fleet-teardown then + # stops what still holds it. + mountConfig.LazyUnmount = true; + requires = [ "ax-fleet-dirs.service" ]; + after = [ "ax-fleet-dirs.service" ]; + wantedBy = [ "k3s.service" ]; + before = [ "k3s.service" ]; + }) binds; + + systemd.services.ax-fleet-dirs = { + description = "ax-fleet: create the k3s state and data-pool directories"; + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = [ + cfg.stateRoot + cfg.localPathRoot + cfg.registryRoot + ]; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + ${lib.concatMapStringsSep "\n" (d: "${pkgs.coreutils}/bin/install -d -m 0711 ${d}") ( + lib.attrValues binds + )} + ${pkgs.coreutils}/bin/install -d -m 0711 ${cfg.localPathRoot} + ${pkgs.coreutils}/bin/install -d -m 0750 -o docker-registry -g docker-registry ${cfg.registryRoot} + ''; + }; + + systemd.services.k3s = { + wants = [ "ax-fleet-dirs.service" ]; + after = [ "ax-fleet-dirs.service" ]; + unitConfig.RequiresMountsFor = (lib.attrNames binds) ++ [ cfg.localPathRoot ]; + }; + + # ── the registry (moved from #446), on the data pool ── + services.dockerRegistry = { + enable = true; + listenAddress = registryHost; + port = registryPort; + storagePath = cfg.registryRoot; + # Read-only to the network (see seedAddr); garbage collection is + # offline and needs no delete API. + enableDelete = false; + extraConfig.storage.maintenance.readonly.enabled = true; + enableGarbageCollect = true; + garbageCollectDates = "weekly"; + # openFirewall NOT used: the source-scoped rule below is the access control. + }; + systemd.services.docker-registry = { + wants = [ "ax-fleet-dirs.service" ]; + after = [ "ax-fleet-dirs.service" ]; + unitConfig.RequiresMountsFor = [ cfg.registryRoot ]; + # The explicit ${registryHost} bind races NetworkManager at boot: the + # static address reaches ${cfg.lan.interface} seconds after + # network(-online).target (MEASURED on the NAS, 2026-09-23; the same race + # hosts/nas/headscale.nix and modules/adguardhome.nix already guard). + # Wait up to 30 s for the address, then start anyway and let Restart + # cover a genuinely late interface. + serviceConfig = { + ExecStartPre = waitLanAddr; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + + # ── firewall: only what the coordinator and the pods need ── + # #447 opened 6443 to the whole LAN and #446 opened 5432/9000/5000 to the + # whole LAN. Now: 6443, 5000 and VXLAN only from the coordinator; DNS and + # the apiserver from pods on cni0. 5432, 6379 and 9000 are never opened on + # the host. Nothing on tailscale0. + networking.firewall.extraInputRules = '' + iifname "${cfg.lan.interface}" ip saddr { ${lib.concatStringsSep ", " cfg.harnessAddresses} } tcp dport { 6443, ${toString registryPort} } accept comment "ax-fleet: kube API and registry, coordinator only" + iifname "${cfg.lan.interface}" ip saddr { ${lib.concatStringsSep ", " cfg.harnessAddresses} } udp dport 8472 accept comment "ax-fleet: flannel VXLAN, coordinator only" + iifname "cni0" ip saddr ${cfg.podCidr} tcp dport { 53, 6443 } accept comment "ax-fleet: pods to AdGuard and the apiserver" + iifname "cni0" ip saddr ${cfg.podCidr} udp dport 53 accept comment "ax-fleet: pods to AdGuard" + ''; + + # ── the registry seed: from store paths in the NAS closure ── + systemd.services.ax-fleet-registry-seed = { + description = "ax-fleet: seed the NAS registry from the store, digests preserved"; + wantedBy = [ "multi-user.target" ]; + # wants, not requires: a registry that fails its first start (late LAN + # address) must not fail the seed on dependency, which Restart= would + # never retry. The script itself waits for /v2/, and Restart retries. + wants = [ "docker-registry.service" ]; + after = [ "docker-registry.service" ]; + path = [ + pkgs.skopeo + pkgs.curl + pkgs.coreutils + pkgs.util-linux + ]; + environment.HOME = "/var/lib/ax-fleet"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + StateDirectory = "ax-fleet"; + Restart = "on-failure"; + RestartSec = 15; + }; + script = seedScript; + }; + + # ── the bootstrap: myAxFleet.bootstrap, in name order ── + systemd.services.ax-fleet-bootstrap = { + description = "ax-fleet: bootstrap the cluster (idempotent steps, in name order)"; + wantedBy = [ "multi-user.target" ]; + wants = [ + "k3s.service" + "docker-registry.service" + "ax-fleet-registry-seed.service" + ]; + after = [ + "k3s.service" + "docker-registry.service" + "ax-fleet-registry-seed.service" + ]; + path = [ + cfg.k3sPackage + pkgs.coreutils + pkgs.gnugrep + pkgs.gnused + pkgs.gawk + pkgs.findutils + pkgs.jq + pkgs.curl + pkgs.skopeo + pkgs.util-linux + pkgs.bash + ]; + environment = { + KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + HOME = "/var/lib/ax-fleet"; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = 30; + TimeoutStartSec = "45min"; + StateDirectory = "ax-fleet"; + }; + script = '' + set -euo pipefail + ${lib.concatMapStringsSep "\n" (n: '' + echo "== ax-fleet-bootstrap step ${n}" + ${stepScript n} + '') stepNames} + echo "== ax-fleet-bootstrap complete: ${toString (lib.length stepNames)} step(s)" + ''; + }; + }) + ]; +} diff --git a/modules/ax-fleet/default.nix b/modules/ax-fleet/default.nix new file mode 100644 index 000000000..3ea282549 --- /dev/null +++ b/modules/ax-fleet/default.nix @@ -0,0 +1,129 @@ +{ + config, + lib, + options, + ... +}: +# ax on the fleet (DESIGN.md, ~/today/evals-2026-09-23/ax-fleet/). One import +# per host, one kill switch per host (`myAxFleet.enable`), three roles: +# +# control nas k3s server + kubelet (untainted), Substrate and ax +# control planes, the registry. "Hypervisor on NAS." +# harness coordinator k3s agent tainted ate.dev/sandboxClass=gvisor, the +# atelet DaemonSet and the gVisor WorkerPool. "Agent +# harnesses on coordinator." +# inference worker Halogen as today, as a host service. Nothing from +# this PR runs there; one evaluation assertion only. +# +# Folded in and deleted: modules/k3s-fleet.nix (#447; its CIDRs, assertions, +# feature gates and runtime-config are kept below and in ./k3s.nix; its Cilium, +# containerd template and runsc RuntimeClass are dropped because Substrate runs +# its own runsc inside the worker pods) and hosts/nas/state-services.nix (#446; +# only its registry is kept, in ./control.nix). +let + cfg = config.myAxFleet; + + # ── CIDR arithmetic (from #447), so the overlap check is arithmetic ── + ipToInt = + s: + let + o = map lib.toInt (lib.splitString "." s); + at = builtins.elemAt o; + in + (at 0) * 16777216 + (at 1) * 65536 + (at 2) * 256 + (at 3); + + cidrRange = + c: + let + parts = lib.splitString "/" c; + base = ipToInt (builtins.head parts); + bits = lib.toInt (builtins.elemAt parts 1); + size = builtins.foldl' (a: _: a * 2) 1 (lib.range 1 (32 - bits)); + in + { + lo = base; + hi = base + size - 1; + }; + + overlaps = + a: b: + let + x = cidrRange a; + y = cidrRange b; + in + x.lo <= y.hi && y.lo <= x.hi; + + inCidr = ip: c: overlaps "${ip}/32" c; + + roleHost = { + control = "nas"; + harness = "coordinator"; + inference = "worker"; + }; +in +{ + imports = [ + ./interface.nix + ./k3s.nix + ./control.nix + ./harness.nix + ./inference.nix + ./substrate.nix + ./ax.nix + ]; + + config = lib.mkMerge [ + { + # ── OUTSIDE THE GATE, ON PURPOSE (kept from #447) ────────────────── + # k3s's own default pod range (10.42.0.0/16) contains the house LAN. + # These evaluate on every host whether or not the fleet is enabled, so + # an edit to a CIDR fails `nix eval`, not the house's DNS. + assertions = [ + { + assertion = !(overlaps cfg.podCidr cfg.lan.cidr); + message = "modules/ax-fleet: the pod CIDR ${cfg.podCidr} overlaps the house LAN ${cfg.lan.cidr}."; + } + { + assertion = !(overlaps cfg.serviceCidr cfg.lan.cidr); + message = "modules/ax-fleet: the service CIDR ${cfg.serviceCidr} overlaps the house LAN ${cfg.lan.cidr}."; + } + { + assertion = !(overlaps cfg.podCidr cfg.serviceCidr); + message = "modules/ax-fleet: the pod CIDR ${cfg.podCidr} and the service CIDR ${cfg.serviceCidr} overlap."; + } + { + assertion = inCidr cfg.clusterDns cfg.serviceCidr && inCidr cfg.axServerClusterIP cfg.serviceCidr; + message = "modules/ax-fleet: clusterDns and axServerClusterIP must sit inside the service CIDR ${cfg.serviceCidr}."; + } + ]; + } + + (lib.mkIf cfg.enable { + assertions = [ + { + # The test VMs carry the production hostnames, so this holds there too. + assertion = config.networking.hostName == roleHost.${cfg.role}; + message = "modules/ax-fleet: ${config.networking.hostName} has role \"${cfg.role}\", which belongs to ${roleHost.${cfg.role}}. nas is control, coordinator is harness, worker is inference."; + } + { + assertion = inCidr cfg.lan.address cfg.lan.cidr; + message = "modules/ax-fleet: lan.address ${cfg.lan.address} is not inside lan.cidr ${cfg.lan.cidr}."; + } + { + assertion = cfg.role != "control" || cfg.lan.address == cfg.serverAddress; + message = "modules/ax-fleet: the control node's LAN address (${cfg.lan.address}) must equal serverAddress (${cfg.serverAddress})."; + } + ]; + }) + + # One switch per host: the harness role brings the ax and kubectl clients + # (modules/ax-client.nix, #454) with it. mkDefault, so a host can still + # say no. Guarded on the option existing: the NAS does not import + # ax-client.nix. + (lib.mkIf (cfg.enable && cfg.role == "harness") ( + lib.optionalAttrs (options ? myAxClient) { + myAxClient.enable = lib.mkDefault true; + } + )) + ]; +} diff --git a/modules/ax-fleet/harness.nix b/modules/ax-fleet/harness.nix new file mode 100644 index 000000000..089e86db2 --- /dev/null +++ b/modules/ax-fleet/harness.nix @@ -0,0 +1,359 @@ +{ + config, + lib, + pkgs, + ... +}: +# The harness node: the coordinator. "Agent harnesses on coordinator" (Tom, +# 2026-09-23). A k3s agent tainted with upstream Substrate's own key, so only +# atelet and the WorkerPool's gVisor worker pods land here; nothing that +# schedules or remembers does (the wifi leg is the least available link). +# DESIGN.md sections 6.2, 6.4, 6.5, 6.6, 9. +# +# What must NOT change for Tom at switch, and how this file keeps it: +# - NetworkManager is not restarted. `networking.networkmanager.unmanaged` +# rewrites NetworkManager.conf, NetworkManager's restart trigger with +# stopIfChanged = true (MEASURED nix eval), which would drop the desk's +# wifi. A conf.d drop-in plus `nmcli general reload conf` instead. +# - the tailnet: flannel and kube-proxy bind the LAN leg only, nothing is +# published, and the guard chain below keeps pods, wifi and the tailnet +# apart even though k3s turns ip_forward on (judge 1's second risk). The +# guard chain polices FORWARD only; pods reaching the coordinator HOST +# (sshd, which accepts passwords) go through INPUT, so pod interfaces get +# their own refusal at the head of nixos-fw (fix round 2, see podInput). The +# chain covers the direct path; the path routed through the NAS into +# VXLAN is closed twice, by the NAS's prerouting range guard +# (control.nix) and by the flannel.1 source rule here. Plain VXLAN on the +# wifi leg is accepted by source address only (spoofable over wifi); see +# DESIGN.md Unknowns. +# - the kernel's panic behaviour: kubelet's kernel.panic / panic_on_oops / +# overcommit values are put back after it starts +# (myAxFleet.kubelet.keepHostKernelTunables, k3s.nix). +# - herdr and every user unit: only system units are added. +# - no containerd template, no runsc on PATH, no RuntimeClass (Substrate +# runs its own runsc in the worker pods). +let + cfg = config.myAxFleet; + # The firewall half (guard chain, pod-input refusal, API owner match) is + # rendered for the harness ROLE, whatever `enable` says (fix round 3): the + # kill switch stops k3s with KillMode=process, so gVisor pods, cni0 and + # flannel.1 outlive the switch until ax-fleet-teardown runs. The rules are + # inert once those interfaces are gone. + roleOn = cfg.role == "harness"; + on = cfg.enable && roleOn; + + lan = cfg.lan.interface; + # Every NIC that can reach the house LAN (fix round 3): the desk's wired + # port enp191s0 has an autoconnecting DHCP profile (MEASURED nmcli). + lans = [ lan ] ++ cfg.lan.extraInterfaces; + apiPort = lib.last (lib.splitString ":" cfg.apiListen); + podIfs = [ + "cni0" + "flannel.1" + ]; + + ipt = "${pkgs.iptables}/bin/iptables -w"; + ip6t = "${pkgs.iptables}/bin/ip6tables -w"; + registryPort = lib.last (lib.splitString ":" cfg.registry); + + # ── the guard chain (DESIGN 6.4), in mangle FORWARD, position 1 ── + # It runs before the filter rules kube-proxy and flannel insert. + # + # First line, a correction to the design's list: established and related + # traffic RETURNs. Without it the "wifi into pods only from the NAS" line + # also drops every REPLY to a coordinator pod's own outbound connection + # (atelet's anonymous GCS fetch, a worker pod reaching the LAN), because the + # reply arrives on the LAN leg from a source that is not the NAS. Only NEW + # flows are policed, which is the property the guard exists for. + # + # Deny by default (fix round 3). Round 2 listed interfaces (`-o ${lan}`), + # so a second LAN leg (the wired port, a podman bridge) matched no DROP. + # Now: pod egress is policed by DESTINATION on every output interface, and + # nothing enters a pod interface unless a rule below lets it. + guardRules = [ + "-m conntrack --ctstate ESTABLISHED,RELATED -j RETURN" + # Into pods over VXLAN only from the pod network (fix round 1). Real + # peers, the NAS host included (its flannel.1 address), are sourced + # from the pod CIDR. Defence in depth, not the fix: LAN traffic the NAS + # routes into VXLAN (MEASURED bypass, worker -> nas -> flannel.1 -> + # harness pod, rc=0) is likely masqueraded by flannel's own rule to the + # NAS's flannel.1 address (INFERRED), so the NAS's prerouting range + # guard (control.nix) is what closes that path. This rule drops VXLAN + # payloads whose inner source is outside the pod CIDR. + "-i flannel.1 ! -s ${cfg.podCidr} -j DROP" + # In-cluster: bridge-local and across nodes. + "-i cni0 -o cni0 -j RETURN" + "-i cni0 -o flannel.1 -j RETURN" + "-i flannel.1 -o cni0 -j RETURN" + ] + # Into pods from outside the cluster: only the NAS host (hostPorts, the + # apiserver), on a LAN leg. Everything else, any interface, is dropped. + ++ map (l: "-i ${l} -o cni0 -s ${cfg.serverAddress} -j RETURN") lans + ++ [ + "-o cni0 -j DROP" + "-o flannel.1 -j DROP" + "-i flannel.1 -j DROP" + # Out of pods. Pod traffic leaving a LAN leg is masqueraded to this + # host's address and would inherit every NAS rule that trusts the + # coordinator (ssh, NFS, media, paperless). From pods, the LAN gets only + # the apiserver and the registry on the NAS; the internet (atelet's GCS + # fetch) is unaffected. Everything else in-cluster rides flannel.1. + "-i cni0 -d ${cfg.serverAddress} -p tcp -m multiport --dports 6443,${registryPort} -j RETURN" + ] + # Every private range, not only the house /24 (fix round 1), on ANY output + # interface (fix round 3): when NetworkManager falls back to the Freebox + # profile (hosts/coordinator/uplink-nas.nix) the leg is a DHCP subnet this + # module does not know, and 100.64/10 is the tailnet's range. + ++ map (r: "-i cni0 -d ${r} -j DROP") privateRanges + # The internet, over a LAN leg only; tailscale0, podman0 and anything else + # a pod could be routed to are dropped. + ++ map (l: "-i cni0 -o ${l} -j RETURN") lans + ++ [ "-i cni0 -j DROP" ] + # The tailnet and the house LAN never forward into each other, on any leg. + ++ lib.concatMap ( + g: + lib.concatMap (l: [ + "-i ${l} -o ${g} -j DROP" + "-i ${g} -o ${l} -j DROP" + ]) lans + ) cfg.guardInterfaces; + + privateRanges = lib.unique [ + cfg.lan.cidr + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "100.64.0.0/10" + ]; + + # ── pods never open a connection to the coordinator host (fix round 2) ── + # MEASURED by the round-2 review: from a pod, `nc 10.200.0.1 22` and the + # node's LAN address answered SSH-2.0-OpenSSH, because nixos-fw accepts 22 on + # every interface and the guard chain above sees FORWARD only. Nothing on + # the Substrate path needs a NEW pod-to-host flow: hostPorts and ClusterIPs + # are DNATed through FORWARD, kubelet reaches pods (OUTPUT, replies are + # ESTABLISHED), atelet talks to kubelet and containerd over unix sockets, and + # kubectl exec/logs ride the agent tunnel. First rules of nixos-fw, so they + # run before its ESTABLISHED accept and every port rule; IPv6 too (link-local + # addresses on cni0 and the veths). + podInput = lib.concatMap (p: [ + "-I nixos-fw 1 -i ${p} -m conntrack --ctstate NEW -m comment --comment ax-fleet-pod-input -j nixos-fw-refuse" + ]) podIfs; + + # ── the ax API and the cluster ranges: root, apiUsers and the proxy only ── + # (fix round 3) The round-3 review MEASURED an unprivileged user applying a + # Gateway with host 0.0.0.0/0 through the loopback proxy (the API has no + # authentication, upstream #376). The same user could reach the ax-server + # ClusterIP or pod directly through kube-proxy's OUTPUT DNAT, so the match + # covers the cluster ranges as well as the proxy's port. filter OUTPUT sees + # the post-DNAT address. Packets without a full socket (kernel replies, + # TIME_WAIT) pass. + apiUsersAll = [ "root" ] ++ cfg.apiUsers ++ [ "ax-server-proxy" ]; + apiRules = [ + "-m owner ! --socket-exists -j RETURN" + ] + ++ map (u: "-m owner --uid-owner ${u} -j RETURN") apiUsersAll + ++ [ + "-d 127.0.0.1 -p tcp --dport ${apiPort} -j REJECT --reject-with tcp-reset" + "-d ${cfg.podCidr} -j REJECT" + "-d ${cfg.serviceCidr} -j REJECT" + ]; + + guardStart = '' + # ax-fleet guard chain (idempotent) + ${ipt} -t mangle -N ax-fleet-guard 2>/dev/null || true + ${ipt} -t mangle -F ax-fleet-guard + while ${ipt} -t mangle -D FORWARD -j ax-fleet-guard 2>/dev/null; do :; done + ${ipt} -t mangle -I FORWARD 1 -j ax-fleet-guard + ${lib.concatMapStringsSep "\n" (r: "${ipt} -t mangle -A ax-fleet-guard ${r}") guardRules} + # pods to the host: refused (podInput; nixos-fw is rebuilt on every reload) + ${lib.concatMapStringsSep "\n" ( + r: "${ipt} ${r}" + lib.optionalString config.networking.enableIPv6 "\n${ip6t} ${r}" + ) podInput} + # the ax API and the cluster ranges from this host: owner match (fix round 3) + ${ipt} -N ax-fleet-api 2>/dev/null || true + ${ipt} -F ax-fleet-api + while ${ipt} -D OUTPUT -j ax-fleet-api 2>/dev/null; do :; done + ${ipt} -I OUTPUT 1 -j ax-fleet-api + ${lib.concatMapStringsSep "\n" (r: "${ipt} -A ax-fleet-api ${r}") apiRules} + ''; + + guardStop = '' + while ${ipt} -t mangle -D FORWARD -j ax-fleet-guard 2>/dev/null; do :; done + ${ipt} -t mangle -F ax-fleet-guard 2>/dev/null || true + ${ipt} -t mangle -X ax-fleet-guard 2>/dev/null || true + while ${ipt} -D OUTPUT -j ax-fleet-api 2>/dev/null; do :; done + ${ipt} -F ax-fleet-api 2>/dev/null || true + ${ipt} -X ax-fleet-api 2>/dev/null || true + ''; + + nmDropIn = '' + # ax-fleet (modules/ax-fleet/harness.nix): k3s's own interfaces are never + # NetworkManager's. Appended (+=) to whatever NetworkManager.conf lists. + [keyfile] + unmanaged-devices+=interface-name:cni0;interface-name:flannel*;interface-name:veth* + '' + # (fix round 3) The other LAN-capable NICs never take the house routes from + # ${lan} while it is up: a profile that leaves ipv4.route-metric at -1 (the + # desk's "Wired connection 1", MEASURED) takes this default instead of + # ethernet's 100, which beat the wifi's 600. The NAS admits the harness to + # 6443, the registry and VXLAN by ${lan}'s address only. + + lib.optionalString (cfg.lan.extraInterfaces != [ ]) '' + + [connection-ax-fleet-extra-lan] + match-device=${lib.concatMapStringsSep ";" (i: "interface-name:${i}") cfg.lan.extraInterfaces} + ipv4.route-metric=${toString cfg.lan.extraRouteMetric} + ipv6.route-metric=${toString cfg.lan.extraRouteMetric} + ''; + + kubeconfigScript = pkgs.writeShellApplication { + name = "ax-fleet-kubeconfig"; + runtimeInputs = [ + pkgs.openssh + pkgs.coreutils + ]; + text = '' + # Fetch the cluster admin kubeconfig from the NAS over the existing ssh + # trust into ~/.kube/config (0600). The content is never printed. + umask 077 + mkdir -p "$HOME/.kube" + tmp=$(mktemp "$HOME/.kube/.config.XXXXXX") + trap 'rm -f "$tmp"' EXIT + ssh "''${AX_FLEET_NAS:-nas}" cat /etc/ax-fleet/admin.kubeconfig > "$tmp" + [ -s "$tmp" ] || { echo "ax-fleet-kubeconfig: empty kubeconfig from the NAS" >&2; exit 1; } + mv "$tmp" "$HOME/.kube/config" + trap - EXIT + echo "wrote $HOME/.kube/config" + ''; + }; +in +{ + config = lib.mkMerge [ + (lib.mkIf roleOn { + networking.firewall.extraCommands = guardStart; + networking.firewall.extraStopCommands = guardStop; + # A fixed uid for the proxy, so the owner match can name it (DynamicUser + # allocates from a range any other DynamicUser unit shares). + users.users.ax-server-proxy = { + isSystemUser = true; + group = "ax-server-proxy"; + }; + users.groups.ax-server-proxy = { }; + # The teardown leaves a guard the generation declares (see pkgs/ax-fleet-teardown). + environment.etc."ax-fleet/guard-declared".text = "harness\n"; + assertions = [ + { + assertion = builtins.all (u: config.users.users ? ${u}) cfg.apiUsers; + message = "modules/ax-fleet/harness.nix: every myAxFleet.apiUsers entry must be a declared user (iptables resolves the name when the firewall starts)."; + } + { + assertion = !(builtins.elem lan cfg.lan.extraInterfaces); + message = "modules/ax-fleet/harness.nix: myAxFleet.lan.extraInterfaces must not repeat lan.interface."; + } + ]; + }) + (lib.mkIf on { + myAxFleet.kubelet = { + # Memory: Tom's seats, Chrome and a coordinator Halogen feel pressure + # after the sandboxes are evicted, never before. CPU is the slices below: + # system-reserved only shrinks kubepods.slice's weight, it protects + # nothing. + systemReserved = lib.mkDefault "cpu=8,memory=32Gi"; + kubeReserved = lib.mkDefault "cpu=1,memory=2Gi"; + evictionHard = lib.mkDefault "memory.available<8Gi"; + }; + + # ── CPU: the desk outweighs the sandboxes (fix round 2) ── + # kubelet gives kubepods.slice cpu.weight = 1 + ((allocatable_mcpu * 1024 + # / 1000 - 2) * 9999) / 262142: MEASURED 274 for 7 allocatable CPUs in the + # review VM, INFERRED 899 for the desk's 32 - 8 - 1 = 23. user.slice and + # system.slice are 100 on the live box (MEASURED), so under contention the + # gVisor workers (no CPU limit, substrate.nix) would take about 90 % of the + # CPU from niri, herdr, the seats and Chrome. Weights are work-conserving: + # idle desk CPU still goes to the sandboxes. Both slices get the same + # weight, so their ratio to each other is unchanged. switch-to-configuration + # never restarts a slice; daemon-reload applies the property. + systemd.slices.user.sliceConfig.CPUWeight = lib.mkDefault cfg.kubelet.deskCpuWeight; + systemd.slices.system.sliceConfig.CPUWeight = lib.mkDefault cfg.kubelet.deskCpuWeight; + + services.k3s = { + role = "agent"; + # The IP, not the name `nas`. + serverAddr = "https://${cfg.serverAddress}:6443"; + nodeTaint = [ cfg.harnessTaint ]; + # Registered WITH the version label: ate-setup only labels nodes that + # exist when it runs (MEASURED version.go:113-131), and the coordinator + # joins after the NAS. Without it atelet (version-keyed) never lands. + nodeLabel = [ + "ax.mecattaf.dev/role=harness" + "ate.dev/substrate-version=${cfg.substrateVersion}" + ]; + }; + + # k3s turns ip_forward on at start anyway; the guard chain is what keeps + # it safe. IPv6 forwarding stays 0 (the wifi leg keeps accepting RAs). + # flannel VXLAN from the NAS only; no TCP port is opened. With the switch + # on only: an accept, unlike the guards above. + networking.firewall.extraCommands = '' + ${ipt} -A nixos-fw -i ${lan} -s ${cfg.serverAddress} -p udp --dport 8472 -j nixos-fw-accept + ''; + + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + # Proxy ARP on the pod interfaces only (fix round 3). Round 2 set + # `conf.default`, but every NIC created after systemd-sysctl copies + # `default`, and the desk's NICs are renamed after it runs (MEASURED boot + # journal: sysctl 6.356 s, enp191s0 6.497 s, wlp192s0 7.493 s), so + # wlp192s0 would have answered ARP for the whole LAN after a reboot. + # systemd's udev rule (99-systemd.rules) runs systemd-sysctl for each new + # interface, which applies these by name and glob as each one appears. + # `flannel/1` is sysctl.d's spelling of flannel.1. + boot.kernel.sysctl."net.ipv4.conf.cni0.proxy_arp" = 1; + boot.kernel.sysctl."net.ipv4.conf.flannel/1.proxy_arp" = 1; + boot.kernel.sysctl."net.ipv4.conf.veth*.proxy_arp" = 1; + + # ── NetworkManager: a drop-in and a config reload, never a restart ── + environment.etc."NetworkManager/conf.d/90-ax-fleet.conf".text = nmDropIn; + systemd.services.ax-fleet-nm-unmanaged = { + description = "ax-fleet: make NetworkManager re-read its conf.d (k3s interfaces unmanaged)"; + wantedBy = [ "multi-user.target" ]; + after = [ "NetworkManager.service" ]; + before = [ "k3s.service" ]; + restartTriggers = [ nmDropIn ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # Tolerated: with NetworkManager down there is nothing to reload. + ExecStart = "-${pkgs.networkmanager}/bin/nmcli general reload conf"; + }; + }; + systemd.services.k3s.wants = [ "ax-fleet-nm-unmanaged.service" ]; + + # ── ax-server on ${cfg.apiListen}, through kube-proxy's OUTPUT rules ── + # The ClusterIP is never a NodePort: ax's API has no authentication + # (upstream #376). Loopback only, and only root, apiUsers and this proxy + # may connect (apiRules). Not 127.0.0.1:8080 (fix round 3): that is + # ax-conwip's default and ax-mockstack's, which must reach nothing live. + systemd.sockets.ax-server-proxy = { + description = "ax-fleet: ax-server on ${cfg.apiListen}"; + wantedBy = [ "sockets.target" ]; + listenStreams = [ cfg.apiListen ]; + }; + systemd.services.ax-server-proxy = { + description = "ax-fleet: proxy ${cfg.apiListen} to the ax-server ClusterIP"; + requires = [ "ax-server-proxy.socket" ]; + after = [ "ax-server-proxy.socket" ]; + serviceConfig = { + ExecStart = "${config.systemd.package}/lib/systemd/systemd-socket-proxyd ${cfg.axServerClusterIP}:8080"; + User = "ax-server-proxy"; + Group = "ax-server-proxy"; + PrivateTmp = true; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + }; + }; + + environment.systemPackages = [ kubeconfigScript ]; + }) + ]; +} diff --git a/modules/ax-fleet/inference.nix b/modules/ax-fleet/inference.nix new file mode 100644 index 000000000..fd44083d7 --- /dev/null +++ b/modules/ax-fleet/inference.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + ... +}: +# The inference node: the worker. "Halogen inference mainly on worker" (Tom, +# 2026-09-23). Halogen stays the worker's host service; sandboxes reach it at +# halogenEndpoint through Substrate's egress gateway on the NAS (SNAT to the +# NAS's LAN address). Nothing from this PR runs here at runtime and the worker +# is not switched in this motion: this file is one evaluation assertion, that +# Halogen's port stays open on the LAN leg. +let + cfg = config.myAxFleet; + on = cfg.enable && cfg.role == "inference"; + port = lib.toInt (lib.last (lib.splitString ":" cfg.halogenEndpoint)); + lanPorts = config.networking.firewall.interfaces.${cfg.lan.interface}.allowedTCPPorts or [ ]; +in +{ + config = lib.mkIf on { + assertions = [ + { + assertion = builtins.elem port lanPorts; + message = "modules/ax-fleet/inference.nix: Halogen's port ${toString port} must stay open on ${cfg.lan.interface}; ax sandboxes reach ${cfg.halogenEndpoint} through the egress gateway on the NAS."; + } + { + assertion = !config.services.k3s.enable; + message = "modules/ax-fleet/inference.nix: the worker is not a k3s node in this motion."; + } + ]; + }; +} diff --git a/modules/ax-fleet/interface.nix b/modules/ax-fleet/interface.nix new file mode 100644 index 000000000..fb2f1add9 --- /dev/null +++ b/modules/ax-fleet/interface.nix @@ -0,0 +1,291 @@ +{ + lib, + inputs, + ... +}: +# ax on the fleet: the options, and nothing else. +# +# Design: ~/today/evals-2026-09-23/ax-fleet/DESIGN.md section 5. Tom's ruling +# (2026-09-23, verbatim): "this is a dotfiles task to do on my nixos fleet. the +# decisions there were already made: hypervisor on NAS, agent harnesses on +# coordinator, halogen inference mainly on worker (can also run on coordinator +# if we need redundancy or a second parallel halogen task)." +# +# `enable` is THE kill switch. Its default is false, so a host that imports +# this module and says nothing renders nothing from it. The substrate and ax +# tracks write only the three extension points at the bottom (`manifests`, +# `registrySeed`, `bootstrap`); everything above them is read by the cluster +# modules next to this file. +let + inherit (lib) mkOption mkEnableOption types; +in +{ + options.myAxFleet = { + enable = mkEnableOption "this host's part of ax on the fleet (k3s, Substrate, ax). THE kill switch"; + + role = mkOption { + type = types.enum [ + "control" + "harness" + "inference" + ]; + description = '' + control = the NAS (k3s server, Substrate and ax control planes, the + registry). harness = the coordinator (k3s agent, gVisor sandboxes). + inference = the worker (Halogen as a host service; nothing from this + PR at runtime). Asserted against the hostname in ./default.nix. + ''; + }; + + lan = { + interface = mkOption { + type = types.str; + example = "enp1s0"; + description = "The LAN leg: enp1s0 (nas), wlp192s0 (coordinator), enp191s0 (worker). Tests use eth1."; + }; + address = mkOption { + type = types.str; + example = "10.42.0.1"; + description = "This host's address on the LAN leg."; + }; + cidr = mkOption { + type = types.str; + default = "10.42.0.0/24"; + }; + extraInterfaces = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "enp191s0" ]; + description = '' + Other NICs that can reach the house LAN (fix round 3): the + coordinator's wired port enp191s0 has an autoconnecting DHCP profile. + On the harness role the guard chain treats them as LAN legs, and a + NetworkManager drop-in gives them `extraRouteMetric`, so the LAN + routes stay on `interface` while it is up. Tests use [ "eth3" ]. + ''; + }; + extraRouteMetric = mkOption { + type = types.int; + default = 700; + description = "Route metric for `extraInterfaces` (NetworkManager's wifi default is 600, ethernet 100)."; + }; + }; + + apiListen = mkOption { + type = types.str; + default = "127.0.0.1:8099"; + description = '' + Loopback address of ax-server-proxy.socket on the harness (fix round 3: + not 127.0.0.1:8080, which ax-conwip's default and ax-mockstack own). + AX_SERVER points here. + ''; + }; + apiUsers = mkOption { + type = types.listOf types.str; + default = [ "tom" ]; + description = '' + Local users (besides root) that may open connections to `apiListen` + and to the cluster ranges from the harness host (fix round 3). The ax + API has no authentication (upstream #376); everyone else is refused by + an owner match in OUTPUT. + ''; + }; + + guardInterfaces = mkOption { + type = types.listOf types.str; + default = [ "tailscale0" ]; + description = "Interfaces the coordinator guard chain isolates from pods and from the LAN leg (the tailnet). Tests use [ \"eth2\" ]."; + }; + + serverAddress = mkOption { + type = types.str; + default = "10.42.0.1"; + description = "The k3s server the harness agent dials: the IP, never the name."; + }; + harnessAddresses = mkOption { + type = types.listOf types.str; + default = [ "10.42.0.2" ]; + description = "LAN addresses of harness nodes: the only sources the NAS admits to 6443, the registry and VXLAN."; + }; + podCidr = mkOption { + type = types.str; + default = "10.200.0.0/16"; + }; + serviceCidr = mkOption { + type = types.str; + default = "10.201.0.0/16"; + }; + clusterDns = mkOption { + type = types.str; + default = "10.201.0.10"; + }; + axServerClusterIP = mkOption { + type = types.str; + default = "10.201.0.80"; + }; + nodePortRange = mkOption { + type = types.str; + default = "30000-30999"; + description = "Kept below 32400 (Plex on the NAS, a listener on the coordinator)."; + }; + + k3sPackage = mkOption { + type = types.package; + default = inputs.nixpkgs.legacyPackages.x86_64-linux.k3s_1_36; + defaultText = lib.literalExpression "inputs.nixpkgs.legacyPackages.x86_64-linux.k3s_1_36"; + description = "One k3s derivation for every node: the one the 2026-09-23 probe ran (1.36.2+k3s1 from the dotfiles nixpkgs pin, not stable's)."; + }; + + k3sTokenFile = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + null (the default) means the agenix secret secrets/k3s-token.age, + declared by ./k3s.nix. Tests set a path to a plain file instead. + ''; + }; + + k3sAgentTokenFile = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + The agent join credential (server --agent-token-file, agent + --token-file). null (the default) means the agenix secret + secrets/k3s-agent-token.age, declared by ./k3s.nix. Tests set a path + to a plain file instead. + ''; + }; + + stateRoot = mkOption { + type = types.str; + default = "/mnt/fast"; + description = "The NAS's fast tier. k3s hot state (/var/lib/rancher, /var/lib/kubelet, /var/log/pods) is bind-mounted from /k3s."; + }; + localPathRoot = mkOption { + type = types.str; + default = "/mnt/nas/services/ax-fleet/local-path"; + description = "Every PersistentVolume (k3s local-path), on the data pool."; + }; + registryRoot = mkOption { + type = types.str; + default = "/mnt/nas/services/ax-fleet/registry"; + }; + registry = mkOption { + type = types.str; + default = "10.42.0.1:5000"; + description = "The NAS registry, plain HTTP, LAN only, scoped to the coordinator by nftables."; + }; + + substrateVersion = mkOption { + type = types.str; + default = "d277088b"; + description = "One string: the harness node label value, the component image tag and ate-setup's VERSION."; + }; + harnessTaint = mkOption { + type = types.str; + default = "ate.dev/sandboxClass=gvisor:NoSchedule"; + description = "Upstream Substrate's own taint key (atelet tolerates it)."; + }; + + kubelet = { + systemReserved = mkOption { + type = types.nullOr types.str; + default = null; + description = "kubelet --system-reserved. Defaults per role in control.nix / harness.nix; tests shrink it."; + }; + kubeReserved = mkOption { + type = types.nullOr types.str; + default = null; + }; + evictionHard = mkOption { + type = types.nullOr types.str; + default = null; + }; + deskCpuWeight = mkOption { + type = types.ints.between 1 10000; + default = 10000; + description = '' + Harness role: cpu.weight for user.slice and system.slice, against + kubepods.slice's kubelet-computed weight (INFERRED 899 on the desk). + The desk wins CPU contention; idle CPU still goes to the sandboxes. + ''; + }; + keepHostKernelTunables = mkOption { + type = types.bool; + default = true; + description = '' + kubelet sets kernel.panic=10, kernel.panic_on_oops=1 and + vm.overcommit_memory=1 when it starts (REPORTED by the VM receipt). + On the desk that turns any kernel oops into a reboot 10 s later, + dropping herdr and every seat. true: ax-fleet-kernel-tunables puts + the three keys back to the values recorded before k3s first ran, + after every kubelet start. false: kubelet's values stay. Tom's + ruling is pending (Waiting on you, 2026-09-23); true is the default + because it changes nothing for Tom at switch. + ''; + }; + }; + + workerPool = { + replicas = mkOption { + type = types.ints.positive; + default = 2; + }; + memoryLimit = mkOption { + type = types.str; + default = "16Gi"; + }; + unreachableTolerationSeconds = mkOption { + type = types.ints.unsigned; + default = 3600; + description = "A configuration value, not an estimate: how long worker pods tolerate an unreachable or not-ready coordinator."; + }; + }; + + halogenEndpoint = mkOption { + type = types.str; + default = "10.42.0.5:8731"; + }; + + # ── Extension points. The substrate and ax tracks write ONLY these. ── + manifests = mkOption { + type = types.attrsOf ( + types.submodule { + options.source = mkOption { + type = types.path; + description = "A YAML file; linked into k3s's auto-deploy directory on the NAS as .yaml."; + }; + } + ); + default = { }; + description = "Our own ax-system objects, auto-deployed by k3s on the NAS. Substrate itself is installed by ate-setup (bootstrap), not here."; + }; + + registrySeed = mkOption { + type = types.attrsOf ( + types.submodule { + options = { + oci = mkOption { + type = types.package; + description = "An OCI image layout at the output root (oci-layout, index.json, blobs/) plus a `digest` file holding the manifest digest (sha256:...)."; + }; + repo = mkOption { type = types.str; }; + tag = mkOption { type = types.str; }; + }; + } + ); + default = { }; + description = "Images the NAS copies into its own registry (skopeo --preserve-digests) before the bootstrap runs."; + }; + + bootstrap = mkOption { + type = types.attrsOf types.lines; + default = { }; + description = '' + "NN-name" -> idempotent bash, run by ax-fleet-bootstrap.service on the + NAS in name order, with KUBECONFIG set to the k3s admin kubeconfig. + The cluster track owns 10-api and 60-kubeconfig. + ''; + }; + }; +} diff --git a/modules/ax-fleet/k3s.nix b/modules/ax-fleet/k3s.nix new file mode 100644 index 000000000..a2f37b2cd --- /dev/null +++ b/modules/ax-fleet/k3s.nix @@ -0,0 +1,329 @@ +{ + config, + lib, + pkgs, + options, + ... +}: +# k3s, common to the two cluster nodes (control = nas, harness = coordinator). +# DESIGN.md sections 6.2, 6.5 and 7. What the 2026-09-23 probe MEASURED +# working on k3s 1.36.2 (probe-build/vm/flake.nix) is the core: k3s_1_36 from +# the dotfiles nixpkgs pin, the three feature gates on every component, the +# certificates.k8s.io/v1beta1 runtime-config, overlay and br_netfilter, the +# inotify raises, a registry mirror in registries.yaml. +# +# What is deliberately NOT here (DESIGN.md D2, D4, 6.2): +# - Cilium. flannel VXLAN plus kube-proxy (iptables), bound to the LAN leg +# only. No BPF socket-LB next to tailscale0. +# - a containerd template, runsc on the unit PATH, a RuntimeClass. Substrate +# runs its own runsc inside the worker pods and uses no RuntimeClass +# (MEASURED, read-substrate.md 2); containerd keeps k3s's generated config. +# - --data-dir. The NixOS module links manifests and images into fixed +# paths under /var/lib/rancher/k3s, so state moves by bind mount instead +# (./control.nix). +let + cfg = config.myAxFleet; + clusterRole = options.myAxFleet.role.isDefined && (cfg.role == "control" || cfg.role == "harness"); + cluster = cfg.enable && clusterRole; + + teardown = pkgs.callPackage ../../pkgs/ax-fleet-teardown { k3s = cfg.k3sPackage; }; + + # The feature gates Substrate's podcertcontroller needs, on all three + # components (A5a and the probe MEASURED that kubelet accepts all three). + gates = "ClusterTrustBundle=true,ClusterTrustBundleProjection=true,PodCertificateRequest=true"; + + kubeletArgs = [ + "feature-gates=${gates}" + ] + ++ lib.optional (cfg.kubelet.systemReserved != null) "system-reserved=${cfg.kubelet.systemReserved}" + ++ lib.optional (cfg.kubelet.kubeReserved != null) "kube-reserved=${cfg.kubelet.kubeReserved}" + ++ lib.optional (cfg.kubelet.evictionHard != null) "eviction-hard=${cfg.kubelet.evictionHard}"; + + commonFlags = [ + "--resolv-conf=/etc/ax-fleet/resolv.conf" + "--flannel-iface=${cfg.lan.interface}" + # NodePorts (none are declared) could only ever bind the LAN /32, and + # kube-proxy never sets route_localnet. + "--kube-proxy-arg=nodeport-addresses=${cfg.lan.address}/32" + "--kube-proxy-arg=iptables-localhost-nodeports=false" + # kube-proxy leaves the host's conntrack table alone (fix round 2). Its + # defaults set nf_conntrack_tcp_timeout_established to 86400 and + # close_wait to 3600 (MEASURED in the review VM; the live NAS and desk + # read 432000 and 60), which on the house router would drop the SNAT + # entry of any NATed flow idle for a day. 0 = do not touch. + "--kube-proxy-arg=conntrack-tcp-timeout-established=0s" + "--kube-proxy-arg=conntrack-tcp-timeout-close-wait=0s" + "--kube-proxy-arg=conntrack-max-per-core=0" + ] + ++ map (a: "--kubelet-arg=${a}") kubeletArgs; + + # containerd pulls every pod image through the NAS registry; for anything + # the mirror lacks it falls back to the upstream registry, so a missing + # seed degrades rather than breaks on the fleet. The offline VM test is what + # proves that nothing is missing. + registriesYaml = '' + mirrors: + "${cfg.registry}": + endpoint: + - "http://${cfg.registry}" + "docker.io": + endpoint: + - "http://${cfg.registry}" + "registry.k8s.io": + endpoint: + - "http://${cfg.registry}" + ''; + + # Values recorded before k3s first runs on this host, restored by + # ax-fleet-teardown. The kubelet sets the three kernel keys itself + # (INFERRED from upstream kubelet behaviour; the VM test records them). + snapshotKeys = [ + "net.ipv4.ip_forward" + "net.ipv6.conf.all.forwarding" + "net.ipv4.conf.all.proxy_arp" + "net.ipv4.conf.default.proxy_arp" + "kernel.panic" + "kernel.panic_on_oops" + "vm.overcommit_memory" + # kube-proxy's conntrack keys (fix round 2): the args above leave them, + # the snapshot and the teardown make sure. Absent when nf_conntrack is + # not loaded yet (boot), then simply not recorded. + "net.netfilter.nf_conntrack_max" + "net.netfilter.nf_conntrack_tcp_timeout_established" + "net.netfilter.nf_conntrack_tcp_timeout_close_wait" + ]; + + # kubelet's values for the three kernel keys (upstream kubelet + # setupKernelTunables; MEASURED 10 1 1 in the VM tests). + kubeletKernel = { + "kernel.panic" = "10"; + "kernel.panic_on_oops" = "1"; + "vm.overcommit_memory" = "1"; + }; + + # ── the host's own declared value for a snapshot key (fix round 2) ── + # At boot the activation script runs before systemd-sysctl, so /proc holds + # kernel defaults, not the generation's values: the round-2 review MEASURED + # a snapshot of ip_forward = 0 on a NAS that declares 1 (the house router), + # which the teardown would then have applied. So at boot, a key some module + # OUTSIDE modules/ax-fleet declares is recorded from that declaration; the + # keys only this module declares (the harness's ip_forward and proxy_arp) + # and the undeclared ones are read from /proc, where they still hold the + # pre-ax value. null = no non-ax declaration, or conflicting ones. + unwrap = + v: + if builtins.isAttrs v && (v._type or null) == "override" then + unwrap v.content + else if builtins.isAttrs v && (v._type or null) == "if" then + (if v.condition then unwrap v.content else null) + else + v; + sysctlString = + v: + if builtins.isBool v then + (if v then "1" else "0") + else if v == null then + null + else + toString v; + hostDeclared = + k: + let + defs = lib.filter ( + d: !(lib.hasInfix "/modules/ax-fleet/" (toString d.file)) + ) options.boot.kernel.sysctl.definitionsWithLocations; + vals = lib.unique ( + lib.filter (v: v != null) ( + map (d: sysctlString (unwrap (d.value.${k} or null))) ( + lib.filter (d: builtins.isAttrs d.value) defs + ) + ) + ); + in + if builtins.length vals == 1 then builtins.head vals else null; + + serverToken = + if cfg.k3sTokenFile != null then cfg.k3sTokenFile else config.age.secrets.k3s-token.path; + agentToken = + if cfg.k3sAgentTokenFile != null then + cfg.k3sAgentTokenFile + else + config.age.secrets.k3s-agent-token.path; +in +{ + config = lib.mkMerge [ + # The kill switch's second half stays on the host whatever the switch says + # (fix round 2): the documented rollback is `enable = false`, switch, then + # `sudo ax-fleet-teardown` ON the host, and the NAS holds no dotfiles + # checkout (MEASURED). Inert until run by hand. `KillMode=process` means + # pods and shims outlive k3s, so the teardown is a real step. + (lib.mkIf clusterRole { environment.systemPackages = [ teardown ]; }) + + (lib.mkIf cluster ( + lib.mkMerge [ + { + services.k3s = { + enable = true; + package = cfg.k3sPackage; + # Two credentials (fix round 2). The server token is the NAS's + # alone; with no agent token k3s gives agents the server password + # (MEASURED deps.go getNodePass), i.e. the k3s:server role on + # /v1-k3s/token, /cacerts and /encrypt/config. The coordinator joins + # with the agent token only. + tokenFile = if cfg.role == "control" then serverToken else agentToken; + agentTokenFile = if cfg.role == "control" then agentToken else null; + nodeIP = cfg.lan.address; + # k3s core images (pause, coredns, local-path and its helper) come + # from the pinned airgap tarball, never from the network. + images = [ cfg.k3sPackage.airgap-images ]; + extraFlags = commonFlags; + }; + + environment.etc."rancher/k3s/registries.yaml".text = registriesYaml; + # The kubelet's (and so CoreDNS's) upstream resolver: AdGuard on the + # NAS, never the coordinator's systemd-resolved stub. + environment.etc."ax-fleet/resolv.conf".text = "nameserver ${cfg.serverAddress}\n"; + + boot.kernelModules = [ + "overlay" + "br_netfilter" + ]; + # Both hosts define 512 / 524288 today (modules/common.nix and the + # nixpkgs default); a plain definition would be an evaluation + # conflict. mkOverride 99 raises them to the probe recipe's values. + boot.kernel.sysctl."fs.inotify.max_user_instances" = lib.mkOverride 99 8192; + boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkOverride 99 1048576; + + systemd.services.k3s = { + # Re-create the manifest and image links AFTER the bind mounts are + # up, whatever order a live switch ran tmpfiles and mounts in. + serviceConfig.ExecStartPre = [ + "${config.systemd.package}/bin/systemd-tmpfiles --create --prefix=/var/lib/rancher/k3s" + # --node-ip and --flannel-iface name the LAN address, which + # NetworkManager adds after network.target at boot (MEASURED on the + # NAS). Bounded wait, then start anyway (k3s's Restart covers it). + "${pkgs.writeShellScript "ax-fleet-k3s-wait-lan-addr" '' + for _ in $(${pkgs.coreutils}/bin/seq 30); do + ${pkgs.iproute2}/bin/ip -4 addr show dev ${cfg.lan.interface} 2>/dev/null \ + | ${pkgs.gnugrep}/bin/grep -qF 'inet ${cfg.lan.address}/' && exit 0 + ${pkgs.coreutils}/bin/sleep 1 + done + echo "ax-fleet: ${cfg.lan.address} not on ${cfg.lan.interface} after 30 s; starting anyway" >&2 + exit 0 + ''}" + ]; + }; + + # ── kubelet's kernel tunables, put back (myAxFleet.kubelet.keepHostKernelTunables) ── + # kubelet sets these whenever its container manager starts, which is + # NOT tied to a k3s restart: an agent whose server is unreachable is + # `active` and runs no kubelet, and sets 10 1 1 whenever the server + # answers, with NRestarts 0 (MEASURED by the round-2 review). So a + # watcher, bound to k3s, not a one-shot after it (the round-1 unit gave + # up after 900 s). Per key: only a value equal to kubelet's that differs + # from the snapshot is put back, so a deliberate host change to any + # other value is left alone. + systemd.services.ax-fleet-kernel-tunables = lib.mkIf cfg.kubelet.keepHostKernelTunables { + description = "ax-fleet: keep the host's kernel.panic, kernel.panic_on_oops, vm.overcommit_memory against kubelet"; + wantedBy = [ "k3s.service" ]; + after = [ "k3s.service" ]; + partOf = [ "k3s.service" ]; + path = [ + pkgs.procps + pkgs.coreutils + pkgs.gnugrep + ]; + serviceConfig = { + Type = "simple"; + Restart = "always"; + RestartSec = 5; + }; + script = '' + snap=/var/lib/ax-fleet/sysctl-before.conf + [ -s "$snap" ] || { echo "no $snap; leaving kernel tunables as they are"; exec sleep infinity; } + while :; do + ${lib.concatStrings ( + lib.mapAttrsToList (k: kv: '' + want=$(grep -E '^${k} = ' "$snap" | cut -d' ' -f3) + cur=$(sysctl -n ${k}) + if [ -n "$want" ] && [ "$cur" = ${kv} ] && [ "$cur" != "$want" ]; then + sysctl -w "${k}=$want" + fi + '') kubeletKernel + )} + sleep 5 + done + ''; + }; + + # ── sysctl snapshot, taken BEFORE this generation's sysctls apply ── + # An activation snippet, not a unit: at a live switch the activation + # script runs before systemd-sysctl is restarted with the new values; + # a unit ordered before k3s would record the ip_forward=1 this very + # generation just set on the harness. switch-to-configuration exports + # NIXOS_ACTION (switch, test) to the activation; the boot activation + # (initrd-nixos-activation on the systemd-initrd hosts, stage 2 + # otherwise) does not, and there /proc holds kernel defaults, so + # declared keys come from hostDeclared (fix round 2). /run/systemd/system + # is no signal: it exists in the systemd initrd. Written once; later + # generations never overwrite it. + system.activationScripts.ax-fleet-sysctl-snapshot = { + text = '' + if [ ! -e /var/lib/ax-fleet/sysctl-before.conf ]; then + mkdir -p /var/lib/ax-fleet + { + ${lib.concatMapStringsSep "\n" ( + k: + let + proc = "/proc/sys/${lib.replaceStrings [ "." ] [ "/" ] k}"; + d = hostDeclared k; + in + if d == null then + "v=$(cat ${proc} 2>/dev/null) && echo \"${k} = $v\" || true" + else + '' + if [ -n "''${NIXOS_ACTION:-}" ]; then + v=$(cat ${proc} 2>/dev/null) && echo "${k} = $v" || true + else + echo "${k} = ${d}" + fi'' + ) snapshotKeys} + } > /var/lib/ax-fleet/sysctl-before.conf.tmp + mv /var/lib/ax-fleet/sysctl-before.conf.tmp /var/lib/ax-fleet/sysctl-before.conf + fi + ''; + }; + } + + # The tokens: agenix secrets, never printed. k3s-token.age is the + # server's (recipients editors ++ nasOnly); k3s-agent-token.age is the + # agent credential (editors ++ coordinatorOnly ++ nasOnly). Only + # declared where agenix is imported and no test token is given. + (lib.optionalAttrs (options ? age) { + age.secrets = lib.mkMerge [ + (lib.mkIf (cfg.role == "control" && cfg.k3sTokenFile == null) { + k3s-token = { + file = ../../secrets/k3s-token.age; + mode = "0400"; + }; + }) + (lib.mkIf (cfg.k3sAgentTokenFile == null) { + k3s-agent-token = { + file = ../../secrets/k3s-agent-token.age; + mode = "0400"; + }; + }) + ]; + assertions = [ + { + assertion = + (cfg.k3sAgentTokenFile != null && (cfg.role != "control" || cfg.k3sTokenFile != null)) + || config.mySecrets.enable or false; + message = "myAxFleet needs agenix delivery (mySecrets.enable) for secrets/k3s-token.age and secrets/k3s-agent-token.age on ${config.networking.hostName}."; + } + ]; + }) + ] + )) + ]; +} diff --git a/modules/ax-fleet/substrate.nix b/modules/ax-fleet/substrate.nix new file mode 100644 index 000000000..cfd1cd522 --- /dev/null +++ b/modules/ax-fleet/substrate.nix @@ -0,0 +1,318 @@ +{ + config, + lib, + pkgs, + inputs, + ... +}: +# Track substrate (DESIGN.md sections 8, 9 and 14 S3): upstream Agent Substrate +# d277088b on the fleet's k3s, installed by its own `ate-setup --kind` exactly +# as the 2026-09-23 probe ran it (MEASURED rc 0, probe-build.md section 4), with +# --image-repo/--image-tag pointing at nix-built images in the NAS registry +# instead of ko. +# +# This file writes ONLY the myAxFleet extension points: +# registrySeed the six component images, the eight third-party images +# bootstrap 20-registry-svc, 30-substrate, 40-gvisor-asset, 50-workerpool +# and an assertion that the version is one string everywhere. It renders +# nothing on a host where myAxFleet.enable is false, and the extension points +# are consumed only on the control role (the NAS), so the coordinator and the +# worker see an unchanged closure from this file. +let + cfg = config.myAxFleet; + on = cfg.enable && cfg.role == "control"; + + inherit (pkgs.stdenv.hostPlatform) system; + + # ONE image set for every evaluator (fix round 2), as ax.nix already does: + # built from the flake's own nixpkgs pin, never the host's `pkgs`. The NAS + # evaluates on nixpkgs-stable, and the round-2 review MEASURED that the six + # component images the real NAS would seed (ateapi, atecontroller, atelet, + # atenet, ateom-gvisor, podcertcontroller) had other digests than the ones + # the VM test ran. The Go toolchain comes from the nixpkgs-go input, the + # same derivation pkgs/ax uses. ax-fleet-topology asserts the parity. + fleetPkgs = inputs.nixpkgs.legacyPackages.${system}; + substrate = fleetPkgs.callPackage ../../pkgs/substrate { + go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; + }; + images = fleetPkgs.callPackage ../../pkgs/substrate/images.nix { inherit substrate; }; + + kubectl = "${cfg.k3sPackage}/bin/kubectl"; + jq = "${pkgs.jq}/bin/jq"; + curl = "${pkgs.curl}/bin/curl"; + + ns = "ate-system"; + + # 10.42.0.1:5000 -> host and port, for the kind-registry EndpointSlice. + registryParts = lib.splitString ":" cfg.registry; + registryHost = lib.head registryParts; + registryPort = lib.toInt (lib.last registryParts); + + # "ate.dev/sandboxClass=gvisor:NoSchedule" -> key, value, effect. + taint = + let + m = builtins.match "([^=]+)=([^:]*):(.+)" cfg.harnessTaint; + in + { + key = lib.elemAt m 0; + value = lib.elemAt m 1; + effect = lib.elemAt m 2; + }; + + # ── 20: the Service atelet's --localhost-registry-replacement names ── + # The kind overlay runs atelet with --localhost-registry-replacement= + # kind-registry:5000 (manifests/ate-install/kind/atelet/kustomization.yaml), + # so every localhost:5000/... image atelet pulls itself (the pause image, the + # Task images ax names) resolves to this Service. The probe needed exactly + # this object pointing at its node's registry (MEASURED probe-build.md 4). + registrySvc = pkgs.writeText "ax-fleet-20-registry-svc.yaml" ( + builtins.toJSON { + apiVersion = "v1"; + kind = "List"; + items = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = ns; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "kind-registry"; + namespace = ns; + labels."app.kubernetes.io/managed-by" = "ax-fleet"; + }; + spec.ports = [ + { + name = "registry"; + port = 5000; + targetPort = registryPort; + protocol = "TCP"; + } + ]; + } + { + apiVersion = "discovery.k8s.io/v1"; + kind = "EndpointSlice"; + metadata = { + name = "kind-registry-nas"; + namespace = ns; + labels = { + "kubernetes.io/service-name" = "kind-registry"; + "endpointslice.kubernetes.io/managed-by" = "ax-fleet"; + }; + }; + addressType = "IPv4"; + ports = [ + { + name = "registry"; + port = registryPort; + protocol = "TCP"; + } + ]; + endpoints = [ + { + addresses = [ registryHost ]; + conditions.ready = true; + } + ]; + } + ]; + } + ); + + # The install is re-run only when what it installs changes: the installer, + # the patched manifests and every image are all in these three paths. The + # component binaries themselves are not in the NAS closure, only the images. + stamp = "${substrate.ate-setup}|${substrate.installTree}|${images}"; + + # ── 50: the gVisor WorkerPool, on the harness only ── + # nodeSelector and tolerations go through spec.template (MEASURED + # pkg/api/v1alpha1/workerpool_types.go). Resources: limits.memory caps each + # worker pod's cgroup, which holds the gVisor sandbox (DESIGN.md section 2, + # replacing ax patch P3). Capacity is read from limits only and an unreported + # dimension is unconstrained (cmd/ateapi/internal/scheduling HasRoom), so no + # CPU limit. The requests are set explicitly: with limits alone Kubernetes + # defaults requests to limits, and 2 x 16Gi would be reserved up front. + workerPoolSpec = { + replicas = cfg.workerPool.replicas; + sandboxClass = "gvisor"; + workerImage = "@WORKER_IMAGE@"; + template = { + labels."ax.mecattaf.dev/pool" = "ateom-gvisor"; + nodeSelector = { + "ax.mecattaf.dev/role" = "harness"; + "ate.dev/substrate-version" = cfg.substrateVersion; + }; + tolerations = [ + { + inherit (taint) key value effect; + operator = "Equal"; + } + { + key = "node.kubernetes.io/unreachable"; + operator = "Exists"; + effect = "NoExecute"; + tolerationSeconds = cfg.workerPool.unreachableTolerationSeconds; + } + { + key = "node.kubernetes.io/not-ready"; + operator = "Exists"; + effect = "NoExecute"; + tolerationSeconds = cfg.workerPool.unreachableTolerationSeconds; + } + ]; + resources = { + limits.memory = cfg.workerPool.memoryLimit; + requests = { + cpu = "250m"; + memory = "1Gi"; + }; + }; + }; + }; + workerPoolTemplate = pkgs.writeText "ax-fleet-50-workerpool.json" ( + builtins.toJSON { + apiVersion = "ate.dev/v1alpha1"; + kind = "WorkerPool"; + metadata = { + name = "ateom-gvisor"; + namespace = ns; + labels."app.kubernetes.io/managed-by" = "ax-fleet"; + }; + spec = workerPoolSpec; + } + ); +in +{ + config = lib.mkIf on { + assertions = [ + { + assertion = cfg.substrateVersion == substrate.version; + message = "myAxFleet.substrateVersion (${cfg.substrateVersion}) must equal pkgs/substrate's version (${substrate.version}): it is the node label, the image tag and ate-setup's VERSION at once."; + } + { + assertion = builtins.match "([^=]+)=([^:]*):(.+)" cfg.harnessTaint != null; + message = "myAxFleet.harnessTaint must read key=value:Effect."; + } + { + assertion = builtins.length registryParts == 2; + message = "myAxFleet.registry must read host:port."; + } + ]; + + # Components under substrate/:; third-party images under + # their upstream repositories, which the docker.io and registry.k8s.io + # mirrors in registries.yaml resolve to. The pause image under `pause`, + # which atelet reaches as kind-registry:5000/pause. + myAxFleet.registrySeed = images.seed; + + myAxFleet.bootstrap = { + "20-registry-svc" = '' + # 20-registry-svc: Namespace ${ns} and the kind-registry Service -> + # ${cfg.registry}, before ate-setup starts atelet. + ${kubectl} apply -f ${registrySvc} + ''; + + "25-rustfs-secret" = '' + # 25-rustfs-secret: the RustFS credential RustFS, ate-api and atelet + # read through secretKeyRef (pkgs/substrate patch 0003), in place of + # the kind overlay's literal default published upstream. Generated + # once on this host into /var/lib/ax-fleet/rustfs.env (0600 root), + # applied before ate-setup starts those pods. Never printed. + f=/var/lib/ax-fleet/rustfs.env + if [ ! -s "$f" ]; then + ( + umask 077 + tmp=$(mktemp /var/lib/ax-fleet/.rustfs.env.XXXXXX) + rnd() { head -c "$1" /dev/urandom | od -An -tx1 | tr -d ' \n'; } + printf 'access-key=ax%s\nsecret-key=%s\n' "$(rnd 10)" "$(rnd 32)" > "$tmp" + mv "$tmp" "$f" + ) + echo "generated $f" + fi + ${kubectl} -n ${ns} create secret generic ax-fleet-rustfs --from-env-file="$f" \ + --dry-run=client -o yaml | ${kubectl} apply -f - >/dev/null + echo "secret ${ns}/ax-fleet-rustfs applied" + ''; + + "30-substrate" = '' + # 30-substrate: upstream's own installer, once per (installer, images) + # pair. The stamp lives in the cluster, so a wiped cluster re-installs. + want='${stamp}' + have=$(${kubectl} -n kube-system get configmap ax-fleet-substrate \ + -o jsonpath='{.data.stamp}' 2>/dev/null || true) + if [ "$have" = "$want" ]; then + echo "substrate ${substrate.version} already installed from this closure" + else + # ate-setup walks up from its working directory to go.mod and reads + # manifests/ from there; it writes nothing under that root. + cd ${substrate.installTree} + env -u GCE_REGION -u CLUSTER_LOCATION -u NETWORK -u SUBNETWORK \ + -u MEMORYSTORE_INSTANCE -u PROJECT_ID \ + VERSION=${cfg.substrateVersion} BUCKET_NAME=ate-snapshots \ + HOME="''${HOME:-/var/lib/ax-fleet}" \ + ${substrate.ate-setup}/bin/ate-setup --kind --no-dev-env \ + --kubeconfig "$KUBECONFIG" --context default \ + --rollout-timeout 10m \ + --image-repo ${cfg.registry}/substrate \ + --image-tag ${cfg.substrateVersion} \ + deploy ate-system + cd / + ${kubectl} -n kube-system create configmap ax-fleet-substrate \ + --from-literal=stamp="$want" \ + --from-literal=version=${cfg.substrateVersion} \ + --dry-run=client -o yaml | ${kubectl} apply -f - + echo "substrate ${substrate.version} installed" + fi + ''; + + "40-gvisor-asset" = '' + # 40-gvisor-asset: put the pinned runsc tarball where atelet's S3 + # fallback looks for gs://${images.gvisor.bucket}/${images.gvisor.key} + # (same bucket and key; the scheme is ignored). The credential is + # read from the Secret 25-rustfs-secret applied and kept off argv. + ${kubectl} -n ${ns} rollout status deploy/rustfs --timeout=10m + ip=$(${kubectl} -n ${ns} get svc rustfs -o jsonpath='{.spec.clusterIP}') + base="http://$ip:9000" + creds=$(mktemp) + trap 'rm -f "$creds"' EXIT + chmod 600 "$creds" + ${kubectl} -n ${ns} get secret ax-fleet-rustfs -o json | ${jq} -r ' + .data + | "user = \"\(.["access-key"] | @base64d):\(.["secret-key"] | @base64d)\""' > "$creds" + s3() { ${curl} -sS --aws-sigv4 "aws:amz:us-east-1:s3" -K "$creds" "$@"; } + code=$(s3 -o /dev/null -w '%{http_code}' -I "$base/${images.gvisor.bucket}") + if [ "$code" != 200 ]; then + s3 -f -X PUT "$base/${images.gvisor.bucket}" -o /dev/null + echo "created bucket ${images.gvisor.bucket}" + fi + obj="$base/${images.gvisor.bucket}/${images.gvisor.key}" + code=$(s3 -o /dev/null -w '%{http_code}' -I "$obj") + if [ "$code" != 200 ]; then + s3 -f -T ${images.gvisor} \ + -H "x-amz-content-sha256: ${images.gvisor.sha256}" \ + -H "Content-Type: application/zstd" "$obj" -o /dev/null + echo "uploaded ${images.gvisor.key}" + fi + got=$(s3 -f "$obj" | sha256sum | cut -d' ' -f1) + if [ "$got" != "${images.gvisor.sha256}" ]; then + echo "gvisor asset sha256 mismatch in RustFS: $got" >&2 + exit 1 + fi + echo "gvisor asset verified (sha256 ${images.gvisor.sha256})" + ''; + + "50-workerpool" = '' + # 50-workerpool: WorkerPool ateom-gvisor on the harness node(s), by + # digest (the CRD's CEL rule wants an @). The digest is read from the + # store at run time, so evaluation needs no IFD. + digest=$(tr -d '[:space:]' < ${images.components.ateom-gvisor}/digest) + ref="${cfg.registry}/substrate/ateom-gvisor:${cfg.substrateVersion}@$digest" + sed "s|@WORKER_IMAGE@|$ref|" ${workerPoolTemplate} | ${kubectl} apply -f - + echo "workerpool ateom-gvisor: ${toString cfg.workerPool.replicas} replica(s), $ref" + ''; + }; + }; +} diff --git a/modules/halogen.nix b/modules/halogen.nix index 5e6c06cb5..04d91a680 100644 --- a/modules/halogen.nix +++ b/modules/halogen.nix @@ -199,7 +199,9 @@ let done [ -n "$unit" ] || exit 0 systemctl start "$unit" - systemctl --no-pager status "$unit" | head -5 + # Not `status | head`: head closing the pipe SIGPIPEs systemctl, and under + # writeShellApplication's pipefail every successful switch exited 141. + systemctl show "$unit" --property=Id,ActiveState,SubState,ActiveEnterTimestamp ''; }; diff --git a/overlays/default.nix b/overlays/default.nix index eec5f8be1..426f1a853 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -1,4 +1,4 @@ -{ torchRocm }: +{ torchRocm, go127 }: final: prev: { qwentts = final.callPackage ../pkgs/qwentts.nix { }; parakeet-service = final.callPackage ../pkgs/parakeet-service { }; @@ -37,6 +37,15 @@ final: prev: { mactahoe-claude-gtk-theme = final.callPackage ../pkgs/mactahoe-gtk-theme.nix { variant = "claude"; }; mactahoe-icon-theme = final.callPackage ../pkgs/mactahoe-icon-theme.nix { }; + # google/ax — the Kubernetes control plane for agent Tasks (v0.3.0, pinned by + # commit). Not in nixpkgs under any name, in any channel: upstream ships no nix + # packaging, no flake and no published image, so there is nothing to take. + # go_1_27 is threaded in from the nixpkgs-go input rather than resolved from + # this fixpoint because ax's go.mod requires exactly 1.27.1 and neither the + # main pin (1.27rc2) nor nixpkgs-fresh (1.27.0) has it — see that input's + # comment in flake.nix, and the `let` in pkgs/ax/default.nix. + ax = final.callPackage ../pkgs/ax { go_1_27 = go127; }; + # Backlog.md — markdown-native task manager CLI (`backlog`). Not in nixpkgs; # packaged from the upstream release binary (Bun compile). See pkgs/backlog-md.nix. backlog-md = final.callPackage ../pkgs/backlog-md.nix { }; diff --git a/pkgs/ax-agent-image/ax-agent.sh b/pkgs/ax-agent-image/ax-agent.sh new file mode 100644 index 000000000..0959c2707 --- /dev/null +++ b/pkgs/ax-agent-image/ax-agent.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# ax-agent: the command an ax Task runs in the fleet's task image. +# +# ax-agent halogen-smoke one chat completion against Halogen with curl +# ax-agent pi pi against Halogen, result validated against a +# JSON Schema (the probe's adapter-pi.sh, e1) +# ax-agent fetch URL GET URL; exits with curl's code (egress checks) +# ax-agent probe what a sandbox can do without a credential: +# `claude --version` (only in an image that carries +# claude-code; the fleet image does not) and one GET +# of Halogen's /v1/models; records /proc/version, +# which names gVisor's kernel inside a sandbox +# ax-agent exit N exit with code N +# +# Every mode writes its result to $AX_RESULT_PATH (default .ax/result.json +# under the workspace), which P1's runner serves to the controller once the +# command exits. No mode reads or needs a secret: Halogen has no auth, and no +# Claude credential exists in this image (DESIGN.md section 11). +set -uo pipefail + +mode="${1:-}" +[ $# -gt 0 ] && shift +out="${AX_RESULT_PATH:-${AX_CONWIP_RESULT_PATH:-.ax/result.json}}" +mkdir -p "$(dirname "$out")" +halogen="${HALOGEN_URL:-http://10.42.0.5:8731}" +model="${AX_AGENT_MODEL:-${AX_CONWIP_MODEL:-halogen-qwen3.8-flash-next}}" +share="@share@" + +case "$mode" in +halogen-smoke) + body=$(jq -nc --arg m "$model" \ + '{model:$m, max_tokens:32, messages:[{role:"user", content:"Reply with the single word: ok"}]}') + curl -sS --max-time 300 -H 'content-type: application/json' \ + -d "$body" "$halogen/v1/chat/completions" >"$out.raw" + rc=$? + content=$(jq -r '.choices[0].message.content // empty' "$out.raw" 2>/dev/null) + jq -n --arg mode "$mode" --arg model "$model" --arg url "$halogen" --argjson rc "$rc" \ + --arg content "$content" \ + '{mode:$mode, model:$model, halogen:$url, curl_rc:$rc, ok:($rc == 0 and ($content | length) > 0), content:$content}' >"$out" + echo "ax-agent halogen-smoke: curl rc=$rc content_bytes=${#content}" + [ "$rc" -eq 0 ] && [ -n "$content" ] + ;; + +pi) + # pi reads its providers from ~/.pi/agent/models.json (pi README). HOME is + # /workspace/.home, which survives a suspend. + mkdir -p "$HOME/.pi/agent" + sed "s#@HALOGEN_URL@#${halogen}#" "$share/pi-models.json" >"$HOME/.pi/agent/models.json" + prompt="${AX_CONWIP_PROMPT:-What is 6 times 7? Answer with the number.}" + default_schema='{"type":"object","required":["answer"],"properties":{"answer":{"type":"integer"}}}' + schema="${AX_CONWIP_SCHEMA_JSON:-$default_schema}" + sys="Return only one JSON object, no prose and no code fence, that validates against this JSON Schema: ${schema}" + t0=$(date +%s.%N) + raw=$(pi -p --provider halogen --model "$model" --thinking "${AX_CONWIP_EFFORT:-low}" \ + --no-session --no-tools --no-context-files --no-skills --no-extensions --no-prompt-templates --offline \ + --append-system-prompt "$sys" "$prompt" 2>"$out.stderr") + rc=$? + t1=$(date +%s.%N) + printf '%s' "$raw" >"$out.raw" + verdict=$(printf '%s' "$raw" | python3 "$share/validate.py" "$schema") + vrc=$? + jq -n --arg label "${AX_CONWIP_LABEL:-}" --arg model "$model" --arg effort "${AX_CONWIP_EFFORT:-low}" \ + --argjson rc "$rc" --argjson v "$verdict" --arg secs "$(python3 -c "print($t1-$t0)")" --arg cwd "$PWD" \ + --arg err "$(tail -c 2000 "$out.stderr" 2>/dev/null)" \ + '{label:$label, model:$model, effort:$effort, harness:"pi", harness_rc:$rc, + valid:$v.valid, errors:$v.errors, result:$v.value, seconds:($secs|tonumber), cwd:$cwd, + stderr_tail:$err}' >"$out" + echo "ax-agent pi: valid=$(jq .valid "$out") rc=$rc vrc=$vrc" + [ "$rc" -eq 0 ] && [ "$vrc" -eq 0 ] + ;; + +fetch) + url="${1:?usage: ax-agent fetch URL}" + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 20 "$url") + rc=$? + jq -n --arg url "$url" --argjson rc "$rc" --arg code "$code" '{mode:"fetch", url:$url, curl_rc:$rc, http_code:$code}' >"$out" + echo "ax-agent fetch: $url rc=$rc http=$code" + exit "$rc" + ;; + +probe) + kernel=$(cat /proc/version 2>/dev/null) + if command -v claude >/dev/null 2>&1; then + cv=$(claude --version 2>&1) + crc=$? + else + cv="claude: not in this image" + crc=127 + fi + code=$(curl -sS -o "$out.models" -w '%{http_code}' --max-time 60 "$halogen/v1/models") + hrc=$? + seen=$(jq -r '.data[0].id // empty' "$out.models" 2>/dev/null) + jq -n --arg cv "$cv" --argjson crc "$crc" --arg url "$halogen" --argjson hrc "$hrc" --arg code "$code" \ + --arg seen "$seen" --arg kernel "$kernel" \ + '{mode:"probe", claude_version:$cv, claude_rc:$crc, halogen:$url, curl_rc:$hrc, http_code:$code, + model:$seen, proc_version:$kernel, ok:($crc == 0 and $hrc == 0 and $code == "200")}' >"$out" + echo "ax-agent probe: claude_rc=$crc curl_rc=$hrc http=$code" + [ "$crc" -eq 0 ] && [ "$hrc" -eq 0 ] && [ "$code" = 200 ] + ;; + +exit) + n="${1:-0}" + jq -n --argjson n "$n" '{mode:"exit", code:$n}' >"$out" + exit "$n" + ;; + +*) + echo "usage: ax-agent halogen-smoke | pi | fetch URL | probe | exit N" >&2 + exit 64 + ;; +esac diff --git a/pkgs/ax-agent-image/default.nix b/pkgs/ax-agent-image/default.nix new file mode 100644 index 000000000..c93ad2886 --- /dev/null +++ b/pkgs/ax-agent-image/default.nix @@ -0,0 +1,169 @@ +{ + lib, + callPackage, + dockerTools, + runCommand, + writeTextDir, + buildEnv, + bashInteractive, + coreutils, + findutils, + gnugrep, + gnused, + gawk, + curl, + jq, + python3, + cacert, + gnutar, + gzip, + git, + shellcheck, + # The patched ax (sandbox-class + P1): its ax-task-runner is PID 1. + ax, + # pi from llm-agents. Halogen only on day one; no claude-code in this image + # until Tom rules on Claude credentials in sandboxes (DESIGN.md section 11). + pi, + # Test-only variants (tests/ax-fleet): extra store paths linked into /bin and + # a name suffix. The fleet image passes neither: pi only, no claude-code. + extraPaths ? [ ], + variant ? "", +}: +# The ax Task image for the fleet (DESIGN.md section 10.2): `ax-agent`. +# +# Substrate starts /usr/local/bin/ax-task-runner (ax's DefaultGuestCommand) as +# PID 1; the runner sets up /workspace and runs the Task's spec.command, here +# `ax-agent `. Output: an OCI layout plus `digest`, seeded into the NAS +# registry as ax/ax-agent and named by Tasks as +# localhost:5000/ax/ax-agent@sha256: (atelet rewrites localhost:5000). +# +# No secret is baked in: pi-models.json names Halogen, which has no auth, and +# the endpoint is filled at run time from $HALOGEN_URL (default the worker, +# http://10.42.0.5:8731), so one image digest serves every host. +let + ociLayout = callPackage ../ax/oci-layout.nix { }; + + share = runCommand "ax-agent-share" { } '' + mkdir -p $out/share/ax-agent + cp ${./pi-models.json} $out/share/ax-agent/pi-models.json + cp ${./validate.py} $out/share/ax-agent/validate.py + ''; + + ax-agent = runCommand "ax-agent" { nativeBuildInputs = [ shellcheck ]; } '' + mkdir -p $out/bin + substitute ${./ax-agent.sh} $out/bin/ax-agent \ + --replace-fail '@share@' '${share}/share/ax-agent' \ + --replace-fail '#!/usr/bin/env bash' '#!${bashInteractive}/bin/bash' + chmod +x $out/bin/ax-agent + shellcheck -S warning $out/bin/ax-agent + ''; + + # The runner at the path Substrate's template names. + # pi with its ELF program headers in the order the ELF spec asks for. pi + # 0.85.1's bun binary lists PT_LOAD out of p_vaddr order; Linux runs it, but + # gVisor's loader refuses it (ENOEXEC, exit 126 inside the sandbox, MEASURED + # in the ax-fleet VM test, INTEGRATE.md). Only table entries move; segments, + # addresses and bytes stay. The name keeps pi's, so the store path length is + # unchanged; only the text wrapper in bin/ is repointed. + piSandbox = runCommand pi.name { nativeBuildInputs = [ python3 ]; } '' + cp -a ${pi} $out + chmod -R u+w $out + find $out -type f -print0 | while IFS= read -r -d "" f; do + if [ "$(head -c 4 "$f" | od -An -c | tr -d ' ')" = '177ELF' ]; then + python3 ${./elf-sort-load.py} "$f" + fi + done + for f in $out/bin/*; do + substituteInPlace "$f" --replace-quiet ${pi} $out + done + chmod -R a-w $out + ''; + + runner = runCommand "ax-task-runner-usr-local" { } '' + mkdir -p $out/usr/local/bin + ln -s ${ax}/bin/ax-task-runner $out/usr/local/bin/ax-task-runner + ln -s ${ax-agent}/bin/ax-agent $out/usr/local/bin/ax-agent + ''; + + etc = [ + (writeTextDir "etc/passwd" '' + root:x:0:0:root:/root:/bin/bash + agent:x:1000:1000:agent:/workspace/.home:/bin/bash + nobody:x:65534:65534:nobody:/var/empty:/bin/false + '') + (writeTextDir "etc/group" '' + root:x:0: + agent:x:1000: + nobody:x:65534: + '') + ]; + + env = buildEnv { + name = "ax-agent-env"; + paths = [ + bashInteractive + coreutils + findutils + gnugrep + gnused + gawk + curl + jq + python3 + cacert + gnutar + gzip + git + piSandbox + ax-agent + ] + ++ extraPaths; + pathsToLink = [ + "/bin" + "/etc/ssl" + "/share/ax-agent" + ]; + }; + + image = dockerTools.buildLayeredImage { + name = "ax/ax-agent${variant}"; + tag = "v${ax.version}-p1"; + contents = [ + env + runner + share + ] + ++ etc; + extraCommands = '' + mkdir -p tmp workspace usr/bin etc/ax-agent/pi + chmod 1777 tmp + ln -s ${coreutils}/bin/env usr/bin/env + cp ${./pi-models.json} etc/ax-agent/pi/models.json + ''; + config = { + Cmd = [ "/usr/local/bin/ax-task-runner" ]; + WorkingDir = "/workspace"; + Env = [ + "PATH=/usr/local/bin:/bin" + "HOME=/workspace/.home" + "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" + "HALOGEN_URL=http://10.42.0.5:8731" + ]; + }; + }; +in +(ociLayout { + name = "ax/ax-agent${variant}"; + tag = "v${ax.version}-p1"; + inherit image; +}).overrideAttrs + (old: { + passthru = old.passthru // { + inherit ax-agent image piSandbox; + }; + meta = { + description = "ax Task image for the fleet: ax-task-runner, pi, and the ax-agent adapter (OCI layout)"; + platforms = [ "x86_64-linux" ]; + license = lib.licenses.asl20; + }; + }) diff --git a/pkgs/ax-agent-image/elf-sort-load.py b/pkgs/ax-agent-image/elf-sort-load.py new file mode 100644 index 000000000..0af001f78 --- /dev/null +++ b/pkgs/ax-agent-image/elf-sort-load.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Reorder an ELF64 program header table so gVisor's loader accepts it. + +The ELF spec says loadable segments appear in ascending p_vaddr order, and +PT_PHDR and PT_INTERP precede every PT_LOAD. Linux tolerates a table that +breaks this; gVisor's loader refuses it with ENOEXEC, which bash reports as +exit 126. pi 0.85.1 from llm-agents ships such a table: its first PT_LOAD maps +vaddr 0x6431000 and a later one 0x1ff000 (MEASURED 2026-09-23, ax-fleet +INTEGRATE.md). The fix moves table entries only: every segment keeps its file +offset, address, size and flags, and the table keeps its place and length. + +Usage: elf-sort-load.py FILE... (files that are not ELF64 LE, or are already +in order, are left alone; prints one line per file it changed) +""" +import struct +import sys + +PT_LOAD, PT_INTERP, PT_PHDR = 1, 3, 6 + + +def fix(path): + with open(path, "r+b") as f: + ident = f.read(64) + if len(ident) < 64 or ident[:4] != b"\x7fELF" or ident[4] != 2 or ident[5] != 1: + return False + (phoff,) = struct.unpack_from(" {order}") + return True + + +if __name__ == "__main__": + for p in sys.argv[1:]: + fix(p) diff --git a/pkgs/ax-agent-image/pi-models.json b/pkgs/ax-agent-image/pi-models.json new file mode 100644 index 000000000..282b8fa89 --- /dev/null +++ b/pkgs/ax-agent-image/pi-models.json @@ -0,0 +1,26 @@ +{ + "providers": { + "halogen": { + "api": "openai-completions", + "apiKey": "none", + "authHeader": false, + "baseUrl": "@HALOGEN_URL@/v1", + "compat": { + "supportsDeveloperRole": false, + "supportsReasoningEffort": false, + "supportsStore": false + }, + "models": [ + { + "id": "halogen-qwen3.8-flash-next", + "name": "Qwen3.8-Flash-Next (Halogen, worker)", + "contextWindow": 262144, + "maxTokens": 32768, + "input": ["text", "image"], + "reasoning": true, + "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 } + } + ] + } + } +} diff --git a/pkgs/ax-agent-image/validate.py b/pkgs/ax-agent-image/validate.py new file mode 100755 index 000000000..7a7fe3d1b --- /dev/null +++ b/pkgs/ax-agent-image/validate.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Minimal JSON Schema subset check (type, required, properties, enum, items, +additionalProperties:false) for the probe. Reads schema from $1 (JSON text) and +candidate text from stdin; extracts the first JSON object; prints a verdict JSON.""" +import json, sys, re +schema = json.loads(sys.argv[1]) +raw = sys.stdin.read() +T = {"object": dict, "array": list, "string": str, "integer": int, "number": (int, float), "boolean": bool, "null": type(None)} +def check(s, v, p="$"): + errs = [] + t = s.get("type") + if t and not isinstance(v, T[t]) or (t == "integer" and isinstance(v, bool)): + return [f"{p}: expected {t}"] + if "enum" in s and v not in s["enum"]: + errs.append(f"{p}: not in enum") + if isinstance(v, dict): + for r in s.get("required", []): + if r not in v: errs.append(f"{p}.{r}: missing") + props = s.get("properties", {}) + for k, sub in props.items(): + if k in v: errs += check(sub, v[k], f"{p}.{k}") + if s.get("additionalProperties") is False: + errs += [f"{p}.{k}: extra" for k in v if k not in props] + if isinstance(v, list) and "items" in s: + for i, x in enumerate(v): errs += check(s["items"], x, f"{p}[{i}]") + return errs +m = re.search(r"\{.*\}", raw, re.S) +if not m: + print(json.dumps({"valid": False, "errors": ["no JSON object in output"], "value": None})); sys.exit(3) +try: + val = json.loads(m.group(0)) +except Exception as e: + print(json.dumps({"valid": False, "errors": [f"parse: {e}"], "value": None})); sys.exit(3) +errs = check(schema, val) +print(json.dumps({"valid": not errs, "errors": errs, "value": val})) +sys.exit(0 if not errs else 4) diff --git a/pkgs/ax-fleet-teardown/default.nix b/pkgs/ax-fleet-teardown/default.nix new file mode 100644 index 000000000..da6380f7e --- /dev/null +++ b/pkgs/ax-fleet-teardown/default.nix @@ -0,0 +1,100 @@ +{ + writeShellApplication, + k3s, + iptables, + nftables, + iproute2, + procps, + coreutils, + gnugrep, + systemd, +}: +# ax-fleet-teardown: the second half of the kill switch (DESIGN.md section 13). +# +# The k3s unit runs with KillMode=process, so pods and containerd shims outlive +# it; switching `myAxFleet.enable = false` alone leaves them, plus cni0, +# flannel.1, the KUBE-/FLANNEL-/CNI- rules and ip_forward=1. This wraps the +# pinned package's own k3s-killall.sh, removes the guard chains only when the +# running generation does not declare them (a pre-ax generation), then +# restores the sysctls recorded before k3s first ran on this host +# (/var/lib/ax-fleet/sysctl-before.conf, written by the activation snippet in +# modules/ax-fleet/k3s.nix). +# +# PATH is pinned (inheritPath = false). The upstream killall's +# remove_interfaces runs `tailscale set --advertise-routes=` whenever a +# `tailscale` binary is on PATH; on the NAS that would withdraw the +# 10.42.0.0/24 subnet route (hosts/nas/headscale.nix). With the caller's PATH +# cut off, `command -v tailscale` fails and the call is skipped; the killall +# wrapper still prefixes its own dependencies. The explicit check below makes +# the teardown refuse to run if tailscale ever becomes reachable anyway. +# +# Left on disk on purpose: /mnt/fast/k3s, the data-pool directories and +# /var/lib/ax-fleet. Deleting them is Tom's call. +writeShellApplication { + name = "ax-fleet-teardown"; + inheritPath = false; + runtimeInputs = [ + k3s + iptables + nftables + iproute2 + procps + coreutils + gnugrep + systemd + ]; + text = '' + if [ "$(id -u)" -ne 0 ]; then + echo "ax-fleet-teardown: run as root (sudo)" >&2 + exit 1 + fi + + if command -v tailscale >/dev/null 2>&1; then + echo "ax-fleet-teardown: tailscale is on PATH; k3s-killall.sh would clear the advertised routes. Refusing." >&2 + exit 1 + fi + + if systemctl is-enabled --quiet k3s.service 2>/dev/null; then + echo "ax-fleet-teardown: note: k3s.service is still part of this generation;" >&2 + echo " k3s-killall.sh stops it now, and it starts again at the next boot or switch." >&2 + fi + + echo "== k3s-killall.sh (${k3s.version})" + ${k3s}/bin/k3s-killall.sh || echo "k3s-killall.sh exited $?; continuing" >&2 + + # Fix round 3: a harness or control generation declares the guards + # whatever `enable` says (the kill switch leaves pods running until this + # script), so they stay; a generation from before ax never had them, and + # then they go. + if [ -e /etc/ax-fleet/guard-declared ]; then + echo "== guards: declared by this generation ($(cat /etc/ax-fleet/guard-declared)); left in place" + else + echo "== guard chains" + while iptables -w -t mangle -D FORWARD -j ax-fleet-guard 2>/dev/null; do :; done + iptables -w -t mangle -F ax-fleet-guard 2>/dev/null || true + iptables -w -t mangle -X ax-fleet-guard 2>/dev/null || true + while iptables -w -D OUTPUT -j ax-fleet-api 2>/dev/null; do :; done + iptables -w -F ax-fleet-api 2>/dev/null || true + iptables -w -X ax-fleet-api 2>/dev/null || true + + echo "== NAS guard table" + nft delete table inet ax-fleet-guard 2>/dev/null || true + fi + + snap=/var/lib/ax-fleet/sysctl-before.conf + if [ -s "$snap" ]; then + echo "== sysctl restore from $snap" + # -e: a conntrack key recorded while nf_conntrack was loaded may be + # absent now; skip it rather than abort the restore. + sysctl -e -p "$snap" + else + echo "ax-fleet-teardown: no $snap; sysctls left as they are" >&2 + fi + + echo "== left behind (should be empty)" + ip -br link show 2>/dev/null | grep -E '^(cni0|flannel\.1|veth)' || true + iptables-save 2>/dev/null | grep -cE 'KUBE-|FLANNEL|CNI-' || true + pgrep -a containerd-shim || true + echo "ax-fleet-teardown: done" + ''; +} diff --git a/pkgs/ax/default.nix b/pkgs/ax/default.nix new file mode 100644 index 000000000..244a1ffea --- /dev/null +++ b/pkgs/ax/default.nix @@ -0,0 +1,132 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, + git, + # The Go toolchain, passed in by overlays/default.nix rather than taken from + # this pkgs fixpoint. See the `go` binding below for why. + go_1_27, +}: +# google/ax — a Kubernetes control plane for agent Tasks, which delegates the +# sandbox to Agent Substrate rather than running workloads itself. Four commands +# ship in cmd/: `ax` (CLI), `ax-controller`, `ax-server`, `ax-task-runner`. +# +# NOT in nixpkgs, in any channel, under any name (checked 2026-09-22): upstream +# tagged v0.3.0 and has no nix packaging of its own, no flake, and no container +# publishing path a fleet could consume. This expression is the whole of it. +# +# Pinned by commit, not by tag, so a retag upstream cannot move what this fleet +# builds. d8ed0fe38bceb7842d3c47817d53d16ccdfcb601 IS tag v0.3.0 as of 2026-09-22. +# +# TWO CARRIED PATCHES in ./patches, applied in order, described at the `patches` +# entry below: sandbox-class.patch (a per-Task sandbox class) and +# p1-completion.patch (a finished command frees its worker). The upstream clone +# stays clean; each patch was extracted from a scratch copy of the fetched +# source, one commit per patch. The vendored Substrate client stays at +# 672533541dbf: no patch touches go.mod or go.sum. +let + # go.mod's first directive is `go 1.27.1` (MEASURED). Go refuses to build a + # module whose `go` line is newer than the running toolchain, and the sandbox + # has no network to fetch one, so the toolchain must be at least 1.27.1: + # this flake's main nixpkgs pin go = 1.26.5, go_1_27 = 1.27rc2 too old + # nixpkgs-fresh (flake.nix:28) go = 1.26.7, go_1_27 = 1.27.0 too old + # so overlays/default.nix resolves go_1_27 from the `nixpkgs-go` input, which + # exists for this one attribute and nothing else. All three versions MEASURED + # 2026-09-23; see that input's comment in flake.nix. + buildGo127Module = buildGoModule.override { go = go_1_27; }; +in +buildGo127Module { + pname = "ax"; + version = "0.3.0"; + + src = fetchFromGitHub { + owner = "google"; + repo = "ax"; + rev = "d8ed0fe38bceb7842d3c47817d53d16ccdfcb601"; # = tag v0.3.0 + hash = "sha256-mGSQ4QsYLdeKDtVMBODCulqQQ0Ze0NjeADPhB6edaYU="; + }; + + # Obtained the ordinary way: build once with lib.fakeHash, read the "got:" + # line off the failure, paste it back. UNCHANGED by both patches, which + # touch no go.mod or go.sum line and so vendors the same module set + # (MEASURED 2026-09-23: the patched build reuses this hash). + vendorHash = "sha256-iC/X6Bg1M7Pn3dT1zWs2YxuPfgl9ZKNEYQsBisIQguY="; + + # sandbox-class.patch adds `string sandbox_class = 11` to TaskSpec, regenerates + # ax.pb.go with the same protoc-gen-go v1.36.11 upstream used, validates the + # value in ValidateTask, and threads it from the reconciler through + # BuildActorTemplate, replacing the SandboxClass_SANDBOX_CLASS_GVISOR hardcode + # at internal/substrate/client.go:273. Empty means gvisor, so every existing + # manifest behaves exactly as before. + # + # The generated Go is part of the patch on purpose: ax bridges YAML through + # protojson with unknown fields REJECTED, so a .proto-only edit would make + # every manifest naming sandboxClass fail strict decode. + # + # It does NOT give ax a workerd sandbox. Agent Substrate's SandboxClass enum + # has exactly three members (UNSPECIFIED, GVISOR, MICROVM; MEASURED 2026-09-23 + # from the vendored ateapipb), so "gvisor" and "microvm" are the only values + # that can reach a real substrate. A workerd class needs an upstream Agent + # Substrate change that does not exist, and ax cannot invent the enum member. + # + # p1-completion.patch (REQUIRED for ax on the fleet). Stock v0.3.0 never learns + # that a Task's command exited: the Task stays Running, its actor keeps its + # worker, and a small WorkerPool is exhausted after a few finished Tasks + # (MEASURED by the 2026-09-23 Substrate probe: the third Task on a 3-worker + # pool failed ResourceExhausted; evals-2026-09-23/substrate/probe-build.md 4). + # The patch: + # - runner: records the command's own exit and serves + # /metadata/v1alpha1/ax/{exit,result,usage} on the metadata port; result + # is the file at AX_RESULT_PATH (default /.ax/result.json), + # capped at 1 MiB (413 above that, never truncated); + # - controller: a terminal guard (Completed, or Failed with Ready reason + # CommandExited, is never resumed again), an exit read after each resume, + # and a --running-resync loop (default 15s) that re-checks Running Tasks, + # because nothing publishes an event when a command exits. On exit it + # writes Completed (0) or Failed (Ready False CommandExited, ExitCode=N), + # TaskStatus.command, usage, stores the result, then SuspendActor frees + # the worker. A CRASHED actor with no exit report becomes Failed + # ActorCrashed instead of being silently recreated; + # - API: TaskStatus.command = 8, UsageStats.tool_calls = 3, rpc + # GetTaskResult (store key task-result::), and + # `ax result task `. ax.pb.go and ax_grpc.pb.go are regenerated + # with protoc-gen-go v1.36.11 and protoc-gen-go-grpc v1.6.2. + # Its tests run in checkPhase below: the exit write-back for 0 and 3, the + # terminal guard (the flipped probe TestProbe_CompletedWriteBackIsOverwritten), + # the resync, a crashed actor, the runner's endpoints, and the floor test: + # four Tasks in a row on a 2-worker pool all Completed, next to the contrast + # that without an exit report the third is refused ResourceExhausted. + patches = [ + ./patches/sandbox-class.patch + ./patches/p1-completion.patch + ]; + + # subPackages left unset so all four commands build, matching upstream's + # `make build-binaries` plus the cross-compiled runner. -s -w mirrors the + # Makefile's ldflags. + ldflags = [ + "-s" + "-w" + ]; + + # Upstream's `make test` is `go test ./...`, and per docs/development.md it + # "Runs everything, including the mock Substrate gRPC server, in-memory store + # validation, and API server tests". It needs no cluster and no network, so it + # is the one cluster-free regression proof this package has. Kept ON. + doCheck = true; + + # MEASURED 2026-09-22: without git on PATH, two internal/workspace tests fail + # with `exit status 127 ... git: command not found` (TestSetupWorkspace_GitSubdir + # at setup_test.go:104, TestSetupWorkspace_GitDepth at setup_test.go:227). They + # shell out to git to build a throwaway repo. This is a packaging fix, not a + # source patch: nothing upstream is wrong. + nativeCheckInputs = [ git ]; + + meta = { + description = "Kubernetes control plane for agent Tasks, delegating sandboxes to Agent Substrate"; + homepage = "https://github.com/google/ax"; + license = lib.licenses.asl20; + platforms = lib.platforms.linux; + mainProgram = "ax"; + }; +} diff --git a/pkgs/ax/images.nix b/pkgs/ax/images.nix new file mode 100644 index 000000000..ff146a41d --- /dev/null +++ b/pkgs/ax/images.nix @@ -0,0 +1,64 @@ +{ + callPackage, + dockerTools, + cacert, + redis, + # The patched ax (pkgs/ax: sandbox-class + P1). Passed in so every host seeds + # and references the same derivation. + ax, +}: +# The ax control-plane images for the fleet (DESIGN.md section 8): ax-server, +# ax-controller and ax-redis, each an OCI layout with a `digest` file, seeded +# into the NAS registry as ax/: and referenced by digest in the +# ax-fleet-40-ax manifest (modules/ax-fleet/ax.nix). +# +# Upstream publishes no image (ko:// references only, deploy/*.yaml), so these +# are built here from the same pkgs/ax the CLI uses. +let + ociLayout = callPackage ./oci-layout.nix { }; + tag = "v${ax.version}-p1"; + + axImage = + cmd: + ociLayout { + name = "ax/${cmd}"; + inherit tag; + image = dockerTools.buildLayeredImage { + name = "ax/${cmd}"; + inherit tag; + contents = [ cacert ]; + config = { + Entrypoint = [ "${ax}/bin/${cmd}" ]; + Env = [ "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" ]; + User = "65532:65532"; + }; + }; + }; +in +{ + inherit tag ociLayout; + + ax-server = axImage "ax-server"; + ax-controller = axImage "ax-controller"; + + # nixpkgs redis, not upstream's redis:7-alpine. Upstream's image patches + # protected mode off; a stock redis-server refuses non-loopback clients when + # it has no password, so the manifest passes --protected-mode no. It is + # reachable only as a ClusterIP (DESIGN.md section 9). + ax-redis = ociLayout { + name = "ax/ax-redis"; + tag = "${redis.version}"; + image = dockerTools.buildLayeredImage { + name = "ax/ax-redis"; + tag = "${redis.version}"; + contents = [ ]; + extraCommands = '' + mkdir -p data + ''; + config = { + Entrypoint = [ "${redis}/bin/redis-server" ]; + WorkingDir = "/data"; + }; + }; + }; +} diff --git a/pkgs/ax/oci-layout.nix b/pkgs/ax/oci-layout.nix new file mode 100644 index 000000000..06ab9b92c --- /dev/null +++ b/pkgs/ax/oci-layout.nix @@ -0,0 +1,36 @@ +{ + lib, + runCommand, + skopeo, + jq, +}: +# Turn a dockerTools image (a docker-archive tarball) into an OCI image layout +# at the output root (oci-layout, index.json, blobs/) plus a `digest` file with +# the manifest digest (sha256:...). That is the shape myAxFleet.registrySeed +# takes: the NAS pushes it with `skopeo copy --preserve-digests`, so the digest +# a manifest or a Task names is the digest the registry serves. +# +# The digest is read at BUILD time, inside this derivation, and consumers read +# it from "${layout}/digest" in their own build steps. No import-from-derivation. +{ + name, + image, + tag ? "latest", +}: +runCommand "${lib.replaceStrings [ "/" ] [ "-" ] name}-oci" + { + nativeBuildInputs = [ + skopeo + jq + ]; + passthru = { inherit image tag; }; + } + '' + export HOME=$TMPDIR + # gzip layers: these images cross the coordinator's wifi leg when pulled. + skopeo --insecure-policy --tmpdir "$TMPDIR" copy --quiet \ + --dest-compress --dest-compress-format gzip \ + docker-archive:${image} oci:$out:${tag} + jq -r '.manifests[0].digest' $out/index.json > $out/digest + grep -Eq '^sha256:[0-9a-f]{64}$' $out/digest + '' diff --git a/pkgs/ax/patches/p1-completion.patch b/pkgs/ax/patches/p1-completion.patch new file mode 100644 index 000000000..9176e8464 --- /dev/null +++ b/pkgs/ax/patches/p1-completion.patch @@ -0,0 +1,3545 @@ +diff --git a/cmd/ax-controller/main.go b/cmd/ax-controller/main.go +index 9d1020e..f882e2d 100644 +--- a/cmd/ax-controller/main.go ++++ b/cmd/ax-controller/main.go +@@ -21,6 +21,7 @@ import ( + "os" + "os/signal" + "syscall" ++ "time" + + "github.com/google/ax/internal/controller" + "github.com/google/ax/internal/store/redis" +@@ -42,6 +43,7 @@ func main() { + redisPassword string + redisGroup string + redisConsumer string ++ runningResync time.Duration + ) + + flag.StringVar(&redisAddr, "redis-addr", "localhost:6379", "Redis server address (e.g. localhost:6379)") +@@ -56,6 +58,7 @@ func main() { + flag.BoolVar(&substratePlaintext, "substrate-plaintext", false, "Use insecure plaintext gRPC connection to Substrate") + flag.StringVar(&defaultTemplate, "template", "default-template", "Default Substrate ActorTemplate name") + flag.StringVar(&defaultTemplateAtespace, "template-atespace", "ax-system", "Default Substrate ActorTemplate atespace") ++ flag.DurationVar(&runningResync, "running-resync", controller.DefaultRunningResync, "How often Running tasks are re-checked for a command exit (0 disables)") + flag.Parse() + + if envRedis := os.Getenv("REDIS_ADDR"); envRedis != "" { +@@ -104,6 +107,7 @@ func main() { + + rStore := redis.NewStore(rClient, redis.Options{}) + worker := controller.NewWorker(rStore, reconciler, redisGroup, redisConsumer) ++ worker.RunningResync = runningResync + if err := worker.Run(ctx); err != nil && err != context.Canceled { + slog.Error("redis worker stopped with error", "error", err) + os.Exit(1) +diff --git a/cmd/ax/main.go b/cmd/ax/main.go +index 7b2922e..fff6379 100644 +--- a/cmd/ax/main.go ++++ b/cmd/ax/main.go +@@ -147,6 +147,8 @@ func main() { + err = runDelete(serverURL, atespace, cleanArgs) + case "ssh": + err = runSSH(serverURL, atespace, kubeContext, cleanArgs) ++ case "result": ++ err = runResult(serverURL, atespace, cleanArgs) + default: + fmt.Fprintf(os.Stderr, "unknown command: %s\n", cmd) + printUsage() +@@ -183,6 +185,7 @@ Available Commands: + ssh [-- cmd] Run a command or shell inside the running task container + suspend task Suspend execution of a task and checkpoint state + resume task Resume execution of a suspended task ++ result task Print the result file a finished task's command wrote + delete task Delete a task + delete gateway Delete a gateway + delete workspace Delete a workspace +@@ -1228,3 +1231,33 @@ func runSSH(serverURL, atespace, kubeContext string, args []string) error { + + return nil + } ++ ++// runResult prints the result content the controller copied from a finished ++// task's sandbox, byte for byte, to stdout. ++func runResult(serverURL, atespace string, args []string) error { ++ name := "" ++ switch { ++ case len(args) == 1: ++ name = args[0] ++ case len(args) >= 2 && (args[0] == "task" || args[0] == "tasks"): ++ name = args[1] ++ default: ++ return fmt.Errorf("usage: ax result task ") ++ } ++ ++ client, conn, err := getAXClient(serverURL) ++ if err != nil { ++ return err ++ } ++ defer conn.Close() ++ ++ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) ++ defer cancel() ++ ++ res, err := client.GetTaskResult(ctx, &v1alpha1.GetTaskResultRequest{Atespace: atespace, Name: name}) ++ if err != nil { ++ return fmt.Errorf("getting task result: %w", err) ++ } ++ _, err = os.Stdout.Write(res.GetContent()) ++ return err ++} +diff --git a/internal/controller/completion_test.go b/internal/controller/completion_test.go +new file mode 100644 +index 0000000..d286b42 +--- /dev/null ++++ b/internal/controller/completion_test.go +@@ -0,0 +1,396 @@ ++// Copyright 2026 Google LLC ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++package controller_test ++ ++import ( ++ "context" ++ "encoding/json" ++ "fmt" ++ "net" ++ "net/http" ++ "net/http/httptest" ++ "sync" ++ "sync/atomic" ++ "testing" ++ "time" ++ ++ "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ++ "github.com/google/ax/internal/controller" ++ "github.com/google/ax/internal/metadata" ++ "github.com/google/ax/internal/store" ++ "github.com/google/ax/internal/store/memory" ++ "github.com/google/ax/internal/substrate" ++ "github.com/google/ax/pkg/apis/v1alpha1" ++ "google.golang.org/grpc" ++ "google.golang.org/grpc/codes" ++ "google.golang.org/grpc/credentials/insecure" ++ "google.golang.org/grpc/status" ++) ++ ++// poolControl is a Substrate mock with a fixed number of workers. A resumed ++// actor holds a worker until it is suspended or deleted; with every worker ++// held, ResumeActor fails ResourceExhausted, as the real scheduler does. ++type poolControl struct { ++ ateapipb.UnimplementedControlServer ++ mu sync.Mutex ++ workers int ++ workerIP string ++ running map[string]bool ++ crashed map[string]bool ++ resumes map[string]int ++ suspends map[string]int ++} ++ ++func newPoolControl(workers int, workerIP string) *poolControl { ++ return &poolControl{workers: workers, workerIP: workerIP, running: map[string]bool{}, crashed: map[string]bool{}, resumes: map[string]int{}, suspends: map[string]int{}} ++} ++ ++func (m *poolControl) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) { ++ return &ateapipb.Atespace{Metadata: req.GetAtespace().GetMetadata()}, nil ++} ++ ++func (m *poolControl) CreateActorTemplate(ctx context.Context, req *ateapipb.CreateActorTemplateRequest) (*ateapipb.ActorTemplate, error) { ++ return req.GetActorTemplate(), nil ++} ++ ++func (m *poolControl) GetActorTemplate(ctx context.Context, req *ateapipb.GetActorTemplateRequest) (*ateapipb.ActorTemplate, error) { ++ return &ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Atespace: req.GetActorTemplate().GetAtespace(), Name: req.GetActorTemplate().GetName()}}, nil ++} ++ ++func (m *poolControl) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) { ++ return &ateapipb.Actor{Metadata: req.GetActor().GetMetadata(), Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}}, nil ++} ++ ++func (m *poolControl) CreateActorEgressPolicy(ctx context.Context, req *ateapipb.CreateActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { ++ return &ateapipb.EgressPolicy{}, nil ++} ++ ++func (m *poolControl) GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) { ++ m.mu.Lock() ++ defer m.mu.Unlock() ++ name := req.GetActor().GetName() ++ state := ateapipb.ActorState_ACTOR_STATE_SUSPENDED ++ switch { ++ case m.crashed[name]: ++ state = ateapipb.ActorState_ACTOR_STATE_CRASHED ++ case m.running[name]: ++ state = ateapipb.ActorState_ACTOR_STATE_RUNNING ++ } ++ return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Name: name}, Status: &ateapipb.ActorStatus{State: state}}, nil ++} ++ ++func (m *poolControl) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { ++ m.mu.Lock() ++ defer m.mu.Unlock() ++ name := req.GetActor().GetName() ++ m.resumes[name]++ ++ if !m.running[name] { ++ if len(m.running) >= m.workers { ++ return nil, status.Error(codes.ResourceExhausted, "no free workers available") ++ } ++ m.running[name] = true ++ } ++ return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ ++ Metadata: &ateapipb.ResourceMetadata{Name: name}, ++ Status: &ateapipb.ActorStatus{ ++ State: ateapipb.ActorState_ACTOR_STATE_RUNNING, ++ WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPod: "w", WorkerPodIp: m.workerIP}, ++ }, ++ }, Resumed: true}, nil ++} ++ ++func (m *poolControl) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) { ++ m.mu.Lock() ++ defer m.mu.Unlock() ++ name := req.GetActor().GetName() ++ m.suspends[name]++ ++ delete(m.running, name) ++ return &ateapipb.SuspendActorResponse{}, nil ++} ++ ++func (m *poolControl) resumeCount(name string) int { ++ m.mu.Lock() ++ defer m.mu.Unlock() ++ return m.resumes[name] ++} ++ ++// fakeRunner serves the runner's metadata endpoints. exited toggles whether ++// the command has exited; exitCode is what it reports. ++type fakeRunner struct { ++ exited atomic.Bool ++ exitCode atomic.Int32 ++ result string ++} ++ ++func (f *fakeRunner) handler() http.Handler { ++ mux := http.NewServeMux() ++ mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) ++ mux.HandleFunc("/metadata/v1alpha1/ax/exit", func(w http.ResponseWriter, r *http.Request) { ++ _ = json.NewEncoder(w).Encode(metadata.CommandExitStatus{Exited: f.exited.Load(), ExitCode: int(f.exitCode.Load()), FinishedAt: time.Now()}) ++ }) ++ mux.HandleFunc("/metadata/v1alpha1/ax/result", func(w http.ResponseWriter, r *http.Request) { ++ if !f.exited.Load() { ++ http.Error(w, "not exited", http.StatusConflict) ++ return ++ } ++ if f.result == "" { ++ http.NotFound(w, r) ++ return ++ } ++ _, _ = w.Write([]byte(f.result)) ++ }) ++ mux.HandleFunc("/metadata/v1alpha1/ax/usage", func(w http.ResponseWriter, r *http.Request) { ++ _, _ = w.Write([]byte(`{"prompt_tokens":11,"completion_tokens":7,"tool_calls":2}`)) ++ }) ++ return mux ++} ++ ++type harness struct { ++ pool *poolControl ++ runner *fakeRunner ++ reconciler *controller.TaskReconciler ++ store *memory.MemoryStore ++} ++ ++func newHarness(t *testing.T, workers int) *harness { ++ t.Helper() ++ fr := &fakeRunner{result: `{"answer":42}`} ++ hs := httptest.NewServer(fr.handler()) ++ t.Cleanup(hs.Close) ++ pool := newPoolControl(workers, hs.Listener.Addr().String()) ++ ++ lis, err := net.Listen("tcp", "127.0.0.1:0") ++ if err != nil { ++ t.Fatal(err) ++ } ++ gs := grpc.NewServer() ++ ateapipb.RegisterControlServer(gs, pool) ++ go gs.Serve(lis) ++ t.Cleanup(gs.Stop) ++ client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) ++ if err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { client.Close() }) ++ r := controller.NewTaskReconciler(client, "default-template", "ax-system") ++ r.SecretResolver = noSecrets ++ r.WorkspaceReadyTimeout = 100 * time.Millisecond ++ return &harness{pool: pool, runner: fr, reconciler: r, store: memory.NewStore()} ++} ++ ++func newTask(name string) *v1alpha1.Task { ++ return &v1alpha1.Task{ ++ Metadata: &v1alpha1.ObjectMeta{Name: name, Atespace: "fleet"}, ++ Spec: &v1alpha1.TaskSpec{Image: "localhost:5000/ax/ax-agent@sha256:00", Command: []string{"true"}}, ++ } ++} ++ ++func readyReason(task *v1alpha1.Task) string { ++ for _, c := range task.GetStatus().GetConditions() { ++ if c.GetType() == "Ready" { ++ return c.GetReason() ++ } ++ } ++ return "" ++} ++ ++func TestReconcile_ExitWriteBack(t *testing.T) { ++ ctx := context.Background() ++ for _, tc := range []struct { ++ code int32 ++ wantPhase string ++ }{{0, "Completed"}, {3, "Failed"}} { ++ t.Run(tc.wantPhase, func(t *testing.T) { ++ h := newHarness(t, 2) ++ h.runner.exited.Store(true) ++ h.runner.exitCode.Store(tc.code) ++ var saved []byte ++ h.reconciler.SaveResult = func(ctx context.Context, atespace, name string, content []byte) error { ++ saved = content ++ return nil ++ } ++ task, err := h.reconciler.Reconcile(ctx, newTask("t"), nil) ++ if err != nil { ++ t.Fatal(err) ++ } ++ if task.Status.Phase != tc.wantPhase { ++ t.Fatalf("phase = %q, want %q", task.Status.Phase, tc.wantPhase) ++ } ++ if readyReason(task) != "CommandExited" { ++ t.Fatalf("Ready reason = %q", readyReason(task)) ++ } ++ if want := fmt.Sprintf("ExitCode=%d", tc.code); task.Status.Conditions[len(task.Status.Conditions)-1].GetMessage() != want && !hasMessage(task, want) { ++ t.Fatalf("no condition message %q", want) ++ } ++ if c := task.Status.GetCommand(); !c.GetExited() || c.GetExitCode() != tc.code || c.GetResultBytes() != int64(len(`{"answer":42}`)) || c.GetResultSha256() == "" { ++ t.Fatalf("command status = %v", c) ++ } ++ if string(saved) != `{"answer":42}` { ++ t.Fatalf("saved result = %q", saved) ++ } ++ if u := task.Status.GetUsage(); u.GetPromptTokens() != 11 || u.GetToolCalls() != 2 { ++ t.Fatalf("usage = %v", u) ++ } ++ if h.pool.suspends["t"] != 1 || task.Status.WorkerIp != "" { ++ t.Fatalf("worker not freed: suspends=%d workerIP=%q", h.pool.suspends["t"], task.Status.WorkerIp) ++ } ++ if !controller.IsTerminal(task) { ++ t.Fatal("finished task is not terminal") ++ } ++ }) ++ } ++} ++ ++func hasMessage(task *v1alpha1.Task, msg string) bool { ++ for _, c := range task.GetStatus().GetConditions() { ++ if c.GetMessage() == msg { ++ return true ++ } ++ } ++ return false ++} ++ ++// The probe measured that stock v0.3.0 turns a Completed write-back into ++// Running and resumes the actor again. With the terminal guard it stays put. ++func TestWorker_CompletedWriteBackIsNotOverwritten(t *testing.T) { ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ h := newHarness(t, 2) ++ task := newTask("done") ++ task.Status = &v1alpha1.TaskStatus{Phase: "Completed"} ++ if err := h.store.SaveTask(ctx, task); err != nil { ++ t.Fatal(err) ++ } ++ w := controller.NewWorker(h.store, h.reconciler, "g", "c") ++ go func() { _ = w.Run(ctx) }() ++ time.Sleep(300 * time.Millisecond) ++ got, err := h.store.GetTask(ctx, "fleet", "done") ++ if err != nil { ++ t.Fatal(err) ++ } ++ if got.Status.Phase != "Completed" { ++ t.Fatalf("phase = %q, want Completed", got.Status.Phase) ++ } ++ if n := h.pool.resumeCount("done"); n != 0 { ++ t.Fatalf("terminal task resumed %d times", n) ++ } ++} ++ ++// runSequence applies n tasks one after another through a worker, each command ++// exiting after its first resume, and returns the final phase of each. ++func runSequence(t *testing.T, h *harness, n int) []*v1alpha1.Task { ++ t.Helper() ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ w := controller.NewWorker(h.store, h.reconciler, "g", "c") ++ w.RunningResync = 50 * time.Millisecond ++ go func() { _ = w.Run(ctx) }() ++ var out []*v1alpha1.Task ++ for i := 0; i < n; i++ { ++ name := fmt.Sprintf("floor-%d", i) ++ if err := h.store.SaveTask(ctx, newTask(name)); err != nil { ++ t.Fatal(err) ++ } ++ var got *v1alpha1.Task ++ deadline := time.Now().Add(5 * time.Second) ++ for time.Now().Before(deadline) { ++ tk, err := h.store.GetTask(ctx, "fleet", name) ++ if err == nil && tk.Status.GetPhase() != "Pending" && tk.Status.GetPhase() != "Running" { ++ got = tk ++ break ++ } ++ if err == nil && tk.Status.GetPhase() == "Running" && !h.runner.exited.Load() { ++ // Without an exit report a Running task never finishes. ++ got = tk ++ break ++ } ++ time.Sleep(20 * time.Millisecond) ++ } ++ if got == nil { ++ t.Fatalf("%s did not settle", name) ++ } ++ out = append(out, got) ++ } ++ return out ++} ++ ++// Four Tasks in a row on a 2-worker pool: every one reaches a terminal phase ++// and none fails ResourceExhausted, because each finished Task frees its worker. ++func TestWorker_FloorFourOnTwoWorkers(t *testing.T) { ++ h := newHarness(t, 2) ++ h.runner.exited.Store(true) ++ for _, tk := range runSequence(t, h, 4) { ++ if tk.Status.Phase != "Completed" { ++ t.Fatalf("%s: phase %q reason %q, want Completed", tk.Metadata.Name, tk.Status.Phase, readyReason(tk)) ++ } ++ } ++} ++ ++// The contrast the probe measured on stock v0.3.0: when no exit is reported, ++// finished work keeps its worker and the third Task on 2 workers is refused. ++func TestWorker_NoExitReportExhaustsPool(t *testing.T) { ++ h := newHarness(t, 2) ++ tasks := runSequence(t, h, 3) ++ if tasks[2].Status.Phase != "Failed" || readyReason(tasks[2]) != "ActorResumeFailed" { ++ t.Fatalf("third task: phase %q reason %q, want Failed ActorResumeFailed", tasks[2].Status.Phase, readyReason(tasks[2])) ++ } ++} ++ ++func TestWorker_ResyncFinishesRunningTask(t *testing.T) { ++ ctx := context.Background() ++ h := newHarness(t, 2) ++ w := controller.NewWorker(h.store, h.reconciler, "g", "c") ++ task, err := h.reconciler.Reconcile(ctx, newTask("slow"), nil) ++ if err != nil || task.Status.Phase != "Running" { ++ t.Fatalf("reconcile: %v phase %q", err, task.GetStatus().GetPhase()) ++ } ++ if err := h.store.SaveTask(ctx, task); err != nil { ++ t.Fatal(err) ++ } ++ if n := w.ResyncOnce(ctx); n != 0 { ++ t.Fatalf("resync changed %d tasks before exit", n) ++ } ++ h.runner.exited.Store(true) ++ if n := w.ResyncOnce(ctx); n != 1 { ++ t.Fatalf("resync changed %d tasks after exit, want 1", n) ++ } ++ got, _ := h.store.GetTask(ctx, "fleet", "slow") ++ if got.Status.Phase != "Completed" { ++ t.Fatalf("phase = %q", got.Status.Phase) ++ } ++ if b, err := h.store.GetTaskResult(ctx, "fleet", "slow"); err != nil || string(b) != `{"answer":42}` { ++ t.Fatalf("stored result = %q, %v", b, err) ++ } ++ if _, err := h.store.GetTaskResult(ctx, "fleet", "nope"); err != store.ErrNotFound { ++ t.Fatalf("missing result err = %v", err) ++ } ++} ++ ++func TestResync_CrashedActorIsFailed(t *testing.T) { ++ ctx := context.Background() ++ h := newHarness(t, 2) ++ task, err := h.reconciler.Reconcile(ctx, newTask("boom"), nil) ++ if err != nil { ++ t.Fatal(err) ++ } ++ h.pool.mu.Lock() ++ h.pool.crashed["boom"] = true ++ h.pool.mu.Unlock() ++ task.Status.WorkerIp = "127.0.0.1:1" // runner unreachable ++ changed, _ := h.reconciler.ResyncRunning(ctx, task) ++ if !changed || task.Status.Phase != "Failed" || readyReason(task) != "ActorCrashed" { ++ t.Fatalf("changed=%v phase=%q reason=%q", changed, task.Status.Phase, readyReason(task)) ++ } ++} +diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go +index c4aebda..a4692ca 100644 +--- a/internal/controller/reconciler.go ++++ b/internal/controller/reconciler.go +@@ -17,8 +17,11 @@ package controller + import ( + "context" + "crypto/sha256" ++ "encoding/hex" ++ "encoding/json" + "errors" + "fmt" ++ "io" + "log/slog" + "net" + "net/http" +@@ -28,6 +31,8 @@ import ( + "strings" + "time" + ++ "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ++ "github.com/google/ax/internal/metadata" + "github.com/google/ax/internal/model" + "github.com/google/ax/internal/substrate" + "github.com/google/ax/pkg/apis/v1alpha1" +@@ -70,6 +75,10 @@ type TaskReconciler struct { + // WorkspaceReadyTimeout bounds how long Reconcile waits for the actor's workspace + // to report ready before recording it as still initializing. + WorkspaceReadyTimeout time.Duration ++ ++ // SaveResult stores the result content copied from a finished task's ++ // sandbox. Nil drops the content; the digest and size still reach status. ++ SaveResult func(ctx context.Context, atespace, name string, content []byte) error + } + + // NewTaskReconciler creates a new TaskReconciler. +@@ -111,6 +120,14 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, gat + task.Spec.Image = v1alpha1.DefaultTaskImage + } + ++ // Terminal guard: a task whose command already exited is never resumed ++ // again. Without it every later event (an apply, a status write-back that ++ // republishes) would resume the actor and flip the phase back to Running. ++ if IsTerminal(task) { ++ slog.Info("task is terminal, not reconciling", "name", task.Metadata.Name, "atespace", atespace, "phase", task.Status.Phase) ++ return task, nil ++ } ++ + slog.Info("reconciling task", "name", task.Metadata.Name, "atespace", atespace, "image", task.Spec.Image) + + now := time.Now() +@@ -293,6 +310,15 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, gat + } + DonePolling: + ++ // A short command may already have finished; if so, record it now. ++ if workerIP != "" { ++ if done, err := r.finishIfExited(ctx, task, host, port); err != nil { ++ slog.Warn("could not read command exit", "name", task.Metadata.Name, "error", err) ++ } else if done { ++ return task, nil ++ } ++ } ++ + // The task is Ready only once its actor is running and the workspace inside it is set up. + if workspaceReady { + if !r.conditionTrue(task, condWorkspaceReady) { +@@ -317,6 +343,11 @@ DonePolling: + + // Condition types reported on Task status. + const ( ++ // reasonCommandExited is the Ready reason on a task whose command exited. ++ reasonCommandExited = "CommandExited" ++ // reasonActorCrashed is the Ready reason on a task whose actor crashed. ++ reasonActorCrashed = "ActorCrashed" ++ + // condReady reports whether the task as a whole is ready to do work: its actor is + // running and the workspace inside it has finished setting up. + condReady = "Ready" +@@ -483,3 +514,172 @@ func marshalWorkspaces(workspaces []*v1alpha1.Workspace) (string, error) { + } + return sb.String(), nil + } ++ ++// IsTerminal reports whether the task's command has finished: phase Completed, ++// or Failed with the Ready reason CommandExited. Such a task is never resumed. ++func IsTerminal(task *v1alpha1.Task) bool { ++ st := task.GetStatus() ++ switch st.GetPhase() { ++ case "Completed": ++ return true ++ case "Failed": ++ for _, c := range st.GetConditions() { ++ if c.GetType() == condReady && c.GetReason() == reasonCommandExited { ++ return true ++ } ++ } ++ } ++ return false ++} ++ ++// ResyncRunning re-checks a Running task, because no event fires when its ++// command exits. It returns true when it changed the task's status: the command ++// exited (Completed or Failed, worker freed) or the actor crashed (Failed). ++func (r *TaskReconciler) ResyncRunning(ctx context.Context, task *v1alpha1.Task) (bool, error) { ++ if task.GetStatus().GetPhase() != "Running" || task.GetMetadata() == nil || IsTerminal(task) { ++ return false, nil ++ } ++ atespace := task.Metadata.Atespace ++ if atespace == "" { ++ atespace = "default" ++ } ++ host, port := task.Status.WorkerIp, "80" ++ if h, p, err := net.SplitHostPort(host); err == nil { ++ host, port = h, p ++ } ++ done, exitErr := r.finishIfExited(ctx, task, host, port) ++ if done { ++ return true, nil ++ } ++ // No exit report: a crashed actor is Failed, not restarted behind the ++ // caller's back. ++ state, err := r.client.ActorState(ctx, atespace, task.Metadata.Name) ++ if err == nil && state == ateapipb.ActorState_ACTOR_STATE_CRASHED { ++ task.Status.Phase = "Failed" ++ task.Status.WorkerIp = "" ++ r.setNotReady(task, reasonActorCrashed, "Substrate reports the actor CRASHED", time.Now()) ++ return true, nil ++ } ++ if exitErr != nil { ++ return false, exitErr ++ } ++ return false, err ++} ++ ++// finishIfExited asks the runner whether the command exited. If it did, it ++// copies the result and usage, writes the outcome into status, and suspends ++// the actor so its worker is free for the next task. ++func (r *TaskReconciler) finishIfExited(ctx context.Context, task *v1alpha1.Task, host, port string) (bool, error) { ++ atespace := task.Metadata.Atespace ++ actorName := task.Metadata.Name ++ body, err := r.metadataGet(ctx, atespace, actorName, host, port, "/metadata/v1alpha1/ax/exit") ++ if err != nil { ++ return false, err ++ } ++ var exit metadata.CommandExitStatus ++ if err := json.Unmarshal(body, &exit); err != nil { ++ return false, fmt.Errorf("decoding exit status: %w", err) ++ } ++ if !exit.Exited { ++ return false, nil ++ } ++ ++ now := time.Now() ++ cmdStatus := &v1alpha1.CommandStatus{Exited: true, ExitCode: int32(exit.ExitCode)} ++ if !exit.FinishedAt.IsZero() { ++ cmdStatus.FinishedAt = timestamppb.New(exit.FinishedAt) ++ } else { ++ cmdStatus.FinishedAt = timestamppb.New(now) ++ } ++ if result, err := r.metadataGet(ctx, atespace, actorName, host, port, "/metadata/v1alpha1/ax/result"); err == nil { ++ sum := sha256.Sum256(result) ++ cmdStatus.ResultBytes = int64(len(result)) ++ cmdStatus.ResultSha256 = hex.EncodeToString(sum[:]) ++ if r.SaveResult != nil { ++ if err := r.SaveResult(ctx, atespace, actorName, result); err != nil { ++ slog.Warn("could not store task result", "name", actorName, "error", err) ++ } ++ } ++ } else { ++ slog.Info("task produced no result", "name", actorName, "reason", err) ++ } ++ if usageBody, err := r.metadataGet(ctx, atespace, actorName, host, port, "/metadata/v1alpha1/ax/usage"); err == nil { ++ var u struct { ++ PromptTokens int32 `json:"prompt_tokens"` ++ CompletionTokens int32 `json:"completion_tokens"` ++ ToolCalls int32 `json:"tool_calls"` ++ } ++ if json.Unmarshal(usageBody, &u) == nil { ++ task.Status.Usage = &v1alpha1.UsageStats{PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, ToolCalls: u.ToolCalls} ++ } ++ } ++ task.Status.Command = cmdStatus ++ ++ msg := fmt.Sprintf("ExitCode=%d", exit.ExitCode) ++ if exit.ExitCode == 0 { ++ task.Status.Phase = "Completed" ++ } else { ++ task.Status.Phase = "Failed" ++ } ++ r.setCondition(task, condReady, "False", reasonCommandExited, msg, now) ++ ++ // Free the worker. The DATA snapshot keeps /workspace for inspection. ++ if err := r.client.SuspendActor(ctx, atespace, actorName); err != nil { ++ slog.Warn("could not suspend finished task's actor", "name", actorName, "error", err) ++ r.setCondition(task, condReady, "False", reasonCommandExited, msg+"; suspend failed: "+err.Error(), now) ++ } else { ++ task.Status.WorkerIp = "" ++ } ++ slog.Info("task command exited", "name", actorName, "atespace", atespace, "exitCode", exit.ExitCode, "phase", task.Status.Phase) ++ return true, nil ++} ++ ++// metadataGet reads a runner metadata path, directly from the worker first and ++// then through the atenet router when ATENET_ROUTER_ADDR is set. ++func (r *TaskReconciler) metadataGet(ctx context.Context, atespace, actorName, host, port, path string) ([]byte, error) { ++ var errs []error ++ if host != "" { ++ b, err := r.fetch(ctx, fmt.Sprintf("http://%s%s", net.JoinHostPort(host, port), path), "") ++ if err == nil { ++ return b, nil ++ } ++ errs = append(errs, err) ++ } ++ if routerAddr := os.Getenv("ATENET_ROUTER_ADDR"); routerAddr != "" { ++ b, err := r.fetch(ctx, fmt.Sprintf("http://%s%s", routerAddr, path), fmt.Sprintf("%s/%s", atespace, actorName)) ++ if err == nil { ++ return b, nil ++ } ++ errs = append(errs, err) ++ } ++ if len(errs) == 0 { ++ return nil, errors.New("no route to the task's metadata server") ++ } ++ return nil, errors.Join(errs...) ++} ++ ++func (r *TaskReconciler) fetch(ctx context.Context, url, targetActor string) ([]byte, error) { ++ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) ++ if err != nil { ++ return nil, err ++ } ++ if targetActor != "" { ++ req.Header.Set("ate-target-actor", targetActor) ++ } ++ resp, err := r.httpClient.Do(req) ++ if err != nil { ++ return nil, err ++ } ++ defer resp.Body.Close() ++ body, err := io.ReadAll(io.LimitReader(resp.Body, metadata.MaxResultBytes+1)) ++ if err != nil { ++ return nil, err ++ } ++ if resp.StatusCode != http.StatusOK { ++ return nil, fmt.Errorf("GET %s: %s", url, resp.Status) ++ } ++ if int64(len(body)) > metadata.MaxResultBytes { ++ return nil, fmt.Errorf("GET %s: body over %d bytes", url, metadata.MaxResultBytes) ++ } ++ return body, nil ++} +diff --git a/internal/controller/worker.go b/internal/controller/worker.go +index 398b090..c783e8a 100644 +--- a/internal/controller/worker.go ++++ b/internal/controller/worker.go +@@ -20,6 +20,7 @@ import ( + "fmt" + "log/slog" + "os" ++ "sync" + "time" + + "github.com/google/ax/internal/store" +@@ -31,6 +32,9 @@ const ( + // readRetryDelay is how long the worker waits after a transient error from the + // event queue before trying again. + readRetryDelay = time.Second ++ // DefaultRunningResync is how often Running tasks are re-checked for a ++ // command exit, since nothing publishes an event when a command exits. ++ DefaultRunningResync = 15 * time.Second + ) + + // Worker consumes task events from the store's event queue and reconciles each +@@ -41,6 +45,14 @@ type Worker struct { + reconciler *TaskReconciler + group string + consumer string ++ ++ // RunningResync is the period of the Running-task re-check. Zero or less ++ // disables it. ++ RunningResync time.Duration ++ ++ // mu serializes event handling and the resync, so one task is never ++ // reconciled twice at once by this worker. ++ mu sync.Mutex + } + + // NewWorker creates a worker that joins group as consumer. An empty group uses the +@@ -53,11 +65,15 @@ func NewWorker(s store.Store, reconciler *TaskReconciler, group, consumer string + hostname, _ := os.Hostname() + consumer = fmt.Sprintf("%s-%d", hostname, time.Now().UnixNano()%10000) + } ++ if reconciler != nil && reconciler.SaveResult == nil { ++ reconciler.SaveResult = s.SaveTaskResult ++ } + return &Worker{ +- store: s, +- reconciler: reconciler, +- group: group, +- consumer: consumer, ++ store: s, ++ reconciler: reconciler, ++ group: group, ++ consumer: consumer, ++ RunningResync: DefaultRunningResync, + } + } + +@@ -73,6 +89,10 @@ func (w *Worker) Run(ctx context.Context) error { + } + defer sub.Close() + ++ if w.RunningResync > 0 { ++ go w.resyncLoop(ctx) ++ } ++ + for { + ev, err := sub.Next(ctx) + if err != nil { +@@ -89,7 +109,10 @@ func (w *Worker) Run(ctx context.Context) error { + continue + } + +- if err := w.processEvent(ctx, ev); err != nil { ++ w.mu.Lock() ++ err = w.processEvent(ctx, ev) ++ w.mu.Unlock() ++ if err != nil { + slog.Error("error processing task event", + "id", ev.ID, + "atespace", ev.Atespace, +@@ -165,3 +188,53 @@ func (w *Worker) processEvent(ctx context.Context, ev store.TaskEvent) error { + + return nil + } ++ ++// resyncLoop re-checks every Running task each RunningResync until ctx is done. ++func (w *Worker) resyncLoop(ctx context.Context) { ++ ticker := time.NewTicker(w.RunningResync) ++ defer ticker.Stop() ++ for { ++ select { ++ case <-ctx.Done(): ++ return ++ case <-ticker.C: ++ w.ResyncOnce(ctx) ++ } ++ } ++} ++ ++// ResyncOnce re-checks every Running task once and writes back any that ++// finished. It returns how many tasks changed. ++func (w *Worker) ResyncOnce(ctx context.Context) int { ++ w.mu.Lock() ++ defer w.mu.Unlock() ++ tasks, err := w.store.ListTasks(ctx, "", 0, 0) ++ if err != nil { ++ slog.Warn("resync: listing tasks", "error", err) ++ return 0 ++ } ++ changed := 0 ++ for _, t := range tasks { ++ if t.GetStatus().GetPhase() != "Running" { ++ continue ++ } ++ // Re-read so a status written since the list is not overwritten. ++ task, err := w.store.GetTask(ctx, t.Metadata.Atespace, t.Metadata.Name) ++ if err != nil || task.GetStatus().GetPhase() != "Running" { ++ continue ++ } ++ ok, err := w.reconciler.ResyncRunning(ctx, task) ++ if err != nil { ++ slog.Debug("resync: task not finished or unreachable", "atespace", task.Metadata.Atespace, "name", task.Metadata.Name, "error", err) ++ } ++ if !ok { ++ continue ++ } ++ if err := w.store.UpdateTaskStatus(ctx, task.Metadata.Atespace, task.Metadata.Name, task.Status); err != nil { ++ slog.Error("resync: writing task status", "name", task.Metadata.Name, "error", err) ++ continue ++ } ++ changed++ ++ } ++ return changed ++} +diff --git a/internal/metadata/exit_test.go b/internal/metadata/exit_test.go +new file mode 100644 +index 0000000..85da1c1 +--- /dev/null ++++ b/internal/metadata/exit_test.go +@@ -0,0 +1,118 @@ ++// Copyright 2026 Google LLC ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++package metadata_test ++ ++import ( ++ "context" ++ "encoding/json" ++ "fmt" ++ "io" ++ "net" ++ "net/http" ++ "os" ++ "path/filepath" ++ "strings" ++ "testing" ++ "time" ++ ++ "github.com/google/ax/internal/metadata" ++ "github.com/google/ax/pkg/apis/v1alpha1" ++) ++ ++func freePort(t *testing.T) int { ++ t.Helper() ++ l, err := net.Listen("tcp", "127.0.0.1:0") ++ if err != nil { ++ t.Fatal(err) ++ } ++ defer l.Close() ++ return l.Addr().(*net.TCPAddr).Port ++} ++ ++func get(t *testing.T, url string) (int, string) { ++ t.Helper() ++ var resp *http.Response ++ var err error ++ for i := 0; i < 50; i++ { ++ resp, err = http.Get(url) ++ if err == nil { ++ break ++ } ++ time.Sleep(20 * time.Millisecond) ++ } ++ if err != nil { ++ t.Fatalf("GET %s: %v", url, err) ++ } ++ defer resp.Body.Close() ++ b, _ := io.ReadAll(resp.Body) ++ return resp.StatusCode, string(b) ++} ++ ++func TestMetadataServer_ExitAndResult(t *testing.T) { ++ dir := t.TempDir() ++ resultPath := filepath.Join(dir, "result.json") ++ usagePath := filepath.Join(dir, "usage.json") ++ port := freePort(t) ++ task := &v1alpha1.Task{Metadata: &v1alpha1.ObjectMeta{Name: "t1"}, Spec: &v1alpha1.TaskSpec{}} ++ s := metadata.NewServer(port, task, nil, metadata.ServerOptions{ResultPath: resultPath, UsagePath: usagePath}) ++ if err := s.Start(); err != nil { ++ t.Fatal(err) ++ } ++ defer s.Stop(context.Background()) ++ base := fmt.Sprintf("http://127.0.0.1:%d/metadata/v1alpha1/ax", port) ++ ++ // Before exit: exit says not exited, result is refused. ++ code, body := get(t, base+"/exit") ++ var st metadata.CommandExitStatus ++ if code != http.StatusOK || json.Unmarshal([]byte(body), &st) != nil || st.Exited { ++ t.Fatalf("exit before command exit = %d %q", code, body) ++ } ++ if code, _ := get(t, base+"/result"); code != http.StatusConflict { ++ t.Fatalf("result before exit = %d, want 409", code) ++ } ++ ++ if err := os.WriteFile(resultPath, []byte(`{"ok":true}`), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.WriteFile(usagePath, []byte(`{"prompt_tokens":3}`), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ s.SetCommandExit(3, time.Now()) ++ s.SetCommandExit(0, time.Now()) // only the first report counts ++ ++ code, body = get(t, base+"/exit") ++ if err := json.Unmarshal([]byte(body), &st); err != nil || code != http.StatusOK || !st.Exited || st.ExitCode != 3 { ++ t.Fatalf("exit after command exit = %d %q", code, body) ++ } ++ if code, body := get(t, base+"/result"); code != http.StatusOK || body != `{"ok":true}` { ++ t.Fatalf("result = %d %q", code, body) ++ } ++ if code, body := get(t, base+"/usage"); code != http.StatusOK || !strings.Contains(body, "prompt_tokens") { ++ t.Fatalf("usage = %d %q", code, body) ++ } ++ ++ // Over the cap: refused, not truncated. ++ if err := os.WriteFile(resultPath, make([]byte, metadata.MaxResultBytes+1), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ if code, _ := get(t, base+"/result"); code != http.StatusRequestEntityTooLarge { ++ t.Fatalf("oversized result = %d, want 413", code) ++ } ++ // Missing: 404. ++ _ = os.Remove(resultPath) ++ if code, _ := get(t, base+"/result"); code != http.StatusNotFound { ++ t.Fatalf("missing result = %d, want 404", code) ++ } ++} +diff --git a/internal/metadata/server.go b/internal/metadata/server.go +index fb4feba..909e879 100644 +--- a/internal/metadata/server.go ++++ b/internal/metadata/server.go +@@ -16,13 +16,17 @@ package metadata + + import ( + "context" ++ "encoding/json" + "errors" + "fmt" ++ "io" + "log/slog" + "net" + "net/http" ++ "os" + "strings" + "sync" ++ "time" + + "github.com/agent-substrate/env/guest" + "github.com/google/ax/pkg/apis/v1alpha1" +@@ -41,12 +45,31 @@ type Server struct { + task *v1alpha1.Task + workspaces []*v1alpha1.Workspace + workspaceReady bool ++ resultPath string ++ usagePath string ++ exit *CommandExitStatus ++} ++ ++// MaxResultBytes caps the result file served at /metadata/v1alpha1/ax/result. ++// A larger file is refused with 413 rather than truncated. ++const MaxResultBytes = 1 << 20 ++ ++// CommandExitStatus is the JSON body of /metadata/v1alpha1/ax/exit. ++type CommandExitStatus struct { ++ Exited bool `json:"exited"` ++ ExitCode int `json:"exitCode"` ++ FinishedAt time.Time `json:"finishedAt,omitempty"` + } + + // ServerOptions configures optional settings for the metadata and guest server. + type ServerOptions struct { + WorkspacePath string + LogDir string ++ // ResultPath is the file served at /metadata/v1alpha1/ax/result once the ++ // command has exited. Empty disables the endpoint. ++ ResultPath string ++ // UsagePath is the JSON usage file served at /metadata/v1alpha1/ax/usage. ++ UsagePath string + } + + // NewServer creates a new metadata and guest server serving the task and its +@@ -65,6 +88,8 @@ func NewServer(port int, task *v1alpha1.Task, workspaces []*v1alpha1.Workspace, + if len(opts) > 0 { + opt = opts[0] + } ++ s.resultPath = opt.ResultPath ++ s.usagePath = opt.UsagePath + + // Guest services expose process execution and file access inside the container, + // so they are only served when the task opts in via spec.debug. +@@ -98,6 +123,12 @@ func NewServer(port int, task *v1alpha1.Task, workspaces []*v1alpha1.Workspace, + // /metadata/v1alpha1/ax/workspaces every bound Workspace, as a YAML stream + mux.HandleFunc("/metadata/v1alpha1/ax/task", s.handleTask) + mux.HandleFunc("/metadata/v1alpha1/ax/workspaces", s.handleWorkspaces) ++ // /metadata/v1alpha1/ax/exit whether the command exited, and its code ++ // /metadata/v1alpha1/ax/result the command's result file, after exit ++ // /metadata/v1alpha1/ax/usage the command's usage file, after exit ++ mux.HandleFunc("/metadata/v1alpha1/ax/exit", s.handleExit) ++ mux.HandleFunc("/metadata/v1alpha1/ax/result", s.handleResult) ++ mux.HandleFunc("/metadata/v1alpha1/ax/usage", s.handleUsage) + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.grpcServer != nil && r.ProtoMajor == 2 && strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { +@@ -236,3 +267,72 @@ func (s *Server) handleWorkspaces(w http.ResponseWriter, r *http.Request) { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + } ++ ++// SetCommandExit records that the task command exited on its own with code. ++// Only the first call counts. ++func (s *Server) SetCommandExit(code int, at time.Time) { ++ s.mu.Lock() ++ defer s.mu.Unlock() ++ if s.exit == nil { ++ s.exit = &CommandExitStatus{Exited: true, ExitCode: code, FinishedAt: at.UTC()} ++ } ++} ++ ++func (s *Server) commandExit() *CommandExitStatus { ++ s.mu.RLock() ++ defer s.mu.RUnlock() ++ if s.exit == nil { ++ return nil ++ } ++ cp := *s.exit ++ return &cp ++} ++ ++// handleExit reports whether the command has exited. It always answers 200 ++// with a JSON body, so a caller can tell "still running" from "unreachable". ++func (s *Server) handleExit(w http.ResponseWriter, r *http.Request) { ++ st := s.commandExit() ++ if st == nil { ++ st = &CommandExitStatus{} ++ } ++ w.Header().Set("Content-Type", "application/json") ++ _ = json.NewEncoder(w).Encode(st) ++} ++ ++func (s *Server) handleResult(w http.ResponseWriter, r *http.Request) { ++ s.serveAfterExit(w, s.resultPath, MaxResultBytes) ++} ++ ++func (s *Server) handleUsage(w http.ResponseWriter, r *http.Request) { ++ s.serveAfterExit(w, s.usagePath, 64<<10) ++} ++ ++// serveAfterExit serves the file at path once the command has exited: 409 ++// before that, 404 when there is no file, 413 when it exceeds limit. ++func (s *Server) serveAfterExit(w http.ResponseWriter, path string, limit int64) { ++ if s.commandExit() == nil { ++ http.Error(w, "command has not exited", http.StatusConflict) ++ return ++ } ++ if path == "" { ++ http.Error(w, "not configured", http.StatusNotFound) ++ return ++ } ++ f, err := os.Open(path) ++ if err != nil { ++ http.Error(w, "no file", http.StatusNotFound) ++ return ++ } ++ defer f.Close() ++ fi, err := f.Stat() ++ if err != nil || !fi.Mode().IsRegular() { ++ http.Error(w, "no file", http.StatusNotFound) ++ return ++ } ++ if fi.Size() > limit { ++ http.Error(w, fmt.Sprintf("file is %d bytes, over the %d byte cap", fi.Size(), limit), http.StatusRequestEntityTooLarge) ++ return ++ } ++ w.Header().Set("Content-Type", "application/octet-stream") ++ _, _ = io.Copy(w, io.LimitReader(f, limit)) ++} +diff --git a/internal/server/server.go b/internal/server/server.go +index 27e89ab..669cbac 100644 +--- a/internal/server/server.go ++++ b/internal/server/server.go +@@ -16,6 +16,8 @@ package server + + import ( + "context" ++ "crypto/sha256" ++ "encoding/hex" + "errors" + "net/http" + "strings" +@@ -88,6 +90,27 @@ func (s *Server) GetTask(ctx context.Context, req *v1alpha1.GetTaskRequest) (*v1 + return task, nil + } + ++// GetTaskResult returns the result content the controller copied from the ++// task's sandbox when its command exited. ++func (s *Server) GetTaskResult(ctx context.Context, req *v1alpha1.GetTaskResultRequest) (*v1alpha1.TaskResult, error) { ++ if req == nil || req.Name == "" { ++ return nil, status.Error(codes.InvalidArgument, "missing task name") ++ } ++ atespace := req.Atespace ++ if atespace == "" { ++ atespace = "default" ++ } ++ content, err := s.store.GetTaskResult(ctx, atespace, req.Name) ++ if err != nil { ++ if errors.Is(err, store.ErrNotFound) { ++ return nil, status.Errorf(codes.NotFound, "no result for task %q in atespace %q", req.Name, atespace) ++ } ++ return nil, status.Errorf(codes.Internal, "getting task result: %v", err) ++ } ++ sum := sha256.Sum256(content) ++ return &v1alpha1.TaskResult{Content: content, Sha256: hex.EncodeToString(sum[:])}, nil ++} ++ + func (s *Server) ListTasks(ctx context.Context, req *v1alpha1.ListTasksRequest) (*v1alpha1.ListTasksResponse, error) { + atespace := "" + limit := int64(50) +diff --git a/internal/store/memory/store.go b/internal/store/memory/store.go +index 8a0b13d..215f9a4 100644 +--- a/internal/store/memory/store.go ++++ b/internal/store/memory/store.go +@@ -47,6 +47,7 @@ type MemoryStore struct { + workspaces map[string]*v1alpha1.Workspace + events chan store.TaskEvent + watchers map[string][]chan *v1alpha1.Task ++ results map[string][]byte + } + + // NewStore creates a new in-memory Store. +@@ -58,9 +59,29 @@ func NewStore() *MemoryStore { + workspaces: make(map[string]*v1alpha1.Workspace), + events: make(chan store.TaskEvent, 1000), + watchers: make(map[string][]chan *v1alpha1.Task), ++ results: make(map[string][]byte), + } + } + ++// SaveTaskResult stores a copy of the task's result content. ++func (s *MemoryStore) SaveTaskResult(ctx context.Context, atespace, name string, content []byte) error { ++ s.mu.Lock() ++ defer s.mu.Unlock() ++ s.results[taskKey(atespace, name)] = append([]byte(nil), content...) ++ return nil ++} ++ ++// GetTaskResult returns a copy of the stored result content. ++func (s *MemoryStore) GetTaskResult(ctx context.Context, atespace, name string) ([]byte, error) { ++ s.mu.RLock() ++ defer s.mu.RUnlock() ++ content, ok := s.results[taskKey(atespace, name)] ++ if !ok { ++ return nil, store.ErrNotFound ++ } ++ return append([]byte(nil), content...), nil ++} ++ + func taskKey(atespace, name string) string { + if atespace == "" { + atespace = "default" +@@ -216,6 +237,7 @@ func (s *MemoryStore) DeleteTask(ctx context.Context, atespace, name string) err + s.mu.Lock() + defer s.mu.Unlock() + delete(s.tasks, taskKey(atespace, name)) ++ delete(s.results, taskKey(atespace, name)) + return nil + } + +diff --git a/internal/store/redis/store.go b/internal/store/redis/store.go +index 0bbf7aa..34da35b 100644 +--- a/internal/store/redis/store.go ++++ b/internal/store/redis/store.go +@@ -82,6 +82,37 @@ func (s *Store) taskKey(atespace, name string) string { + return fmt.Sprintf("%s:task:%s:%s", s.opts.KeyPrefix, atespace, name) + } + ++func (s *Store) taskResultKey(atespace, name string) string { ++ return fmt.Sprintf("%s:task-result:%s:%s", s.opts.KeyPrefix, atespace, name) ++} ++ ++// SaveTaskResult stores the task's result content under its own key, so task ++// records, lists and watches stay small. It publishes no event. ++func (s *Store) SaveTaskResult(ctx context.Context, atespace, name string, content []byte) error { ++ if atespace == "" { ++ atespace = "default" ++ } ++ if err := s.client.Set(ctx, s.taskResultKey(atespace, name), content, s.opts.TTL).Err(); err != nil { ++ return fmt.Errorf("saving task result in redis: %w", err) ++ } ++ return nil ++} ++ ++// GetTaskResult returns the stored result content, or store.ErrNotFound. ++func (s *Store) GetTaskResult(ctx context.Context, atespace, name string) ([]byte, error) { ++ if atespace == "" { ++ atespace = "default" ++ } ++ content, err := s.client.Get(ctx, s.taskResultKey(atespace, name)).Bytes() ++ if err != nil { ++ if errors.Is(err, redis.Nil) { ++ return nil, store.ErrNotFound ++ } ++ return nil, fmt.Errorf("getting task result from redis: %w", err) ++ } ++ return content, nil ++} ++ + func (s *Store) gwKey(atespace, name string) string { + return fmt.Sprintf("%s:gw:%s:%s", s.opts.KeyPrefix, atespace, name) + } +@@ -335,6 +366,7 @@ func (s *Store) DeleteTask(ctx context.Context, atespace, name string) error { + + pipe := s.client.TxPipeline() + pipe.Del(ctx, s.taskKey(atespace, name)) ++ pipe.Del(ctx, s.taskResultKey(atespace, name)) + pipe.ZRem(ctx, s.taskIndexKey(), member) + pipe.ZRem(ctx, s.taskAtespaceIndexKey(atespace), name) + if _, err := pipe.Exec(ctx); err != nil { +diff --git a/internal/store/store.go b/internal/store/store.go +index 1e84a0f..684f140 100644 +--- a/internal/store/store.go ++++ b/internal/store/store.go +@@ -86,5 +86,11 @@ type Store interface { + DeleteModel(ctx context.Context, atespace, name string) error + + WatchTask(ctx context.Context, atespace, name string) (<-chan *v1alpha1.Task, io.Closer, error) ++ ++ // SaveTaskResult stores the result file a task's command wrote, as copied ++ // by the controller once the command exited. It publishes no event. ++ SaveTaskResult(ctx context.Context, atespace, name string, content []byte) error ++ // GetTaskResult returns the stored result. ErrNotFound when none was saved. ++ GetTaskResult(ctx context.Context, atespace, name string) ([]byte, error) + Close() error + } +diff --git a/internal/substrate/client.go b/internal/substrate/client.go +index 2840ce2..3693e99 100644 +--- a/internal/substrate/client.go ++++ b/internal/substrate/client.go +@@ -389,6 +389,17 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA + return actor, nil + } + ++// ActorState returns the actor's current state as Substrate reports it. ++func (c *Client) ActorState(ctx context.Context, atespace, actorName string) (ateapipb.ActorState, error) { ++ actor, err := c.control.GetActor(ctx, &ateapipb.GetActorRequest{ ++ Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: actorName}, ++ }) ++ if err != nil { ++ return ateapipb.ActorState_ACTOR_STATE_UNSPECIFIED, fmt.Errorf("getting actor %s/%s: %w", atespace, actorName, err) ++ } ++ return actor.GetStatus().GetState(), nil ++} ++ + // ResumeActor resumes the specified actor onto a worker and returns the worker details. + func (c *Client) ResumeActor(ctx context.Context, atespace, actorName string) (*ateapipb.Actor, string, error) { + req := &ateapipb.ResumeActorRequest{ +diff --git a/pkg/apis/v1alpha1/ax.pb.go b/pkg/apis/v1alpha1/ax.pb.go +index 9a71dc0..0ce2bf0 100644 +--- a/pkg/apis/v1alpha1/ax.pb.go ++++ b/pkg/apis/v1alpha1/ax.pb.go +@@ -571,8 +571,10 @@ type TaskStatus struct { + PendingApproval *PendingApproval `protobuf:"bytes,5,opt,name=pending_approval,json=pendingApproval,proto3" json:"pending_approval,omitempty"` + Usage *UsageStats `protobuf:"bytes,6,opt,name=usage,proto3" json:"usage,omitempty"` + Conditions []*Condition `protobuf:"bytes,7,rep,name=conditions,proto3" json:"conditions,omitempty"` +- unknownFields protoimpl.UnknownFields +- sizeCache protoimpl.SizeCache ++ // command reports how the task's command finished. Unset while it runs. ++ Command *CommandStatus `protobuf:"bytes,8,opt,name=command,proto3" json:"command,omitempty"` ++ unknownFields protoimpl.UnknownFields ++ sizeCache protoimpl.SizeCache + } + + func (x *TaskStatus) Reset() { +@@ -654,6 +656,92 @@ func (x *TaskStatus) GetConditions() []*Condition { + return nil + } + ++func (x *TaskStatus) GetCommand() *CommandStatus { ++ if x != nil { ++ return x.Command ++ } ++ return nil ++} ++ ++// CommandStatus is written by the controller once the runner reports that the ++// task's command exited. The result content itself is served by GetTaskResult. ++type CommandStatus struct { ++ state protoimpl.MessageState `protogen:"open.v1"` ++ Exited bool `protobuf:"varint,1,opt,name=exited,proto3" json:"exited,omitempty"` ++ ExitCode int32 `protobuf:"varint,2,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` ++ FinishedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=finished_at,json=finishedAt,proto3" json:"finished_at,omitempty"` ++ // result_bytes is the size of the copied result, 0 when there was none. ++ ResultBytes int64 `protobuf:"varint,4,opt,name=result_bytes,json=resultBytes,proto3" json:"result_bytes,omitempty"` ++ ResultSha256 string `protobuf:"bytes,5,opt,name=result_sha256,json=resultSha256,proto3" json:"result_sha256,omitempty"` ++ unknownFields protoimpl.UnknownFields ++ sizeCache protoimpl.SizeCache ++} ++ ++func (x *CommandStatus) Reset() { ++ *x = CommandStatus{} ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[9] ++ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ++ ms.StoreMessageInfo(mi) ++} ++ ++func (x *CommandStatus) String() string { ++ return protoimpl.X.MessageStringOf(x) ++} ++ ++func (*CommandStatus) ProtoMessage() {} ++ ++func (x *CommandStatus) ProtoReflect() protoreflect.Message { ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[9] ++ if x != nil { ++ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ++ if ms.LoadMessageInfo() == nil { ++ ms.StoreMessageInfo(mi) ++ } ++ return ms ++ } ++ return mi.MessageOf(x) ++} ++ ++// Deprecated: Use CommandStatus.ProtoReflect.Descriptor instead. ++func (*CommandStatus) Descriptor() ([]byte, []int) { ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{9} ++} ++ ++func (x *CommandStatus) GetExited() bool { ++ if x != nil { ++ return x.Exited ++ } ++ return false ++} ++ ++func (x *CommandStatus) GetExitCode() int32 { ++ if x != nil { ++ return x.ExitCode ++ } ++ return 0 ++} ++ ++func (x *CommandStatus) GetFinishedAt() *timestamppb.Timestamp { ++ if x != nil { ++ return x.FinishedAt ++ } ++ return nil ++} ++ ++func (x *CommandStatus) GetResultBytes() int64 { ++ if x != nil { ++ return x.ResultBytes ++ } ++ return 0 ++} ++ ++func (x *CommandStatus) GetResultSha256() string { ++ if x != nil { ++ return x.ResultSha256 ++ } ++ return "" ++} ++ + type PendingApproval struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +@@ -665,7 +753,7 @@ type PendingApproval struct { + + func (x *PendingApproval) Reset() { + *x = PendingApproval{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[9] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -677,7 +765,7 @@ func (x *PendingApproval) String() string { + func (*PendingApproval) ProtoMessage() {} + + func (x *PendingApproval) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[9] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -690,7 +778,7 @@ func (x *PendingApproval) ProtoReflect() protoreflect.Message { + + // Deprecated: Use PendingApproval.ProtoReflect.Descriptor instead. + func (*PendingApproval) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{9} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{10} + } + + func (x *PendingApproval) GetId() string { +@@ -718,13 +806,14 @@ type UsageStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + PromptTokens int32 `protobuf:"varint,1,opt,name=prompt_tokens,json=promptTokens,proto3" json:"prompt_tokens,omitempty"` + CompletionTokens int32 `protobuf:"varint,2,opt,name=completion_tokens,json=completionTokens,proto3" json:"completion_tokens,omitempty"` ++ ToolCalls int32 `protobuf:"varint,3,opt,name=tool_calls,json=toolCalls,proto3" json:"tool_calls,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache + } + + func (x *UsageStats) Reset() { + *x = UsageStats{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[10] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -736,7 +825,7 @@ func (x *UsageStats) String() string { + func (*UsageStats) ProtoMessage() {} + + func (x *UsageStats) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[10] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -749,7 +838,7 @@ func (x *UsageStats) ProtoReflect() protoreflect.Message { + + // Deprecated: Use UsageStats.ProtoReflect.Descriptor instead. + func (*UsageStats) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{10} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{11} + } + + func (x *UsageStats) GetPromptTokens() int32 { +@@ -766,6 +855,13 @@ func (x *UsageStats) GetCompletionTokens() int32 { + return 0 + } + ++func (x *UsageStats) GetToolCalls() int32 { ++ if x != nil { ++ return x.ToolCalls ++ } ++ return 0 ++} ++ + type Condition struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` +@@ -779,7 +875,7 @@ type Condition struct { + + func (x *Condition) Reset() { + *x = Condition{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[11] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -791,7 +887,7 @@ func (x *Condition) String() string { + func (*Condition) ProtoMessage() {} + + func (x *Condition) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[11] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -804,7 +900,7 @@ func (x *Condition) ProtoReflect() protoreflect.Message { + + // Deprecated: Use Condition.ProtoReflect.Descriptor instead. + func (*Condition) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{11} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{12} + } + + func (x *Condition) GetType() string { +@@ -854,7 +950,7 @@ type Gateway struct { + + func (x *Gateway) Reset() { + *x = Gateway{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[12] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -866,7 +962,7 @@ func (x *Gateway) String() string { + func (*Gateway) ProtoMessage() {} + + func (x *Gateway) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[12] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -879,7 +975,7 @@ func (x *Gateway) ProtoReflect() protoreflect.Message { + + // Deprecated: Use Gateway.ProtoReflect.Descriptor instead. + func (*Gateway) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{12} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{13} + } + + func (x *Gateway) GetApiVersion() string { +@@ -920,7 +1016,7 @@ type GatewaySpec struct { + + func (x *GatewaySpec) Reset() { + *x = GatewaySpec{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[13] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -932,7 +1028,7 @@ func (x *GatewaySpec) String() string { + func (*GatewaySpec) ProtoMessage() {} + + func (x *GatewaySpec) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[13] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -945,7 +1041,7 @@ func (x *GatewaySpec) ProtoReflect() protoreflect.Message { + + // Deprecated: Use GatewaySpec.ProtoReflect.Descriptor instead. + func (*GatewaySpec) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{13} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{14} + } + + func (x *GatewaySpec) GetListeners() []*Listener { +@@ -973,7 +1069,7 @@ type Listener struct { + + func (x *Listener) Reset() { + *x = Listener{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[14] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -985,7 +1081,7 @@ func (x *Listener) String() string { + func (*Listener) ProtoMessage() {} + + func (x *Listener) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[14] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -998,7 +1094,7 @@ func (x *Listener) ProtoReflect() protoreflect.Message { + + // Deprecated: Use Listener.ProtoReflect.Descriptor instead. + func (*Listener) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{14} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{15} + } + + func (x *Listener) GetName() string { +@@ -1031,7 +1127,7 @@ type EgressConfig struct { + + func (x *EgressConfig) Reset() { + *x = EgressConfig{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[15] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1043,7 +1139,7 @@ func (x *EgressConfig) String() string { + func (*EgressConfig) ProtoMessage() {} + + func (x *EgressConfig) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[15] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1056,7 +1152,7 @@ func (x *EgressConfig) ProtoReflect() protoreflect.Message { + + // Deprecated: Use EgressConfig.ProtoReflect.Descriptor instead. + func (*EgressConfig) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{15} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{16} + } + + func (x *EgressConfig) GetAllowlist() *EgressAllowlist { +@@ -1075,7 +1171,7 @@ type EgressAllowlist struct { + + func (x *EgressAllowlist) Reset() { + *x = EgressAllowlist{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[16] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1087,7 +1183,7 @@ func (x *EgressAllowlist) String() string { + func (*EgressAllowlist) ProtoMessage() {} + + func (x *EgressAllowlist) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[16] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1100,7 +1196,7 @@ func (x *EgressAllowlist) ProtoReflect() protoreflect.Message { + + // Deprecated: Use EgressAllowlist.ProtoReflect.Descriptor instead. + func (*EgressAllowlist) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{16} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{17} + } + + func (x *EgressAllowlist) GetHosts() []*HostRule { +@@ -1120,7 +1216,7 @@ type HostRule struct { + + func (x *HostRule) Reset() { + *x = HostRule{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[17] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1132,7 +1228,7 @@ func (x *HostRule) String() string { + func (*HostRule) ProtoMessage() {} + + func (x *HostRule) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[17] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1145,7 +1241,7 @@ func (x *HostRule) ProtoReflect() protoreflect.Message { + + // Deprecated: Use HostRule.ProtoReflect.Descriptor instead. + func (*HostRule) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{17} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{18} + } + + func (x *HostRule) GetHost() string { +@@ -1174,7 +1270,7 @@ type Workspace struct { + + func (x *Workspace) Reset() { + *x = Workspace{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[18] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1186,7 +1282,7 @@ func (x *Workspace) String() string { + func (*Workspace) ProtoMessage() {} + + func (x *Workspace) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[18] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1199,7 +1295,7 @@ func (x *Workspace) ProtoReflect() protoreflect.Message { + + // Deprecated: Use Workspace.ProtoReflect.Descriptor instead. + func (*Workspace) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{18} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{19} + } + + func (x *Workspace) GetApiVersion() string { +@@ -1241,7 +1337,7 @@ type WorkspaceSpec struct { + + func (x *WorkspaceSpec) Reset() { + *x = WorkspaceSpec{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[19] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1253,7 +1349,7 @@ func (x *WorkspaceSpec) String() string { + func (*WorkspaceSpec) ProtoMessage() {} + + func (x *WorkspaceSpec) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[19] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1266,7 +1362,7 @@ func (x *WorkspaceSpec) ProtoReflect() protoreflect.Message { + + // Deprecated: Use WorkspaceSpec.ProtoReflect.Descriptor instead. + func (*WorkspaceSpec) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{19} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{20} + } + + func (x *WorkspaceSpec) GetGit() []*GitRepo { +@@ -1303,7 +1399,7 @@ type GitRepo struct { + + func (x *GitRepo) Reset() { + *x = GitRepo{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[20] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1315,7 +1411,7 @@ func (x *GitRepo) String() string { + func (*GitRepo) ProtoMessage() {} + + func (x *GitRepo) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[20] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1328,7 +1424,7 @@ func (x *GitRepo) ProtoReflect() protoreflect.Message { + + // Deprecated: Use GitRepo.ProtoReflect.Descriptor instead. + func (*GitRepo) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{20} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{21} + } + + func (x *GitRepo) GetName() string { +@@ -1376,7 +1472,7 @@ type MCPConfig struct { + + func (x *MCPConfig) Reset() { + *x = MCPConfig{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[21] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1388,7 +1484,7 @@ func (x *MCPConfig) String() string { + func (*MCPConfig) ProtoMessage() {} + + func (x *MCPConfig) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[21] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1401,7 +1497,7 @@ func (x *MCPConfig) ProtoReflect() protoreflect.Message { + + // Deprecated: Use MCPConfig.ProtoReflect.Descriptor instead. + func (*MCPConfig) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{21} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{22} + } + + func (x *MCPConfig) GetRegistries() []*MCPRegistry { +@@ -1430,7 +1526,7 @@ type MCPRegistry struct { + + func (x *MCPRegistry) Reset() { + *x = MCPRegistry{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[22] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1442,7 +1538,7 @@ func (x *MCPRegistry) String() string { + func (*MCPRegistry) ProtoMessage() {} + + func (x *MCPRegistry) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[22] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1455,7 +1551,7 @@ func (x *MCPRegistry) ProtoReflect() protoreflect.Message { + + // Deprecated: Use MCPRegistry.ProtoReflect.Descriptor instead. + func (*MCPRegistry) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{22} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{23} + } + + func (x *MCPRegistry) GetProvider() string { +@@ -1498,7 +1594,7 @@ type MCPServer struct { + + func (x *MCPServer) Reset() { + *x = MCPServer{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[23] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1510,7 +1606,7 @@ func (x *MCPServer) String() string { + func (*MCPServer) ProtoMessage() {} + + func (x *MCPServer) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[23] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1523,7 +1619,7 @@ func (x *MCPServer) ProtoReflect() protoreflect.Message { + + // Deprecated: Use MCPServer.ProtoReflect.Descriptor instead. + func (*MCPServer) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{23} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{24} + } + + func (x *MCPServer) GetName() string { +@@ -1564,7 +1660,7 @@ type SkillsConfig struct { + + func (x *SkillsConfig) Reset() { + *x = SkillsConfig{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[24] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1576,7 +1672,7 @@ func (x *SkillsConfig) String() string { + func (*SkillsConfig) ProtoMessage() {} + + func (x *SkillsConfig) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[24] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1589,7 +1685,7 @@ func (x *SkillsConfig) ProtoReflect() protoreflect.Message { + + // Deprecated: Use SkillsConfig.ProtoReflect.Descriptor instead. + func (*SkillsConfig) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{24} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{25} + } + + func (x *SkillsConfig) GetRegistries() []*SkillRegistry { +@@ -1617,7 +1713,7 @@ type SkillRegistry struct { + + func (x *SkillRegistry) Reset() { + *x = SkillRegistry{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[25] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1629,7 +1725,7 @@ func (x *SkillRegistry) String() string { + func (*SkillRegistry) ProtoMessage() {} + + func (x *SkillRegistry) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[25] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1642,7 +1738,7 @@ func (x *SkillRegistry) ProtoReflect() protoreflect.Message { + + // Deprecated: Use SkillRegistry.ProtoReflect.Descriptor instead. + func (*SkillRegistry) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{25} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{26} + } + + func (x *SkillRegistry) GetProvider() string { +@@ -1678,7 +1774,7 @@ type Model struct { + + func (x *Model) Reset() { + *x = Model{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[26] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1690,7 +1786,7 @@ func (x *Model) String() string { + func (*Model) ProtoMessage() {} + + func (x *Model) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[26] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1703,7 +1799,7 @@ func (x *Model) ProtoReflect() protoreflect.Message { + + // Deprecated: Use Model.ProtoReflect.Descriptor instead. + func (*Model) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{26} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{27} + } + + func (x *Model) GetApiVersion() string { +@@ -1748,7 +1844,7 @@ type ModelSpec struct { + + func (x *ModelSpec) Reset() { + *x = ModelSpec{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[27] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1760,7 +1856,7 @@ func (x *ModelSpec) String() string { + func (*ModelSpec) ProtoMessage() {} + + func (x *ModelSpec) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[27] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1773,7 +1869,7 @@ func (x *ModelSpec) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ModelSpec.ProtoReflect.Descriptor instead. + func (*ModelSpec) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{27} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{28} + } + + func (x *ModelSpec) GetProvider() string { +@@ -1814,7 +1910,7 @@ type SecretKeyRef struct { + + func (x *SecretKeyRef) Reset() { + *x = SecretKeyRef{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[28] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1826,7 +1922,7 @@ func (x *SecretKeyRef) String() string { + func (*SecretKeyRef) ProtoMessage() {} + + func (x *SecretKeyRef) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[28] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1839,7 +1935,7 @@ func (x *SecretKeyRef) ProtoReflect() protoreflect.Message { + + // Deprecated: Use SecretKeyRef.ProtoReflect.Descriptor instead. + func (*SecretKeyRef) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{28} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{29} + } + + func (x *SecretKeyRef) GetName() string { +@@ -1867,7 +1963,7 @@ type GetTaskRequest struct { + + func (x *GetTaskRequest) Reset() { + *x = GetTaskRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[29] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1879,7 +1975,7 @@ func (x *GetTaskRequest) String() string { + func (*GetTaskRequest) ProtoMessage() {} + + func (x *GetTaskRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[29] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1892,7 +1988,7 @@ func (x *GetTaskRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use GetTaskRequest.ProtoReflect.Descriptor instead. + func (*GetTaskRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{29} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{30} + } + + func (x *GetTaskRequest) GetAtespace() string { +@@ -1909,6 +2005,110 @@ func (x *GetTaskRequest) GetName() string { + return "" + } + ++type GetTaskResultRequest struct { ++ state protoimpl.MessageState `protogen:"open.v1"` ++ Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` ++ Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` ++ unknownFields protoimpl.UnknownFields ++ sizeCache protoimpl.SizeCache ++} ++ ++func (x *GetTaskResultRequest) Reset() { ++ *x = GetTaskResultRequest{} ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[31] ++ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ++ ms.StoreMessageInfo(mi) ++} ++ ++func (x *GetTaskResultRequest) String() string { ++ return protoimpl.X.MessageStringOf(x) ++} ++ ++func (*GetTaskResultRequest) ProtoMessage() {} ++ ++func (x *GetTaskResultRequest) ProtoReflect() protoreflect.Message { ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[31] ++ if x != nil { ++ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ++ if ms.LoadMessageInfo() == nil { ++ ms.StoreMessageInfo(mi) ++ } ++ return ms ++ } ++ return mi.MessageOf(x) ++} ++ ++// Deprecated: Use GetTaskResultRequest.ProtoReflect.Descriptor instead. ++func (*GetTaskResultRequest) Descriptor() ([]byte, []int) { ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{31} ++} ++ ++func (x *GetTaskResultRequest) GetAtespace() string { ++ if x != nil { ++ return x.Atespace ++ } ++ return "" ++} ++ ++func (x *GetTaskResultRequest) GetName() string { ++ if x != nil { ++ return x.Name ++ } ++ return "" ++} ++ ++type TaskResult struct { ++ state protoimpl.MessageState `protogen:"open.v1"` ++ Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` ++ Sha256 string `protobuf:"bytes,2,opt,name=sha256,proto3" json:"sha256,omitempty"` ++ unknownFields protoimpl.UnknownFields ++ sizeCache protoimpl.SizeCache ++} ++ ++func (x *TaskResult) Reset() { ++ *x = TaskResult{} ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[32] ++ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ++ ms.StoreMessageInfo(mi) ++} ++ ++func (x *TaskResult) String() string { ++ return protoimpl.X.MessageStringOf(x) ++} ++ ++func (*TaskResult) ProtoMessage() {} ++ ++func (x *TaskResult) ProtoReflect() protoreflect.Message { ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[32] ++ if x != nil { ++ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ++ if ms.LoadMessageInfo() == nil { ++ ms.StoreMessageInfo(mi) ++ } ++ return ms ++ } ++ return mi.MessageOf(x) ++} ++ ++// Deprecated: Use TaskResult.ProtoReflect.Descriptor instead. ++func (*TaskResult) Descriptor() ([]byte, []int) { ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{32} ++} ++ ++func (x *TaskResult) GetContent() []byte { ++ if x != nil { ++ return x.Content ++ } ++ return nil ++} ++ ++func (x *TaskResult) GetSha256() string { ++ if x != nil { ++ return x.Sha256 ++ } ++ return "" ++} ++ + type ListTasksRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` +@@ -1920,7 +2120,7 @@ type ListTasksRequest struct { + + func (x *ListTasksRequest) Reset() { + *x = ListTasksRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[30] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1932,7 +2132,7 @@ func (x *ListTasksRequest) String() string { + func (*ListTasksRequest) ProtoMessage() {} + + func (x *ListTasksRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[30] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -1945,7 +2145,7 @@ func (x *ListTasksRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListTasksRequest.ProtoReflect.Descriptor instead. + func (*ListTasksRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{30} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{33} + } + + func (x *ListTasksRequest) GetAtespace() string { +@@ -1978,7 +2178,7 @@ type ListTasksResponse struct { + + func (x *ListTasksResponse) Reset() { + *x = ListTasksResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[31] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -1990,7 +2190,7 @@ func (x *ListTasksResponse) String() string { + func (*ListTasksResponse) ProtoMessage() {} + + func (x *ListTasksResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[31] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2003,7 +2203,7 @@ func (x *ListTasksResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListTasksResponse.ProtoReflect.Descriptor instead. + func (*ListTasksResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{31} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{34} + } + + func (x *ListTasksResponse) GetTasks() []*Task { +@@ -2022,7 +2222,7 @@ type UpdateTaskRequest struct { + + func (x *UpdateTaskRequest) Reset() { + *x = UpdateTaskRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[32] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2034,7 +2234,7 @@ func (x *UpdateTaskRequest) String() string { + func (*UpdateTaskRequest) ProtoMessage() {} + + func (x *UpdateTaskRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[32] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2047,7 +2247,7 @@ func (x *UpdateTaskRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use UpdateTaskRequest.ProtoReflect.Descriptor instead. + func (*UpdateTaskRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{32} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{35} + } + + func (x *UpdateTaskRequest) GetTask() *Task { +@@ -2067,7 +2267,7 @@ type DeleteTaskRequest struct { + + func (x *DeleteTaskRequest) Reset() { + *x = DeleteTaskRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[33] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2079,7 +2279,7 @@ func (x *DeleteTaskRequest) String() string { + func (*DeleteTaskRequest) ProtoMessage() {} + + func (x *DeleteTaskRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[33] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2092,7 +2292,7 @@ func (x *DeleteTaskRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteTaskRequest.ProtoReflect.Descriptor instead. + func (*DeleteTaskRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{33} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{36} + } + + func (x *DeleteTaskRequest) GetAtespace() string { +@@ -2117,7 +2317,7 @@ type DeleteTaskResponse struct { + + func (x *DeleteTaskResponse) Reset() { + *x = DeleteTaskResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[34] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2129,7 +2329,7 @@ func (x *DeleteTaskResponse) String() string { + func (*DeleteTaskResponse) ProtoMessage() {} + + func (x *DeleteTaskResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[34] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2142,7 +2342,7 @@ func (x *DeleteTaskResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteTaskResponse.ProtoReflect.Descriptor instead. + func (*DeleteTaskResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{34} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{37} + } + + type SuspendTaskRequest struct { +@@ -2155,7 +2355,7 @@ type SuspendTaskRequest struct { + + func (x *SuspendTaskRequest) Reset() { + *x = SuspendTaskRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[35] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2167,7 +2367,7 @@ func (x *SuspendTaskRequest) String() string { + func (*SuspendTaskRequest) ProtoMessage() {} + + func (x *SuspendTaskRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[35] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[38] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2180,7 +2380,7 @@ func (x *SuspendTaskRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use SuspendTaskRequest.ProtoReflect.Descriptor instead. + func (*SuspendTaskRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{35} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{38} + } + + func (x *SuspendTaskRequest) GetAtespace() string { +@@ -2207,7 +2407,7 @@ type ResumeTaskRequest struct { + + func (x *ResumeTaskRequest) Reset() { + *x = ResumeTaskRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[36] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2219,7 +2419,7 @@ func (x *ResumeTaskRequest) String() string { + func (*ResumeTaskRequest) ProtoMessage() {} + + func (x *ResumeTaskRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[36] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[39] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2232,7 +2432,7 @@ func (x *ResumeTaskRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ResumeTaskRequest.ProtoReflect.Descriptor instead. + func (*ResumeTaskRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{36} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{39} + } + + func (x *ResumeTaskRequest) GetAtespace() string { +@@ -2259,7 +2459,7 @@ type WatchTaskRequest struct { + + func (x *WatchTaskRequest) Reset() { + *x = WatchTaskRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[37] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[40] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2271,7 +2471,7 @@ func (x *WatchTaskRequest) String() string { + func (*WatchTaskRequest) ProtoMessage() {} + + func (x *WatchTaskRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[37] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[40] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2284,7 +2484,7 @@ func (x *WatchTaskRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use WatchTaskRequest.ProtoReflect.Descriptor instead. + func (*WatchTaskRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{37} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{40} + } + + func (x *WatchTaskRequest) GetAtespace() string { +@@ -2311,7 +2511,7 @@ type WatchTaskResponse struct { + + func (x *WatchTaskResponse) Reset() { + *x = WatchTaskResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[38] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2323,7 +2523,7 @@ func (x *WatchTaskResponse) String() string { + func (*WatchTaskResponse) ProtoMessage() {} + + func (x *WatchTaskResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[38] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[41] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2336,7 +2536,7 @@ func (x *WatchTaskResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use WatchTaskResponse.ProtoReflect.Descriptor instead. + func (*WatchTaskResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{38} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{41} + } + + func (x *WatchTaskResponse) GetTask() *Task { +@@ -2364,7 +2564,7 @@ type GetGatewayRequest struct { + + func (x *GetGatewayRequest) Reset() { + *x = GetGatewayRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[39] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2376,7 +2576,7 @@ func (x *GetGatewayRequest) String() string { + func (*GetGatewayRequest) ProtoMessage() {} + + func (x *GetGatewayRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[39] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[42] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2389,7 +2589,7 @@ func (x *GetGatewayRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use GetGatewayRequest.ProtoReflect.Descriptor instead. + func (*GetGatewayRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{39} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{42} + } + + func (x *GetGatewayRequest) GetAtespace() string { +@@ -2415,7 +2615,7 @@ type ListGatewaysRequest struct { + + func (x *ListGatewaysRequest) Reset() { + *x = ListGatewaysRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[40] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2427,7 +2627,7 @@ func (x *ListGatewaysRequest) String() string { + func (*ListGatewaysRequest) ProtoMessage() {} + + func (x *ListGatewaysRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[40] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[43] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2440,7 +2640,7 @@ func (x *ListGatewaysRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListGatewaysRequest.ProtoReflect.Descriptor instead. + func (*ListGatewaysRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{40} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{43} + } + + func (x *ListGatewaysRequest) GetAtespace() string { +@@ -2459,7 +2659,7 @@ type ListGatewaysResponse struct { + + func (x *ListGatewaysResponse) Reset() { + *x = ListGatewaysResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[41] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[44] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2471,7 +2671,7 @@ func (x *ListGatewaysResponse) String() string { + func (*ListGatewaysResponse) ProtoMessage() {} + + func (x *ListGatewaysResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[41] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[44] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2484,7 +2684,7 @@ func (x *ListGatewaysResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListGatewaysResponse.ProtoReflect.Descriptor instead. + func (*ListGatewaysResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{41} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{44} + } + + func (x *ListGatewaysResponse) GetGateways() []*Gateway { +@@ -2503,7 +2703,7 @@ type UpdateGatewayRequest struct { + + func (x *UpdateGatewayRequest) Reset() { + *x = UpdateGatewayRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[42] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[45] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2515,7 +2715,7 @@ func (x *UpdateGatewayRequest) String() string { + func (*UpdateGatewayRequest) ProtoMessage() {} + + func (x *UpdateGatewayRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[42] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[45] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2528,7 +2728,7 @@ func (x *UpdateGatewayRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use UpdateGatewayRequest.ProtoReflect.Descriptor instead. + func (*UpdateGatewayRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{42} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{45} + } + + func (x *UpdateGatewayRequest) GetGateway() *Gateway { +@@ -2548,7 +2748,7 @@ type DeleteGatewayRequest struct { + + func (x *DeleteGatewayRequest) Reset() { + *x = DeleteGatewayRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[43] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[46] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2560,7 +2760,7 @@ func (x *DeleteGatewayRequest) String() string { + func (*DeleteGatewayRequest) ProtoMessage() {} + + func (x *DeleteGatewayRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[43] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[46] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2573,7 +2773,7 @@ func (x *DeleteGatewayRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteGatewayRequest.ProtoReflect.Descriptor instead. + func (*DeleteGatewayRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{43} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{46} + } + + func (x *DeleteGatewayRequest) GetAtespace() string { +@@ -2598,7 +2798,7 @@ type DeleteGatewayResponse struct { + + func (x *DeleteGatewayResponse) Reset() { + *x = DeleteGatewayResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[44] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[47] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2610,7 +2810,7 @@ func (x *DeleteGatewayResponse) String() string { + func (*DeleteGatewayResponse) ProtoMessage() {} + + func (x *DeleteGatewayResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[44] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[47] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2623,7 +2823,7 @@ func (x *DeleteGatewayResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteGatewayResponse.ProtoReflect.Descriptor instead. + func (*DeleteGatewayResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{44} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{47} + } + + // Workspaces +@@ -2637,7 +2837,7 @@ type GetWorkspaceRequest struct { + + func (x *GetWorkspaceRequest) Reset() { + *x = GetWorkspaceRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[45] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2649,7 +2849,7 @@ func (x *GetWorkspaceRequest) String() string { + func (*GetWorkspaceRequest) ProtoMessage() {} + + func (x *GetWorkspaceRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[45] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[48] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2662,7 +2862,7 @@ func (x *GetWorkspaceRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use GetWorkspaceRequest.ProtoReflect.Descriptor instead. + func (*GetWorkspaceRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{45} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{48} + } + + func (x *GetWorkspaceRequest) GetAtespace() string { +@@ -2688,7 +2888,7 @@ type ListWorkspacesRequest struct { + + func (x *ListWorkspacesRequest) Reset() { + *x = ListWorkspacesRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[46] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2700,7 +2900,7 @@ func (x *ListWorkspacesRequest) String() string { + func (*ListWorkspacesRequest) ProtoMessage() {} + + func (x *ListWorkspacesRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[46] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[49] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2713,7 +2913,7 @@ func (x *ListWorkspacesRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListWorkspacesRequest.ProtoReflect.Descriptor instead. + func (*ListWorkspacesRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{46} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{49} + } + + func (x *ListWorkspacesRequest) GetAtespace() string { +@@ -2732,7 +2932,7 @@ type ListWorkspacesResponse struct { + + func (x *ListWorkspacesResponse) Reset() { + *x = ListWorkspacesResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[47] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2744,7 +2944,7 @@ func (x *ListWorkspacesResponse) String() string { + func (*ListWorkspacesResponse) ProtoMessage() {} + + func (x *ListWorkspacesResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[47] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[50] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2757,7 +2957,7 @@ func (x *ListWorkspacesResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListWorkspacesResponse.ProtoReflect.Descriptor instead. + func (*ListWorkspacesResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{47} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{50} + } + + func (x *ListWorkspacesResponse) GetWorkspaces() []*Workspace { +@@ -2776,7 +2976,7 @@ type UpdateWorkspaceRequest struct { + + func (x *UpdateWorkspaceRequest) Reset() { + *x = UpdateWorkspaceRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[48] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[51] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2788,7 +2988,7 @@ func (x *UpdateWorkspaceRequest) String() string { + func (*UpdateWorkspaceRequest) ProtoMessage() {} + + func (x *UpdateWorkspaceRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[48] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[51] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2801,7 +3001,7 @@ func (x *UpdateWorkspaceRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use UpdateWorkspaceRequest.ProtoReflect.Descriptor instead. + func (*UpdateWorkspaceRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{48} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{51} + } + + func (x *UpdateWorkspaceRequest) GetWorkspace() *Workspace { +@@ -2821,7 +3021,7 @@ type DeleteWorkspaceRequest struct { + + func (x *DeleteWorkspaceRequest) Reset() { + *x = DeleteWorkspaceRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[49] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[52] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2833,7 +3033,7 @@ func (x *DeleteWorkspaceRequest) String() string { + func (*DeleteWorkspaceRequest) ProtoMessage() {} + + func (x *DeleteWorkspaceRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[49] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[52] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2846,7 +3046,7 @@ func (x *DeleteWorkspaceRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteWorkspaceRequest.ProtoReflect.Descriptor instead. + func (*DeleteWorkspaceRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{49} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{52} + } + + func (x *DeleteWorkspaceRequest) GetAtespace() string { +@@ -2871,7 +3071,7 @@ type DeleteWorkspaceResponse struct { + + func (x *DeleteWorkspaceResponse) Reset() { + *x = DeleteWorkspaceResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[50] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[53] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2883,7 +3083,7 @@ func (x *DeleteWorkspaceResponse) String() string { + func (*DeleteWorkspaceResponse) ProtoMessage() {} + + func (x *DeleteWorkspaceResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[50] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[53] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2896,7 +3096,7 @@ func (x *DeleteWorkspaceResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteWorkspaceResponse.ProtoReflect.Descriptor instead. + func (*DeleteWorkspaceResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{50} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{53} + } + + // Models +@@ -2910,7 +3110,7 @@ type GetModelRequest struct { + + func (x *GetModelRequest) Reset() { + *x = GetModelRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[51] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2922,7 +3122,7 @@ func (x *GetModelRequest) String() string { + func (*GetModelRequest) ProtoMessage() {} + + func (x *GetModelRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[51] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[54] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2935,7 +3135,7 @@ func (x *GetModelRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use GetModelRequest.ProtoReflect.Descriptor instead. + func (*GetModelRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{51} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{54} + } + + func (x *GetModelRequest) GetAtespace() string { +@@ -2961,7 +3161,7 @@ type ListModelsRequest struct { + + func (x *ListModelsRequest) Reset() { + *x = ListModelsRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[52] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[55] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -2973,7 +3173,7 @@ func (x *ListModelsRequest) String() string { + func (*ListModelsRequest) ProtoMessage() {} + + func (x *ListModelsRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[52] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[55] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -2986,7 +3186,7 @@ func (x *ListModelsRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListModelsRequest.ProtoReflect.Descriptor instead. + func (*ListModelsRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{52} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{55} + } + + func (x *ListModelsRequest) GetAtespace() string { +@@ -3005,7 +3205,7 @@ type ListModelsResponse struct { + + func (x *ListModelsResponse) Reset() { + *x = ListModelsResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[53] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -3017,7 +3217,7 @@ func (x *ListModelsResponse) String() string { + func (*ListModelsResponse) ProtoMessage() {} + + func (x *ListModelsResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[53] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -3030,7 +3230,7 @@ func (x *ListModelsResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use ListModelsResponse.ProtoReflect.Descriptor instead. + func (*ListModelsResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{53} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{56} + } + + func (x *ListModelsResponse) GetModels() []*Model { +@@ -3049,7 +3249,7 @@ type UpdateModelRequest struct { + + func (x *UpdateModelRequest) Reset() { + *x = UpdateModelRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[54] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -3061,7 +3261,7 @@ func (x *UpdateModelRequest) String() string { + func (*UpdateModelRequest) ProtoMessage() {} + + func (x *UpdateModelRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[54] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -3074,7 +3274,7 @@ func (x *UpdateModelRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use UpdateModelRequest.ProtoReflect.Descriptor instead. + func (*UpdateModelRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{54} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{57} + } + + func (x *UpdateModelRequest) GetModel() *Model { +@@ -3094,7 +3294,7 @@ type DeleteModelRequest struct { + + func (x *DeleteModelRequest) Reset() { + *x = DeleteModelRequest{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[55] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[58] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -3106,7 +3306,7 @@ func (x *DeleteModelRequest) String() string { + func (*DeleteModelRequest) ProtoMessage() {} + + func (x *DeleteModelRequest) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[55] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[58] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -3119,7 +3319,7 @@ func (x *DeleteModelRequest) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteModelRequest.ProtoReflect.Descriptor instead. + func (*DeleteModelRequest) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{55} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{58} + } + + func (x *DeleteModelRequest) GetAtespace() string { +@@ -3144,7 +3344,7 @@ type DeleteModelResponse struct { + + func (x *DeleteModelResponse) Reset() { + *x = DeleteModelResponse{} +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[56] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[59] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +@@ -3156,7 +3356,7 @@ func (x *DeleteModelResponse) String() string { + func (*DeleteModelResponse) ProtoMessage() {} + + func (x *DeleteModelResponse) ProtoReflect() protoreflect.Message { +- mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[56] ++ mi := &file_pkg_apis_v1alpha1_ax_proto_msgTypes[59] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { +@@ -3169,7 +3369,7 @@ func (x *DeleteModelResponse) ProtoReflect() protoreflect.Message { + + // Deprecated: Use DeleteModelResponse.ProtoReflect.Descriptor instead. + func (*DeleteModelResponse) Descriptor() ([]byte, []int) { +- return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{56} ++ return file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP(), []int{59} + } + + var File_pkg_apis_v1alpha1_ax_proto protoreflect.FileDescriptor +@@ -3218,7 +3418,7 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "\x04goal\x18\x03 \x01(\tR\x04goal\" \n" + + "\n" + + "GatewayRef\x12\x12\n" + +- "\x04name\x18\x01 \x01(\tR\x04name\"\x95\x02\n" + ++ "\x04name\x18\x01 \x01(\tR\x04name\"\xcb\x02\n" + + "\n" + + "TaskStatus\x12\x14\n" + + "\x05phase\x18\x01 \x01(\tR\x05phase\x12\x0e\n" + +@@ -3229,15 +3429,25 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "\x05usage\x18\x06 \x01(\v2\x17.ax.v1alpha1.UsageStatsR\x05usage\x126\n" + + "\n" + + "conditions\x18\a \x03(\v2\x16.ax.v1alpha1.ConditionR\n" + +- "conditions\"x\n" + ++ "conditions\x124\n" + ++ "\acommand\x18\b \x01(\v2\x1a.ax.v1alpha1.CommandStatusR\acommand\"\xc9\x01\n" + ++ "\rCommandStatus\x12\x16\n" + ++ "\x06exited\x18\x01 \x01(\bR\x06exited\x12\x1b\n" + ++ "\texit_code\x18\x02 \x01(\x05R\bexitCode\x12;\n" + ++ "\vfinished_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\n" + ++ "finishedAt\x12!\n" + ++ "\fresult_bytes\x18\x04 \x01(\x03R\vresultBytes\x12#\n" + ++ "\rresult_sha256\x18\x05 \x01(\tR\fresultSha256\"x\n" + + "\x0fPendingApproval\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x16\n" + + "\x06action\x18\x02 \x01(\tR\x06action\x12=\n" + +- "\frequested_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\vrequestedAt\"^\n" + ++ "\frequested_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\vrequestedAt\"}\n" + + "\n" + + "UsageStats\x12#\n" + + "\rprompt_tokens\x18\x01 \x01(\x05R\fpromptTokens\x12+\n" + +- "\x11completion_tokens\x18\x02 \x01(\x05R\x10completionTokens\"\xb7\x01\n" + ++ "\x11completion_tokens\x18\x02 \x01(\x05R\x10completionTokens\x12\x1d\n" + ++ "\n" + ++ "tool_calls\x18\x03 \x01(\x05R\ttoolCalls\"\xb7\x01\n" + + "\tCondition\x12\x12\n" + + "\x04type\x18\x01 \x01(\tR\x04type\x12\x16\n" + + "\x06status\x18\x02 \x01(\tR\x06status\x12L\n" + +@@ -3324,7 +3534,14 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "\x03key\x18\x02 \x01(\tR\x03key\"@\n" + + "\x0eGetTaskRequest\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + +- "\x04name\x18\x02 \x01(\tR\x04name\"\\\n" + ++ "\x04name\x18\x02 \x01(\tR\x04name\"F\n" + ++ "\x14GetTaskResultRequest\x12\x1a\n" + ++ "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + ++ "\x04name\x18\x02 \x01(\tR\x04name\">\n" + ++ "\n" + ++ "TaskResult\x12\x18\n" + ++ "\acontent\x18\x01 \x01(\fR\acontent\x12\x16\n" + ++ "\x06sha256\x18\x02 \x01(\tR\x06sha256\"\\\n" + + "\x10ListTasksRequest\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x14\n" + + "\x05limit\x18\x02 \x01(\x03R\x05limit\x12\x16\n" + +@@ -3389,7 +3606,7 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "\x12DeleteModelRequest\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\"\x15\n" + +- "\x13DeleteModelResponse2\x9e\v\n" + ++ "\x13DeleteModelResponse2\xeb\v\n" + + "\x02AX\x129\n" + + "\aGetTask\x12\x1b.ax.v1alpha1.GetTaskRequest\x1a\x11.ax.v1alpha1.Task\x12J\n" + + "\tListTasks\x12\x1d.ax.v1alpha1.ListTasksRequest\x1a\x1e.ax.v1alpha1.ListTasksResponse\x12?\n" + +@@ -3400,7 +3617,8 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "\vSuspendTask\x12\x1f.ax.v1alpha1.SuspendTaskRequest\x1a\x11.ax.v1alpha1.Task\x12?\n" + + "\n" + + "ResumeTask\x12\x1e.ax.v1alpha1.ResumeTaskRequest\x1a\x11.ax.v1alpha1.Task\x12L\n" + +- "\tWatchTask\x12\x1d.ax.v1alpha1.WatchTaskRequest\x1a\x1e.ax.v1alpha1.WatchTaskResponse0\x01\x12B\n" + ++ "\tWatchTask\x12\x1d.ax.v1alpha1.WatchTaskRequest\x1a\x1e.ax.v1alpha1.WatchTaskResponse0\x01\x12K\n" + ++ "\rGetTaskResult\x12!.ax.v1alpha1.GetTaskResultRequest\x1a\x17.ax.v1alpha1.TaskResult\x12B\n" + + "\n" + + "GetGateway\x12\x1e.ax.v1alpha1.GetGatewayRequest\x1a\x14.ax.v1alpha1.Gateway\x12S\n" + + "\fListGateways\x12 .ax.v1alpha1.ListGatewaysRequest\x1a!.ax.v1alpha1.ListGatewaysResponse\x12H\n" + +@@ -3428,7 +3646,7 @@ func file_pkg_apis_v1alpha1_ax_proto_rawDescGZIP() []byte { + return file_pkg_apis_v1alpha1_ax_proto_rawDescData + } + +-var file_pkg_apis_v1alpha1_ax_proto_msgTypes = make([]protoimpl.MessageInfo, 57) ++var file_pkg_apis_v1alpha1_ax_proto_msgTypes = make([]protoimpl.MessageInfo, 60) + var file_pkg_apis_v1alpha1_ax_proto_goTypes = []any{ + (*ObjectMeta)(nil), // 0: ax.v1alpha1.ObjectMeta + (*Task)(nil), // 1: ax.v1alpha1.Task +@@ -3439,59 +3657,62 @@ var file_pkg_apis_v1alpha1_ax_proto_goTypes = []any{ + (*WorkspaceRef)(nil), // 6: ax.v1alpha1.WorkspaceRef + (*GatewayRef)(nil), // 7: ax.v1alpha1.GatewayRef + (*TaskStatus)(nil), // 8: ax.v1alpha1.TaskStatus +- (*PendingApproval)(nil), // 9: ax.v1alpha1.PendingApproval +- (*UsageStats)(nil), // 10: ax.v1alpha1.UsageStats +- (*Condition)(nil), // 11: ax.v1alpha1.Condition +- (*Gateway)(nil), // 12: ax.v1alpha1.Gateway +- (*GatewaySpec)(nil), // 13: ax.v1alpha1.GatewaySpec +- (*Listener)(nil), // 14: ax.v1alpha1.Listener +- (*EgressConfig)(nil), // 15: ax.v1alpha1.EgressConfig +- (*EgressAllowlist)(nil), // 16: ax.v1alpha1.EgressAllowlist +- (*HostRule)(nil), // 17: ax.v1alpha1.HostRule +- (*Workspace)(nil), // 18: ax.v1alpha1.Workspace +- (*WorkspaceSpec)(nil), // 19: ax.v1alpha1.WorkspaceSpec +- (*GitRepo)(nil), // 20: ax.v1alpha1.GitRepo +- (*MCPConfig)(nil), // 21: ax.v1alpha1.MCPConfig +- (*MCPRegistry)(nil), // 22: ax.v1alpha1.MCPRegistry +- (*MCPServer)(nil), // 23: ax.v1alpha1.MCPServer +- (*SkillsConfig)(nil), // 24: ax.v1alpha1.SkillsConfig +- (*SkillRegistry)(nil), // 25: ax.v1alpha1.SkillRegistry +- (*Model)(nil), // 26: ax.v1alpha1.Model +- (*ModelSpec)(nil), // 27: ax.v1alpha1.ModelSpec +- (*SecretKeyRef)(nil), // 28: ax.v1alpha1.SecretKeyRef +- (*GetTaskRequest)(nil), // 29: ax.v1alpha1.GetTaskRequest +- (*ListTasksRequest)(nil), // 30: ax.v1alpha1.ListTasksRequest +- (*ListTasksResponse)(nil), // 31: ax.v1alpha1.ListTasksResponse +- (*UpdateTaskRequest)(nil), // 32: ax.v1alpha1.UpdateTaskRequest +- (*DeleteTaskRequest)(nil), // 33: ax.v1alpha1.DeleteTaskRequest +- (*DeleteTaskResponse)(nil), // 34: ax.v1alpha1.DeleteTaskResponse +- (*SuspendTaskRequest)(nil), // 35: ax.v1alpha1.SuspendTaskRequest +- (*ResumeTaskRequest)(nil), // 36: ax.v1alpha1.ResumeTaskRequest +- (*WatchTaskRequest)(nil), // 37: ax.v1alpha1.WatchTaskRequest +- (*WatchTaskResponse)(nil), // 38: ax.v1alpha1.WatchTaskResponse +- (*GetGatewayRequest)(nil), // 39: ax.v1alpha1.GetGatewayRequest +- (*ListGatewaysRequest)(nil), // 40: ax.v1alpha1.ListGatewaysRequest +- (*ListGatewaysResponse)(nil), // 41: ax.v1alpha1.ListGatewaysResponse +- (*UpdateGatewayRequest)(nil), // 42: ax.v1alpha1.UpdateGatewayRequest +- (*DeleteGatewayRequest)(nil), // 43: ax.v1alpha1.DeleteGatewayRequest +- (*DeleteGatewayResponse)(nil), // 44: ax.v1alpha1.DeleteGatewayResponse +- (*GetWorkspaceRequest)(nil), // 45: ax.v1alpha1.GetWorkspaceRequest +- (*ListWorkspacesRequest)(nil), // 46: ax.v1alpha1.ListWorkspacesRequest +- (*ListWorkspacesResponse)(nil), // 47: ax.v1alpha1.ListWorkspacesResponse +- (*UpdateWorkspaceRequest)(nil), // 48: ax.v1alpha1.UpdateWorkspaceRequest +- (*DeleteWorkspaceRequest)(nil), // 49: ax.v1alpha1.DeleteWorkspaceRequest +- (*DeleteWorkspaceResponse)(nil), // 50: ax.v1alpha1.DeleteWorkspaceResponse +- (*GetModelRequest)(nil), // 51: ax.v1alpha1.GetModelRequest +- (*ListModelsRequest)(nil), // 52: ax.v1alpha1.ListModelsRequest +- (*ListModelsResponse)(nil), // 53: ax.v1alpha1.ListModelsResponse +- (*UpdateModelRequest)(nil), // 54: ax.v1alpha1.UpdateModelRequest +- (*DeleteModelRequest)(nil), // 55: ax.v1alpha1.DeleteModelRequest +- (*DeleteModelResponse)(nil), // 56: ax.v1alpha1.DeleteModelResponse +- (*timestamppb.Timestamp)(nil), // 57: google.protobuf.Timestamp +- (*structpb.Struct)(nil), // 58: google.protobuf.Struct ++ (*CommandStatus)(nil), // 9: ax.v1alpha1.CommandStatus ++ (*PendingApproval)(nil), // 10: ax.v1alpha1.PendingApproval ++ (*UsageStats)(nil), // 11: ax.v1alpha1.UsageStats ++ (*Condition)(nil), // 12: ax.v1alpha1.Condition ++ (*Gateway)(nil), // 13: ax.v1alpha1.Gateway ++ (*GatewaySpec)(nil), // 14: ax.v1alpha1.GatewaySpec ++ (*Listener)(nil), // 15: ax.v1alpha1.Listener ++ (*EgressConfig)(nil), // 16: ax.v1alpha1.EgressConfig ++ (*EgressAllowlist)(nil), // 17: ax.v1alpha1.EgressAllowlist ++ (*HostRule)(nil), // 18: ax.v1alpha1.HostRule ++ (*Workspace)(nil), // 19: ax.v1alpha1.Workspace ++ (*WorkspaceSpec)(nil), // 20: ax.v1alpha1.WorkspaceSpec ++ (*GitRepo)(nil), // 21: ax.v1alpha1.GitRepo ++ (*MCPConfig)(nil), // 22: ax.v1alpha1.MCPConfig ++ (*MCPRegistry)(nil), // 23: ax.v1alpha1.MCPRegistry ++ (*MCPServer)(nil), // 24: ax.v1alpha1.MCPServer ++ (*SkillsConfig)(nil), // 25: ax.v1alpha1.SkillsConfig ++ (*SkillRegistry)(nil), // 26: ax.v1alpha1.SkillRegistry ++ (*Model)(nil), // 27: ax.v1alpha1.Model ++ (*ModelSpec)(nil), // 28: ax.v1alpha1.ModelSpec ++ (*SecretKeyRef)(nil), // 29: ax.v1alpha1.SecretKeyRef ++ (*GetTaskRequest)(nil), // 30: ax.v1alpha1.GetTaskRequest ++ (*GetTaskResultRequest)(nil), // 31: ax.v1alpha1.GetTaskResultRequest ++ (*TaskResult)(nil), // 32: ax.v1alpha1.TaskResult ++ (*ListTasksRequest)(nil), // 33: ax.v1alpha1.ListTasksRequest ++ (*ListTasksResponse)(nil), // 34: ax.v1alpha1.ListTasksResponse ++ (*UpdateTaskRequest)(nil), // 35: ax.v1alpha1.UpdateTaskRequest ++ (*DeleteTaskRequest)(nil), // 36: ax.v1alpha1.DeleteTaskRequest ++ (*DeleteTaskResponse)(nil), // 37: ax.v1alpha1.DeleteTaskResponse ++ (*SuspendTaskRequest)(nil), // 38: ax.v1alpha1.SuspendTaskRequest ++ (*ResumeTaskRequest)(nil), // 39: ax.v1alpha1.ResumeTaskRequest ++ (*WatchTaskRequest)(nil), // 40: ax.v1alpha1.WatchTaskRequest ++ (*WatchTaskResponse)(nil), // 41: ax.v1alpha1.WatchTaskResponse ++ (*GetGatewayRequest)(nil), // 42: ax.v1alpha1.GetGatewayRequest ++ (*ListGatewaysRequest)(nil), // 43: ax.v1alpha1.ListGatewaysRequest ++ (*ListGatewaysResponse)(nil), // 44: ax.v1alpha1.ListGatewaysResponse ++ (*UpdateGatewayRequest)(nil), // 45: ax.v1alpha1.UpdateGatewayRequest ++ (*DeleteGatewayRequest)(nil), // 46: ax.v1alpha1.DeleteGatewayRequest ++ (*DeleteGatewayResponse)(nil), // 47: ax.v1alpha1.DeleteGatewayResponse ++ (*GetWorkspaceRequest)(nil), // 48: ax.v1alpha1.GetWorkspaceRequest ++ (*ListWorkspacesRequest)(nil), // 49: ax.v1alpha1.ListWorkspacesRequest ++ (*ListWorkspacesResponse)(nil), // 50: ax.v1alpha1.ListWorkspacesResponse ++ (*UpdateWorkspaceRequest)(nil), // 51: ax.v1alpha1.UpdateWorkspaceRequest ++ (*DeleteWorkspaceRequest)(nil), // 52: ax.v1alpha1.DeleteWorkspaceRequest ++ (*DeleteWorkspaceResponse)(nil), // 53: ax.v1alpha1.DeleteWorkspaceResponse ++ (*GetModelRequest)(nil), // 54: ax.v1alpha1.GetModelRequest ++ (*ListModelsRequest)(nil), // 55: ax.v1alpha1.ListModelsRequest ++ (*ListModelsResponse)(nil), // 56: ax.v1alpha1.ListModelsResponse ++ (*UpdateModelRequest)(nil), // 57: ax.v1alpha1.UpdateModelRequest ++ (*DeleteModelRequest)(nil), // 58: ax.v1alpha1.DeleteModelRequest ++ (*DeleteModelResponse)(nil), // 59: ax.v1alpha1.DeleteModelResponse ++ (*timestamppb.Timestamp)(nil), // 60: google.protobuf.Timestamp ++ (*structpb.Struct)(nil), // 61: google.protobuf.Struct + } + var file_pkg_apis_v1alpha1_ax_proto_depIdxs = []int32{ +- 57, // 0: ax.v1alpha1.ObjectMeta.creation_timestamp:type_name -> google.protobuf.Timestamp ++ 60, // 0: ax.v1alpha1.ObjectMeta.creation_timestamp:type_name -> google.protobuf.Timestamp + 0, // 1: ax.v1alpha1.Task.metadata:type_name -> ax.v1alpha1.ObjectMeta + 2, // 2: ax.v1alpha1.Task.spec:type_name -> ax.v1alpha1.TaskSpec + 8, // 3: ax.v1alpha1.Task.status:type_name -> ax.v1alpha1.TaskStatus +@@ -3501,82 +3722,86 @@ var file_pkg_apis_v1alpha1_ax_proto_depIdxs = []int32{ + 7, // 7: ax.v1alpha1.TaskSpec.gateway:type_name -> ax.v1alpha1.GatewayRef + 5, // 8: ax.v1alpha1.ResourceReqs.requests:type_name -> ax.v1alpha1.ResourceList + 5, // 9: ax.v1alpha1.ResourceReqs.limits:type_name -> ax.v1alpha1.ResourceList +- 9, // 10: ax.v1alpha1.TaskStatus.pending_approval:type_name -> ax.v1alpha1.PendingApproval +- 10, // 11: ax.v1alpha1.TaskStatus.usage:type_name -> ax.v1alpha1.UsageStats +- 11, // 12: ax.v1alpha1.TaskStatus.conditions:type_name -> ax.v1alpha1.Condition +- 57, // 13: ax.v1alpha1.PendingApproval.requested_at:type_name -> google.protobuf.Timestamp +- 57, // 14: ax.v1alpha1.Condition.last_transition_time:type_name -> google.protobuf.Timestamp +- 0, // 15: ax.v1alpha1.Gateway.metadata:type_name -> ax.v1alpha1.ObjectMeta +- 13, // 16: ax.v1alpha1.Gateway.spec:type_name -> ax.v1alpha1.GatewaySpec +- 14, // 17: ax.v1alpha1.GatewaySpec.listeners:type_name -> ax.v1alpha1.Listener +- 15, // 18: ax.v1alpha1.GatewaySpec.egress:type_name -> ax.v1alpha1.EgressConfig +- 16, // 19: ax.v1alpha1.EgressConfig.allowlist:type_name -> ax.v1alpha1.EgressAllowlist +- 17, // 20: ax.v1alpha1.EgressAllowlist.hosts:type_name -> ax.v1alpha1.HostRule +- 0, // 21: ax.v1alpha1.Workspace.metadata:type_name -> ax.v1alpha1.ObjectMeta +- 19, // 22: ax.v1alpha1.Workspace.spec:type_name -> ax.v1alpha1.WorkspaceSpec +- 20, // 23: ax.v1alpha1.WorkspaceSpec.git:type_name -> ax.v1alpha1.GitRepo +- 21, // 24: ax.v1alpha1.WorkspaceSpec.mcp:type_name -> ax.v1alpha1.MCPConfig +- 24, // 25: ax.v1alpha1.WorkspaceSpec.skills:type_name -> ax.v1alpha1.SkillsConfig +- 22, // 26: ax.v1alpha1.MCPConfig.registries:type_name -> ax.v1alpha1.MCPRegistry +- 23, // 27: ax.v1alpha1.MCPConfig.servers:type_name -> ax.v1alpha1.MCPServer +- 23, // 28: ax.v1alpha1.MCPRegistry.servers:type_name -> ax.v1alpha1.MCPServer +- 25, // 29: ax.v1alpha1.SkillsConfig.registries:type_name -> ax.v1alpha1.SkillRegistry +- 0, // 30: ax.v1alpha1.Model.metadata:type_name -> ax.v1alpha1.ObjectMeta +- 27, // 31: ax.v1alpha1.Model.spec:type_name -> ax.v1alpha1.ModelSpec +- 28, // 32: ax.v1alpha1.ModelSpec.secret_key:type_name -> ax.v1alpha1.SecretKeyRef +- 58, // 33: ax.v1alpha1.ModelSpec.parameters:type_name -> google.protobuf.Struct +- 1, // 34: ax.v1alpha1.ListTasksResponse.tasks:type_name -> ax.v1alpha1.Task +- 1, // 35: ax.v1alpha1.UpdateTaskRequest.task:type_name -> ax.v1alpha1.Task +- 1, // 36: ax.v1alpha1.WatchTaskResponse.task:type_name -> ax.v1alpha1.Task +- 12, // 37: ax.v1alpha1.ListGatewaysResponse.gateways:type_name -> ax.v1alpha1.Gateway +- 12, // 38: ax.v1alpha1.UpdateGatewayRequest.gateway:type_name -> ax.v1alpha1.Gateway +- 18, // 39: ax.v1alpha1.ListWorkspacesResponse.workspaces:type_name -> ax.v1alpha1.Workspace +- 18, // 40: ax.v1alpha1.UpdateWorkspaceRequest.workspace:type_name -> ax.v1alpha1.Workspace +- 26, // 41: ax.v1alpha1.ListModelsResponse.models:type_name -> ax.v1alpha1.Model +- 26, // 42: ax.v1alpha1.UpdateModelRequest.model:type_name -> ax.v1alpha1.Model +- 29, // 43: ax.v1alpha1.AX.GetTask:input_type -> ax.v1alpha1.GetTaskRequest +- 30, // 44: ax.v1alpha1.AX.ListTasks:input_type -> ax.v1alpha1.ListTasksRequest +- 32, // 45: ax.v1alpha1.AX.UpdateTask:input_type -> ax.v1alpha1.UpdateTaskRequest +- 33, // 46: ax.v1alpha1.AX.DeleteTask:input_type -> ax.v1alpha1.DeleteTaskRequest +- 35, // 47: ax.v1alpha1.AX.SuspendTask:input_type -> ax.v1alpha1.SuspendTaskRequest +- 36, // 48: ax.v1alpha1.AX.ResumeTask:input_type -> ax.v1alpha1.ResumeTaskRequest +- 37, // 49: ax.v1alpha1.AX.WatchTask:input_type -> ax.v1alpha1.WatchTaskRequest +- 39, // 50: ax.v1alpha1.AX.GetGateway:input_type -> ax.v1alpha1.GetGatewayRequest +- 40, // 51: ax.v1alpha1.AX.ListGateways:input_type -> ax.v1alpha1.ListGatewaysRequest +- 42, // 52: ax.v1alpha1.AX.UpdateGateway:input_type -> ax.v1alpha1.UpdateGatewayRequest +- 43, // 53: ax.v1alpha1.AX.DeleteGateway:input_type -> ax.v1alpha1.DeleteGatewayRequest +- 45, // 54: ax.v1alpha1.AX.GetWorkspace:input_type -> ax.v1alpha1.GetWorkspaceRequest +- 46, // 55: ax.v1alpha1.AX.ListWorkspaces:input_type -> ax.v1alpha1.ListWorkspacesRequest +- 48, // 56: ax.v1alpha1.AX.UpdateWorkspace:input_type -> ax.v1alpha1.UpdateWorkspaceRequest +- 49, // 57: ax.v1alpha1.AX.DeleteWorkspace:input_type -> ax.v1alpha1.DeleteWorkspaceRequest +- 51, // 58: ax.v1alpha1.AX.GetModel:input_type -> ax.v1alpha1.GetModelRequest +- 52, // 59: ax.v1alpha1.AX.ListModels:input_type -> ax.v1alpha1.ListModelsRequest +- 54, // 60: ax.v1alpha1.AX.UpdateModel:input_type -> ax.v1alpha1.UpdateModelRequest +- 55, // 61: ax.v1alpha1.AX.DeleteModel:input_type -> ax.v1alpha1.DeleteModelRequest +- 1, // 62: ax.v1alpha1.AX.GetTask:output_type -> ax.v1alpha1.Task +- 31, // 63: ax.v1alpha1.AX.ListTasks:output_type -> ax.v1alpha1.ListTasksResponse +- 1, // 64: ax.v1alpha1.AX.UpdateTask:output_type -> ax.v1alpha1.Task +- 34, // 65: ax.v1alpha1.AX.DeleteTask:output_type -> ax.v1alpha1.DeleteTaskResponse +- 1, // 66: ax.v1alpha1.AX.SuspendTask:output_type -> ax.v1alpha1.Task +- 1, // 67: ax.v1alpha1.AX.ResumeTask:output_type -> ax.v1alpha1.Task +- 38, // 68: ax.v1alpha1.AX.WatchTask:output_type -> ax.v1alpha1.WatchTaskResponse +- 12, // 69: ax.v1alpha1.AX.GetGateway:output_type -> ax.v1alpha1.Gateway +- 41, // 70: ax.v1alpha1.AX.ListGateways:output_type -> ax.v1alpha1.ListGatewaysResponse +- 12, // 71: ax.v1alpha1.AX.UpdateGateway:output_type -> ax.v1alpha1.Gateway +- 44, // 72: ax.v1alpha1.AX.DeleteGateway:output_type -> ax.v1alpha1.DeleteGatewayResponse +- 18, // 73: ax.v1alpha1.AX.GetWorkspace:output_type -> ax.v1alpha1.Workspace +- 47, // 74: ax.v1alpha1.AX.ListWorkspaces:output_type -> ax.v1alpha1.ListWorkspacesResponse +- 18, // 75: ax.v1alpha1.AX.UpdateWorkspace:output_type -> ax.v1alpha1.Workspace +- 50, // 76: ax.v1alpha1.AX.DeleteWorkspace:output_type -> ax.v1alpha1.DeleteWorkspaceResponse +- 26, // 77: ax.v1alpha1.AX.GetModel:output_type -> ax.v1alpha1.Model +- 53, // 78: ax.v1alpha1.AX.ListModels:output_type -> ax.v1alpha1.ListModelsResponse +- 26, // 79: ax.v1alpha1.AX.UpdateModel:output_type -> ax.v1alpha1.Model +- 56, // 80: ax.v1alpha1.AX.DeleteModel:output_type -> ax.v1alpha1.DeleteModelResponse +- 62, // [62:81] is the sub-list for method output_type +- 43, // [43:62] is the sub-list for method input_type +- 43, // [43:43] is the sub-list for extension type_name +- 43, // [43:43] is the sub-list for extension extendee +- 0, // [0:43] is the sub-list for field type_name ++ 10, // 10: ax.v1alpha1.TaskStatus.pending_approval:type_name -> ax.v1alpha1.PendingApproval ++ 11, // 11: ax.v1alpha1.TaskStatus.usage:type_name -> ax.v1alpha1.UsageStats ++ 12, // 12: ax.v1alpha1.TaskStatus.conditions:type_name -> ax.v1alpha1.Condition ++ 9, // 13: ax.v1alpha1.TaskStatus.command:type_name -> ax.v1alpha1.CommandStatus ++ 60, // 14: ax.v1alpha1.CommandStatus.finished_at:type_name -> google.protobuf.Timestamp ++ 60, // 15: ax.v1alpha1.PendingApproval.requested_at:type_name -> google.protobuf.Timestamp ++ 60, // 16: ax.v1alpha1.Condition.last_transition_time:type_name -> google.protobuf.Timestamp ++ 0, // 17: ax.v1alpha1.Gateway.metadata:type_name -> ax.v1alpha1.ObjectMeta ++ 14, // 18: ax.v1alpha1.Gateway.spec:type_name -> ax.v1alpha1.GatewaySpec ++ 15, // 19: ax.v1alpha1.GatewaySpec.listeners:type_name -> ax.v1alpha1.Listener ++ 16, // 20: ax.v1alpha1.GatewaySpec.egress:type_name -> ax.v1alpha1.EgressConfig ++ 17, // 21: ax.v1alpha1.EgressConfig.allowlist:type_name -> ax.v1alpha1.EgressAllowlist ++ 18, // 22: ax.v1alpha1.EgressAllowlist.hosts:type_name -> ax.v1alpha1.HostRule ++ 0, // 23: ax.v1alpha1.Workspace.metadata:type_name -> ax.v1alpha1.ObjectMeta ++ 20, // 24: ax.v1alpha1.Workspace.spec:type_name -> ax.v1alpha1.WorkspaceSpec ++ 21, // 25: ax.v1alpha1.WorkspaceSpec.git:type_name -> ax.v1alpha1.GitRepo ++ 22, // 26: ax.v1alpha1.WorkspaceSpec.mcp:type_name -> ax.v1alpha1.MCPConfig ++ 25, // 27: ax.v1alpha1.WorkspaceSpec.skills:type_name -> ax.v1alpha1.SkillsConfig ++ 23, // 28: ax.v1alpha1.MCPConfig.registries:type_name -> ax.v1alpha1.MCPRegistry ++ 24, // 29: ax.v1alpha1.MCPConfig.servers:type_name -> ax.v1alpha1.MCPServer ++ 24, // 30: ax.v1alpha1.MCPRegistry.servers:type_name -> ax.v1alpha1.MCPServer ++ 26, // 31: ax.v1alpha1.SkillsConfig.registries:type_name -> ax.v1alpha1.SkillRegistry ++ 0, // 32: ax.v1alpha1.Model.metadata:type_name -> ax.v1alpha1.ObjectMeta ++ 28, // 33: ax.v1alpha1.Model.spec:type_name -> ax.v1alpha1.ModelSpec ++ 29, // 34: ax.v1alpha1.ModelSpec.secret_key:type_name -> ax.v1alpha1.SecretKeyRef ++ 61, // 35: ax.v1alpha1.ModelSpec.parameters:type_name -> google.protobuf.Struct ++ 1, // 36: ax.v1alpha1.ListTasksResponse.tasks:type_name -> ax.v1alpha1.Task ++ 1, // 37: ax.v1alpha1.UpdateTaskRequest.task:type_name -> ax.v1alpha1.Task ++ 1, // 38: ax.v1alpha1.WatchTaskResponse.task:type_name -> ax.v1alpha1.Task ++ 13, // 39: ax.v1alpha1.ListGatewaysResponse.gateways:type_name -> ax.v1alpha1.Gateway ++ 13, // 40: ax.v1alpha1.UpdateGatewayRequest.gateway:type_name -> ax.v1alpha1.Gateway ++ 19, // 41: ax.v1alpha1.ListWorkspacesResponse.workspaces:type_name -> ax.v1alpha1.Workspace ++ 19, // 42: ax.v1alpha1.UpdateWorkspaceRequest.workspace:type_name -> ax.v1alpha1.Workspace ++ 27, // 43: ax.v1alpha1.ListModelsResponse.models:type_name -> ax.v1alpha1.Model ++ 27, // 44: ax.v1alpha1.UpdateModelRequest.model:type_name -> ax.v1alpha1.Model ++ 30, // 45: ax.v1alpha1.AX.GetTask:input_type -> ax.v1alpha1.GetTaskRequest ++ 33, // 46: ax.v1alpha1.AX.ListTasks:input_type -> ax.v1alpha1.ListTasksRequest ++ 35, // 47: ax.v1alpha1.AX.UpdateTask:input_type -> ax.v1alpha1.UpdateTaskRequest ++ 36, // 48: ax.v1alpha1.AX.DeleteTask:input_type -> ax.v1alpha1.DeleteTaskRequest ++ 38, // 49: ax.v1alpha1.AX.SuspendTask:input_type -> ax.v1alpha1.SuspendTaskRequest ++ 39, // 50: ax.v1alpha1.AX.ResumeTask:input_type -> ax.v1alpha1.ResumeTaskRequest ++ 40, // 51: ax.v1alpha1.AX.WatchTask:input_type -> ax.v1alpha1.WatchTaskRequest ++ 31, // 52: ax.v1alpha1.AX.GetTaskResult:input_type -> ax.v1alpha1.GetTaskResultRequest ++ 42, // 53: ax.v1alpha1.AX.GetGateway:input_type -> ax.v1alpha1.GetGatewayRequest ++ 43, // 54: ax.v1alpha1.AX.ListGateways:input_type -> ax.v1alpha1.ListGatewaysRequest ++ 45, // 55: ax.v1alpha1.AX.UpdateGateway:input_type -> ax.v1alpha1.UpdateGatewayRequest ++ 46, // 56: ax.v1alpha1.AX.DeleteGateway:input_type -> ax.v1alpha1.DeleteGatewayRequest ++ 48, // 57: ax.v1alpha1.AX.GetWorkspace:input_type -> ax.v1alpha1.GetWorkspaceRequest ++ 49, // 58: ax.v1alpha1.AX.ListWorkspaces:input_type -> ax.v1alpha1.ListWorkspacesRequest ++ 51, // 59: ax.v1alpha1.AX.UpdateWorkspace:input_type -> ax.v1alpha1.UpdateWorkspaceRequest ++ 52, // 60: ax.v1alpha1.AX.DeleteWorkspace:input_type -> ax.v1alpha1.DeleteWorkspaceRequest ++ 54, // 61: ax.v1alpha1.AX.GetModel:input_type -> ax.v1alpha1.GetModelRequest ++ 55, // 62: ax.v1alpha1.AX.ListModels:input_type -> ax.v1alpha1.ListModelsRequest ++ 57, // 63: ax.v1alpha1.AX.UpdateModel:input_type -> ax.v1alpha1.UpdateModelRequest ++ 58, // 64: ax.v1alpha1.AX.DeleteModel:input_type -> ax.v1alpha1.DeleteModelRequest ++ 1, // 65: ax.v1alpha1.AX.GetTask:output_type -> ax.v1alpha1.Task ++ 34, // 66: ax.v1alpha1.AX.ListTasks:output_type -> ax.v1alpha1.ListTasksResponse ++ 1, // 67: ax.v1alpha1.AX.UpdateTask:output_type -> ax.v1alpha1.Task ++ 37, // 68: ax.v1alpha1.AX.DeleteTask:output_type -> ax.v1alpha1.DeleteTaskResponse ++ 1, // 69: ax.v1alpha1.AX.SuspendTask:output_type -> ax.v1alpha1.Task ++ 1, // 70: ax.v1alpha1.AX.ResumeTask:output_type -> ax.v1alpha1.Task ++ 41, // 71: ax.v1alpha1.AX.WatchTask:output_type -> ax.v1alpha1.WatchTaskResponse ++ 32, // 72: ax.v1alpha1.AX.GetTaskResult:output_type -> ax.v1alpha1.TaskResult ++ 13, // 73: ax.v1alpha1.AX.GetGateway:output_type -> ax.v1alpha1.Gateway ++ 44, // 74: ax.v1alpha1.AX.ListGateways:output_type -> ax.v1alpha1.ListGatewaysResponse ++ 13, // 75: ax.v1alpha1.AX.UpdateGateway:output_type -> ax.v1alpha1.Gateway ++ 47, // 76: ax.v1alpha1.AX.DeleteGateway:output_type -> ax.v1alpha1.DeleteGatewayResponse ++ 19, // 77: ax.v1alpha1.AX.GetWorkspace:output_type -> ax.v1alpha1.Workspace ++ 50, // 78: ax.v1alpha1.AX.ListWorkspaces:output_type -> ax.v1alpha1.ListWorkspacesResponse ++ 19, // 79: ax.v1alpha1.AX.UpdateWorkspace:output_type -> ax.v1alpha1.Workspace ++ 53, // 80: ax.v1alpha1.AX.DeleteWorkspace:output_type -> ax.v1alpha1.DeleteWorkspaceResponse ++ 27, // 81: ax.v1alpha1.AX.GetModel:output_type -> ax.v1alpha1.Model ++ 56, // 82: ax.v1alpha1.AX.ListModels:output_type -> ax.v1alpha1.ListModelsResponse ++ 27, // 83: ax.v1alpha1.AX.UpdateModel:output_type -> ax.v1alpha1.Model ++ 59, // 84: ax.v1alpha1.AX.DeleteModel:output_type -> ax.v1alpha1.DeleteModelResponse ++ 65, // [65:85] is the sub-list for method output_type ++ 45, // [45:65] is the sub-list for method input_type ++ 45, // [45:45] is the sub-list for extension type_name ++ 45, // [45:45] is the sub-list for extension extendee ++ 0, // [0:45] is the sub-list for field type_name + } + + func init() { file_pkg_apis_v1alpha1_ax_proto_init() } +@@ -3590,7 +3815,7 @@ func file_pkg_apis_v1alpha1_ax_proto_init() { + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_pkg_apis_v1alpha1_ax_proto_rawDesc), len(file_pkg_apis_v1alpha1_ax_proto_rawDesc)), + NumEnums: 0, +- NumMessages: 57, ++ NumMessages: 60, + NumExtensions: 0, + NumServices: 1, + }, +diff --git a/pkg/apis/v1alpha1/ax.proto b/pkg/apis/v1alpha1/ax.proto +index cf35316..072305b 100644 +--- a/pkg/apis/v1alpha1/ax.proto ++++ b/pkg/apis/v1alpha1/ax.proto +@@ -34,6 +34,9 @@ service AX { + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); ++ // GetTaskResult returns the result file the task's command wrote, as copied ++ // by the controller when the command exited. NotFound until then. ++ rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); +@@ -143,6 +146,19 @@ message TaskStatus { + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; ++ // command reports how the task's command finished. Unset while it runs. ++ CommandStatus command = 8; ++} ++ ++// CommandStatus is written by the controller once the runner reports that the ++// task's command exited. The result content itself is served by GetTaskResult. ++message CommandStatus { ++ bool exited = 1; ++ int32 exit_code = 2; ++ google.protobuf.Timestamp finished_at = 3; ++ // result_bytes is the size of the copied result, 0 when there was none. ++ int64 result_bytes = 4; ++ string result_sha256 = 5; + } + + message PendingApproval { +@@ -154,6 +170,7 @@ message PendingApproval { + message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; ++ int32 tool_calls = 3; + } + + message Condition { +@@ -285,6 +302,16 @@ message GetTaskRequest { + string name = 2; + } + ++message GetTaskResultRequest { ++ string atespace = 1; ++ string name = 2; ++} ++ ++message TaskResult { ++ bytes content = 1; ++ string sha256 = 2; ++} ++ + message ListTasksRequest { + string atespace = 1; + int64 limit = 2; +diff --git a/pkg/apis/v1alpha1/ax_grpc.pb.go b/pkg/apis/v1alpha1/ax_grpc.pb.go +index d106157..a8b2ca8 100644 +--- a/pkg/apis/v1alpha1/ax_grpc.pb.go ++++ b/pkg/apis/v1alpha1/ax_grpc.pb.go +@@ -14,8 +14,8 @@ + + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. + // versions: +-// - protoc-gen-go-grpc v1.6.0 +-// - protoc v7.34.1 ++// - protoc-gen-go-grpc v1.6.2 ++// - protoc v7.35.1 + // source: pkg/apis/v1alpha1/ax.proto + + package v1alpha1 +@@ -40,6 +40,7 @@ const ( + AX_SuspendTask_FullMethodName = "/ax.v1alpha1.AX/SuspendTask" + AX_ResumeTask_FullMethodName = "/ax.v1alpha1.AX/ResumeTask" + AX_WatchTask_FullMethodName = "/ax.v1alpha1.AX/WatchTask" ++ AX_GetTaskResult_FullMethodName = "/ax.v1alpha1.AX/GetTaskResult" + AX_GetGateway_FullMethodName = "/ax.v1alpha1.AX/GetGateway" + AX_ListGateways_FullMethodName = "/ax.v1alpha1.AX/ListGateways" + AX_UpdateGateway_FullMethodName = "/ax.v1alpha1.AX/UpdateGateway" +@@ -68,6 +69,9 @@ type AXClient interface { + SuspendTask(ctx context.Context, in *SuspendTaskRequest, opts ...grpc.CallOption) (*Task, error) + ResumeTask(ctx context.Context, in *ResumeTaskRequest, opts ...grpc.CallOption) (*Task, error) + WatchTask(ctx context.Context, in *WatchTaskRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WatchTaskResponse], error) ++ // GetTaskResult returns the result file the task's command wrote, as copied ++ // by the controller when the command exited. NotFound until then. ++ GetTaskResult(ctx context.Context, in *GetTaskResultRequest, opts ...grpc.CallOption) (*TaskResult, error) + // Gateways + GetGateway(ctx context.Context, in *GetGatewayRequest, opts ...grpc.CallOption) (*Gateway, error) + ListGateways(ctx context.Context, in *ListGatewaysRequest, opts ...grpc.CallOption) (*ListGatewaysResponse, error) +@@ -172,6 +176,16 @@ func (c *aXClient) WatchTask(ctx context.Context, in *WatchTaskRequest, opts ... + // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. + type AX_WatchTaskClient = grpc.ServerStreamingClient[WatchTaskResponse] + ++func (c *aXClient) GetTaskResult(ctx context.Context, in *GetTaskResultRequest, opts ...grpc.CallOption) (*TaskResult, error) { ++ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) ++ out := new(TaskResult) ++ err := c.cc.Invoke(ctx, AX_GetTaskResult_FullMethodName, in, out, cOpts...) ++ if err != nil { ++ return nil, err ++ } ++ return out, nil ++} ++ + func (c *aXClient) GetGateway(ctx context.Context, in *GetGatewayRequest, opts ...grpc.CallOption) (*Gateway, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Gateway) +@@ -306,6 +320,9 @@ type AXServer interface { + SuspendTask(context.Context, *SuspendTaskRequest) (*Task, error) + ResumeTask(context.Context, *ResumeTaskRequest) (*Task, error) + WatchTask(*WatchTaskRequest, grpc.ServerStreamingServer[WatchTaskResponse]) error ++ // GetTaskResult returns the result file the task's command wrote, as copied ++ // by the controller when the command exited. NotFound until then. ++ GetTaskResult(context.Context, *GetTaskResultRequest) (*TaskResult, error) + // Gateways + GetGateway(context.Context, *GetGatewayRequest) (*Gateway, error) + ListGateways(context.Context, *ListGatewaysRequest) (*ListGatewaysResponse, error) +@@ -352,6 +369,9 @@ func (UnimplementedAXServer) ResumeTask(context.Context, *ResumeTaskRequest) (*T + func (UnimplementedAXServer) WatchTask(*WatchTaskRequest, grpc.ServerStreamingServer[WatchTaskResponse]) error { + return status.Error(codes.Unimplemented, "method WatchTask not implemented") + } ++func (UnimplementedAXServer) GetTaskResult(context.Context, *GetTaskResultRequest) (*TaskResult, error) { ++ return nil, status.Error(codes.Unimplemented, "method GetTaskResult not implemented") ++} + func (UnimplementedAXServer) GetGateway(context.Context, *GetGatewayRequest) (*Gateway, error) { + return nil, status.Error(codes.Unimplemented, "method GetGateway not implemented") + } +@@ -528,6 +548,24 @@ func _AX_WatchTask_Handler(srv interface{}, stream grpc.ServerStream) error { + // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. + type AX_WatchTaskServer = grpc.ServerStreamingServer[WatchTaskResponse] + ++func _AX_GetTaskResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { ++ in := new(GetTaskResultRequest) ++ if err := dec(in); err != nil { ++ return nil, err ++ } ++ if interceptor == nil { ++ return srv.(AXServer).GetTaskResult(ctx, in) ++ } ++ info := &grpc.UnaryServerInfo{ ++ Server: srv, ++ FullMethod: AX_GetTaskResult_FullMethodName, ++ } ++ handler := func(ctx context.Context, req interface{}) (interface{}, error) { ++ return srv.(AXServer).GetTaskResult(ctx, req.(*GetTaskResultRequest)) ++ } ++ return interceptor(ctx, in, info, handler) ++} ++ + func _AX_GetGateway_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetGatewayRequest) + if err := dec(in); err != nil { +@@ -775,6 +813,10 @@ var AX_ServiceDesc = grpc.ServiceDesc{ + MethodName: "ResumeTask", + Handler: _AX_ResumeTask_Handler, + }, ++ { ++ MethodName: "GetTaskResult", ++ Handler: _AX_GetTaskResult_Handler, ++ }, + { + MethodName: "GetGateway", + Handler: _AX_GetGateway_Handler, +diff --git a/runner/exit_test.go b/runner/exit_test.go +new file mode 100644 +index 0000000..a340da9 +--- /dev/null ++++ b/runner/exit_test.go +@@ -0,0 +1,65 @@ ++// Copyright 2026 Google LLC ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++package runner_test ++ ++import ( ++ "encoding/json" ++ "fmt" ++ "io" ++ "net/http" ++ "testing" ++ "time" ++ ++ "github.com/google/ax/internal/metadata" ++) ++ ++// The runner reports the command's exit and serves its result file from the ++// default path under the workspace, which is what the controller reads back. ++func TestRun_ServesExitAndResult(t *testing.T) { ++ h := newHarness(t) ++ h.task.Spec.Command = []string{"/bin/sh", "-c", `mkdir -p .ax && printf '{"ok":1}' > .ax/result.json && exit 5`} ++ h.start(t) ++ ++ select { ++ case e := <-h.exited: ++ if e.ExitCode != 5 { ++ t.Fatalf("exit code = %d, want 5", e.ExitCode) ++ } ++ case <-time.After(10 * time.Second): ++ t.Fatal("command did not exit") ++ } ++ ++ base := fmt.Sprintf("http://127.0.0.1:%d/metadata/v1alpha1/ax", h.cfg.Port) ++ resp, err := http.Get(base + "/exit") ++ if err != nil { ++ t.Fatal(err) ++ } ++ var st metadata.CommandExitStatus ++ err = json.NewDecoder(resp.Body).Decode(&st) ++ resp.Body.Close() ++ if err != nil || !st.Exited || st.ExitCode != 5 { ++ t.Fatalf("exit status = %+v, %v", st, err) ++ } ++ ++ resp, err = http.Get(base + "/result") ++ if err != nil { ++ t.Fatal(err) ++ } ++ body, _ := io.ReadAll(resp.Body) ++ resp.Body.Close() ++ if resp.StatusCode != http.StatusOK || string(body) != `{"ok":1}` { ++ t.Fatalf("result = %d %q", resp.StatusCode, body) ++ } ++} +diff --git a/runner/runner.go b/runner/runner.go +index 4eaadb0..7ec8da9 100644 +--- a/runner/runner.go ++++ b/runner/runner.go +@@ -28,6 +28,7 @@ import ( + "log/slog" + "os" + "os/exec" ++ "path/filepath" + "syscall" + "time" + +@@ -44,6 +45,12 @@ const ( + // binds no workspaces. + DefaultWorkspacePath = v1alpha1.DefaultWorkspacePath + ++ // DefaultResultPath is where the task command writes its result, relative ++ // to the first workspace, unless AX_RESULT_PATH says otherwise. ++ DefaultResultPath = ".ax/result.json" ++ // DefaultUsagePath is where the task command writes its usage JSON. ++ DefaultUsagePath = ".ax/usage.json" ++ + // stopGracePeriod is how long a running task command gets to exit after + // SIGTERM before it is killed during shutdown. + stopGracePeriod = 10 * time.Second +@@ -143,7 +150,11 @@ func Run(ctx context.Context, cfg Config) error { + _ = os.Setenv(e.GetName(), e.GetValue()) + } + +- metaServer := metadata.NewServer(port, cfg.Task, workspaces, metadata.ServerOptions{WorkspacePath: wsPath}) ++ metaServer := metadata.NewServer(port, cfg.Task, workspaces, metadata.ServerOptions{ ++ WorkspacePath: wsPath, ++ ResultPath: outputPath(wsPath, DefaultResultPath, "AX_RESULT_PATH", "AX_CONWIP_RESULT_PATH"), ++ UsagePath: outputPath(wsPath, DefaultUsagePath, "AX_USAGE_PATH", "AX_CONWIP_USAGE_PATH"), ++ }) + if err := metaServer.Start(); err != nil { + return fmt.Errorf("starting metadata server: %w", err) + } +@@ -191,6 +202,13 @@ func Run(ctx context.Context, cfg Config) error { + + select { + case err := <-exited: ++ // Only a command that exited on its own is reported as finished; a ++ // command stopped at shutdown is not, so a suspend never reads as exit. ++ code := -1 ++ if cmd.ProcessState != nil { ++ code = cmd.ProcessState.ExitCode() ++ } ++ metaServer.SetCommandExit(code, time.Now()) + reportExit(cfg, cmd, err) + // Keep the sandbox up and inspectable until told to stop. + <-ctx.Done() +@@ -244,3 +262,19 @@ func commandEnv(task *v1alpha1.Task, port int) []string { + } + return env + } ++ ++// outputPath resolves a command output file: the first of envVars that is set, ++// else def; a relative path is taken relative to the workspace. ++func outputPath(wsPath, def string, envVars ...string) string { ++ p := def ++ for _, v := range envVars { ++ if val := os.Getenv(v); val != "" { ++ p = val ++ break ++ } ++ } ++ if !filepath.IsAbs(p) { ++ p = filepath.Join(wsPath, p) ++ } ++ return p ++} diff --git a/pkgs/ax/patches/sandbox-class.patch b/pkgs/ax/patches/sandbox-class.patch new file mode 100644 index 000000000..68675ea94 --- /dev/null +++ b/pkgs/ax/patches/sandbox-class.patch @@ -0,0 +1,304 @@ +diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go +index 31c399b..c4aebda 100644 +--- a/internal/controller/reconciler.go ++++ b/internal/controller/reconciler.go +@@ -164,11 +164,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, gat + } + + // If a custom image, workspace, or extra environment is specified, provision or use a dedicated ActorTemplate +- if task.Spec != nil && (task.Spec.Image != "" || len(extraEnv) > 0) { ++ if task.Spec != nil && (task.Spec.Image != "" || task.Spec.GetSandboxClass() != "" || len(extraEnv) > 0) { + slog.Info("ensuring custom ActorTemplate for task", "image", task.Spec.Image) +- customTemplateName := taskTemplateName(task.Metadata.Name, task.Spec.Image, extraEnv) ++ customTemplateName := taskTemplateName(task.Metadata.Name, task.Spec.Image, task.Spec.GetSandboxClass(), extraEnv) + +- tmpl, err := r.client.EnsureActorTemplateWithImage(ctx, templateAtespace, templateName, atespace, customTemplateName, task.Spec.Image, extraEnv) ++ tmpl, err := r.client.EnsureActorTemplateWithImage(ctx, templateAtespace, templateName, atespace, customTemplateName, task.Spec.Image, task.Spec.GetSandboxClass(), extraEnv) + if err != nil { + slog.Warn("could not create custom ActorTemplate, falling back to default template", "error", err) + } else if tmpl != nil && tmpl.Metadata != nil { +@@ -386,9 +386,14 @@ func (r *TaskReconciler) lookupGeminiKey(ctx context.Context, atespace string) s + + // taskTemplateName derives the per-task ActorTemplate name from the task name and a + // digest of the image and container environment, so a spec change yields a new template. +-func taskTemplateName(taskName, image string, env map[string]string) string { ++func taskTemplateName(taskName, image, sandboxClass string, env map[string]string) string { + h := sha256.New() + h.Write([]byte(image)) ++ // Only mixed in when set, so that a task that does not name a sandbox class ++ // keeps the template name it had before the field existed. ++ if sandboxClass != "" { ++ h.Write([]byte("sandboxClass=" + sandboxClass + ";")) ++ } + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) +diff --git a/internal/substrate/client.go b/internal/substrate/client.go +index bacb078..2840ce2 100644 +--- a/internal/substrate/client.go ++++ b/internal/substrate/client.go +@@ -210,7 +210,7 @@ const ( + ) + + // BuildActorTemplate constructs a Substrate ActorTemplate based on the standard ate-env specification. +-func BuildActorTemplate(atespace, name, image string, envMap map[string]string, command []string, snapshotsBucket string) *ateapipb.ActorTemplate { ++func BuildActorTemplate(atespace, name, image string, envMap map[string]string, command []string, snapshotsBucket, sandboxClass string) *ateapipb.ActorTemplate { + if atespace == "" { + atespace = "default" + } +@@ -269,15 +269,38 @@ func BuildActorTemplate(atespace, name, image string, envMap map[string]string, + FromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + }, + }, +- SandboxConfig: &ateapipb.SandboxConfig{ +- SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, +- ConfigName: "gvisor-default", +- }, ++ SandboxConfig: sandboxConfigFor(sandboxClass), ++ } ++} ++ ++// sandboxConfigFor maps an ax spec.sandboxClass onto the substrate's own ++// SandboxConfig. The empty string means gVisor, which is what this function's ++// caller hardcoded before the field existed, so the default is unchanged. ++// ++// config_name names a cluster-scoped substrate SandboxConfig object and must ++// match the class. "gvisor-default" is the name ax has always used; the microVM ++// name follows the same convention, and ax neither creates nor verifies it. ++// ++// An unrecognised value should never reach here: v1alpha1.ValidateTask refuses ++// it at admission. If one does, gVisor is the fallback because it is the ++// historical default and the only class whose config object ax has ever named ++// successfully. The substrate enum has no workerd member, so no value of ++// sandboxClass can select one. ++func sandboxConfigFor(sandboxClass string) *ateapipb.SandboxConfig { ++ if sandboxClass == v1alpha1.SandboxClassMicroVM { ++ return &ateapipb.SandboxConfig{ ++ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM, ++ ConfigName: "microvm-default", ++ } ++ } ++ return &ateapipb.SandboxConfig{ ++ SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ++ ConfigName: "gvisor-default", + } + } + + // EnsureActorTemplateWithImage creates an ActorTemplate using the specified container image and optional environment variables. +-func (c *Client) EnsureActorTemplateWithImage(ctx context.Context, baseAtespace, baseTemplate, targetAtespace, targetTemplate, image string, extraEnv ...map[string]string) (*ateapipb.ActorTemplate, error) { ++func (c *Client) EnsureActorTemplateWithImage(ctx context.Context, baseAtespace, baseTemplate, targetAtespace, targetTemplate, image, sandboxClass string, extraEnv ...map[string]string) (*ateapipb.ActorTemplate, error) { + existing, err := c.GetActorTemplate(ctx, targetAtespace, targetTemplate) + if err == nil && existing != nil { + return existing, nil +@@ -290,7 +313,7 @@ func (c *Client) EnsureActorTemplateWithImage(ctx context.Context, baseAtespace, + } + } + +- tmpl := BuildActorTemplate(targetAtespace, targetTemplate, image, envMap, nil, "") ++ tmpl := BuildActorTemplate(targetAtespace, targetTemplate, image, envMap, nil, "", sandboxClass) + req := &ateapipb.CreateActorTemplateRequest{ + ActorTemplate: tmpl, + } +diff --git a/pkg/apis/v1alpha1/ax.pb.go b/pkg/apis/v1alpha1/ax.pb.go +index b1adb26..9a71dc0 100644 +--- a/pkg/apis/v1alpha1/ax.pb.go ++++ b/pkg/apis/v1alpha1/ax.pb.go +@@ -15,7 +15,7 @@ + // Code generated by protoc-gen-go. DO NOT EDIT. + // versions: + // protoc-gen-go v1.36.11 +-// protoc v7.34.1 ++// protoc v7.35.1 + // source: pkg/apis/v1alpha1/ax.proto + + package v1alpha1 +@@ -187,7 +187,23 @@ type TaskSpec struct { + Gateway *GatewayRef `protobuf:"bytes,8,opt,name=gateway,proto3" json:"gateway,omitempty"` + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. +- Debug bool `protobuf:"varint,10,opt,name=debug,proto3" json:"debug,omitempty"` ++ Debug bool `protobuf:"varint,10,opt,name=debug,proto3" json:"debug,omitempty"` ++ // sandbox_class selects the sandbox runtime family the task's actor runs in. ++ // Empty means "gvisor", which is what every task got before this field ++ // existed, so the default is unchanged. ++ // ++ // The accepted values are exactly the members Agent Substrate's own ++ // SandboxClass enum offers, lowercased: "gvisor" and "microvm". Any other ++ // value is refused by ValidateTask rather than downgraded, because a task ++ // that silently runs in a weaker sandbox than it asked for is worse than a ++ // task that does not start. In particular there is no "workerd" value: the ++ // substrate enum has no such member, and ax cannot add one. ++ // ++ // "microvm" additionally requires a cluster-scoped substrate SandboxConfig ++ // object named "microvm-default", by analogy with the "gvisor-default" object ++ // the gVisor path has always named. ax neither creates nor verifies it; if it ++ // is absent the substrate rejects the ActorTemplate. ++ SandboxClass string `protobuf:"bytes,11,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache + } +@@ -278,6 +294,13 @@ func (x *TaskSpec) GetDebug() bool { + return false + } + ++func (x *TaskSpec) GetSandboxClass() string { ++ if x != nil { ++ return x.SandboxClass ++ } ++ return "" ++} ++ + type EnvVar struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +@@ -3165,7 +3188,7 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "\x04kind\x18\x02 \x01(\tR\x04kind\x123\n" + + "\bmetadata\x18\x03 \x01(\v2\x17.ax.v1alpha1.ObjectMetaR\bmetadata\x12)\n" + + "\x04spec\x18\x04 \x01(\v2\x15.ax.v1alpha1.TaskSpecR\x04spec\x12/\n" + +- "\x06status\x18\x05 \x01(\v2\x17.ax.v1alpha1.TaskStatusR\x06status\"\xd4\x02\n" + ++ "\x06status\x18\x05 \x01(\v2\x17.ax.v1alpha1.TaskStatusR\x06status\"\xf9\x02\n" + + "\bTaskSpec\x12\x18\n" + + "\asuspend\x18\x02 \x01(\bR\asuspend\x12\x14\n" + + "\x05image\x18\x03 \x01(\tR\x05image\x12\x18\n" + +@@ -3177,7 +3200,8 @@ const file_pkg_apis_v1alpha1_ax_proto_rawDesc = "" + + "workspaces\x121\n" + + "\agateway\x18\b \x01(\v2\x17.ax.v1alpha1.GatewayRefR\agateway\x12\x14\n" + + "\x05debug\x18\n" + +- " \x01(\bR\x05debugJ\x04\b\x01\x10\x02J\x04\b\t\x10\n" + ++ " \x01(\bR\x05debug\x12#\n" + ++ "\rsandbox_class\x18\v \x01(\tR\fsandboxClassJ\x04\b\x01\x10\x02J\x04\b\t\x10\n" + + "R\x04goalR\bpolicies\"2\n" + + "\x06EnvVar\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + +diff --git a/pkg/apis/v1alpha1/ax.proto b/pkg/apis/v1alpha1/ax.proto +index 62bd57a..cf35316 100644 +--- a/pkg/apis/v1alpha1/ax.proto ++++ b/pkg/apis/v1alpha1/ax.proto +@@ -91,6 +91,22 @@ message TaskSpec { + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; ++ // sandbox_class selects the sandbox runtime family the task's actor runs in. ++ // Empty means "gvisor", which is what every task got before this field ++ // existed, so the default is unchanged. ++ // ++ // The accepted values are exactly the members Agent Substrate's own ++ // SandboxClass enum offers, lowercased: "gvisor" and "microvm". Any other ++ // value is refused by ValidateTask rather than downgraded, because a task ++ // that silently runs in a weaker sandbox than it asked for is worse than a ++ // task that does not start. In particular there is no "workerd" value: the ++ // substrate enum has no such member, and ax cannot add one. ++ // ++ // "microvm" additionally requires a cluster-scoped substrate SandboxConfig ++ // object named "microvm-default", by analogy with the "gvisor-default" object ++ // the gVisor path has always named. ax neither creates nor verifies it; if it ++ // is absent the substrate rejects the ActorTemplate. ++ string sandbox_class = 11; + } + + message EnvVar { +diff --git a/pkg/apis/v1alpha1/types.go b/pkg/apis/v1alpha1/types.go +index a65e626..820c19b 100644 +--- a/pkg/apis/v1alpha1/types.go ++++ b/pkg/apis/v1alpha1/types.go +@@ -36,6 +36,16 @@ const ( + + DefaultTaskImage = "gcr.io/ax-substrate/ate-images/ax-task-runner" + ++ // The sandbox classes TaskSpec.SandboxClass accepts. They are exactly the ++ // members of Agent Substrate's own SandboxClass enum, lowercased; ax does ++ // not own that enum and cannot add to it. ++ SandboxClassGVisor = "gvisor" ++ SandboxClassMicroVM = "microvm" ++ ++ // DefaultSandboxClass is what an empty spec.sandboxClass resolves to. It is ++ // the class every task ran in before the field existed. ++ DefaultSandboxClass = SandboxClassGVisor ++ + // PhaseTerminating marks a task whose deletion has been requested and whose + // actor is being torn down. The record disappears once cleanup completes. + PhaseTerminating = "Terminating" +@@ -252,6 +262,15 @@ func ValidateTask(t *Task) error { + if spec == nil { + return nil + } ++ switch spec.GetSandboxClass() { ++ case "", SandboxClassGVisor, SandboxClassMicroVM: ++ default: ++ // Refused, not downgraded: a task that quietly runs in a different ++ // sandbox class than it asked for is worse than a task that does not ++ // start. Agent Substrate offers no other class today. ++ return fmt.Errorf("spec.sandboxClass: %q is not a sandbox class Agent Substrate offers; supported values are %q and %q", ++ spec.GetSandboxClass(), SandboxClassGVisor, SandboxClassMicroVM) ++ } + refs := spec.WorkspaceRefs() + paths := spec.WorkspacePaths() + names := make(map[string]bool, len(refs)) +diff --git a/pkg/apis/v1alpha1/types_test.go b/pkg/apis/v1alpha1/types_test.go +index 83deaa4..2431b97 100644 +--- a/pkg/apis/v1alpha1/types_test.go ++++ b/pkg/apis/v1alpha1/types_test.go +@@ -137,6 +137,43 @@ func TestStrictDecoding_RejectsUnknownFields(t *testing.T) { + } + } + ++func TestTask_SandboxClass_YAML(t *testing.T) { ++ // The point of this test is that the field is real to protojson, not just ++ // present in the .proto: strict decoding rejects anything it does not know. ++ var task v1alpha1.Task ++ if err := yaml.Unmarshal([]byte("kind: Task\nspec:\n sandboxClass: microvm\n"), &task); err != nil { ++ t.Fatalf("decoding a manifest naming sandboxClass: %v", err) ++ } ++ if got := task.GetSpec().GetSandboxClass(); got != v1alpha1.SandboxClassMicroVM { ++ t.Errorf("spec.sandboxClass = %q, want %q", got, v1alpha1.SandboxClassMicroVM) ++ } ++ ++ out, err := yaml.Marshal(&task) ++ if err != nil { ++ t.Fatalf("marshalling: %v", err) ++ } ++ if !strings.Contains(string(out), "sandboxClass: microvm") { ++ t.Errorf("round trip lost the field, got:\n%s", out) ++ } ++ ++ // Empty stays empty rather than being rendered as the default, so existing ++ // manifests keep their existing YAML shape. ++ var plain v1alpha1.Task ++ if err := yaml.Unmarshal([]byte("kind: Task\nspec:\n image: img\n"), &plain); err != nil { ++ t.Fatalf("decoding a manifest without sandboxClass: %v", err) ++ } ++ if got := plain.GetSpec().GetSandboxClass(); got != "" { ++ t.Errorf("spec.sandboxClass = %q, want empty", got) ++ } ++ out, err = yaml.Marshal(&plain) ++ if err != nil { ++ t.Fatalf("marshalling: %v", err) ++ } ++ if strings.Contains(string(out), "sandboxClass") { ++ t.Errorf("unset field should not be rendered, got:\n%s", out) ++ } ++} ++ + func TestGateway_RoundTrip(t *testing.T) { + want := &v1alpha1.Gateway{ + ApiVersion: v1alpha1.APIVersion, +@@ -410,6 +447,19 @@ func TestValidateTask(t *testing.T) { + wantErr string + }{ + {name: "nil spec"}, ++ {name: "sandbox class unset", spec: &v1alpha1.TaskSpec{}}, ++ {name: "sandbox class gvisor", spec: &v1alpha1.TaskSpec{SandboxClass: v1alpha1.SandboxClassGVisor}}, ++ {name: "sandbox class microvm", spec: &v1alpha1.TaskSpec{SandboxClass: v1alpha1.SandboxClassMicroVM}}, ++ { ++ name: "sandbox class workerd", ++ spec: &v1alpha1.TaskSpec{SandboxClass: "workerd"}, ++ wantErr: `spec.sandboxClass: "workerd" is not a sandbox class Agent Substrate offers`, ++ }, ++ { ++ name: "sandbox class wrong case", ++ spec: &v1alpha1.TaskSpec{SandboxClass: "gVisor"}, ++ wantErr: `spec.sandboxClass: "gVisor" is not a sandbox class Agent Substrate offers`, ++ }, + {name: "no workspaces", spec: &v1alpha1.TaskSpec{}}, + {name: "list of one", spec: &v1alpha1.TaskSpec{Workspaces: []*v1alpha1.WorkspaceRef{{Name: "a"}}}}, + { diff --git a/pkgs/substrate/default.nix b/pkgs/substrate/default.nix new file mode 100644 index 000000000..2c186d350 --- /dev/null +++ b/pkgs/substrate/default.nix @@ -0,0 +1,146 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, + applyPatches, + runCommand, + # Passed in by the caller from the nixpkgs-go input, exactly as pkgs/ax gets + # it (see overlays/default.nix and the nixpkgs-go comment in flake.nix). + # Substrate's go.mod says `go 1.27.0`; the probe built it with 1.27.1 + # (MEASURED, evals-2026-09-23/substrate/probe-build.md section 2), and one + # toolchain for ax and Substrate keeps the fleet on one Go. + go_1_27, +}: +# Agent Substrate, the sandbox control plane ax delegates to. Upstream +# github.com/agent-substrate/substrate, pinned by commit to d277088b: the +# SERVER version the 2026-09-23 version rule names. ax keeps its own vendored +# CLIENT pin (672533541dbf); wire compatibility between the two is MEASURED +# (probe-build.md section 4). +# +# Not in nixpkgs. Upstream publishes no release images the fleet may use (the +# kagent-dev fork's ghcr images are ruled out), so every component image is +# built here and seeded into the NAS registry by pkgs/substrate/images.nix. +# +# The four patches touch only manifests/, never Go code: +# 0001 pauseImage -> localhost:5000/pause (atelet pulls it itself; the fleet +# must not depend on registry.k8s.io at sandbox start). +# 0002 third-party images -> their linux/amd64 child digests, so the NAS +# seeds ~0.7 GB instead of every platform (see the patch header). +# 0003 the kind overlay's literal RustFS credential (public in the upstream +# repo) -> secretKeyRef to Secret ate-system/ax-fleet-rustfs, which the +# bootstrap step 25-rustfs-secret generates once on the NAS. +# 0004 the kind overlay's tracing stack, bounded (fix round 3): jaeger +# keeps at most 10000 traces in memory, the collector gets a +# memory_limiter and debug at basic, and both get limits.memory +# 512Mi. Upstream leaves them unbounded (MEASURED at d277088b), and +# they run on the NAS next to Immich, Paperless and NFS. +# ate-setup reads the manifests from its working directory's repository root +# (it walks up to go.mod). What it reads for `deploy ate-system` is go.mod, +# manifests/ and hack/ (kustomize overlays, CSI manifests; MEASURED grep of +# cmd/ate-setup/internal/steps), so passthru.installTree is those three, +# about 1.3 MB instead of the 172 MB tree, and passthru.ate-setup is the +# installer alone (55 MB instead of all seven binaries). Only those two sit in +# the NAS closure at run time; the component binaries reach the NAS inside +# the images. +let + version = "d277088b"; + rev = "d277088bc1d081ef716d81dd7986d05d0a36ad3a"; + + source = applyPatches { + name = "substrate-source-${version}"; + src = fetchFromGitHub { + owner = "agent-substrate"; + repo = "substrate"; + inherit rev; + # nix-prefetch-url --unpack of the GitHub archive; the unpacked tree is + # byte-identical to /home/tom/Downloads/substrate at this rev (MEASURED + # diff -rq, 2026-09-23). + hash = "sha256-/KY4vYgHbeiVnpRUsVFOVfoN3SVEtWJrXTj0Pb4Zeqw="; + }; + patches = [ + ./patches/0001-sandboxconfig-pause-localhost.patch + ./patches/0002-images-linux-amd64-digests.patch + ./patches/0003-kind-rustfs-credential-secret.patch + ./patches/0004-kind-otel-memory-bounds.patch + ]; + }; + + buildGo127Module = buildGoModule.override { go = go_1_27; }; + + common = { + inherit version; + src = source; + + # The tree is vendored (vendor/modules.txt), so no module download. + vendorHash = null; + + env.CGO_ENABLED = "0"; + + # The Makefile's LDFLAGS (Makefile:44-45), with the version the node label, + # the image tag and ate-setup's VERSION all share. + ldflags = [ + "-s" + "-w" + "-X=github.com/agent-substrate/substrate/internal/version.Version=${version}" + ]; + + # Upstream's suite needs Docker (225 PostgreSQL testcontainer tests), root + # (19 tests) and an FHS /bin/sleep; the probe ran it outside nix (1422 + # pass, 2 environment-only failures, MEASURED probe-build.md section 3). + # Inside the nix sandbox it can only fail for the same reasons. + doCheck = false; + }; + + ate-setup = buildGo127Module ( + common + // { + pname = "ate-setup"; + subPackages = [ "cmd/ate-setup" ]; + meta.mainProgram = "ate-setup"; + } + ); + + installTree = runCommand "substrate-install-tree-${version}" { } '' + mkdir -p $out + cp ${source}/go.mod $out/ + cp -r ${source}/manifests ${source}/hack $out/ + ''; + + # The six components the kind install and the WorkerPool run, plus the + # installer. Image names are the import paths' last element, which is what + # ate-setup's --image-repo mode looks up (cmd/ate-setup/internal/images). + components = [ + "ateapi" + "atecontroller" + "atelet" + "atenet" + "podcertcontroller" + "ateom-gvisor" + ]; +in +buildGo127Module ( + common + // { + pname = "substrate"; + + subPackages = [ "cmd/ate-setup" ] ++ map (c: "cmd/${c}") components; + + passthru = { + inherit + source + installTree + ate-setup + rev + components + ; + }; + + meta = { + description = "Agent Substrate (upstream main ${version}): ate-setup and the gVisor control plane"; + homepage = "https://github.com/agent-substrate/substrate"; + license = lib.licenses.asl20; + platforms = [ "x86_64-linux" ]; + mainProgram = "ate-setup"; + }; + } +) diff --git a/pkgs/substrate/images.nix b/pkgs/substrate/images.nix new file mode 100644 index 000000000..83da1c65c --- /dev/null +++ b/pkgs/substrate/images.nix @@ -0,0 +1,298 @@ +{ + lib, + dockerTools, + runCommand, + linkFarm, + fetchurl, + skopeo, + jq, + cacert, + tzdata, + # pkgs/substrate/default.nix, built with the fleet's Go 1.27.1. + substrate, +}: +# Every image the Substrate install and the gVisor WorkerPool pull, as OCI +# layouts the NAS seeds into its registry (modules/ax-fleet/substrate.nix, +# myAxFleet.registrySeed). Each layout carries a `digest` file holding the +# manifest digest, so references are written by digest without IFD: whatever +# needs the digest reads the file at build or run time. +# +# components the six Substrate components, nix-built from d277088b, tagged +# with the version (ate-setup --image-repo/--image-tag resolves +# each tag to a digest with a HEAD request, then pins it). +# thirdParty the images the kind install names by digest, fetched as the +# linux/amd64 child manifest of the upstream-pinned index +# (patch 0002), bytes and digest preserved. Fixed-output. +# gvisor the runsc tarball SandboxConfig gvisor-default names, fetched +# with the sha256 that manifest carries. +# +# The whole set is one derivation (a linkFarm) so `nix build` of the flake's +# substrate-images output proves every image; the parts are in passthru. +let + inherit (substrate) version; + + # ko's default base is gcr.io/distroless/static-debian13 (.ko.yaml). What a + # static Go binary needs from it: CA roots, zoneinfo, a passwd naming root and + # nonroot 65532 (atenet runs as 65532, atelet as 0; MEASURED + # manifests/ate-install), and a world-writable /tmp. + staticBase = '' + mkdir -p etc tmp home/nonroot + chmod 1777 tmp + cat > etc/passwd <<'EOF' + root:x:0:0:root:/root:/sbin/nologin + nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin + nobody:x:65534:65534:nobody:/nonexistent:/sbin/nologin + EOF + cat > etc/group <<'EOF' + root:x:0: + nonroot:x:65532: + nobody:x:65534: + EOF + sed -i 's/^ //' etc/passwd etc/group + ''; + + # docker-archive -> OCI layout plus its manifest digest. Layers are gzipped + # so the registry and containerd hold compressed blobs, as they would for a + # ko push. + toOci = + { + image, + tag, + repo, + }: + runCommand "${lib.replaceStrings [ "/" ] [ "-" ] repo}-${tag}-oci" + { + nativeBuildInputs = [ + skopeo + jq + ]; + passthru = { + inherit repo tag image; + }; + } + '' + export HOME=$TMPDIR + skopeo --insecure-policy --tmpdir "$TMPDIR" copy \ + --dest-compress --dest-compress-format gzip \ + docker-archive:${image} oci:$out:${tag} + jq -r '.manifests | if length == 1 then .[0].digest else error("expected one manifest") end' \ + $out/index.json > $out/digest + grep -Eq '^sha256:[0-9a-f]{64}$' $out/digest + ''; + + component = + name: + let + image = dockerTools.buildLayeredImage { + name = "substrate/${name}"; + tag = version; + contents = [ + cacert + tzdata + ]; + extraCommands = staticBase + '' + mkdir -p ko-app + cp ${substrate}/bin/${name} ko-app/${name} + ''; + config = { + # ko's layout: the binary at /ko-app/ is the entrypoint. + Entrypoint = [ "/ko-app/${name}" ]; + Env = [ + "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" + "ZONEINFO=${tzdata}/share/zoneinfo" + "PATH=/ko-app" + ]; + WorkingDir = "/"; + Labels = { + "org.opencontainers.image.source" = "https://github.com/agent-substrate/substrate"; + "org.opencontainers.image.revision" = substrate.rev; + "org.opencontainers.image.version" = version; + }; + }; + }; + in + toOci { + inherit image; + tag = version; + repo = "substrate/${name}"; + }; + + components = lib.genAttrs substrate.components component; + + # A digest-preserving copy of one upstream manifest. The build fetches the + # upstream-pinned index, checks that it lists `digest` for linux/amd64, then + # copies that manifest and its blobs byte for byte. outputHash pins the result. + thirdPartyImage = + { + name, + upstream, + index, + digest, + repo, + tag, + hash, + }: + runCommand "${name}-${tag}-oci" + { + nativeBuildInputs = [ + skopeo + jq + ]; + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = hash; + passthru = { + inherit + repo + tag + digest + index + upstream + ; + }; + } + '' + export HOME=$TMPDIR + skopeo --insecure-policy --tmpdir "$TMPDIR" inspect --raw docker://${upstream}@${index} > index.json + echo "${lib.removePrefix "sha256:" index} index.json" | sha256sum -c - + jq -e --arg d ${digest} \ + '[.manifests[] | select(.digest == $d and .platform.os == "linux" and .platform.architecture == "amd64")] | length == 1' \ + index.json + skopeo --insecure-policy --tmpdir "$TMPDIR" copy --preserve-digests \ + docker://${upstream}@${digest} oci:$out:${tag} + jq -e --arg d ${digest} '.manifests[0].digest == $d' $out/index.json + echo ${digest} > $out/digest + ''; + + # upstream index digests: manifests/ate-install at d277088b (MEASURED grep). + # linux/amd64 children: `skopeo inspect --raw` of each index (MEASURED + # 2026-09-23); the same values are in patches/0002. + thirdParty = lib.mapAttrs (name: a: thirdPartyImage (a // { inherit name; })) { + envoy = { + upstream = "docker.io/envoyproxy/envoy"; + repo = "envoyproxy/envoy"; + tag = "v1.39-latest"; + index = "sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4"; + digest = "sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b"; + hash = "sha256-8eny55hV9lIrOBFpRt9Zw563sJHEf5IsJsAgvyYrSNs="; + }; + postgres = { + upstream = "docker.io/library/postgres"; + repo = "library/postgres"; + tag = "18-alpine"; + index = "sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15"; + digest = "sha256:b6a16ed0eb96e2c362811f7eeb951eac8b459e7b40be4149ea5444aa7c65569b"; + hash = "sha256-s0f5LZJr0644wB7RDApe03GXwwNIvvUZD3JXta5qcQQ="; + }; + rustfs = { + upstream = "docker.io/rustfs/rustfs"; + repo = "rustfs/rustfs"; + tag = "1.0.0-beta.3"; + index = "sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358"; + digest = "sha256:d441111efe3af5bbd6e29eba7cd124f96bb7d49a0612390907f9216d5f970382"; + hash = "sha256-TKKJLC0H9huZDQQNgx8s9xGvpre2bSa2GGArAFfv53s="; + }; + aws-cli = { + upstream = "docker.io/amazon/aws-cli"; + repo = "amazon/aws-cli"; + tag = "2.17.0"; + index = "sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73"; + digest = "sha256:7b7edf789765c22d75e61ad6f307c06950e15357b79ea1749104641ce3a11fec"; + hash = "sha256-kZF9g+VfTk5gmtNsZhJNBaX9KceOvDl2s/eWD/ucY00="; + }; + otel-collector = { + upstream = "docker.io/otel/opentelemetry-collector-contrib"; + repo = "otel/opentelemetry-collector-contrib"; + tag = "0.157.0"; + index = "sha256:f2f01157055a9b2aab9df7118e1f1c9abf345e99b23bc7a2bc791db374a7d0f6"; + digest = "sha256:4eb842091c796156d4d3c994eb22ba793590f5723719dbf6b8436cb4dfc17f48"; + hash = "sha256-/btIX9XM2O9wkPwAiN6mLJw8JZ2qoB1yzChWcYiEaGY="; + }; + jaeger = { + upstream = "docker.io/jaegertracing/all-in-one"; + repo = "jaegertracing/all-in-one"; + tag = "1.55"; + index = "sha256:f6b5d09073f14f76873d300f565a6691d815e81bea8e07e1dc3ff67e0596dd4e"; + digest = "sha256:d5bbf80eb37e3a0d1b1644f17d1c3a7b88abd74177ec06198d4a289b58b41798"; + hash = "sha256-Ag47jLKCMmMczbW0muNZuaOYkMtEHxVYEt9+BkgYWQ0="; + }; + prometheus = { + upstream = "docker.io/prom/prometheus"; + repo = "prom/prometheus"; + tag = "v3.5.3"; + index = "sha256:ddc2493835a1509976d5e4e0c94199c4f843ce1f42dd6bcfc8231ba734a93ff7"; + digest = "sha256:442634af681c5988a3ccbd4c6e6ab57e077dc89eead4a31e2abf410501874a92"; + hash = "sha256-yf0l7RsqNY+X6muqRkaK+PlfuUP8w2+2UMaM8I1oUxs="; + }; + # Named by the patched SandboxConfig as localhost:5000/pause:3.10.2@...; + # atelet rewrites localhost:5000 to kind-registry:5000, so repo `pause`. + pause = { + upstream = "registry.k8s.io/pause"; + repo = "pause"; + tag = "3.10.2"; + index = "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"; + digest = "sha256:412c4a7219cb8a299a37337f3d87810c5340095322e15594a1637785adad0f17"; + hash = "sha256-wQV0nZBm2siOhrsElV7IS1dyk3ppsRzC0TtugyGGxA0="; + }; + }; + + # SandboxConfig gvisor-default's amd64 asset (sandboxconfig-gvisor.yaml:34-35 + # at d277088b). atelet tries anonymous GCS, then its S3 client with the same + # bucket and key against the in-cluster RustFS; bootstrap step 40 puts this + # file there, so a node without internet still gets runsc. + gvisor = + let + bucket = "gvisor"; + key = "releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd"; + sha256 = "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"; + in + fetchurl { + name = "gvisor-nightly-2026-09-02-x86_64.tar.zstd"; + url = "https://storage.googleapis.com/${bucket}/${key}"; + inherit sha256; + passthru = { + inherit bucket key sha256; + }; + }; + + # The shape myAxFleet.registrySeed takes: name -> { oci, repo, tag }. + seed = + lib.mapAttrs' ( + n: v: + lib.nameValuePair "substrate-${n}" { + oci = v; + inherit (v) repo tag; + } + ) components + // lib.mapAttrs (_: v: { + oci = v; + inherit (v) repo tag; + }) thirdParty; +in +linkFarm "substrate-images-${version}" ( + lib.mapAttrsToList (n: v: { + name = "components/${n}"; + path = v; + }) components + ++ lib.mapAttrsToList (n: v: { + name = "third-party/${n}"; + path = v; + }) thirdParty + ++ [ + { + name = "gvisor/gvisor.tar.zstd"; + path = gvisor; + } + ] +) +// { + inherit + components + thirdParty + gvisor + seed + version + ; +} diff --git a/pkgs/substrate/patches/0001-sandboxconfig-pause-localhost.patch b/pkgs/substrate/patches/0001-sandboxconfig-pause-localhost.patch new file mode 100644 index 000000000..94eab02b6 --- /dev/null +++ b/pkgs/substrate/patches/0001-sandboxconfig-pause-localhost.patch @@ -0,0 +1,15 @@ +--- a/manifests/ate-install/sandboxconfig-gvisor.yaml ++++ b/manifests/ate-install/sandboxconfig-gvisor.yaml +@@ -27,7 +27,11 @@ + sandboxClass: gvisor + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. +- pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" ++ # mecattaf fleet: the same upstream image, seeded digest-preserved into the ++ # NAS registry as its linux/amd64 manifest (a child of the upstream index ++ # f548e0e8...). atelet pulls it itself and rewrites localhost:5000 through ++ # --localhost-registry-replacement, so no node needs registry.k8s.io. ++ pauseImage: "localhost:5000/pause:3.10.2@sha256:412c4a7219cb8a299a37337f3d87810c5340095322e15594a1637785adad0f17" + assets: + amd64: + gvisor: diff --git a/pkgs/substrate/patches/0002-images-linux-amd64-digests.patch b/pkgs/substrate/patches/0002-images-linux-amd64-digests.patch new file mode 100644 index 000000000..ec5c4ea82 --- /dev/null +++ b/pkgs/substrate/patches/0002-images-linux-amd64-digests.patch @@ -0,0 +1,122 @@ +Pin every third-party image the kind install pulls to its linux/amd64 manifest. + +Upstream pins each image by its multi-arch index digest. Seeding an index +digest-preserved means copying every platform it lists (postgres:18-alpine +lists 16 manifests), several GB that would sit in the NAS's /nix/store on its +57 GB eMMC root. Each digest below is the linux/amd64 child listed in the +upstream-pinned index (MEASURED 2026-09-23 with `skopeo inspect --raw`), so +the content pin is the same Merkle chain, one level down. The fleet is +x86_64-only. pkgs/substrate/images.nix fetches each child by this digest and +checks at fetch time that the upstream index lists it for linux/amd64. + +diff -ru a/manifests/ate-install/atenet-egress-with-sdsmint.yaml b/manifests/ate-install/atenet-egress-with-sdsmint.yaml +--- a/manifests/ate-install/atenet-egress-with-sdsmint.yaml ++++ b/manifests/ate-install/atenet-egress-with-sdsmint.yaml +@@ -1111,7 +1111,7 @@ + readOnly: true + containers: + - name: envoy +- image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 ++ image: envoyproxy/envoy:v1.39-latest@sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b + securityContext: + allowPrivilegeEscalation: false + capabilities: +diff -ru a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml +--- a/manifests/ate-install/atenet-egress.yaml ++++ b/manifests/ate-install/atenet-egress.yaml +@@ -601,7 +601,7 @@ + terminationGracePeriodSeconds: 60 + containers: + - name: envoy +- image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 ++ image: envoyproxy/envoy:v1.39-latest@sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b + securityContext: + allowPrivilegeEscalation: false + capabilities: +diff -ru a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml +--- a/manifests/ate-install/atenet-router.yaml ++++ b/manifests/ate-install/atenet-router.yaml +@@ -243,7 +243,7 @@ + - name: "drain-signal" + mountPath: "/var/run/atenet" + - name: envoy +- image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 ++ image: envoyproxy/envoy:v1.39-latest@sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b + command: + - "/usr/local/bin/envoy" + - "-c" +diff -ru a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml +--- a/manifests/ate-install/kind/otel-collector.yaml ++++ b/manifests/ate-install/kind/otel-collector.yaml +@@ -177,7 +177,7 @@ + spec: + containers: + - name: otel-collector +- image: otel/opentelemetry-collector-contrib:0.157.0@sha256:f2f01157055a9b2aab9df7118e1f1c9abf345e99b23bc7a2bc791db374a7d0f6 ++ image: otel/opentelemetry-collector-contrib:0.157.0@sha256:4eb842091c796156d4d3c994eb22ba793590f5723719dbf6b8436cb4dfc17f48 + args: + - --config=/conf/otel-collector-config.yaml + volumeMounts: +@@ -242,7 +242,7 @@ + spec: + containers: + - name: jaeger +- image: jaegertracing/all-in-one:1.55@sha256:f6b5d09073f14f76873d300f565a6691d815e81bea8e07e1dc3ff67e0596dd4e ++ image: jaegertracing/all-in-one:1.55@sha256:d5bbf80eb37e3a0d1b1644f17d1c3a7b88abd74177ec06198d4a289b58b41798 + ports: + - name: otlp-grpc + containerPort: 4317 +diff -ru a/manifests/ate-install/kind/prometheus.yaml b/manifests/ate-install/kind/prometheus.yaml +--- a/manifests/ate-install/kind/prometheus.yaml ++++ b/manifests/ate-install/kind/prometheus.yaml +@@ -172,7 +172,7 @@ + type: RuntimeDefault + containers: + - name: prometheus +- image: prom/prometheus:v3.5.3@sha256:ddc2493835a1509976d5e4e0c94199c4f843ce1f42dd6bcfc8231ba734a93ff7 ++ image: prom/prometheus:v3.5.3@sha256:442634af681c5988a3ccbd4c6e6ab57e077dc89eead4a31e2abf410501874a92 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus +diff -ru a/manifests/ate-install/kind/rustfs.yaml b/manifests/ate-install/kind/rustfs.yaml +--- a/manifests/ate-install/kind/rustfs.yaml ++++ b/manifests/ate-install/kind/rustfs.yaml +@@ -62,7 +62,7 @@ + fsGroup: 10001 + containers: + - name: rustfs +- image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 ++ image: rustfs/rustfs:1.0.0-beta.3@sha256:d441111efe3af5bbd6e29eba7cd124f96bb7d49a0612390907f9216d5f970382 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 +@@ -102,7 +102,7 @@ + restartPolicy: OnFailure + containers: + - name: create-bucket +- image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 ++ image: amazon/aws-cli:2.17.0@sha256:7b7edf789765c22d75e61ad6f307c06950e15357b79ea1749104641ce3a11fec + env: + - name: AWS_ACCESS_KEY_ID + value: rustfsadmin +diff -ru a/manifests/ate-install/postgres/postgres.yaml b/manifests/ate-install/postgres/postgres.yaml +--- a/manifests/ate-install/postgres/postgres.yaml ++++ b/manifests/ate-install/postgres/postgres.yaml +@@ -120,7 +120,7 @@ + initContainers: + - name: tls-reloader + restartPolicy: Always +- image: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 ++ image: postgres:18-alpine@sha256:b6a16ed0eb96e2c362811f7eeb951eac8b459e7b40be4149ea5444aa7c65569b + securityContext: + runAsUser: 70 + command: +@@ -143,7 +143,7 @@ + memory: 32Mi + containers: + - name: postgres +- image: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 ++ image: postgres:18-alpine@sha256:b6a16ed0eb96e2c362811f7eeb951eac8b459e7b40be4149ea5444aa7c65569b + lifecycle: + postStart: + exec: diff --git a/pkgs/substrate/patches/0003-kind-rustfs-credential-secret.patch b/pkgs/substrate/patches/0003-kind-rustfs-credential-secret.patch new file mode 100644 index 000000000..8ebef8952 --- /dev/null +++ b/pkgs/substrate/patches/0003-kind-rustfs-credential-secret.patch @@ -0,0 +1,85 @@ +--- a/manifests/ate-install/kind/atelet/kustomization.yaml ++++ b/manifests/ate-install/kind/atelet/kustomization.yaml +@@ -52,7 +52,15 @@ + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: AWS_ACCESS_KEY_ID +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: AWS_SECRET_ACCESS_KEY +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key +--- a/manifests/ate-install/kind/kustomization.yaml ++++ b/manifests/ate-install/kind/kustomization.yaml +@@ -66,8 +66,16 @@ + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: AWS_ACCESS_KEY_ID +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: AWS_SECRET_ACCESS_KEY +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key + +--- a/manifests/ate-install/kind/rustfs.yaml ++++ b/manifests/ate-install/kind/rustfs.yaml +@@ -78,10 +78,18 @@ + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: RUSTFS_ACCESS_KEY +- value: "rustfsadmin" ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: RUSTFS_SECRET_KEY +- value: "rustfsadmin" ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key + volumeMounts: + - name: data + mountPath: /data +@@ -104,10 +112,18 @@ + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: AWS_ACCESS_KEY_ID +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: AWS_SECRET_ACCESS_KEY +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL diff --git a/pkgs/substrate/patches/0004-kind-otel-memory-bounds.patch b/pkgs/substrate/patches/0004-kind-otel-memory-bounds.patch new file mode 100644 index 000000000..c097991d9 --- /dev/null +++ b/pkgs/substrate/patches/0004-kind-otel-memory-bounds.patch @@ -0,0 +1,77 @@ +--- a/manifests/ate-install/kind/otel-collector.yaml 2026-09-23 13:06:34.261419536 +0200 ++++ b/manifests/ate-install/kind/otel-collector.yaml 2026-09-23 13:06:45.024736968 +0200 +@@ -32,6 +32,13 @@ + http: + endpoint: 0.0.0.0:4318 + processors: ++ # mecattaf fleet: the collector runs on the NAS (22 GB RAM, shared with ++ # Immich, Paperless and NFS). memory_limiter refuses data before the ++ # container limit below is reached, instead of being OOM-killed. ++ memory_limiter: ++ check_interval: 1s ++ limit_mib: 400 ++ spike_limit_mib: 100 + batch: + + # The count connector below sends deltas, and the Prometheus exporter +@@ -95,17 +102,19 @@ + insecure: true + prometheus: + endpoint: 0.0.0.0:8889 ++ # mecattaf fleet: basic, not detailed; detailed writes every span and ++ # log record to the pod log. + debug: +- verbosity: detailed ++ verbosity: basic + service: + pipelines: + traces: + receivers: [otlp] +- processors: [batch] ++ processors: [memory_limiter, batch] + exporters: [otlp/jaeger, debug] + metrics: + receivers: [otlp] +- processors: [batch] ++ processors: [memory_limiter, batch] + exporters: [prometheus, debug] + + # Every ate event, plus any workload that sends OTLP logs. debug is the +@@ -115,7 +124,7 @@ + # record reports a stale state with no sign that anything is missing. + logs: + receivers: [otlp] +- processors: [batch] ++ processors: [memory_limiter, batch] + exporters: [debug] + + # The count pipelines read the same receiver as the pipelines above, +@@ -180,6 +189,12 @@ + image: otel/opentelemetry-collector-contrib:0.157.0@sha256:4eb842091c796156d4d3c994eb22ba793590f5723719dbf6b8436cb4dfc17f48 + args: + - --config=/conf/otel-collector-config.yaml ++ # mecattaf fleet: bounded (see memory_limiter above). ++ resources: ++ requests: ++ memory: 128Mi ++ limits: ++ memory: 512Mi + volumeMounts: + - name: config + mountPath: /conf +@@ -243,6 +258,15 @@ + containers: + - name: jaeger + image: jaegertracing/all-in-one:1.55@sha256:d5bbf80eb37e3a0d1b1644f17d1c3a7b88abd74177ec06198d4a289b58b41798 ++ # mecattaf fleet: in-memory storage keeps every trace with no bound ++ # by default. Cap the trace count and the container. ++ args: ++ - --memory.max-traces=10000 ++ resources: ++ requests: ++ memory: 128Mi ++ limits: ++ memory: 512Mi + ports: + - name: otlp-grpc + containerPort: 4317 diff --git a/secrets.nix b/secrets.nix index 5d94f6b0b..05b09be96 100644 --- a/secrets.nix +++ b/secrets.nix @@ -92,6 +92,17 @@ in # NAS private media HTTPS: zone-limited DNS-01 token, no broad Wrangler OAuth # authority. Ciphertext is provisioned before enabling personal-https.nix. "secrets/nas-cloudflare-dns.age".publicKeys = editors ++ nasOnly; + # The k3s credentials (ax on the fleet, modules/ax-fleet/k3s.nix), split in + # fix round 2 (2026-09-23). k3s-token.age is the SERVER token: read by the + # NAS only; with it comes the k3s:server role (/v1-k3s/token, /cacerts, + # /encrypt/config). k3s-agent-token.age is the agent join credential: the + # NAS passes it as --agent-token-file, the coordinator joins with it. Both + # were minted fresh from /dev/urandom with `age -R` over these lists, never + # displayed; the 643a4196 ciphertext (also decryptable by the worker and the + # client) was replaced, not re-encrypted, before any cluster used it. + # Rotate with: nix develop -c agenix -e secrets/.age + "secrets/k3s-token.age".publicKeys = editors ++ nasOnly; + "secrets/k3s-agent-token.age".publicKeys = editors ++ coordinatorOnly ++ nasOnly; # --- wifi PSK tier: the coordinator, whose Freebox uplink # (wlp192s0) is now declarative too (migrated from an imperative profile on # flash night — refs #37). Rekey after this change: nix develop -c agenix -r diff --git a/secrets/k3s-agent-token.age b/secrets/k3s-agent-token.age new file mode 100644 index 000000000..091edc6bd --- /dev/null +++ b/secrets/k3s-agent-token.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> X25519 RYDnXPmLP7LRAQUOkbt8OfXKlRrAkxw3ifwxFPxO1U0 +lPdZbsFZ0Q+tIikHyjq4zuiSRO6D/BmWQw4ilwGcrG8 +-> ssh-ed25519 b4VnIg 1Ew3degzdeyANd3OUkRJ4zkCFgTMpeZUaPcvqaNiQXs +b4tGOKtANgMigmVK9kDTjxVRGFaooM0/X6BfFMAai5w +-> ssh-ed25519 sKUETw Yir5k5hDZ2wzoziKlhicGETCYMpKURtD6bv+0A4ykjI +Wb8h0aOXwq3VpycN8k9WjRitw8ooiz8hHirbcb1un20 +--- W0F7q+18w3EXzMQBF7OT5o5kLcHo30Pkarp3RMTJxPQ +) x:{Tzeh~5lLʚvd7ox:=̲/r9m? \ No newline at end of file diff --git a/secrets/k3s-token.age b/secrets/k3s-token.age new file mode 100644 index 000000000..35d03103a --- /dev/null +++ b/secrets/k3s-token.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> X25519 dtiPpi1BLiotWQN+ZoSMCY16V39onbZjA2tXCedlGhw +O8vycEoqUfgd4RjUn2TC2CBweycUR9YHaus2/b3IuQg +-> ssh-ed25519 sKUETw zN+hojttTBmbuORJJbvGBJo+feofYkS9R8s7eHnllWw ++wLRUCZuZv7n1sDj4Djnuu2mTDi2sUfNaPTQ0MuVeD4 +--- hvbZ54nuJjxpSPeHNTegaKSMV92TALVThkKhTcK9pbw +4|HxmSWfHme;uҭp>aHM`l~큿  0ϝ{eݮH[6$%ίF"­WH2$: \ No newline at end of file diff --git a/tests/ax-fleet-boot/default.nix b/tests/ax-fleet-boot/default.nix new file mode 100644 index 000000000..954e4a760 --- /dev/null +++ b/tests/ax-fleet-boot/default.nix @@ -0,0 +1,107 @@ +{ + pkgs, + lib, + inputs, +}: +# checks.x86_64-linux.ax-fleet-boot (DESIGN.md 12.2): a NAS node with ax ON +# FROM BOOT. The 4-VM test proves the live-switch ordering; this proves the +# boot ordering: the bind mounts come up before k3s, the tmpfiles links land +# on the /mnt/fast side, the registry seed and the bootstrap succeed, and the +# node is Ready and untainted. The substrate track adds its Available checks +# here through the same bootstrap. +# +# Built from nixpkgs-STABLE, as the real NAS is (fix round 2): the round-2 +# review MEASURED that every other ax-fleet VM runs NixOS 26.11 and systemd +# 261 while the NAS runs 26.05 and systemd 260, so the NAS's own module set +# (k3s, docker-registry, nftables, systemd) had never run. The NAS boots with +# the systemd initrd, as the real one does (boot.initrd.systemd.enable is +# true on it, MEASURED nix eval), because that is where the boot-time +# activation runs. +let + stablePkgs = inputs.nixpkgs-stable.legacyPackages.x86_64-linux; + nodes = import ../ax-fleet/nodes.nix { + pkgs = stablePkgs; + inherit lib inputs; + }; +in +stablePkgs.testers.runNixOSTest { + name = "ax-fleet-boot"; + node.specialArgs = { inherit inputs; }; + nodes.nas = { + imports = [ nodes.nas ]; + myAxFleet.enable = true; + boot.initrd.systemd.enable = true; + # The NAS's kernel (hosts/nas/kernel.nix: freshPkgs.linuxPackages_7_2); + # ax-fleet-topology asserts it is the same derivation. + boot.kernelPackages = + (import inputs.nixpkgs-fresh { + system = "x86_64-linux"; + config.allowUnfree = true; + }).linuxPackages_7_2; + myAxFleet.kubelet.systemReserved = "cpu=1,memory=1Gi"; + # The real NAS gets 10.42.0.1 from NetworkManager seconds AFTER + # network(-online).target (MEASURED 2026-09-23: target at 11.78 s, address + # at 20.15 s). Reproduce that: no static address, then a unit nothing + # waits for adds it late. The delay is a test parameter longer than the + # units' 30 s address wait, so both the wait and Restart= are exercised. + networking.interfaces.eth1.ipv4.addresses = lib.mkOverride 10 [ ]; + systemd.services.late-lan-addr = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ + pkgs.iproute2 + pkgs.coreutils + ]; + script = '' + sleep 40 + ip link set eth1 up + ip addr add 10.42.0.1/24 dev eth1 + date +%s > /run/late-lan-addr.done + ''; + }; + }; + globalTimeout = 3600; + testScript = '' + nas.start() + nas.wait_for_file("/run/late-lan-addr.done", timeout=600) + nas.wait_for_unit("k3s.service", timeout=900) + nas.wait_for_unit("ax-fleet-bootstrap.service", timeout=1800) + for p in ("/var/lib/rancher/k3s", "/var/lib/kubelet", "/var/log/pods"): + src = nas.succeed(f"findmnt -n -o SOURCE -T {p}").strip() + assert src.startswith("/dev/vdb"), f"{p} is on {src}, not /mnt/fast" + nas.succeed("ls /mnt/fast/k3s/rancher/k3s/agent/images/ | grep -q airgap") + nas.succeed("systemctl show ax-fleet-registry-seed.service -p Result --value | grep -x success") + # The registry came up on the late address and stays up. + nas.succeed("systemctl is-active docker-registry.service") + nas.succeed("curl -sf --max-time 10 http://10.42.0.1:5000/v2/") + addr_at = int(nas.succeed("cat /run/late-lan-addr.done").strip()) + reg_at = int(nas.succeed( + "date -d \"$(systemctl show docker-registry.service -p ActiveEnterTimestamp --value)\" +%s" + ).strip()) + assert reg_at >= addr_at, f"registry active at {reg_at}, before the address at {addr_at}" + print("AXFLEET-BOOT registry NRestarts=" + nas.succeed("systemctl show docker-registry.service -p NRestarts --value").strip()) + nas.succeed("systemctl show ax-fleet-bootstrap.service -p Result --value | grep -x success") + nas.wait_until_succeeds( + "k3s kubectl get node nas -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -x True", + timeout=600, + ) + # k3s's own transient taints (uninitialized, not-ready) clear on their own; + # the NAS carries no taint of ours. + nas.wait_until_succeeds("test -z \"$(k3s kubectl get node nas -o jsonpath='{.spec.taints}')\"", timeout=300) + nas.succeed("k3s kubectl -n kube-system wait --for=condition=Available deploy/coredns deploy/local-path-provisioner --timeout=600s") + nas.succeed("test -s /etc/ax-fleet/admin.kubeconfig") + # The snapshot was taken at first activation, before k3s ever ran, and at + # boot it carries the host's declared values, not the kernel defaults the + # boot activation sees (fix round 2: it had recorded ip_forward = 0 on + # the router, which the teardown would have applied). + print("AXFLEET-BOOT snapshot:\n" + nas.succeed("cat /var/lib/ax-fleet/sysctl-before.conf")) + print("AXFLEET-BOOT release=" + nas.succeed("nixos-version").strip() + " systemd=" + nas.succeed("systemctl --version | head -1").strip()) + nas.succeed("test \"$(sysctl -n net.ipv4.ip_forward)\" = 1") + nas.succeed("grep -x 'net.ipv4.ip_forward = 1' /var/lib/ax-fleet/sysctl-before.conf") + nas.succeed("grep -q '^kernel.panic = ' /var/lib/ax-fleet/sysctl-before.conf") + # kubelet ran; its panic values are not what the host is left with. + snap_panic = nas.succeed("sed -n 's/^kernel.panic = //p' /var/lib/ax-fleet/sysctl-before.conf").strip() + nas.wait_until_succeeds(f"test \"$(sysctl -n kernel.panic)\" = '{snap_panic}'", timeout=300) + nas.fail("findmnt -n -T /var/lib/rancher/k3s -o SOURCE | grep -q vda") + ''; +} diff --git a/tests/ax-fleet-topology/default.nix b/tests/ax-fleet-topology/default.nix new file mode 100644 index 000000000..f64ad81d0 --- /dev/null +++ b/tests/ax-fleet-topology/default.nix @@ -0,0 +1,289 @@ +{ + pkgs, + lib, + self, +}: +# checks.x86_64-linux.ax-fleet-topology (DESIGN.md 12.3): evaluation-only +# assertions over the REAL host configurations. Every assert is eval-time and +# sits in front of the runCommand, so each runs under --no-build. +# +# Not asserted here, on purpose: "no 8731 on the coordinator's wlp192s0". That +# is #461's change; nas-topology already asserts exactly it and fails on +# origin/main until #461 is merged (inherited, not introduced). Duplicating it +# here would make this check red for a reason outside this PR. +let + hostCfg = host: self.nixosConfigurations.${host}.config; + offCfg = + host: + (self.nixosConfigurations.${host}.extendModules { + modules = [ { myAxFleet.enable = lib.mkForce false; } ]; + }).config; + + flagsOf = + cfg: + let + raw = cfg.systemd.services.k3s.serviceConfig.ExecStart; + s = if builtins.isList raw then lib.concatStringsSep " " raw else raw; + in + lib.filter (w: w != "" && w != "\\") (lib.splitString " " (lib.replaceStrings [ "\n" ] [ " " ] s)); + + has = cfg: flag: builtins.elem flag (flagsOf cfg); + hasPrefix = cfg: p: builtins.any (lib.hasPrefix p) (flagsOf cfg); + + nas = hostCfg "nas"; + coord = hostCfg "coordinator"; + worker = hostCfg "worker"; + client = hostCfg "client"; + ax = cfg: cfg.myAxFleet; + + axLines = text: lib.filter (l: lib.hasInfix "ax-fleet:" l) (lib.splitString "\n" text); + + # ── the kill switch: with enable = mkForce false nothing from this PR renders ── + killed = + host: + let + c = offCfg host; + units = lib.attrNames c.systemd.services ++ lib.attrNames c.systemd.sockets; + in + !c.services.k3s.enable + && !c.services.dockerRegistry.enable + && !(builtins.any (lib.hasPrefix "ax-fleet") units) + && !(builtins.any (lib.hasPrefix "ax-server-proxy") units) + && !(builtins.any (m: m.where == "/var/lib/rancher") c.systemd.mounts) + && !(c.environment.etc ? "NetworkManager/conf.d/90-ax-fleet.conf") + && !(c.environment.etc ? "rancher/k3s/registries.yaml") + && !(c.system.activationScripts ? ax-fleet-sysctl-snapshot) + && !(lib.hasInfix "ax-fleet:" (c.networking.firewall.extraInputRules or "")) + && !(c.boot.kernel.sysctl ? "net.ipv4.conf.veth*.proxy_arp"); + + # ...except the guards, which stay for the role whatever `enable` says (fix + # round 3): the kill switch leaves pods running until the teardown. + guardsKept = + host: + let + c = offCfg host; + in + if host == "coordinator" then + lib.hasInfix "ax-fleet-guard" c.networking.firewall.extraCommands + && lib.hasInfix "ax-fleet-pod-input" c.networking.firewall.extraCommands + && lib.hasInfix "ax-fleet-api" c.networking.firewall.extraCommands + else if host == "nas" then + lib.hasInfix "hook forward" c.networking.nftables.tables.ax-fleet-guard.content + else + !(lib.hasInfix "ax-fleet-guard" c.networking.firewall.extraCommands) + && !(c.networking.nftables.tables ? ax-fleet-guard); + + # ── parity with the VM test: flags and firewall text, interfaces substituted ── + testNodes = self.checks.x86_64-linux.ax-fleet.nodes; + testOn = name: testNodes.${name}.specialisation.ax-on.configuration; + # Flags that legitimately differ: the token source and the VM's kubelet + # reservations (a VM has 8 GiB, the desk 128). + volatile = + f: + f == "--token-file" + || f == "--agent-token-file" + || lib.hasSuffix "/k3s-token" f + || lib.hasSuffix "/k3s-agent-token" f + || lib.hasSuffix "-ax-fleet-vm-token" f + || lib.hasSuffix "-ax-fleet-vm-agent-token" f + || lib.hasInfix "reserved=" f + || lib.hasInfix "eviction-hard=" f; + normFlags = + subst: cfg: + lib.sort (a: b: a < b) ( + map (lib.replaceStrings (lib.attrNames subst) (lib.attrValues subst)) ( + lib.filter (f: !(volatile f)) (flagsOf cfg) + ) + ); + nasSubst = { + "enp1s0" = "eth1"; + }; + coordSubst = { + "wlp192s0" = "eth1"; + "tailscale0" = "eth2"; + "enp191s0" = "eth3"; + }; + guardText = + subst: cfg: + map (lib.replaceStrings (lib.attrNames subst) (lib.attrValues subst)) ( + lib.filter ( + l: + lib.hasInfix "ax-fleet-guard" l + || lib.hasInfix "8472" l + || lib.hasInfix "ax-fleet-pod-input" l + || lib.hasInfix "ax-fleet-api" l + ) (lib.splitString "\n" cfg.networking.firewall.extraCommands) + ); + pkgNames = cfg: map (p: p.pname or p.name or "") cfg.environment.systemPackages; + hasTeardown = cfg: builtins.elem "ax-fleet-teardown" (pkgNames cfg); + + # every image the real NAS seeds is the store path the VM NAS seeds (fix round 2) + seedPaths = cfg: lib.mapAttrs (_: s: s.oci.outPath) cfg.myAxFleet.registrySeed; + nasSeeds = seedPaths nas; + vmSeeds = seedPaths (testOn "nas"); +in +# nas: the control node +assert (ax nas).enable && (ax nas).role == "control"; +assert has nas "--flannel-iface=enp1s0"; +assert has nas "--node-ip=10.42.0.1"; +assert has nas "--flannel-backend=vxlan"; +assert !(hasPrefix nas "--node-taint"); +assert has nas "--node-label=ate.dev/substrate-version=none"; +assert has nas "--node-label=ax.mecattaf.dev/role=control"; +assert has nas "--default-local-storage-path=/mnt/nas/services/ax-fleet/local-path"; +assert !(builtins.any (lib.hasInfix "local-storage") nas.services.k3s.disable); +assert has nas "--kube-apiserver-arg=runtime-config=certificates.k8s.io/v1beta1=true"; +assert has nas "--service-node-port-range=30000-30999"; +assert !(hasPrefix nas "--data-dir"); +assert !(hasPrefix nas "--cluster-init"); +assert nas.services.k3s.containerdConfigTemplate == null; +assert nas.services.k3s.package == (ax nas).k3sPackage; +assert (ax nas).k3sPackage.version == "1.36.2+k3s1"; +# every ax-fleet NAS rule is scoped to a source and to an interface, none to the tailnet +assert builtins.length (axLines nas.networking.firewall.extraInputRules) == 4; +assert builtins.all (l: lib.hasInfix "ip saddr" l && lib.hasInfix "iifname" l) ( + axLines nas.networking.firewall.extraInputRules +); +assert + !(builtins.any (lib.hasInfix "tailscale0") (axLines nas.networking.firewall.extraInputRules)); +assert nas.services.dockerRegistry.listenAddress == "10.42.0.1"; +assert !nas.services.dockerRegistry.openFirewall; +assert lib.hasPrefix "/mnt/nas/" nas.services.dockerRegistry.storagePath; +assert builtins.all (m: lib.hasPrefix "/mnt/fast/" m.what) ( + lib.filter ( + m: m.where == "/var/lib/rancher" || m.where == "/var/lib/kubelet" || m.where == "/var/log/pods" + ) nas.systemd.mounts +); +assert + builtins.length (lib.filter (m: lib.hasPrefix "/mnt/fast/k3s" m.what) nas.systemd.mounts) == 3; +# the shared PostgreSQL is not touched: identical settings with the switch off +assert nas.services.postgresql.settings == (offCfg "nas").services.postgresql.settings; +assert nas.services.postgresql.authentication == (offCfg "nas").services.postgresql.authentication; + +# coordinator: the harness node +assert (ax coord).enable && (ax coord).role == "harness"; +assert has coord "--flannel-iface=wlp192s0"; +assert has coord "--node-ip=10.42.0.2"; +assert has coord "--server"; +assert has coord "https://10.42.0.1:6443"; +assert has coord "--node-taint=${(ax coord).harnessTaint}"; +assert (ax coord).harnessTaint == "ate.dev/sandboxClass=gvisor:NoSchedule"; +assert has coord "--node-label=ate.dev/substrate-version=${(ax coord).substrateVersion}"; +assert (ax coord).substrateVersion == (ax nas).substrateVersion; +assert coord.services.k3s.containerdConfigTemplate == null; +assert coord.services.k3s.package == nas.services.k3s.package; +assert lib.hasInfix "ax-fleet-guard" coord.networking.firewall.extraCommands; +assert lib.hasInfix "-s 10.42.0.1 -p udp --dport 8472" coord.networking.firewall.extraCommands; +assert !(coord.networking.firewall.interfaces ? cni0); +assert coord.environment.etc ? "NetworkManager/conf.d/90-ax-fleet.conf"; +# NO NetworkManager restart trigger: NetworkManager.conf renders byte-identical with the switch off +assert + coord.environment.etc."NetworkManager/NetworkManager.conf".source == (offCfg "coordinator") + .environment.etc."NetworkManager/NetworkManager.conf".source; +assert + coord.networking.networkmanager.unmanaged == (offCfg "coordinator") + .networking.networkmanager.unmanaged; +# proxy ARP only on the pod interfaces, never through all/default (fix round 3) +assert coord.boot.kernel.sysctl."net.ipv4.conf.veth*.proxy_arp" == 1; +assert coord.boot.kernel.sysctl."net.ipv4.conf.cni0.proxy_arp" == 1; +assert coord.boot.kernel.sysctl."net.ipv4.conf.flannel/1.proxy_arp" == 1; +assert builtins.all ( + k: !(coord.boot.kernel.sysctl ? ${k}) || coord.boot.kernel.sysctl.${k} == null +) [ "net.ipv4.conf.all.proxy_arp" "net.ipv4.conf.default.proxy_arp" ]; +# the wired port is a guarded LAN leg with a route metric above the wifi's +assert (ax coord).lan.extraInterfaces == [ "enp191s0" ]; +assert lib.hasInfix "-i cni0 -o enp191s0 -j RETURN" coord.networking.firewall.extraCommands; +assert lib.hasInfix "-i cni0 -d 10.0.0.0/8 -j DROP" coord.networking.firewall.extraCommands; +assert lib.hasInfix "-o cni0 -j DROP" coord.networking.firewall.extraCommands; +assert lib.hasInfix "match-device=interface-name:enp191s0" coord.environment.etc."NetworkManager/conf.d/90-ax-fleet.conf".text; +assert lib.hasInfix "ipv4.route-metric=700" coord.environment.etc."NetworkManager/conf.d/90-ax-fleet.conf".text; +# the ax API: only root, tom and the proxy (fix round 3) +assert lib.hasInfix "--uid-owner tom -j RETURN" coord.networking.firewall.extraCommands; +assert coord.systemd.services.ax-server-proxy.serviceConfig.User == "ax-server-proxy"; +assert (coord.boot.kernel.sysctl."net.ipv6.conf.all.forwarding" or 0) == 0; +assert coord.services.tailscale.useRoutingFeatures == "none"; +assert coord.systemd.sockets.ax-server-proxy.listenStreams == [ "127.0.0.1:8099" ]; +assert coord.environment.sessionVariables.AX_SERVER == "http://127.0.0.1:8099"; +# never the address ax-conwip's default (and the mock stack) points at +assert !(builtins.elem coord.home-manager.users.tom.myAxConwip.serverUrl coord.systemd.sockets.ax-server-proxy.listenStreams); +assert coord.myAxClient.enable; + +# worker: inference, nothing at runtime +assert (ax worker).enable && (ax worker).role == "inference"; +assert !worker.services.k3s.enable; +assert builtins.elem 8731 worker.networking.firewall.interfaces.enp191s0.allowedTCPPorts; +# client: untouched +assert !(client ? myAxFleet); +assert !client.services.k3s.enable; + +# the registry is read-only to the network; the seed writes on loopback only +assert nas.services.dockerRegistry.extraConfig.storage.maintenance.readonly.enabled; +assert !nas.services.dockerRegistry.enableDelete; +# two k3s credentials: the server token stays on the NAS +assert lib.hasSuffix "/k3s-token" nas.services.k3s.tokenFile; +assert lib.hasSuffix "/k3s-agent-token" nas.services.k3s.agentTokenFile; +assert lib.hasSuffix "/k3s-agent-token" coord.services.k3s.tokenFile; +assert !(coord.age.secrets ? k3s-token); +# pods never open a connection to the desk itself; the desk outweighs kubepods +assert lib.hasInfix + "-i cni0 -m conntrack --ctstate NEW -m comment --comment ax-fleet-pod-input -j nixos-fw-refuse" + coord.networking.firewall.extraCommands; +assert coord.systemd.slices.user.sliceConfig.CPUWeight == 10000; +assert coord.systemd.slices.system.sliceConfig.CPUWeight == 10000; +# kube-proxy leaves the host's conntrack table as it is +assert builtins.all + ( + h: + has h "--kube-proxy-arg=conntrack-tcp-timeout-established=0s" + && has h "--kube-proxy-arg=conntrack-max-per-core=0" + ) + [ + nas + coord + ]; +# the VM coordinator runs the desk's kernel +assert + coord.boot.kernelPackages.kernel.outPath == (testOn "coordinator") + .boot.kernelPackages.kernel.outPath; +# the NAS-from-boot VM runs the NAS's release line and kernel +assert + self.checks.x86_64-linux.ax-fleet-boot.nodes.nas.system.nixos.release == nas.system.nixos.release; +assert + self.checks.x86_64-linux.ax-fleet-boot.nodes.nas.boot.kernelPackages.kernel.outPath + == nas.boot.kernelPackages.kernel.outPath; +# image parity with the VM +assert builtins.all (n: vmSeeds ? ${n} && vmSeeds.${n} == nasSeeds.${n}) (lib.attrNames nasSeeds); + +# the kill switch; the teardown stays on the host's PATH with the switch off +assert builtins.all (h: hasTeardown (offCfg h)) [ + "nas" + "coordinator" +]; +assert !(hasTeardown worker); +assert builtins.all killed [ + "nas" + "coordinator" + "worker" +]; +assert builtins.all guardsKept [ + "nas" + "coordinator" + "worker" +]; +# the NAS's pods: Halogen on its port, no other private address (fix round 3) +assert lib.hasInfix "ip daddr 10.42.0.5 tcp dport 8731 return" nas.networking.nftables.tables.ax-fleet-guard.content; +assert + lib.replaceStrings [ "tailscale0" ] [ "eth2" ] nas.networking.nftables.tables.ax-fleet-guard.content + == (testOn "nas").networking.nftables.tables.ax-fleet-guard.content; + +# parity with the VM test +assert normFlags nasSubst nas == normFlags { } (testOn "nas"); +assert normFlags coordSubst coord == normFlags { } (testOn "coordinator"); +assert + map (lib.replaceStrings [ "enp1s0" ] [ "eth1" ]) (axLines nas.networking.firewall.extraInputRules) + == axLines (testOn "nas").networking.firewall.extraInputRules; +assert guardText coordSubst coord == guardText { } (testOn "coordinator"); + +pkgs.runCommand "ax-fleet-topology" { } '' + touch "$out" +'' diff --git a/tests/ax-fleet/default.nix b/tests/ax-fleet/default.nix new file mode 100644 index 000000000..d91976b36 --- /dev/null +++ b/tests/ax-fleet/default.nix @@ -0,0 +1,174 @@ +{ + pkgs, + lib, + inputs, +}: +# checks.x86_64-linux.ax-fleet: the 4-VM proof before any switch (DESIGN.md +# 12.1). The script mirrors the real motion: baseline, switch the NAS, switch +# the coordinator (the worker is not switched), Tasks, resilience, rollback. +# +# The test script is phases/*.py concatenated in name order, after the +# prelude below: 10-cluster and 90-rollback (cluster track), 20-substrate +# (substrate track), 30-ax (ax track). Every subtest a phase runs through +# `step(...)` is named in $out/receipt.json with the values it recorded. +let + nodes = import ./nodes.nix { inherit pkgs lib inputs; }; + teardown = pkgs.callPackage ../../pkgs/ax-fleet-teardown { + k3s = inputs.nixpkgs.legacyPackages.x86_64-linux.k3s_1_36; + }; + phaseDir = ./phases; + phaseFiles = lib.sort (a: b: a < b) ( + lib.filter (n: lib.hasSuffix ".py" n) (lib.attrNames (builtins.readDir phaseDir)) + ); + prelude = '' + import json + import os + import time + from contextlib import contextmanager + + # 90-rollback runs the teardown from the host's PATH, as documented; this + # store path is only compared against it. + TEARDOWN = "${teardown}/bin/ax-fleet-teardown" + PROBE_IMAGE = "ax-fleet-probe:test" + PROBE_TARBALL = "${nodes.probeImage}" + # Test-only task image variant with claude-code (nodes.nix); its OCI layout + # carries the manifest digest the Task names. + CLAUDE_PROBE_OCI = "${nodes.claudeProbeImage}" + LOCAL_PATH_ROOT = "/mnt/nas/services/ax-fleet/local-path" + SYSCTLS = [ + "net.ipv4.ip_forward", + "net.ipv6.conf.all.forwarding", + "kernel.panic", + "kernel.panic_on_oops", + "vm.overcommit_memory", + # kube-proxy's conntrack keys (fix round 2): the house router's NAT + # table must keep the host's timeouts. + "net.netfilter.nf_conntrack_max", + "net.netfilter.nf_conntrack_tcp_timeout_established", + "net.netfilter.nf_conntrack_tcp_timeout_close_wait", + ] + + from typing import Any + + receipt_subtests: list[dict[str, Any]] = [] + receipt_values: dict[str, Any] = {} + receipt: dict[str, Any] = {"test": "ax-fleet", "subtests": receipt_subtests, "values": receipt_values} + + + def save_receipt(): + out = os.environ.get("out", ".") + os.makedirs(out, exist_ok=True) + with open(os.path.join(out, "receipt.json"), "w") as f: + json.dump(receipt, f, indent=2, sort_keys=True) + + + def record(key, value): + receipt_values[key] = value + # Also into the build log: a failed build keeps no $out. + print(f"AXFLEET-RECORD {key} = {json.dumps(value, sort_keys=True)}") + save_receipt() + + + @contextmanager + def step(name): + t0 = time.monotonic() + with subtest(name): + yield + receipt_subtests.append({"name": name, "result": "pass", "seconds": round(time.monotonic() - t0, 1)}) + save_receipt() + + + def kubectl(args): + """kubectl on the NAS, as root, with the k3s admin kubeconfig.""" + return nas.succeed(f"k3s kubectl {args}") + + + def jsonpath(obj, path): + return kubectl(f"get {obj} -o jsonpath='{path}'").strip() + + + def sysctls(machine): + return {k: machine.succeed(f"sysctl -n {k}").strip() for k in SYSCTLS} + + + def node_ready(name): + nas.wait_until_succeeds( + f"k3s kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}' | grep -x True", + timeout=600, + ) + + + KERNEL_KEYS = ("kernel.panic", "kernel.panic_on_oops", "vm.overcommit_memory") + + + def kernel_keys_back(machine, baseline): + """myAxFleet.kubelet.keepHostKernelTunables: kubelet's values are put back.""" + for k in KERNEL_KEYS: + machine.wait_until_succeeds(f"test \"$(sysctl -n {k})\" = '{baseline[k]}'", timeout=300) + + + def flap_until_unreachable(tag): + """Take the coordinator's LAN leg down until the control plane has + reacted (Ready=Unknown and the unreachable taint), then bring it back. + A test parameter, not an estimate: it waits for the transition.""" + t0 = time.monotonic() + coordinator.succeed("ip link set eth1 down") + try: + nas.wait_until_succeeds( + "k3s kubectl get node coordinator -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -qx Unknown", + timeout=900, + ) + nas.wait_until_succeeds( + "k3s kubectl get node coordinator -o jsonpath='{.spec.taints[*].key}' | grep -qw node.kubernetes.io/unreachable", + timeout=600, + ) + record(f"flap_{tag}_taints_while_down", nas.succeed("k3s kubectl get node coordinator -o jsonpath='{.spec.taints}'").strip()) + finally: + coordinator.succeed("ip link set eth1 up") + outage = round(time.monotonic() - t0, 1) + record(f"flap_{tag}_outage_seconds", outage) + return outage + + + def user_unit_pid(unit): + return coordinator.succeed( + "runuser -u alice -- env XDG_RUNTIME_DIR=/run/user/$(id -u alice) " + f"systemctl --user show {unit} -p MainPID --value" + ).strip() + + + def nm_invocation(): + return coordinator.succeed("systemctl show NetworkManager -p InvocationID --value").strip() + + + def unit_invocation(machine, unit): + return machine.succeed(f"systemctl show {unit} -p InvocationID --value").strip() + + ''; +in +pkgs.testers.runNixOSTest { + name = "ax-fleet"; + node.specialArgs = { inherit inputs; }; + nodes = { + inherit (nodes) + nas + coordinator + worker + peer + ; + }; + # Long waits are test parameters (k3s start, image import), not estimates. + globalTimeout = 3 * 3600; + testScript = + prelude + + lib.concatMapStrings (f: '' + + # ───────────── phases/${f} ───────────── + ${builtins.readFile (phaseDir + "/${f}")} + '') phaseFiles + + '' + + save_receipt() + ''; + passthru.axFleet = { inherit nodes teardown; }; +} diff --git a/tests/ax-fleet/halogen_stub.py b/tests/ax-fleet/halogen_stub.py new file mode 100644 index 000000000..e5efe6f37 --- /dev/null +++ b/tests/ax-fleet/halogen_stub.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Halogen stand-in for the ax-fleet VM test (DESIGN.md 12.1, node `worker`). + +OpenAI-compatible enough for the ax task image's two model paths: + - `halogen-smoke`: one non-streaming POST /v1/chat/completions (curl); + - `pi`: streaming POST /v1/chat/completions (SSE, `data: {...}` chunks, then + `data: [DONE]`). +Also GET /v1/models and GET /health, like the real server. + +Every request is appended to --log as one JSON line with the source address +and a per-request id, so the test can prove that sandbox traffic arrived +SNAT'd from the NAS (10.42.0.1) and count requests per Task. +""" + +import argparse +import json +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +MODEL = "halogen-qwen3.8-flash-next" +REPLY = "halogen-stub-ok" +# The pi mode asks for one JSON object against a JSON Schema in its system +# prompt; the adapter's default schema wants {"answer": }. +JSON_REPLY = '{"answer": 42}' + + +def reply_for(req): + """REPLY, or JSON_REPLY when any message mentions a JSON Schema.""" + for msg in req.get("messages") or []: + content = msg.get("content") if isinstance(msg, dict) else None + if isinstance(content, list): + content = " ".join(str(p.get("text", "")) for p in content if isinstance(p, dict)) + if isinstance(content, str) and "JSON Schema" in content: + return JSON_REPLY + return REPLY +LOCK = threading.Lock() + + +def make_handler(log_path): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): # quiet stderr; the jsonl is the log + pass + + def _record(self, body): + entry = { + "id": uuid.uuid4().hex, + "ts": time.time(), + "src": self.client_address[0], + "method": self.command, + "path": self.path, + "bytes": len(body), + } + with LOCK, open(log_path, "a", encoding="utf-8") as f: + f.write(json.dumps(entry) + "\n") + return entry["id"] + + def _json(self, code, obj): + data = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + rid = self._record(b"") + if self.path.rstrip("/") == "/health": + self._json(200, {"status": "ok", "in_flight": 0, "queued": 0, "request_id": rid}) + elif self.path.rstrip("/") == "/v1/models": + self._json(200, {"object": "list", "data": [{"id": MODEL, "object": "model", "owned_by": "halogen"}]}) + else: + self._json(404, {"error": "not found", "request_id": rid}) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + rid = self._record(body) + if self.path.rstrip("/") != "/v1/chat/completions": + self._json(404, {"error": "not found", "request_id": rid}) + return + try: + req = json.loads(body or b"{}") + except json.JSONDecodeError: + self._json(400, {"error": "bad json", "request_id": rid}) + return + created = int(time.time()) + cid = "chatcmpl-" + rid + if not req.get("stream"): + self._json( + 200, + { + "id": cid, + "object": "chat.completion", + "created": created, + "model": MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": reply_for(req)}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + chunks = [ + {"role": "assistant", "content": ""}, + {"content": reply_for(req)}, + ] + for i, delta in enumerate(chunks + [None]): + choice = {"index": 0, "delta": delta or {}, "finish_reason": None if delta else "stop"} + obj = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": MODEL, "choices": [choice]} + if delta is None: + obj["usage"] = {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + self.wfile.write(b"data: " + json.dumps(obj).encode() + b"\n\n") + self.wfile.flush() + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + self.close_connection = True + + return Handler + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8731) + ap.add_argument("--log", default="/tmp/halogen-stub.jsonl") + args = ap.parse_args() + ThreadingHTTPServer(("0.0.0.0", args.port), make_handler(args.log)).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/ax-fleet/nodes.nix b/tests/ax-fleet/nodes.nix new file mode 100644 index 000000000..9feaa3892 --- /dev/null +++ b/tests/ax-fleet/nodes.nix @@ -0,0 +1,369 @@ +{ + pkgs, + lib, + inputs, +}: +# The four VMs of checks.x86_64-linux.ax-fleet (DESIGN.md 12.1), shared with +# checks.x86_64-linux.ax-fleet-boot. They carry the fleet's real hostnames and +# LAN addresses, so every manifest and firewall rule the modules render is the +# production one; only interface names (eth1, eth2), the token and the kubelet +# reservations differ, and ax-fleet-topology pins that parity. +# +# Each base config is "today", with ax OFF. `specialisation.ax-on` sets +# myAxFleet.enable = true, and the test script switches to it live, NAS first, +# exactly as Tom will. +let + # A plain file, test-only, not a secret: the fleet reads agenix instead. + token = pkgs.writeText "ax-fleet-vm-token" "ax-fleet-vm-test-token-0123456789abcdef"; + # The agent credential, distinct from the server token as on the fleet. + agentToken = pkgs.writeText "ax-fleet-vm-agent-token" "ax-fleet-vm-agent-token-fedcba9876543210"; + + # busybox (httpd, nslookup) plus curl, imported by k3s from the images + # directory on both nodes, so probe pods need no registry and no network. + probeImage = pkgs.dockerTools.buildImage { + name = "ax-fleet-probe"; + tag = "test"; + copyToRoot = pkgs.buildEnv { + name = "ax-fleet-probe-root"; + paths = [ + pkgs.busybox + pkgs.curl + pkgs.cacert + ]; + pathsToLink = [ + "/bin" + "/etc" + ]; + }; + config.Cmd = [ + "/bin/sh" + "-c" + "sleep 1000000" + ]; + }; + + # Test-only: the fleet task image (pkgs/ax-agent-image, the same ax and pi + # the fleet seeds) plus claude-code, for 25-harness-probe's `claude + # --version`. No credential exists in it or in any Task. The fleet image + # itself stays pi-only on day one (DESIGN.md 10.2 and 11); only the NAS test + # node seeds this variant, from its ax-on specialisation. + claudeProbeImage = + inputs.nixpkgs.legacyPackages.x86_64-linux.callPackage ../../pkgs/ax-agent-image + { + ax = inputs.self.packages.x86_64-linux.ax; + pi = inputs.llm-agents.packages.x86_64-linux.pi; + extraPaths = [ inputs.llm-agents.packages.x86_64-linux.claude-code ]; + variant = "-claude-probe"; + }; + + lanAddr = address: { + interface = "eth1"; + inherit address; + }; + + setAddr = iface: address: prefixLength: { + networking.interfaces.${iface}.ipv4.addresses = lib.mkForce [ { inherit address prefixLength; } ]; + }; + + # Everything that makes a node a fleet node in the test. + fleetNode = + { role, address }: + { + imports = [ + ../../modules/ax-fleet + # As on the real hosts (hosts/{coordinator,worker,client}): the harness + # role turns myAxClient on by mkDefault, which puts `ax` and `kubectl` + # on the coordinator's PATH for the Task phases. + ../../modules/ax-client.nix + inputs.agenix.nixosModules.default + ]; + system.switch.enable = true; + myAxFleet = { + inherit role; + lan = lanAddr address; + k3sTokenFile = "${token}"; + k3sAgentTokenFile = "${agentToken}"; + guardInterfaces = [ "eth2" ]; + }; + environment.systemPackages = [ + pkgs.jq + pkgs.curl + pkgs.iptables + pkgs.nftables + pkgs.iproute2 + pkgs.dnsutils + ]; + }; + + axOn = extra: { + specialisation.ax-on.configuration = lib.mkMerge [ + { + myAxFleet.enable = true; + services.k3s.images = [ probeImage ]; + } + extra + ]; + }; + + vmReservations = { + systemReserved = "cpu=500m,memory=512Mi"; + kubeReserved = "cpu=250m,memory=256Mi"; + evictionHard = "memory.available<256Mi"; + }; +in +{ + inherit + probeImage + token + agentToken + claudeProbeImage + ; + + nas = + { ... }: + { + imports = [ + (fleetNode { + role = "control"; + address = "10.42.0.1"; + }) + (axOn { + myAxFleet.kubelet = { + systemReserved = "cpu=1,memory=1Gi"; + }; + myAxFleet.registrySeed.ax-agent-claude-probe = { + oci = claudeProbeImage; + repo = "ax/ax-agent-claude-probe"; + inherit (claudeProbeImage.passthru) tag; + }; + }) + (setAddr "eth1" "10.42.0.1" 24) + ]; + networking.hostName = "nas"; + virtualisation = { + vlans = [ 1 ]; + memorySize = 10240; + cores = 6; + diskSize = 8192; + emptyDiskImages = [ + 8192 + 16384 + ]; + fileSystems = { + "/mnt/fast" = { + device = "/dev/vdb"; + fsType = "ext4"; + autoFormat = true; + }; + "/mnt/nas" = { + device = "/dev/vdc"; + fsType = "btrfs"; + autoFormat = true; + }; + }; + }; + boot.supportedFilesystems = [ "btrfs" ]; + # The house router, as hosts/nas/router.nix declares it (fix round 2): + # the snapshot and the teardown must keep it. + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + + # The NAS firewall as the real one is shaped: nftables, interface-scoped + # extraInputRules, filterForward off, strict rpfilter, and a reload that + # keeps other tables (flushRuleset = false). + networking.nftables.enable = true; + networking.nftables.flushRuleset = false; + networking.firewall = { + enable = true; + filterForward = false; + checkReversePath = "strict"; + extraInputRules = '' + iifname "eth1" udp dport 53 accept comment "stand-in AdGuard" + iifname "eth1" tcp dport 53 accept comment "stand-in AdGuard" + ''; + }; + + # AdGuard stand-in: answers one record nobody else serves. + services.dnsmasq = { + enable = true; + resolveLocalQueries = false; + settings = { + listen-address = [ "10.42.0.1" ]; + bind-interfaces = true; + no-resolv = true; + address = [ "/only-nas.test/10.42.0.77" ]; + }; + }; + + # A recording `tailscale` on the system PATH, as the real NAS has one: + # k3s-killall.sh's remove_interfaces runs `tailscale set + # --advertise-routes=` whenever the binary is reachable, which would + # withdraw the house subnet route. 90-rollback asserts the teardown + # never calls it. + environment.systemPackages = [ + (pkgs.writeShellScriptBin "tailscale" '' + echo "$*" >> /var/log/tailscale-stub.log + '') + ]; + + # The bystander: the NAS's shared PostgreSQL (Paperless, Immich) must + # not restart and must not change. + services.postgresql = { + enable = true; + ensureDatabases = [ "paperless" ]; + }; + }; + + coordinator = + { ... }: + { + imports = [ + (fleetNode { + role = "harness"; + address = "10.42.0.2"; + }) + (axOn { myAxFleet.kubelet = vmReservations; }) + (setAddr "eth1" "10.42.0.2" 24) + (setAddr "eth2" "100.105.121.73" 10) + ]; + networking.hostName = "coordinator"; + # The desk's wired port (fix round 3): eth3, NetworkManager-managed, + # DHCP from the worker's second leg, as enp191s0's "Wired connection 1". + myAxFleet.lan.extraInterfaces = [ "eth3" ]; + # myAxFleet.apiUsers defaults to [ "tom" ]; alice is the other local user. + users.users.tom.isNormalUser = true; + virtualisation = { + vlans = [ + 1 + 2 + 3 + ]; + memorySize = 8192; + cores = 4; + diskSize = 8192; + podman.enable = true; + }; + # The desk's shape: iptables firewall, strict rpfilter, NetworkManager + # running (its InvocationID must not change at switch), zram swap. + networking.nftables.enable = false; + networking.firewall = { + enable = true; + checkReversePath = "strict"; + allowedTCPPorts = [ 80 ]; + }; + networking.networkmanager.enable = true; + # eth1/eth2 are the test driver's static addresses; identical in base + # and specialisation, so NetworkManager.conf does not change at switch. + networking.networkmanager.unmanaged = [ + "eth1" + "eth2" + ]; + zramSwap.enable = true; + # The desk's kernel, where runsc runs (fix round 2: the VM had run the + # pin's default kernel). The same expression as modules/strix.nix, so + # the same derivation; ax-fleet-topology asserts it. + boot.kernelPackages = + (import inputs.nixpkgs-fresh { + system = "x86_64-linux"; + config.allowUnfree = true; + }).linuxPackages_7_2; + # The desk's sshd as modules/common.nix renders it: port 22 open on every + # interface, passwords accepted (fix round 2). Pods must not reach it. + services.openssh = { + enable = true; + openFirewall = true; + settings.PasswordAuthentication = true; + settings.KbdInteractiveAuthentication = true; + }; + + services.caddy = { + enable = true; + virtualHosts.":80".extraConfig = '' + respond "caddy-ok" + ''; + }; + + # herdr stand-in: a lingering user's long-lived user unit with the same + # X-SwitchMethod herdr has. Its MainPID must survive switch and rollback. + users.users.alice = { + isNormalUser = true; + linger = true; + }; + systemd.user.services.herdr-standin = { + wantedBy = [ "default.target" ]; + unitConfig."X-SwitchMethod" = "keep-old"; + serviceConfig.ExecStart = "${pkgs.coreutils}/bin/sleep 1000000"; + }; + }; + + worker = + { ... }: + { + imports = [ + (fleetNode { + role = "inference"; + address = "10.42.0.5"; + }) + (setAddr "eth1" "10.42.0.5" 24) + (setAddr "eth2" "192.168.43.5" 24) + ]; + networking.hostName = "worker"; + virtualisation.vlans = [ + 1 + 3 + ]; + # vlan 3: a second LAN segment for the coordinator's wired leg (fix + # round 3). DHCP with no router option, so no default route moves. + services.dnsmasq = { + enable = true; + resolveLocalQueries = false; + settings = { + port = 0; + interface = [ "eth2" ]; + bind-interfaces = true; + dhcp-range = [ "192.168.43.100,192.168.43.150,1h" ]; + dhcp-option = [ "3" ]; + }; + }; + # Another worker port (fix round 3): the real worker opens 22 with + # passwords on every interface; pods and the egress gateway must reach + # 8731 and nothing else. + systemd.services.worker-2222 = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.busybox}/bin/httpd -f -p 2222 -h ${pkgs.writeTextDir "index.html" "worker-port-2222-reached\n"}"; + }; + # The worker is not switched in the motion; its fleet role is ON from + # boot and renders only the 8731 assertion, as on the real host. + myAxFleet.enable = true; + networking.firewall = { + enable = true; + interfaces.eth1.allowedTCPPorts = [ + 8731 + 2222 + ]; + interfaces.eth2.allowedTCPPorts = [ 8731 ]; + interfaces.eth2.allowedUDPPorts = [ 67 ]; + }; + systemd.services.halogen-stub = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${pkgs.python3}/bin/python3 ${./halogen_stub.py} --port 8731 --log /var/lib/halogen-stub/requests.jsonl"; + StateDirectory = "halogen-stub"; + }; + }; + }; + + peer = + { ... }: + { + imports = [ (setAddr "eth1" "100.64.0.9" 10) ]; + networking.hostName = "peer"; + virtualisation.vlans = [ 2 ]; + networking.firewall.enable = false; + environment.systemPackages = [ pkgs.curl ]; + # Something to reach: the guard must stop pods from getting here. + systemd.services.peer-http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.busybox}/bin/httpd -f -p 8000 -h /etc"; + }; + }; +} diff --git a/tests/ax-fleet/phases/10-cluster.py b/tests/ax-fleet/phases/10-cluster.py new file mode 100644 index 000000000..ae9bed9cb --- /dev/null +++ b/tests/ax-fleet/phases/10-cluster.py @@ -0,0 +1,392 @@ +# Phase 1 (baseline), 2 and 3 (switch nas, then coordinator): the cluster +# assertions. Track cluster. DESIGN.md 12.1 and 6.6. The substrate and ax +# phases (20, 30) run after this one, on the cluster it leaves behind. + +AX_ON = "/run/current-system/specialisation/ax-on/bin/switch-to-configuration test" + +PROBE_POD = """ +apiVersion: v1 +kind: Pod +metadata: + name: {name} + namespace: default + labels: {{app: ax-fleet-probe}} +spec: + nodeSelector: {{ax.mecattaf.dev/role: {role}}} + tolerations: + - {{key: ate.dev/sandboxClass, operator: Exists, effect: NoSchedule}} + containers: + - name: probe + image: ax-fleet-probe:test + imagePullPolicy: Never + command: ["/bin/sh", "-c", "echo probe-started-{name}; mkdir -p /www; echo pod-ok > /www/index.html; exec httpd -f -p 8000 -h /www"] + ports: + - {{containerPort: 8000, hostPort: {host_port}}} + {volume_mounts} + {volumes} +""" + + +def apply_probe(name, role, host_port, pvc=None): + vm = "volumeMounts: [{name: data, mountPath: /data}]" if pvc else "" + vols = f"volumes: [{{name: data, persistentVolumeClaim: {{claimName: {pvc}}}}}]" if pvc else "" + doc = PROBE_POD.format(name=name, role=role, host_port=host_port, volume_mounts=vm, volumes=vols) + nas.succeed(f"cat > /tmp/{name}.yaml <<'EOF'\n{doc}\nEOF") + kubectl(f"apply -f /tmp/{name}.yaml") + kubectl(f"wait --for=condition=Ready pod/{name} --timeout=300s") + + +with step("baseline"): + start_all() + for m in (nas, coordinator, worker, peer): + m.wait_for_unit("multi-user.target") + coordinator.wait_for_unit("caddy.service") + coordinator.wait_for_unit("NetworkManager.service") + nas.wait_for_unit("postgresql.service") + nas.wait_for_unit("dnsmasq.service") + worker.wait_for_unit("halogen-stub.service") + worker.wait_for_open_port(8731) + coordinator.wait_until_succeeds( + "runuser -u alice -- env XDG_RUNTIME_DIR=/run/user/$(id -u alice) systemctl --user is-active herdr-standin" + ) + worker.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + peer.succeed("curl -sf --max-time 10 http://100.105.121.73/ | grep -x caddy-ok") + # Not a k3s node, never switched: nothing from the fleet module runs here. + worker.fail("systemctl cat k3s.service") + nas.fail("systemctl cat k3s.service") + coordinator.fail("systemctl cat k3s.service") + + base = { + "herdr_pid": user_unit_pid("herdr-standin"), + "nm_invocation": nm_invocation(), + "postgres_invocation": unit_invocation(nas, "postgresql.service"), + "nas_databases": sorted(nas.succeed("runuser -u postgres -- psql -Atc 'select datname from pg_database'").split()), + "nas_nft": nas.succeed("nft -s list ruleset"), + "nas_nixos_fw": nas.succeed("nft -s list table inet nixos-fw"), + "sysctl_nas": sysctls(nas), + "sysctl_coordinator": sysctls(coordinator), + } + record("baseline", {k: v for k, v in base.items() if k not in ("nas_nft", "nas_nixos_fw")}) + + +with step("switch nas"): + nas.succeed(f"{AX_ON} >&2") + nas.succeed("systemctl is-active k3s.service docker-registry.service") + for u in ("ax-fleet-registry-seed.service", "ax-fleet-bootstrap.service"): + nas.succeed(f"systemctl is-active {u}") + nas.succeed(f"systemctl show {u} -p Result --value | grep -x success") + + +with step("nas: k3s state on the fast tier, links on that disk"): + src = nas.succeed("findmnt -n -o SOURCE -T /var/lib/rancher/k3s").strip() + record("nas_rancher_source", src) + assert src.startswith("/dev/vdb"), f"/var/lib/rancher/k3s is on {src}, not the /mnt/fast disk" + for p in ("/var/lib/kubelet", "/var/log/pods"): + s = nas.succeed(f"findmnt -n -o SOURCE -T {p}").strip() + assert s.startswith("/dev/vdb"), f"{p} is on {s}" + # The airgap images link lives under the bind mount, i.e. on /mnt/fast. + nas.succeed("ls -l /var/lib/rancher/k3s/agent/images/ | grep -q airgap") + nas.succeed("ls -l /mnt/fast/k3s/rancher/k3s/agent/images/ | grep -q airgap") + nas.succeed("test -s /etc/ax-fleet/admin.kubeconfig") + nas.succeed("stat -c '%a %U %G' /etc/ax-fleet/admin.kubeconfig | grep -x '640 root wheel'") + nas.succeed("grep -q 'server: https://10.42.0.1:6443' /etc/ax-fleet/admin.kubeconfig") + nas.succeed("test -s /var/lib/ax-fleet/sysctl-before.conf") + # The house router's forwarding is what the snapshot says (fix round 2). + nas.succeed("grep -x 'net.ipv4.ip_forward = 1' /var/lib/ax-fleet/sysctl-before.conf") + + +with step("nas: node Ready, untainted, control labels"): + node_ready("nas") + # k3s's own transient taints (uninitialized, not-ready) clear on their own. + nas.wait_until_succeeds("test -z \"$(k3s kubectl get node nas -o jsonpath='{.spec.taints}')\"", timeout=300) + labels = json.loads(kubectl("get node nas -o jsonpath='{.metadata.labels}'")) + assert labels.get("ax.mecattaf.dev/role") == "control", labels + assert labels.get("ate.dev/substrate-version") == "none", labels + ip = jsonpath("node nas", "{.status.addresses[?(@.type==\"InternalIP\")].address}") + assert ip == "10.42.0.1", ip + kubectl("-n kube-system wait --for=condition=Available deploy/coredns deploy/local-path-provisioner --timeout=600s") + + +with step("nas: certificates.k8s.io/v1beta1 serves the Substrate resources"): + res = json.loads(kubectl("get --raw /apis/certificates.k8s.io/v1beta1")) + names = sorted(r["name"] for r in res["resources"]) + record("certificates_v1beta1", names) + assert "clustertrustbundles" in names and "podcertificaterequests" in names, names + + +with step("nas: PersistentVolumes land on the data pool"): + nas.succeed( + "cat > /tmp/pvc.yaml <<'EOF'\n" + "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata: {name: ax-fleet-probe-pvc, namespace: default}\n" + "spec: {accessModes: [ReadWriteOnce], storageClassName: local-path, resources: {requests: {storage: 16Mi}}}\n" + "EOF" + ) + kubectl("apply -f /tmp/pvc.yaml") + apply_probe("probe-nas", "control", 18086, pvc="ax-fleet-probe-pvc") + kubectl("exec probe-nas -- sh -c 'echo data-pool > /data/marker'") + paths = kubectl( + "get pv -o jsonpath='{range .items[*]}{.spec.hostPath.path}{.spec.local.path}{\"\\n\"}{end}'" + ).split() + record("pv_paths", paths) + assert paths and all(p.startswith(LOCAL_PATH_ROOT) for p in paths), paths + nas.succeed(f"grep -rqx data-pool {LOCAL_PATH_ROOT}") + src = nas.succeed(f"findmnt -n -o SOURCE -T {LOCAL_PATH_ROOT}").strip() + assert src.startswith("/dev/vdc"), f"local-path root is on {src}, not the data pool" + + +with step("nas: bystanders untouched"): + assert unit_invocation(nas, "postgresql.service") == base["postgres_invocation"], "the shared PostgreSQL restarted" + dbs = sorted(nas.succeed("runuser -u postgres -- psql -Atc 'select datname from pg_database'").split()) + assert dbs == base["nas_databases"], (dbs, base["nas_databases"]) + nas.succeed("dig +short @10.42.0.1 only-nas.test | grep -x 10.42.0.77") + ruleset = nas.succeed("nft list ruleset") + record("nas_kube_services_in_nft", "KUBE-SERVICES" in ruleset) + nas.succeed("nft list chain inet ax-fleet-guard prerouting | grep -q 'ax-fleet: cluster ranges'") + record("sysctl_nas_after_switch", sysctls(nas)) + kernel_keys_back(nas, base["sysctl_nas"]) + + +with step("switch coordinator"): + coordinator.succeed(f"{AX_ON} >&2") + coordinator.succeed("systemctl is-active k3s.service ax-fleet-nm-unmanaged.service ax-server-proxy.socket") + node_ready("coordinator") + + +with step("coordinator: Ready with the harness taint and labels"): + taints = json.loads(kubectl("get node coordinator -o jsonpath='{.spec.taints}'")) + assert {"key": "ate.dev/sandboxClass", "value": "gvisor", "effect": "NoSchedule"} in taints, taints + labels = json.loads(kubectl("get node coordinator -o jsonpath='{.metadata.labels}'")) + assert labels.get("ax.mecattaf.dev/role") == "harness", labels + assert labels.get("ate.dev/substrate-version") == "d277088b", labels + ip = jsonpath("node coordinator", "{.status.addresses[?(@.type==\"InternalIP\")].address}") + assert ip == "10.42.0.2", ip + # flannel.1 appears shortly after the node registers; wait for it. + coordinator.wait_until_succeeds("ip -d link show flannel.1 | grep -q 'dev eth1'", timeout=180) + record("sysctl_coordinator_after_switch", sysctls(coordinator)) + + +with step("coordinator: only tolerating pods land here; control stays on the NAS"): + apply_probe("probe-coord", "harness", 18085) + placed = kubectl( + "get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.spec.nodeName}{\"\\n\"}{end}'" + ) + record("pod_placement", placed.strip().splitlines()) + for line in placed.strip().splitlines(): + name, node = line.split() + if node == "coordinator": + assert ( + name == "default/probe-coord" or "/atelet" in name or "/ateom-gvisor" in name + ), f"{name} landed on the coordinator" + # kubectl logs of a coordinator pod rides the agent tunnel (egress-selector agent). + kubectl("logs probe-coord | grep -q probe-started-probe-coord") + + +with step("coordinator: nothing changed for Tom"): + assert user_unit_pid("herdr-standin") == base["herdr_pid"], "herdr stand-in restarted" + assert nm_invocation() == base["nm_invocation"], "NetworkManager restarted" + coordinator.wait_until_succeeds("nmcli -t -f DEVICE,STATE d | grep -x 'cni0:unmanaged'", timeout=60) + coordinator.succeed("nmcli -t -f DEVICE,STATE d | grep -x 'flannel.1:unmanaged'") + record("nmcli_devices", coordinator.succeed("nmcli -t -f DEVICE,STATE d").strip().splitlines()) + worker.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + peer.succeed("curl -sf --max-time 10 http://100.105.121.73/ | grep -x caddy-ok") + # A podman container on the default bridge still reaches the host's Caddy + # (br_netfilter is loaded now, rpfilter is strict). + loaded = coordinator.succeed(f"podman load -i {PROBE_TARBALL}") + ref = [l.split("Loaded image:")[1].strip() for l in loaded.splitlines() if "Loaded image" in l][0] + coordinator.succeed(f"podman run --rm --network bridge {ref} curl -sf --max-time 10 http://10.88.0.1/ | grep -x caddy-ok") + coordinator.succeed("lsmod | grep -q br_netfilter") + # kubelet's panic tunables are put back: an oops does not reboot the desk. + kernel_keys_back(coordinator, base["sysctl_coordinator"]) + record("sysctl_coordinator_after_restore", sysctls(coordinator)) + + +with step("coordinator: kubelet's tunables are put back whenever they appear, not only after a k3s start"): + # Fix round 2. kubelet applies them when its container manager starts, + # which on an agent is when the server first answers, possibly long after + # k3s.service started (MEASURED by the review). Write kubelet's values with + # no k3s restart, well after the switch: the watcher must restore them. + nrestarts = coordinator.succeed("systemctl show k3s.service -p NRestarts --value").strip() + coordinator.succeed("sysctl -w kernel.panic=10 kernel.panic_on_oops=1 vm.overcommit_memory=1") + kernel_keys_back(coordinator, base["sysctl_coordinator"]) + assert coordinator.succeed("systemctl show k3s.service -p NRestarts --value").strip() == nrestarts + coordinator.succeed("systemctl is-active ax-fleet-kernel-tunables.service") + + +with step("coordinator: the desk outweighs kubepods for CPU"): + w = {s: int(coordinator.succeed(f"cat /sys/fs/cgroup/{s}/cpu.weight").strip()) for s in ("user.slice", "system.slice", "kubepods.slice")} + record("cpu_weights", w) + assert w["user.slice"] > w["kubepods.slice"] and w["system.slice"] > w["kubepods.slice"], w + + +with step("coordinator: the guards hold"): + pod_ip = jsonpath("pod probe-coord", "{.status.podIP}") + record("probe_coord_ip", pod_ip) + # Make the negative tests discriminating: the peer routes the LAN and the + # pod network through the coordinator, and the worker routes the tailnet + # back through it. Without the guard chain, ip_forward=1 would carry these. + peer.succeed("ip route replace 10.42.0.0/24 via 100.105.121.73") + peer.succeed("ip route replace 10.200.0.0/16 via 100.105.121.73") + worker.succeed("ip route replace 100.64.0.0/10 via 10.42.0.2") + peer.fail("curl -sf --max-time 5 http://10.42.0.5:8731/health") + peer.fail(f"curl -sf --max-time 5 http://{pod_ip}:8000/") + for port in (8085, 9090, 10250, 6443, 18085): + peer.fail(f"curl -sk --max-time 5 -o /dev/null https://100.105.121.73:{port}/") + peer.fail(f"curl -s --max-time 5 -o /dev/null http://100.105.121.73:{port}/") + # hostPorts: from the NAS yes (atelet's peers), from the worker no. + nas.succeed("curl -sf --max-time 10 http://10.42.0.2:18085/ | grep -x pod-ok") + worker.fail("curl -sf --max-time 5 http://10.42.0.2:18085/") + # Pods never reach the tailnet; the coordinator host itself still does. + coordinator.succeed("curl -sf --max-time 5 http://100.64.0.9:8000/hostname") + kubectl("exec probe-coord -- sh -c '! curl -sf --max-time 5 http://100.64.0.9:8000/hostname'") + # Pods reach the apiserver Service (any HTTP answer is reachability). + code = kubectl("exec probe-coord -- curl -sk -o /dev/null -w '%{http_code}' --max-time 10 https://10.201.0.1/readyz").strip() + assert code != "000", "a coordinator pod cannot reach the apiserver Service" + # Pods never reach the LAN except the NAS's apiserver and registry. + kubectl("exec probe-coord -- sh -c '! curl -sf --max-time 5 http://10.42.0.5:8731/health'") + worker.succeed("ip route del 100.64.0.0/10 via 10.42.0.2") + assert coordinator.succeed("sysctl -n net.ipv6.conf.all.forwarding").strip() == "0" + coordinator.succeed("iptables -t mangle -S nixos-fw-rpfilter | grep -q rpfilter") + coordinator.succeed("iptables -t mangle -S FORWARD 1 | grep -q ax-fleet-guard") + coordinator.fail("ip -br link | grep -qi cilium") + record("guard_chain", coordinator.succeed("iptables -t mangle -S ax-fleet-guard").strip().splitlines()) + + +with step("coordinator: pods never reach the coordinator host (sshd accepts passwords)"): + # Fix round 2. Discriminating: sshd answers the worker on the LAN, and + # port 22 is open on every interface (openFirewall), so only the pod-input + # refusal stops a pod. + worker.succeed("timeout 10 bash -c 'exec 3<>/dev/tcp/10.42.0.2/22; head -c 7 <&3' | grep -x SSH-2.0") + gw = coordinator.succeed("ip -4 -o addr show dev cni0 | awk '{print $4}' | cut -d/ -f1").strip() + fl = coordinator.succeed("ip -4 -o addr show dev flannel.1 | awk '{print $4}' | cut -d/ -f1").strip() + record("pod_input_targets", {"cni0": gw, "flannel.1": fl}) + for ip in ("10.42.0.2", gw, fl, "100.105.121.73"): + kubectl(f"exec probe-coord -- sh -c '! (nc -w 5 {ip} 22 /dev/null | grep -q SSH)'") + # The NAS's pods, over VXLAN, neither. + kubectl(f"exec probe-nas -- sh -c '! (nc -w 5 {fl} 22 /dev/null | grep -q SSH)'") + # Pod reachability of the host's own hostPort and of Services is unchanged. + code = kubectl("exec probe-coord -- curl -sk -o /dev/null -w '%{http_code}' --max-time 10 https://10.201.0.1/readyz").strip() + assert code != "000", "a coordinator pod lost the apiserver Service" + refused = coordinator.succeed("iptables -S nixos-fw | grep -c ax-fleet-pod-input").strip() + assert refused == "2", refused + coordinator.succeed("ip6tables -S nixos-fw | grep -q 'cni0.*ax-fleet-pod-input'") + + +with step("coordinator: pods reach no private range on the LAN leg (the Freebox fallback case)"): + # A private subnet this module does not know, on the same leg, as when + # NetworkManager falls back to the Freebox profile. Discriminating: the + # coordinator host reaches it; a pod must not. + worker.succeed("ip addr add 192.168.77.5/24 dev eth1") + coordinator.succeed("ip route replace 192.168.77.0/24 dev eth1") + try: + coordinator.succeed("curl -sf --max-time 10 http://192.168.77.5:8731/health") + kubectl("exec probe-coord -- sh -c '! curl -s --max-time 5 -o /dev/null http://192.168.77.5:8731/health'") + finally: + coordinator.succeed("ip route del 192.168.77.0/24 dev eth1") + worker.succeed("ip addr del 192.168.77.5/24 dev eth1") + + +with step("coordinator: a second LAN leg (the desk's wired port) is guarded and never takes the routes"): + # Fix round 3. eth3 stands in for enp191s0: NetworkManager-managed, DHCP + # from the worker's second leg (vlan 3). Round 2's guard named only eth1, + # so pod egress and inbound hostPorts over eth3 matched no DROP. + rc, _ = coordinator.execute("timeout 120 sh -c 'until ip -4 -o addr show dev eth3 | grep -q \"inet 192.168.43.\"; do sleep 2; done'") + if rc != 0: + record("eth3_profile_added", True) + coordinator.succeed("nmcli con add type ethernet ifname eth3 con-name wired-eth3 ipv4.method auto ipv6.method ignore") + record("eth3_routes_before_reactivation", coordinator.succeed("ip -4 route show dev eth3").strip().splitlines()) + # The drop-in's metric applies at the next activation, as on the desk, + # where the wired port is down today. + coordinator.succeed("nmcli device disconnect eth3 && nmcli device connect eth3") + coordinator.wait_until_succeeds("ip -4 -o addr show dev eth3 | grep -q 'inet 192.168.43.'", timeout=120) + routes = coordinator.succeed("ip -4 route show dev eth3").strip().splitlines() + record("eth3_routes", routes) + assert routes and all("metric 700" in r for r in routes), routes + addr3 = coordinator.succeed("ip -4 -o addr show dev eth3 | awk '{print $4}' | cut -d/ -f1").strip() + pod_ip = jsonpath("pod probe-coord", "{.status.podIP}") + # Out: the host reaches the worker's second leg; a pod does not. + coordinator.succeed("curl -sf --max-time 10 http://192.168.43.5:8731/health") + kubectl("exec probe-coord -- sh -c '! curl -s --max-time 5 -o /dev/null http://192.168.43.5:8731/health'") + # In: Caddy answers on the second leg; the pod hostPort and a routed + # path into the pod network do not. + worker.succeed(f"curl -sf --max-time 10 http://{addr3}/ | grep -x caddy-ok") + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{addr3}:18085/") + worker.succeed(f"ip route replace 10.200.0.0/16 via {addr3}") + try: + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{pod_ip}:8000/") + finally: + worker.succeed(f"ip route del 10.200.0.0/16 via {addr3}") + + +with step("coordinator: proxy ARP on the pod interfaces only, never on a LAN leg or a NIC that appears later"): + # Fix round 3. Round 2 set conf.default, which every later NIC copies + # (the desk's NICs appear after systemd-sysctl at boot). + veth = coordinator.succeed("ip -o link show type veth | awk -F': ' '{print $2}' | cut -d@ -f1 | head -1").strip() + coordinator.succeed("ip link add axprobe0 type dummy && udevadm settle") + try: + parp = { + i: coordinator.succeed(f"cat /proc/sys/net/ipv4/conf/{i}/proxy_arp").strip() + for i in ("default", "all", "eth1", "eth2", "eth3", "axprobe0", "cni0", "flannel.1", veth) + } + finally: + coordinator.succeed("ip link del axprobe0") + record("proxy_arp", parp) + assert parp["cni0"] == "1" and parp["flannel.1"] == "1" and parp[veth] == "1", parp + assert all(parp[i] == "0" for i in ("default", "all", "eth1", "eth2", "eth3", "axprobe0")), parp + + +with step("coordinator: LAN traffic routed through the NAS never reaches a harness pod"): + # The house default gateway is the NAS. Before fix round 1 this path + # (worker -> nas -> flannel.1 -> harness pod) answered (MEASURED). + pod_ip = jsonpath("pod probe-coord", "{.status.podIP}") + nas.succeed(f"curl -sf --max-time 10 http://{pod_ip}:8000/ | grep -x pod-ok") # the path itself works + worker.succeed("ip route replace 10.200.0.0/16 via 10.42.0.1") + try: + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{pod_ip}:8000/") + finally: + worker.succeed("ip route del 10.200.0.0/16 via 10.42.0.1") + + +def diag(cmds): + out = {} + for name, (machine, cmd) in cmds.items(): + _, text = machine.execute(cmd + " 2>&1") + out[name] = text[-3000:] + return out + + +with step("cluster plumbing: pod to pod across nodes (flannel VXLAN)"): + nas_pod_ip = jsonpath("pod probe-nas", "{.status.podIP}") + status, _ = nas.execute(f"k3s kubectl exec probe-coord -- curl -sf --max-time 10 http://{nas_pod_ip}:8000/") + if status != 0: + record("diag_flannel", diag({ + "coord_routes": (coordinator, "ip route; ip -d link show flannel.1"), + "nas_routes": (nas, "ip route; ip -d link show flannel.1"), + "coord_fw": (coordinator, "iptables -S nixos-fw; iptables -t mangle -S ax-fleet-guard"), + "nas_nft_input": (nas, "nft list chain inet nixos-fw input-allow"), + })) + nas.succeed(f"k3s kubectl exec probe-coord -- curl -sf --max-time 10 http://{nas_pod_ip}:8000/ | grep -x pod-ok") + + +with step("cluster plumbing: DNS through the NAS stand-in"): + # A records only: the stand-in has no upstream, so AAAA answers REFUSED. + status, _ = nas.execute("k3s kubectl exec probe-coord -- nslookup -type=a only-nas.test 2>&1 | grep -q 10.42.0.77") + if status != 0: + record("diag_dns", diag({ + "coord_nslookup": (nas, "k3s kubectl exec probe-coord -- nslookup only-nas.test"), + "coord_nslookup_svc": (nas, "k3s kubectl exec probe-coord -- nslookup kubernetes.default.svc.cluster.local"), + "nas_pod_direct": (nas, "k3s kubectl exec probe-nas -- nslookup only-nas.test 10.42.0.1"), + "nas_pod_coredns": (nas, "k3s kubectl exec probe-nas -- nslookup only-nas.test"), + "coredns_logs": (nas, "k3s kubectl -n kube-system logs deploy/coredns --tail=50"), + "coredns_pod_resolv": (nas, "k3s kubectl -n kube-system exec deploy/coredns -- cat /etc/resolv.conf"), + "dnsmasq": (nas, "journalctl -u dnsmasq --no-pager | tail -30; ss -lunp | grep :53"), + "nas_nft": (nas, "nft list table inet nixos-fw"), + })) + kubectl("exec probe-coord -- nslookup -type=a only-nas.test | grep -q 10.42.0.77") + kubectl("exec probe-nas -- nslookup -type=a only-nas.test | grep -q 10.42.0.77") + + +# The coordinator link flap runs at the END of 30-ax (fix round 1), together +# with the Substrate/ax flap: a flap long enough to take the node NotReady +# can leave connections to the NAS stale (INFERRED), and the Task phases before it +# must run on a cluster that has not seen an outage. probe-coord stays up +# until then; that subtest asserts its uid survives. diff --git a/tests/ax-fleet/phases/20-substrate.py b/tests/ax-fleet/phases/20-substrate.py new file mode 100644 index 000000000..a173dc933 --- /dev/null +++ b/tests/ax-fleet/phases/20-substrate.py @@ -0,0 +1,160 @@ +# Phase 20: Substrate (track substrate; DESIGN.md sections 12.1 phases 2 and 3, +# and section 14 S4). Concatenated after 10-cluster.py, so both switches have +# happened: the NAS runs k3s, the registry seed and the bootstrap, and the +# coordinator has joined as the tainted harness node. +# +# Uses only the machine objects `nas` and `coordinator`. Every kubectl runs on +# the NAS as root against /etc/rancher/k3s/k3s.yaml. +import json + +SUB_NS = "ate-system" +SUB_LOCAL_PATH = "/mnt/nas/services/ax-fleet/local-path/" +SUB_CONTROL_DEPLOYMENTS = [ + "ate-api-server", + "ate-controller", + "atenet-router", +] +# ate-setup installs the pod-certificate controller in its own namespace +# (manifests/ate-install/pod-certificate-controller.yaml), not in ate-system. +SUB_PODCERT_NS = "podcertificate-controller-system" + + +def sub_k(args, timeout=120): + return nas.succeed(f"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl {args}", timeout=timeout) + + +def sub_json(args): + return json.loads(sub_k(f"{args} -o json")) + + +with step("substrate: bootstrap steps 20-50 ran"): + nas.wait_for_unit("ax-fleet-bootstrap.service", timeout=3600) + boot_log = nas.succeed("journalctl -b -u ax-fleet-bootstrap.service --no-pager") + # Not `step`: that name is the prelude's receipt context manager, which + # 90-rollback still needs. + for step_name in ["20-registry-svc", "30-substrate", "40-gvisor-asset", "50-workerpool"]: + assert f"step {step_name}" in boot_log, f"bootstrap step {step_name} did not run" + assert "gvisor asset verified" in boot_log, "40-gvisor-asset did not verify the tarball" + stamp = sub_k( + "-n kube-system get configmap ax-fleet-substrate -o jsonpath='{.data.version}'" + ).strip() + assert stamp == "d277088b", f"install stamp version {stamp!r}" + +with step("substrate: every control workload Available, on the NAS"): + nas.wait_until_succeeds( + f"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n {SUB_NS} wait " + "--for=condition=Available deploy --all --timeout=10s", + timeout=900, + ) + names = {d["metadata"]["name"] for d in sub_json(f"-n {SUB_NS} get deploy")["items"]} + for want in SUB_CONTROL_DEPLOYMENTS: + assert want in names, f"deployment {want} missing: {sorted(names)}" + # atenet-egress is the actor egress gateway; its kind differs across + # dataplanes, so it is found by name among Deployments and StatefulSets. + sts = {s["metadata"]["name"] for s in sub_json(f"-n {SUB_NS} get statefulset")["items"]} + assert any(n.startswith("atenet-egress") for n in names | sts), "no atenet-egress" + assert "postgres" in sts, f"postgres StatefulSet missing: {sorted(sts)}" + sub_k(f"-n {SUB_NS} rollout status statefulset/postgres --timeout=600s", timeout=660) + sub_k( + f"-n {SUB_PODCERT_NS} wait --for=condition=Available deploy/podcertificate-controller --timeout=600s", + timeout=660, + ) + for pod in sub_json(f"-n {SUB_PODCERT_NS} get pods")["items"]: + assert pod["spec"].get("nodeName") == "nas", f"{pod['metadata']['name']} on {pod['spec'].get('nodeName')!r}" + for pod in sub_json(f"-n {SUB_NS} get pods")["items"]: + owner = (pod["metadata"].get("ownerReferences") or [{}])[0].get("kind", "") + name = pod["metadata"]["name"] + node = pod["spec"].get("nodeName", "") + if name.startswith("atelet") or pod["metadata"].get("labels", {}).get( + "ax.mecattaf.dev/pool" + ): + continue + if owner == "Job" and pod["status"].get("phase") == "Succeeded": + continue + assert node == "nas", f"control pod {name} landed on {node!r}" + +with step("substrate: nas keeps substrate-version=none, coordinator carries the version"): + nodes = {n["metadata"]["name"]: n for n in sub_json("get nodes")["items"]} + assert nodes["nas"]["metadata"]["labels"].get("ate.dev/substrate-version") == "none" + assert ( + nodes["coordinator"]["metadata"]["labels"].get("ate.dev/substrate-version") + == "d277088b" + ) + +with step("substrate: atelet runs on the coordinator only"): + # The atelet DaemonSet is version-keyed by ate-setup; find it by label. + ds = sub_k(f"-n {SUB_NS} get ds -l app=atelet -o jsonpath='{{.items[0].metadata.name}}'").strip() + sub_k(f"-n {SUB_NS} rollout status ds/{ds} --timeout=600s", timeout=660) + atelet = [ + p + for p in sub_json(f"-n {SUB_NS} get pods")["items"] + if p["metadata"]["name"].startswith("atelet") + ] + assert atelet, "no atelet pod" + for p in atelet: + assert p["spec"]["nodeName"] == "coordinator", ( + f"atelet on {p['spec']['nodeName']}" + ) + +with step("substrate: ClusterTrustBundles and pod certificates served"): + res = sub_k("get --raw /apis/certificates.k8s.io/v1beta1") + assert "clustertrustbundles" in res and "podcertificaterequests" in res, res + assert sub_json("get clustertrustbundles")["items"], "no ClusterTrustBundle objects" + +with step("substrate: WorkerPool ateom-gvisor Ready 2 on the coordinator"): + nas.wait_until_succeeds( + "test \"$(KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n ate-system get " + "workerpool ateom-gvisor -o jsonpath='{.status.readyReplicas}')\" = 2", + timeout=900, + ) + wp = sub_json(f"-n {SUB_NS} get workerpool ateom-gvisor") + assert "@sha256:" in wp["spec"]["workerImage"], wp["spec"]["workerImage"] + workers = [ + p + for p in sub_json(f"-n {SUB_NS} get pods -l ax.mecattaf.dev/pool=ateom-gvisor")["items"] + ] + assert len(workers) == 2, f"{len(workers)} worker pods" + for p in workers: + assert p["spec"]["nodeName"] == "coordinator", p["spec"]["nodeName"] + lims = [c.get("resources", {}).get("limits", {}).get("memory") for c in p["spec"]["containers"]] + assert any(lims), f"worker pod has no memory limit: {lims}" + +with step("substrate: gVisor fetched through the RustFS fallback"): + # No internet in the VM: atelet's anonymous GCS open of gs://gvisor/... + # fails, then its S3 client reads the same bucket and key from the + # in-cluster RustFS (40-gvisor-asset). The prewarmer logs "Sandbox assets + # prewarmed" for gvisor-default only once the pause image and the gVisor + # tarball (sha256-checked by atelet) are both on the node + # (cmd/atelet/sandbox_prewarm.go). + nas.wait_until_succeeds( + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n ate-system logs -l app=atelet " + "--all-containers --tail=-1 | grep 'Sandbox assets prewarmed' | grep -q gvisor-default", + timeout=900, + ) + logs = sub_k(f"-n {SUB_NS} logs -l app=atelet --all-containers --tail=-1", timeout=300) + print("\n".join(l for l in logs.splitlines() if "gvisor" in l.lower())[-4000:]) + assert "gVisor release download complete" in logs or "gvisor" in logs.lower() + +with step("substrate: every PersistentVolume on the data pool"): + pvs = sub_json("get pv")["items"] + assert pvs, "no PersistentVolumes" + for pv in pvs: + spec = pv["spec"] + path = (spec.get("hostPath") or spec.get("local") or {}).get("path", "") + assert path.startswith(SUB_LOCAL_PATH), (pv["metadata"]["name"], path) + node_terms = ( + spec.get("nodeAffinity", {}).get("required", {}).get("nodeSelectorTerms", []) + ) + values = [ + v + for t in node_terms + for e in t.get("matchExpressions", []) + for v in e.get("values", []) + ] + assert "coordinator" not in values, f"PV {pv['metadata']['name']} on the coordinator" + +with step("substrate: every image came from the NAS registry by digest"): + pods = sub_json(f"-n {SUB_NS} get pods")["items"] + for p in pods: + for c in p["spec"].get("containers", []) + p["spec"].get("initContainers", []): + assert "@sha256:" in c["image"], f"{p['metadata']['name']}: {c['image']}" diff --git a/tests/ax-fleet/phases/25-harness-probe.py b/tests/ax-fleet/phases/25-harness-probe.py new file mode 100644 index 000000000..d94cbd47b --- /dev/null +++ b/tests/ax-fleet/phases/25-harness-probe.py @@ -0,0 +1,128 @@ +# Phase 25: the integration's harness probe (ax-fleet INTEGRATE.md). It runs +# after 10-cluster (both switches) and 20-substrate, before the T-cases of +# 30-ax, so its result is in the log even if a later case fails. +# +# One ax Task with spec.sandboxClass gvisor, run by `ax-fleet-smoke probe` +# (the script Tom runs on the real coordinator) in a test-only variant of the +# fleet task image that also carries claude-code. No credential exists in the +# image, the Task or the cluster. The Task runs `claude --version` and one GET +# of the worker's Halogen stand-in, must reach Completed with exit 0, and must +# stay Completed for PROBE_HOLD seconds, sampled every PROBE_EVERY seconds +# (four and more of P1's 15 s resync periods). +# +# There is no containerd RuntimeClass on this path: Substrate's ateom-gvisor +# worker pods run runsc themselves (DESIGN.md D2, 6.2). "gVisor" is proven from +# inside the Task: its /proc/version is not the coordinator VM's kernel. +import json +import re + +PROBE_HOLD = 60 +PROBE_EVERY = 5 +PROBE_AX = "AX_SERVER=http://127.0.0.1:8099 ax -a fleet" +PROBE_STUB_LOG = "/var/lib/halogen-stub/requests.jsonl" + + +def probe_task_phase(name): + # `ax get task NAME` prints YAML; status.phase is its only `phase:` key. + out = coordinator.succeed(f"{PROBE_AX} get task {name}") + m = re.search(r"^\s+phase:\s*(\S+)", out, re.M) + assert m, f"no phase for {name}: {out!r}" + return m.group(1).strip("\"'") + + +def probe_stub_lines(): + return worker.succeed(f"cat {PROBE_STUB_LOG} 2>/dev/null || true").splitlines() + + +with step("probe: k3s nodes Ready, Substrate healthy, ax-server and ax-controller up"): + node_ready("nas") + node_ready("coordinator") + kubectl("-n ate-system wait --for=condition=Available deploy --all --timeout=600s") + kubectl("-n ate-system rollout status statefulset/postgres --timeout=600s") + nas.wait_until_succeeds( + "test \"$(k3s kubectl -n ate-system get workerpool ateom-gvisor -o jsonpath='{.status.readyReplicas}')\" = 2", + timeout=900, + ) + for d in ("ax-redis", "ax-server", "ax-controller"): + kubectl(f"-n ax-system rollout status deploy/{d} --timeout=600s") + coordinator.wait_until_succeeds("curl -sf http://127.0.0.1:8099/healthz", timeout=300) + record("probe_nodes", kubectl("get nodes -o wide").strip().splitlines()) + record("probe_ax_pods", kubectl("-n ax-system get pods -o wide").strip().splitlines()) + workers_wide = kubectl("-n ate-system get pods -l ax.mecattaf.dev/pool=ateom-gvisor -o wide") + record("probe_worker_pods", workers_wide.strip().splitlines()) + worker_nodes = kubectl( + "-n ate-system get pods -l ax.mecattaf.dev/pool=ateom-gvisor " + "-o jsonpath='{range .items[*]}{.spec.nodeName}{\"\\n\"}{end}'" + ).split() + assert worker_nodes and all(n == "coordinator" for n in worker_nodes), worker_nodes + + +with step("probe: a gVisor ax Task on the coordinator runs claude --version and reaches Halogen, Completed and still Completed 60 s later"): + digest = coordinator.succeed(f"cat {CLAUDE_PROBE_OCI}/digest").strip() + assert digest.startswith("sha256:"), digest + image = f"localhost:5000/ax/ax-agent-claude-probe@{digest}" + stub_before = len(probe_stub_lines()) + host_kernel = coordinator.succeed("cat /proc/version").strip() + + rc, out = coordinator.execute( + f"ax-fleet-smoke probe --image {image} --keep --timeout 1500 2>/tmp/probe-smoke.stderr", + timeout=1800, + ) + lines = [l for l in out.strip().splitlines() if l.startswith("{")] + if not lines or rc != 0: + _, err = coordinator.execute("tail -c 4000 /tmp/probe-smoke.stderr") + _, tasks = coordinator.execute(f"{PROBE_AX} get tasks 2>&1 | tail -20") + _, atelet = nas.execute("k3s kubectl -n ate-system logs -l app=atelet --all-containers --tail=80 2>&1") + _, wp = nas.execute("k3s kubectl -n ate-system logs -l ax.mecattaf.dev/pool=ateom-gvisor --all-containers --tail=60 2>&1") + _, ctl = nas.execute("k3s kubectl -n ax-system logs deploy/ax-controller --tail=80 2>&1") + record("probe_diag", {"stderr": err[-4000:], "tasks": tasks[-3000:], "atelet": atelet[-6000:], + "workers": wp[-6000:], "controller": ctl[-6000:]}) + assert lines, f"ax-fleet-smoke probe printed no receipt (rc={rc}): {out!r}" + r = json.loads(lines[-1]) + record("probe_receipt", r) + assert rc == 0 and r.get("pass") is True, r + name = r["task"] + + # Completed, and STAYS Completed: sample for PROBE_HOLD seconds. + samples = [] + t_end = time.monotonic() + PROBE_HOLD + while True: + samples.append(probe_task_phase(name)) + if time.monotonic() >= t_end: + break + time.sleep(PROBE_EVERY) + record("probe_phase_samples", {"hold_seconds": PROBE_HOLD, "every_seconds": PROBE_EVERY, "phases": samples}) + assert len(samples) >= PROBE_HOLD // PROBE_EVERY and all(p == "Completed" for p in samples), samples + record("probe_task_yaml", coordinator.succeed(f"{PROBE_AX} get task {name}").strip().splitlines()) + + res = r["result"] + # claude-code answered inside the sandbox, with no credential anywhere. + assert res["claude_rc"] == 0 and re.search(r"\d+\.\d+\.\d+", res["claude_version"]), res + # The Halogen stand-in on the worker VM answered through the Gateway. + assert res["curl_rc"] == 0 and res["http_code"] == "200", res + assert res["model"] == "halogen-qwen3.8-flash-next", res + # gVisor, not runc: a runc container would report the VM's own kernel. + record("probe_kernels", {"sandbox": res["proc_version"], "coordinator_vm": host_kernel}) + assert res["proc_version"] and res["proc_version"] != host_kernel, (res["proc_version"], host_kernel) + + new = [json.loads(l) for l in probe_stub_lines()[stub_before:] if l.strip()] + models = [q for q in new if q.get("path", "").rstrip("/") == "/v1/models"] + record("probe_stub_requests", models) + assert models, f"the Halogen stand-in logged no /v1/models request: {new!r}" + + # The image was pulled by atelet, which runs on the coordinator only. + atelet_logs = kubectl("-n ate-system logs -l app=atelet --all-containers --tail=-1") + hits = [ + l for l in atelet_logs.splitlines() if "ax-agent-claude-probe" in l or digest.split(":", 1)[1][:16] in l + ] + # Recorded, not asserted: the placement proof is the pool (every worker pod + # on the coordinator, asserted in the first step) and the sandbox kernel above. + record("probe_atelet_image_lines", hits[-5:]) + # '[r]unsc' so the pattern does not match the shell that runs pgrep. + record("probe_runsc_processes", { + "coordinator": coordinator.succeed("pgrep -c -f '[r]unsc' || true").strip(), + "nas": nas.succeed("pgrep -c -f '[r]unsc' || true").strip(), + }) + nas.fail("pgrep -f '[r]unsc'") + + coordinator.succeed(f"{PROBE_AX} delete task {name}") diff --git a/tests/ax-fleet/phases/30-ax.py b/tests/ax-fleet/phases/30-ax.py new file mode 100644 index 000000000..8a5d73a1e --- /dev/null +++ b/tests/ax-fleet/phases/30-ax.py @@ -0,0 +1,174 @@ +# Phase 4 (Tasks) and phase 5 (resilience) of checks.x86_64-linux.ax-fleet. +# Track ax; DESIGN.md section 12.1. Concatenated after 10-cluster and +# 20-substrate by tests/ax-fleet/default.nix, so `nas`, `coordinator` and +# `worker` are the test nodes and the cluster, Substrate and the WorkerPool are +# already up. Every Task runs through ax-fleet-smoke, the script Tom runs on the +# real coordinator after the switch. +import json +import re + + +smoke_receipts = [] + + +def ax_smoke(case, *args, expect_pass=True): + cmd = "ax-fleet-smoke " + " ".join([case, *map(str, args)]) + rc, out = coordinator.execute(cmd + " 2>/dev/null") + lines = [l for l in out.strip().splitlines() if l.startswith("{")] + assert lines, f"{cmd}: no receipt (rc={rc}): {out!r}" + receipt = json.loads(lines[-1]) + print(f"RECEIPT {cmd}: {json.dumps(receipt)}") + # Into receipt.json too (fix round 2): every Task outcome, in order. + smoke_receipts.append({"cmd": cmd, "rc": rc, "receipt": receipt}) + record("ax_smoke", smoke_receipts) + if expect_pass: + assert rc == 0 and receipt.get("pass") is True, f"{cmd} failed: {receipt}" + return receipt + + +def kubectl(args): + return nas.succeed(f"k3s kubectl {args}") + + +AX = "AX_SERVER=http://127.0.0.1:8099 ax -a fleet" + + +def task_phase(name): + # `ax get task NAME` prints the Task as YAML; status.phase is its only + # `phase:` key. + out = coordinator.succeed(f"{AX} get task {name}") + m = re.search(r"^\s+phase:\s*(\S+)", out, re.M) + assert m, f"no phase for {name}: {out!r}" + return m.group(1).strip("\"'") + + +with step("ax control plane is Available on the NAS, nowhere else"): + for d in ["ax-redis", "ax-server", "ax-controller"]: + kubectl(f"-n ax-system rollout status deploy/{d} --timeout=600s") + node = kubectl( + f"-n ax-system get pods -l app.kubernetes.io/name={d} -o jsonpath='{{.items[0].spec.nodeName}}'" + ).strip() + assert node == "nas", f"{d} runs on {node}" + pv = kubectl( + "get pv -o jsonpath='{range .items[?(@.spec.claimRef.name==\"ax-redis-data\")]}{.spec.local.path}{.spec.hostPath.path}{end}'" + ) + assert pv.startswith("/mnt/nas/services/ax-fleet/local-path"), f"ax-redis volume at {pv!r}" + coordinator.wait_until_succeeds("curl -sf http://127.0.0.1:8099/healthz", timeout=300) + # No Claude credential, and no secret, in any ax object. + kubectl("-n ax-system get secrets -o name | (! grep -q .)") + +with step("T1 halogen: Completed, and still Completed after the hold"): + t1 = ax_smoke("halogen", "--hold", 90, "--keep") + assert t1["exit_code"] == 0 and t1["phase_after_hold"] == "Completed", t1 + # The golden-snapshot double execution judge 2 inferred: record, do not fail. + rc, count = worker.execute("curl -sf http://127.0.0.1:8731/stub/requests") + print(f"RECEIPT halogen stub requests after T1: rc={rc} {count.strip()!r}") + +with step("T2 pi: Completed with a schema-valid result read back through P1"): + t2 = ax_smoke("pi", "--hold", 30) + assert t2["result"]["valid"] is True, t2 + assert t2["result_bytes"] and t2["result_sha256"], t2 + +with step("T3 exit 3: Failed with ExitCode=3"): + t3 = ax_smoke("exit", 3, "--hold", 30) + assert t3["phase"] == "Failed" and t3["ready"]["message"].startswith("ExitCode=3"), t3 + +with step("T4 egress-deny: a non-allowlisted target is refused by the Gateway"): + # The coordinator's LAN address answers the worker in this run (checked + # first), so a failure inside the sandbox is the Gateway's refusal. + worker.succeed("curl -s -o /dev/null --max-time 10 http://10.42.0.2/") + t4 = ax_smoke("egress-deny", "http://10.42.0.2/") + assert t4["phase"] == "Failed", t4 + + +with step("T4b egress-deny: the Gateway's host on another port is refused (the NAS enforces the port)"): + # Fix round 3. The Gateway names 10.42.0.5/32 port 8731, but ax drops the + # port and Substrate never compares one: the round-3 review MEASURED + # curl_rc 0 from worker:2222 through the egress gateway. The NAS's pod + # egress chain (control.nix) is what refuses it now. Discriminating: the + # NAS host reaches that port. + nas.succeed("curl -sf --max-time 10 http://10.42.0.5:2222/ | grep -q worker-port-2222-reached") + t4b = ax_smoke("egress-deny", "http://10.42.0.5:2222/") + # The egress gateway answers 503 itself (its upstream connect is dropped + # on the NAS); before the fix the target answered 200 (MEASURED by the + # round-3 review). + assert t4b["refused_by"] in ("gateway-connection", "egress-upstream"), t4b + assert t4b["result"]["http_code"] != "200", t4b + +with step("security: only root and apiUsers reach the ax API and the cluster ranges from the desk"): + # Fix round 3. The round-3 review MEASURED alice applying a Gateway with + # host 0.0.0.0/0 through the loopback proxy. The owner match also covers + # the ClusterIP and the pod, which kube-proxy's OUTPUT DNAT would carry. + ax_pod = kubectl( + "-n ax-system get pods -l app.kubernetes.io/name=ax-server -o jsonpath='{.items[0].status.podIP}'" + ).strip() + coordinator.succeed("runuser -u tom -- curl -sf --max-time 10 http://127.0.0.1:8099/healthz") + coordinator.succeed("curl -sf --max-time 10 http://10.201.0.80:8080/healthz") + for url in ("http://127.0.0.1:8099/healthz", "http://10.201.0.80:8080/healthz", f"http://{ax_pod}:8080/healthz"): + coordinator.fail(f"runuser -u alice -- curl -s --max-time 5 -o /dev/null {url}") + # 127.0.0.1:8080 is left to ax-conwip's default and the mock stack. + coordinator.fail("ss -ltn | grep -q '127.0.0.1:8080 '") + record("api_owner_chain", coordinator.succeed("iptables -S ax-fleet-api").strip().splitlines()) + + +with step("T5 floor 4: four Tasks in a row on the 2-worker pool, none ResourceExhausted"): + t5 = ax_smoke("floor", 4) + for t in t5["tasks"]: + assert t["phase"] == "Completed", t + occupancy = kubectl( + "-n ate-system get pods --field-selector spec.nodeName=coordinator -o name | grep -c ateom || true" + ).strip() + print(f"RECEIPT worker pods on coordinator after {t['task']}: {occupancy}") + +t1_name = t1["task"] + +with step("resilience: a restarted ax-controller leaves a finished Task Completed"): + kubectl("-n ax-system delete pod -l app.kubernetes.io/name=ax-controller --wait=true") + kubectl("-n ax-system rollout status deploy/ax-controller --timeout=300s") + coordinator.sleep(45) # three resync periods + assert task_phase(t1_name) == "Completed" + +with step("resilience: a restarted ax-redis keeps every Task (AOF on the volume)"): + before = coordinator.succeed(f"{AX} get tasks | tail -n +2 | wc -l").strip() + kubectl("-n ax-system delete pod -l app.kubernetes.io/name=ax-redis --wait=true") + kubectl("-n ax-system rollout status deploy/ax-redis --timeout=300s") + coordinator.wait_until_succeeds(f"{AX} get tasks >/dev/null", timeout=120) + after = coordinator.succeed(f"{AX} get tasks | tail -n +2 | wc -l").strip() + assert before == after and int(after) >= 1, f"tasks before={before} after={after}" + assert task_phase(t1_name) == "Completed" + +with step("resilience: the LAN leg is down until the coordinator is NotReady; pods keep their names and uid, a new T1 completes"): + pods = lambda: kubectl( + "-n ate-system get pods --field-selector spec.nodeName=coordinator -o name | grep ateom | sort" + ) + before = pods() + probe_uid = kubectl("get pod probe-coord -o jsonpath='{.metadata.uid}'").strip() + flap_until_unreachable("ax") + nas.wait_until_succeeds( + "k3s kubectl get node coordinator -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -qx True", + timeout=300, + ) + nas.wait_until_succeeds( + "! k3s kubectl get node coordinator -o jsonpath='{.spec.taints[*].key}' | grep -qw node.kubernetes.io/unreachable", + timeout=300, + ) + assert pods() == before, f"worker pods changed: {before!r}" + kubectl("wait --for=condition=Ready pod/probe-coord --timeout=300s") + assert kubectl("get pod probe-coord -o jsonpath='{.metadata.uid}'").strip() == probe_uid, "the probe pod was replaced by the flap" + nas.wait_until_succeeds("curl -sf --max-time 5 http://10.42.0.2:18085/ | grep -x pod-ok", timeout=120) + # Tasks after a real outage are recorded as they are. Fix round 1 + # MEASURED that each stale gRPC connection left by the outage costs one + # Task: ActorResumeFailed, Unavailable, "connection reset by peer", once + # NAS -> atelet :8085 and once atelet -> NAS :443 ("mint actor + # certificate"). ax does not retry Unavailable. The bound is a test + # parameter: the fleet must recover within it without a restart. + attempts = [] + for _ in range(6): + r = ax_smoke("halogen", expect_pass=False) + attempts.append({k: r.get(k) for k in ("pass", "phase", "ready")}) + if r.get("pass") is True: + break + record("post_outage_tasks", attempts) + assert attempts[-1]["pass"] is True, f"no Task completed after the outage: {attempts}" + +coordinator.succeed(f"{AX} delete task {t1_name}") diff --git a/tests/ax-fleet/phases/35-lan-guard.py b/tests/ax-fleet/phases/35-lan-guard.py new file mode 100644 index 000000000..927bf7b36 --- /dev/null +++ b/tests/ax-fleet/phases/35-lan-guard.py @@ -0,0 +1,112 @@ +# Phase: the house LAN cannot reach the cluster ranges through the NAS, and +# the credential and RBAC surface is what fix round 1 left (2026-09-23). Runs +# after 30-ax, so ax-server, ax-redis and RustFS all exist. The worker is a +# plain LAN host; the NAS is its default gateway on the real LAN. + + +def tcp_open(ip, port): + return f"timeout 5 bash -c 'exec 3<>/dev/tcp/{ip}/{port}'" + + +with step("lan: routed LAN traffic never reaches a ClusterIP or pod IP"): + def svc_ip(ns, name): + return jsonpath(f"-n {ns} svc {name}", "{.spec.clusterIP}") + + ax_ip = svc_ip("ax-system", "ax-server") + redis_ip = svc_ip("ax-system", "ax-redis") + rustfs_ip = svc_ip("ate-system", "rustfs") + ax_pod = kubectl( + "-n ax-system get pods -l app.kubernetes.io/name=ax-server -o jsonpath='{.items[0].status.podIP}'" + ).strip() + record("lan_probe_targets", {"ax": ax_ip, "redis": redis_ip, "rustfs": rustfs_ip, "ax_pod": ax_pod}) + + # Positive controls from the NAS host (OUTPUT path, not prerouting): the + # endpoints are up, so a failure from the worker is the guard. + nas.succeed(f"curl -sf --max-time 10 http://{ax_ip}:8080/healthz") + nas.succeed(tcp_open(redis_ip, 6379)) + nas.succeed(f"curl -s --max-time 10 -o /dev/null http://{rustfs_ip}:9000/") + + worker.succeed("ip route replace 10.200.0.0/16 via 10.42.0.1") + worker.succeed("ip route replace 10.201.0.0/16 via 10.42.0.1") + try: + # curl without -f: rc 0 on ANY HTTP answer, so fail() means no answer. + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{ax_ip}:8080/healthz") + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{ax_pod}:8080/healthz") + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{rustfs_ip}:9000/") + worker.fail(tcp_open(redis_ip, 6379)) + worker.fail("dig +time=2 +tries=1 @10.201.0.10 ax-server.ax-system.svc.cluster.local") + finally: + worker.succeed("ip route del 10.200.0.0/16 via 10.42.0.1") + worker.succeed("ip route del 10.201.0.0/16 via 10.42.0.1") + record("nas_range_guard", nas.succeed("nft list chain inet ax-fleet-guard prerouting").strip().splitlines()) + + +with step("nas: pods reach Halogen on its port and no other private address"): + # Fix round 3. The round-3 review MEASURED postgres-0 reaching worker:2222; + # the NAS had no pod egress guard. Discriminating: the NAS host reaches + # both targets. + nas.succeed("curl -sf --max-time 10 http://10.42.0.5:2222/ | grep -q worker-port-2222-reached") + nas.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + kubectl("exec probe-nas -- curl -sf --max-time 10 http://10.42.0.5:8731/health") + kubectl("exec probe-nas -- sh -c '! curl -s --max-time 5 -o /dev/null http://10.42.0.5:2222/'") + kubectl("exec probe-nas -- sh -c '! curl -s --max-time 5 -o /dev/null http://10.42.0.2/'") + record("nas_pod_egress", nas.succeed("nft list chain inet ax-fleet-guard forward").strip().splitlines()) + + +with step("security: the registry is read-only to the coordinator; the seed wrote everything"): + # Fix round 2. The round-2 review MEASURED 202 for an upload and 201 for a + # tag overwrite from an unprivileged coordinator user. + reg = "http://10.42.0.1:5000" + alice = "runuser -u alice -- curl -s -o /dev/null -w '%{http_code}' --max-time 10" + assert coordinator.succeed(f"{alice} {reg}/v2/_catalog").strip() == "200" + codes = { + "upload": coordinator.succeed(f"{alice} -X POST {reg}/v2/secprobe/blobs/uploads/").strip(), + "mount": coordinator.succeed(f"{alice} -X POST '{reg}/v2/secprobe/blobs/uploads/?mount=sha256:0000000000000000000000000000000000000000000000000000000000000000&from=ax/ax-redis'").strip(), + "delete": coordinator.succeed(f"{alice} -X DELETE {reg}/v2/ax/ax-redis/manifests/sha256:0000000000000000000000000000000000000000000000000000000000000000").strip(), + } + record("registry_write_codes", codes) + assert all(c not in ("201", "202") for c in codes.values()), codes + nas.fail("ss -ltn | grep -q '127.0.0.1:5001'") # the seed's writer is gone + nas.succeed("ss -ltn | grep -q '10.42.0.1:5000'") + + +with step("security: ax-controller holds no Secret grant and no API token"): + nas.fail("k3s kubectl get clusterrole ax-controller") + nas.fail("k3s kubectl get clusterrolebinding ax-controller") + auto = jsonpath("-n ax-system deploy ax-controller", "{.spec.template.spec.automountServiceAccountToken}") + assert auto == "false", auto + vols = kubectl( + "-n ax-system get pods -l app.kubernetes.io/name=ax-controller -o jsonpath='{.items[0].spec.volumes[*].name}'" + ).split() + record("ax_controller_volumes", vols) + assert not any(v.startswith("kube-api-access") for v in vols), vols + can = nas.succeed( + "k3s kubectl auth can-i list secrets --all-namespaces --as=system:serviceaccount:ax-system:ax-controller || true" + ).strip() + assert can == "no", can + + +with step("security: RustFS and its clients read a generated credential, not the kind default"): + nas.succeed("stat -c '%a %U' /var/lib/ax-fleet/rustfs.env | grep -x '600 root'") + nas.succeed("k3s kubectl -n ate-system get secret ax-fleet-rustfs -o name") + wanted = {"RUSTFS_ACCESS_KEY", "RUSTFS_SECRET_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} + seen = [] + # The bucket-init Job uses the credential too; fix round 1 first missed it + # and it crash-looped (MEASURED), so it must complete here. + kubectl("-n ate-system wait --for=condition=complete job/rustfs-bucket-init --timeout=300s") + for obj in ("deploy rustfs", "deploy ate-api-server", "ds atelet", "job rustfs-bucket-init"): + kind, name = obj.split() + names = kubectl(f"-n ate-system get {kind} -o name").split() + target = [n for n in names if n.split("/")[-1].startswith(name)] + for t in target: + doc = json.loads(kubectl(f"-n ate-system get {t} -o json")) + for c in doc["spec"]["template"]["spec"]["containers"]: + for e in c.get("env", []): + if e["name"] in wanted: + assert "value" not in e, f"{t} {e['name']} carries a literal value" + assert e["valueFrom"]["secretKeyRef"]["name"] == "ax-fleet-rustfs", (t, e["name"]) + seen.append(f"{t}:{e['name']}") + record("rustfs_credential_refs", sorted(seen)) + assert any(s.endswith("RUSTFS_SECRET_KEY") for s in seen), seen + assert any(s.endswith("AWS_SECRET_ACCESS_KEY") for s in seen), seen + assert any(s.startswith("job") and s.endswith("AWS_SECRET_ACCESS_KEY") for s in seen), seen diff --git a/tests/ax-fleet/phases/90-rollback.py b/tests/ax-fleet/phases/90-rollback.py new file mode 100644 index 000000000..a11575735 --- /dev/null +++ b/tests/ax-fleet/phases/90-rollback.py @@ -0,0 +1,85 @@ +# Phase 6: rollback, the kill switch proven (DESIGN.md 12.1, 13). Track +# cluster. Reverse order: coordinator, then nas. Each host goes back to its +# base toplevel (ax off) and then ax-fleet-teardown runs, as Tom would. + +BASE = "/run/booted-system/bin/switch-to-configuration test" +LEFTOVER_RULES = "iptables-save 2>/dev/null | grep -E 'KUBE-|FLANNEL|CNI-'" + + +def pod_netns_pid(machine): + # A pod sandbox's pause process in a network namespace other than the host's. + return machine.succeed( + "host=$(readlink /proc/1/ns/net); for p in $(pgrep -x pause); do " + "[ \"$(readlink /proc/$p/ns/net)\" != \"$host\" ] && { echo $p; break; }; done" + ).strip() + + +with step("rollback coordinator"): + coordinator.succeed(f"{BASE} >&2") + coordinator.fail("systemctl is-active k3s.service") + # Fix round 3: between the kill switch and the teardown the pods still + # run (KillMode=process). The role-scoped guards keep them off the host. + pid = pod_netns_pid(coordinator) + assert pid, "no pod network namespace survived the kill switch" + coordinator.succeed("timeout 10 bash -c 'exec 3<>/dev/tcp/10.42.0.2/22'") # sshd is up + coordinator.fail(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/10.42.0.2/22'") + coordinator.fail(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/100.105.121.73/22'") + coordinator.fail(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/10.42.0.5/8731'") + assert coordinator.succeed("iptables -S nixos-fw | grep -c ax-fleet-pod-input").strip() == "2" + coordinator.succeed("iptables -t mangle -S FORWARD 1 | grep -q ax-fleet-guard") + # As documented (fix round 2): the teardown from the rolled-back host's + # own PATH, no checkout, no injected store path. + coordinator.succeed("test -x /run/current-system/sw/bin/ax-fleet-teardown") + coordinator.succeed("ax-fleet-teardown >&2") + coordinator.fail("ip link show cni0") + coordinator.fail("ip link show flannel.1") + coordinator.fail(LEFTOVER_RULES) + coordinator.fail("pgrep -f containerd-shim") + # The guards belong to the harness role's every generation (fix round 3); + # the teardown leaves them, inert without cni0 and flannel.1. + coordinator.succeed("iptables -t mangle -S ax-fleet-guard | grep -q DROP") + coordinator.succeed("iptables -S OUTPUT 1 | grep -q ax-fleet-api") + after = sysctls(coordinator) + record("sysctl_coordinator_after_rollback", after) + assert after == base["sysctl_coordinator"], (after, base["sysctl_coordinator"]) + assert user_unit_pid("herdr-standin") == base["herdr_pid"], "herdr stand-in restarted" + assert nm_invocation() == base["nm_invocation"], "NetworkManager restarted" + coordinator.fail("test -e /etc/NetworkManager/conf.d/90-ax-fleet.conf") + worker.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + peer.succeed("curl -sf --max-time 10 http://100.105.121.73/ | grep -x caddy-ok") + + +with step("rollback nas"): + # Diagnostic only: which processes hold the kubelet bind before the switch. + _, holders = nas.execute("ls -l /proc/[0-9]*/cwd /proc/[0-9]*/root 2>/dev/null | grep -c /var/lib/kubelet") + record("nas_kubelet_holders_before_rollback", holders.strip()) + nas.succeed(f"{BASE} >&2") + nas.fail("systemctl is-active k3s.service") + nas.fail("systemctl is-active docker-registry.service") + nas.succeed("command -v tailscale") # the stub is reachable from a root shell + nas.succeed("test -x /run/current-system/sw/bin/ax-fleet-teardown") + nas.succeed("ax-fleet-teardown >&2") + # The teardown's pinned PATH keeps k3s-killall.sh away from tailscale. + nas.fail("test -e /var/log/tailscale-stub.log") + nas.fail("ip link show cni0") + nas.fail("ip link show flannel.1") + nas.fail("pgrep -f containerd-shim") + ruleset = nas.succeed("nft -s list ruleset") + for marker in ("KUBE-", "FLANNEL", "CNI-"): + assert marker not in ruleset, f"{marker} left in the NAS ruleset" + # The control role's guard table stays in every generation (fix round 3). + nas.succeed("nft list chain inet ax-fleet-guard forward | grep -q 'tcp dport 8731'") + nixos_fw = nas.succeed("nft -s list table inet nixos-fw") + assert nixos_fw == base["nas_nixos_fw"], "the NAS firewall table differs from the baseline" + record("nas_ruleset_equal_baseline", ruleset == base["nas_nft"]) + if ruleset != base["nas_nft"]: + record("nas_ruleset_after_rollback", ruleset) + after = sysctls(nas) + record("sysctl_nas_after_rollback", after) + assert after == base["sysctl_nas"], (after, base["sysctl_nas"]) + assert unit_invocation(nas, "postgresql.service") == base["postgres_invocation"], "the shared PostgreSQL restarted" + dbs = sorted(nas.succeed("runuser -u postgres -- psql -Atc 'select datname from pg_database'").split()) + assert dbs == base["nas_databases"], dbs + # Left on disk on purpose; deleting it is Tom's call. + nas.succeed("test -d /mnt/fast/k3s/rancher/k3s") + nas.fail("findmnt /var/lib/rancher")